Add clickhouse support (#42)

Author: KirIgor

.github/clickhouse/clickhouse_keeper.xml

@@ -0,0 +1,42 @@
+<yandex>
+  <remote_servers>
+    <click_cluster>
+      <shard>
+        <internal_replication>true</internal_replication>
+        <replica>
+          <host>localhost</host>
+          <port>9000</port>
+        </replica>
+      </shard>
+    </click_cluster>
+  </remote_servers>
+
+  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <tcp_port_secure>0</tcp_port_secure>
+    <server_id>1</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+      <operation_timeout_ms>10000</operation_timeout_ms>
+      <session_timeout_ms>30000</session_timeout_ms>
+      <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+      <server>
+        <id>1</id>
+        <hostname>localhost</hostname>
+        <port>9234</port>
+      </server>
+    </raft_configuration>
+  </keeper_server>
+
+  <zookeeper>
+    <node>
+      <host>localhost</host>
+      <port>9181</port>
+    </node>
+  </zookeeper>
+</yandex>

.github/workflows/test.yml

@@ -17,34 +17,59 @@ jobs:
       matrix:
         ruby: ["3.1", "3.2", "3.3", "3.4"]
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
     env:
-      PGHOST: localhost
-      PGUSER: root
-
+      PGHOST: pg
+      PGUSER: user
+      PGPASSWORD: pass
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Create docker network
+        run: docker network create dbnet
+
+      - name: Start PostgreSQL
+        run: |
+          docker run -d \
+            --name pg \
+            --network dbnet \
+            -e POSTGRES_PASSWORD=pass \
+            -e POSTGRES_USER=user \
+            -e POSTGRES_DB=umbrellio_utils_test \
+            -p 5432:5432 \
+            postgres:14
+
+      - name: Start ClickHouse
+        run: |
+          docker run -d \
+            --name ch \
+            --network dbnet \
+            -e CLICKHOUSE_SKIP_USER_SETUP=1 -e CLICKHOUSE_DB=umbrellio_utils_test \
+            -p 9000:9000 -p 8123:8123 \
+            -v ${{ github.workspace }}/.github/clickhouse/clickhouse_keeper.xml:/etc/clickhouse-server/config.d/keeper.xml \
+            clickhouse/clickhouse-server:25.3.6.56-alpine
+
+      - name: Wait for Postgres
+        run: |
+          for i in {1..30}; do
+            if docker exec pg pg_isready -U user; then exit 0; fi
+            sleep 1
+          done
+          exit 1
+
+      - name: Wait for ClickHouse
+        run: |
+          for i in {1..30}; do
+            if docker exec ch clickhouse-client --query "SELECT 1"; then exit 0; fi
+            sleep 1
+          done
+          exit 1
 
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           rubygems: latest
           bundler-cache: true
 
-      - run: psql -c 'CREATE DATABASE umbrellio_utils_test'
-
       - name: Run Linter
         run: bundle exec ci-helper RubocopLint
 

.rubocop.yml

@@ -12,6 +12,9 @@ Naming/MethodParameterName:
 RSpec/EmptyLineAfterHook:
   Enabled: false
 
+Metrics/ModuleLength:
+  Enabled: false
+
 Naming/FileName:
   Exclude:
     - lib/umbrellio-utils.rb

Gemfile

@@ -8,6 +8,8 @@ gemspec
 gem "activesupport"
 gem "bundler"
 gem "ci-helper"
+gem "click_house", github: "umbrellio/click_house", branch: "master"
+gem "csv"
 gem "http"
 gem "net-pop"
 gem "nokogiri"

Gemfile.lock

@@ -1,7 +1,16 @@
+GIT
+  remote: https://github.com/umbrellio/click_house.git
+  revision: 1bbf8cb909a248b401d0ba9a9f6f1de2e2c068db
+  branch: master
+  specs:
+    click_house (2.1.2)
+      activesupport
+      faraday (>= 1.7, < 3)
+
 PATH
   remote: .
   specs:
-    umbrellio-utils (1.9.0)
+    umbrellio-utils (1.10.0)
       memery (~> 1)
 
 GEM
@@ -100,6 +109,7 @@ GEM
     concurrent-ruby (1.3.5)
     connection_pool (2.5.4)
     crass (1.0.6)
+    csv (3.3.5)
     date (3.4.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
@@ -109,6 +119,12 @@ GEM
     erb (4.0.4)
       cgi (>= 0.3.3)
     erubi (1.13.1)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
     ffi (1.17.2-aarch64-linux-gnu)
     ffi (1.17.2-aarch64-linux-musl)
     ffi (1.17.2-arm-linux-gnu)
@@ -163,6 +179,8 @@ GEM
     method_source (1.1.0)
     mini_mime (1.1.5)
     minitest (5.25.5)
+    net-http (0.8.0)
+      uri (>= 0.11.1)
     net-imap (0.5.10)
       date
       net-protocol
@@ -364,6 +382,7 @@ GEM
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.1.0)
+    uri (1.1.1)
     useragent (0.16.11)
     websocket-driver (0.8.0)
       base64
@@ -388,6 +407,8 @@ DEPENDENCIES
   activesupport
   bundler
   ci-helper
+  click_house!
+  csv
   http
   net-pop
   nokogiri

bin/clickhouse-server

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+docker stop clickhouse-server || true
+docker rm clickhouse-server || true
+
+docker run \
+  --detach \
+  --network host \
+  --name clickhouse-server \
+  --ulimit nofile=262144:262144 \
+  $CLICKHOUSE_IMAGE_TAG
+
+# Wait for ClickHouse server to become available
+until docker exec clickhouse-server clickhouse-client --query "SELECT 1" &>/dev/null; do
+  echo "Waiting for ClickHouse to be ready..."
+  sleep 1
+done
+
+rails ch:create ch:migrate

lib/umbrellio-utils.rb

@@ -1,3 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "umbrellio_utils"
+
+if defined?(Rake)
+  Dir[File.join(__dir__, "umbrellio_utils/tasks/**/*.rake")].each { |f| load f }
+end

lib/umbrellio_utils.rb

@@ -48,18 +48,21 @@ def synchronize(&)
 
 require_relative "umbrellio_utils/cards"
 require_relative "umbrellio_utils/checks"
+require_relative "umbrellio_utils/click_house"
 require_relative "umbrellio_utils/constants"
 require_relative "umbrellio_utils/control"
 require_relative "umbrellio_utils/database"
 require_relative "umbrellio_utils/formatting"
 require_relative "umbrellio_utils/http_client"
 require_relative "umbrellio_utils/jobs"
+require_relative "umbrellio_utils/migrations"
 require_relative "umbrellio_utils/misc"
 require_relative "umbrellio_utils/parsing"
 require_relative "umbrellio_utils/passwords"
 require_relative "umbrellio_utils/random"
 require_relative "umbrellio_utils/request_wrapper"
 require_relative "umbrellio_utils/rounding"
+require_relative "umbrellio_utils/sql"
 require_relative "umbrellio_utils/semantic_logger/tiny_json_formatter"
 require_relative "umbrellio_utils/store"
 require_relative "umbrellio_utils/vault"