misc: Enable clickhouse CI (getlago#1525)

* misc: Enable clickhouse CI * fix dclickhouse config * Allow connection to clockhouse host in specs --------- Co-authored-by: Jeremy Denquin <[email protected]>
zeeshanmuhammad · Dec 15, 2023 · 288d0ce · 288d0ce
1 parent d361e3e
commit 288d0ce
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 107 deletions.
diff --git a/.github/workflows/spec.yml b/.github/workflows/spec.yml
@@ -2,7 +2,7 @@ name: Run Spec
 on:
   push:
     branches:
-      - 'main'
+      - "main"
   pull_request:
     types: [opened, synchronize, reopened]
 jobs:
@@ -30,17 +30,28 @@ jobs:
       LAGO_PDF_URL: https://pdf.lago.dev
       SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY }}
       LAGO_FROM_EMAIL: [email protected]
+      LAGO_CLICKHOUSE_ENABLED: true
+      LAGO_CLICKHOUSE_HOST: localhost
+      LAGO_CLICKHOUSE_DATABASE: default
+      LAGO_CLICKHOUSE_USERNAME: ""
+      LAGO_CLICKHOUSE_PASSWORD: ""
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Install Ruby and gems
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.2.2'
+          ruby-version: "3.2.2"
           bundler-cache: true
+      - name: Start Clickhouse database
+        run: |
+          docker run -d --rm -p 8123:8123 -p 9000:9000 --ulimit nofile=262144:262144 -v ./clickhouse-s3:/var/lib/clickhouse-s3 -v ./ci/clickhouse/config.xml:/etc/clickhouse-server/config.d/config.xml clickhouse/clickhouse-server
+        shell: bash
       - name: Generate RSA keys
         run: ./scripts/generate.rsa.sh
-      - name: Set up database schema
+      - name: Set up Postgres database schema
         run: bin/rails db:schema:load:primary
+      - name: Set up Clickhouse database schema
+        run: bin/rails db:schema:load:clickhouse
       - name: Run tests
         run: bundle exec rspec
diff --git a/ci/clickhouse/config.xml b/ci/clickhouse/config.xml
@@ -0,0 +1,44 @@
+<clickhouse replace="true">
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+        <console>1</console>
+    </logger>
+    <display_name>clickhouse_dev</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <user_directories>
+        <!-- <users_xml>
+            <path>users.xml</path>
+        </users_xml> -->
+        <local_directory>
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <storage_configuration>
+        <disks>
+            <default>
+            </default>
+            <s3>
+              <type>local</type>
+              <path>/var/lib/clickhouse-s3/</path>
+            </s3>
+        </disks>
+        <policies>
+            <hot_cold>
+              <volumes>
+                <hot>
+                  <disk>default</disk>
+                </hot>
+                <cold>
+                  <disk>s3</disk>
+                </cold>
+              </volumes>
+            </hot_cold>
+        </policies>
+    </storage_configuration>
+</clickhouse>
diff --git a/config/database.yml b/config/database.yml
@@ -36,13 +36,13 @@ test:
     url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
   clickhouse:
     adapter: clickhouse
-    database: default_test
-    host: clickhouse
-    port: 8123
-    username: default
-    password: default
+    database: <%= ENV.fetch('LAGO_CLICKHOUSE_DATABASE', 'default_test') %>
+    host: <%= ENV.fetch('LAGO_CLICKHOUSE_HOST', 'clickhouse') %>
+    port: <%= ENV.fetch('LAGO_CLICKHOUSE_PORT', 8123) %>
+    username: <%= ENV.fetch('LAGO_CLICKHOUSE_USERNAME', 'default') %>
+    password: <%= ENV.fetch('LAGO_CLICKHOUSE_PASSWORD', 'default') %>
     debug: true
-    database_tasks: false
+    database_tasks: <% if ENV['LAGO_CLICKHOUSE_ENABLED'].present? %> true <% else %> false <% end %>
 
 staging:
   primary: