feat(clickhouse): Add support for clickhouse migrations (getlago#1409)

* feat(clickhouse): Add support for clickhouse migrations

* Ignore rubocop in clickhouse generated schema

* feat(clickhouse): Add events_raw model

* feat(clickhouse): Update gem

.rubocop.yml

@@ -10,6 +10,7 @@ AllCops:
   Exclude:
     - 'bin/**/*'
     - 'db/schema.rb'
+    - 'db/*_schema.rb'
 
 Layout/BlockAlignment:
   Description: 'Align block ends correctly.'

Gemfile

@@ -22,7 +22,7 @@ gem 'oauth2'
 gem 'rack-cors'
 
 # Database
-gem 'clickhouse-activerecord'
+gem 'clickhouse-activerecord', git: 'https://github.com/getlago/clickhouse-activerecord.git'
 gem 'discard', '~> 1.2'
 gem 'kaminari-activerecord'
 gem 'paper_trail'

Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/getlago/clickhouse-activerecord.git
+  revision: ec9c648ae9e7a77d4e2361bfb64ffb25dede3714
+  specs:
+    clickhouse-activerecord (0.6.0)
+      activerecord (>= 5.2)
+      bundler (>= 1.13.4)
+
 GIT
   remote: https://github.com/glebm/i18n-tasks.git
   revision: 249f0dd61b4e2a07befad96bed8e1f89b7527d3e
@@ -132,9 +140,6 @@ GEM
       msgpack (~> 1.2)
     builder (3.2.4)
     byebug (11.1.3)
-    clickhouse-activerecord (0.5.15)
-      activerecord (>= 5.2)
-      bundler (>= 1.13.4)
     clockwork (3.0.2)
       activesupport
       tzinfo
@@ -531,7 +536,7 @@ DEPENDENCIES
   bcrypt
   bootsnap
   byebug
-  clickhouse-activerecord
+  clickhouse-activerecord!
   clockwork
   clockwork-test
   coffee-rails

app/models/clickhouse/events_raw.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Clickhouse
+  class EventsRaw < BaseRecord
+    self.table_name = 'events_raw'
+  end
+end

config/database.yml

@@ -22,8 +22,10 @@ development:
     host: clickhouse
     port: 8123
     username: default
+    password: default
+    migrations_paths: db/clickhouse_migrate
     debug: true
-    database_tasks: <%= ENV['LAGO_CLICKHOUSE_ENABLED'].present? %>
+    database_tasks: <% if ENV['LAGO_CLICKHOUSE_ENABLED'].present? %> true <% else %> false <% end %>
 
 test:
   primary:
@@ -38,8 +40,9 @@ test:
     host: clickhouse
     port: 8123
     username: default
+    password: default
     debug: true
-    database_tasks: <%= ENV['LAGO_CLICKHOUSE_ENABLED'].present? %>
+    database_tasks: false
 
 staging:
   primary:
@@ -57,7 +60,7 @@ staging:
     username: <%= ENV['LAGO_CLICKHOUSE_USERNAME'] %>
     password: <%= ENV['LAGO_CLICKHOUSE_PASSWORD'] %>
     debug: false
-    database_tasks: <%= ENV['LAGO_CLICKHOUSE_ENABLED'].present? %>
+    database_tasks: <% if ENV['LAGO_CLICKHOUSE_ENABLED'].present? %> true <% else %> false <% end %>
 
 production:
   primary:
@@ -79,4 +82,4 @@ production:
     username: <%= ENV['LAGO_CLICKHOUSE_USERNAME'] %>
     password: <%= ENV['LAGO_CLICKHOUSE_PASSWORD'] %>
     debug: false
-    database_tasks: <%= ENV['LAGO_CLICKHOUSE_ENABLED'].present? %>
+    database_tasks: <% if ENV['LAGO_CLICKHOUSE_ENABLED'].present? %> true <% else %> false <% end %>

db/clickhouse_migrate/20231024084411_create_events_raw.rb

@@ -0,0 +1,23 @@
+class CreateEventsRaw < ActiveRecord::Migration[7.0]
+  def change
+    options = <<-SQL
+      MergeTree
+      PRIMARY KEY (organization_id, external_subscription_id, code, toStartOfDay(timestamp))
+      TTL
+        timestamp TO VOLUME 'hot',
+        timestamp + INTERVAL 90 DAY TO VOLUME 'cold'
+      SETTINGS
+        storage_policy = 'hot_cold';
+    SQL
+
+    create_table :events_raw, id: false, options: do |t|
+      t.string :organization_id, null: false
+      t.string :external_customer_id, null: false
+      t.string :external_subscription_id, null: false
+      t.string :transaction_id, null: false
+      t.datetime :timestamp, null: false
+      t.string :code, null: false
+      t.string :properties, null: false
+    end
+  end
+end

db/clickhouse_schema.rb

@@ -10,6 +10,18 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ClickhouseActiverecord::Schema.define(version: 0) do
+ClickhouseActiverecord::Schema.define(version: 2023_10_24_084411) do
+
+  # TABLE: events_raw
+  # SQL: CREATE TABLE default.events_raw ( `organization_id` String, `external_customer_id` String, `external_subscription_id` String, `transaction_id` String, `timestamp` DateTime, `code` String, `properties` String ) ENGINE = MergeTree PRIMARY KEY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) ORDER BY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) TTL timestamp TO VOLUME 'hot', timestamp + toIntervalDay(90) TO VOLUME 'cold' SETTINGS storage_policy = 'hot_cold', index_granularity = 8192
+  create_table "events_raw", id: false, options: "MergeTree PRIMARY KEY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) ORDER BY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) TTL timestamp TO VOLUME 'hot', timestamp + toIntervalDay(90) TO VOLUME 'cold' SETTINGS storage_policy = 'hot_cold', index_granularity = 8192", force: :cascade do |t|
+    t.string "organization_id", null: false
+    t.string "external_customer_id", null: false
+    t.string "external_subscription_id", null: false
+    t.string "transaction_id", null: false
+    t.datetime "timestamp", precision: nil, null: false
+    t.string "code", null: false
+    t.string "properties", null: false
+  end
 
 end