From 850b9381cb676cfca54febac1cca6a9fd8f0e8d4 Mon Sep 17 00:00:00 2001 From: Adam Clarke Date: Sat, 11 May 2024 15:49:50 -0400 Subject: [PATCH 01/17] Add analytics container --- packer/supabase/docker-compose.yml | 42 ++++++++++++++++++++++++++++++ terraform/files/.env.tftpl | 7 +++++ 2 files changed, 49 insertions(+) diff --git a/packer/supabase/docker-compose.yml b/packer/supabase/docker-compose.yml index 2fa5a0d..537b357 100644 --- a/packer/supabase/docker-compose.yml +++ b/packer/supabase/docker-compose.yml @@ -251,6 +251,48 @@ services: - --main-service - /home/deno/functions/main + analytics: + container_name: supabase-analytics + image: supabase/logflare:1.4.0 + healthcheck: + test: [ "CMD", "curl", "http://localhost:4000/health" ] + timeout: 5s + interval: 5s + retries: 10 + restart: unless-stopped + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + # Uncomment to use Big Query backend for analytics + # volumes: + # - type: bind + # source: ${PWD}/gcloud.json + # target: /opt/app/rel/logflare/bin/gcloud.json + # read_only: true + environment: + LOGFLARE_NODE_HOST: 127.0.0.1 + DB_USERNAME: supabase_admin + DB_DATABASE: ${POSTGRES_DB} + DB_HOSTNAME: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_SCHEMA: _analytics + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_SINGLE_TENANT: true + LOGFLARE_SUPABASE_MODE: true + LOGFLARE_MIN_CLUSTER_SIZE: 1 + + # Comment variables to use Big Query backend for analytics + POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + POSTGRES_BACKEND_SCHEMA: _analytics + LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true + # Uncomment to use Big Query backend for analytics + # GOOGLE_PROJECT_ID: ${GOOGLE_PROJECT_ID} + # GOOGLE_PROJECT_NUMBER: ${GOOGLE_PROJECT_NUMBER} + ports: + - 4000:4000 + db: container_name: supabase-db image: supabase/postgres:15.1.0.90 diff --git a/terraform/files/.env.tftpl b/terraform/files/.env.tftpl index 5424abb..0edb72a 100644 --- a/terraform/files/.env.tftpl +++ b/terraform/files/.env.tftpl @@ -100,3 +100,10 @@ IMGPROXY_ENABLE_WEBP_DETECTION=true ############ # NOTE: VERIFY_JWT applies to all functions. Per-function VERIFY_JWT is not supported yet. FUNCTIONS_VERIFY_JWT=false + +############ +# Logs - Configuration for Logflare +# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction +############ + +LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key From 85f4684c77a38f5f1b0404676b524846adb3eef5 Mon Sep 17 00:00:00 2001 From: Adam Clarke Date: Sat, 11 May 2024 15:54:34 -0400 Subject: [PATCH 02/17] Use supabase GA with analytics --- packer/supabase/docker-compose.yml | 13 ++++++++++++- terraform/files/.env.tftpl | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packer/supabase/docker-compose.yml b/packer/supabase/docker-compose.yml index 537b357..c1afb9d 100644 --- a/packer/supabase/docker-compose.yml +++ b/packer/supabase/docker-compose.yml @@ -30,13 +30,16 @@ services: studio: container_name: supabase-studio - image: supabase/studio:20230622-d8395eb + image: supabase/studio:v1.24.04 restart: unless-stopped healthcheck: test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})" ] timeout: 5s interval: 5s retries: 3 + depends_on: + analytics: + condition: service_healthy ports: - ${STUDIO_PORT}:3000/tcp environment: @@ -51,6 +54,14 @@ services: SUPABASE_ANON_KEY: ${ANON_KEY} SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_URL: http://analytics:4000 + NEXT_PUBLIC_ENABLE_LOGS: true + # Comment to use Big Query backend for analytics + NEXT_ANALYTICS_BACKEND_PROVIDER: postgres + # Uncomment to use Big Query backend for analytics + # NEXT_ANALYTICS_BACKEND_PROVIDER: bigquery + kong: container_name: supabase-kong image: kong:2.8.1 diff --git a/terraform/files/.env.tftpl b/terraform/files/.env.tftpl index 0edb72a..bab6b15 100644 --- a/terraform/files/.env.tftpl +++ b/terraform/files/.env.tftpl @@ -107,3 +107,7 @@ FUNCTIONS_VERIFY_JWT=false ############ LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key + +# Google Cloud Project details +GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID +GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER \ No newline at end of file From cb84299acc7040eb0c5ba1bdf1f861fd5afdf500 Mon Sep 17 00:00:00 2001 From: Adam Clarke Date: Sat, 11 May 2024 16:00:11 -0400 Subject: [PATCH 03/17] get hostname from env to support ipv6 --- packer/supabase/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/supabase/docker-compose.yml b/packer/supabase/docker-compose.yml index c1afb9d..bf63c16 100644 --- a/packer/supabase/docker-compose.yml +++ b/packer/supabase/docker-compose.yml @@ -33,7 +33,7 @@ services: image: supabase/studio:v1.24.04 restart: unless-stopped healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})" ] + test: [ "CMD", "node", "-e", "require('http').get('http://'+process.env.HOSTNAME+':3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})" ] timeout: 5s interval: 5s retries: 3 From 9441eeb78ee72e327ac8a46343f0de385d1e9ec2 Mon Sep 17 00:00:00 2001 From: Adam Clarke Date: Sat, 11 May 2024 17:00:28 -0400 Subject: [PATCH 04/17] update postgres, add vector --- packer/supabase/docker-compose.yml | 44 ++++- packer/supabase/volumes/db/jwt.sql | 5 + packer/supabase/volumes/db/logs.sql | 4 + packer/supabase/volumes/logs/vector.yml | 232 ++++++++++++++++++++++++ terraform/files/.env.tftpl | 3 + 5 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 packer/supabase/volumes/db/jwt.sql create mode 100644 packer/supabase/volumes/db/logs.sql create mode 100644 packer/supabase/volumes/logs/vector.yml diff --git a/packer/supabase/docker-compose.yml b/packer/supabase/docker-compose.yml index bf63c16..3f7c30e 100644 --- a/packer/supabase/docker-compose.yml +++ b/packer/supabase/docker-compose.yml @@ -66,6 +66,7 @@ services: container_name: supabase-kong image: kong:2.8.1 restart: unless-stopped + #entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' ports: - ${KONG_HTTP_PORT}:8000/tcp - ${KONG_HTTPS_PORT}:8443/tcp @@ -306,12 +307,15 @@ services: db: container_name: supabase-db - image: supabase/postgres:15.1.0.90 + image: supabase/postgres:15.1.1.41 healthcheck: test: pg_isready -U postgres -h localhost interval: 5s timeout: 5s retries: 10 + depends_on: + vector: + condition: service_healthy command: - postgres - -c @@ -330,10 +334,46 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} PGDATABASE: ${POSTGRES_DB} POSTGRES_DB: ${POSTGRES_DB} + JWT_SECRET: ${JWT_SECRET} + JWT_EXP: ${JWT_EXPIRY} volumes: - ./volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:Z # Must be superuser to create event trigger - ./volumes/db/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:Z # Must be superuser to alter reserved role - ./volumes/db/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:Z - - /mnt/supabase_volume/supabase/data/:/var/lib/postgresql/data:Z \ No newline at end of file + # Initialize the database settings with JWT_SECRET and JWT_EXP + - ./volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:Z + # PGDATA directory is persisted between restarts + - /mnt/supabase_volume/supabase/data/:/var/lib/postgresql/data:Z + # Changes required for Analytics support + - ./volumes/db/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:Z + # Use named volume to persist pgsodium decryption key between restarts + - db-config:/etc/postgresql-custom + + vector: + container_name: supabase-vector + image: timberio/vector:0.28.1-alpine + healthcheck: + test: + [ + + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://vector:9001/health" + ] + timeout: 5s + interval: 5s + retries: 3 + volumes: + - ./volumes/logs/vector.yml:/etc/vector/vector.yml:ro + - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro + environment: + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + command: [ "--config", "etc/vector/vector.yml" ] + +volumes: + db-config: \ No newline at end of file diff --git a/packer/supabase/volumes/db/jwt.sql b/packer/supabase/volumes/db/jwt.sql new file mode 100644 index 0000000..075bf74 --- /dev/null +++ b/packer/supabase/volumes/db/jwt.sql @@ -0,0 +1,5 @@ +\set jwt_secret `echo "$JWT_SECRET"` +\set jwt_exp `echo "$JWT_EXP"` + +ALTER DATABASE postgres SET "app.settings.jwt_secret" TO :'jwt_secret'; +ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; \ No newline at end of file diff --git a/packer/supabase/volumes/db/logs.sql b/packer/supabase/volumes/db/logs.sql new file mode 100644 index 0000000..ff52c65 --- /dev/null +++ b/packer/supabase/volumes/db/logs.sql @@ -0,0 +1,4 @@ +\set pguser `echo "$POSTGRES_USER"` + +create schema if not exists _analytics; +alter schema _analytics owner to :pguser; \ No newline at end of file diff --git a/packer/supabase/volumes/logs/vector.yml b/packer/supabase/volumes/logs/vector.yml new file mode 100644 index 0000000..5e9675b --- /dev/null +++ b/packer/supabase/volumes/logs/vector.yml @@ -0,0 +1,232 @@ +api: + enabled: true + address: 0.0.0.0:9001 + +sources: + docker_host: + type: docker_logs + exclude_containers: + - supabase-vector + +transforms: + project_logs: + type: remap + inputs: + - docker_host + source: |- + .project = "default" + .event_message = del(.message) + .appname = del(.container_name) + del(.container_created_at) + del(.container_id) + del(.source_type) + del(.stream) + del(.label) + del(.image) + del(.host) + del(.stream) + router: + type: route + inputs: + - project_logs + route: + kong: '.appname == "supabase-kong"' + auth: '.appname == "supabase-auth"' + rest: '.appname == "supabase-rest"' + realtime: '.appname == "supabase-realtime"' + storage: '.appname == "supabase-storage"' + functions: '.appname == "supabase-functions"' + db: '.appname == "supabase-db"' + # Ignores non nginx errors since they are related with kong booting up + kong_logs: + type: remap + inputs: + - router.kong + source: |- + req, err = parse_nginx_log(.event_message, "combined") + if err == null { + .timestamp = req.timestamp + .metadata.request.headers.referer = req.referer + .metadata.request.headers.user_agent = req.agent + .metadata.request.headers.cf_connecting_ip = req.client + .metadata.request.method = req.method + .metadata.request.path = req.path + .metadata.request.protocol = req.protocol + .metadata.response.status_code = req.status + } + if err != null { + abort + } + # Ignores non nginx errors since they are related with kong booting up + kong_err: + type: remap + inputs: + - router.kong + source: |- + .metadata.request.method = "GET" + .metadata.response.status_code = 200 + parsed, err = parse_nginx_log(.event_message, "error") + if err == null { + .timestamp = parsed.timestamp + .severity = parsed.severity + .metadata.request.host = parsed.host + .metadata.request.headers.cf_connecting_ip = parsed.client + url, err = split(parsed.request, " ") + if err == null { + .metadata.request.method = url[0] + .metadata.request.path = url[1] + .metadata.request.protocol = url[2] + } + } + if err != null { + abort + } + # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. + auth_logs: + type: remap + inputs: + - router.auth + source: |- + parsed, err = parse_json(.event_message) + if err == null { + .metadata.timestamp = parsed.time + .metadata = merge!(.metadata, parsed) + } + # PostgREST logs are structured so we separate timestamp from message using regex + rest_logs: + type: remap + inputs: + - router.rest + source: |- + parsed, err = parse_regex(.event_message, r'^(?P