Skip to content

Commit 13a6393

Browse files
vibhathavibhatha
authored
GH-444: [Eval] Remote DB Usage via Docker Compose (#445)
* feat: Implement production deployment configuration for Choreo using remote databases, along with a deployment guide and a Supabase restore script. * docs: Add documentation for database restoration from GitHub backups to remote PostgreSQL databases. * chore: Remove the explicit volumes section from docker-compose-prod.yml. * fix(reviews): adding correct tool
1 parent a291d0a commit 13a6393

File tree

5 files changed

+342
-0
lines changed

5 files changed

+342
-0
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ target/
2323
artifacts and env configs not to be committed
2424
opengin/core-api/.env
2525
.env
26+
.env.*
2627
opengin/core-api/core-service
2728

2829
# Neo4j

‎deployment/choreo/development/docker/postgres/restore_from_github_to_supabase.sh‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Configuration
5+
GITHUB_REPO="LDFLK/data-backups"
6+
ENVIRONMENT="${ENVIRONMENT:-development}"
7+
VERSION="${1:-latest}"
8+
9+
# Colors for output
10+
GREEN='\033[0;32m'
11+
RED='\033[0;31m'
12+
BLUE='\033[0;34m'
13+
NC='\033[0m'
14+
15+
log() {
16+
local level=$1
17+
shift
18+
local message="$*"
19+
case $level in
20+
"INFO") echo -e "${BLUE}[INFO]${NC} $message" ;;
21+
"SUCCESS") echo -e "${GREEN}[SUCCESS]${NC} $message" ;;
22+
"ERROR") echo -e "${RED}[ERROR]${NC} $message" ;;
23+
esac
24+
}
25+
26+
# Prerequisites Checks
27+
if [ -z "$SUPABASE_DB_URL" ]; then
28+
log "ERROR" "SUPABASE_DB_URL environment variable is required."
29+
echo -e "Example: export SUPABASE_DB_URL='postgresql://user:pass@host:5432/db'"
30+
exit 1
31+
fi
32+
33+
if ! command -v psql &> /dev/null; then
34+
log "ERROR" "psql could not be found. Please install PostgreSQL client tools."
35+
exit 1
36+
fi
37+
38+
if ! command -v wget &> /dev/null; then
39+
log "ERROR" "wget could not be found. Please install wget."
40+
exit 1
41+
fi
42+
43+
if ! command -v unzip &> /dev/null; then
44+
log "ERROR" "unzip could not be found. Please install unzip."
45+
exit 1
46+
fi
47+
48+
# Create temporary directory
49+
TEMP_DIR=$(mktemp -d)
50+
log "INFO" "Created temporary directory: $TEMP_DIR"
51+
52+
cleanup() {
53+
log "INFO" "Cleaning up temporary files..."
54+
rm -rf "$TEMP_DIR"
55+
}
56+
trap cleanup EXIT
57+
58+
# 1. Download Release
59+
ARCHIVE_URL="https://github.com/$GITHUB_REPO/archive/refs/tags/$VERSION.zip"
60+
ARCHIVE_FILE="$TEMP_DIR/archive.zip"
61+
62+
log "INFO" "Downloading GitHub archive for version $VERSION from $ARCHIVE_URL..."
63+
64+
if wget -q "$ARCHIVE_URL" -O "$ARCHIVE_FILE"; then
65+
log "SUCCESS" "Download complete."
66+
else
67+
log "ERROR" "Failed to download archive. Please check the version/tag."
68+
exit 1
69+
fi
70+
71+
# 2. Extract Archive
72+
log "INFO" "Extracting archive..."
73+
unzip -q "$ARCHIVE_FILE" -d "$TEMP_DIR"
74+
EXTRACTED_DIR="$TEMP_DIR/data-backups-$VERSION"
75+
76+
# Note: If version is 'latest', the folder name inside might be based on the commit hash or tag name
77+
# We should try to find the directory if the predictable name fails, or just assume the single directory extracted
78+
if [ ! -d "$EXTRACTED_DIR" ]; then
79+
EXTRACTED_DIR=$(find "$TEMP_DIR" -maxdepth 1 -type d -name "data-backups-*" | head -n 1)
80+
fi
81+
82+
log "INFO" "Extracted to: $EXTRACTED_DIR"
83+
84+
# 3. Locate Postgres Backup
85+
# Structure: opengin/$environment/postgres/opengin.tar.gz
86+
BACKUP_ARCHIVE="$EXTRACTED_DIR/opengin/$ENVIRONMENT/postgres/opengin.tar.gz"
87+
88+
if [ ! -f "$BACKUP_ARCHIVE" ]; then
89+
log "ERROR" "PostgreSQL backup not found at: $BACKUP_ARCHIVE"
90+
exit 1
91+
fi
92+
93+
log "INFO" "Found backup archive: $BACKUP_ARCHIVE"
94+
95+
# 4. Extract SQL Dump
96+
log "INFO" "Extracting SQL dump from archive..."
97+
tar -xzf "$BACKUP_ARCHIVE" -C "$TEMP_DIR"
98+
99+
# The dump file name inside the tar usually matches the tar name base or is specifically named.
100+
# Based on init.sh, backup_postgres creates `opengin.tar.gz` containing `${backup_file}.sql` which defaults to `opengin.sql`
101+
DUMP_FILE="$TEMP_DIR/opengin.sql"
102+
103+
if [ ! -f "$DUMP_FILE" ]; then
104+
# Fallback: find any .sql file in temp dir
105+
DUMP_FILE=$(find "$TEMP_DIR" -maxdepth 1 -name "*.sql" | head -n 1)
106+
fi
107+
108+
if [ ! -f "$DUMP_FILE" ]; then
109+
log "ERROR" "Could not find extracted SQL file."
110+
exit 1
111+
fi
112+
113+
log "INFO" "SQL Dump ready: $DUMP_FILE"
114+
115+
# 5. Restore to Supabase
116+
log "INFO" "Using local psql: $(psql --version)"
117+
log "INFO" "Restoring to Supabase..."
118+
119+
if psql -d "$SUPABASE_DB_URL" -f "$DUMP_FILE"; then
120+
log "SUCCESS" "Migration completed successfully!"
121+
else
122+
log "ERROR" "Migration failed."
123+
exit 1
124+
fi

‎deployment/choreo/docs/PRODUCTION_DEPLOYMENT.md‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Production Deployment Guide
2+
3+
This guide describes how to deploy the OpenGIN services in a production-like environment using the Choreo-optimized Docker configuration (`docker-compose-prod.yml`).
4+
5+
## Overview
6+
7+
The production setup differs from development in the following ways:
8+
* **Databases**: Connects to remote managed databases (Supabase for PostgreSQL, MongoDB Atlas, Neo4j Aura) instead of valid local containers.
9+
* **Images**: Uses `.choreo` Dockerfiles which are optimized for cloud deployment (e.g., using specific build contexts and startup scripts).
10+
* **Configuration**: All secrets and connection details are managed via environment variables.
11+
12+
## Prerequisites
13+
14+
* Rancher Desktop (or Docker Engine + Compose plugin) installed.
15+
* Access to the remote database instances.
16+
17+
## Configuration (.env)
18+
19+
Create a `.env` file in the project root to store your database credentials.
20+
21+
> **Important**: This file contains secrets. Ensure it is added to your `.gitignore`.
22+
23+
### Required Variables
24+
25+
Add the following variables to your `.env` file, replacing the values with your actual credentials.
26+
27+
#### PostgreSQL (Supabase)
28+
```bash
29+
POSTGRES_HOST=your-db-host.supabase.co
30+
POSTGRES_PORT=5432
31+
POSTGRES_USER=your-db-user
32+
POSTGRES_PASSWORD=your-db-password
33+
POSTGRES_DB=postgres
34+
# SSL Mode is usually 'require' for managed databases
35+
POSTGRES_SSL_MODE=require
36+
```
37+
![Supabase Connection URL Settings](supabase-url.png)
38+
*Find your connection details in Project Settings > Database > Connection Parameters*
39+
40+
#### MongoDB (Atlas)
41+
```bash
42+
MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net/
43+
MONGO_DB_NAME=opengin
44+
MONGO_ADMIN_USER=your-admin-user
45+
MONGO_ADMIN_PASSWORD=your-admin-pass
46+
```
47+
48+
#### Neo4j (Aura)
49+
```bash
50+
NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
51+
NEO4J_USER=neo4j
52+
NEO4J_PASSWORD=your-neo4j-password
53+
NEO4J_DATABASE=neo4j
54+
```
55+
56+
## Running the Application
57+
58+
To start the application using the production configuration:
59+
60+
```bash
61+
docker compose -f docker-compose-prod.yml --env-file .env up --build
62+
```
63+
64+
### Detached Mode
65+
To run in the background:
66+
```bash
67+
docker compose -f docker-compose-prod.yml --env-file .env up -d --build
68+
```
69+
70+
### Stopping the Application
71+
```bash
72+
docker compose -f docker-compose-prod.yml down
73+
```
74+
75+
## Troubleshooting
76+
77+
### Environment Variable Collisions (Auth Failures)
78+
If you encounter authentication failures (e.g., `password authentication failed for user "postgres"`), check if you have local shell environment variables set that conflict with your `.env` file.
79+
80+
Docker Compose prioritizes shell variables over the `.env` file. Common conflicts include:
81+
* `POSTGRES_USER`
82+
* `POSTGRES_PASSWORD`
83+
84+
**Solution**: Unset the conflicting variables in your shell before running docker-compose:
85+
```bash
86+
unset POSTGRES_USER POSTGRES_PASSWORD
87+
docker compose -f docker-compose-prod.yml --env-file .env up --build
88+
```
89+
90+
### Hostname Resolution
91+
If services cannot communicate (e.g., `Could not resolve host: core-choreo`), ensure that `docker-compose-prod.yml` explicitly overrides the `BAL_CONFIG_VAR_CORESERVICEURL` environment variable for dependent services (ingestion, read).
92+
93+
```yaml
94+
environment:
95+
- BAL_CONFIG_VAR_CORESERVICEURL=http://core:50051
96+
```
97+
This is already configured in the provided `docker-compose-prod.yml`.
98+
99+
## Database Restoration
100+
101+
To restore the latest backup from the GitHub repository (`LDFLK/data-backups`) to your remote PostgreSQL database (Supabase or NeonDB), use the helper script:
102+
103+
`deployment/choreo/development/docker/postgres/restore_from_github_to_supabase.sh`
104+
105+
### Usage
106+
107+
1. **Set the Connection String**:
108+
The script requires the `SUPABASE_DB_URL` environment variable. You can use the value from your `.env` file (construct it from the individual variables if needed, or if you have a full connection string variable).
109+
110+
2. **Run the Script**:
111+
You can optionally specify a version tag (defaults to `latest`).
112+
113+
```bash
114+
# Example: Restoring to the DB defined in your current shell
115+
export SUPABASE_DB_URL='postgresql://user:password@host:port/dbname?sslmode=require'
116+
./deployment/choreo/development/docker/postgres/restore_from_github_to_supabase.sh [version]
117+
```
118+
119+
* `[version]`: Optional. The tag version to download from GitHub (e.g., `0.0.1`). Defaults to `latest`.

‎deployment/choreo/docs/supabase-url.png‎

67.4 KB
Loading

‎docker-compose-prod.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
version: '3.8'
2+
3+
services:
4+
# MongoDB service is removed in production as we use a remote DB
5+
6+
# Neo4j service is removed in production as we use a remote DB
7+
8+
# Postgres service is removed in production as we use a remote DB
9+
10+
core:
11+
build:
12+
context: ./opengin/core-api
13+
dockerfile: docker/Dockerfile.choreo
14+
container_name: core
15+
ports:
16+
- "50051:50051"
17+
environment:
18+
# Neo4j Config - Remote DB
19+
- NEO4J_URI=${NEO4J_URI}
20+
- NEO4J_USER=${NEO4J_USER}
21+
- NEO4J_PASSWORD=${NEO4J_PASSWORD}
22+
# MongoDB Config - Remote DB
23+
- MONGO_URI=${MONGO_URI}
24+
- MONGO_DB_NAME=${MONGO_DB_NAME}
25+
- MONGO_COLLECTION=metadata
26+
- MONGO_ADMIN_USER=${MONGO_ADMIN_USER}
27+
- MONGO_ADMIN_PASSWORD=${MONGO_ADMIN_PASSWORD}
28+
# Postgres Config - Remote DB
29+
- POSTGRES_HOST=${POSTGRES_HOST}
30+
- POSTGRES_PORT=${POSTGRES_PORT:-5432}
31+
- POSTGRES_USER=${POSTGRES_USER}
32+
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
33+
- POSTGRES_DB=${POSTGRES_DB:-postgres}
34+
- CORE_SERVICE_HOST=0.0.0.0
35+
- CORE_SERVICE_PORT=50051
36+
# No depends_on needed as all databases are remote
37+
networks:
38+
- ldf-network
39+
healthcheck:
40+
test: [ "CMD", "nc", "-zv", "localhost", "50051" ]
41+
interval: 30s
42+
timeout: 10s
43+
retries: 5
44+
start_period: 120s
45+
46+
ingestion:
47+
build:
48+
context: ./opengin/ingestion-api
49+
dockerfile: docker/Dockerfile.choreo
50+
container_name: ingestion
51+
ports:
52+
- "8080:8080"
53+
networks:
54+
- ldf-network
55+
environment:
56+
# Explicitly set Ballerina Config Var to override image default (core-choreo)
57+
- BAL_CONFIG_VAR_CORESERVICEURL=http://core:50051
58+
- CORE_SERVICE_URL=http://core:50051
59+
- INGESTION_SERVICE_HOST=0.0.0.0
60+
- INGESTION_SERVICE_PORT=8080
61+
depends_on:
62+
core:
63+
condition: service_healthy
64+
healthcheck:
65+
test: [ "CMD", "nc", "-z", "localhost", "8080" ]
66+
interval: 15s
67+
timeout: 10s
68+
retries: 5
69+
start_period: 120s
70+
71+
read:
72+
build:
73+
context: ./opengin/read-api
74+
dockerfile: docker/Dockerfile.choreo
75+
container_name: read
76+
ports:
77+
- "8081:8081"
78+
networks:
79+
- ldf-network
80+
environment:
81+
# Explicitly set Ballerina Config Var to override image default (core-choreo)
82+
- BAL_CONFIG_VAR_CORESERVICEURL=http://core:50051
83+
- CORE_SERVICE_URL=http://core:50051
84+
- READ_SERVICE_HOST=0.0.0.0
85+
- READ_SERVICE_PORT=8081
86+
depends_on:
87+
core:
88+
condition: service_healthy
89+
healthcheck:
90+
test: [ "CMD", "nc", "-zv", "localhost", "8081" ]
91+
interval: 15s
92+
timeout: 10s
93+
retries: 5
94+
start_period: 120s
95+
96+
networks:
97+
ldf-network:
98+
driver: bridge

0 commit comments

Comments
 (0)