Refactor choreo docker (#423) #5

Workflow file for this run

.github/workflows/ingestion-api-test-choreo.yml at 5ff94eb

	name: Ingestion API Test (Choreo)

	on:
	push:
	branches: [ main ]
	paths:
	- 'opengin/ingestion-api/**'
	- 'opengin/ingestion-api/docker/Dockerfile.choreo'
	- 'docker-compose-choreo.yml'
	pull_request:
	branches: [ main ]
	paths:
	- 'opengin/ingestion-api/**'
	- 'opengin/ingestion-api/docker/Dockerfile.choreo'
	- 'docker-compose-choreo.yml'

	jobs:
	ingestion-api-workflow:
	runs-on: ubuntu-latest

	steps:

	- uses: actions/checkout@v3

	- name: Clear GitHub Actions cache
	run: \|
	echo "🧹 Clearing GitHub Actions cache..."
	rm -rf ~/.cache \|\| true
	rm -rf /tmp/* \|\| true

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Build and test Ingestion service (Choreo)
	run: \|
	# Complete cache and container cleanup for fresh start
	echo "🧹 Complete cleanup - removing all containers, images, and cache..."
	docker compose -f docker-compose-choreo.yml down --remove-orphans \|\| true
	docker system prune -af \|\| true # -a removes all unused images, -f forces without prompt
	docker builder prune -af \|\| true # Clean build cache
	docker volume prune -f \|\| true # Clean unused volumes

	# Build Ingestion service (force rebuild to pick up environment variable changes)
	docker compose -f docker-compose-choreo.yml build --no-cache ingestion-choreo

	# Start required dependencies
	docker compose -f docker-compose-choreo.yml up -d mongodb-choreo neo4j-choreo postgres-choreo core-choreo

	# Wait for databases to be fully initialized
	echo "⏳ Waiting for databases to be fully initialized..."
	sleep 30

	# Verify databases are fresh and ready
	echo "🔍 Verifying database state..."
	echo "MongoDB collections:"
	docker exec mongodb-choreo mongosh --eval "db.getMongo().getDBNames()" \|\| echo "MongoDB not ready yet"
	echo "Neo4j nodes:"
	docker exec neo4j-choreo cypher-shell -u neo4j -p neo4j123 "MATCH (n) RETURN count(n) as nodeCount" \|\| echo "Neo4j not ready yet"
	echo "PostgreSQL tables:"
	docker exec postgres-choreo psql -U postgres -d opengin -c "\dt" \|\| echo "PostgreSQL not ready yet"

	# Wait for dependencies to be healthy
	echo "Waiting for dependencies to be healthy..."
	timeout=300
	while [ $timeout -gt 0 ]; do
	if docker compose -f docker-compose-choreo.yml ps \| grep -q "healthy"; then
	echo "Dependencies are starting to be healthy..."
	break
	fi
	# Check if any services have failed
	if docker compose -f docker-compose-choreo.yml ps \| grep -q "Exit"; then
	echo "❌ Some services have failed during startup!"
	echo "=== Service Status ==="
	docker compose -f docker-compose-choreo.yml ps
	echo "=== Failed Service Logs ==="
	docker compose -f docker-compose-choreo.yml logs mongodb-choreo neo4j-choreo postgres-choreo core-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi
	echo "Waiting for health checks... ($timeout seconds remaining)"
	sleep 5
	timeout=$((timeout - 5))
	done

	if [ $timeout -le 0 ]; then
	echo "❌ Timeout waiting for dependencies to be healthy!"
	echo "=== Service Status ==="
	docker compose -f docker-compose-choreo.yml ps
	echo "=== Dependency Logs ==="
	docker compose -f docker-compose-choreo.yml logs mongodb-choreo neo4j-choreo postgres-choreo core-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi

	# Wait for CORE service to become healthy
	echo "Waiting for CORE service to become healthy..."
	timeout=180
	while [ $timeout -gt 0 ]; do
	if docker compose -f docker-compose-choreo.yml ps core-choreo \| grep -q "healthy"; then
	echo "✅ CORE service is healthy!"
	break
	fi
	# Check if service has failed
	if docker compose -f docker-compose-choreo.yml ps core-choreo \| grep -q "Exit\\|unhealthy"; then
	echo "❌ CORE service failed to start or became unhealthy!"
	echo "CORE service logs:"
	docker compose -f docker-compose-choreo.yml logs core-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi
	echo "Waiting for CORE service to be healthy... ($timeout seconds remaining)"
	sleep 5
	timeout=$((timeout - 5))
	done

	if [ $timeout -le 0 ]; then
	echo "❌ Timeout waiting for CORE service to become healthy!"
	echo "CORE service logs:"
	docker compose -f docker-compose-choreo.yml logs core-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi

	# Start Ingestion service in detached mode
	echo "Starting Ingestion service (Choreo deployment)..."
	if ! docker compose -f docker-compose-choreo.yml up -d ingestion-choreo; then
	echo "❌ Failed to start Ingestion service!"
	echo "=== All Service Status ==="
	docker compose -f docker-compose-choreo.yml ps
	echo "=== CORE Service Logs ==="
	docker compose -f docker-compose-choreo.yml logs core-choreo
	echo "=== Ingestion Service Logs ==="
	docker compose -f docker-compose-choreo.yml logs ingestion-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi

	# Wait for Ingestion service to become healthy
	echo "Waiting for Ingestion service to become healthy..."
	timeout=120
	while [ $timeout -gt 0 ]; do
	if docker compose -f docker-compose-choreo.yml ps ingestion-choreo \| grep -q "healthy"; then
	echo "✅ Ingestion service is healthy!"
	break
	fi
	# Check if service has failed
	if docker compose -f docker-compose-choreo.yml ps ingestion-choreo \| grep -q "Exit\\|unhealthy"; then
	echo "❌ Ingestion service failed to start or became unhealthy!"
	echo "Ingestion service logs:"
	docker compose -f docker-compose-choreo.yml logs ingestion-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi
	echo "Waiting for Ingestion service to be healthy... ($timeout seconds remaining)"
	sleep 5
	timeout=$((timeout - 5))
	done

	if [ $timeout -le 0 ]; then
	echo "❌ Timeout waiting for Ingestion service to become healthy!"
	echo "Service status:"
	docker compose -f docker-compose-choreo.yml ps ingestion-choreo
	echo "Ingestion service logs:"
	docker compose -f docker-compose-choreo.yml logs ingestion-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi

	# Verify service started successfully - check logs for known startup message
	echo "Verifying service started successfully..."
	if docker compose -f docker-compose-choreo.yml logs ingestion-choreo \| grep -q "Starting ingestion service with Ballerina configurable variables"; then
	echo "✅ Ingestion service startup message found in logs!"
	else
	echo "⚠️ Could not find expected startup message in logs, but service is healthy"
	fi

	# Check for connectivity errors (these would indicate real issues)
	if docker compose -f docker-compose-choreo.yml logs ingestion-choreo \| grep -q "Connection refused\\|connect: connection refused"; then
	echo "⚠️ Found connection refused errors - checking CORE service status"
	if ! docker compose -f docker-compose-choreo.yml ps core-choreo \| grep -q "healthy"; then
	echo "❌ CORE service is not healthy - this may be the cause"
	docker compose -f docker-compose-choreo.yml logs core-choreo
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi
	fi

	echo "=== Ingestion Service Logs ==="
	docker compose -f docker-compose-choreo.yml logs ingestion-choreo

	echo "=== CORE Service Logs ==="
	docker compose -f docker-compose-choreo.yml logs core-choreo

	echo "=== All Service Status ==="
	docker compose -f docker-compose-choreo.yml ps

	# Final verification - service should be healthy at this point
	if docker compose -f docker-compose-choreo.yml ps ingestion-choreo \| grep -q "healthy"; then
	echo "✅ Ingestion API service (Choreo) is healthy and ready!"
	else
	echo "❌ Ingestion service is not healthy!"
	docker compose -f docker-compose-choreo.yml down
	exit 1
	fi

	# Cleanup - ensure complete cleanup for next CI run
	echo "🧹 Final cleanup - removing all containers..."
	docker compose -f docker-compose-choreo.yml down --remove-orphans
	docker system prune -f

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Refactor choreo docker (#423) #5

Workflow file

Refactor choreo docker (#423) #5

Uh oh!

Workflow file for this run