Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
version: 2
updates:
- package-ecosystem: cargo
directory: /
schedule:
interval: weekly
groups:
arrow:
patterns: ["arrow", "parquet"]
iceberg:
patterns: ["iceberg", "iceberg-catalog-rest"]
- package-ecosystem: docker
directories:
- etl-api
- etl-replicator
schedule:
interval: weekly
- package-ecosystem: docker-compose
directory: scripts
schedule:
interval: weekly
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
6 changes: 4 additions & 2 deletions .github/workflows/audit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ jobs:
runs-on: blacksmith-2vcpu-ubuntu-2404
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Install cargo-deny
uses: taiki-e/install-action@cargo-deny
uses: taiki-e/install-action@1ee706eb04986370fc60419ba172594c51067f29 # v2.62.58
with:
tool: cargo-deny

- name: Scan for Vulnerabilities
run: cargo deny check advisories
50 changes: 24 additions & 26 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,17 @@ jobs:
strategy:
matrix:
check: [fmt, clippy, sort]
rust: ["1.88", stable]
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Set up Rust
uses: dtolnay/rust-toolchain@1.88.0
- name: Set up Rust ${{ matrix.rust }}
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
components: ${{ matrix.check == 'fmt' && 'rustfmt' || 'clippy' }}

- name: Cache Cargo
uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.check }}
toolchain: ${{ matrix.rust }}
cache-key: ${{ matrix.check }}

- name: Install cargo-sort
if: matrix.check == 'sort'
Expand All @@ -63,18 +61,17 @@ jobs:
contents: read
strategy:
matrix:
postgres_version: [17, 16, 15, 14]
postgres_version: [18, 17, 16, 15, 14]
rust: ["1.88", stable]
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Rust
uses: dtolnay/[email protected]
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Cache Cargo
uses: Swatinem/rust-cache@v2
- name: Set up Rust ${{ matrix.rust }}
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
key: test-partial
toolchain: ${{ matrix.rust }}
cache-key: test-partial

- name: Start Docker Compose (Postgres ${{ matrix.postgres_version }})
run: |
Expand Down Expand Up @@ -122,18 +119,17 @@ jobs:
id-token: write
strategy:
matrix:
postgres_version: [17, 16, 15, 14]
postgres_version: [18, 17, 16, 15, 14]
rust: ["1.88", stable]
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Set up Rust
uses: dtolnay/[email protected]

- name: Cache Cargo
uses: Swatinem/rust-cache@v2
- name: Set up Rust ${{ matrix.rust }}
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
key: test-full
toolchain: ${{ matrix.rust }}
cache-key: test-full

- name: Start Docker Compose (Postgres ${{ matrix.postgres_version }})
run: |
Expand All @@ -158,7 +154,9 @@ jobs:
./etl-api/scripts/run_migrations.sh

- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
uses: taiki-e/install-action@1ee706eb04986370fc60419ba172594c51067f29 # v2.62.58
with:
tool: cargo-llvm-cov

- name: Set up BigQuery Credentials
run: |
Expand All @@ -175,7 +173,7 @@ jobs:
--lcov --output-path lcov.info

- name: Upload Coverage to Coveralls
uses: coverallsapp/github-action@v2
uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6
with:
fail-on-error: false
github-token: ${{ secrets.GITHUB_TOKEN }}
Expand Down
22 changes: 11 additions & 11 deletions .github/workflows/docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,27 +68,27 @@ jobs:

- name: Checkout (specific ref)
if: inputs.checkout_ref != ''
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
ref: ${{ inputs.checkout_ref }}

- name: Checkout (default)
if: inputs.checkout_ref == ''
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Set up Docker Buildx
uses: useblacksmith/setup-docker-builder@v1
uses: useblacksmith/setup-docker-builder@53647ab5afe8827af5623b35bd4302eabd41619f # v1.2.0

- name: Log in to Docker Hub
if: inputs.push == true
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Build and Push Single-Platform Image
id: build
uses: useblacksmith/build-push-action@v2
uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2
with:
context: ${{ inputs.context }}
file: ${{ inputs.file }}
Expand All @@ -108,7 +108,7 @@ jobs:

- name: Upload Digest
if: inputs.push == true
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: digests-${{ steps.extract-name.outputs.name }}-${{ matrix.arch }}
path: /tmp/digests/*
Expand All @@ -130,30 +130,30 @@ jobs:
echo "name=$NAME" >> "$GITHUB_OUTPUT"

- name: Download Digests
uses: actions/download-artifact@v4
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
path: /tmp/digests
pattern: digests-${{ steps.extract-name.outputs.name }}-*
merge-multiple: true

- name: Set up Docker Buildx
uses: useblacksmith/setup-docker-builder@v1
uses: useblacksmith/setup-docker-builder@53647ab5afe8827af5623b35bd4302eabd41619f # v1.2.0

- name: Log in to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Checkout (specific ref)
if: inputs.checkout_ref != ''
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
ref: ${{ inputs.checkout_ref }}

- name: Checkout (default)
if: inputs.checkout_ref == ''
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Create and Push Multi-Arch Manifest
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docker-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:
if: github.event_name == 'workflow_dispatch' && inputs.experimental == true
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
ref: ${{ needs.resolve-ref.outputs.ref }}
fetch-depth: 0
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ jobs:
if: github.event_name == 'push'
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
- name: Configure Git Credentials
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
with:
python-version: 3.x
- name: Compute Cache Key
run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
- name: Restore Docs Cache
uses: actions/cache@v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
key: mkdocs-material-${{ env.cache_id }}
path: ~/.cache
Expand All @@ -47,15 +47,15 @@ jobs:
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
with:
python-version: 3.x
- name: Compute Cache Key
run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
- name: Restore Docs Cache
uses: actions/cache@v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
key: mkdocs-material-${{ env.cache_id }}
path: ~/.cache
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/prepare-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
CARGO_TERM_COLOR: always
steps:
- name: Checkout (Full History)
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
fetch-depth: 0

Expand Down Expand Up @@ -86,7 +86,7 @@ jobs:
git cliff --config cliff.toml --tag "v${NEXT_VERSION}" --unreleased --prepend CHANGELOG.md

- name: Create Release Pull Request
uses: peter-evans/create-pull-request@v6
uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9
with:
commit-message: "chore(release): v${{ steps.ver.outputs.next }}"
title: "chore(release): v${{ steps.ver.outputs.next }}"
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ jobs:
steps:
- name: Checkout merge commit (PR merge)
if: github.event_name == 'pull_request'
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
ref: ${{ github.event.pull_request.merge_commit_sha }}

- name: Checkout main (manual dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v4
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0

- name: Create and Push Tag
shell: bash
Expand Down Expand Up @@ -146,7 +146,7 @@ jobs:

- name: Create Release
id: create_release
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe # v2.4.2
with:
tag_name: ${{ needs.version.outputs.tag }}
name: Release ${{ needs.version.outputs.tag }}
Expand Down
10 changes: 5 additions & 5 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ etl-telemetry = { path = "etl-telemetry", default-features = false }

actix-web = { version = "4.11.0", default-features = false }
actix-web-httpauth = { version = "0.8.2", default-features = false }
actix-web-metrics = { version = "0.3.0", default-features = false }
actix-web-metrics = { version = "0.4.0", default-features = false }
anyhow = { version = "1.0.98", default-features = false }
arrow = { version = "55.0", default-features = false }
async-trait = { version = "0.1.88" }
Expand All @@ -40,7 +40,7 @@ byteorder = { version = "1.5.0", default-features = false }
bytes = { version = "1.10.1" }
chrono = { version = "0.4.41", default-features = false }
clap = { version = "4.5.42", default-features = false }
config = { version = "0.14", default-features = false }
config = { version = "0.15", default-features = false }
configcat = { version = "0.1.3", default-features = false }
const-oid = { version = "0.9.6", default-features = false }
constant_time_eq = { version = "0.4.2" }
Expand All @@ -50,8 +50,8 @@ gcp-bigquery-client = { version = "0.27.0", default-features = false }
iceberg = { version = "0.7.0", default-features = false }
iceberg-catalog-rest = { version = "0.7.0", default-features = false }
insta = { version = "1.43.1", default-features = false }
k8s-openapi = { version = "0.25.0", default-features = false }
kube = { version = "1.1.0", default-features = false }
k8s-openapi = { version = "0.26.0", default-features = false }
kube = { version = "2.0.1", default-features = false }
metrics = { version = "0.24.2", default-features = false }
metrics-exporter-prometheus = { version = "0.17.2", default-features = false }
parquet = { version = "55.0", default-features = false }
Expand All @@ -65,7 +65,7 @@ ring = { version = "0.17.14", default-features = false }
rustls = { version = "0.23.31", default-features = false }
rustls-pemfile = { version = "2.2.0", default-features = false }
secrecy = { version = "0.10.3", default-features = false }
sentry = { version = "0.42.0" }
sentry = { version = "0.46.0" }
serde = { version = "1.0.219", default-features = false }
serde_json = { version = "1.0.141", default-features = false }
serde_yaml = { version = "0.9.34", default-features = false }
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ ETL is a Rust framework by [Supabase](https://supabase.com) for building high‑

## Requirements

**PostgreSQL Version:** ETL officially supports and tests against **PostgreSQL 14, 15, 16, and 17**.
**PostgreSQL Version:** ETL officially supports and tests against **PostgreSQL 14, 15, 16, 17, and 18**.

- **PostgreSQL 15+** is recommended for access to advanced publication features including:
- Column-level filtering
Expand Down
4 changes: 2 additions & 2 deletions docs/how-to/configure-postgres.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ This guide covers the essential Postgres concepts and configuration needed for l

## Prerequisites

- **PostgreSQL 14, 15, 16, or 17** (officially supported and tested versions)
- **PostgreSQL 14, 15, 16, 17, or 18** (officially supported and tested versions)
- PostgreSQL 15+ is recommended for advanced publication filtering features (column-level and row-level filters, `FOR ALL TABLES IN SCHEMA` syntax)
- PostgreSQL 14 is supported but has limited publication filtering capabilities
- Superuser access to the Postgres server
Expand Down Expand Up @@ -161,7 +161,7 @@ DROP PUBLICATION my_publication;

## Version-Specific Features

ETL supports PostgreSQL versions 14 through 17, with enhanced features available in newer versions:
ETL supports PostgreSQL versions 14 through 18, with enhanced features available in newer versions:

### PostgreSQL 15+ Features

Expand Down
4 changes: 2 additions & 2 deletions etl-api/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Build stage with cargo-chef for better layer caching
# Native build: each runner builds for its own architecture
FROM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef
FROM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-trixie AS chef
WORKDIR /app

# Install system dependencies
Expand Down Expand Up @@ -37,7 +37,7 @@ RUN cargo build --release -p etl-api && \
strip target/release/etl-api

# Runtime stage with distroless for security
FROM gcr.io/distroless/cc-debian12:nonroot
FROM gcr.io/distroless/cc-debian13:nonroot
WORKDIR /app

# Create non-root user (distroless already has nonroot user)
Expand Down
Loading