Give each S3 reporter its own credentials and endpoint, and test against rustfs

.env-template

@@ -1,8 +1,25 @@
-# AWS 
+# AWS
+# Used only by reporters with no HPR_*_AWS_ENDPOINT set.
 AWS_ACCESS_KEY_ID=testkey
 AWS_SECRET_ACCESS_KEY=testsecret
 AWS_DEFAULT_REGION=us-east-1
-PACKET_REPORTER_BUCKET_NAME=test-bucket
+
+# Packet reporter
+# Set AWS_ENDPOINT for an S3-compatible provider; AWS_KEY/AWS_SECRET are then
+# required and AWS_REGION is ignored. Leave AWS_ENDPOINT empty for real AWS S3,
+# which uses AWS_REGION and the AWS_* credentials above.
+HPR_PACKET_REPORTER_AWS_BUCKET=test-bucket
+HPR_PACKET_REPORTER_AWS_ENDPOINT=
+HPR_PACKET_REPORTER_AWS_BUCKET_REGION=us-east-1
+HPR_PACKET_REPORTER_AWS_KEY=
+HPR_PACKET_REPORTER_AWS_SECRET=
+
+# Gateway liveness reporter
+HPR_LIVENESS_REPORTER_AWS_BUCKET=test-bucket
+HPR_LIVENESS_REPORTER_AWS_ENDPOINT=
+HPR_LIVENESS_REPORTER_AWS_BUCKET_REGION=us-east-1
+HPR_LIVENESS_REPORTER_AWS_KEY=
+HPR_LIVENESS_REPORTER_AWS_SECRET=
 
 # Config Service
 HPR_CONFIG_WORKER_HOST=127.0.0.1

.github/workflows/ci.yml

@@ -164,11 +164,14 @@ jobs:
           AWS_SUITES="["
           for F in ./test/*_SUITE.erl; do
             SUITE=$(basename "$F")
-            if [ "$SUITE" = "hpr_packet_reporter_SUITE.erl" ]; then
-              AWS_SUITES+="\"${SUITE}\","
-            else
-              CT_SUITES+="\"${SUITE}\","
-            fi
+            case "$SUITE" in
+              hpr_packet_reporter_SUITE.erl|hpr_gateway_liveness_reporter_SUITE.erl)
+                AWS_SUITES+="\"${SUITE}\","
+                ;;
+              *)
+                CT_SUITES+="\"${SUITE}\","
+                ;;
+            esac
           done
           CT_SUITES=${CT_SUITES%,}
           CT_SUITES+="]"
@@ -218,34 +221,37 @@ jobs:
       group: ${{ github.workflow }}-${{ github.ref }}-ct-aws-${{ matrix.suite }}
       cancel-in-progress: true
     services:
-      localstack:
-        image: localstack/localstack:1.4
+      rustfs:
+        image: rustfs/rustfs:1.0.0-alpha.90
         env:
-          SERVICES: s3
-        ports:
-          - "4566-4583:4566-4583"
-        volumes:
-          - "/var/lib/localstack:/var/lib/localstack"
-          - "/var/run/docker.sock:/var/run/docker.sock"
+          RUSTFS_ACCESS_KEY: admin
+          RUSTFS_SECRET_KEY: admin
+        options: >-
+          --health-cmd "curl -f http://127.0.0.1:9000/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+          --health-start-period 10s
     env:
-      HPR_PACKET_REPORTER_LOCAL_HOST: ${{ secrets.HPR_PACKET_REPORTER_LOCAL_HOST }}
-      HPR_PACKET_REPORTER_LOCAL_PORT: ${{ secrets.HPR_PACKET_REPORTER_LOCAL_PORT }}
+      # The job runs inside a container, so the rustfs service is reachable by
+      # its label rather than on localhost as it is under docker compose.
+      HPR_TEST_S3_ENDPOINT: http://rustfs:9000
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: admin
+      AWS_DEFAULT_REGION: us-east-1
     steps:
       - name: checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
       - name: Install AWS CLI
         run: apk add --no-cache aws-cli
-      - name: Localstack init
-        timeout-minutes: 1
+      - name: Create rustfs bucket
+        timeout-minutes: 2
         run: |
-          until curl -sf http://localstack:4566/health > /dev/null 2>&1; do
-            echo "...waiting for Localstack..."
+          until curl -sf http://rustfs:9000/health > /dev/null 2>&1; do
+            echo "...waiting for rustfs..."
             sleep 2
           done
-          aws configure set aws_access_key_id testkey
-          aws configure set aws_secret_access_key testsecret
-          aws configure set default.region us-east-1
-          aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket
+          aws --endpoint-url=http://rustfs:9000 s3 mb s3://test-bucket
       - name: Restore cache
         uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
         with:

Makefile

@@ -24,12 +24,15 @@ test: | $(grpc_services_directory)
 	$(REBAR) dialyzer
 	$(REBAR) release
 	$(REBAR) eunit -v
+	docker compose up -d --wait;
 	$(REBAR) ct --readable=true
+	docker compose down -v
 
+# Reporter suites need rustfs; ct.config already points at localhost:9000.
 test-aws:
-	docker compose -f docker-compose-ct.yaml up -d;
-	HPR_PACKET_REPORTER_LOCAL_HOST=localhost $(REBAR) ct --readable=true --suite=hpr_packet_reporter_SUITE;
-	docker compose -f docker-compose-ct.yaml down
+	docker compose up -d --wait;
+	$(REBAR) ct --readable=true --suite=hpr_packet_reporter_SUITE,hpr_gateway_liveness_reporter_SUITE;
+	docker compose down -v
 
 
 rel: | $(grpc_services_directory)

config/ct.config

@@ -5,13 +5,19 @@
         {hpr_free_net_ids, [16#C00053]},
         {packet_reporter, #{
             aws_bucket => <<"test-bucket">>,
-            aws_bucket_region => <<"local">>,
+            %% rustfs from docker-compose.yaml. CI reaches it under a different
+            %% hostname and overrides this via HPR_TEST_S3_ENDPOINT.
+            aws_endpoint => <<"http://localhost:9000">>,
+            aws_access_key_id => <<"admin">>,
+            aws_secret_access_key => <<"admin">>,
             report_interval => 300_000,
             report_max_size => 50_000_000
         }},
         {gateway_liveness_reporter, #{
             aws_bucket => <<"test-bucket">>,
-            aws_bucket_region => <<"local">>,
+            aws_endpoint => <<"http://localhost:9000">>,
+            aws_access_key_id => <<"admin">>,
+            aws_secret_access_key => <<"admin">>,
             %% far in the future; tests trigger checkpoint/report directly via cast
             report_interval => <<"0 0 0 1 1 *">>,
             checkpoint_interval => <<"0 0 0 1 1 *">>,

config/sys.config.src

@@ -10,16 +10,24 @@
         {metrics_port, "${HPR_METRICS_PORT}"},
         {packet_reporter, #{
             aws_bucket => <<"${HPR_PACKET_REPORTER_AWS_BUCKET}">>,
-            aws_bucket_region => <<"${HPR_PACKET_REPORTER_AWS_BUCKET_REGION}">>,
-            %% Optional S3-compatible endpoint, e.g.
-            %% <<"https://t3.storageapi.dev">>. Leave empty for real AWS S3.
+            %% Set aws_endpoint for an S3-compatible provider, e.g.
+            %% <<"https://t3.storageapi.dev">>; the key/secret below are then
+            %% required. Leave it empty for real AWS S3 in aws_bucket_region, with
+            %% credentials from the aws_credentials provider chain.
             aws_endpoint => <<"${HPR_PACKET_REPORTER_AWS_ENDPOINT}">>,
+            aws_bucket_region => <<"${HPR_PACKET_REPORTER_AWS_BUCKET_REGION}">>,
+            aws_access_key_id => <<"${HPR_PACKET_REPORTER_AWS_KEY}">>,
+            aws_secret_access_key => <<"${HPR_PACKET_REPORTER_AWS_SECRET}">>,
             report_interval => 300_000,
             report_max_size => 50_000_000
         }},
         {gateway_liveness_reporter, #{
             aws_bucket => <<"${HPR_LIVENESS_REPORTER_AWS_BUCKET}">>,
+            %% See packet_reporter above for the endpoint / region rule.
+            aws_endpoint => <<"${HPR_LIVENESS_REPORTER_AWS_ENDPOINT}">>,
             aws_bucket_region => <<"${HPR_LIVENESS_REPORTER_AWS_BUCKET_REGION}">>,
+            aws_access_key_id => <<"${HPR_LIVENESS_REPORTER_AWS_KEY}">>,
+            aws_secret_access_key => <<"${HPR_LIVENESS_REPORTER_AWS_SECRET}">>,
             %% Name this instance reports itself as. Leave empty to use the
             %% hostname; set it where the hostname is meaningless.
             server_name => <<"${HPR_LIVENESS_REPORTER_SERVER_NAME}">>,
@@ -80,12 +88,11 @@
         {devaddr_cache_ttl_ms, 43200000},
         {ics_stream_worker_checkpoint_secs, 300}
     ]},
+    %% Used only by reporters that set no aws_endpoint. A reporter pointed at an
+    %% S3-compatible endpoint authenticates with its own key/secret above and
+    %% never consults this chain.
     {aws_credentials, [
         {credential_providers, [aws_credentials_env, aws_credentials_ec2]}
-        %% uncomment for env usage
-        %% {aws_access_key_id, "${HPR_PACKET_REPORTER_AWS_KEY}"},
-        %% {aws_secret_access_key, "${HPR_PACKET_REPORTER_AWS_SECRET}"},
-        %% {aws_default_region, "${HPR_PACKET_REPORTER_AWS_REGION}"}
     ]},
     {chatterbox, [
         {client_initial_window_size, 16777216},

docker-compose.yaml

@@ -1,29 +1,43 @@
-version: '3'
 services:
-  hpr:
-    build:
-      context: .
-    image: quay.io/team-helium/hpr:local
-    container_name: helium_packet_router
+  rustfs:
+    image: rustfs/rustfs:1.0.0-alpha.90
+    container_name: hpr_rustfs
     restart: unless-stopped
-    env_file: .env
     ports:
-      - 8080:8080
+      - 9000:9000
+      - 9001:9001
     volumes:
-      - /var/data/hpr:/var/data
-      - /var/log/hpr:/var/log/hpr
-  promtail:
-    image: grafana/promtail
-    container_name: hpr_promtail
-    restart: unless-stopped
+      - rustfs_data:/data
+    environment:
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=admin
+      - RUSTFS_SECRET_KEY=admin
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh",
+          "-c",
+          "curl -f http://127.0.0.1:9000/health && curl -f
+            http://127.0.0.1:9001/rustfs/console/health",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  rustfs-init:
+    image: amazon/aws-cli:2.34.18
+    depends_on:
+      rustfs:
+        condition: service_healthy
+    restart: no
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=admin
     volumes:
-      - /var/log:/var/log
-      - /var/data/hpr/promtail-config.yaml:/etc/promtail/promtail-config.yaml
-    command: -config.file=/etc/promtail/promtail-config.yaml
-  prometheus:
-      image: prom/prometheus
-      container_name: hpr_prometheus
-      restart: unless-stopped
-      volumes:
-          - /var/data/hpr/prometheus.yml:/etc/prometheus/prometheus.yml
-
+      - ./infra/rustfs/init.sh:/init.sh:ro
+    entrypoint: [ "/init.sh" ]
+
+volumes:
+  rustfs_data:

infra/rustfs/init.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+echo "Initializing RustFS buckets..."
+
+# Bucket both report uploaders write to in CT
+aws --endpoint-url http://rustfs:9000 s3 mb s3://test-bucket 2>/dev/null || {
+    echo "Bucket test-bucket already exists"
+}
+
+echo "RustFS initialization complete"