add s3 smoke test

you-rgb · you-rgb · commit 0886a447d040 · 2025-09-19T11:27:40.000+10:00
diff --git a/.github/workflows/infinia-deploy-aws.yml b/.github/workflows/infinia-deploy-aws.yml
@@ -120,3 +120,17 @@ jobs:
           done
           echo "Verification failed after ${ATTEMPTS} attempts ❌"
           exit 1
+
+      
+      # ---------- S3 Smoke test --------------------------
+      - name: S3 API smoke test (Python via SSM)
+        working-directory: ${{ github.workspace }}
+        env:
+          AWS_REGION: us-east-1
+        run: |
+          python3 scripts/s3_smoke_test.py \
+            --region "${{ inputs.aws_region }}" \
+            --admin-password "${{ secrets.INFINIA_ADMIN_PASSWORD }}" \
+            --realm-tag-key "Role" --realm-tag-value "realm" \
+            --client-tag-key "Role" --client-tag-value "client" \
+            --endpoint-port 8111
diff --git a/.github/workflows/main-aws.yml b/.github/workflows/main-aws.yml
@@ -39,11 +39,11 @@ jobs:
       infinia_version: ${{ inputs.infinia_version }}
     secrets: inherit
 
-  infinia-destroy:
-    name: Destroy Infinia Infrastructure
-    needs: infinia-deploy
-    if: always()
-    uses: ./.github/workflows/infinia-destroy-aws.yml
-    with:
-      aws_region: us-east-1
-    secrets: inherit
+  # infinia-destroy:
+  #   name: Destroy Infinia Infrastructure
+  #   needs: infinia-deploy
+  #   if: always()
+  #   uses: ./.github/workflows/infinia-destroy-aws.yml
+  #   with:
+  #     aws_region: us-east-1
+  #   secrets: inherit
diff --git a/scripts/s3_smoke_test.py b/scripts/s3_smoke_test.py
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+End-to-end S3 API smoke test for Infinia, driven via AWS SSM.
+
+Flow:
+1) Find the realm EC2 by tag (e.g., Role=realm). Get its instance ID and public IP.
+2) On the realm instance (via SSM), run redcli to create s3_test_user and mint S3 access keys.
+3) Parse S3_KEY and S3_SECRET from the SSM output.
+4) Find all client EC2 instances by tag (e.g., Role=client).
+5) On each client (via SSM), write ~/.aws/credentials with the generated keys, then:
+   - create a unique bucket (ignore “already exists”),
+   - upload a timestamped file via aws s3 cp to the Infinia endpoint,
+   - list bucket objects and verify the file appears.
+
+Exit codes:
+  0 on success; non-zero on failure.
+
+Requirements on the runner:
+  - AWS credentials set (GitHub action `configure-aws-credentials` is fine)
+  - boto3, botocore installed (your pipeline already installs boto3)
+
+Assumptions:
+  - Clients already have AWS CLI v2 installed (as per your note).
+  - redcli is installed on the realm instance.
+  - The S3 endpoint is https://<realm-public-ip>:<endpoint_port>.
+"""
+
+import argparse
+import os
+import re
+import sys
+import time
+import json
+import random
+import datetime
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def find_one_instance_id_by_tag(ec2, key, value):
+    resp = ec2.describe_instances(
+        Filters=[
+            {"Name": f"tag:{key}", "Values": [value]},
+            {"Name": "instance-state-name", "Values": ["running"]},
+        ],
+        MaxResults=50,
+    )
+    for r in resp.get("Reservations", []):
+        for i in r.get("Instances", []):
+            return i["InstanceId"]
+    return None
+
+
+def get_public_ip_by_instance_id(ec2, instance_id):
+    resp = ec2.describe_instances(InstanceIds=[instance_id])
+    inst = resp["Reservations"][0]["Instances"][0]
+    return inst.get("PublicIpAddress")
+
+
+def list_instance_ids_by_tag(ec2, key, value):
+    ids = []
+    paginator = ec2.get_paginator("describe_instances")
+    for page in paginator.paginate(
+        Filters=[
+            {"Name": f"tag:{key}", "Values": [value]},
+            {"Name": "instance-state-name", "Values": ["running"]},
+        ]
+    ):
+        for r in page.get("Reservations", []):
+            for i in r.get("Instances", []):
+                ids.append(i["InstanceId"])
+    return ids
+
+
+def ssm_send_and_wait(ssm, instance_ids, commands, comment, timeout_sec=600, poll_sec=5):
+    """Send AWS-RunShellScript to one or many instances and wait for completion.
+
+    Returns dict: {instance_id: {"Status": str, "StdOut": str, "StdErr": str}}
+    """
+    if isinstance(instance_ids, str):
+        instance_ids = [instance_ids]
+
+    resp = ssm.send_command(
+        DocumentName="AWS-RunShellScript",
+        InstanceIds=instance_ids,
+        Comment=comment,
+        Parameters={"commands": commands},
+    )
+    cmd_id = resp["Command"]["CommandId"]
+
+    deadline = time.time() + timeout_sec
+    results = {iid: None for iid in instance_ids}
+
+    while time.time() < deadline:
+        done = True
+        for iid in instance_ids:
+            if results[iid] is not None:
+                continue
+            try:
+                inv = ssm.get_command_invocation(CommandId=cmd_id, InstanceId=iid)
+            except ssm.exceptions.InvocationDoesNotExist:
+                done = False
+                continue
+
+            status = inv.get("Status")
+            if status in ("Success", "Failed", "Cancelled", "TimedOut"):
+                results[iid] = {
+                    "Status": status,
+                    "StdOut": inv.get("StandardOutputContent", ""),
+                    "StdErr": inv.get("StandardErrorContent", ""),
+                }
+            else:
+                done = False
+
+        if done:
+            break
+        time.sleep(poll_sec)
+
+    # Fill any missing as timeout
+    for iid in instance_ids:
+        if results[iid] is None:
+            results[iid] = {"Status": "TimedOut", "StdOut": "", "StdErr": ""}
+
+    return results
+
+
+def parse_redcli_creds_from_text(text):
+    """
+    Extract S3_KEY and S3_SECRET from redcli pretty-table-like output.
+
+    Accepts lines like:
+      │ S3_KEY    │ AKIA... │
+      │ S3_SECRET │ wJalrXUtnFEMI/K7MDENG/bPxRfiCY... │
+    """
+    key_match = re.search(r"S3_KEY[^\w-]*([A-Za-z0-9_-]+)", text)
+    sec_match = re.search(r"S3_SECRET[^\w+/=_-]*([A-Za-z0-9+/=_-]+)", text)
+    s3_key = key_match.group(1) if key_match else ""
+    s3_secret = sec_match.group(1) if sec_match else ""
+    return s3_key.strip(), s3_secret.strip()
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Infinia S3 API smoke test via SSM")
+    ap.add_argument("--region", required=True, help="AWS region (e.g. us-east-1)")
+    ap.add_argument("--admin-password", required=True, help="Infinia realm_admin password")
+    ap.add_argument("--realm-tag-key", default="Role")
+    ap.add_argument("--realm-tag-value", default="realm")
+    ap.add_argument("--client-tag-key", default="Role")
+    ap.add_argument("--client-tag-value", default="client")
+    ap.add_argument("--endpoint-port", type=int, default=8111)
+    ap.add_argument("--no-verify-ssl", action="store_true", default=True,
+                    help="Use --no-verify-ssl against the endpoint")
+    ap.add_argument("--timeout-sec", type=int, default=900)
+    args = ap.parse_args()
+
+    session = boto3.Session(region_name=args.region)
+    ec2 = session.client("ec2")
+    ssm = session.client("ssm")
+
+    # 1) Realm
+    realm_id = find_one_instance_id_by_tag(ec2, args.realm_tag_key, args.realm_tag_value)
+    if not realm_id:
+        eprint(f"❌ No running realm instance found with tag {args.realm_tag_key}={args.realm_tag_value}")
+        return 2
+    eprint(f"Realm instance: {realm_id}")
+
+    realm_ip = get_public_ip_by_instance_id(ec2, realm_id)
+    if not realm_ip:
+        eprint("❌ Realm public IP not found")
+        return 2
+    endpoint_url = f"https://{realm_ip}:{args.endpoint_port}"
+    eprint(f"Realm endpoint: {endpoint_url}")
+
+    # 2) Create S3 creds on realm (SSM → redcli)
+    redcli_cmds = [
+        'set -euo pipefail',
+        f'redcli user login realm_admin -p "{args.admin_password}"',
+        'redcli user add s3_test_user -p Adminpassword -r red -t red || true',
+        'redcli user grant s3_test_user red/red -t red || true',
+        'OUT="$(redcli s3 access add s3_test_user -t red -r red/red/redobj -e 10y)"',
+        'echo "===BEGIN-OUT==="',
+        'echo "$OUT"',
+        'echo "===END-OUT==="',
+        # Friendly parse lines for easier regex later
+        'echo "S3_KEY=$(echo "$OUT" | sed -n \'s/.*S3_KEY[^A-Za-z0-9_-]*\\([A-Za-z0-9_-]\\+\\).*/\\1/p\' | head -n1)"',
+        'echo "S3_SECRET=$(echo "$OUT" | sed -n \'s/.*S3_SECRET[^A-Za-z0-9+\\/=_-]*\\([A-Za-z0-9+\\/=_-]\\+\\).*/\\1/p\' | head -n1)"',
+    ]
+    realm_result = ssm_send_and_wait(
+        ssm, realm_id, redcli_cmds, "Create s3_test_user and S3 access", timeout_sec=min(args.timeout_sec, 600)
+    )[realm_id]
+
+    if realm_result["Status"] != "Success":
+        eprint("❌ redcli SSM command failed on realm")
+        eprint("STDERR:\n" + realm_result["StdErr"])
+        eprint("STDOUT:\n" + realm_result["StdOut"])
+        return 3
+
+    s3_key, s3_secret = parse_redcli_creds_from_text(realm_result["StdOut"])
+    if not s3_key or not s3_secret:
+        eprint("❌ Failed to parse S3 credentials from realm output")
+        eprint("STDOUT:\n" + realm_result["StdOut"])
+        return 4
+
+    # Do not print secrets in logs
+    eprint(f"S3_KEY parsed: {s3_key[:4]}… (masked)")
+    eprint("S3_SECRET parsed: **** (masked)")
+
+    # 3) Find clients
+    client_ids = list_instance_ids_by_tag(ec2, args.client_tag_key, args.client_tag_value)
+    if not client_ids:
+        eprint(f"❌ No running client instances found with tag {args.client_tag_key}={args.client_tag_value}")
+        return 5
+    eprint(f"Client instances: {', '.join(client_ids)}")
+
+    # 4) S3 test on each client
+    now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
+    bucket = f"infinia-smoke-{now}-{random.randint(1000,9999)}"
+    test_file_cmd = r'TS=$(date +%Y%m%dT%H%M%S); echo "S3 test file created at $TS" > /home/ubuntu/test-$TS.txt'
+
+    create_list_upload_cmds = [
+        "set -euo pipefail",
+        "mkdir -p /home/ubuntu/.aws && chmod 700 /home/ubuntu/.aws",
+        # write credentials (no echo in logs)
+        f"cat > /home/ubuntu/.aws/credentials <<'EOF'\n[default]\naws_access_key_id={s3_key}\naws_secret_access_key={s3_secret}\nEOF",
+        test_file_cmd,
+        "export AWS_SHARED_CREDENTIALS_FILE=/home/ubuntu/.aws/credentials",
+        "aws --version || true",
+        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3api create-bucket --bucket {bucket} || true",
+        r"FILE=$(ls -1t /home/ubuntu/test-*.txt | head -n1)",
+        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3 cp \"$FILE\" s3://{bucket}/",
+        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3api list-objects-v2 --bucket {bucket} --output json --query 'Contents[].Key'",
+        "echo OK",
+    ]
+
+    client_results = ssm_send_and_wait(
+        ssm, client_ids, create_list_upload_cmds, "Infinia S3 smoke test", timeout_sec=args.timeout_sec
+    )
+
+    had_fail = False
+    for iid, res in client_results.items():
+        print(f"---- Client {iid} ({res['Status']}) ----")
+        # Show stdout for visibility but keep credentials masked (we never echo them above)
+        print(res["StdOut"])
+        if res["Status"] != "Success" or "OK" not in res["StdOut"]:
+            eprint(f"❌ S3 smoke test failed on client {iid}")
+            eprint(res["StdErr"])
+            had_fail = True
+
+    if had_fail:
+        return 6
+
+    print("✅ S3 smoke test passed on all clients")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
