Add temporl worker connection failure

Author: Harsh9485

rules/cre-2025-0074/temporal-worker-connection-failure.yaml

@@ -0,0 +1,67 @@
+rules:
+  - cre:
+      id: CRE-2025-0074
+      severity: 0
+      title: Temporal Worker → Server Downtime → Connection Refused Failure
+      category: workflow-orchestration-connectivity
+      author: Prequel
+      description: |
+        Detects failure when a Temporal worker is unable to reach the Temporal server.
+        - This typically occurs during startup or after server downtime.
+        - Worker log contains gRPC error: "connection refused".
+      
+      cause: |
+        ROOT CAUSES:
+        - Temporal server was stopped or not reachable on port 7233.
+        - Worker container started before the server was healthy.
+        - Network misconfiguration or resource contention during startup.
+      
+      impact: |
+        BUSINESS IMPACT:
+        - Temporal workflows do not start or progress.
+        - Tasks queue up without being picked up by a worker.
+        - Can go unnoticed without active monitoring.
+      
+      impactScore: 10
+      tags:
+        - temporal
+        - worker
+        - grpc
+        - connection-refused
+        - startup-failure
+        
+      mitigation: |
+        IMMEDIATE:
+        - Confirm server is running: `docker ps`, `docker logs temporal`
+        - Ensure worker starts only after server is available.
+        - Restart worker manually or with retry logic.
+        RECOVERY ACTIONS:
+        - Add health checks for the server container.
+        - Use startup dependencies in Docker Compose.
+        - Improve alerting for “connection refused” messages.
+        PREVENTION STRATEGIES:
+        - Monitor port 7233 availability.
+        - Delay worker startup until Temporal is healthy.
+        - Add retry logic in the worker code.
+      mitigationScore: 6
+      references:
+        - https://docs.temporal.io/
+        - https://docs.temporal.io/production-deployment/production-deployment-overview
+        - https://grpc.io/docs/guides/
+        - https://github.com/temporalio/samples-go
+
+      applications:
+        - name: "temporal"
+          version: ">=1.27.0"
+    metadata:
+      kind: prequel
+      id: 7AUUSrCz9ssNvDvPwBVfAK
+
+    rule:
+      sequence:
+        window: 120s
+        event:
+          source: cre.log.temporal.worker
+        order:
+          - regex: "Failed to connect to Temporal: failed reaching server"
+          - regex: "connection refused"

rules/cre-2025-0074/test.log

@@ -0,0 +1,8 @@
+2025-06-05T09:40:02Z worker-1  | 2025/06/05 09:35:42 INFO  No logger configured for temporal client. Created default one.
+2025-06-05T09:40:02Z worker-1  | 2025/06/05 09:35:42 Failed to connect to Temporal: failed reaching server: Frontend is not healthy yet
+2025-06-05T09:40:02Z worker-1  | 2025/06/05 09:39:47 INFO  No logger configured for temporal client. Created default one.
+2025-06-05T09:40:02Z worker-1  | 2025/06/05 09:39:47 INFO  Started Worker Namespace default TaskQueue example-task-queue WorkerID 1@4e2f67868860@
+2025-06-05T09:40:06Z worker-1  | 2025/06/05 09:40:06 WARN  Failed to poll for task. Namespace default TaskQueue example-task-queue WorkerID 1@4e2f67868860@ WorkerType ActivityWorker Error closing transport due to: connection error: desc = "error reading from server: EOF", received prior goaway: code: NO_ERROR, debug data: "graceful_stop"
+2025-06-05T09:40:06Z worker-1  | 2025/06/05 09:40:06 WARN  Failed to poll for task. Namespace default TaskQueue example-task-queue WorkerID 1@4e2f67868860@ WorkerType WorkflowWorker Error closing transport due to: connection error: desc = "error reading from server: EOF", received prior goaway: code: NO_ERROR, debug data: "graceful_stop"
+2025-06-05T09:40:06Z worker-1  | 2025/06/05 09:40:06 WARN  Failed to poll for task. Namespace default TaskQueue example-task-queue WorkerID 1@4e2f67868860@ WorkerType WorkflowWorker Error last connection error: connection error: desc = "transport: Error while dialing: dial tcp 172.22.0.3:7233: connect: connection refused"
+2025-06-05T09:40:06Z worker-1  | 2025/06/05 09:40:06 WARN  Failed to poll for task. Namespace default TaskQueue example-task-queue WorkerID 1@4e2f67868860@ WorkerType ActivityWorker Error last connection error: connection error: desc = "transport: Error while dialing: dial tcp 172.22.0.3:7233: connect: connection refused"

rules/tags/categories.yaml

@@ -138,3 +138,6 @@ categories:
   - name: demo-problems
     displayName: Demo Problems
     description: This is a category for demos
+  - name: distributed-worker-connectivity
+    displayName: Distributed Worker Connectivity Issues
+    description: Failures where a distributed systems worker fails to reach or stay connected to the orchestration backend (e.g., Temporal, Celery).

rules/tags/tags.yaml

@@ -111,6 +111,9 @@ tags:
   - name: prometheus
     displayName: Prometheus
     description: Problems with scraping, rule evaluation, or querying Prometheus data.
+  - name: connection-refused-startup
+    displayName: Connection Refused on Startup
+    description: Failures that occur when a service (e.g., Temporal worker) tries to connect to a backend and receives a connection refused error.
   - name: psycopg2
     displayName: Psycopg2
     description: Python client errors related to connecting or querying PostgreSQL using psycopg2.