chore: sync develop → main

.github/workflows/collect-lime-results.yml

@@ -22,6 +22,7 @@ on:
 
 permissions:
   contents: write
+  pull-requests: write
 
 concurrency:
   group: collect-lime-results
@@ -124,16 +125,38 @@ jobs:
           echo "Collected $new_files new report.xml file(s)."
           echo "NEW_FILES=$new_files" >> "$GITHUB_ENV"
 
-      - name: Commit and push results
+      # `develop` is protected (signed commits required, PRs only), so we
+      # cannot push directly from the bot. Open a PR with API-signed
+      # commits via peter-evans/create-pull-request and auto-merge it.
+      - name: Create PR with collected results
         if: env.NEW_FILES != '0'
+        id: cpr
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          base: develop
+          branch: bot/lime-results
+          delete-branch: true
+          sign-commits: true
+          commit-message: |
+            ci: collect lime-packages test results (${{ env.NEW_FILES }} new)
+          title: 'ci: collect lime-packages test results'
+          body: |
+            Automated PR with ${{ env.NEW_FILES }} freshly downloaded `report.xml`
+            file(s) from `fcefyn-testbed/lime-packages` CI artifacts.
+
+            Triggered by the `Collect lime-packages test results` workflow.
+          labels: |
+            ci
+            automated
+          add-paths: |
+            docs/ci-results/results/
+
+      - name: Enable auto-merge
+        if: steps.cpr.outputs.pull-request-number
+        env:
+          GH_TOKEN: ${{ secrets.BOT_PR_TOKEN }}
         run: |
-          set -euo pipefail
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add docs/ci-results/results/
-          if git diff --cached --quiet; then
-            echo "Nothing to commit."
-            exit 0
-          fi
-          git commit -m "ci: collect lime-packages test results ($NEW_FILES new)"
-          git push origin develop
+          gh pr merge --auto --squash \
+            --repo "${{ github.repository }}" \
+            "${{ steps.cpr.outputs.pull-request-number }}"

ansible/roles/observability/defaults/main.yml

@@ -106,3 +106,12 @@ grafana_config:
   cookie_secure: "true"
   cookie_samesite: "lax"
   allow_sign_up: "false"
+
+# Anonymous access: let anyone browse dashboards as read-only Viewer
+# without logging in. No VPS changes needed; Grafana handles auth.
+# Set enabled: false to revert to login-only access.
+grafana_anonymous:
+  enabled: "true"
+  org_name: "Main Org."
+  org_role: "Viewer"
+  hide_version: "true"

ansible/roles/observability/files/alerting/fcefyn-alerts.yaml

@@ -0,0 +1,279 @@
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: fcefyn-testbed
+    folder: FCEFyN Alerts
+    interval: 1m
+    rules:
+
+      # -----------------------------------------------------------------------
+      # Target down
+      # Fires when any Prometheus target (DUT or orchestrator) stops responding.
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-target-down
+        title: Target down
+        condition: C
+        for: 2m
+        annotations:
+          summary: "{{ $labels.job }} / {{ $labels.instance }} is down"
+          description: "Prometheus target has been unreachable for more than 2 minutes."
+        labels:
+          severity: critical
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "up == 0"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # High CPU (orchestrator host)
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-high-cpu
+        title: High CPU usage
+        condition: C
+        for: 5m
+        annotations:
+          summary: "High CPU on {{ $labels.instance }}"
+          description: "CPU usage has been above 90% for more than 5 minutes."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "100 - (avg by (instance, dut) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [90]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # High memory
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-high-memory
+        title: High memory usage
+        condition: C
+        for: 5m
+        annotations:
+          summary: "High memory on {{ $labels.instance }}"
+          description: "Memory usage has been above 90% for more than 5 minutes."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [90]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # Low disk space
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-low-disk
+        title: Low disk space
+        condition: C
+        for: 0m
+        annotations:
+          summary: "Low disk on {{ $labels.instance }} ({{ $labels.mountpoint }})"
+          description: "Disk usage is above 85% on {{ $labels.mountpoint }}."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "(1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\"}) * 100"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [85]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # DUT metrics tunnel down
+      # Checks that autossh tunnel units are not in failed state.
+      # Metric comes from the orchestrator's node_exporter.
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-tunnel-down
+        title: DUT metrics tunnel down
+        condition: C
+        for: 2m
+        annotations:
+          summary: "Metrics tunnel failed: {{ $labels.name }}"
+          description: "autossh unit {{ $labels.name }} is in failed state — DUT metrics may be stale."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "node_systemd_unit_state{name=~\"dut-metrics-tunnel-.+\\.service\", state=\"failed\"} == 1"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # Systemd units in failed state (orchestrator host)
+      # Catches prometheus, grafana, labgrid-coordinator, etc.
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-systemd-failed
+        title: Systemd unit failed
+        condition: C
+        for: 1m
+        annotations:
+          summary: "Systemd unit failed on {{ $labels.instance }}: {{ $labels.name }}"
+          description: "A systemd unit is in failed state on the orchestration host."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "node_systemd_unit_state{state=\"failed\", name!~\"dut-metrics-tunnel-.+\\.service\"} == 1"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last
+
+      # -----------------------------------------------------------------------
+      # Network errors (orchestrator host)
+      # Sustained error rate on physical interfaces.
+      # -----------------------------------------------------------------------
+      - uid: fcefyn-network-errors
+        title: Network interface errors
+        condition: C
+        for: 5m
+        annotations:
+          summary: "Network errors on {{ $labels.instance }} / {{ $labels.device }}"
+          description: "Interface {{ $labels.device }} has a sustained error/drop rate over 5 minutes."
+        labels:
+          severity: warning
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: "rate(node_network_receive_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_transmit_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_receive_drop_total{device!~\"lo|veth.+|docker.+\"}[5m]) > 0.1"
+              instant: true
+              refId: A
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: classic_conditions
+              refId: C
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+                  operator:
+                    type: and
+                  query:
+                    params: [A]
+                  reducer:
+                    type: last