From ba450781315cb28cd2decc4783ea446018b77635 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:17:43 -0300 Subject: [PATCH 01/24] docs(ci-results): rewrite publishing.md for pull-based flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish pipeline now lives entirely in fcefyn_testbed_utils via the collect-lime-results.yml workflow that pulls report.xml artifacts from lime-packages instead of relying on a workflow change there. Replaces the old TESTBED_UTILS_TOKEN-based push diagram with the new pull sequence, lists artifact→path mappings, and documents required secrets (LIME_PACKAGES_TOKEN, BOT_PR_TOKEN) plus the repo-level toggles. --- docs/ci-results/publishing.md | 116 +++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/docs/ci-results/publishing.md b/docs/ci-results/publishing.md index 9da9efa..f637ac4 100644 --- a/docs/ci-results/publishing.md +++ b/docs/ci-results/publishing.md @@ -1,68 +1,94 @@ # Publishing Test Results -Test results are published automatically to this repository after each scheduled CI run in `lime-packages`. Once published, the dashboard shows individual test case details instead of "Test details not yet published". +Test reports are pulled into this repository from `fcefyn-testbed/lime-packages` CI runs on a schedule, then served alongside the dashboard. Once published, the dashboard renders individual test-case details instead of "Test details not yet published". ## How it works -The `publish-results` job in `lime-packages/.github/workflows/build-firmware.yml` runs after all test jobs complete on the daily schedule (`0 6 * * *`): +The pipeline runs entirely from this repository — `lime-packages` does not need any changes. The `collect-lime-results.yml` workflow uses the GitHub Actions REST API to download `report.xml` artifacts from recent CI runs and commits them via an auto-merged PR. ```mermaid sequenceDiagram - participant CI as CI (lime-packages) - participant GHA as GitHub Actions Artifacts - participant Utils as fcefyn_testbed_utils - - CI->>GHA: Upload test-results-* artifacts - CI->>CI: publish-results job starts - CI->>Utils: git checkout (develop, TESTBED_UTILS_TOKEN) - CI->>GHA: Download all test-results-* artifacts - CI->>Utils: Copy report.xml files to docs/ci-results/results/ - CI->>Utils: git commit + push - Utils->>Utils: pages.yml triggers GitHub Pages deploy + participant Cron as Cron (every 6h) + participant Workflow as collect-lime-results.yml + participant LimeAPI as lime-packages Actions API + participant Bot as bot/lime-results branch + participant Develop as develop + participant Pages as GitHub Pages + + Cron->>Workflow: Trigger (schedule or workflow_dispatch) + Workflow->>LimeAPI: List recent build-firmware.yml + tests.yml runs
(LIME_PACKAGES_TOKEN) + LimeAPI-->>Workflow: Run IDs and artifact metadata + Workflow->>LimeAPI: Download test-results-* artifact zips + Workflow->>Workflow: Extract report.xml and
map artifact name → results path + Workflow->>Bot: peter-evans/create-pull-request
(commits via API, BOT_PR_TOKEN) + Bot->>Develop: gh pr merge --auto --squash + Develop->>Pages: pages.yml builds + deploys ``` +### Why this direction + +The original plan was for `lime-packages` to push its results into this repo after every CI run, but that requires a workflow change to land in `lime-packages` master — which the upstream LibreMesh project has not accepted. Inverting the direction (this repo pulls from there) means the integration ships on our side without any upstream coupling. + +## Artifact → results path mapping + +The collect workflow recognises these artifact name patterns produced by `lime-packages` CI and routes them to the layout the dashboard expects: + +| Artifact name | Destination | +|--|--| +| `test-results-unit` | `docs/ci-results/results/unit/report.xml` | +| `test-results-qemu-single-{device}-{release}` | `docs/ci-results/results/qemu-single/{device}-{release}/report.xml` | +| `test-results-qemu-mesh-{device}-{release}` | `docs/ci-results/results/qemu-mesh/{device}-{release}/report.xml` | +| `test-results-mesh-pairs-{pair}-{release}` | `docs/ci-results/results/mesh-pairs/{pair}-{release}/report.xml` | +| `test-results-mesh-{release}` | `docs/ci-results/results/mesh/{release}/report.xml` | +| `test-results-{place}-{release}` | `docs/ci-results/results/physical/{place}-{release}/report.xml` | + +Already-collected reports are skipped, so re-running the workflow is idempotent and only picks up new runs. + ## Results directory structure ``` docs/ci-results/results/ -├── devices.json # device registry +├── devices.json ├── physical/ -│ ├── belkin_rt3200_1-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_2-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_3-24.10.6/ -│ │ └── report.xml -│ ├── bpi_r4_1-24.10.6/ -│ │ └── report.xml -│ ├── openwrt_one_1-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_1-25.12.2/ -│ │ └── report.xml -│ ├── belkin_rt3200_2-25.12.2/ -│ │ └── report.xml -│ ├── belkin_rt3200_3-25.12.2/ -│ │ └── report.xml -│ ├── bpi_r4_1-25.12.2/ -│ │ └── report.xml -│ └── openwrt_one_1-25.12.2/ -│ └── report.xml +│ ├── belkin_rt3200_1-24.10.6/report.xml +│ ├── belkin_rt3200_2-24.10.6/report.xml +│ ├── belkin_rt3200_3-24.10.6/report.xml +│ ├── bananapi_bpi-r4-24.10.6/report.xml +│ └── openwrt_one-24.10.6/report.xml +├── mesh/ +│ └── 24.10.6/report.xml +├── mesh-pairs/ +│ ├── 1-24.10.6/report.xml +│ ├── 2-24.10.6/report.xml +│ └── 3-24.10.6/report.xml ├── qemu-single/ -│ ├── qemu_x86_64-24.10.6/ -│ │ └── report.xml -│ └── qemu_x86_64-25.12.2/ -│ └── report.xml +│ ├── qemu_x86_64-24.10.6/report.xml +│ └── qemu_x86_64-25.12.2/report.xml └── qemu-mesh/ - ├── qemu_x86_64-24.10.6/ - │ └── report.xml - └── qemu_x86_64-25.12.2/ - └── report.xml + ├── qemu_x86_64-24.10.6/report.xml + └── qemu_x86_64-25.12.2/report.xml ``` -## Required secret +## Required secrets + +| Secret | Stored in | Purpose | Permissions | +|--|--|--|--| +| `LIME_PACKAGES_TOKEN` | `fcefyn_testbed_utils` | Read CI runs and download artifacts from `lime-packages` | Fine-grained PAT scoped to `fcefyn-testbed/lime-packages` with **Actions: Read** | +| `BOT_PR_TOKEN` | `fcefyn_testbed_utils` | Open the auto-merged PR with the collected results | Fine-grained PAT scoped to `fcefyn-testbed/fcefyn_testbed_utils` with **Contents: Read and write** + **Pull requests: Read and write** | + +## Repository settings required -The publish pipeline requires a fine-grained PAT stored as `TESTBED_UTILS_TOKEN` in `lime-packages` repository secrets, with `Contents: Read and write` permission on this repository. +These need to be enabled once for the auto-merge step to work: + +- **Settings → Actions → General → Workflow permissions** → "Allow GitHub Actions to create and approve pull requests" (toggled via the `BOT_PR_TOKEN`, but the repo-level toggle must also be on) +- **Settings → General → Pull Requests** → "Allow auto-merge" +- **`develop` branch protection** → keep "Require signed commits" **off** (PRs created via the GitHub API with a user PAT are not auto-signed by GitHub; enabling this would block the bot) ## Triggering manually -To publish results outside the daily schedule, trigger `build-firmware.yml` via `workflow_dispatch` — the `publish-results` job runs after the test jobs complete. +To force an immediate collection (e.g. after a fresh CI run in `lime-packages`): + +1. Go to **Actions → Collect lime-packages test results**. +2. **Run workflow** → branch `develop` → optionally set `runs` (how many recent CI runs to scan, default 10). + +A bot PR (`bot/lime-results`) appears and auto-merges as soon as required checks pass. If there are no new reports the workflow exits cleanly without opening a PR. From 45dabe0a340483634a914198c7ae6ed902eb406e Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:17:57 -0300 Subject: [PATCH 02/24] docs(ci-results): fix architecture.md report-source URLs PAGES_BASE in dashboard.html points to this repo's Pages (fcefyn-testbed.github.io/fcefyn_testbed_utils/...), not lime-packages. Also remove the stale "publish-results job in build-firmware.yml" reference; reports now come from collect-lime-results.yml on this side. --- docs/ci-results/architecture.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ci-results/architecture.md b/docs/ci-results/architecture.md index 5b8ca07..5d5937e 100644 --- a/docs/ci-results/architecture.md +++ b/docs/ci-results/architecture.md @@ -41,13 +41,13 @@ Used for: **job status, run history, duration, links**. Used for: **per-test-case breakdown** (pass / fail / skip counts and individual test names). -- Served from GitHub Pages alongside `dashboard.html` +- Served from this repo's GitHub Pages alongside `dashboard.html` - JUnit XML format, generated by pytest (`--junitxml`) -- Published automatically after each scheduled CI run by the `publish-results` job in `build-firmware.yml` +- Populated by the `collect-lime-results.yml` workflow, which pulls artifacts from `fcefyn-testbed/lime-packages` CI every 6h — see [Publishing results](publishing.md) - Path convention: `results/{type}/{identifier}/report.xml` - Direct links to reports use `PAGES_BASE` (configured at the top of `dashboard.html`): ```js - const PAGES_BASE = 'https://fcefyn-testbed.github.io/lime-packages/ci-results/results/'; + const PAGES_BASE = 'https://fcefyn-testbed.github.io/fcefyn_testbed_utils/ci-results/results/'; ``` ### Source 3 — `devices.json` From 90e801d690add3ee2298d5d2efe29cb91aca443e Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:18:30 -0300 Subject: [PATCH 03/24] docs(ci-results): rewrite maintenance.md for pull pipeline Replace the TESTBED_UTILS_TOKEN rotation procedure with the two fine-grained PATs the new pipeline uses (LIME_PACKAGES_TOKEN for reading lime-packages artifacts, BOT_PR_TOKEN for opening the auto-merged PR). Expand the troubleshooting table with every failure mode we hit while wiring this up: org-level Actions toggle, repo-level auto-merge toggle, develop branch protection signed-commits gotcha, and the PAGES_BASE host mismatch that surfaced as 404 on Report links. --- docs/ci-results/maintenance.md | 53 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/docs/ci-results/maintenance.md b/docs/ci-results/maintenance.md index 51ef15e..38dd1d6 100644 --- a/docs/ci-results/maintenance.md +++ b/docs/ci-results/maintenance.md @@ -6,29 +6,50 @@ Edit `docs/ci-results/results/devices.json` and open a PR to `develop`. Changes See [Device Registry](devices.md) for the full field reference. -## Rotating the TESTBED_UTILS_TOKEN +## Rotating the tokens -The publish pipeline uses a fine-grained PAT stored in `lime-packages` repository secrets. PATs expire — when that happens the `publish-results` job will fail with a 401 error. +The pull pipeline uses two fine-grained PATs stored as repository secrets in `fcefyn_testbed_utils`. PATs expire — when they do, the `collect-lime-results.yml` workflow fails on the step that uses the expired token. -To rotate: +### `LIME_PACKAGES_TOKEN` -1. Go to `github.com/settings/tokens` → generate a new fine-grained token: +Used to list runs and download artifacts from `lime-packages`. A 401/403 on the "Download lime-packages report.xml artifacts" step means this token expired. + +1. Generate a new fine-grained PAT at `github.com/settings/personal-access-tokens/new`: - Resource owner: `fcefyn-testbed` - - Repository: `fcefyn_testbed_utils` - - Permission: `Contents: Read and write` -2. Go to `github.com/fcefyn-testbed/lime-packages/settings/secrets/actions` -3. Update `TESTBED_UTILS_TOKEN` with the new value + - Repository access: only `lime-packages` + - Permissions: **Actions: Read-only** (Metadata: Read is added automatically) +2. `Settings → Secrets and variables → Actions` on `fcefyn_testbed_utils` +3. Update `LIME_PACKAGES_TOKEN` with the new value + +### `BOT_PR_TOKEN` + +Used to open the auto-merged PR with the collected results. A 401/403 on the "Create PR with collected results" or "Enable auto-merge" step means this one expired. + +1. Generate a new fine-grained PAT at `github.com/settings/personal-access-tokens/new`: + - Resource owner: `fcefyn-testbed` + - Repository access: only `fcefyn_testbed_utils` + - Permissions: **Contents: Read and write** + **Pull requests: Read and write** +2. Update `BOT_PR_TOKEN` in the same secrets page + +## Re-collecting results manually + +To force an immediate pull (e.g. after a fresh CI run in `lime-packages`): -## Re-publishing results manually +1. **Actions → Collect lime-packages test results** +2. **Run workflow** → branch `develop` → optionally set `runs` (default 10 — how many recent CI runs to scan) -If a scheduled run published incomplete results (e.g. some jobs were skipped), trigger `build-firmware.yml` via `workflow_dispatch` to republish. +The workflow is idempotent: it skips reports already present, so running it repeatedly is safe. ## Troubleshooting | Symptom | Likely cause | Fix | -|---------|-------------|-----| -| Cards show "Test details not yet published" | `publish-results` job hasn't run yet | Wait for next schedule or trigger manually | -| `publish-results` fails with 401 | `TESTBED_UTILS_TOKEN` expired | Rotate the token (see above) | -| `publish-results` fails with 403 | Token lacks `Contents: write` on `fcefyn_testbed_utils` | Regenerate with correct permissions | -| Dashboard shows no cards | GitHub API rate limit hit (60 req/h) | Wait ~1 hour and reload | -| Old results not updating | CI skipped `publish-results` (non-schedule event) | Trigger `workflow_dispatch` manually | +|---|---|---| +| Cards show "Test details not yet published" | `report.xml` for that device/release hasn't been collected yet | Trigger `collect-lime-results.yml` manually, or wait for the cron | +| "Report ↗" link returns 404 | Either the report hasn't been collected yet, or `PAGES_BASE` in `dashboard.html` is pointing at the wrong host | Open the link and check the URL host — it must be `fcefyn-testbed.github.io/fcefyn_testbed_utils/` | +| Collect workflow fails with 401/403 on Download step | `LIME_PACKAGES_TOKEN` expired | Rotate (see above) | +| Collect workflow fails with 401/403 on Create PR step | `BOT_PR_TOKEN` expired | Rotate (see above) | +| Collect workflow fails with "GitHub Actions is not permitted to create or approve pull requests" | Org-level Actions setting disabled | Use a PAT (`BOT_PR_TOKEN`) — already the default; if blocked, check **Settings → Actions → General → Workflow permissions** at the org level | +| Collect workflow fails with "Auto merge is not allowed for this repository" | Repo setting disabled | **Settings → General → Pull Requests → Allow auto-merge** | +| Bot PR fails to merge with "Commits must have verified signatures" | `develop` branch protection has signed-commits required | Disable it: the PR is opened with the user PAT and not signed by GitHub — `gh api -X DELETE repos/fcefyn-testbed/fcefyn_testbed_utils/branches/develop/protection/required_signatures` | +| Dashboard shows no cards at all | GitHub API rate limit hit (60 req/h, unauthenticated) | Wait ~1 hour and reload | +| Old results not refreshing | Cron runs every 6h; no new CI runs in `lime-packages` since last pull | Trigger manually if needed | From 8f2c99673d955e93a9eb3c2319caee3bc313de0e Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:18:50 -0300 Subject: [PATCH 04/24] docs(ci-results): fix device coverage table Place names use bananapi_bpi-r4 / openwrt_one (no _1 suffix), single Belkin entries are 24.10.6-only on the lab, and 25.12.2 currently runs only on QEMU. Add the physical mesh / mesh-pairs rows that the dashboard now surfaces. --- docs/ci-results/index.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/ci-results/index.md b/docs/ci-results/index.md index f463b25..30282f5 100644 --- a/docs/ci-results/index.md +++ b/docs/ci-results/index.md @@ -40,10 +40,11 @@ The summary bar at the top shows aggregate counts across all tracked jobs: | Device | Place | Type | Releases | |--------|-------|------|---------| -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_1` | Physical | 24.10.6, 25.12.2 | -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_2` | Physical | 24.10.6, 25.12.2 | -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_3` | Physical | 24.10.6, 25.12.2 | -| Banana Pi BPi-R4 | `bpi_r4_1` | Physical | 24.10.6, 25.12.2 | -| OpenWrt One | `openwrt_one_1` | Physical | 24.10.6, 25.12.2 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_1` | Physical | 24.10.6 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_2` | Physical | 24.10.6 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_3` | Physical | 24.10.6 | +| Banana Pi BPi-R4 | `bananapi_bpi-r4` | Physical | 24.10.6 | +| OpenWrt One | `openwrt_one` | Physical | 24.10.6 | +| 2-node and 3-node physical mesh | — | Mesh / Mesh pairs | 24.10.6 | | QEMU x86-64 | — | QEMU single | 24.10.6, 25.12.2 | | QEMU x86-64 | — | QEMU mesh | 24.10.6, 25.12.2 | From 5d8f6b78d55d99a0db28d27f9e8928f1d180ed41 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:19:32 -0300 Subject: [PATCH 05/24] docs(observability): list Lab Overview dashboard alongside the others Adds the lab-overview.json dashboard to the dashboards table and bumps the Orchestrator Host panel count to ~40 to reflect the new Lab Services and WireGuard sections. --- docs/configuracion/observabilidad.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index b4a7363..b9d809a 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -78,12 +78,13 @@ VPS, Certbot, and tunnel unit: [grafana-public-access.md](grafana-public-access. ## Grafana dashboards -Two dashboards: +Three dashboards: | Dashboard | Source | Description | |-----------|--------|---------------| +| **FCEFyN Testbed - Lab Overview** | Provisioned (JSON in repo) | Lab-wide view: every DUT and the gateway at a glance. Single-row stat panels (scrape `up`, uptime, CPU %, RAM %) plus a consolidated table and a firmware/target table. Uses `dut=~".+"` selectors so the whole fleet renders without picking a device. | | **FCEFyN Testbed - DUTs & gateway** | Provisioned (JSON in repo) | DUTs + WDR3500 gateway. **device** variable with `label_values(up{dut!="lab-orchestrator"}, dut)`: **does not** include the orchestration host. All queries use `dut="$device"` and datasource `uid: prometheus`. | -| **FCEFyN Testbed - Orchestrator Host** | Provisioned (JSON in repo) | Orchestration host (~30 panels). Job `orchestrator-host`, label `dut=lab-orchestrator`. | +| **FCEFyN Testbed - Orchestrator Host** | Provisioned (JSON in repo) | Orchestration host (~40 panels). Job `orchestrator-host`, label `dut=lab-orchestrator`. | ### DUTs & gateway dashboard sections From dbf4064e6b230b8556404c724c711bb7b8050b74 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:19:54 -0300 Subject: [PATCH 06/24] docs(observability): document Lab Services and WireGuard sections Adds rows to the Orchestrator Host sections table describing the new "Lab Services" panels (labgrid-exporter, pdudaemon, ser2net via node_systemd_unit_state) and the "WireGuard (wg0)" tunnel panels (bandwidth, packets, errors, link status). --- docs/configuracion/observabilidad.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index b9d809a..3cb5d68 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -112,6 +112,8 @@ Three dashboards: | Network - Physical | Bandwidth (bps), Packets/s, Errors & Drops, TCP Connections | | Network - VLANs | Bandwidth and packets for vlan100-108, vlan200 (collapsible) | | System Internals | File Descriptors, Entropy, Sockets by Protocol, Systemd Units (active/failed), Socket Memory | +| Lab Services | Active/inactive stats for `labgrid-exporter.service`, `pdudaemon.service`, `ser2net.service` plus a state-timeline showing transitions over the selected range. Source: `node_systemd_unit_state{job="orchestrator-host",name=…,state="active"}`. | +| WireGuard (wg0) | Tunnel bandwidth (rx positive, tx negative), packet rate, error rate, and a UP/DOWN pill from `node_network_up{device="wg0"}`. Same metrics any other ethernet device reports, just filtered to the tunnel. | For **orchestration host only** metrics, always use **Orchestrator Host**; the DUT dashboard excludes it on purpose from the **device** dropdown. From 805607d2cb84c7231405561c3625605a9b695086 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:20:17 -0300 Subject: [PATCH 07/24] docs(observability): list lab-overview.json in Key files table So the inventory of dashboard JSONs in the repo matches what Ansible actually provisions. --- docs/configuracion/observabilidad.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index 3cb5d68..7c460e6 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -302,3 +302,4 @@ Paths are relative to the repository root. | `ansible/roles/observability/templates/grafana-dashboards-provider.yml.j2` | File-based dashboard provider in Grafana | | `ansible/roles/observability/files/dashboards/orchestrator-node.json` | Orchestrator host dashboard JSON | | `ansible/roles/observability/files/dashboards/duts-node.json` | DUTs + gateway dashboard JSON (variable excludes `lab-orchestrator`) | +| `ansible/roles/observability/files/dashboards/lab-overview.json` | Lab-wide overview dashboard JSON (every DUT in one screen) | From eae4a406e9d9fdc8a31a43c9f7ce251eacc43001 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:20:44 -0300 Subject: [PATCH 08/24] docs(observability): describe Lab Overview dashboard sections Per-section breakdown of lab-overview.json so the doc matches what Grafana actually shows: scrape up cards, uptime/CPU/RAM stat panels, consolidated device table, and the firmware/target table sourced from node_openwrt_info labels. --- docs/configuracion/observabilidad.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index 7c460e6..b3051b5 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -100,6 +100,19 @@ Three dashboards: | Wi-Fi | `wifi_network_*` (AP), `wifi_stations` / `wifi_station_signal_dbm` (stations, if opkg packages present) | | Labels | Table of scrape labels (`firmware`, `target`, etc.) from `up{dut="$device"}` | +### Lab Overview dashboard sections + +This dashboard has no `device` variable — every panel iterates over all DUTs at once using `dut=~".+"`. Useful for a quick glance at the whole fleet. + +| Section | Panels | +|---------|--------| +| Device health at a glance | `up{dut=~".+"}` rendered as one stat-card per device with UP/DOWN value mapping | +| Uptime | `node_time_seconds - node_boot_time_seconds`, color-thresholded green→yellow→orange | +| CPU | `100 - avg(rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100` per DUT | +| Memory | `100 - MemAvailable / MemTotal * 100` per DUT | +| Device table | One row per DUT consolidating scrape status, uptime, CPU %, RAM % via `merge` + `organize` transformations | +| Firmware & target | Instant table of `node_openwrt_info{dut=~".+"}` with labels-to-fields, surfacing the `firmware` and `target` labels set in the scrape config | + ### Orchestrator Host dashboard sections | Section | Panels | From 0060e7604e06c1be4b7b74f7b172989fc91395a4 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 16:21:09 -0300 Subject: [PATCH 09/24] docs(observability): note network errors and drops panel on DUTs The duts-node.json dashboard has a per-interface node_network_*_errs_total / *_drop_total rate panel. Was missing from the section table. --- docs/configuracion/observabilidad.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index b3051b5..c587547 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -94,7 +94,7 @@ Three dashboards: | Device info | Instant tables `node_uname_info`, `node_openwrt_info` | | CPU & load | CPU by mode (stacked), load 1/5/15m | | Memory | Total / available / used | -| Network | Traffic and packets per interface (excluding `lo`) | +| Network | Traffic and packets per interface (excluding `lo`); plus a per-interface errors and drops panel with `rate(node_network_{receive,transmit}_{errs,drop}_total[2m])` | | Disk | Usage % per mountpoint, free space | | Temperature | `node_hwmon_temp_celsius`, `node_thermal_zone_temp`, CPU stats / max / ieee80211 radios | | Wi-Fi | `wifi_network_*` (AP), `wifi_stations` / `wifi_station_signal_dbm` (stations, if opkg packages present) | From 9b3f9a7364be15d68d83e3e6d40ba3268671eea0 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 17:58:19 -0300 Subject: [PATCH 10/24] docs(ci-results): fix registered devices table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bpi_r4_1 / openwrt_one_1 place names never existed in the labgrid config — the actual places are bananapi_bpi-r4 and openwrt_one. Also drop the 25.12.2 physical rows (that release is QEMU-only on the lab right now) and add the physical mesh / mesh-pair entries that prepare_matrix.sh emits. The "Adding a new device" steps now point at the pull-based collect workflow. --- docs/ci-results/devices.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/ci-results/devices.md b/docs/ci-results/devices.md index 3c64c64..e912d93 100644 --- a/docs/ci-results/devices.md +++ b/docs/ci-results/devices.md @@ -69,13 +69,17 @@ The dashboard recognises the following CI job name patterns and maps them to typ | `physical-belkin_rt3200_1-24.10.6` | `linksys_e8450` | `belkin_rt3200_1` | 24.10.6 | | `physical-belkin_rt3200_2-24.10.6` | `linksys_e8450` | `belkin_rt3200_2` | 24.10.6 | | `physical-belkin_rt3200_3-24.10.6` | `linksys_e8450` | `belkin_rt3200_3` | 24.10.6 | -| `physical-bpi_r4_1-24.10.6` | `bananapi_bpi-r4` | `bpi_r4_1` | 24.10.6 | -| `physical-openwrt_one_1-24.10.6` | `openwrt_one` | `openwrt_one_1` | 24.10.6 | -| `physical-belkin_rt3200_1-25.12.2` | `linksys_e8450` | `belkin_rt3200_1` | 25.12.2 | -| `physical-belkin_rt3200_2-25.12.2` | `linksys_e8450` | `belkin_rt3200_2` | 25.12.2 | -| `physical-belkin_rt3200_3-25.12.2` | `linksys_e8450` | `belkin_rt3200_3` | 25.12.2 | -| `physical-bpi_r4_1-25.12.2` | `bananapi_bpi-r4` | `bpi_r4_1` | 25.12.2 | -| `physical-openwrt_one_1-25.12.2` | `openwrt_one` | `openwrt_one_1` | 25.12.2 | +| `physical-bananapi_bpi-r4-24.10.6` | `bananapi_bpi-r4` | `bananapi_bpi-r4` | 24.10.6 | +| `physical-openwrt_one-24.10.6` | `openwrt_one` | `openwrt_one` | 24.10.6 | + +### Physical mesh + +| ID | Type | Release | +|----|------|---------| +| `physical-mesh-24.10.6` | Multi-node mesh (N=2 or N=3 from `prepare_matrix.sh`) | 24.10.6 | +| `mesh-pair-1-24.10.6` | Pair #1 — `openwrt_one + bananapi_bpi-r4` | 24.10.6 | +| `mesh-pair-2-24.10.6` | Pair #2 — `openwrt_one + belkin_rt3200_2` | 24.10.6 | +| `mesh-pair-3-24.10.6` | Pair #3 — `bananapi_bpi-r4 + belkin_rt3200_2` | 24.10.6 | ### QEMU @@ -88,6 +92,6 @@ The dashboard recognises the following CI job name patterns and maps them to typ ## Adding a new device -1. Add the device to `targets.yml` in `lime-packages` with `test_firmware: true` and the corresponding `test_places` entry +1. Add the device to `targets.yml` in `lime-packages` with `test_firmware: true` (and `test_places` if the labgrid place name differs from the device name) 2. Add an entry to `devices.json` with the correct `id`, `place`, `release`, and `results_path` -3. The CI publish pipeline will start populating `report.xml` after the next scheduled run +3. The next scheduled run of `collect-lime-results.yml` (every 6h) will start pulling the new `report.xml`; trigger it manually if you don't want to wait From 49fca32fedec3a79e130a10e275c4f74700b2e66 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 17:58:59 -0300 Subject: [PATCH 11/24] docs(ci-results): update search examples to current place names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit \`bpi\` no longer matches anything — the place is bananapi_bpi-r4. Adds \`bananapi\` and \`openwrt\` as more useful examples that match the real labgrid place names. --- docs/ci-results/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ci-results/usage.md b/docs/ci-results/usage.md index 6e3f918..a95c310 100644 --- a/docs/ci-results/usage.md +++ b/docs/ci-results/usage.md @@ -53,7 +53,7 @@ Use the **release dropdown** to filter cards by OpenWrt release (e.g. `24.10.6`, ### By name -The **search box** filters by device name or place as you type (e.g. `belkin`, `bpi`, `qemu`, `25.12`). +The **search box** filters by device name or place as you type (e.g. `belkin`, `bananapi`, `openwrt`, `qemu`, `25.12`). All three filters (type tab, release dropdown, search) apply simultaneously. From 771a315f8693fa9206ae8a06c9a8a9371e7061eb Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 17:59:25 -0300 Subject: [PATCH 12/24] docs(glossary): add auto-merge entry Central to the collect-lime-results.yml pipeline, worth defining alongside the other CI/Labgrid terms. --- docs/glosario.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/glosario.md b/docs/glosario.md index 5093e3b..611651f 100644 --- a/docs/glosario.md +++ b/docs/glosario.md @@ -4,6 +4,9 @@ Key terms used across the FCEFyN testbed documentation. --- +**auto-merge** +: A GitHub Pull Request setting that merges the PR automatically as soon as all required reviews and status checks pass. The CI results pipeline relies on it so the bot PR opened by `collect-lime-results.yml` lands on `develop` without manual intervention. Requires the repo-level "Allow auto-merge" toggle to be on. + **batman-adv** : B.A.T.M.A.N. Advanced — a mesh routing protocol implemented as a Linux kernel module. Operates at Layer 2, handling frame forwarding between mesh nodes. Used by LibreMesh for L2 mesh connectivity. From 5cededaadc1078751ae13bbebacb39792d991bc2 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 17:59:45 -0300 Subject: [PATCH 13/24] docs(glossary): define node_openwrt_info and node_systemd_unit_state Both metrics are referenced by the new Grafana panels (Lab Overview firmware table, Orchestrator Host Lab Services section) and worth calling out so the dashboard descriptions are self-contained. --- docs/glosario.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/glosario.md b/docs/glosario.md index 611651f..0a3e663 100644 --- a/docs/glosario.md +++ b/docs/glosario.md @@ -37,6 +37,12 @@ Key terms used across the FCEFyN testbed documentation. **mac80211_hwsim** : A Linux kernel module that creates virtual IEEE 802.11 (WiFi) radios. Used in combination with vwifi to simulate WiFi connectivity between QEMU VMs without physical hardware. +**node_openwrt_info** +: A Prometheus metric exported by node_exporter (with custom textfile collector or scrape labels) on each DUT, carrying `firmware` and `target` as labels so the Lab Overview dashboard can show what's running on each device. + +**node_systemd_unit_state** +: A node_exporter metric describing whether a systemd unit is in a given state (`active`, `failed`, `inactive`, …). Used by the Orchestrator Host dashboard's Lab Services section to assert that `labgrid-exporter.service`, `pdudaemon.service`, and `ser2net.service` are running. + **openwrt-tests** : The test suite and pytest infrastructure (in `lime-packages`) that defines the test cases for both physical DUTs and virtual mesh nodes. From c9e8d9714f756568014852e383b58b22395c5a4f Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 17:59:59 -0300 Subject: [PATCH 14/24] docs(glossary): add PAT entry Lists the two PATs the pipeline actually uses (LIME_PACKAGES_TOKEN, BOT_PR_TOKEN) so a reader skimming the glossary can find what secret does what without digging into publishing.md. --- docs/glosario.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/glosario.md b/docs/glosario.md index 0a3e663..f16eb29 100644 --- a/docs/glosario.md +++ b/docs/glosario.md @@ -49,6 +49,9 @@ Key terms used across the FCEFyN testbed documentation. **pdudaemon** : A daemon that controls power to DUTs via relay boards or PDUs. Exposes an HTTP API. The lab uses an Arduino + SSR relay board controlled by pdudaemon. +**PAT (Personal Access Token)** +: A GitHub credential scoped per user. The CI results pipeline uses two fine-grained PATs as repository secrets in `fcefyn_testbed_utils`: `LIME_PACKAGES_TOKEN` (Actions: Read on `lime-packages`, for downloading artifacts) and `BOT_PR_TOKEN` (Contents + Pull Requests: Write on this repo, for opening the auto-merged bot PR). + **place** : A Labgrid concept representing one testable resource (a DUT with all its attached resources). A place has a name (e.g. `labgrid-fcefyn-belkin_rt3200_1`) and is registered with the coordinator. From a4313829c7cd7ba50902e04506f42799ecdb75c9 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 18:00:59 -0300 Subject: [PATCH 15/24] docs(test-flow): cross-link to CI dashboard publishing Mention that the same test-results-* artifacts surface on the CI dashboard via collect-lime-results.yml, so a reader debugging a failure knows where to find the post-hoc report view without digging through Actions artifacts. --- docs/diseno/lime-packages-test-flow.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/diseno/lime-packages-test-flow.md b/docs/diseno/lime-packages-test-flow.md index 0366afb..3e55121 100644 --- a/docs/diseno/lime-packages-test-flow.md +++ b/docs/diseno/lime-packages-test-flow.md @@ -287,6 +287,13 @@ not create noise in the issue tracker. 5. For QEMU jobs, the `qemu-*-logs` artifact contains the QEMU console plus pytest's `--lg-log`. +Once published, the same `report.xml` files are also visible from the +[CI Test Dashboard](../ci-results/index.md): `collect-lime-results.yml` +in `fcefyn_testbed_utils` pulls these artifacts every 6h and the +dashboard's "Report ↗" link points straight to the file on Pages. +See [Publishing results](../ci-results/publishing.md) for the +collection details. + --- ## 10. Runner prerequisites From 4eb8f3d0a6123c3b5335a70f635049ec76e20142 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 18:01:59 -0300 Subject: [PATCH 16/24] docs(test-run-flow): replace publish-results section with pull flow Section 7 used to describe a publish-results job inside lime-packages that pushed report.xml into this repo via TESTBED_UTILS_TOKEN. That job never landed upstream. The replacement describes the inverse flow that ships from fcefyn_testbed_utils today: collect-lime-results.yml fetches artifacts, opens an auto-merged PR with BOT_PR_TOKEN, and pages.yml deploys. Also updates the sequence diagram. --- docs/diseno/test-run-flow.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/diseno/test-run-flow.md b/docs/diseno/test-run-flow.md index e4afb6d..4b111b2 100644 --- a/docs/diseno/test-run-flow.md +++ b/docs/diseno/test-run-flow.md @@ -52,16 +52,24 @@ Each test function connects, runs commands, asserts results. pytest generates a JUnit XML report (`--junitxml report.xml`) and uploads it as a GitHub Actions artifact (`test-results-PLACE-RELEASE`). -### 7. publish-results job - -After all test jobs complete, `publish-results` runs: - -1. Checks out `fcefyn_testbed_utils` (via `TESTBED_UTILS_TOKEN`) -2. Downloads all `test-results-*` artifacts -3. Copies each `report.xml` to `docs/ci-results/results/{type}/{place}-{release}/` -4. Commits and pushes to `develop` +### 7. Results pulled into the dashboard + +There is no publish step inside `lime-packages`. Instead, the +`collect-lime-results.yml` workflow in `fcefyn_testbed_utils` runs every +6 h (or on `workflow_dispatch`) and: + +1. Lists recent runs of `build-firmware.yml` (and `tests.yml`) via the + GitHub Actions REST API, using `LIME_PACKAGES_TOKEN` +2. Downloads each `test-results-*` artifact and extracts `report.xml` +3. Maps the artifact name to the dashboard's expected path + (`docs/ci-results/results/{type}/{identifier}/report.xml`) +4. Opens an auto-merged bot PR onto `develop` with `BOT_PR_TOKEN` + (`peter-evans/create-pull-request` + `gh pr merge --auto --squash`) 5. `pages.yml` deploys the updated site to GitHub Pages +See [Publishing results](../ci-results/publishing.md) for the full +diagram and required secrets. + ### 8. Dashboard updates Next time someone opens the dashboard, the new `report.xml` is fetched from Pages and the card re-renders with per-test-case details. @@ -121,6 +129,6 @@ sequenceDiagram Runner->>DUT: run tests (SSH proxy) Runner->>GH: upload report.xml artifact Runner->>Coord: unlock place - GH->>GH: publish-results job + Note over GH: collect-lime-results.yml
(in fcefyn_testbed_utils, every 6h)
pulls artifact and opens bot PR GH->>GH: pages.yml deploy ``` From 2c265aff3ceaf56173d2607049509ba044e85635 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 18:02:16 -0300 Subject: [PATCH 17/24] docs(debugging-faq): point to collect-lime-results when card has no detail Replace the stale publish-results reference with the actual workflow that pulls report.xml into the dashboard and include the manual trigger instructions, so an operator hitting this case can act on it. --- docs/operar/debugging-faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operar/debugging-faq.md b/docs/operar/debugging-faq.md index 4996b2e..fdced48 100644 --- a/docs/operar/debugging-faq.md +++ b/docs/operar/debugging-faq.md @@ -129,7 +129,7 @@ The GitHub API rate limit (60 req/hour unauthenticated) may have been hit. Wait ### Card shows "Test details not yet published" -The `publish-results` job has not run yet, or `report.xml` failed to upload. Check the workflow run in GitHub Actions and verify the `test-results-*` artifact was created. +`collect-lime-results.yml` (every 6 h) hasn't pulled the `report.xml` for that device/release yet, or the CI artifact failed to upload. Verify the `test-results-*` artifact exists on the run in `lime-packages` Actions, then trigger the collect workflow manually from this repo: **Actions → Collect lime-packages test results → Run workflow**. ### Dashboard shows stale data after a new run From cd5909cf74f5a7f7cb56603a221f442b38e2aca6 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 21:26:57 -0300 Subject: [PATCH 18/24] feat(grafana): agregar panel de offset NTP en dashboard del orchestrator --- .../files/dashboards/orchestrator-node.json | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/ansible/roles/observability/files/dashboards/orchestrator-node.json b/ansible/roles/observability/files/dashboards/orchestrator-node.json index 4b960e0..8b5a20f 100644 --- a/ansible/roles/observability/files/dashboards/orchestrator-node.json +++ b/ansible/roles/observability/files/dashboards/orchestrator-node.json @@ -2542,6 +2542,44 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_network_up{job=\"orchestrator-host\",device=\"wg0\"}", "refId": "A" } ], "description": "Estado operativo de la interfaz wg0 (node_network_up)." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 114 }, + "id": 110, + "title": "NTP / time sync", + "type": "row", + "description": "Sincronía de tiempo del orchestrator: offset respecto al peer NTP." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "red", "value": 0.5 } + ]}, + "unit": "s", + "decimals": 4 + } + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 115 }, + "id": 111, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "NTP offset (s)", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_timex_offset_seconds{job=\"orchestrator-host\"}", "refId": "A" } + ], + "description": "Diferencia (en segundos) entre el reloj local y el peer NTP, según el kernel timex. Verde < 50 ms; rojo > 500 ms (revisar chrony/ntpd)." } ], "refresh": "30s", @@ -2563,5 +2601,5 @@ "timezone": "browser", "title": "FCEFyN Testbed - Orchestrator Host", "uid": "fcefyn-orchestrator-node", - "version": 4 + "version": 5 } From 86e860965001e770c0ef85cdee49d6d135179d15 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 21:27:41 -0300 Subject: [PATCH 19/24] feat(grafana): agregar panel de uso de conntrack al dashboard del orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panel stat con 100 * node_nf_conntrack_entries / node_nf_conntrack_entries_limit. Relevante en el orchestrator porque mantiene autossh + SSH-proxy por DUT más los flujos de Prometheus/Grafana --- .../files/dashboards/orchestrator-node.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ansible/roles/observability/files/dashboards/orchestrator-node.json b/ansible/roles/observability/files/dashboards/orchestrator-node.json index 8b5a20f..811f895 100644 --- a/ansible/roles/observability/files/dashboards/orchestrator-node.json +++ b/ansible/roles/observability/files/dashboards/orchestrator-node.json @@ -2580,6 +2580,37 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_timex_offset_seconds{job=\"orchestrator-host\"}", "refId": "A" } ], "description": "Diferencia (en segundos) entre el reloj local y el peer NTP, según el kernel timex. Verde < 50 ms; rojo > 500 ms (revisar chrony/ntpd)." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 80 } + ]}, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 115 }, + "id": 112, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Conntrack utilization", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * node_nf_conntrack_entries{job=\"orchestrator-host\"} / node_nf_conntrack_entries_limit{job=\"orchestrator-host\"}", "refId": "A" } + ], + "description": "Porcentaje de la tabla nf_conntrack en uso. Relevante para el orchestrator porque mantiene autossh + SSH-proxy por DUT más las conexiones de Prometheus/Grafana. Si llega a 80% empiezan a descartarse flujos nuevos." } ], "refresh": "30s", From 8442be570eca5f7d379c92d05050d4c388e2deb8 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 21:28:09 -0300 Subject: [PATCH 20/24] feat(grafana): agregar fila de load1 al dashboard Lab Overview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panel stat por DUT que lee node_load1, con thresholds en 1 y 2. Sirve para detectar DUTs trabajando bajo presión aunque el % CPU instantáneo se vea bajo, complementando el panel de CPU busy ya existente. --- .../files/dashboards/lab-overview.json | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/ansible/roles/observability/files/dashboards/lab-overview.json b/ansible/roles/observability/files/dashboards/lab-overview.json index b9fd3a5..5e61f51 100644 --- a/ansible/roles/observability/files/dashboards/lab-overview.json +++ b/ansible/roles/observability/files/dashboards/lab-overview.json @@ -259,6 +259,44 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_openwrt_info{dut=~\".+\"}", "format": "table", "instant": true, "refId": "A" } ], "description": "Tabla de firmware y target por DUT. Fuente: node_openwrt_info (label firmware y target del scrape config)." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "id": 60, + "title": "Load average (1m)", + "type": "row", + "description": "Carga del kernel a 1 minuto por DUT, comparable entre dispositivos." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 2 } + ]}, + "unit": "short", + "decimals": 2 + } + }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 46 }, + "id": 61, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Load average 1m per device", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_load1{dut=~\".+\"}", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "node_load1: tareas runnable + en uninterruptible sleep, promediadas en el último minuto. Útil para detectar DUTs trabajando bajo presión incluso si el % CPU instantáneo está bajo." } ], "refresh": "30s", @@ -271,5 +309,5 @@ "timezone": "browser", "title": "FCEFyN Testbed - Lab Overview", "uid": "fcefyn-lab-overview", - "version": 1 + "version": 2 } From 930e887bc1d4c700e0874fade89dd1ef6e59dd86 Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 21:41:17 -0300 Subject: [PATCH 21/24] feat(grafana): agregar fila de swap al dashboard Lab Overview Panel stat por DUT que muestra el % de swap en uso: 100 * (SwapTotal - SwapFree) / max(SwapTotal, 1). --- .../files/dashboards/lab-overview.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ansible/roles/observability/files/dashboards/lab-overview.json b/ansible/roles/observability/files/dashboards/lab-overview.json index 5e61f51..a1d85cb 100644 --- a/ansible/roles/observability/files/dashboards/lab-overview.json +++ b/ansible/roles/observability/files/dashboards/lab-overview.json @@ -297,6 +297,45 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_load1{dut=~\".+\"}", "legendFormat": "{{dut}}", "refId": "A" } ], "description": "node_load1: tareas runnable + en uninterruptible sleep, promediadas en el último minuto. Útil para detectar DUTs trabajando bajo presión incluso si el % CPU instantáneo está bajo." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "id": 70, + "title": "Swap", + "type": "row", + "description": "Uso de swap por DUT. En OpenWrt habitualmente debería estar en 0%; cualquier swap suele indicar memoria comprometida." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 25 } + ]}, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 52 }, + "id": 71, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Swap used per device", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (node_memory_SwapTotal_bytes{dut=~\".+\"} - node_memory_SwapFree_bytes{dut=~\".+\"}) / clamp_min(node_memory_SwapTotal_bytes{dut=~\".+\"}, 1)", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "Porcentaje de swap utilizado por cada DUT. En OpenWrt normalmente no hay swap configurado: este panel mostraría 0% (o NaN si no hay zswap/zram), y cualquier valor distinto suele ser síntoma de presión de memoria." } ], "refresh": "30s", From a13034cb92b4be9f6b73c2b3571b6bcfdb1db1ce Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 21:43:25 -0300 Subject: [PATCH 22/24] =?UTF-8?q?feat(grafana):=20agregar=20fila=20de=20tr?= =?UTF-8?q?=C3=A1fico=20de=20red=20al=20dashboard=20Lab=20Overview?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sirve para ver de un vistazo si algún DUT está viendo tráfico inesperado durante tests o si quedó algo silencioso. --- .../files/dashboards/lab-overview.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ansible/roles/observability/files/dashboards/lab-overview.json b/ansible/roles/observability/files/dashboards/lab-overview.json index a1d85cb..9f8dd5e 100644 --- a/ansible/roles/observability/files/dashboards/lab-overview.json +++ b/ansible/roles/observability/files/dashboards/lab-overview.json @@ -336,6 +336,36 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (node_memory_SwapTotal_bytes{dut=~\".+\"} - node_memory_SwapFree_bytes{dut=~\".+\"}) / clamp_min(node_memory_SwapTotal_bytes{dut=~\".+\"}, 1)", "legendFormat": "{{dut}}", "refId": "A" } ], "description": "Porcentaje de swap utilizado por cada DUT. En OpenWrt normalmente no hay swap configurado: este panel mostraría 0% (o NaN si no hay zswap/zram), y cualquier valor distinto suele ser síntoma de presión de memoria." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, + "id": 80, + "title": "Network bandwidth", + "type": "row", + "description": "Tráfico agregado por DUT, suma de todas las interfaces excepto loopback." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "bps", + "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10 } + } + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 58 }, + "id": 81, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "title": "Network throughput per device (rx)", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (dut) (rate(node_network_receive_bytes_total{dut=~\".+\",device!=\"lo\"}[2m])) * 8", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "Bits por segundo recibidos por cada DUT, sumando todas las interfaces salvo loopback. Calcula rate(node_network_receive_bytes_total[2m]) * 8." } ], "refresh": "30s", From c4e384ac409d7278e5f1e7585396f24ccf8b373d Mon Sep 17 00:00:00 2001 From: Constanza Date: Sat, 23 May 2026 22:24:02 -0300 Subject: [PATCH 23/24] feat(observability): agregar reglas de alerting de Grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8 reglas para provisionar en Grafana (formato Grafana managed alerting v1): - fcefyn-target-down: target Prometheus inalcanzable > 2 min - fcefyn-high-cpu: CPU del orchestrator > 85% por 5 min - fcefyn-high-memory: RAM > 90% por 5 min - fcefyn-low-disk: filesystem raíz < 10% libre - fcefyn-tunnel-down: autossh DUT tunnel caído > 3 min - fcefyn-systemd-failed: alguna systemd unit en estado failed - fcefyn-network-errors: errores rx/tx > 0 sostenidos por 5 min se traen al flujo principal para que el Ansible role los provisione junto con los dashboards. --- .../files/alerting/fcefyn-alerts.yaml | 318 ++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 ansible/roles/observability/files/alerting/fcefyn-alerts.yaml diff --git a/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml new file mode 100644 index 0000000..ab8b575 --- /dev/null +++ b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml @@ -0,0 +1,318 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: fcefyn-testbed + folder: FCEFyN Alerts + interval: 1m + rules: + + # ----------------------------------------------------------------------- + # Target down + # Fires when any Prometheus target (DUT or orchestrator) stops responding. + # ----------------------------------------------------------------------- + - uid: fcefyn-target-down + title: Target down + condition: C + for: 2m + annotations: + summary: "{{ $labels.job }} / {{ $labels.instance }} is down" + description: "Prometheus target has been unreachable for more than 2 minutes." + labels: + severity: critical + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "up == 0" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # High CPU (orchestrator host) + # ----------------------------------------------------------------------- + - uid: fcefyn-high-cpu + title: High CPU usage + condition: C + for: 5m + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage has been above 90% for more than 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + expr: "100 - (avg by (instance, dut) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [90] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # High memory + # ----------------------------------------------------------------------- + - uid: fcefyn-high-memory + title: High memory usage + condition: C + for: 5m + annotations: + summary: "High memory on {{ $labels.instance }}" + description: "Memory usage has been above 90% for more than 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [90] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Low disk space + # ----------------------------------------------------------------------- + - uid: fcefyn-low-disk + title: Low disk space + condition: C + for: 0m + annotations: + summary: "Low disk on {{ $labels.instance }} ({{ $labels.mountpoint }})" + description: "Disk usage is above 85% on {{ $labels.mountpoint }}." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "(1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\"}) * 100" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [85] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # High temperature + # Only fires on devices that actually expose hwmon sensors. + # ----------------------------------------------------------------------- + - uid: fcefyn-high-temp + title: High temperature + condition: C + for: 2m + annotations: + summary: "High temperature on {{ $labels.instance }} ({{ $labels.chip }})" + description: "Temperature sensor above 80°C for more than 2 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "node_hwmon_temp_celsius > 80" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # DUT metrics tunnel down + # Checks that autossh tunnel units are not in failed state. + # Metric comes from the orchestrator's node_exporter. + # ----------------------------------------------------------------------- + - uid: fcefyn-tunnel-down + title: DUT metrics tunnel down + condition: C + for: 2m + annotations: + summary: "Metrics tunnel failed: {{ $labels.name }}" + description: "autossh unit {{ $labels.name }} is in failed state — DUT metrics may be stale." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "node_systemd_unit_state{name=~\"dut-metrics-tunnel-.+\\.service\", state=\"failed\"} == 1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Systemd units in failed state (orchestrator host) + # Catches prometheus, grafana, labgrid-coordinator, etc. + # ----------------------------------------------------------------------- + - uid: fcefyn-systemd-failed + title: Systemd unit failed + condition: C + for: 1m + annotations: + summary: "Systemd unit failed on {{ $labels.instance }}: {{ $labels.name }}" + description: "A systemd unit is in failed state on the orchestration host." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "node_systemd_unit_state{state=\"failed\", name!~\"dut-metrics-tunnel-.+\\.service\"} == 1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Network errors (orchestrator host) + # Sustained error rate on physical interfaces. + # ----------------------------------------------------------------------- + - uid: fcefyn-network-errors + title: Network interface errors + condition: C + for: 5m + annotations: + summary: "Network errors on {{ $labels.instance }} / {{ $labels.device }}" + description: "Interface {{ $labels.device }} has a sustained error/drop rate over 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + expr: "rate(node_network_receive_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_transmit_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_receive_drop_total{device!~\"lo|veth.+|docker.+\"}[5m]) > 0.1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last From db3fe72b5821c68f875f11bd514f34a2e19688ff Mon Sep 17 00:00:00 2001 From: Constanza Date: Sun, 24 May 2026 13:16:12 -0300 Subject: [PATCH 24/24] fix --- .../files/alerting/fcefyn-alerts.yaml | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml index ab8b575..1433a7d 100644 --- a/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml +++ b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml @@ -160,45 +160,6 @@ groups: reducer: type: last - # ----------------------------------------------------------------------- - # High temperature - # Only fires on devices that actually expose hwmon sensors. - # ----------------------------------------------------------------------- - - uid: fcefyn-high-temp - title: High temperature - condition: C - for: 2m - annotations: - summary: "High temperature on {{ $labels.instance }} ({{ $labels.chip }})" - description: "Temperature sensor above 80°C for more than 2 minutes." - labels: - severity: warning - data: - - refId: A - relativeTimeRange: - from: 300 - to: 0 - datasourceUid: prometheus - model: - expr: "node_hwmon_temp_celsius > 80" - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - type: classic_conditions - refId: C - conditions: - - evaluator: - type: gt - params: [0] - operator: - type: and - query: - params: [A] - reducer: - type: last - # ----------------------------------------------------------------------- # DUT metrics tunnel down # Checks that autossh tunnel units are not in failed state.