diff --git a/.github/workflows/collect-lime-results.yml b/.github/workflows/collect-lime-results.yml index 92b7ed7..d80645e 100644 --- a/.github/workflows/collect-lime-results.yml +++ b/.github/workflows/collect-lime-results.yml @@ -22,6 +22,7 @@ on: permissions: contents: write + pull-requests: write concurrency: group: collect-lime-results @@ -124,16 +125,38 @@ jobs: echo "Collected $new_files new report.xml file(s)." echo "NEW_FILES=$new_files" >> "$GITHUB_ENV" - - name: Commit and push results + # `develop` is protected (signed commits required, PRs only), so we + # cannot push directly from the bot. Open a PR with API-signed + # commits via peter-evans/create-pull-request and auto-merge it. + - name: Create PR with collected results if: env.NEW_FILES != '0' + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.BOT_PR_TOKEN }} + base: develop + branch: bot/lime-results + delete-branch: true + sign-commits: true + commit-message: | + ci: collect lime-packages test results (${{ env.NEW_FILES }} new) + title: 'ci: collect lime-packages test results' + body: | + Automated PR with ${{ env.NEW_FILES }} freshly downloaded `report.xml` + file(s) from `fcefyn-testbed/lime-packages` CI artifacts. + + Triggered by the `Collect lime-packages test results` workflow. + labels: | + ci + automated + add-paths: | + docs/ci-results/results/ + + - name: Enable auto-merge + if: steps.cpr.outputs.pull-request-number + env: + GH_TOKEN: ${{ secrets.BOT_PR_TOKEN }} run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/ci-results/results/ - if git diff --cached --quiet; then - echo "Nothing to commit." - exit 0 - fi - git commit -m "ci: collect lime-packages test results ($NEW_FILES new)" - git push origin develop + gh pr merge --auto --squash \ + --repo "${{ github.repository }}" \ + "${{ steps.cpr.outputs.pull-request-number }}" diff --git a/ansible/roles/observability/defaults/main.yml b/ansible/roles/observability/defaults/main.yml index 91223df..41c5f93 100644 --- a/ansible/roles/observability/defaults/main.yml +++ b/ansible/roles/observability/defaults/main.yml @@ -106,3 +106,12 @@ grafana_config: cookie_secure: "true" cookie_samesite: "lax" allow_sign_up: "false" + +# Anonymous access: let anyone browse dashboards as read-only Viewer +# without logging in. No VPS changes needed; Grafana handles auth. +# Set enabled: false to revert to login-only access. +grafana_anonymous: + enabled: "true" + org_name: "Main Org." + org_role: "Viewer" + hide_version: "true" diff --git a/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml new file mode 100644 index 0000000..1433a7d --- /dev/null +++ b/ansible/roles/observability/files/alerting/fcefyn-alerts.yaml @@ -0,0 +1,279 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: fcefyn-testbed + folder: FCEFyN Alerts + interval: 1m + rules: + + # ----------------------------------------------------------------------- + # Target down + # Fires when any Prometheus target (DUT or orchestrator) stops responding. + # ----------------------------------------------------------------------- + - uid: fcefyn-target-down + title: Target down + condition: C + for: 2m + annotations: + summary: "{{ $labels.job }} / {{ $labels.instance }} is down" + description: "Prometheus target has been unreachable for more than 2 minutes." + labels: + severity: critical + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "up == 0" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # High CPU (orchestrator host) + # ----------------------------------------------------------------------- + - uid: fcefyn-high-cpu + title: High CPU usage + condition: C + for: 5m + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage has been above 90% for more than 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + expr: "100 - (avg by (instance, dut) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [90] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # High memory + # ----------------------------------------------------------------------- + - uid: fcefyn-high-memory + title: High memory usage + condition: C + for: 5m + annotations: + summary: "High memory on {{ $labels.instance }}" + description: "Memory usage has been above 90% for more than 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [90] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Low disk space + # ----------------------------------------------------------------------- + - uid: fcefyn-low-disk + title: Low disk space + condition: C + for: 0m + annotations: + summary: "Low disk on {{ $labels.instance }} ({{ $labels.mountpoint }})" + description: "Disk usage is above 85% on {{ $labels.mountpoint }}." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "(1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\"}) * 100" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [85] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # DUT metrics tunnel down + # Checks that autossh tunnel units are not in failed state. + # Metric comes from the orchestrator's node_exporter. + # ----------------------------------------------------------------------- + - uid: fcefyn-tunnel-down + title: DUT metrics tunnel down + condition: C + for: 2m + annotations: + summary: "Metrics tunnel failed: {{ $labels.name }}" + description: "autossh unit {{ $labels.name }} is in failed state — DUT metrics may be stale." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "node_systemd_unit_state{name=~\"dut-metrics-tunnel-.+\\.service\", state=\"failed\"} == 1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Systemd units in failed state (orchestrator host) + # Catches prometheus, grafana, labgrid-coordinator, etc. + # ----------------------------------------------------------------------- + - uid: fcefyn-systemd-failed + title: Systemd unit failed + condition: C + for: 1m + annotations: + summary: "Systemd unit failed on {{ $labels.instance }}: {{ $labels.name }}" + description: "A systemd unit is in failed state on the orchestration host." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: "node_systemd_unit_state{state=\"failed\", name!~\"dut-metrics-tunnel-.+\\.service\"} == 1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last + + # ----------------------------------------------------------------------- + # Network errors (orchestrator host) + # Sustained error rate on physical interfaces. + # ----------------------------------------------------------------------- + - uid: fcefyn-network-errors + title: Network interface errors + condition: C + for: 5m + annotations: + summary: "Network errors on {{ $labels.instance }} / {{ $labels.device }}" + description: "Interface {{ $labels.device }} has a sustained error/drop rate over 5 minutes." + labels: + severity: warning + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + expr: "rate(node_network_receive_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_transmit_errs_total{device!~\"lo|veth.+|docker.+\"}[5m]) + rate(node_network_receive_drop_total{device!~\"lo|veth.+|docker.+\"}[5m]) > 0.1" + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + type: classic_conditions + refId: C + conditions: + - evaluator: + type: gt + params: [0] + operator: + type: and + query: + params: [A] + reducer: + type: last diff --git a/ansible/roles/observability/files/dashboards/lab-overview.json b/ansible/roles/observability/files/dashboards/lab-overview.json index b9fd3a5..9f8dd5e 100644 --- a/ansible/roles/observability/files/dashboards/lab-overview.json +++ b/ansible/roles/observability/files/dashboards/lab-overview.json @@ -259,6 +259,113 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_openwrt_info{dut=~\".+\"}", "format": "table", "instant": true, "refId": "A" } ], "description": "Tabla de firmware y target por DUT. Fuente: node_openwrt_info (label firmware y target del scrape config)." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "id": 60, + "title": "Load average (1m)", + "type": "row", + "description": "Carga del kernel a 1 minuto por DUT, comparable entre dispositivos." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 2 } + ]}, + "unit": "short", + "decimals": 2 + } + }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 46 }, + "id": 61, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Load average 1m per device", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_load1{dut=~\".+\"}", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "node_load1: tareas runnable + en uninterruptible sleep, promediadas en el último minuto. Útil para detectar DUTs trabajando bajo presión incluso si el % CPU instantáneo está bajo." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "id": 70, + "title": "Swap", + "type": "row", + "description": "Uso de swap por DUT. En OpenWrt habitualmente debería estar en 0%; cualquier swap suele indicar memoria comprometida." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 25 } + ]}, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 52 }, + "id": 71, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Swap used per device", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (node_memory_SwapTotal_bytes{dut=~\".+\"} - node_memory_SwapFree_bytes{dut=~\".+\"}) / clamp_min(node_memory_SwapTotal_bytes{dut=~\".+\"}, 1)", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "Porcentaje de swap utilizado por cada DUT. En OpenWrt normalmente no hay swap configurado: este panel mostraría 0% (o NaN si no hay zswap/zram), y cualquier valor distinto suele ser síntoma de presión de memoria." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, + "id": 80, + "title": "Network bandwidth", + "type": "row", + "description": "Tráfico agregado por DUT, suma de todas las interfaces excepto loopback." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "bps", + "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10 } + } + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 58 }, + "id": 81, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "title": "Network throughput per device (rx)", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (dut) (rate(node_network_receive_bytes_total{dut=~\".+\",device!=\"lo\"}[2m])) * 8", "legendFormat": "{{dut}}", "refId": "A" } + ], + "description": "Bits por segundo recibidos por cada DUT, sumando todas las interfaces salvo loopback. Calcula rate(node_network_receive_bytes_total[2m]) * 8." } ], "refresh": "30s", @@ -271,5 +378,5 @@ "timezone": "browser", "title": "FCEFyN Testbed - Lab Overview", "uid": "fcefyn-lab-overview", - "version": 1 + "version": 2 } diff --git a/ansible/roles/observability/files/dashboards/orchestrator-node.json b/ansible/roles/observability/files/dashboards/orchestrator-node.json index 4b960e0..811f895 100644 --- a/ansible/roles/observability/files/dashboards/orchestrator-node.json +++ b/ansible/roles/observability/files/dashboards/orchestrator-node.json @@ -2542,6 +2542,75 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_network_up{job=\"orchestrator-host\",device=\"wg0\"}", "refId": "A" } ], "description": "Estado operativo de la interfaz wg0 (node_network_up)." + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 114 }, + "id": 110, + "title": "NTP / time sync", + "type": "row", + "description": "Sincronía de tiempo del orchestrator: offset respecto al peer NTP." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "red", "value": 0.5 } + ]}, + "unit": "s", + "decimals": 4 + } + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 115 }, + "id": 111, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "NTP offset (s)", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_timex_offset_seconds{job=\"orchestrator-host\"}", "refId": "A" } + ], + "description": "Diferencia (en segundos) entre el reloj local y el peer NTP, según el kernel timex. Verde < 50 ms; rojo > 500 ms (revisar chrony/ntpd)." + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 80 } + ]}, + "unit": "percent", + "min": 0, + "max": 100 + } + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 115 }, + "id": 112, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "textMode": "auto" + }, + "title": "Conntrack utilization", + "type": "stat", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * node_nf_conntrack_entries{job=\"orchestrator-host\"} / node_nf_conntrack_entries_limit{job=\"orchestrator-host\"}", "refId": "A" } + ], + "description": "Porcentaje de la tabla nf_conntrack en uso. Relevante para el orchestrator porque mantiene autossh + SSH-proxy por DUT más las conexiones de Prometheus/Grafana. Si llega a 80% empiezan a descartarse flujos nuevos." } ], "refresh": "30s", @@ -2563,5 +2632,5 @@ "timezone": "browser", "title": "FCEFyN Testbed - Orchestrator Host", "uid": "fcefyn-orchestrator-node", - "version": 4 + "version": 5 } diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml index e0207ec..01fd0c8 100644 --- a/ansible/roles/observability/tasks/main.yml +++ b/ansible/roles/observability/tasks/main.yml @@ -240,6 +240,46 @@ mode: "0640" notify: Restart grafana +# ------------------------------------------------------------------------- +# Grafana anonymous access — public read-only viewer (no login required) +# ------------------------------------------------------------------------- + +- name: Configure grafana.ini - anonymous auth enabled + community.general.ini_file: + path: /etc/grafana/grafana.ini + section: auth.anonymous + option: enabled + value: "{{ grafana_anonymous.enabled }}" + mode: "0640" + notify: Restart grafana + +- name: Configure grafana.ini - anonymous org_name + community.general.ini_file: + path: /etc/grafana/grafana.ini + section: auth.anonymous + option: org_name + value: "{{ grafana_anonymous.org_name }}" + mode: "0640" + notify: Restart grafana + +- name: Configure grafana.ini - anonymous org_role + community.general.ini_file: + path: /etc/grafana/grafana.ini + section: auth.anonymous + option: org_role + value: "{{ grafana_anonymous.org_role }}" + mode: "0640" + notify: Restart grafana + +- name: Configure grafana.ini - anonymous hide_version + community.general.ini_file: + path: /etc/grafana/grafana.ini + section: auth.anonymous + option: hide_version + value: "{{ grafana_anonymous.hide_version }}" + mode: "0640" + notify: Restart grafana + # ------------------------------------------------------------------------- # Orchestrator node exporter — metrics from the orchestration host itself # ------------------------------------------------------------------------- diff --git a/docs/ci-results/architecture.md b/docs/ci-results/architecture.md index 5b8ca07..5d5937e 100644 --- a/docs/ci-results/architecture.md +++ b/docs/ci-results/architecture.md @@ -41,13 +41,13 @@ Used for: **job status, run history, duration, links**. Used for: **per-test-case breakdown** (pass / fail / skip counts and individual test names). -- Served from GitHub Pages alongside `dashboard.html` +- Served from this repo's GitHub Pages alongside `dashboard.html` - JUnit XML format, generated by pytest (`--junitxml`) -- Published automatically after each scheduled CI run by the `publish-results` job in `build-firmware.yml` +- Populated by the `collect-lime-results.yml` workflow, which pulls artifacts from `fcefyn-testbed/lime-packages` CI every 6h — see [Publishing results](publishing.md) - Path convention: `results/{type}/{identifier}/report.xml` - Direct links to reports use `PAGES_BASE` (configured at the top of `dashboard.html`): ```js - const PAGES_BASE = 'https://fcefyn-testbed.github.io/lime-packages/ci-results/results/'; + const PAGES_BASE = 'https://fcefyn-testbed.github.io/fcefyn_testbed_utils/ci-results/results/'; ``` ### Source 3 — `devices.json` diff --git a/docs/ci-results/dashboard.html b/docs/ci-results/dashboard.html index 823450d..db2a151 100644 --- a/docs/ci-results/dashboard.html +++ b/docs/ci-results/dashboard.html @@ -372,7 +372,7 @@ const RESULTS_BASE = 'results/'; // served alongside this page // Base URL of the GitHub Pages site where test reports are published. // Set to empty string to disable direct report links. - const PAGES_BASE = 'https://fcefyn-testbed.github.io/lime-packages/ci-results/results/'; + const PAGES_BASE = 'https://fcefyn-testbed.github.io/fcefyn_testbed_utils/ci-results/results/'; // ── GitHub API (no auth, public repo) ───────────────────────────────────── async function ghFetch(path) { diff --git a/docs/ci-results/devices.md b/docs/ci-results/devices.md index 3c64c64..e912d93 100644 --- a/docs/ci-results/devices.md +++ b/docs/ci-results/devices.md @@ -69,13 +69,17 @@ The dashboard recognises the following CI job name patterns and maps them to typ | `physical-belkin_rt3200_1-24.10.6` | `linksys_e8450` | `belkin_rt3200_1` | 24.10.6 | | `physical-belkin_rt3200_2-24.10.6` | `linksys_e8450` | `belkin_rt3200_2` | 24.10.6 | | `physical-belkin_rt3200_3-24.10.6` | `linksys_e8450` | `belkin_rt3200_3` | 24.10.6 | -| `physical-bpi_r4_1-24.10.6` | `bananapi_bpi-r4` | `bpi_r4_1` | 24.10.6 | -| `physical-openwrt_one_1-24.10.6` | `openwrt_one` | `openwrt_one_1` | 24.10.6 | -| `physical-belkin_rt3200_1-25.12.2` | `linksys_e8450` | `belkin_rt3200_1` | 25.12.2 | -| `physical-belkin_rt3200_2-25.12.2` | `linksys_e8450` | `belkin_rt3200_2` | 25.12.2 | -| `physical-belkin_rt3200_3-25.12.2` | `linksys_e8450` | `belkin_rt3200_3` | 25.12.2 | -| `physical-bpi_r4_1-25.12.2` | `bananapi_bpi-r4` | `bpi_r4_1` | 25.12.2 | -| `physical-openwrt_one_1-25.12.2` | `openwrt_one` | `openwrt_one_1` | 25.12.2 | +| `physical-bananapi_bpi-r4-24.10.6` | `bananapi_bpi-r4` | `bananapi_bpi-r4` | 24.10.6 | +| `physical-openwrt_one-24.10.6` | `openwrt_one` | `openwrt_one` | 24.10.6 | + +### Physical mesh + +| ID | Type | Release | +|----|------|---------| +| `physical-mesh-24.10.6` | Multi-node mesh (N=2 or N=3 from `prepare_matrix.sh`) | 24.10.6 | +| `mesh-pair-1-24.10.6` | Pair #1 — `openwrt_one + bananapi_bpi-r4` | 24.10.6 | +| `mesh-pair-2-24.10.6` | Pair #2 — `openwrt_one + belkin_rt3200_2` | 24.10.6 | +| `mesh-pair-3-24.10.6` | Pair #3 — `bananapi_bpi-r4 + belkin_rt3200_2` | 24.10.6 | ### QEMU @@ -88,6 +92,6 @@ The dashboard recognises the following CI job name patterns and maps them to typ ## Adding a new device -1. Add the device to `targets.yml` in `lime-packages` with `test_firmware: true` and the corresponding `test_places` entry +1. Add the device to `targets.yml` in `lime-packages` with `test_firmware: true` (and `test_places` if the labgrid place name differs from the device name) 2. Add an entry to `devices.json` with the correct `id`, `place`, `release`, and `results_path` -3. The CI publish pipeline will start populating `report.xml` after the next scheduled run +3. The next scheduled run of `collect-lime-results.yml` (every 6h) will start pulling the new `report.xml`; trigger it manually if you don't want to wait diff --git a/docs/ci-results/index.md b/docs/ci-results/index.md index f463b25..30282f5 100644 --- a/docs/ci-results/index.md +++ b/docs/ci-results/index.md @@ -40,10 +40,11 @@ The summary bar at the top shows aggregate counts across all tracked jobs: | Device | Place | Type | Releases | |--------|-------|------|---------| -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_1` | Physical | 24.10.6, 25.12.2 | -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_2` | Physical | 24.10.6, 25.12.2 | -| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_3` | Physical | 24.10.6, 25.12.2 | -| Banana Pi BPi-R4 | `bpi_r4_1` | Physical | 24.10.6, 25.12.2 | -| OpenWrt One | `openwrt_one_1` | Physical | 24.10.6, 25.12.2 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_1` | Physical | 24.10.6 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_2` | Physical | 24.10.6 | +| Linksys E8450 (Belkin RT3200) | `belkin_rt3200_3` | Physical | 24.10.6 | +| Banana Pi BPi-R4 | `bananapi_bpi-r4` | Physical | 24.10.6 | +| OpenWrt One | `openwrt_one` | Physical | 24.10.6 | +| 2-node and 3-node physical mesh | — | Mesh / Mesh pairs | 24.10.6 | | QEMU x86-64 | — | QEMU single | 24.10.6, 25.12.2 | | QEMU x86-64 | — | QEMU mesh | 24.10.6, 25.12.2 | diff --git a/docs/ci-results/maintenance.md b/docs/ci-results/maintenance.md index 51ef15e..38dd1d6 100644 --- a/docs/ci-results/maintenance.md +++ b/docs/ci-results/maintenance.md @@ -6,29 +6,50 @@ Edit `docs/ci-results/results/devices.json` and open a PR to `develop`. Changes See [Device Registry](devices.md) for the full field reference. -## Rotating the TESTBED_UTILS_TOKEN +## Rotating the tokens -The publish pipeline uses a fine-grained PAT stored in `lime-packages` repository secrets. PATs expire — when that happens the `publish-results` job will fail with a 401 error. +The pull pipeline uses two fine-grained PATs stored as repository secrets in `fcefyn_testbed_utils`. PATs expire — when they do, the `collect-lime-results.yml` workflow fails on the step that uses the expired token. -To rotate: +### `LIME_PACKAGES_TOKEN` -1. Go to `github.com/settings/tokens` → generate a new fine-grained token: +Used to list runs and download artifacts from `lime-packages`. A 401/403 on the "Download lime-packages report.xml artifacts" step means this token expired. + +1. Generate a new fine-grained PAT at `github.com/settings/personal-access-tokens/new`: - Resource owner: `fcefyn-testbed` - - Repository: `fcefyn_testbed_utils` - - Permission: `Contents: Read and write` -2. Go to `github.com/fcefyn-testbed/lime-packages/settings/secrets/actions` -3. Update `TESTBED_UTILS_TOKEN` with the new value + - Repository access: only `lime-packages` + - Permissions: **Actions: Read-only** (Metadata: Read is added automatically) +2. `Settings → Secrets and variables → Actions` on `fcefyn_testbed_utils` +3. Update `LIME_PACKAGES_TOKEN` with the new value + +### `BOT_PR_TOKEN` + +Used to open the auto-merged PR with the collected results. A 401/403 on the "Create PR with collected results" or "Enable auto-merge" step means this one expired. + +1. Generate a new fine-grained PAT at `github.com/settings/personal-access-tokens/new`: + - Resource owner: `fcefyn-testbed` + - Repository access: only `fcefyn_testbed_utils` + - Permissions: **Contents: Read and write** + **Pull requests: Read and write** +2. Update `BOT_PR_TOKEN` in the same secrets page + +## Re-collecting results manually + +To force an immediate pull (e.g. after a fresh CI run in `lime-packages`): -## Re-publishing results manually +1. **Actions → Collect lime-packages test results** +2. **Run workflow** → branch `develop` → optionally set `runs` (default 10 — how many recent CI runs to scan) -If a scheduled run published incomplete results (e.g. some jobs were skipped), trigger `build-firmware.yml` via `workflow_dispatch` to republish. +The workflow is idempotent: it skips reports already present, so running it repeatedly is safe. ## Troubleshooting | Symptom | Likely cause | Fix | -|---------|-------------|-----| -| Cards show "Test details not yet published" | `publish-results` job hasn't run yet | Wait for next schedule or trigger manually | -| `publish-results` fails with 401 | `TESTBED_UTILS_TOKEN` expired | Rotate the token (see above) | -| `publish-results` fails with 403 | Token lacks `Contents: write` on `fcefyn_testbed_utils` | Regenerate with correct permissions | -| Dashboard shows no cards | GitHub API rate limit hit (60 req/h) | Wait ~1 hour and reload | -| Old results not updating | CI skipped `publish-results` (non-schedule event) | Trigger `workflow_dispatch` manually | +|---|---|---| +| Cards show "Test details not yet published" | `report.xml` for that device/release hasn't been collected yet | Trigger `collect-lime-results.yml` manually, or wait for the cron | +| "Report ↗" link returns 404 | Either the report hasn't been collected yet, or `PAGES_BASE` in `dashboard.html` is pointing at the wrong host | Open the link and check the URL host — it must be `fcefyn-testbed.github.io/fcefyn_testbed_utils/` | +| Collect workflow fails with 401/403 on Download step | `LIME_PACKAGES_TOKEN` expired | Rotate (see above) | +| Collect workflow fails with 401/403 on Create PR step | `BOT_PR_TOKEN` expired | Rotate (see above) | +| Collect workflow fails with "GitHub Actions is not permitted to create or approve pull requests" | Org-level Actions setting disabled | Use a PAT (`BOT_PR_TOKEN`) — already the default; if blocked, check **Settings → Actions → General → Workflow permissions** at the org level | +| Collect workflow fails with "Auto merge is not allowed for this repository" | Repo setting disabled | **Settings → General → Pull Requests → Allow auto-merge** | +| Bot PR fails to merge with "Commits must have verified signatures" | `develop` branch protection has signed-commits required | Disable it: the PR is opened with the user PAT and not signed by GitHub — `gh api -X DELETE repos/fcefyn-testbed/fcefyn_testbed_utils/branches/develop/protection/required_signatures` | +| Dashboard shows no cards at all | GitHub API rate limit hit (60 req/h, unauthenticated) | Wait ~1 hour and reload | +| Old results not refreshing | Cron runs every 6h; no new CI runs in `lime-packages` since last pull | Trigger manually if needed | diff --git a/docs/ci-results/publishing.md b/docs/ci-results/publishing.md index 9da9efa..f637ac4 100644 --- a/docs/ci-results/publishing.md +++ b/docs/ci-results/publishing.md @@ -1,68 +1,94 @@ # Publishing Test Results -Test results are published automatically to this repository after each scheduled CI run in `lime-packages`. Once published, the dashboard shows individual test case details instead of "Test details not yet published". +Test reports are pulled into this repository from `fcefyn-testbed/lime-packages` CI runs on a schedule, then served alongside the dashboard. Once published, the dashboard renders individual test-case details instead of "Test details not yet published". ## How it works -The `publish-results` job in `lime-packages/.github/workflows/build-firmware.yml` runs after all test jobs complete on the daily schedule (`0 6 * * *`): +The pipeline runs entirely from this repository — `lime-packages` does not need any changes. The `collect-lime-results.yml` workflow uses the GitHub Actions REST API to download `report.xml` artifacts from recent CI runs and commits them via an auto-merged PR. ```mermaid sequenceDiagram - participant CI as CI (lime-packages) - participant GHA as GitHub Actions Artifacts - participant Utils as fcefyn_testbed_utils - - CI->>GHA: Upload test-results-* artifacts - CI->>CI: publish-results job starts - CI->>Utils: git checkout (develop, TESTBED_UTILS_TOKEN) - CI->>GHA: Download all test-results-* artifacts - CI->>Utils: Copy report.xml files to docs/ci-results/results/ - CI->>Utils: git commit + push - Utils->>Utils: pages.yml triggers GitHub Pages deploy + participant Cron as Cron (every 6h) + participant Workflow as collect-lime-results.yml + participant LimeAPI as lime-packages Actions API + participant Bot as bot/lime-results branch + participant Develop as develop + participant Pages as GitHub Pages + + Cron->>Workflow: Trigger (schedule or workflow_dispatch) + Workflow->>LimeAPI: List recent build-firmware.yml + tests.yml runs
(LIME_PACKAGES_TOKEN) + LimeAPI-->>Workflow: Run IDs and artifact metadata + Workflow->>LimeAPI: Download test-results-* artifact zips + Workflow->>Workflow: Extract report.xml and
map artifact name → results path + Workflow->>Bot: peter-evans/create-pull-request
(commits via API, BOT_PR_TOKEN) + Bot->>Develop: gh pr merge --auto --squash + Develop->>Pages: pages.yml builds + deploys ``` +### Why this direction + +The original plan was for `lime-packages` to push its results into this repo after every CI run, but that requires a workflow change to land in `lime-packages` master — which the upstream LibreMesh project has not accepted. Inverting the direction (this repo pulls from there) means the integration ships on our side without any upstream coupling. + +## Artifact → results path mapping + +The collect workflow recognises these artifact name patterns produced by `lime-packages` CI and routes them to the layout the dashboard expects: + +| Artifact name | Destination | +|--|--| +| `test-results-unit` | `docs/ci-results/results/unit/report.xml` | +| `test-results-qemu-single-{device}-{release}` | `docs/ci-results/results/qemu-single/{device}-{release}/report.xml` | +| `test-results-qemu-mesh-{device}-{release}` | `docs/ci-results/results/qemu-mesh/{device}-{release}/report.xml` | +| `test-results-mesh-pairs-{pair}-{release}` | `docs/ci-results/results/mesh-pairs/{pair}-{release}/report.xml` | +| `test-results-mesh-{release}` | `docs/ci-results/results/mesh/{release}/report.xml` | +| `test-results-{place}-{release}` | `docs/ci-results/results/physical/{place}-{release}/report.xml` | + +Already-collected reports are skipped, so re-running the workflow is idempotent and only picks up new runs. + ## Results directory structure ``` docs/ci-results/results/ -├── devices.json # device registry +├── devices.json ├── physical/ -│ ├── belkin_rt3200_1-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_2-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_3-24.10.6/ -│ │ └── report.xml -│ ├── bpi_r4_1-24.10.6/ -│ │ └── report.xml -│ ├── openwrt_one_1-24.10.6/ -│ │ └── report.xml -│ ├── belkin_rt3200_1-25.12.2/ -│ │ └── report.xml -│ ├── belkin_rt3200_2-25.12.2/ -│ │ └── report.xml -│ ├── belkin_rt3200_3-25.12.2/ -│ │ └── report.xml -│ ├── bpi_r4_1-25.12.2/ -│ │ └── report.xml -│ └── openwrt_one_1-25.12.2/ -│ └── report.xml +│ ├── belkin_rt3200_1-24.10.6/report.xml +│ ├── belkin_rt3200_2-24.10.6/report.xml +│ ├── belkin_rt3200_3-24.10.6/report.xml +│ ├── bananapi_bpi-r4-24.10.6/report.xml +│ └── openwrt_one-24.10.6/report.xml +├── mesh/ +│ └── 24.10.6/report.xml +├── mesh-pairs/ +│ ├── 1-24.10.6/report.xml +│ ├── 2-24.10.6/report.xml +│ └── 3-24.10.6/report.xml ├── qemu-single/ -│ ├── qemu_x86_64-24.10.6/ -│ │ └── report.xml -│ └── qemu_x86_64-25.12.2/ -│ └── report.xml +│ ├── qemu_x86_64-24.10.6/report.xml +│ └── qemu_x86_64-25.12.2/report.xml └── qemu-mesh/ - ├── qemu_x86_64-24.10.6/ - │ └── report.xml - └── qemu_x86_64-25.12.2/ - └── report.xml + ├── qemu_x86_64-24.10.6/report.xml + └── qemu_x86_64-25.12.2/report.xml ``` -## Required secret +## Required secrets + +| Secret | Stored in | Purpose | Permissions | +|--|--|--|--| +| `LIME_PACKAGES_TOKEN` | `fcefyn_testbed_utils` | Read CI runs and download artifacts from `lime-packages` | Fine-grained PAT scoped to `fcefyn-testbed/lime-packages` with **Actions: Read** | +| `BOT_PR_TOKEN` | `fcefyn_testbed_utils` | Open the auto-merged PR with the collected results | Fine-grained PAT scoped to `fcefyn-testbed/fcefyn_testbed_utils` with **Contents: Read and write** + **Pull requests: Read and write** | + +## Repository settings required -The publish pipeline requires a fine-grained PAT stored as `TESTBED_UTILS_TOKEN` in `lime-packages` repository secrets, with `Contents: Read and write` permission on this repository. +These need to be enabled once for the auto-merge step to work: + +- **Settings → Actions → General → Workflow permissions** → "Allow GitHub Actions to create and approve pull requests" (toggled via the `BOT_PR_TOKEN`, but the repo-level toggle must also be on) +- **Settings → General → Pull Requests** → "Allow auto-merge" +- **`develop` branch protection** → keep "Require signed commits" **off** (PRs created via the GitHub API with a user PAT are not auto-signed by GitHub; enabling this would block the bot) ## Triggering manually -To publish results outside the daily schedule, trigger `build-firmware.yml` via `workflow_dispatch` — the `publish-results` job runs after the test jobs complete. +To force an immediate collection (e.g. after a fresh CI run in `lime-packages`): + +1. Go to **Actions → Collect lime-packages test results**. +2. **Run workflow** → branch `develop` → optionally set `runs` (how many recent CI runs to scan, default 10). + +A bot PR (`bot/lime-results`) appears and auto-merges as soon as required checks pass. If there are no new reports the workflow exits cleanly without opening a PR. diff --git a/docs/ci-results/results/mesh-pairs/1-24.10.6/report.xml b/docs/ci-results/results/mesh-pairs/1-24.10.6/report.xml new file mode 100644 index 0000000..8413442 --- /dev/null +++ b/docs/ci-results/results/mesh-pairs/1-24.10.6/report.xml @@ -0,0 +1 @@ +/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/mesh-pairs/2-24.10.6/report.xml b/docs/ci-results/results/mesh-pairs/2-24.10.6/report.xml new file mode 100644 index 0000000..6dac2ad --- /dev/null +++ b/docs/ci-results/results/mesh-pairs/2-24.10.6/report.xml @@ -0,0 +1 @@ +/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/mesh-pairs/3-24.10.6/report.xml b/docs/ci-results/results/mesh-pairs/3-24.10.6/report.xml new file mode 100644 index 0000000..e5e292f --- /dev/null +++ b/docs/ci-results/results/mesh-pairs/3-24.10.6/report.xml @@ -0,0 +1 @@ +/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/mesh/24.10.6/report.xml b/docs/ci-results/results/mesh/24.10.6/report.xml new file mode 100644 index 0000000..b42097b --- /dev/null +++ b/docs/ci-results/results/mesh/24.10.6/report.xml @@ -0,0 +1 @@ +/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/laryc/testbed_fcefyn/actions-runner/_work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/physical/bananapi_bpi-r4-24.10.6/report.xml b/docs/ci-results/results/physical/bananapi_bpi-r4-24.10.6/report.xml new file mode 100644 index 0000000..91cf75f --- /dev/null +++ b/docs/ci-results/results/physical/bananapi_bpi-r4-24.10.6/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ci-results/results/physical/belkin_rt3200_1-24.10.6/report.xml b/docs/ci-results/results/physical/belkin_rt3200_1-24.10.6/report.xml new file mode 100644 index 0000000..1fdadeb --- /dev/null +++ b/docs/ci-results/results/physical/belkin_rt3200_1-24.10.6/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ci-results/results/physical/belkin_rt3200_2-24.10.6/report.xml b/docs/ci-results/results/physical/belkin_rt3200_2-24.10.6/report.xml new file mode 100644 index 0000000..356486d --- /dev/null +++ b/docs/ci-results/results/physical/belkin_rt3200_2-24.10.6/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ci-results/results/physical/belkin_rt3200_3-24.10.6/report.xml b/docs/ci-results/results/physical/belkin_rt3200_3-24.10.6/report.xml new file mode 100644 index 0000000..4ce0e5e --- /dev/null +++ b/docs/ci-results/results/physical/belkin_rt3200_3-24.10.6/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ci-results/results/physical/openwrt_one-24.10.6/report.xml b/docs/ci-results/results/physical/openwrt_one-24.10.6/report.xml new file mode 100644 index 0000000..d0632d8 --- /dev/null +++ b/docs/ci-results/results/physical/openwrt_one-24.10.6/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ci-results/results/qemu-mesh/qemu_x86_64-24.10.6/report.xml b/docs/ci-results/results/qemu-mesh/qemu_x86_64-24.10.6/report.xml new file mode 100644 index 0000000..a158f01 --- /dev/null +++ b/docs/ci-results/results/qemu-mesh/qemu_x86_64-24.10.6/report.xml @@ -0,0 +1 @@ +/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/qemu-mesh/qemu_x86_64-25.12.2/report.xml b/docs/ci-results/results/qemu-mesh/qemu_x86_64-25.12.2/report.xml new file mode 100644 index 0000000..7f1ff6d --- /dev/null +++ b/docs/ci-results/results/qemu-mesh/qemu_x86_64-25.12.2/report.xml @@ -0,0 +1 @@ +/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:238: babeld not running on this build/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_mesh.py:245: babeld not running \ No newline at end of file diff --git a/docs/ci-results/results/qemu-single/qemu_x86_64-24.10.6/report.xml b/docs/ci-results/results/qemu-single/qemu_x86_64-24.10.6/report.xml new file mode 100644 index 0000000..6d55adb --- /dev/null +++ b/docs/ci-results/results/qemu-single/qemu_x86_64-24.10.6/report.xml @@ -0,0 +1 @@ +/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_base.py:83: Skipping because feature "{'rootfs'}" is not supported/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_base.py:95: Skipping because feature "{'rootfs'}" is not supported/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_lan.py:102: No br-lan peer present (single-node deployment). Multi-node coverage lives in test-mesh-* jobs. \ No newline at end of file diff --git a/docs/ci-results/results/qemu-single/qemu_x86_64-25.12.2/report.xml b/docs/ci-results/results/qemu-single/qemu_x86_64-25.12.2/report.xml new file mode 100644 index 0000000..a08f637 --- /dev/null +++ b/docs/ci-results/results/qemu-single/qemu_x86_64-25.12.2/report.xml @@ -0,0 +1 @@ +/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_base.py:83: Skipping because feature "{'rootfs'}" is not supported/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_base.py:95: Skipping because feature "{'rootfs'}" is not supported/home/runner/work/lime-packages/lime-packages/libremesh-tests/tests/test_lan.py:102: No br-lan peer present (single-node deployment). Multi-node coverage lives in test-mesh-* jobs. \ No newline at end of file diff --git a/docs/ci-results/usage.md b/docs/ci-results/usage.md index 6e3f918..a95c310 100644 --- a/docs/ci-results/usage.md +++ b/docs/ci-results/usage.md @@ -53,7 +53,7 @@ Use the **release dropdown** to filter cards by OpenWrt release (e.g. `24.10.6`, ### By name -The **search box** filters by device name or place as you type (e.g. `belkin`, `bpi`, `qemu`, `25.12`). +The **search box** filters by device name or place as you type (e.g. `belkin`, `bananapi`, `openwrt`, `qemu`, `25.12`). All three filters (type tab, release dropdown, search) apply simultaneously. diff --git a/docs/configuracion/ci-runner.md b/docs/configuracion/ci-runner.md index ff3ae53..5e9813b 100644 --- a/docs/configuracion/ci-runner.md +++ b/docs/configuracion/ci-runner.md @@ -73,3 +73,28 @@ To move the runner from one repo to another (or user to org): ## 6. Ownership transfer (if needed in near future) When the repo transfers to an org, attached runners move with it. The systemd service name may still reference the old owner; this should not affect operation. + +--- + +## 7. Workflows that use this runner + +The following workflows in `fcefyn-testbed/fcefyn_testbed_utils` target this runner with `runs-on: [self-hosted, testbed-fcefyn]`: + +| Workflow | Trigger | What it does on the runner | +|----------|---------|---------------------------| +| `build-and-test-libremesh.yml` | Manual (`workflow_dispatch`) | Downloads firmware artifact built on GitHub-hosted runners, reserves the DUT via labgrid, loads firmware, and runs libremesh-tests with pytest. | + +The `build` job of that workflow runs on GitHub-hosted runners (no lab hardware needed). Only the `flash_and_test` job lands on this runner. + +See [CI: Build & Test](../operar/ci-build-and-test.md) for full usage instructions. + +--- + +## 8. Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| Runner shows **Offline** in GitHub | Service stopped | `sudo systemctl restart actions.runner.*` | +| `flash_and_test` job queued but never starts | Runner offline or label mismatch | Check runner labels include `testbed-fcefyn` | +| `PermissionError` on labgrid coordinator | Wrong `/etc/labgrid` ownership | See §4 above | +| Job fails at "Reserve DUT" | DUT locked by a previous run | `labgrid-client -p unlock` | diff --git a/docs/configuracion/observabilidad.md b/docs/configuracion/observabilidad.md index b4a7363..c587547 100644 --- a/docs/configuracion/observabilidad.md +++ b/docs/configuracion/observabilidad.md @@ -78,12 +78,13 @@ VPS, Certbot, and tunnel unit: [grafana-public-access.md](grafana-public-access. ## Grafana dashboards -Two dashboards: +Three dashboards: | Dashboard | Source | Description | |-----------|--------|---------------| +| **FCEFyN Testbed - Lab Overview** | Provisioned (JSON in repo) | Lab-wide view: every DUT and the gateway at a glance. Single-row stat panels (scrape `up`, uptime, CPU %, RAM %) plus a consolidated table and a firmware/target table. Uses `dut=~".+"` selectors so the whole fleet renders without picking a device. | | **FCEFyN Testbed - DUTs & gateway** | Provisioned (JSON in repo) | DUTs + WDR3500 gateway. **device** variable with `label_values(up{dut!="lab-orchestrator"}, dut)`: **does not** include the orchestration host. All queries use `dut="$device"` and datasource `uid: prometheus`. | -| **FCEFyN Testbed - Orchestrator Host** | Provisioned (JSON in repo) | Orchestration host (~30 panels). Job `orchestrator-host`, label `dut=lab-orchestrator`. | +| **FCEFyN Testbed - Orchestrator Host** | Provisioned (JSON in repo) | Orchestration host (~40 panels). Job `orchestrator-host`, label `dut=lab-orchestrator`. | ### DUTs & gateway dashboard sections @@ -93,12 +94,25 @@ Two dashboards: | Device info | Instant tables `node_uname_info`, `node_openwrt_info` | | CPU & load | CPU by mode (stacked), load 1/5/15m | | Memory | Total / available / used | -| Network | Traffic and packets per interface (excluding `lo`) | +| Network | Traffic and packets per interface (excluding `lo`); plus a per-interface errors and drops panel with `rate(node_network_{receive,transmit}_{errs,drop}_total[2m])` | | Disk | Usage % per mountpoint, free space | | Temperature | `node_hwmon_temp_celsius`, `node_thermal_zone_temp`, CPU stats / max / ieee80211 radios | | Wi-Fi | `wifi_network_*` (AP), `wifi_stations` / `wifi_station_signal_dbm` (stations, if opkg packages present) | | Labels | Table of scrape labels (`firmware`, `target`, etc.) from `up{dut="$device"}` | +### Lab Overview dashboard sections + +This dashboard has no `device` variable — every panel iterates over all DUTs at once using `dut=~".+"`. Useful for a quick glance at the whole fleet. + +| Section | Panels | +|---------|--------| +| Device health at a glance | `up{dut=~".+"}` rendered as one stat-card per device with UP/DOWN value mapping | +| Uptime | `node_time_seconds - node_boot_time_seconds`, color-thresholded green→yellow→orange | +| CPU | `100 - avg(rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100` per DUT | +| Memory | `100 - MemAvailable / MemTotal * 100` per DUT | +| Device table | One row per DUT consolidating scrape status, uptime, CPU %, RAM % via `merge` + `organize` transformations | +| Firmware & target | Instant table of `node_openwrt_info{dut=~".+"}` with labels-to-fields, surfacing the `firmware` and `target` labels set in the scrape config | + ### Orchestrator Host dashboard sections | Section | Panels | @@ -111,6 +125,8 @@ Two dashboards: | Network - Physical | Bandwidth (bps), Packets/s, Errors & Drops, TCP Connections | | Network - VLANs | Bandwidth and packets for vlan100-108, vlan200 (collapsible) | | System Internals | File Descriptors, Entropy, Sockets by Protocol, Systemd Units (active/failed), Socket Memory | +| Lab Services | Active/inactive stats for `labgrid-exporter.service`, `pdudaemon.service`, `ser2net.service` plus a state-timeline showing transitions over the selected range. Source: `node_systemd_unit_state{job="orchestrator-host",name=…,state="active"}`. | +| WireGuard (wg0) | Tunnel bandwidth (rx positive, tx negative), packet rate, error rate, and a UP/DOWN pill from `node_network_up{device="wg0"}`. Same metrics any other ethernet device reports, just filtered to the tunnel. | For **orchestration host only** metrics, always use **Orchestrator Host**; the DUT dashboard excludes it on purpose from the **device** dropdown. @@ -299,3 +315,4 @@ Paths are relative to the repository root. | `ansible/roles/observability/templates/grafana-dashboards-provider.yml.j2` | File-based dashboard provider in Grafana | | `ansible/roles/observability/files/dashboards/orchestrator-node.json` | Orchestrator host dashboard JSON | | `ansible/roles/observability/files/dashboards/duts-node.json` | DUTs + gateway dashboard JSON (variable excludes `lab-orchestrator`) | +| `ansible/roles/observability/files/dashboards/lab-overview.json` | Lab-wide overview dashboard JSON (every DUT in one screen) | diff --git a/docs/demos.md b/docs/demos.md index 276a5e3..5c55afd 100644 --- a/docs/demos.md +++ b/docs/demos.md @@ -2,6 +2,11 @@ This section collects demo videos showing various operations on the testbed. +| Demo | Topic | +|------|-------| +| [Remote access to the HIL testbed](#remote-access-to-the-hil-testbed) | Administrator access and lab infrastructure recovery | +| [Developer remote access through Labgrid](#developer-remote-access-through-labgrid) | Developer SSH + multi-node mesh test run | + --- ## Remote access to the HIL testbed diff --git a/docs/diseno/integration-overview.md b/docs/diseno/integration-overview.md index 2235a26..c745bb1 100644 --- a/docs/diseno/integration-overview.md +++ b/docs/diseno/integration-overview.md @@ -165,3 +165,6 @@ sequenceDiagram | Set up the WireGuard tunnel and contribute hardware to openwrt-tests | [openwrt-tests onboarding](openwrt-tests-onboarding.md) | | VLAN design, `switch-vlan` CLI, switch configuration | [Lab architecture](lab-architecture.md) | | Virtual mesh tests with QEMU and vwifi | [Virtual mesh](virtual-mesh.md) | +| Build and test lime-packages from a PR via CI | [CI: Build & Test](../operar/ci-build-and-test.md) | +| Understand the lime-packages fork CI build pipeline | [lime-packages CI build](lime-packages-ci-flow.md) | +| Understand the lime-packages fork CI test pipeline | [lime-packages CI tests](lime-packages-test-flow.md) | diff --git a/docs/diseno/lime-packages-ci-flow.md b/docs/diseno/lime-packages-ci-flow.md index 16e7ea4..9096cb6 100644 --- a/docs/diseno/lime-packages-ci-flow.md +++ b/docs/diseno/lime-packages-ci-flow.md @@ -1,3 +1,4 @@ + # lime-packages CI: firmware build pipeline How the **fcefyn-testbed/lime-packages** fork builds per-device LibreMesh @@ -271,3 +272,4 @@ consumes this artifact. To add a new board to this pipeline see the [add-device guide][add]. [add]: lime-packages-add-device.md + diff --git a/docs/diseno/lime-packages-test-flow.md b/docs/diseno/lime-packages-test-flow.md index 0366afb..d0a96d8 100644 --- a/docs/diseno/lime-packages-test-flow.md +++ b/docs/diseno/lime-packages-test-flow.md @@ -260,14 +260,15 @@ manages GitHub issues for failing devices: | Test passes, issue open | Comments "passed" and closes the issue. | Each issue body contains a metadata table (place, device, release, run -link, date) and the full output of `lime-report.sh -m` (markdown -mode, see [PR #1242](https://github.com/libremesh/lime-packages/pull/1242)) -inside a collapsible `
` block. +link, date) and the output of `lime-report.sh` (sections with `### FILE` +/ `### CMD` headers) inside a collapsible `
` block. Upstream +[PR #1242](https://github.com/libremesh/lime-packages/pull/1242) adds a +`-m` flag; this fork invokes the script without flags until that lands. `lime-report` is collected from the DUT **before** poweroff/unlock via -`labgrid-client ssh -- lime-report.sh -m`. If the device is -unreachable or the command does not exist yet, a fallback message is -stored instead. +`labgrid-client ssh -- lime-report.sh` with `LG_PROXY` and `LG_ENV` set. +If SSH fails or the command is missing, a fallback message plus +`labgrid-client` stderr is stored instead. This only triggers on `schedule` so that PRs and manual dispatches do not create noise in the issue tracker. @@ -287,6 +288,13 @@ not create noise in the issue tracker. 5. For QEMU jobs, the `qemu-*-logs` artifact contains the QEMU console plus pytest's `--lg-log`. +Once published, the same `report.xml` files are also visible from the +[CI Test Dashboard](../ci-results/index.md): `collect-lime-results.yml` +in `fcefyn_testbed_utils` pulls these artifacts every 6h and the +dashboard's "Report ↗" link points straight to the file on Pages. +See [Publishing results](../ci-results/publishing.md) for the +collection details. + --- ## 10. Runner prerequisites @@ -375,4 +383,4 @@ The workflow only references `environment: physical-lab` by name. All governance (teams, rulesets, environment reviewers) lives in GitHub repository/org settings, not in the YAML. Any organisation adopting this workflow creates its own `physical-lab` environment and teams -without modifying the workflow file. +without modifying the workflow file. \ No newline at end of file diff --git a/docs/diseno/new-lab-contribution.md b/docs/diseno/new-lab-contribution.md index dd5e741..3ed74d5 100644 --- a/docs/diseno/new-lab-contribution.md +++ b/docs/diseno/new-lab-contribution.md @@ -126,3 +126,4 @@ A lab can contribute to both paths at once: the infrastructure is shared. Scenar - [Switch](../configuracion/switch-config.md) - VLAN layout, `switch.conf` multi-user setup, `switch-vlan` invocation. - [Lab architecture](lab-architecture.md) - per-lab coordinator, VLAN scheduling, locking. - [CI runner](../configuracion/ci-runner.md) - self-hosted runner setup (Scenario B). +- [CI: Build & Test](../operar/ci-build-and-test.md) - automated firmware build and test workflow using the self-hosted runner. diff --git a/docs/diseno/test-run-flow.md b/docs/diseno/test-run-flow.md index e4afb6d..4b111b2 100644 --- a/docs/diseno/test-run-flow.md +++ b/docs/diseno/test-run-flow.md @@ -52,16 +52,24 @@ Each test function connects, runs commands, asserts results. pytest generates a JUnit XML report (`--junitxml report.xml`) and uploads it as a GitHub Actions artifact (`test-results-PLACE-RELEASE`). -### 7. publish-results job - -After all test jobs complete, `publish-results` runs: - -1. Checks out `fcefyn_testbed_utils` (via `TESTBED_UTILS_TOKEN`) -2. Downloads all `test-results-*` artifacts -3. Copies each `report.xml` to `docs/ci-results/results/{type}/{place}-{release}/` -4. Commits and pushes to `develop` +### 7. Results pulled into the dashboard + +There is no publish step inside `lime-packages`. Instead, the +`collect-lime-results.yml` workflow in `fcefyn_testbed_utils` runs every +6 h (or on `workflow_dispatch`) and: + +1. Lists recent runs of `build-firmware.yml` (and `tests.yml`) via the + GitHub Actions REST API, using `LIME_PACKAGES_TOKEN` +2. Downloads each `test-results-*` artifact and extracts `report.xml` +3. Maps the artifact name to the dashboard's expected path + (`docs/ci-results/results/{type}/{identifier}/report.xml`) +4. Opens an auto-merged bot PR onto `develop` with `BOT_PR_TOKEN` + (`peter-evans/create-pull-request` + `gh pr merge --auto --squash`) 5. `pages.yml` deploys the updated site to GitHub Pages +See [Publishing results](../ci-results/publishing.md) for the full +diagram and required secrets. + ### 8. Dashboard updates Next time someone opens the dashboard, the new `report.xml` is fetched from Pages and the card re-renders with per-test-case details. @@ -121,6 +129,6 @@ sequenceDiagram Runner->>DUT: run tests (SSH proxy) Runner->>GH: upload report.xml artifact Runner->>Coord: unlock place - GH->>GH: publish-results job + Note over GH: collect-lime-results.yml
(in fcefyn_testbed_utils, every 6h)
pulls artifact and opens bot PR GH->>GH: pages.yml deploy ``` diff --git a/docs/glosario.md b/docs/glosario.md index 5093e3b..f16eb29 100644 --- a/docs/glosario.md +++ b/docs/glosario.md @@ -4,6 +4,9 @@ Key terms used across the FCEFyN testbed documentation. --- +**auto-merge** +: A GitHub Pull Request setting that merges the PR automatically as soon as all required reviews and status checks pass. The CI results pipeline relies on it so the bot PR opened by `collect-lime-results.yml` lands on `develop` without manual intervention. Requires the repo-level "Allow auto-merge" toggle to be on. + **batman-adv** : B.A.T.M.A.N. Advanced — a mesh routing protocol implemented as a Linux kernel module. Operates at Layer 2, handling frame forwarding between mesh nodes. Used by LibreMesh for L2 mesh connectivity. @@ -34,12 +37,21 @@ Key terms used across the FCEFyN testbed documentation. **mac80211_hwsim** : A Linux kernel module that creates virtual IEEE 802.11 (WiFi) radios. Used in combination with vwifi to simulate WiFi connectivity between QEMU VMs without physical hardware. +**node_openwrt_info** +: A Prometheus metric exported by node_exporter (with custom textfile collector or scrape labels) on each DUT, carrying `firmware` and `target` as labels so the Lab Overview dashboard can show what's running on each device. + +**node_systemd_unit_state** +: A node_exporter metric describing whether a systemd unit is in a given state (`active`, `failed`, `inactive`, …). Used by the Orchestrator Host dashboard's Lab Services section to assert that `labgrid-exporter.service`, `pdudaemon.service`, and `ser2net.service` are running. + **openwrt-tests** : The test suite and pytest infrastructure (in `lime-packages`) that defines the test cases for both physical DUTs and virtual mesh nodes. **pdudaemon** : A daemon that controls power to DUTs via relay boards or PDUs. Exposes an HTTP API. The lab uses an Arduino + SSR relay board controlled by pdudaemon. +**PAT (Personal Access Token)** +: A GitHub credential scoped per user. The CI results pipeline uses two fine-grained PATs as repository secrets in `fcefyn_testbed_utils`: `LIME_PACKAGES_TOKEN` (Actions: Read on `lime-packages`, for downloading artifacts) and `BOT_PR_TOKEN` (Contents + Pull Requests: Write on this repo, for opening the auto-merged bot PR). + **place** : A Labgrid concept representing one testable resource (a DUT with all its attached resources). A place has a name (e.g. `labgrid-fcefyn-belkin_rt3200_1`) and is registered with the coordinator. diff --git a/docs/index.md b/docs/index.md index fe455f5..2488f9f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,3 +21,16 @@ Relationship between orchestration host, switch, gateway, DUTs, power, and seria ![High-level diagram of the testbed and main components](img/diagrams/general-design-overview.png) The design builds on the **remote lab** model from [openwrt-tests](https://github.com/aparcar/openwrt-tests), but scope is not limited to **adding devices** to that network. It also **reuses and extends** the approach with **local infrastructure**, along the same axes (orchestration, network, power, serial) with a focus on **LibreMesh** testing. + +--- + +## Quick actions + +| I want to… | Go to | +|---|---| +| Run tests on lab hardware from my machine | [Developer quickstart](operar/developer-remote-access.md) | +| Build a LibreMesh firmware and test it automatically | [CI: Build & Test](operar/ci-build-and-test.md) | +| SSH into a DUT | [SSH access to DUTs](operar/dut-ssh-access.md) | +| Check the rack layout and device IPs | [Rack quick reference](operar/rack-cheatsheets.md) | +| Build firmware manually | [Build firmware](operar/build-firmware-manual.md) | +| Add a new device to the lab | [Adding a DUT](operar/dut-onboarding.md) | diff --git a/docs/operar/build-firmware-manual.md b/docs/operar/build-firmware-manual.md index d499e00..016ebb3 100644 --- a/docs/operar/build-firmware-manual.md +++ b/docs/operar/build-firmware-manual.md @@ -182,6 +182,34 @@ Firmware appears under `bin/` as in standard OpenWrt. --- +## Pre-built firmwares (`firmwares/`) + +The `firmwares/` directory in this repo contains pre-built images ready to use in the lab without compiling. Useful for quick tests or when the build pipeline is unavailable. + +``` +firmwares/ +├── belkin_rt3200/ +│ ├── libremesh/ ← LibreMesh images (initramfs + sysupgrade) +│ └── openwrt/ ← Stock OpenWrt images +├── bananapi_bpi-r4/ +├── librerouter_librerouter-v1/ +├── openwrt_one/ +├── tplink-wdr3500/ +└── qemu/ +``` + +Each device folder has: + +| File pattern | Purpose | +|---|---| +| `*initramfs*.bin` / `*initramfs*.itb` | TFTP boot — loads into RAM, flash untouched | +| `*sysupgrade*.bin` / `*sysupgrade*.itb` | Flash write via `sysupgrade` | +| `*sdcard*.img.gz` | SD card image (Banana Pi R4 only) | + +To use a pre-built initramfs in tests, set `LG_IMAGE` to the file path. See [Running tests](lab-running-tests.md). + +--- + ## Automatic builds via PR (lime-packages fork) The **fcefyn-testbed/lime-packages** fork includes `.github/workflows/build-firmware.yml`: on pull requests (and manual dispatch), GitHub Actions builds a **local lime_packages feed** with the OpenWrt SDK, then one **firmware image per row** in `.github/ci/targets.yml` using ImageBuilder. Successful runs upload **`firmware-.*`** and **`lime-feed-`** artifacts. @@ -192,3 +220,5 @@ For architecture, caching, and feed indexing details, see: - [lime-packages CI: hardware tests](../diseno/lime-packages-test-flow.md) (downstream **libremesh-tests** on the `testbed-fcefyn` self-hosted runner) Manual procedures above still apply when you build outside CI or need a custom `menuconfig` / full Buildroot tree. + +For the automated CI workflow that builds and tests firmware directly in this repo, see [CI: Build & Test](ci-build-and-test.md). diff --git a/docs/operar/ci-build-and-test.md b/docs/operar/ci-build-and-test.md new file mode 100644 index 0000000..2c922d6 --- /dev/null +++ b/docs/operar/ci-build-and-test.md @@ -0,0 +1,167 @@ +# CI: Build & Test LibreMesh + +This workflow builds a LibreMesh firmware image from source and runs automated +tests on a physical device in the FCEFYN lab. + +It is triggered **manually** — it does not run automatically on every commit. +You decide when to run it and with which parameters. + +--- + +## When to use it + +Run this workflow when you want to: + +- Test a specific version or branch of + [lime-packages](https://github.com/libremesh/lime-packages) on real hardware +- Verify that a set of packages installs and boots correctly on a lab device +- Produce a firmware image with custom packages for a specific device + +--- + +## How to run it + +1. Go to the repository on GitHub +2. Click the **Actions** tab +3. Select **Build LibreMesh and Test on DUT** in the left panel +4. Click **Run workflow** +5. Fill in the inputs (see below) and click the green **Run workflow** button + +--- + +## Inputs + +| Input | Required | Default | Description | +|-------|----------|---------|-------------| +| `duts` | yes | `belkin_rt3200` | Device(s) to build and test. Comma-separated. Use `all` for every lab device. | +| `lime_ref` | yes | `v2024.1` | Branch, tag, or commit SHA of lime-packages to build from. | +| `openwrt_version` | no | `23.05.5` | OpenWrt version to use. Must be compatible with `lime_ref`. | +| `extra_packages` | no | _(empty)_ | Space-separated packages to add or remove from the base set. Prefix with `-` to remove. Example: `luci-app-dawn -lime-proto-batadv` | +| `config_file` | no | _(empty)_ | Repo-relative path to a config file to inject as `/etc/config/` in the firmware. Example: `firmware/configs/belkin_rt3200.conf` | + +### Supported devices + +| `duts` value | Hardware | OpenWrt target | +|---|---|---| +| `belkin_rt3200` | Belkin RT3200 / Linksys E8450 | `mediatek/mt7622` | +| `openwrt_one` | OpenWrt One | `mediatek/filogic` | +| `bananapi_r4` | Banana Pi R4 | `mediatek/filogic` | +| `librerouter` | LibreRouter v1 | `ath79/generic` | + +### OpenWrt / lime-packages compatibility + +| `openwrt_version` | Compatible `lime_ref` | +|---|---| +| `23.05.5` | `v2024.1` | +| `24.10.5` | `master` or newer tags | +| `25.12.0` | `master` or newer tags | + +--- + +## What happens when you run it + +The workflow has three jobs that run in sequence: + +### 1. Resolve matrix (~5 seconds, GitHub-hosted) + +Parses the `duts` input and builds a job matrix so each device runs in +parallel. For example, `"belkin_rt3200,librerouter"` becomes two independent +build jobs. + +### 2. Build (~20–25 min per device, GitHub-hosted) + +For each device, two Docker containers run back to back: + +**Step 1 — OpenWrt SDK** +Downloads `ghcr.io/openwrt/sdk:--v` and +compiles the lime-packages listed below from source, using the exact git ref +you specified in `lime_ref`. This is the slow step. + +Packages compiled: + +- `lime-system` +- `lime-proto-babeld` +- `lime-proto-batadv` +- `lime-proto-anygw` +- `lime-hwd-openwrt-wan` +- `lime-app` +- `shared-state` + `shared-state-babeld_hosts` + `shared-state-bat_hosts` + `shared-state-nodes_and_links` +- `babeld-auto-gw-mode` +- anything you add via `extra_packages` + +**Step 2 — OpenWrt ImageBuilder** +Downloads `ghcr.io/openwrt/imagebuilder:--v` +and assembles the compiled `.ipk` packages into a complete firmware image +(`.bin` or `.itb`). This step takes ~2–3 minutes. + +The firmware is uploaded as a GitHub Actions artifact named +`firmware---` and kept for 7 days. + +### 3. Flash and test (lab self-hosted runner, `testbed-fcefyn`) + +Runs on the physical T430 machine in the FCEFYN lab. + +1. Downloads the firmware artifact from step 2 +2. Reserves the target device via [labgrid](https://labgrid.readthedocs.io) + (waits if the device is busy) +3. Loads the firmware onto the device +4. Runs the [libremesh-tests](https://github.com/fcefyn-testbed/libremesh-tests) + test suite with pytest +5. Releases the device when done (even if tests fail) + +--- + +## Examples + +**Test the latest stable lime-packages on the Belkin RT3200:** +``` +duts: belkin_rt3200 +lime_ref: v2024.1 +openwrt_version: 23.05.5 +``` + +**Test a feature branch on all devices:** +``` +duts: all +lime_ref: my-feature-branch +openwrt_version: 24.10.5 +``` + +**Add a package and remove another:** +``` +duts: belkin_rt3200 +lime_ref: v2024.1 +extra_packages: luci-app-dawn -lime-proto-batadv +``` + +**Inject a custom network config:** +``` +duts: belkin_rt3200 +lime_ref: v2024.1 +config_file: firmware/configs/belkin_rt3200.conf +``` + +--- + +## Testing the build locally with `act` + +[act](https://github.com/nektos/act) lets you run the build job on your own +machine without pushing to GitHub. Only the `build` job works locally — the +`flash_and_test` job requires physical lab hardware and is automatically +skipped. + +```bash +act workflow_dispatch \ + --workflows .github/workflows/build-and-test-libremesh.yml \ + --job build \ + --input duts="belkin_rt3200" \ + --input lime_ref="v2024.1" \ + --input openwrt_version="23.05.5" \ + --input config_file="" \ + --input extra_packages="" \ + -P ubuntu-latest=catthehacker/ubuntu:act-22.04 \ + --artifact-server-path /tmp/act-artifacts +``` + +The firmware is saved to `/tmp/act-artifacts` and to `./images/` in the repo +root (not committed). diff --git a/docs/operar/debugging-faq.md b/docs/operar/debugging-faq.md index 4996b2e..fdced48 100644 --- a/docs/operar/debugging-faq.md +++ b/docs/operar/debugging-faq.md @@ -129,7 +129,7 @@ The GitHub API rate limit (60 req/hour unauthenticated) may have been hit. Wait ### Card shows "Test details not yet published" -The `publish-results` job has not run yet, or `report.xml` failed to upload. Check the workflow run in GitHub Actions and verify the `test-results-*` artifact was created. +`collect-lime-results.yml` (every 6 h) hasn't pulled the `report.xml` for that device/release yet, or the CI artifact failed to upload. Verify the `test-results-*` artifact exists on the run in `lime-packages` Actions, then trigger the collect workflow manually from this repo: **Actions → Collect lime-packages test results → Run workflow**. ### Dashboard shows stale data after a new run diff --git a/docs/operar/developer-remote-access.md b/docs/operar/developer-remote-access.md index fba251f..ec583ac 100644 --- a/docs/operar/developer-remote-access.md +++ b/docs/operar/developer-remote-access.md @@ -305,7 +305,24 @@ ZeroTier is **not** required for developer access. It is used only by lab admins --- -## 10. Reference +## 10. Triggering the CI workflow as a developer + +Instead of running tests manually, you can trigger the full build + test pipeline from GitHub: + +1. Go to [fcefyn-testbed/fcefyn_testbed_utils → Actions → Build LibreMesh and Test on DUT](https://github.com/fcefyn-testbed/fcefyn_testbed_utils/actions/workflows/build-and-test-libremesh.yml) +2. Click **Run workflow** and fill in: + - `duts`: the device you want to test + - `lime_ref`: the lime-packages branch, tag, or commit you want to validate + - `openwrt_version`: must be compatible with `lime_ref` +3. The build runs on GitHub's servers (~20 min). The `flash_and_test` job then runs automatically on the lab hardware. + +This does **not** require SSH access to the lab. The only requirement is having a GitHub account with access to the repository. + +For full workflow documentation: [CI: Build & Test](ci-build-and-test.md). + +--- + +## 11. Reference - [Running tests (host-side)](lab-running-tests.md) - [SSH access to DUTs](dut-ssh-access.md) - VLAN lifecycle and mesh SSH diff --git a/docs/operar/lab-routine-operations.md b/docs/operar/lab-routine-operations.md index 88bf028..d981711 100644 --- a/docs/operar/lab-routine-operations.md +++ b/docs/operar/lab-routine-operations.md @@ -136,3 +136,26 @@ systemctl status arduino-relay-daemon # Restart exporter sudo systemctl restart labgrid-exporter ``` + +--- + +## Verify CI runner {: #verify-ci-runner } + +```bash +# Check runner service +sudo systemctl status actions.runner.* + +# Restart runner +sudo systemctl restart actions.runner.* +``` + +The runner should show **Idle** in GitHub → Settings → Actions → Runners. +If a `flash_and_test` job is queued but never starts, check that the runner is online +and that the DUT is not locked by a previous run: + +```bash +labgrid-client reservations # check for stale reservations +labgrid-client -p unlock # release if needed +``` + +See [CI runner](../configuracion/ci-runner.md) for full setup and troubleshooting. diff --git a/docs/operar/lab-running-tests.md b/docs/operar/lab-running-tests.md index 91cc01a..eb0fe47 100644 --- a/docs/operar/lab-running-tests.md +++ b/docs/operar/lab-running-tests.md @@ -69,6 +69,24 @@ The test suite locates `labnet.yaml` via `LABNET_PATH`, `OPENWRT_TESTS_DIR/labne --- +## Using CI-built firmware in tests + +The `build-and-test-libremesh.yml` workflow builds a firmware artifact and runs `flash_and_test` automatically. If you want to run the tests manually with a firmware built by CI: + +1. Go to GitHub → Actions → **Build LibreMesh and Test on DUT** → select a run +2. Download the artifact `firmware---` +3. Extract the `.bin` or `.itb` file and point `LG_IMAGE` to it: + +```bash +LG_PLACE=labgrid-fcefyn-belkin_rt3200_1 \ +LG_IMAGE=/path/to/openwrt-23.05.5-mediatek-mt7622-linksys_e8450-squashfs-sysupgrade.bin \ +uv run pytest tests/ -v +``` + +Artifacts are kept for 7 days. For automated end-to-end runs, trigger the workflow directly — it handles the full build → flash → test pipeline without manual steps. + +--- + ## Remote coordinator access (openwrt-tests) ```bash diff --git a/docs/operar/labgrid-useful-commands.md b/docs/operar/labgrid-useful-commands.md index 644df81..e579050 100644 --- a/docs/operar/labgrid-useful-commands.md +++ b/docs/operar/labgrid-useful-commands.md @@ -134,3 +134,49 @@ ansible-playbook playbook_labgrid.yml -l labgrid-fcefyn (Playbook path and inventory as in [ansible-labgrid](../configuracion/ansible-labgrid.md).) VLAN or switch issues are outside `labgrid-client`; see [Routine operations - DUTs and VLANs](lab-routine-operations.md#duts-and-vlans) and [switch-config](../configuracion/switch-config.md). + +--- + +## Helper scripts + +These scripts live in `scripts/` and complement the labgrid workflow. + +### generate_places_yaml.py + +Generates `places.yaml` for the labgrid coordinator from `labnet.yaml` and a Jinja2 template. Run this after adding or removing DUTs from `labnet.yaml`. + +```bash +# Generate for the default lab (labgrid-fcefyn) +python3 scripts/generate_places_yaml.py + +# Generate for a different lab +python3 scripts/generate_places_yaml.py --lab labgrid-hsn + +# Custom paths +python3 scripts/generate_places_yaml.py \ + --labnet /path/to/labnet.yaml \ + --output ~/labgrid-coordinator/places.yaml +``` + +### resolve_target.py + +Resolves the labgrid target file for a given device name. Useful for debugging which `targets/.yaml` would be used by pytest. + +```bash +python3 scripts/resolve_target.py belkin_rt3200_1 +# Output: targets/linksys_e8450.yaml +``` + +!!! note + This script is optional. When running pytest, `LG_ENV` is resolved automatically from `LG_PLACE`. Use this only for debugging or manual environment setup. + +### provision_mesh_ip.py + +Sets up the per-DUT mesh SSH/control IP (`10.13.200.x`) on `br-lan` via serial. Run once per DUT after flashing. + +```bash +python3 scripts/provision_mesh_ip.py --all # all DUTs +python3 scripts/provision_mesh_ip.py --all --dry-run # verify without applying +``` + +See [duts-config](../configuracion/duts-config.md) for the per-DUT IP table. diff --git a/docs/operar/rack-cheatsheets.md b/docs/operar/rack-cheatsheets.md index 6ddf164..c621333 100644 --- a/docs/operar/rack-cheatsheets.md +++ b/docs/operar/rack-cheatsheets.md @@ -37,6 +37,24 @@ LibreMesh feeds, `menuconfig`, lime packages, QEMU/vwifi: [build-firmware-manual --- +## CI workflow quick reference + +Trigger a firmware build and test from GitHub Actions without touching the lab manually. + +**Go to:** GitHub → Actions → **Build LibreMesh and Test on DUT** → Run workflow + +| Input | Example | Notes | +|-------|---------|-------| +| `duts` | `belkin_rt3200` or `all` | Comma-separated or `all` | +| `lime_ref` | `v2024.1` | Branch, tag, or commit SHA | +| `openwrt_version` | `23.05.5` | Must match `lime_ref` | +| `extra_packages` | `luci-app-dawn` | Prefix with `-` to remove | +| `config_file` | `firmware/configs/belkin_rt3200.conf` | Optional, injected as `/etc/config/` | + +The `flash_and_test` job runs on the **T430Runner** (`testbed-fcefyn`). Full guide: [CI: Build & Test](ci-build-and-test.md). + +--- + ## SSH: Oracle VPS and OpenWrt gateway From the **orchestration host** (same machine as Labgrid). Requires `~/.ssh/config` per repo templates. @@ -48,6 +66,25 @@ From the **orchestration host** (same machine as Labgrid). Requires `~/.ssh/conf --- +## Switch and power scripts + +| Script | Purpose | Example | +|--------|---------|---------| +| `scripts/switch/poe_switch_control.py` | Control PoE ports on the TP-Link switch | `python3 scripts/switch/poe_switch_control.py off 1` | +| `scripts/switch/dut_gateway.py` | Update default gateway on DUTs via SSH after VLAN change | `python3 scripts/switch/dut_gateway.py --dut belkin_rt3200` | + +```bash +# Power cycle OpenWRT One (PoE port 1) +python3 scripts/switch/poe_switch_control.py off 1 +sleep 3 +python3 scripts/switch/poe_switch_control.py on 1 + +# Set SWITCH_PASSWORD env var or use ~/.config/switch.conf +export SWITCH_PASSWORD=yourpassword +``` + +--- + ## TP-Link SG2016P switch | Field | Value | diff --git a/docs/operar/system-operation-manual.md b/docs/operar/system-operation-manual.md index 4d072ba..ba9449a 100644 --- a/docs/operar/system-operation-manual.md +++ b/docs/operar/system-operation-manual.md @@ -78,6 +78,7 @@ Automation of necessary operations is achieved using: - **Ansible:** `ansible/playbook_labgrid.yml` (exporter, places, users). See [ansible-labgrid](../configuracion/ansible-labgrid.md). - **Dynamic VLAN:** `labgrid-switch-abstraction` (used by libremesh-tests in CI); manual ops: `switch-vlan`. Details in [Lab architecture](../diseno/lab-architecture.md). +- **CI firmware build:** `build-and-test-libremesh.yml` compiles lime-packages from a specific git ref using the OpenWrt SDK and ImageBuilder on GitHub-hosted runners, then runs libremesh-tests on the physical DUTs via the `testbed-fcefyn` self-hosted runner. Triggered manually. See [CI: Build & Test](ci-build-and-test.md). --- diff --git a/docs/operar/virtual-mesh-local.md b/docs/operar/virtual-mesh-local.md new file mode 100644 index 0000000..d1ccfa4 --- /dev/null +++ b/docs/operar/virtual-mesh-local.md @@ -0,0 +1,111 @@ +# Running the virtual mesh locally + +How to launch QEMU-based virtual mesh nodes on a developer machine using +[vwifi](https://github.com/sysprog21/vwifi) for WiFi simulation, and run the +libremesh-tests suite against them without physical hardware. + +--- + +## 1. Check the source image + +The build source image lives in: + +``` +firmwares/qemu/libremesh/lime--viwifi-x86-64-generic-ext4-combined.img +``` + +The `vms/node*.img` files are **working copies** created by QEMU at runtime — +they are not the source image and should not be used as a base. + +```bash +# Check size (should be ~121 MB) and date +ls -lh firmwares/qemu/libremesh/*.img + +# Confirm it is a bootable x86 disk +file firmwares/qemu/libremesh/lime-*.img +# Expected: "DOS/MBR boot sector" + +# Check the LibreMesh version inside the image +strings firmwares/qemu/libremesh/lime-*.img | grep -i 'DISTRIB_RELEASE\|lime_release' + +# Hash for comparison with previous builds +md5sum firmwares/qemu/libremesh/lime-*.img +``` + +--- + +## 2. Launch the VMs + +```bash +# From the repo root +VIRTUAL_MESH_IMAGE=firmwares/qemu/libremesh/lime-*.img ./vms/launch_debug_vms.sh +``` + +Available environment variables: + +| Variable | Default | Description | +|---|---|---| +| `VIRTUAL_MESH_IMAGE` | — | Path to the image (required) | +| `VIRTUAL_MESH_NODES` | `2` | Number of VMs to start | +| `VIRTUAL_MESH_BOOT_TIMEOUT` | `120` | Seconds to wait for boot | +| `VIRTUAL_MESH_CONVERGENCE_WAIT` | `60` | Seconds to wait for mesh convergence | +| `VIRTUAL_MESH_SKIP_VWIFI` | `0` | Skip vwifi setup (useful for quick debugging) | + +### SSH into the nodes + +```bash +ssh -o StrictHostKeyChecking=no -p 2222 root@127.0.0.1 # VM 1 +ssh -o StrictHostKeyChecking=no -p 2223 root@127.0.0.1 # VM 2 +``` + +--- + +## 3. Run the tests + +With the VMs already running: + +```bash +# All tests +pytest tests/mesh/ -v + +# Node health only (interfaces, services, UCI, kernel) +pytest tests/mesh/test_mesh_node_basic.py -v + +# Network connectivity (ping bat0, unique IPs, inter-node visibility) +pytest tests/mesh/test_mesh_basic.py -v + +# batman-adv (TQ, originators, symmetry, statistics) +pytest tests/mesh/test_mesh_batman.py -v +``` + +With 3 nodes: + +```bash +VIRTUAL_MESH_NODES=3 VIRTUAL_MESH_IMAGE=firmwares/qemu/libremesh/lime-*.img \ + ./vms/launch_debug_vms.sh & + +# Wait for "Debug session ready", then: +VIRTUAL_MESH_NODES=3 pytest tests/mesh/ -v +``` + +--- + +## 4. Useful commands inside a node + +```bash +batctl n # direct batman-adv neighbours +batctl o # originator table (all mesh routes) +batctl if # batman slave interfaces +batctl s # traffic statistics +ip addr show bat0 # node IP on the mesh +logread | grep lime-config # verify lime-config ran +uci show vwifi # vwifi client config +``` + +--- + +## 5. Related pages + +- [Virtual mesh design](../diseno/virtual-mesh.md) — architecture, CI integration, fixture variables +- [Build firmware](build-firmware-manual.md) — how to build the QEMU vwifi image +- [Running tests](lab-running-tests.md) — physical DUT tests diff --git a/mkdocs.yml b/mkdocs.yml index 34dccc1..5d48f48 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,6 +88,8 @@ nav: - Adding a DUT: operar/dut-onboarding.md - Belkin U-Boot TFTP — PC Ethernet IP: operar/belkin-uboot-tftp-pc-network.md - Build firmware: operar/build-firmware-manual.md + - "CI: Build & Test": operar/ci-build-and-test.md + - Virtual mesh (local): operar/virtual-mesh-local.md - DUT provisioning (full): operar/provision-dut.md - DUT exporter setup: operar/setup-dut-exporter.md - Mesh IP provisioning: operar/provision-mesh-ip.md