From 423855ea625c67cdbcc0a952c11d9630d6ab2be6 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Tue, 19 May 2026 18:20:38 +0200 Subject: [PATCH 1/3] Add $schema versioning to fleet.json; nest broker reference. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts the pod-specification pattern (a $schema URL string identifies the format) for the fleet file. The new on-disk shape: { "$schema": "https://toit.io/schemas/artemis/fleet/v2.json", "id": ..., "broker": { "ref": , "scope": }, ... } Replaces the legacy top-level "organization" UUID and bare-string "broker" with a single "broker" object that nests its ref alongside its scope. The scope is opaque to the fleet schema; for the current Toit-hosted setup it happens to be the organization-id UUID stringified. Future per-service backends (pod-store, fleet-store) get the same nested shape with their own scopes. Reader accepts both shapes: - Files with "$schema" use the new layout. - Files without it fall back to the legacy fields. Writer always emits the new layout — any edit to an old fleet.json upgrades it in place on next write. In-memory model is unchanged: FleetFile still exposes 'organization-id' as a Uuid. Renaming that to a Scope-typed field is a follow-up. Tests: - cmd-fleet-init-test and cmd-fleet-create-reference-test now read fleet-json["broker"]["ref"] when looking up the broker name. - fleet-root-test gains a back-compat case that downgrades the new format to the legacy shape and exercises the reader. --- src/cli/fleet.toit | 50 +++++++++++++++++++--- tests/cmd-fleet-create-reference-test.toit | 2 +- tests/cmd-fleet-init-test.toit | 2 +- tests/fleet-root-test.toit | 27 ++++++++++++ 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/cli/fleet.toit b/src/cli/fleet.toit index 19cfd7aa..c2090c25 100644 --- a/src/cli/fleet.toit +++ b/src/cli/fleet.toit @@ -72,6 +72,8 @@ class Status_: return is-fully-updated and missed-checkins == 0 class FleetFile: + static JSON-SCHEMA ::= "https://toit.io/schemas/artemis/fleet/v2.json" + path/string id/Uuid organization-id/Uuid @@ -105,7 +107,17 @@ class FleetFile: ui.abort "Fleet file '$path' has invalid format." if not fleet-contents.contains "id": ui.abort "Fleet file '$path' does not contain an ID." - if not fleet-contents.contains "organization": + + schema := fleet-contents.get "\$schema" + if schema and schema != JSON-SCHEMA: + ui.abort "Fleet file '$path' has an unsupported schema: $schema" + is-new-format := schema != null + + // The legacy format had a top-level "organization" and a top-level + // "broker" string. The new format nests the broker reference under + // a "broker" object with "ref" and "scope" fields and has no + // top-level "organization". + if not is-new-format and not fleet-contents.contains "organization": ui.abort "Fleet file '$path' does not contain an organization ID." is-reference := fleet-contents.get "is-reference" --if-absent=: false @@ -130,7 +142,28 @@ class FleetFile: ui.abort "Fleet file '$path' has invalid format for 'pod' in group '$group-name'." PodReference.parse entry["pod"] --cli=cli - broker-name := fleet-contents.get "broker" + broker-entry := fleet-contents.get "broker" + broker-name/string? := null + organization-id/Uuid? := null + if is-new-format: + if broker-entry is not Map: + ui.abort "Fleet file '$path' has invalid format for 'broker' (expected an object)." + broker-map := broker-entry as Map + ref-value := broker-map.get "ref" + scope-value := broker-map.get "scope" + if ref-value is not string: + ui.abort "Fleet file '$path' has invalid format for 'broker.ref'." + if scope-value is not string: + ui.abort "Fleet file '$path' has invalid format for 'broker.scope'." + broker-name = ref-value + organization-id = Uuid.parse scope-value + else: + // Legacy format: broker is either a string or absent. + if broker-entry and broker-entry is not string: + ui.abort "Fleet file '$path' has invalid format for 'broker'." + broker-name = broker-entry + organization-id = Uuid.parse fleet-contents["organization"] + migrating-from-entry := fleet-contents.get "migrating-from" servers-entry := fleet-contents.get "servers" @@ -141,8 +174,8 @@ class FleetFile: ui.abort "Fleet file '$path' has invalid format for 'broker' and 'servers'." if servers-entry is not Map: ui.abort "Fleet file '$path' has invalid format for 'servers'." - broker-entry := servers-entry.get broker-name - if not broker-entry: + broker-server-entry := servers-entry.get broker-name + if not broker-server-entry: ui.abort "Fleet file '$path' does not contain a server entry for broker '$broker-name'." servers = servers-entry.map: | server-name/string encoded-server | @@ -187,7 +220,7 @@ class FleetFile: return FleetFile --path=path --id=Uuid.parse fleet-contents["id"] - --organization-id=Uuid.parse fleet-contents["organization"] + --organization-id=organization-id --group-pods=group-pods --is-reference=is-reference --broker-name=broker-name @@ -236,8 +269,8 @@ class FleetFile: to-json_ --reference/bool=false -> Map: result := { + "\$schema": JSON-SCHEMA, "id": "$id", - "organization": "$organization-id", } if reference: result["is-reference"] = true @@ -258,7 +291,10 @@ class FleetFile: result["groups"] = groups // Add the servers last, so that the file is easier to read. - result["broker"] = broker-name + result["broker"] = { + "ref": broker-name, + "scope": "$organization-id", + } if migrating-from and not migrating-from.is-empty: result["migrating-from"] = migrating-from result["servers"] = servers.map: | server-name/string server-config/ServerConfig | diff --git a/tests/cmd-fleet-create-reference-test.toit b/tests/cmd-fleet-create-reference-test.toit index e3bda51c..687ca654 100644 --- a/tests/cmd-fleet-create-reference-test.toit +++ b/tests/cmd-fleet-create-reference-test.toit @@ -33,7 +33,7 @@ run-test fleet/TestFleet: fleet-json := json.decode (file.read-contents ref-file) // Check that we have a broker entry. - broker-name := fleet-json["broker"] + broker-name := fleet-json["broker"]["ref"] broker-entry := fleet-json["servers"][broker-name] // We can still use the ref file to list pods. diff --git a/tests/cmd-fleet-init-test.toit b/tests/cmd-fleet-init-test.toit index ec4ac74b..9b8596a1 100644 --- a/tests/cmd-fleet-init-test.toit +++ b/tests/cmd-fleet-init-test.toit @@ -28,7 +28,7 @@ run-test tester/Tester: fleet-json := json.decode (file.read-contents "$fleet-tmp-dir/fleet.json") // Check that we have a broker entry. - broker-name := fleet-json["broker"] + broker-name := fleet-json["broker"]["ref"] broker-entry := fleet-json["servers"][broker-name] // We are not allowed to initialize a folder twice. diff --git a/tests/fleet-root-test.toit b/tests/fleet-root-test.toit index ff391a6c..2c36c150 100644 --- a/tests/fleet-root-test.toit +++ b/tests/fleet-root-test.toit @@ -1,5 +1,6 @@ // Copyright (C) 2023 Toitware ApS. +import encoding.json import host.file import host.os import expect show * @@ -35,3 +36,29 @@ run-test tester/Tester: expect (file.is-file "$fleet-tmp-dir/fleet.json") expect (file.is-file "$fleet-tmp-dir/devices.json") expect (file.is-file "$fleet-tmp-dir/my-pod.yaml") + + // Verify that the reader still understands the legacy fleet.json + // shape (no $schema, broker as a string, organization at top level). + // We initialize a new fleet, rewrite the file in the legacy shape, + // then exercise a command that has to parse it. + with-tmp-directory: | fleet-tmp-dir | + tester.run [ + "fleet", + "--fleet-root", fleet-tmp-dir, + "init", + "--organization-id", "$TEST-ORGANIZATION-UUID", + ] + fleet-path := "$fleet-tmp-dir/fleet.json" + new-format := json.decode (file.read-contents fleet-path) + broker-entry := new-format["broker"] + legacy := new-format.copy + legacy.remove "\$schema" + legacy["broker"] = broker-entry["ref"] + legacy["organization"] = broker-entry["scope"] + file.write-contents --path=fleet-path (json.encode legacy) + tester.run [ + "fleet", + "--fleet-root", fleet-tmp-dir, + "group", + "list", + ] From 3b27fb3456ce5fe3b85be5dfcd241c078f11c23d Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 20 May 2026 21:42:32 +0200 Subject: [PATCH 2/3] Move broker scope onto each server entry. Restructures the new fleet.json shape: the broker reference is again a plain string referring to a server name, and each server entry carries its own scope alongside its connection info: { "$schema": "...", "broker": "", "servers": { "": { "type": ..., "host": ..., "scope": "" } } } Migration-from servers automatically carry their own scope without needing a parallel scope map at the top level. Picking a different broker is just changing the 'broker' string. The reader pulls the scope field out of each server entry before handing the rest to ServerConfig.from-json, so ServerConfig stays scope-free in memory. For now only the broker's scope is actually used (it populates organization-id); scopes on other entries are read but ignored. They become load-bearing once we track per-server scope in memory. The writer emits the new shape with the fleet's single organization-id copied into every server entry. The in-memory FleetFile is unchanged. Tests reverted to reading fleet-json["broker"] as a string and the back-compat downgrade in fleet-root-test now strips the scope field from server entries (and moves the broker's scope back to a top-level 'organization'). --- src/cli/fleet.toit | 58 +++++++++++++--------- tests/cmd-fleet-create-reference-test.toit | 2 +- tests/cmd-fleet-init-test.toit | 2 +- tests/fleet-root-test.toit | 18 ++++--- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/cli/fleet.toit b/src/cli/fleet.toit index c2090c25..224ff2f8 100644 --- a/src/cli/fleet.toit +++ b/src/cli/fleet.toit @@ -142,26 +142,18 @@ class FleetFile: ui.abort "Fleet file '$path' has invalid format for 'pod' in group '$group-name'." PodReference.parse entry["pod"] --cli=cli + // Broker reference is a string in both the new and legacy layouts. + // In the new layout the broker's scope lives inside its server entry + // (see below); in the legacy layout it's read from the top-level + // "organization" field. broker-entry := fleet-contents.get "broker" broker-name/string? := null + if broker-entry and broker-entry is not string: + ui.abort "Fleet file '$path' has invalid format for 'broker'." + broker-name = broker-entry + organization-id/Uuid? := null - if is-new-format: - if broker-entry is not Map: - ui.abort "Fleet file '$path' has invalid format for 'broker' (expected an object)." - broker-map := broker-entry as Map - ref-value := broker-map.get "ref" - scope-value := broker-map.get "scope" - if ref-value is not string: - ui.abort "Fleet file '$path' has invalid format for 'broker.ref'." - if scope-value is not string: - ui.abort "Fleet file '$path' has invalid format for 'broker.scope'." - broker-name = ref-value - organization-id = Uuid.parse scope-value - else: - // Legacy format: broker is either a string or absent. - if broker-entry and broker-entry is not string: - ui.abort "Fleet file '$path' has invalid format for 'broker'." - broker-name = broker-entry + if not is-new-format: organization-id = Uuid.parse fleet-contents["organization"] migrating-from-entry := fleet-contents.get "migrating-from" @@ -178,12 +170,29 @@ class FleetFile: if not broker-server-entry: ui.abort "Fleet file '$path' does not contain a server entry for broker '$broker-name'." + // The new layout stores each server's scope alongside its + // connection info. Strip the scope field before handing the entry + // to ServerConfig.from-json. For now only the broker's scope is + // actually used (it populates organization-id); scopes on other + // server entries are read but ignored. They become load-bearing + // once we track per-server scope in memory. servers = servers-entry.map: | server-name/string encoded-server | if encoded-server is not Map: ui.abort "Fleet file '$path' has invalid format for server '$server-name'." - ServerConfig.from-json server-name encoded-server + encoded-map := encoded-server as Map + cleaned := encoded-map + if encoded-map.contains "scope": + cleaned = encoded-map.copy + cleaned.remove "scope" + ServerConfig.from-json server-name cleaned --der-deserializer=: base64.decode it + if is-new-format: + broker-scope-value := (broker-server-entry as Map).get "scope" + if broker-scope-value is not string: + ui.abort "Fleet file '$path' is missing 'scope' on broker server '$broker-name'." + organization-id = Uuid.parse broker-scope-value + if migrating-from-entry: if migrating-from-entry is not List: ui.abort "Fleet file '$path' has invalid format for 'migrating-from'." @@ -291,14 +300,17 @@ class FleetFile: result["groups"] = groups // Add the servers last, so that the file is easier to read. - result["broker"] = { - "ref": broker-name, - "scope": "$organization-id", - } + result["broker"] = broker-name if migrating-from and not migrating-from.is-empty: result["migrating-from"] = migrating-from + // Each server entry carries its own scope. For now every entry + // uses the fleet's single organization-id; this anticipates a + // future world where each server can be scoped independently. + scope-string := "$organization-id" result["servers"] = servers.map: | server-name/string server-config/ServerConfig | - server-config.to-json --der-serializer=: base64.encode it + encoded := server-config.to-json --der-serializer=: base64.encode it + encoded["scope"] = scope-string + encoded result["recovery-urls"] = recovery-urls return result diff --git a/tests/cmd-fleet-create-reference-test.toit b/tests/cmd-fleet-create-reference-test.toit index 687ca654..e3bda51c 100644 --- a/tests/cmd-fleet-create-reference-test.toit +++ b/tests/cmd-fleet-create-reference-test.toit @@ -33,7 +33,7 @@ run-test fleet/TestFleet: fleet-json := json.decode (file.read-contents ref-file) // Check that we have a broker entry. - broker-name := fleet-json["broker"]["ref"] + broker-name := fleet-json["broker"] broker-entry := fleet-json["servers"][broker-name] // We can still use the ref file to list pods. diff --git a/tests/cmd-fleet-init-test.toit b/tests/cmd-fleet-init-test.toit index 9b8596a1..ec4ac74b 100644 --- a/tests/cmd-fleet-init-test.toit +++ b/tests/cmd-fleet-init-test.toit @@ -28,7 +28,7 @@ run-test tester/Tester: fleet-json := json.decode (file.read-contents "$fleet-tmp-dir/fleet.json") // Check that we have a broker entry. - broker-name := fleet-json["broker"]["ref"] + broker-name := fleet-json["broker"] broker-entry := fleet-json["servers"][broker-name] // We are not allowed to initialize a folder twice. diff --git a/tests/fleet-root-test.toit b/tests/fleet-root-test.toit index 2c36c150..85311cad 100644 --- a/tests/fleet-root-test.toit +++ b/tests/fleet-root-test.toit @@ -38,9 +38,9 @@ run-test tester/Tester: expect (file.is-file "$fleet-tmp-dir/my-pod.yaml") // Verify that the reader still understands the legacy fleet.json - // shape (no $schema, broker as a string, organization at top level). - // We initialize a new fleet, rewrite the file in the legacy shape, - // then exercise a command that has to parse it. + // shape (no $schema, no scope inside server entries, organization at + // the top level). We initialize a new fleet, rewrite the file in the + // legacy shape, then exercise a command that has to parse it. with-tmp-directory: | fleet-tmp-dir | tester.run [ "fleet", @@ -50,11 +50,17 @@ run-test tester/Tester: ] fleet-path := "$fleet-tmp-dir/fleet.json" new-format := json.decode (file.read-contents fleet-path) - broker-entry := new-format["broker"] legacy := new-format.copy legacy.remove "\$schema" - legacy["broker"] = broker-entry["ref"] - legacy["organization"] = broker-entry["scope"] + broker-name := legacy["broker"] + legacy-servers := (legacy["servers"] as Map).copy + legacy-servers.do: | name encoded-server | + cleaned := (encoded-server as Map).copy + if name == broker-name: + legacy["organization"] = cleaned["scope"] + cleaned.remove "scope" + legacy-servers[name] = cleaned + legacy["servers"] = legacy-servers file.write-contents --path=fleet-path (json.encode legacy) tester.run [ "fleet", From ab0996b79d1452e2ce0d0f887b32992d807b3e52 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 20 May 2026 21:44:34 +0200 Subject: [PATCH 3/3] Fix comment. --- src/cli/fleet.toit | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli/fleet.toit b/src/cli/fleet.toit index 224ff2f8..452d301f 100644 --- a/src/cli/fleet.toit +++ b/src/cli/fleet.toit @@ -113,10 +113,10 @@ class FleetFile: ui.abort "Fleet file '$path' has an unsupported schema: $schema" is-new-format := schema != null - // The legacy format had a top-level "organization" and a top-level - // "broker" string. The new format nests the broker reference under - // a "broker" object with "ref" and "scope" fields and has no - // top-level "organization". + // The legacy format had a top-level "organization" UUID; the new + // format drops it and stores a per-server "scope" inside each + // server entry instead. The "broker" field is a server-name string + // in both layouts. if not is-new-format and not fleet-contents.contains "organization": ui.abort "Fleet file '$path' does not contain an organization ID."