diff --git a/src/cli/broker.toit b/src/cli/broker.toit index 9568e9cf..af303b87 100644 --- a/src/cli/broker.toit +++ b/src/cli/broker.toit @@ -17,7 +17,7 @@ import .config import .device import .pod import .pod-specification -import .scope show Scope +import ..shared.scope show Scope import .utils import .utils.patch-build show build-diff-patch build-trivial-patch @@ -67,7 +67,6 @@ Manages devices that have an Artemis service running on them. */ class Broker: fleet-id/Uuid - organization-id/Uuid server-config/ServerConfig cli_/Cli network_/net.Client? := null @@ -83,7 +82,6 @@ class Broker: constructor --.fleet-id/Uuid - --.organization-id/Uuid --.server-config --cli/Cli --tmp-directory/string @@ -104,16 +102,6 @@ class Broker: cli_.ui.abort "$error-message (broker)." return broker-connection__ - /** - The $Scope to use when talking to the broker. - - For now derived directly from $organization-id. When the fleet file gains - a per-service scope field this will return the broker's own configured - scope instead. - */ - scope -> Scope: - return Scope.from-organization-id organization-id - short-string-for_ --device-id/Uuid -> string: if not device-short-strings_: throw "Access to device in non-device fleet." return device-short-strings_[device-id] @@ -142,7 +130,8 @@ class Broker: return error.contains "duplicate key value" or error.contains "already exists" /** - Uploads the given $pod to the broker for the given $fleet-id in $organization-id. + Uploads the given $pod to the broker for the given $fleet-id under the + broker's configured scope. Also uploads the trivial patches. */ @@ -154,25 +143,23 @@ class Broker: // Only upload if we don't have it in our cache. key := cache-key-pod-parts --broker-config=server-config - --organization-id=organization-id --part-id=id cli_.cache.get-file-path key: | store/FileStore | broker-connection_.pod-registry-upload-pod-part contents --part-id=id - --scope=scope + --scope=server-config.scope store.save contents key := cache-key-pod-manifest --broker-config=server-config - --organization-id=organization-id --pod-id=pod.id cli_.cache.get-file-path key: | store/FileStore | encoded := ubjson.encode manifest broker-connection_.pod-registry-upload-pod-manifest encoded --pod-id=pod.id - --scope=scope + --scope=server-config.scope store.save encoded description-ids := broker-connection_.pod-registry-descriptions --fleet-id=fleet-id - --scope=scope + --scope=server-config.scope --names=[pod.name] --create-if-absent @@ -215,7 +202,7 @@ class Broker: upload-patch_ it /** - Uploads the given $patch to the server under the given $organization-id. + Uploads the given $patch to the broker. */ upload-patch_ patch/FirmwarePatch: diff-and-upload_ patch @@ -228,12 +215,11 @@ class Broker: trivial-id := id_ --to=patch.to_ cache-key := cache-key-patch --broker-config=server-config - --organization-id=organization-id --patch-id=trivial-id cli_.cache.get cache-key: | store/FileStore | trivial := build-trivial-patch patch.bits_ broker-connection_.upload-firmware trivial - --scope=scope + --scope=server-config.scope --firmware-id=trivial-id store.save-via-writer: | writer/io.Writer | trivial.do: writer.write it @@ -245,12 +231,11 @@ class Broker: old-id := id_ --to=patch.from_ cache-key = cache-key-patch --broker-config=server-config - --organization-id=organization-id --patch-id=old-id trivial-old := cli_.cache.get cache-key: | store/FileStore | downloaded := null catch: downloaded = broker-connection_.download-firmware - --scope=scope + --scope=server-config.scope --id=old-id if not downloaded: cli_.ui.emit --warning "Failed to download old firmware for patch $old-id -> $trivial-id." @@ -274,7 +259,6 @@ class Broker: diff-id := id_ --from=patch.from_ --to=patch.to_ cache-key = cache-key-patch --broker-config=server-config - --organization-id=organization-id --patch-id=diff-id cli_.cache.get cache-key: | store/FileStore | // Build the diff and verify that we can apply it and get the @@ -289,7 +273,7 @@ class Broker: to64 := base64.encode patch.to_ --url-mode cli_.ui.emit --info "Uploading patch $from64 -> $to64 ($diff-size)." broker-connection_.upload-firmware diff - --scope=scope + --scope=server-config.scope --firmware-id=diff-id store.save-via-writer: | writer/io.Writer | diff.do: writer.write it @@ -312,19 +296,17 @@ class Broker: is-cached --pod-id/Uuid -> bool: manifest-key := cache-key-pod-manifest --broker-config=server-config - --organization-id=organization-id --pod-id=pod-id return cli_.cache.contains manifest-key download --pod-id/Uuid -> Pod: manifest-key := cache-key-pod-manifest --broker-config=server-config - --organization-id=organization-id --pod-id=pod-id encoded-manifest := cli_.cache.get manifest-key: | store/FileStore | bytes := broker-connection_.pod-registry-download-pod-manifest --pod-id=pod-id - --scope=scope + --scope=server-config.scope store.save bytes manifest := ubjson.decode encoded-manifest return Pod.from-manifest @@ -333,12 +315,11 @@ class Broker: --download=: | part-id/string | key := cache-key-pod-parts --broker-config=server-config - --organization-id=organization-id --part-id=part-id cli_.cache.get key: | store/FileStore | bytes := broker-connection_.pod-registry-download-pod-part part-id - --scope=scope + --scope=server-config.scope store.save bytes list-pods --names/List -> Map: @@ -348,7 +329,7 @@ class Broker: else: descriptions = broker-connection_.pod-registry-descriptions --fleet-id=fleet-id - --scope=scope + --scope=server-config.scope --names=names --no-create-if-absent result := {:} @@ -360,7 +341,7 @@ class Broker: delete --description-names/List: descriptions := broker-connection_.pod-registry-descriptions --fleet-id=fleet-id - --scope=scope + --scope=server-config.scope --names=description-names --no-create-if-absent unknown-pod-descriptions := [] @@ -429,7 +410,7 @@ class Broker: descriptions := broker-connection_.pod-registry-descriptions --fleet-id=fleet-id - --scope=scope + --scope=server-config.scope --names=names.to-list --no-create-if-absent diff --git a/src/cli/brokers/broker.toit b/src/cli/brokers/broker.toit index 28c8331a..302057a7 100644 --- a/src/cli/brokers/broker.toit +++ b/src/cli/brokers/broker.toit @@ -11,7 +11,7 @@ import ..config import ..event import ..device import ..pod-registry -import ..scope show Scope +import ...shared.scope show Scope import ...shared.server-config import .supabase import .http.base diff --git a/src/cli/brokers/http/base.toit b/src/cli/brokers/http/base.toit index 2a58dcca..08cfeef0 100644 --- a/src/cli/brokers/http/base.toit +++ b/src/cli/brokers/http/base.toit @@ -13,7 +13,7 @@ import ..broker import ...device import ...event import ...pod-registry -import ...scope show Scope +import ....shared.scope show Scope import ....shared.server-config import ....shared.utils as utils import ....shared.constants show * diff --git a/src/cli/cache.toit b/src/cli/cache.toit index 99dc830a..0e3910a1 100644 --- a/src/cli/cache.toit +++ b/src/cli/cache.toit @@ -15,21 +15,18 @@ cache-key-application-image id/Uuid --broker-config/ServerConfig -> string: cache-key-pod-parts -> string --broker-config/ServerConfig - --organization-id/Uuid --part-id/string: - return "$broker-config.cache-key/$organization-id/pod/parts/$part-id" + return "$broker-config.cache-key/$broker-config.scope.as-uuid/pod/parts/$part-id" cache-key-pod-manifest -> string --broker-config/ServerConfig - --organization-id/Uuid --pod-id/Uuid: - return "$broker-config.cache-key/$organization-id/pod/manifest/$pod-id" + return "$broker-config.cache-key/$broker-config.scope.as-uuid/pod/manifest/$pod-id" cache-key-patch -> string --broker-config/ServerConfig - --organization-id/Uuid --patch-id/string: - return "$broker-config.cache-key/$organization-id/patches/$patch-id" + return "$broker-config.cache-key/$broker-config.scope.as-uuid/patches/$patch-id" CACHE-ARTIFACT-KIND-ENVELOPE ::= "envelope" CACHE-ARTIFACT-KIND-PARTITION-TABLE ::= "partitions" diff --git a/src/cli/fleet.toit b/src/cli/fleet.toit index 452d301f..d6e8f1b9 100644 --- a/src/cli/fleet.toit +++ b/src/cli/fleet.toit @@ -19,6 +19,7 @@ import .pod import .pod-specification import .pod-registry import .utils +import ..shared.scope show Scope import .utils.names import .server-config import ..shared.json-diff @@ -76,18 +77,16 @@ class FleetFile: path/string id/Uuid - organization-id/Uuid group-pods/Map is-reference/bool broker-name/string migrating-from/List - servers/Map // From broker-name to ServerConfig. + servers/Map // From broker-name to ServerConfig (each carries its scope). recovery-urls/List constructor --.path --.id - --.organization-id --.group-pods --.is-reference --.broker-name @@ -95,6 +94,23 @@ class FleetFile: --.servers --.recovery-urls: + /** + The $Scope to use when talking to the configured broker. + + Derived from the broker server entry's $ServerConfig.scope. + */ + broker-scope -> Scope: + return (servers[broker-name] as ServerConfig).scope + + /** + The organization-id encoded inside $broker-scope. + + Kept as a derived view for callers that talk to the auth provider + (which is still org-id concrete). + */ + organization-id -> Uuid: + return broker-scope.as-uuid + static parse path/string --default-broker-config/ServerConfig --cli/Cli -> FleetFile: ui := cli.ui fleet-contents := null @@ -171,27 +187,25 @@ class FleetFile: ui.abort "Fleet file '$path' does not contain a server entry for broker '$broker-name'." // The new layout stores each server's scope alongside its - // connection info. Strip the scope field before handing the entry - // to ServerConfig.from-json. For now only the broker's scope is - // actually used (it populates organization-id); scopes on other - // server entries are read but ignored. They become load-bearing - // once we track per-server scope in memory. + // connection info; ServerConfig.from-json reads it. servers = servers-entry.map: | server-name/string encoded-server | if encoded-server is not Map: ui.abort "Fleet file '$path' has invalid format for server '$server-name'." - encoded-map := encoded-server as Map - cleaned := encoded-map - if encoded-map.contains "scope": - cleaned = encoded-map.copy - cleaned.remove "scope" - ServerConfig.from-json server-name cleaned + ServerConfig.from-json server-name encoded-server --der-deserializer=: base64.decode it + broker-server/ServerConfig := servers[broker-name] if is-new-format: - broker-scope-value := (broker-server-entry as Map).get "scope" - if broker-scope-value is not string: + if not broker-server.scope: ui.abort "Fleet file '$path' is missing 'scope' on broker server '$broker-name'." - organization-id = Uuid.parse broker-scope-value + organization-id = broker-server.scope.as-uuid + else: + // Legacy format: the top-level organization-id was the same for + // every server. Pin it onto every entry so the new in-memory + // shape is consistent. + legacy-scope := Scope.from-organization-id organization-id + servers.do --values: | server-config/ServerConfig | + server-config.scope = legacy-scope if migrating-from-entry: if migrating-from-entry is not List: @@ -206,6 +220,11 @@ class FleetFile: if migrating-from-entry or servers-entry: ui.abort "Fleet file '$path' has invalid format for 'broker', 'migrating-from' and 'servers'." broker-name = default-broker-config.name + // Very-legacy fleet file with no broker/servers entry. Attach the + // legacy top-level organization-id to the default broker config. + // TODO: avoid mutating the default broker config (shared with the + // global CLI config); clone with scope set instead. + default-broker-config.scope = Scope.from-organization-id organization-id servers = { default-broker-config.name: default-broker-config, } @@ -229,7 +248,6 @@ class FleetFile: return FleetFile --path=path --id=Uuid.parse fleet-contents["id"] - --organization-id=organization-id --group-pods=group-pods --is-reference=is-reference --broker-name=broker-name @@ -250,7 +268,6 @@ class FleetFile: with -> FleetFile --path/string?=null --id/Uuid?=null - --organization-id/Uuid?=null --group-pods/Map?=null --is-reference/bool?=null --broker-name/string?=null @@ -260,7 +277,6 @@ class FleetFile: return FleetFile --path=(path or this.path) --id=(id or this.id) - --organization-id=(organization-id or this.organization-id) --group-pods=(group-pods or this.group-pods) --is-reference=(is-reference or this.is-reference) --broker-name=(broker-name or this.broker-name) @@ -303,14 +319,10 @@ class FleetFile: result["broker"] = broker-name if migrating-from and not migrating-from.is-empty: result["migrating-from"] = migrating-from - // Each server entry carries its own scope. For now every entry - // uses the fleet's single organization-id; this anticipates a - // future world where each server can be scoped independently. - scope-string := "$organization-id" + // Each server entry carries its own scope (serialized by + // ServerConfig.to-json when the scope is non-null). result["servers"] = servers.map: | server-name/string server-config/ServerConfig | - encoded := server-config.to-json --der-serializer=: base64.encode it - encoded["scope"] = scope-string - encoded + server-config.to-json --der-serializer=: base64.encode it result["recovery-urls"] = recovery-urls return result @@ -377,7 +389,12 @@ class Fleet: static FLEET-FILE_ ::= "fleet.json" id/Uuid - organization-id/Uuid + /** + The $Scope to use when talking to the fleet's broker. + + Mirrors $FleetFile.broker-scope. + */ + broker-scope/Scope artemis/Artemis broker/Broker cli_/Cli @@ -401,12 +418,11 @@ class Fleet: --cli/Cli: fleet-file_ = fleet-file id = fleet-file.id - organization-id = fleet-file.organization-id + broker-scope = fleet-file.broker-scope cli_ = cli broker = Broker --server-config=fleet-file.broker-config --fleet-id=id - --organization-id=organization-id --tmp-directory=artemis.tmp-directory --short-strings=short-strings --cli=cli @@ -416,6 +432,15 @@ class Fleet: if not org: cli.ui.abort "Organization $organization-id does not exist or is not accessible." + /** + The organization-id encoded inside $broker-scope. + + Kept as a derived view for callers that talk to the auth provider + (which is still org-id concrete). + */ + organization-id -> Uuid: + return broker-scope.as-uuid + static load-fleet-file -> FleetFile fleet-root-or-ref/string --default-broker-config/ServerConfig @@ -616,10 +641,12 @@ class FleetWithDevices extends Fleet: fleet-id := random-uuid recovery-urls := recovery-url-prefixes.map: | prefix | "$prefix/recover-$(fleet-id).json" + // TODO: avoid mutating broker-config (it may come from the global + // config); clone with scope set instead. + broker-config.scope = Scope.from-organization-id organization-id fleet-file := FleetFile --path="$fleet-root/$FLEET-FILE_" --id=fleet-id - --organization-id=organization-id --group-pods={ DEFAULT-GROUP: PodReference.parse "$INITIAL-POD-NAME@latest" --cli=cli, } @@ -750,7 +777,6 @@ class FleetWithDevices extends Fleet: --server-config=server-config --short-strings=device-short-strings_ --fleet-id=id - --organization-id=organization-id --tmp-directory=artemis.tmp-directory --cli=cli_ old-broker.update --device-id=device-id --pod=pod @@ -794,7 +820,6 @@ class FleetWithDevices extends Fleet: --server-config=server-config --short-strings=device-short-strings_ --fleet-id=id - --organization-id=organization-id --tmp-directory=artemis.tmp-directory --cli=cli_ // We could filter out devices that were already known in the new broker, but @@ -888,7 +913,6 @@ class FleetWithDevices extends Fleet: Broker --server-config=config --fleet-id=id - --organization-id=organization-id --short-strings=device-short-strings_ --cli=cli_ --tmp-directory=artemis.tmp-directory @@ -1108,11 +1132,17 @@ class FleetWithDevices extends Fleet: migration-start_ broker-config migration-start_ new-broker-config/ServerConfig: + // The new broker operates under the same scope as the rest of the + // fleet. The new-broker-config typically comes from the global CLI + // config (which never carries a scope), so attach the fleet's scope + // here. + // TODO: avoid mutating new-broker-config; clone with scope set. + if not new-broker-config.scope: + new-broker-config.scope = fleet-file_.broker-scope new-broker := Broker --server-config=new-broker-config --short-strings=device-short-strings_ --fleet-id=id - --organization-id=organization-id --tmp-directory=artemis.tmp-directory --cli=cli_ @@ -1167,7 +1197,6 @@ class FleetWithDevices extends Fleet: --server-config=fleet-file.servers[name] --short-strings=device-short-strings_ --fleet-id=id - --organization-id=organization-id --tmp-directory=artemis.tmp-directory --cli=cli_ current-detailed-devices := current-broker.get-devices --device-ids=device-ids diff --git a/src/cli/scope.toit b/src/shared/scope.toit similarity index 100% rename from src/cli/scope.toit rename to src/shared/scope.toit diff --git a/src/shared/server-config.toit b/src/shared/server-config.toit index cb41948a..b41054e1 100644 --- a/src/shared/server-config.toit +++ b/src/shared/server-config.toit @@ -4,14 +4,25 @@ import crypto.sha1 import encoding.base64 as base64-lib import supabase import tls +import uuid show Uuid + +import .scope show Scope abstract class ServerConfig: name/string + /** + The $Scope this fleet uses when talking to the server. + + Null outside of fleet-file contexts (e.g., entries in the global CLI + config don't carry a scope — scope is a fleet-level concept). + */ + scope/Scope? := null + cache-key_/string? := null ders-already-installed_/bool := false - constructor.from-sub_ .name: + constructor.from-sub_ .name --.scope=null: /** Creates a new broker-config from a JSON map. @@ -121,6 +132,8 @@ class ServerConfigSupabase extends ServerConfig implements supabase.ServerConfig root-der = root-der-id and (der-deserializer.call root-der-id) use-tls := json.get "use_tls" if use-tls == null: use-tls = json.contains "root_certificate_name" + scope-value := json.get "scope" + scope/Scope? := scope-value and (Scope.from-organization-id (Uuid.parse scope-value)) return ServerConfigSupabase name --host=json["host"] @@ -128,14 +141,16 @@ class ServerConfigSupabase extends ServerConfig implements supabase.ServerConfig --poll-interval=Duration --us=json["poll_interval"] --use-tls=use-tls --root-certificate-der=root-der + --scope=scope constructor name/string --.host --.anon --.use-tls=true --.root-certificate-der=null - --.poll-interval=DEFAULT-POLL-INTERVAL: - super.from-sub_ name + --.poll-interval=DEFAULT-POLL-INTERVAL + --scope/Scope?=null: + super.from-sub_ name --scope=scope operator== other: if other is not ServerConfigSupabase: return false @@ -164,6 +179,8 @@ class ServerConfigSupabase extends ServerConfig implements supabase.ServerConfig serialized := der-serializer.call root-certificate-der if serialized: result["root_certificate_der_id"] = serialized + if scope: + result["scope"] = "$scope.as-uuid" return result to-service-json [--der-serializer] --base64/bool=false -> Map: @@ -202,6 +219,7 @@ class ServerConfigSupabase extends ServerConfig implements supabase.ServerConfig --use-tls=use-tls --root-certificate-der=root-certificate-der --poll-interval=poll-interval + --scope=scope /** A broker configuration for an HTTP-based broker. @@ -228,6 +246,8 @@ class ServerConfigHttp extends ServerConfig: root-certificates-ders = config["root_certificate_ders"].map: der-deserializer.call it use-tls := config.get "use_tls" if use-tls == null: use-tls = config.contains "root_certificate_names" + scope-value := config.get "scope" + scope/Scope? := scope-value and (Scope.from-organization-id (Uuid.parse scope-value)) return ServerConfigHttp name --host=config["host"] --port=config.get "port" @@ -237,6 +257,7 @@ class ServerConfigHttp extends ServerConfig: --device-headers=config.get "device_headers" --admin-headers=config.get "admin_headers" --poll-interval=Duration --us=config["poll_interval"] + --scope=scope constructor name/string --.host @@ -246,9 +267,10 @@ class ServerConfigHttp extends ServerConfig: --.root-certificate-ders --.device-headers --.admin-headers - --.poll-interval=DEFAULT-POLL-INTERVAL: + --.poll-interval=DEFAULT-POLL-INTERVAL + --scope/Scope?=null: - super.from-sub_ name + super.from-sub_ name --scope=scope operator== other: if other is not ServerConfigHttp: return false @@ -276,11 +298,14 @@ class ServerConfigHttp extends ServerConfig: result["device_headers"] = device-headers if admin-headers: result["admin_headers"] = admin-headers + if scope: + result["scope"] = "$scope.as-uuid" return result to-service-json [--der-serializer] --base64/bool=false -> Map: result := to-json --der-serializer=der-serializer --base64=base64 result.remove "admin_headers" + result.remove "scope" return result compute-cache-key_ -> string: diff --git a/tests/utils.toit b/tests/utils.toit index 32e5a40f..69e4054d 100644 --- a/tests/utils.toit +++ b/tests/utils.toit @@ -20,7 +20,7 @@ import uuid show Uuid import artemis.cli as artemis-pkg import artemis.cli.server-config as cli-server-config import artemis.cli.cache as artemis-cache -import artemis.cli.scope as cli-scope +import artemis.shared.scope as cli-scope import artemis.cli.utils show read-json write-json-to-file untar import artemis.shared.server-config import artemis.shared.version as configured-version