diff --git a/Cargo.lock b/Cargo.lock index 7b2d3744fe6..6d1684a8cc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6815,6 +6815,7 @@ dependencies = [ "internal-dns-resolver", "internal-dns-types", "ipnet", + "ipnetwork", "newtype-uuid", "nexus-config", "nexus-db-lookup", diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index 9c64111efcb..40a38d32f96 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -553,7 +553,7 @@ T ENA ID PARENT > blueprint-plan ade5749d-bdf3-4fab-a8ae-00bea01b3a5a INFO skipping noop image source check for all sleds, reason: no target release is currently set WARN cannot issue more MGS-driven updates (no current artifacts) -INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: InstallDataset } }] +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: InstallDataset } }] generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a planning report for blueprint 86db3308-f817-4626-8838-4085949a6a41: * discretionary zone placement waiting for NTP zones on sleds: 89d02b1b-478c-401a-8e28-7a26f74fa41b diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout index a6dc7a3e94e..d3c024dd095 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout @@ -843,7 +843,7 @@ WARN cannot issue more MGS-driven updates (no current artifacts) generated blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 based on parent blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0 planning report for blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2: * discretionary zones placed: - * 1 zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a: external_dns + * external_dns zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a from source install dataset * zone updates waiting on discretionary zones @@ -1340,7 +1340,7 @@ parent: 366b0b68-d80e-4bc1-abd3-dc69837847e0 planning report for blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2: * discretionary zones placed: - * 1 zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a: external_dns + * external_dns zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a from source install dataset * zone updates waiting on discretionary zones diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout index b04dc65912b..19fdfd99bf4 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout @@ -651,7 +651,7 @@ WARN cannot issue more MGS-driven updates (no current artifacts) generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1: * discretionary zones placed: - * 1 zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: internal_dns + * internal_dns zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c from source install dataset * zone updates waiting on discretionary zones diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 75faf3d2360..bc6cea0e732 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -508,6 +508,7 @@ planning report for blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736: - sleds have remove mupdate override set in blueprint: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, d81c6a84-79b8-4958-ae41-ea46c9b19763 - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -779,6 +780,7 @@ planning report for blueprint 626487fa-7139-45ec-8416-902271fc730b: - sleds have remove mupdate override set in blueprint: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, d81c6a84-79b8-4958-ae41-ea46c9b19763 - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -903,6 +905,7 @@ planning report for blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b: - sleds have remove mupdate override set in blueprint: d81c6a84-79b8-4958-ae41-ea46c9b19763 - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -1088,6 +1091,7 @@ planning report for blueprint afb09faf-a586-4483-9289-04d4f1d8ba23: - current target release generation (3) is lower than minimum required by blueprint (4) - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-show latest @@ -1263,6 +1267,7 @@ planning report for blueprint afb09faf-a586-4483-9289-04d4f1d8ba23: - current target release generation (3) is lower than minimum required by blueprint (4) - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -1401,6 +1406,7 @@ planning report for blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700: * zone adds and updates are blocked: - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-show latest @@ -1576,6 +1582,7 @@ planning report for blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700: * zone adds and updates are blocked: - sleds have mupdate override errors: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -1691,7 +1698,9 @@ planning report for blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) +* only placed 0/2 desired nexus zones * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-show latest @@ -1871,7 +1880,9 @@ planning report for blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) +* only placed 0/2 desired nexus zones * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -1988,7 +1999,7 @@ INFO skipping board for MGS-driven update, serial_number: serial0, part_number: INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -INFO some zones not yet up-to-date, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: 0c71b3b2-6ceb-4e8f-b020-b08675e83038 (service), zone_kind: Nexus, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 427ec88f-f467-42fa-9bbb-66a91a36103c (service), zone_kind: InternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ffbf1373f7ee08dddd74c53ed2a94e7c4c572a982d3a9bc94000c6956b700c6a") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 5199c033-4cf9-4ab6-8ae7-566bd7606363 (service), zone_kind: Crucible, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("6f17cf65fb5a5bec5542dd07c03cd0acc01e59130f02c532c8d848ecae810047") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 6444f8a5-6465-4f0b-a549-1993c113569c (service), zone_kind: InternalNtp, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 803bfb63-c246-41db-b0da-d3b87ddfc63d (service), zone_kind: ExternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ccca13ed19b8731f9adaf0d6203b02ea3b9ede4fa426b9fac0a07ce95440046d") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: ba4994a8-23f9-4b1a-a84f-a08d74591389 (service), zone_kind: CruciblePantry, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("21f0ada306859c23917361f2e0b9235806c32607ec689c7e8cf16bb898bc5a02") }, inv_image_source: InstallDataset } }] +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 0c71b3b2-6ceb-4e8f-b020-b08675e83038 (service), zone_kind: Nexus, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 427ec88f-f467-42fa-9bbb-66a91a36103c (service), zone_kind: InternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ffbf1373f7ee08dddd74c53ed2a94e7c4c572a982d3a9bc94000c6956b700c6a") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 5199c033-4cf9-4ab6-8ae7-566bd7606363 (service), zone_kind: Crucible, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("6f17cf65fb5a5bec5542dd07c03cd0acc01e59130f02c532c8d848ecae810047") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 6444f8a5-6465-4f0b-a549-1993c113569c (service), zone_kind: InternalNtp, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 803bfb63-c246-41db-b0da-d3b87ddfc63d (service), zone_kind: ExternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ccca13ed19b8731f9adaf0d6203b02ea3b9ede4fa426b9fac0a07ce95440046d") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: ba4994a8-23f9-4b1a-a84f-a08d74591389 (service), zone_kind: CruciblePantry, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("21f0ada306859c23917361f2e0b9235806c32607ec689c7e8cf16bb898bc5a02") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 (service), zone_kind: Nexus, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("e9b7035f41848a987a798c15ac424cc91dd662b1af0920d58d8aa1ebad7467b6") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: 75b220ba-a0f4-4872-8202-dc7c87f062d0 (service), zone_kind: CruciblePantry, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("3ff26dad96faa8f67251f5de40458b4f809d536bfe8572134da0e42c2fa12674") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: ea5b4030-b52f-44b2-8d70-45f15f987d01 (service), zone_kind: InternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("de30657a72b066b8ef1f56351a0a5d4d7000da0a62c4be9b2e949a107ca8a389") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: f10a4fb9-759f-4a65-b25e-5794ad2d07d8 (service), zone_kind: InternalNtp, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("d76e26198daed69cdae04490d7477f8c842e0dbe37d463eac0d0a8d3fb803095") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: f55647d4-5500-4ad3-893a-df45bd50d622 (service), zone_kind: Crucible, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("866f6a7c2e51c056fb722b5113e80181cc9cd8b712a0d3dbf1edc4ce29e5229e") }, inv_image_source: InstallDataset } }, ZonePropagationIncomplete { zone_id: f6ec9c67-946a-4da3-98d5-581f72ce8bf0 (service), zone_kind: ExternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("2.0.0") }, hash: ArtifactHash("f282c45771429f7bebf71f0cc668521066db57c6bb07fcfccdfb44825d3d930f") }, inv_image_source: InstallDataset } }] generated blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300 based on parent blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1 planning report for blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300: * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts @@ -1996,6 +2007,8 @@ planning report for blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300: * noop converting host phase 2 slot B to Artifact on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * noop converting host phase 2 slot B to Artifact on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 * noop converting host phase 2 slot B to Artifact on sled d81c6a84-79b8-4958-ae41-ea46c9b19763 +* only placed 0/2 desired nexus zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -2233,6 +2246,7 @@ planning report for blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839: - current target release generation (4) is lower than minimum required by blueprint (5) - sleds have remove mupdate override set in blueprint: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -2362,7 +2376,9 @@ planner config: * adding zones despite being blocked, as specified by the `add_zones_with_mupdate_override` planner config option * discretionary zone placement waiting for NTP zones on sleds: c3bc4c6d-fdde-4fc4-8493-89d2a1e5ee6b * missing NTP zone on sled c3bc4c6d-fdde-4fc4-8493-89d2a1e5ee6b +* only placed 0/2 desired nexus zones * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout index ebbcd906c45..6e60b254a19 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout @@ -184,6 +184,7 @@ planning report for blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4: * zone adds and updates are blocked: - sleds have remove mupdate override set in blueprint: d81c6a84-79b8-4958-ae41-ea46c9b19763 * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates @@ -422,6 +423,7 @@ planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1: * zone adds and updates are blocked: - sleds have remove mupdate override set in blueprint: d81c6a84-79b8-4958-ae41-ea46c9b19763 * zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: pending non-nexus zone updates diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 62c38a877cd..3cb84aaf5cf 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -218,13 +218,203 @@ generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent bluepri planning report for blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* discretionary zones placed: + * nexus zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c from source artifact: version 1.0.0 + * nexus zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 from source artifact: version 1.0.0 + * nexus zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763 from source artifact: version 1.0.0 +* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 + MODIFIED SLEDS: + + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 2 -> 3): + + host phase 2 contents: + ------------------------ + slot boot image source + ------------------------ + A current contents + B current contents + + + physical disks: + ------------------------------------------------------------------------------------ + vendor model serial disposition + ------------------------------------------------------------------------------------ + fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service + fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service + fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service + + + datasets: + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 ++ oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_nexus_a67ad53f-d551-40e7-abae-57664779b27b 9edcc144-9dd9-4bf9-a26d-26f265400b0b in service none none off + + + omicron zones: + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 install dataset in service fd00:1122:3344:102::23 + crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 + external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 ++ nexus a67ad53f-d551-40e7-abae-57664779b27b artifact: version 1.0.0 in service fd00:1122:3344:102::29 + + + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 2 -> 3): + + host phase 2 contents: + ------------------------ + slot boot image source + ------------------------ + A current contents + B current contents + + + physical disks: + ------------------------------------------------------------------------------------ + vendor model serial disposition + ------------------------------------------------------------------------------------ + fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service + fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service + fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service + + + datasets: + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 ++ oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_nexus_43cbc3a6-e640-43f5-a9a2-f83eff427870 7f7a2971-de10-4d5a-a814-901adc52bb00 in service none none off + + + omicron zones: + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 058fd5f9-60a8-4e11-9302-15172782e17d install dataset in service fd00:1122:3344:101::27 + crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 + crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 + crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 + external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 + internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset in service fd00:1122:3344:2::1 + internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 + nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 ++ nexus 43cbc3a6-e640-43f5-a9a2-f83eff427870 artifact: version 1.0.0 in service fd00:1122:3344:101::28 + + + sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 2 -> 3): + + host phase 2 contents: + ------------------------ + slot boot image source + ------------------------ + A current contents + B current contents + + + physical disks: + ------------------------------------------------------------------------------------ + vendor model serial disposition + ------------------------------------------------------------------------------------ + fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service + fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service + fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service + + + datasets: + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off + oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off + oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off + oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 + oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 ++ oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_nexus_26fbf986-e560-4449-a351-547d1721b90e e0b86bc5-6a64-432b-bcbe-482e228a4e7d in service none none off + + + omicron zones: + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 install dataset in service fd00:1122:3344:103::26 + crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 + crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 + crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset in service fd00:1122:3344:103::24 + external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 + internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 + internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 + nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 ++ nexus 26fbf986-e560-4449-a351-547d1721b90e artifact: version 1.0.0 in service fd00:1122:3344:103::28 + + COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -249,25 +439,51 @@ to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) +* DNS zone: "control-plane.oxide.internal": ++ name: 26fbf986-e560-4449-a351-547d1721b90e.host (records: 1) ++ AAAA fd00:1122:3344:103::28 ++ name: 43cbc3a6-e640-43f5-a9a2-f83eff427870.host (records: 1) ++ AAAA fd00:1122:3344:101::28 +* name: _nexus._tcp (records: 3 -> 6) +- SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal +- SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal +- SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal ++ SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal ++ SRV port 12221 26fbf986-e560-4449-a351-547d1721b90e.host.control-plane.oxide.internal ++ SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal ++ SRV port 12221 43cbc3a6-e640-43f5-a9a2-f83eff427870.host.control-plane.oxide.internal ++ SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal ++ SRV port 12221 a67ad53f-d551-40e7-abae-57664779b27b.host.control-plane.oxide.internal ++ name: a67ad53f-d551-40e7-abae-57664779b27b.host (records: 1) ++ AAAA fd00:1122:3344:102::29 + unchanged names: 50 (records: 62) external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) +* DNS zone: "oxide.example": +* name: example-silo.sys (records: 3 -> 6) +- A 192.0.2.2 +- A 192.0.2.3 +- A 192.0.2.4 ++ A 192.0.2.2 ++ A 192.0.2.7 ++ A 192.0.2.3 ++ A 192.0.2.6 ++ A 192.0.2.5 ++ A 192.0.2.4 + unchanged names: 4 (records: 6) > # If we generate another plan, there should be no change. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -277,6 +493,7 @@ planning report for blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -300,11 +517,11 @@ to: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -320,13 +537,13 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT bootloader versions: stage0 -> generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -337,6 +554,7 @@ planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1: * 1 pending MGS update: * model0:serial0: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -369,11 +587,11 @@ to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -389,13 +607,13 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: slot b -> 1.0.0, act generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -406,6 +624,7 @@ planning report for blueprint df06bb57-ad42-4431-9206-abff322896c7: * 1 pending MGS update: * model0:serial0: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -438,11 +657,11 @@ to: blueprint df06bb57-ad42-4431-9206-abff322896c7 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -459,13 +678,13 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 68465b8e3f808f475510b525cfd62086d37ddd57688bd854184fdafb2b2198a4, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -476,6 +695,7 @@ planning report for blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -484,7 +704,7 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba MODIFIED SLEDS: - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 2 -> 3): + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 3 -> 4): host phase 2 contents: -------------------------------- @@ -523,6 +743,7 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_nexus_43cbc3a6-e640-43f5-a9a2-f83eff427870 7f7a2971-de10-4d5a-a814-901adc52bb00 in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 @@ -530,17 +751,18 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba omicron zones: - --------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - --------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d install dataset in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 058fd5f9-60a8-4e11-9302-15172782e17d install dataset in service fd00:1122:3344:101::27 + crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 + crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 + crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 + external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 + internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset in service fd00:1122:3344:2::1 + internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 + nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 + nexus 43cbc3a6-e640-43f5-a9a2-f83eff427870 artifact: version 1.0.0 in service fd00:1122:3344:101::28 COCKROACHDB SETTINGS: @@ -569,23 +791,23 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) > # If we generate another plan, there should be no change. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -595,6 +817,7 @@ planning report for blueprint 9034c710-3e57-45f3-99e5-4316145e87ac: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -618,11 +841,11 @@ to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -639,12 +862,12 @@ generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configu > # Planning after only phase 2 has changed should make no changes. We're still > # waiting on phase 1 to change. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -654,6 +877,7 @@ planning report for blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -677,11 +901,11 @@ to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -697,12 +921,12 @@ generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configu > # Planning _still_ shouldn't make any new changes; the OS update as a whole > # isn't done until sled-agent reports it has booted from the new image. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO keeping apparently-impossible MGS-driven update (waiting for recent update to be applied), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -712,6 +936,7 @@ planning report for blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -735,11 +960,11 @@ to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -755,12 +980,12 @@ generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configu > # Planning should now remove the host OS update and plan the next RoT bootloader > # update. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 @@ -772,6 +997,7 @@ planning report for blueprint 626487fa-7139-45ec-8416-902271fc730b: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -804,11 +1030,11 @@ to: blueprint 626487fa-7139-45ec-8416-902271fc730b internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -827,12 +1053,12 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT bootloader versions: stage0_ne generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -843,6 +1069,7 @@ planning report for blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: Version(ArtifactVersion("0.5.0")) }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -875,11 +1102,11 @@ to: blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -893,12 +1120,12 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT bootloader versions: stage0 -> generated inventory collection 04bc9001-0836-4fec-b9cb-9d4760caf8b4 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: Version(ArtifactVersion("0.5.0")), expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -909,6 +1136,7 @@ planning report for blueprint afb09faf-a586-4483-9289-04d4f1d8ba23: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -941,11 +1169,11 @@ to: blueprint afb09faf-a586-4483-9289-04d4f1d8ba23 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -963,12 +1191,12 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 0.5.0 generated inventory collection 08abe624-4b5f-491c-90cb-d74a84e4ba3e from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -979,6 +1207,7 @@ planning report for blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1011,11 +1240,11 @@ to: blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1029,12 +1258,12 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 1.0.0, act generated inventory collection 005f6a30-7f65-4593-9f78-ee68f766f42b from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -1045,6 +1274,7 @@ planning report for blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1077,11 +1307,11 @@ to: blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1098,12 +1328,12 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 generated inventory collection b5263998-e486-4cea-8842-b32bd326fa3a from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 68465b8e3f808f475510b525cfd62086d37ddd57688bd854184fdafb2b2198a4, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -1114,6 +1344,7 @@ planning report for blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1146,11 +1377,11 @@ to: blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1165,12 +1396,12 @@ generated inventory collection 68767302-7fed-4eb1-9611-3dfd807ff0cd from configu > # Planning should remove this update and add an OS update for this sled. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 68465b8e3f808f475510b525cfd62086d37ddd57688bd854184fdafb2b2198a4, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -1181,6 +1412,7 @@ planning report for blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1189,7 +1421,7 @@ to: blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 MODIFIED SLEDS: - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 2 -> 3): + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 3 -> 4): host phase 2 contents: -------------------------------- @@ -1230,6 +1462,7 @@ to: blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_nexus_a67ad53f-d551-40e7-abae-57664779b27b 9edcc144-9dd9-4bf9-a26d-26f265400b0b in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 @@ -1237,18 +1470,19 @@ to: blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 omicron zones: - --------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - --------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 install dataset in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 install dataset in service fd00:1122:3344:102::23 + crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 + external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 + nexus a67ad53f-d551-40e7-abae-57664779b27b artifact: version 1.0.0 in service fd00:1122:3344:102::29 COCKROACHDB SETTINGS: @@ -1277,11 +1511,11 @@ to: blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1300,11 +1534,11 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c host phase 1 details: B -> fffffff generated inventory collection 62898097-2ff1-48d0-8bc1-91b475daa33d from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:102::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -1315,6 +1549,7 @@ planning report for blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1347,11 +1582,11 @@ to: blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1370,11 +1605,11 @@ generated inventory collection 3086f142-62d3-4f77-bda3-674afbb42d0d from configu > # Another planning step should try to update the last sled, starting with the > # RoT bootloader. > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:102::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 @@ -1387,6 +1622,7 @@ planning report for blueprint 9f89efdf-a23e-4137-b7cc-79f4a91cbe1f: * 1 pending MGS update: * model2:serial2: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1419,11 +1655,11 @@ to: blueprint 9f89efdf-a23e-4137-b7cc-79f4a91cbe1f internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1437,11 +1673,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT bootloader versions: stage0 -> generated inventory collection ae5b3bb4-ce21-465f-b18e-857614732d66 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1452,6 +1688,7 @@ planning report for blueprint 9a9e6c32-5a84-4020-a159-33dceff18d35: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1484,11 +1721,11 @@ to: blueprint 9a9e6c32-5a84-4020-a159-33dceff18d35 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1506,11 +1743,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot b -> 1.0.0, act generated inventory collection 34c3258c-b2ab-4da9-9720-41a3a703c3d7 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1521,6 +1758,7 @@ planning report for blueprint 13cfdd24-52ba-4e94-8c83-02e3a48fc746: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("1.0.0")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1553,11 +1791,11 @@ to: blueprint 13cfdd24-52ba-4e94-8c83-02e3a48fc746 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1577,11 +1815,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot b -> 1.1.0, act generated inventory collection 5e106b73-6a14-4955-b8a8-a4f8afed6405 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: B, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("1.0.0")), component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1592,6 +1830,7 @@ planning report for blueprint b82656b0-a9be-433d-83d0-e2bdf371777a: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: Some(B), expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1624,11 +1863,11 @@ to: blueprint b82656b0-a9be-433d-83d0-e2bdf371777a internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1645,11 +1884,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot b -> 1.1.0, act generated inventory collection 36ef425f-a672-4bf4-8d29-14815a84ccad from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: Some(B), expected_persistent_boot_preference: B, expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1660,6 +1899,7 @@ planning report for blueprint 31c84831-be52-4630-bc3f-128d72cd8f22: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: Some(B) }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1692,11 +1932,11 @@ to: blueprint 31c84831-be52-4630-bc3f-128d72cd8f22 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1711,11 +1951,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot b -> 1.0.0, act generated inventory collection 70bea701-e212-4877-8e6c-925f1f73ddd2 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, expected_transient_boot_preference: Some(B), expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: B, expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1726,6 +1966,7 @@ planning report for blueprint 778e3f3a-58b1-4a5e-acff-d23c5d7124c2: * 1 pending MGS update: * model2:serial2: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1758,11 +1999,11 @@ to: blueprint 778e3f3a-58b1-4a5e-acff-d23c5d7124c2 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1776,11 +2017,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 generated inventory collection 8187f847-81c7-4750-88ac-d691937461af from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 68465b8e3f808f475510b525cfd62086d37ddd57688bd854184fdafb2b2198a4, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 @@ -1791,6 +2032,7 @@ planning report for blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6: * 1 pending MGS update: * model2:serial2: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) * zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -1799,7 +2041,7 @@ to: blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 MODIFIED SLEDS: - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 2 -> 3): + sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 3 -> 4): host phase 2 contents: -------------------------------- @@ -1837,6 +2079,7 @@ to: blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_nexus_26fbf986-e560-4449-a351-547d1721b90e e0b86bc5-6a64-432b-bcbe-482e228a4e7d in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 @@ -1845,17 +2088,18 @@ to: blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 omicron zones: - --------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - --------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 install dataset in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset in service fd00:1122:3344:103::24 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 install dataset in service fd00:1122:3344:103::26 + crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 + crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 + crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset in service fd00:1122:3344:103::24 + external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 + internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 + internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 + nexus 26fbf986-e560-4449-a351-547d1721b90e artifact: version 1.0.0 in service fd00:1122:3344:103::28 + nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 COCKROACHDB SETTINGS: @@ -1884,11 +2128,11 @@ to: blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -1906,122 +2150,57 @@ generated inventory collection 45c1c7bb-984a-43f7-bb3f-4a5437ed7b82 from configu > # Do another planning run. This should start updating zones (one at a time). > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, sled_agent_address: [fd00:1122:3344:103::1]:12345, expected_inactive_phase_2_hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: a67ad53f-d551-40e7-abae-57664779b27b (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint e54a0836-53e1-4948-a3af-0b77165289b5 based on parent blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 planning report for blueprint e54a0836-53e1-4948-a3af-0b77165289b5: -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 353b3b65-20f7-48c3-88f7-495bd5d31545 (clickhouse) -* 25 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 to: blueprint e54a0836-53e1-4948-a3af-0b77165289b5 - MODIFIED SLEDS: + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 3 -> 4): + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + PENDING MGS UPDATES: - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 - - - omicron zones: - ------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------- - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 - install dataset in service fd00:1122:3344:102::23 - └─ + artifact: version 1.0.0 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 2 model2 serial2 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89 1.0.0 HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) + Pending MGS-managed updates (all baseboards): + -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 2 model2 serial2 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89 1.0.0 HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -2035,92 +2214,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection ca7f27e8-5949-4ac1-8f32-18ad76d9c217 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 459a45a5-616e-421f-873b-2fb08c36205c based on parent blueprint e54a0836-53e1-4948-a3af-0b77165289b5 planning report for blueprint 459a45a5-616e-421f-873b-2fb08c36205c: -* 1 out-of-date zone expunged: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 62620961-fc4a-481e-968b-f5acbac0dc63 (internal_ntp) -* 24 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e54a0836-53e1-4948-a3af-0b77165289b5 to: blueprint 459a45a5-616e-421f-873b-2fb08c36205c - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 4 -> 5): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset - in service fd00:1122:3344:102::21 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2137,20 +2250,12 @@ to: blueprint 459a45a5-616e-421f-873b-2fb08c36205c internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) -- AAAA fd00:1122:3344:102::21 -* name: _internal-ntp._tcp (records: 3 -> 2) -- SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal -- SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal -- SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -+ SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal -+ SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -2168,94 +2273,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 8a02a1c6-9e86-4dc0-9293-cd17da34f319 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 8, num_already_artifact: 1, num_eligible: 0, num_ineligible: 7 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -INFO some zones not yet up-to-date, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: f83ade6d-9ab1-4679-813b-b9457e039c0b (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint b2295597-5788-482e-acf9-1731ec63fbd2 based on parent blueprint 459a45a5-616e-421f-873b-2fb08c36205c planning report for blueprint b2295597-5788-482e-acf9-1731ec63fbd2: -* waiting for NTP zones to appear in inventory on sleds: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -* sleds getting NTP zones and which have other services already, making them eligible for discretionary zones: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -* missing NTP zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 459a45a5-616e-421f-873b-2fb08c36205c to: blueprint b2295597-5788-482e-acf9-1731ec63fbd2 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 5 -> 6): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset - expunged ⏳ fd00:1122:3344:102::21 - └─ + expunged ✓ -+ internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2272,20 +2309,12 @@ to: blueprint b2295597-5788-482e-acf9-1731ec63fbd2 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _internal-ntp._tcp (records: 2 -> 3) -- SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal -- SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -+ SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal -+ SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ name: f83ade6d-9ab1-4679-813b-b9457e039c0b.host (records: 1) -+ AAAA fd00:1122:3344:102::29 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -2297,95 +2326,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection c1adcd42-121f-4580-bfb9-d8a9937ca9e1 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 2, num_eligible: 0, num_ineligible: 7 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 6fad8fd4-e825-433f-b76d-495484e068ce based on parent blueprint b2295597-5788-482e-acf9-1731ec63fbd2 planning report for blueprint 6fad8fd4-e825-433f-b76d-495484e068ce: -* 1 out-of-date zone expunged: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 6c3ae381-04f7-41ea-b0ac-74db387dbc3a (external_dns) -* 23 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint b2295597-5788-482e-acf9-1731ec63fbd2 to: blueprint 6fad8fd4-e825-433f-b76d-495484e068ce - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 6 -> 7): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c - in service none none off - └─ + expunged -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset - in service fd00:1122:3344:102::24 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2402,34 +2362,12 @@ to: blueprint 6fad8fd4-e825-433f-b76d-495484e068ce internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) -- AAAA fd00:1122:3344:102::24 -* name: _external-dns._tcp (records: 3 -> 2) -- SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal -- SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -+ SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 3 -> 2) -- NS ns1.oxide.example -- NS ns2.oxide.example -- NS ns3.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -* name: ns1 (records: 1 -> 1) -- A 198.51.100.1 -+ A 198.51.100.2 -* name: ns2 (records: 1 -> 1) -- A 198.51.100.2 -+ A 198.51.100.3 -- name: ns3 (records: 1) -- A 198.51.100.3 - unchanged names: 1 (records: 3) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -2441,96 +2379,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 94b231f9-80a3-48a9-8d25-70f9b42b64ca from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 24b6e243-100c-428d-8ea6-35b504226f55 based on parent blueprint 6fad8fd4-e825-433f-b76d-495484e068ce planning report for blueprint 24b6e243-100c-428d-8ea6-35b504226f55: -* discretionary zones placed: - * 1 zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: external_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 6fad8fd4-e825-433f-b76d-495484e068ce to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 7 -> 8): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset - expunged ⏳ fd00:1122:3344:102::24 - └─ + expunged ✓ -+ external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2547,34 +2415,12 @@ to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _external-dns._tcp (records: 2 -> 3) -- SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -+ SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ name: ba87399e-e9b7-4ee4-8cb7-0032822630e9.host (records: 1) -+ AAAA fd00:1122:3344:102::2a - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 2 -> 3) -- NS ns1.oxide.example -- NS ns2.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -+ NS ns3.oxide.example -* name: ns1 (records: 1 -> 1) -- A 198.51.100.2 -+ A 198.51.100.1 -* name: ns2 (records: 1 -> 1) -- A 198.51.100.3 -+ A 198.51.100.2 -+ name: ns3 (records: 1) -+ A 198.51.100.3 - unchanged names: 1 (records: 3) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -2586,118 +2432,48 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 756aecb6-8353-46ad-a6c4-10ad0f2bbb7f from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 3, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce based on parent blueprint 24b6e243-100c-428d-8ea6-35b504226f55 planning report for blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce: -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 86a22a56-0168-453d-9df1-cb2a7c64b5d3 (crucible) -* 22 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce - MODIFIED SLEDS: + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 8 -> 9): + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 - install dataset in service fd00:1122:3344:102::28 - └─ + artifact: version 1.0.0 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -2709,98 +2485,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 84152e52-8c2e-46ab-880e-4cc2a1fb9dcb from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 4, num_eligible: 0, num_ineligible: 5 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 based on parent blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce planning report for blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312: -* 1 out-of-date zone expunged: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 99e2f30b-3174-40bf-a78a-90da8abba8ca (internal_dns) -* 21 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce to: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 9 -> 10): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da - in service none none off - └─ + expunged -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset - in service fd00:1122:3344:1::1 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2817,34 +2521,12 @@ to: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) -- AAAA fd00:1122:3344:1::1 -* name: @ (records: 3 -> 2) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -- NS ns3.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -* name: _nameservice._tcp (records: 3 -> 2) -- SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal -- SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal -+ SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -* name: ns1 (records: 1 -> 1) -- AAAA fd00:1122:3344:1::1 -+ AAAA fd00:1122:3344:2::1 -* name: ns2 (records: 1 -> 1) -- AAAA fd00:1122:3344:2::1 -+ AAAA fd00:1122:3344:3::1 -- name: ns3 (records: 1) -- AAAA fd00:1122:3344:3::1 - unchanged names: 45 (records: 55) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -2856,99 +2538,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection bcfc7436-77de-47e4-8158-daad15a54da2 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 8, num_already_artifact: 4, num_eligible: 0, num_ineligible: 4 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d based on parent blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 planning report for blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d: -* discretionary zones placed: - * 1 zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: internal_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 to: blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 10 -> 11): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset - expunged ⏳ fd00:1122:3344:1::1 - └─ + expunged ✓ -+ internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -2965,34 +2574,12 @@ to: blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: @ (records: 2 -> 3) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -+ NS ns3.control-plane.oxide.internal -* name: _nameservice._tcp (records: 2 -> 3) -- SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ name: c821c39d-2b2c-4c55-8874-ac12315ba1e4.host (records: 1) -+ AAAA fd00:1122:3344:1::1 -* name: ns1 (records: 1 -> 1) -- AAAA fd00:1122:3344:2::1 -+ AAAA fd00:1122:3344:1::1 -* name: ns2 (records: 1 -> 1) -- AAAA fd00:1122:3344:3::1 -+ AAAA fd00:1122:3344:2::1 -+ name: ns3 (records: 1) -+ AAAA fd00:1122:3344:3::1 - unchanged names: 45 (records: 55) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3004,100 +2591,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 6dbdc88a-4828-480e-b41d-8946f41a3134 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 5, num_eligible: 0, num_ineligible: 4 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint e2125c83-b255-45c9-bc9b-802cff09a812 based on parent blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d planning report for blueprint e2125c83-b255-45c9-bc9b-802cff09a812: -* 1 out-of-date zone expunged: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone ad6a3a03-8d0f-4504-99a4-cbf73d69b973 (crucible_pantry) -* 20 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d to: blueprint e2125c83-b255-45c9-bc9b-802cff09a812 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 11 -> 12): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -* oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset - in service fd00:1122:3344:102::25 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3114,20 +2627,12 @@ to: blueprint e2125c83-b255-45c9-bc9b-802cff09a812 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _crucible-pantry._tcp (records: 3 -> 2) -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -- SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal -- SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal -- name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) -- AAAA fd00:1122:3344:102::25 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3139,101 +2644,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection eb500068-cd91-484b-a532-51081571ecbe from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 8, num_already_artifact: 5, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 based on parent blueprint e2125c83-b255-45c9-bc9b-802cff09a812 planning report for blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3: -* discretionary zones placed: - * 1 zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: crucible_pantry -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e2125c83-b255-45c9-bc9b-802cff09a812 to: blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 12 -> 13): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_698d1d82-0620-4978-93ac-0ba5d40f3da9 dfe5586b-e4a8-4b98-ad72-eabc34988177 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset - expunged ⏳ fd00:1122:3344:102::25 - └─ + expunged ✓ -+ crucible_pantry 698d1d82-0620-4978-93ac-0ba5d40f3da9 artifact: version 1.0.0 in service fd00:1122:3344:102::2b - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3250,20 +2680,12 @@ to: blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 698d1d82-0620-4978-93ac-0ba5d40f3da9.host (records: 1) -+ AAAA fd00:1122:3344:102::2b -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -- SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3275,101 +2697,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 4492baf6-5638-4c1f-bba2-608163519022 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 6, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c based on parent blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 planning report for blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c: -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone bd354eef-d8a6-4165-9124-283fb5e46d77 (crucible) -* 19 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 to: blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 13 -> 14): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_698d1d82-0620-4978-93ac-0ba5d40f3da9 dfe5586b-e4a8-4b98-ad72-eabc34988177 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry 698d1d82-0620-4978-93ac-0ba5d40f3da9 artifact: version 1.0.0 in service fd00:1122:3344:102::2b - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset expunged ✓ fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible bd354eef-d8a6-4165-9124-283fb5e46d77 - install dataset in service fd00:1122:3344:102::26 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3387,11 +2734,11 @@ to: blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3403,101 +2750,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 73f58d4d-6be7-4007-811c-0e578279410e from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 7, num_eligible: 0, num_ineligible: 2 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 based on parent blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c planning report for blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1: -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone e2fdefe7-95b2-4fd2-ae37-56929a06d58c (crucible) -* 18 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c to: blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 14 -> 15): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_698d1d82-0620-4978-93ac-0ba5d40f3da9 dfe5586b-e4a8-4b98-ad72-eabc34988177 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 1.0.0 in service fd00:1122:3344:102::26 - crucible_pantry 698d1d82-0620-4978-93ac-0ba5d40f3da9 artifact: version 1.0.0 in service fd00:1122:3344:102::2b - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset expunged ✓ fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c - install dataset in service fd00:1122:3344:102::27 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3515,11 +2787,11 @@ to: blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3534,88 +2806,26 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest bluepri generated inventory collection 74448e29-ef07-4d7f-9d31-39079eba8296 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 43cbc3a6-e640-43f5-a9a2-f83eff427870 (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }, ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 based on parent blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 planning report for blueprint e2deb7c0-2262-49fe-855f-4250c22afb36: -* 1 out-of-date zone updated in-place: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 058fd5f9-60a8-4e11-9302-15172782e17d (crucible) -* 17 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 to: blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 3 -> 4): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 - - - omicron zones: - ------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------- - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible 058fd5f9-60a8-4e11-9302-15172782e17d - install dataset in service fd00:1122:3344:101::27 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3633,11 +2843,11 @@ to: blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3650,90 +2860,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection a815c282-5564-4cea-b667-a7a5295fc2c1 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 1, num_eligible: 0, num_ineligible: 7 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf based on parent blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 planning report for blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf: -* 1 out-of-date zone expunged: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 427ec88f-f467-42fa-9bbb-66a91a36103c (internal_dns) -* 16 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 to: blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 4 -> 5): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 - in service none none off - └─ + expunged -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset - in service fd00:1122:3344:2::1 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3750,31 +2896,12 @@ to: blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) -- AAAA fd00:1122:3344:2::1 -* name: @ (records: 3 -> 2) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -- NS ns3.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -* name: _nameservice._tcp (records: 3 -> 2) -- SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal -- SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -* name: ns2 (records: 1 -> 1) -- AAAA fd00:1122:3344:2::1 -+ AAAA fd00:1122:3344:3::1 -- name: ns3 (records: 1) -- AAAA fd00:1122:3344:3::1 - unchanged names: 46 (records: 56) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3786,91 +2913,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 18ca4fd2-190d-4ac5-b0f3-14a384bcd254 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 7, num_already_artifact: 1, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d based on parent blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf planning report for blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d: -* discretionary zones placed: - * 1 zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: internal_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf to: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 5 -> 6): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset - expunged ⏳ fd00:1122:3344:2::1 - └─ + expunged ✓ -+ internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -3887,31 +2949,12 @@ to: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 71f71743-8c73-43c6-b080-427ec28ef4c9.host (records: 1) -+ AAAA fd00:1122:3344:2::1 -* name: @ (records: 2 -> 3) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -+ NS ns3.control-plane.oxide.internal -* name: _nameservice._tcp (records: 2 -> 3) -- SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -* name: ns2 (records: 1 -> 1) -- AAAA fd00:1122:3344:3::1 -+ AAAA fd00:1122:3344:2::1 -+ name: ns3 (records: 1) -+ AAAA fd00:1122:3344:3::1 - unchanged names: 46 (records: 56) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -3923,91 +2966,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection b460bcc7-664d-4dff-92fb-f250def5537c from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 60b55d33-5fec-4277-9864-935197eaead7 based on parent blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d planning report for blueprint 60b55d33-5fec-4277-9864-935197eaead7: -* 1 out-of-date zone updated in-place: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 5199c033-4cf9-4ab6-8ae7-566bd7606363 (crucible) -* 15 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d to: blueprint 60b55d33-5fec-4277-9864-935197eaead7 - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 6 -> 7): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 - install dataset in service fd00:1122:3344:101::25 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4025,11 +3003,11 @@ to: blueprint 60b55d33-5fec-4277-9864-935197eaead7 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4041,122 +3019,48 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection f8212fb6-115e-4568-a05c-b241e2e8ffb9 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 3, num_eligible: 0, num_ineligible: 5 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 based on parent blueprint 60b55d33-5fec-4277-9864-935197eaead7 planning report for blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544: -* 1 out-of-date zone expunged: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 6444f8a5-6465-4f0b-a549-1993c113569c (internal_ntp) -* 14 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 60b55d33-5fec-4277-9864-935197eaead7 to: blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 - MODIFIED SLEDS: + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 7 -> 8): + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset - in service fd00:1122:3344:101::21 - └─ + expunged ⏳ - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) -- AAAA fd00:1122:3344:101::21 -* name: _internal-ntp._tcp (records: 3 -> 2) -- SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal -- SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -- SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4168,94 +3072,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection f7602eed-bc12-42db-8eec-6f98a05d9796 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 7, num_already_artifact: 3, num_eligible: 0, num_ineligible: 4 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -INFO some zones not yet up-to-date, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: cc6fdaf4-0195-4cef-950d-7bacd7059787 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 based on parent blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 planning report for blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111: -* waiting for NTP zones to appear in inventory on sleds: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -* sleds getting NTP zones and which have other services already, making them eligible for discretionary zones: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -* missing NTP zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 to: blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 8 -> 9): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset - expunged ⏳ fd00:1122:3344:101::21 - └─ + expunged ✓ -+ internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4272,20 +3108,12 @@ to: blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _internal-ntp._tcp (records: 2 -> 3) -- SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -- SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -+ SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ name: cc6fdaf4-0195-4cef-950d-7bacd7059787.host (records: 1) -+ AAAA fd00:1122:3344:101::28 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4297,95 +3125,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection af824d9a-296d-4a2f-b704-c985c7470a1a from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 4, num_eligible: 0, num_ineligible: 4 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a based on parent blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 planning report for blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a: -* 1 out-of-date zone expunged: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 803bfb63-c246-41db-b0da-d3b87ddfc63d (external_dns) -* 13 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 to: blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 9 -> 10): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e - in service none none off - └─ + expunged -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset - in service fd00:1122:3344:101::23 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4402,31 +3161,12 @@ to: blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) -- AAAA fd00:1122:3344:101::23 -* name: _external-dns._tcp (records: 3 -> 2) -- SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -- SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -+ SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 3 -> 2) -- NS ns1.oxide.example -- NS ns2.oxide.example -- NS ns3.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -* name: ns2 (records: 1 -> 1) -- A 198.51.100.2 -+ A 198.51.100.3 -- name: ns3 (records: 1) -- A 198.51.100.3 - unchanged names: 2 (records: 4) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -4438,96 +3178,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 2d5a41c5-bf7b-464c-a4b7-b14ab35982c4 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 7, num_already_artifact: 4, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 based on parent blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a planning report for blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110: -* discretionary zones placed: - * 1 zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: external_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a to: blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 10 -> 11): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset - expunged ⏳ fd00:1122:3344:101::23 - └─ + expunged ✓ -+ external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4544,31 +3214,12 @@ to: blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _external-dns._tcp (records: 2 -> 3) -- SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -+ SRV port 5353 e14f91b0-0c41-48a0-919d-e5078d2b89b0.host.control-plane.oxide.internal -+ SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ name: e14f91b0-0c41-48a0-919d-e5078d2b89b0.host (records: 1) -+ AAAA fd00:1122:3344:101::29 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 2 -> 3) -- NS ns1.oxide.example -- NS ns2.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -+ NS ns3.oxide.example -* name: ns2 (records: 1 -> 1) -- A 198.51.100.3 -+ A 198.51.100.2 -+ name: ns3 (records: 1) -+ A 198.51.100.3 - unchanged names: 2 (records: 4) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -4580,97 +3231,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 94c793da-83b2-4fdf-b085-d7ef476bf204 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 5, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 based on parent blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 planning report for blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482: -* 1 out-of-date zone expunged: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone ba4994a8-23f9-4b1a-a84f-a08d74591389 (crucible_pantry) -* 12 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 to: blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 11 -> 12): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset expunged ✓ fd00:1122:3344:101::23 - external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset - in service fd00:1122:3344:101::24 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4687,20 +3267,12 @@ to: blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _crucible-pantry._tcp (records: 3 -> 2) -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -- SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -- name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) -- AAAA fd00:1122:3344:101::24 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4712,98 +3284,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 388a8c73-4ec0-4a23-9f82-225e652d8f37 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 7, num_already_artifact: 5, num_eligible: 0, num_ineligible: 2 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec based on parent blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 planning report for blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec: -* discretionary zones placed: - * 1 zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: crucible_pantry -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 to: blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 12 -> 13): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_26bdd109-c842-43a9-95cb-15aba9b0832b 16a8c618-d062-4bde-8ca4-301b5f14ccf2 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset expunged ✓ fd00:1122:3344:101::23 - external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset - expunged ⏳ fd00:1122:3344:101::24 - └─ + expunged ✓ -+ crucible_pantry 26bdd109-c842-43a9-95cb-15aba9b0832b artifact: version 1.0.0 in service fd00:1122:3344:101::2a - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -4820,20 +3320,12 @@ to: blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 26bdd109-c842-43a9-95cb-15aba9b0832b.host (records: 1) -+ AAAA fd00:1122:3344:101::2a -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4845,120 +3337,48 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 2d608b8f-bf88-4707-ac27-6be62f3d5146 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 6, num_eligible: 0, num_ineligible: 2 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint a2c6496d-98fc-444d-aa36-99508aa72367 based on parent blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec planning report for blueprint a2c6496d-98fc-444d-aa36-99508aa72367: -* 1 out-of-date zone updated in-place: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone dfac80b4-a887-430a-ae87-a4e065dba787 (crucible) -* 11 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec to: blueprint a2c6496d-98fc-444d-aa36-99508aa72367 - MODIFIED SLEDS: + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 13 -> 14): + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_26bdd109-c842-43a9-95cb-15aba9b0832b 16a8c618-d062-4bde-8ca4-301b5f14ccf2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible_pantry 26bdd109-c842-43a9-95cb-15aba9b0832b artifact: version 1.0.0 in service fd00:1122:3344:101::2a - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset expunged ✓ fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset expunged ✓ fd00:1122:3344:101::23 - external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible dfac80b4-a887-430a-ae87-a4e065dba787 - install dataset in service fd00:1122:3344:101::26 - └─ + artifact: version 1.0.0 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -4973,88 +3393,26 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest bluepri generated inventory collection 21e24074-fdd0-438e-a4e7-11665b7071bb from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 26fbf986-e560-4449-a351-547d1721b90e (service), zone_kind: Nexus, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") } } }] generated blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 based on parent blueprint a2c6496d-98fc-444d-aa36-99508aa72367 planning report for blueprint 6ed56354-5941-40d1-a06c-b0e940701d52: -* 1 out-of-date zone updated in-place: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone 694bd14f-cb24-4be4-bb19-876e79cda2c8 (crucible) -* 10 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint a2c6496d-98fc-444d-aa36-99508aa72367 to: blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 3 -> 4): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 - - - omicron zones: - ------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------- - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset in service fd00:1122:3344:103::24 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 - install dataset in service fd00:1122:3344:103::26 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5072,11 +3430,11 @@ to: blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5089,11 +3447,11 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection ed8ea2c4-4271-407e-9c84-54129418d171 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 1, num_eligible: 0, num_ineligible: 9 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 1, num_eligible: 0, num_ineligible: 7 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 @@ -5101,9 +3459,10 @@ INFO skipping board for MGS-driven update, serial_number: serial2, part_number: INFO ran out of boards for MGS-driven update generated blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 based on parent blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 planning report for blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2: -* 1 out-of-date zone expunged: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone 75b220ba-a0f4-4872-8202-dc7c87f062d0 (crucible_pantry) -* 9 remaining out-of-date zones +* 1 out-of-date zone updated in-place: + * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 353b3b65-20f7-48c3-88f7-495bd5d31545 (clickhouse) +* 25 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest @@ -5112,7 +3471,7 @@ to: blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 MODIFIED SLEDS: - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 4 -> 5): + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 4 -> 5): host phase 2 contents: ------------------------------ @@ -5126,50 +3485,54 @@ to: blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 ------------------------------------------------------------------------------------ vendor model serial disposition ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service + fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service + fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service + fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be - in service none none off - └─ + expunged + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_nexus_a67ad53f-d551-40e7-abae-57664779b27b 9edcc144-9dd9-4bf9-a26d-26f265400b0b in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 + oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset - in service fd00:1122:3344:103::24 - └─ + expunged ⏳ + ------------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ------------------------------------------------------------------------------------------------------------------------- + crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 + external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 + nexus a67ad53f-d551-40e7-abae-57664779b27b artifact: version 1.0.0 in service fd00:1122:3344:102::29 +* clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 - install dataset in service fd00:1122:3344:102::23 + └─ + artifact: version 1.0.0 COCKROACHDB SETTINGS: @@ -5188,20 +3551,12 @@ to: blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 internal DNS: -* DNS zone: "control-plane.oxide.internal": -- name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) -- AAAA fd00:1122:3344:103::24 -* name: _crucible-pantry._tcp (records: 3 -> 2) -- SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5213,90 +3568,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection 336dbc73-f973-4962-a210-3c9d424bd6a3 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 1, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 based on parent blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 planning report for blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152: -* discretionary zones placed: - * 1 zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: crucible_pantry -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 to: blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 5 -> 6): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset - expunged ⏳ fd00:1122:3344:103::24 - └─ + expunged ✓ -+ crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5313,20 +3604,12 @@ to: blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 c88fcd7d-9509-470e-8c4f-3e6f09104cdc.host.control-plane.oxide.internal -+ name: c88fcd7d-9509-470e-8c4f-3e6f09104cdc.host (records: 1) -+ AAAA fd00:1122:3344:103::28 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5338,90 +3621,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection 897721fc-b087-41be-a566-809d59c8aeea from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 based on parent blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 planning report for blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88: -* 1 out-of-date zone updated in-place: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone 7c252b64-c5af-4ec1-989e-9a03f3b0f111 (crucible) -* 8 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 to: blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 6 -> 7): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 - install dataset in service fd00:1122:3344:103::27 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5439,11 +3658,11 @@ to: blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5455,92 +3674,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection 5d0b9686-48df-4642-a39c-e2dea04d5330 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 3, num_eligible: 0, num_ineligible: 5 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 based on parent blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 planning report for blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4: -* 1 out-of-date zone expunged: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone ea5b4030-b52f-44b2-8d70-45f15f987d01 (internal_dns) -* 7 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 to: blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 7 -> 8): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 - in service none none off - └─ + expunged -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset - in service fd00:1122:3344:3::1 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5553,129 +3706,46 @@ to: blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 OXIMETER SETTINGS: generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: @ (records: 3 -> 2) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -- NS ns3.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -* name: _nameservice._tcp (records: 3 -> 2) -- SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -- SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -- name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) -- AAAA fd00:1122:3344:3::1 -- name: ns3 (records: 1) -- AAAA fd00:1122:3344:3::1 - unchanged names: 47 (records: 57) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest blueprint (7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4) - -> inventory-generate -generated inventory collection 90f0f757-fd33-4744-abee-36616a645b87 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 3, num_eligible: 0, num_ineligible: 4 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 59630e63-c953-4807-9e84-9e750a79f68e based on parent blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 -planning report for blueprint 59630e63-c953-4807-9e84-9e750a79f68e: -* discretionary zones placed: - * 1 zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: internal_dns -* zone updates waiting on discretionary zones - - -> blueprint-diff latest -from: blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 -to: blueprint 59630e63-c953-4807-9e84-9e750a79f68e - - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 8 -> 9): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off + read from:: SingleNode (unchanged) - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset - expunged ⏳ fd00:1122:3344:3::1 - └─ + expunged ✓ -+ internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) + +external DNS: + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) + + + + +> sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest +set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest blueprint (7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4) + +> inventory-generate +generated inventory collection 90f0f757-fd33-4744-abee-36616a645b87 from configured sleds +> blueprint-plan latest latest +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 +INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 +INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 +INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a +INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] +generated blueprint 59630e63-c953-4807-9e84-9e750a79f68e based on parent blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 +planning report for blueprint 59630e63-c953-4807-9e84-9e750a79f68e: +* waiting to update top-level nexus_generation: pending non-nexus zone updates + + +> blueprint-diff latest +from: blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 +to: blueprint 59630e63-c953-4807-9e84-9e750a79f68e COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -5693,28 +3763,12 @@ to: blueprint 59630e63-c953-4807-9e84-9e750a79f68e internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 7fbd0103-d7f8-48a5-b95e-29bf812cac1f.host (records: 1) -+ AAAA fd00:1122:3344:3::1 -* name: @ (records: 2 -> 3) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -+ NS ns3.control-plane.oxide.internal -* name: _nameservice._tcp (records: 2 -> 3) -- SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -- SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -+ SRV port 5353 7fbd0103-d7f8-48a5-b95e-29bf812cac1f.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ name: ns3 (records: 1) -+ AAAA fd00:1122:3344:3::1 - unchanged names: 47 (records: 57) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5726,94 +3780,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection ee9bc64a-70f7-4d81-b39d-a709754ce118 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 4, num_eligible: 0, num_ineligible: 4 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 based on parent blueprint 59630e63-c953-4807-9e84-9e750a79f68e planning report for blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194: -* 1 out-of-date zone expunged: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone f10a4fb9-759f-4a65-b25e-5794ad2d07d8 (internal_ntp) -* 6 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 59630e63-c953-4807-9e84-9e750a79f68e to: blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 9 -> 10): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset - in service fd00:1122:3344:103::21 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5830,20 +3816,12 @@ to: blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _internal-ntp._tcp (records: 3 -> 2) -- SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -- SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal -- SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -- name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) -- AAAA fd00:1122:3344:103::21 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5855,96 +3833,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection 3dc9d8c8-8f50-4d6e-9396-97058d1d2722 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 4, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -INFO some zones not yet up-to-date, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: d5fd048a-8786-42d3-938e-820eae95d7f4 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 90650737-8142-47a6-9a48-a10efc487e57 based on parent blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 planning report for blueprint 90650737-8142-47a6-9a48-a10efc487e57: -* waiting for NTP zones to appear in inventory on sleds: d81c6a84-79b8-4958-ae41-ea46c9b19763 -* sleds getting NTP zones and which have other services already, making them eligible for discretionary zones: d81c6a84-79b8-4958-ae41-ea46c9b19763 -* missing NTP zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763 +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 to: blueprint 90650737-8142-47a6-9a48-a10efc487e57 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 10 -> 11): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset - expunged ⏳ fd00:1122:3344:103::21 - └─ + expunged ✓ -+ internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -5961,20 +3869,12 @@ to: blueprint 90650737-8142-47a6-9a48-a10efc487e57 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _internal-ntp._tcp (records: 2 -> 3) -- SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -- SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -+ SRV port 123 d5fd048a-8786-42d3-938e-820eae95d7f4.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ name: d5fd048a-8786-42d3-938e-820eae95d7f4.host (records: 1) -+ AAAA fd00:1122:3344:103::29 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -5986,95 +3886,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection a4dab274-0fff-47fa-bc22-b98d11ec54d2 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 5, num_eligible: 0, num_ineligible: 3 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa based on parent blueprint 90650737-8142-47a6-9a48-a10efc487e57 planning report for blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa: -* 1 out-of-date zone updated in-place: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone f55647d4-5500-4ad3-893a-df45bd50d622 (crucible) -* 5 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 90650737-8142-47a6-9a48-a10efc487e57 to: blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 11 -> 12): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset expunged ✓ fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible f55647d4-5500-4ad3-893a-df45bd50d622 - install dataset in service fd00:1122:3344:103::25 - └─ + artifact: version 1.0.0 - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -6092,11 +3923,11 @@ to: blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 54 (records: 71) external DNS: DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) + unchanged names: 5 (records: 12) @@ -6108,97 +3939,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection d483be68-4bf3-4133-aed1-661cba8e1194 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 6, num_eligible: 0, num_ineligible: 2 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece based on parent blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa planning report for blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece: -* 1 out-of-date zone expunged: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone f6ec9c67-946a-4da3-98d5-581f72ce8bf0 (external_dns) -* 4 remaining out-of-date zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa to: blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 12 -> 13): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 - in service none none off - └─ + expunged -* oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 artifact: version 1.0.0 in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset expunged ✓ fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset - in service fd00:1122:3344:103::23 - └─ + expunged ⏳ - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -6215,28 +3975,12 @@ to: blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _external-dns._tcp (records: 3 -> 2) -- SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -- SRV port 5353 e14f91b0-0c41-48a0-919d-e5078d2b89b0.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -+ SRV port 5353 e14f91b0-0c41-48a0-919d-e5078d2b89b0.host.control-plane.oxide.internal -- name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) -- AAAA fd00:1122:3344:103::23 - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 3 -> 2) -- NS ns1.oxide.example -- NS ns2.oxide.example -- NS ns3.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -- name: ns3 (records: 1) -- A 198.51.100.3 - unchanged names: 3 (records: 5) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -6248,98 +3992,26 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest bluepri generated inventory collection 74b742c1-01da-4461-a011-785e2e11a5b2 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 10, num_already_artifact: 2, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 6, num_eligible: 0, num_ineligible: 1 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 1, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update +INFO some zones not yet up-to-date, zones_currently_updating: [ZonePropagationIncomplete { zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545 (service), zone_kind: Clickhouse, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659") }, inv_image_source: InstallDataset } }] generated blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 based on parent blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece planning report for blueprint 810ea95a-4730-43dd-867e-1984aeb9d873: -* discretionary zones placed: - * 1 zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: external_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates > blueprint-diff latest from: blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece to: blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 13 -> 14): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 850d04b8-c706-46e9-b405-a7a800b744b5 in service none none off -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01 b24bee8e-82a0-4b4d-a57c-77a1010f3e38 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 artifact: version 1.0.0 in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset expunged ✓ fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset - expunged ⏳ fd00:1122:3344:103::23 - └─ + expunged ✓ -+ external_dns f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01 artifact: version 1.0.0 in service fd00:1122:3344:103::2a - - COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -6356,28 +4028,12 @@ to: blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _external-dns._tcp (records: 2 -> 3) -- SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -- SRV port 5353 e14f91b0-0c41-48a0-919d-e5078d2b89b0.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal -+ SRV port 5353 e14f91b0-0c41-48a0-919d-e5078d2b89b0.host.control-plane.oxide.internal -+ SRV port 5353 f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01.host.control-plane.oxide.internal -+ name: f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01.host (records: 1) -+ AAAA fd00:1122:3344:103::2a - unchanged names: 49 (records: 61) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 54 (records: 71) external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 2 -> 3) -- NS ns1.oxide.example -- NS ns2.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -+ NS ns3.oxide.example -+ name: ns3 (records: 1) -+ A 198.51.100.3 - unchanged names: 3 (records: 5) + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 12) @@ -6389,7 +4045,7 @@ external DNS: blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 parent: e8b088a8-7da0-480b-a2dc-75ffef068ece - sled: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 15) + sled: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 5) host phase 2 contents: ------------------------------ @@ -6416,10 +4072,8 @@ parent: e8b088a8-7da0-480b-a2dc-75ffef068ece oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off @@ -6427,41 +4081,35 @@ parent: e8b088a8-7da0-480b-a2dc-75ffef068ece oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_698d1d82-0620-4978-93ac-0ba5d40f3da9 dfe5586b-e4a8-4b98-ad72-eabc34988177 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off + oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_nexus_a67ad53f-d551-40e7-abae-57664779b27b 9edcc144-9dd9-4bf9-a26d-26f265400b0b in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 omicron zones: - ------------------------------------------------------------------------------------------------------------------------ - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------ - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 1.0.0 in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c artifact: version 1.0.0 in service fd00:1122:3344:102::27 - crucible_pantry 698d1d82-0620-4978-93ac-0ba5d40f3da9 artifact: version 1.0.0 in service fd00:1122:3344:102::2b - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset expunged ✓ fd00:1122:3344:102::25 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 - - - - sled: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 14) + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 + crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 + external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset in service fd00:1122:3344:102::24 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset in service fd00:1122:3344:102::21 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 + nexus a67ad53f-d551-40e7-abae-57664779b27b artifact: version 1.0.0 in service fd00:1122:3344:102::29 + + + + sled: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 4) host phase 2 contents: ------------------------------ @@ -6487,50 +4135,42 @@ parent: e8b088a8-7da0-480b-a2dc-75ffef068ece oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_26bdd109-c842-43a9-95cb-15aba9b0832b 16a8c618-d062-4bde-8ca4-301b5f14ccf2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_nexus_43cbc3a6-e640-43f5-a9a2-f83eff427870 7f7a2971-de10-4d5a-a814-901adc52bb00 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 omicron zones: - ------------------------------------------------------------------------------------------------------------------------ - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------ - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 artifact: version 1.0.0 in service fd00:1122:3344:101::26 - crucible_pantry 26bdd109-c842-43a9-95cb-15aba9b0832b artifact: version 1.0.0 in service fd00:1122:3344:101::2a - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset expunged ✓ fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset expunged ✓ fd00:1122:3344:101::23 - external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 - - - - sled: d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 14) + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 058fd5f9-60a8-4e11-9302-15172782e17d install dataset in service fd00:1122:3344:101::27 + crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 + crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 + crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 + external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 + internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset in service fd00:1122:3344:2::1 + internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 + nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 + nexus 43cbc3a6-e640-43f5-a9a2-f83eff427870 artifact: version 1.0.0 in service fd00:1122:3344:101::28 + + + + sled: d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 4) host phase 2 contents: ------------------------------ @@ -6556,46 +4196,38 @@ parent: e8b088a8-7da0-480b-a2dc-75ffef068ece oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 850d04b8-c706-46e9-b405-a7a800b744b5 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01 b24bee8e-82a0-4b4d-a57c-77a1010f3e38 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off + oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_nexus_26fbf986-e560-4449-a351-547d1721b90e e0b86bc5-6a64-432b-bcbe-482e228a4e7d in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off + oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 omicron zones: - ------------------------------------------------------------------------------------------------------------------------ - zone type zone id image source disposition underlay IP - ------------------------------------------------------------------------------------------------------------------------ - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 artifact: version 1.0.0 in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f4d7ec7b-5e5c-4c90-97f2-2ac9d4588a01 artifact: version 1.0.0 in service fd00:1122:3344:103::2a - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset expunged ✓ fd00:1122:3344:103::23 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset expunged ✓ fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 + ----------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ----------------------------------------------------------------------------------------------------------------------- + crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 install dataset in service fd00:1122:3344:103::26 + crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 + crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 + crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset in service fd00:1122:3344:103::24 + external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 + internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 + internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 + nexus 26fbf986-e560-4449-a351-547d1721b90e artifact: version 1.0.0 in service fd00:1122:3344:103::28 + nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 COCKROACHDB SETTINGS: @@ -6618,9 +4250,7 @@ parent: e8b088a8-7da0-480b-a2dc-75ffef068ece PENDING MGS-MANAGED UPDATES: 0 planning report for blueprint 810ea95a-4730-43dd-867e-1984aeb9d873: -* discretionary zones placed: - * 1 zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: external_dns -* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: pending non-nexus zone updates diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 6dd529cd40a..29b99f38e4f 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -23,8 +23,8 @@ use nexus_db_model::DbMetadataNexusState; use nexus_db_model::EARLIEST_SUPPORTED_VERSION; use nexus_db_model::SchemaUpgradeStep; use nexus_db_model::SchemaVersion; -use nexus_types::deployment::BlueprintZoneDisposition; use omicron_common::api::external::Error; +use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use semver::Version; @@ -300,6 +300,23 @@ impl DatastoreSetupAction { } impl DataStore { + /// Returns [`DbMetadataNexus`] records in any of the supplied states. + pub async fn get_db_metadata_nexus_in_state( + &self, + opctx: &OpContext, + states: &[DbMetadataNexusState], + ) -> Result, Error> { + use nexus_db_schema::schema::db_metadata_nexus::dsl; + + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + + dsl::db_metadata_nexus + .filter(dsl::state.eq_any(states.to_vec())) + .load_async(&*self.pool_connection_authorized(&opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + // Checks if the specified Nexus has access to the database. async fn check_nexus_access( &self, @@ -686,8 +703,8 @@ impl DataStore { Ok(()) } - // Returns the access this Nexus has to the database - async fn database_nexus_access( + /// Returns the access this Nexus has to the database + pub async fn database_nexus_access( &self, nexus_id: OmicronZoneUuid, ) -> Result, Error> { @@ -757,24 +774,19 @@ impl DataStore { pub async fn database_nexus_access_create( &self, opctx: &OpContext, - blueprint: &nexus_types::deployment::Blueprint, + blueprint_id: BlueprintUuid, + active: &Vec, + not_yet: &Vec, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; - // TODO: Without https://github.com/oxidecomputer/omicron/pull/8863, we - // treat all Nexuses as active. Some will become "not_yet", depending on - // the Nexus Generation, once it exists. - let active_nexus_zones = blueprint - .all_omicron_zones(BlueprintZoneDisposition::is_in_service) - .filter_map(|(_sled, zone_cfg)| { - if zone_cfg.zone_type.is_nexus() { - Some(zone_cfg) - } else { - None - } - }); - let new_nexuses = active_nexus_zones - .map(|z| DbMetadataNexus::new(z.id, DbMetadataNexusState::Active)) + let active_nexuses = active + .into_iter() + .map(|id| DbMetadataNexus::new(*id, DbMetadataNexusState::Active)) + .collect::>(); + let not_yet_nexuses = not_yet + .into_iter() + .map(|id| DbMetadataNexus::new(*id, DbMetadataNexusState::NotYet)) .collect::>(); let conn = &*self.pool_connection_authorized(&opctx).await?; @@ -782,14 +794,15 @@ impl DataStore { &conn, "database_nexus_access_create", opctx, - blueprint.id, + blueprint_id, |conn| { - let new_nexuses = new_nexuses.clone(); + let nexus_records = + [&active_nexuses[..], ¬_yet_nexuses[..]].concat(); async move { use nexus_db_schema::schema::db_metadata_nexus::dsl; diesel::insert_into(dsl::db_metadata_nexus) - .values(new_nexuses) + .values(nexus_records) .on_conflict(dsl::nexus_id) .do_nothing() .execute_async(conn) @@ -825,6 +838,22 @@ impl DataStore { Ok(()) } + /// Updates a nexus access record to "Quiesced" + pub async fn database_nexus_access_quiesce( + &self, + nexus_id: OmicronZoneUuid, + ) -> Result<(), Error> { + use nexus_db_schema::schema::db_metadata_nexus::dsl; + + diesel::update(dsl::db_metadata_nexus) + .filter(dsl::nexus_id.eq(nexus_id.into_untyped_uuid())) + .set(dsl::state.eq(DbMetadataNexusState::Quiesced)) + .execute_async(&*self.pool_connection_unauthorized().await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } /// Initializes Nexus database access records from a blueprint using an /// existing connection /// @@ -1136,35 +1165,8 @@ mod test { use crate::db::pub_test_utils::TestDatabase; use camino::Utf8Path; use camino_tempfile::Utf8TempDir; - use id_map::IdMap; use nexus_db_model::SCHEMA_VERSION; - use nexus_inventory::now_db_precision; - use nexus_types::deployment::Blueprint; - use nexus_types::deployment::BlueprintHostPhase2DesiredSlots; - use nexus_types::deployment::BlueprintSledConfig; - use nexus_types::deployment::BlueprintTarget; - use nexus_types::deployment::BlueprintZoneConfig; - use nexus_types::deployment::BlueprintZoneDisposition; - use nexus_types::deployment::BlueprintZoneImageSource; - use nexus_types::deployment::BlueprintZoneType; - use nexus_types::deployment::CockroachDbPreserveDowngrade; - use nexus_types::deployment::OximeterReadMode; - use nexus_types::deployment::PendingMgsUpdates; - use nexus_types::deployment::PlanningReport; - use nexus_types::deployment::blueprint_zone_type; - use nexus_types::external_api::views::SledState; - use nexus_types::inventory::NetworkInterface; - use nexus_types::inventory::NetworkInterfaceKind; - use omicron_common::api::external::Generation; - use omicron_common::api::external::MacAddr; - use omicron_common::api::external::Vni; - use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; - use omicron_uuid_kinds::BlueprintUuid; - use omicron_uuid_kinds::ExternalIpUuid; - use omicron_uuid_kinds::SledUuid; - use omicron_uuid_kinds::ZpoolUuid; - use std::collections::BTreeMap; // Confirms that calling the internal "ensure_schema" function can succeed // when the database is already at that version. @@ -2036,334 +2038,6 @@ mod test { logctx.cleanup_successful(); } - fn create_test_blueprint( - nexus_zones: Vec<(OmicronZoneUuid, BlueprintZoneDisposition)>, - ) -> Blueprint { - let blueprint_id = BlueprintUuid::new_v4(); - let sled_id = SledUuid::new_v4(); - - let zones: IdMap = nexus_zones - .into_iter() - .map(|(zone_id, disposition)| BlueprintZoneConfig { - disposition, - id: zone_id, - filesystem_pool: ZpoolName::new_external(ZpoolUuid::new_v4()), - zone_type: BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { - internal_address: "[::1]:0".parse().unwrap(), - external_dns_servers: Vec::new(), - external_ip: nexus_types::deployment::OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), - }, - external_tls: true, - nic: NetworkInterface { - id: uuid::Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: zone_id.into_untyped_uuid(), - }, - name: "test-nic".parse().unwrap(), - ip: "192.168.1.1".parse().unwrap(), - mac: MacAddr::random_system(), - subnet: ipnetwork::IpNetwork::V4( - "192.168.1.0/24".parse().unwrap() - ).into(), - vni: Vni::try_from(100).unwrap(), - primary: true, - slot: 0, - transit_ips: Vec::new(), - }, - nexus_generation: Generation::new(), - }), - image_source: BlueprintZoneImageSource::InstallDataset, - }) - .collect(); - - let mut sleds = BTreeMap::new(); - sleds.insert( - sled_id, - BlueprintSledConfig { - state: SledState::Active, - sled_agent_generation: Generation::new(), - zones, - disks: IdMap::new(), - datasets: IdMap::new(), - remove_mupdate_override: None, - host_phase_2: BlueprintHostPhase2DesiredSlots::current_contents( - ), - }, - ); - - Blueprint { - id: blueprint_id, - sleds, - pending_mgs_updates: PendingMgsUpdates::new(), - parent_blueprint_id: None, - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - target_release_minimum_generation: Generation::new(), - nexus_generation: Generation::new(), - cockroachdb_fingerprint: String::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - clickhouse_cluster_config: None, - oximeter_read_mode: OximeterReadMode::SingleNode, - oximeter_read_version: Generation::new(), - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - report: PlanningReport::new(blueprint_id), - } - } - - #[tokio::test] - async fn test_database_nexus_access_create() { - let logctx = dev::test_setup_log("test_database_nexus_access_create"); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let datastore = db.datastore(); - let opctx = db.opctx(); - - // Create a blueprint with two in-service Nexus zones, - // and one expunged Nexus. - let nexus1_id = OmicronZoneUuid::new_v4(); - let nexus2_id = OmicronZoneUuid::new_v4(); - let expunged_nexus = OmicronZoneUuid::new_v4(); - let blueprint = create_test_blueprint(vec![ - (nexus1_id, BlueprintZoneDisposition::InService), - (nexus2_id, BlueprintZoneDisposition::InService), - ( - expunged_nexus, - BlueprintZoneDisposition::Expunged { - as_of_generation: Generation::new(), - ready_for_cleanup: true, - }, - ), - ]); - - // Insert the blueprint and make it the target - datastore - .blueprint_insert(&opctx, &blueprint) - .await - .expect("Failed to insert blueprint"); - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: blueprint.id, - enabled: false, - time_made_target: chrono::Utc::now(), - }, - ) - .await - .expect("Failed to set blueprint target"); - - // Create nexus access records - datastore - .database_nexus_access_create(&opctx, &blueprint) - .await - .expect("Failed to create nexus access"); - - // Verify records were created with Active state - let nexus1_access = datastore - .database_nexus_access(nexus1_id) - .await - .expect("Failed to get nexus1 access"); - let nexus2_access = datastore - .database_nexus_access(nexus2_id) - .await - .expect("Failed to get nexus2 access"); - let expunged_access = datastore - .database_nexus_access(expunged_nexus) - .await - .expect("Failed to get expunged access"); - - assert!(nexus1_access.is_some(), "nexus1 should have access record"); - assert!(nexus2_access.is_some(), "nexus2 should have access record"); - assert!( - expunged_access.is_none(), - "expunged nexus should not have access record" - ); - - let nexus1_record = nexus1_access.unwrap(); - let nexus2_record = nexus2_access.unwrap(); - assert_eq!(nexus1_record.state(), DbMetadataNexusState::Active); - assert_eq!(nexus2_record.state(), DbMetadataNexusState::Active); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn test_database_nexus_access_create_idempotent() { - let logctx = - dev::test_setup_log("test_database_nexus_access_create_idempotent"); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let datastore = db.datastore(); - let opctx = db.opctx(); - - // Create a blueprint with one Nexus zone - let nexus_id = OmicronZoneUuid::new_v4(); - let blueprint = create_test_blueprint(vec![( - nexus_id, - BlueprintZoneDisposition::InService, - )]); - - // Insert the blueprint and make it the target - datastore - .blueprint_insert(&opctx, &blueprint) - .await - .expect("Failed to insert blueprint"); - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: blueprint.id, - enabled: false, - time_made_target: chrono::Utc::now(), - }, - ) - .await - .expect("Failed to set blueprint target"); - - // Create nexus access records (first time) - datastore - .database_nexus_access_create(&opctx, &blueprint) - .await - .expect("Failed to create nexus access (first time)"); - - // Verify record was created - async fn confirm_state( - datastore: &DataStore, - nexus_id: OmicronZoneUuid, - expected_state: DbMetadataNexusState, - ) { - let state = datastore - .database_nexus_access(nexus_id) - .await - .expect("Failed to get nexus access after first create") - .expect("Entry for Nexus should have been inserted"); - assert_eq!(state.state(), expected_state); - } - - confirm_state(datastore, nexus_id, DbMetadataNexusState::Active).await; - - // Creating the record again: not an error. - datastore - .database_nexus_access_create(&opctx, &blueprint) - .await - .expect("Failed to create nexus access (first time)"); - confirm_state(datastore, nexus_id, DbMetadataNexusState::Active).await; - - // Manually make the record "Quiesced". - use nexus_db_schema::schema::db_metadata_nexus::dsl; - diesel::update(dsl::db_metadata_nexus) - .filter(dsl::nexus_id.eq(nexus_id.into_untyped_uuid())) - .set(dsl::state.eq(DbMetadataNexusState::Quiesced)) - .execute_async( - &*datastore.pool_connection_unauthorized().await.unwrap(), - ) - .await - .expect("Failed to update record"); - confirm_state(datastore, nexus_id, DbMetadataNexusState::Quiesced) - .await; - - // Create nexus access records another time - should be idempotent, - // but should be "on-conflict, ignore". - datastore - .database_nexus_access_create(&opctx, &blueprint) - .await - .expect("Failed to create nexus access (second time)"); - confirm_state(datastore, nexus_id, DbMetadataNexusState::Quiesced) - .await; - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn test_database_nexus_access_create_fails_wrong_target_blueprint() { - let logctx = dev::test_setup_log( - "test_database_nexus_access_create_fails_wrong_target_blueprint", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let datastore = db.datastore(); - let opctx = db.opctx(); - - // Create two different blueprints - let nexus_id = OmicronZoneUuid::new_v4(); - let target_blueprint = create_test_blueprint(vec![( - nexus_id, - BlueprintZoneDisposition::InService, - )]); - let non_target_blueprint = create_test_blueprint(vec![( - nexus_id, - BlueprintZoneDisposition::InService, - )]); - - // Insert both blueprints - datastore - .blueprint_insert(&opctx, &target_blueprint) - .await - .expect("Failed to insert target blueprint"); - datastore - .blueprint_insert(&opctx, &non_target_blueprint) - .await - .expect("Failed to insert non-target blueprint"); - - // Set the first blueprint as the target - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: target_blueprint.id, - enabled: false, - time_made_target: chrono::Utc::now(), - }, - ) - .await - .expect("Failed to set target blueprint"); - - // Try to create nexus access records using the non-target blueprint. - // This should fail because the transaction should check if the - // blueprint is the current target - let result = datastore - .database_nexus_access_create(&opctx, &non_target_blueprint) - .await; - assert!( - result.is_err(), - "Creating nexus access with wrong target blueprint should fail" - ); - - // Verify no records were created for the nexus - let access = datastore - .database_nexus_access(nexus_id) - .await - .expect("Failed to get nexus access"); - assert!( - access.is_none(), - "No access record should exist when wrong blueprint is used" - ); - - // Verify that using the correct target blueprint works - datastore - .database_nexus_access_create(&opctx, &target_blueprint) - .await - .expect( - "Creating nexus access with correct blueprint should succeed", - ); - - let access_after_correct = datastore - .database_nexus_access(nexus_id) - .await - .expect("Failed to get nexus access after correct blueprint"); - assert!( - access_after_correct.is_some(), - "Access record should exist after using correct target blueprint" - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn test_database_nexus_access_delete() { let logctx = dev::test_setup_log("test_database_nexus_access_delete"); diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index c832d54a4d7..618bd990769 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -3339,12 +3339,16 @@ mod tests { ) .expect("ensured disks"); } + let external_tls = false; + let external_dns_servers = vec![]; + let nexus_generation = builder.parent_blueprint().nexus_generation; builder .sled_add_zone_nexus_with_config( sled_ids[2], - false, - Vec::new(), + external_tls, + external_dns_servers, BlueprintZoneImageSource::InstallDataset, + nexus_generation, ) .expect("added nexus to third sled"); builder.build() @@ -3414,12 +3418,17 @@ mod tests { ) .expect("created blueprint builder"); for &sled_id in &sled_ids { + let external_tls = false; + let external_dns_servers = vec![]; + let nexus_generation = + builder.parent_blueprint().nexus_generation; builder .sled_add_zone_nexus_with_config( sled_id, - false, - Vec::new(), + external_tls, + external_dns_servers, BlueprintZoneImageSource::InstallDataset, + nexus_generation, ) .expect("added nexus to third sled"); } diff --git a/nexus/reconfigurator/blippy/src/blippy.rs b/nexus/reconfigurator/blippy/src/blippy.rs index 07192c7f276..f7382fe1242 100644 --- a/nexus/reconfigurator/blippy/src/blippy.rs +++ b/nexus/reconfigurator/blippy/src/blippy.rs @@ -14,10 +14,12 @@ use nexus_types::inventory::ZpoolName; use omicron_common::address::DnsSubnet; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; +use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::disk::DatasetKind; use omicron_common::disk::M2Slot; use omicron_uuid_kinds::MupdateOverrideUuid; +use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeSet; @@ -51,47 +53,70 @@ impl fmt::Display for Severity { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum Kind { + Blueprint(BlueprintKind), Sled { sled_id: SledUuid, kind: SledKind }, } impl Kind { pub fn display_component(&self) -> impl fmt::Display + '_ { enum Component<'a> { + Blueprint, Sled(&'a SledUuid), } impl fmt::Display for Component<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Component::Blueprint => write!(f, "blueprint"), Component::Sled(id) => write!(f, "sled {id}"), } } } match self { + Kind::Blueprint { .. } => Component::Blueprint, Kind::Sled { sled_id, .. } => Component::Sled(sled_id), } } pub fn display_subkind(&self) -> impl fmt::Display + '_ { enum Subkind<'a> { + Blueprint(&'a BlueprintKind), Sled(&'a SledKind), } impl fmt::Display for Subkind<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Subkind::Blueprint(kind) => write!(f, "{kind}"), Subkind::Sled(kind) => write!(f, "{kind}"), } } } match self { + Kind::Blueprint(kind) => Subkind::Blueprint(kind), Kind::Sled { kind, .. } => Subkind::Sled(kind), } } } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum BlueprintKind { + /// No zones exist in the blueprint using the active Nexus generation + NoZonesWithActiveNexusGeneration(Generation), +} + +impl fmt::Display for BlueprintKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BlueprintKind::NoZonesWithActiveNexusGeneration(gen) => { + write!(f, "No zones with active nexus generation @ {gen}",) + } + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum SledKind { /// Two running zones have the same underlay IP address. @@ -193,6 +218,19 @@ pub enum SledKind { version: BlueprintArtifactVersion, hash: ArtifactHash, }, + /// A Nexus zone exists which is too new, relative to the "active + /// generation". + NexusZoneGenerationTooNew { + active_generation: Generation, + zone_generation: Generation, + id: OmicronZoneUuid, + }, + /// Nexus zones with the same generation have different image sources. + NexusZoneGenerationImageSourceMismatch { + zone1: BlueprintZoneConfig, + zone2: BlueprintZoneConfig, + generation: Generation, + }, } impl fmt::Display for SledKind { @@ -415,6 +453,29 @@ impl fmt::Display for SledKind { (version {version}, hash {hash})", ) } + SledKind::NexusZoneGenerationTooNew { + active_generation, + zone_generation, + id, + } => { + write!( + f, + "Nexus zone {id} has a generation {zone_generation}, which \ + is too new relative to the active generation {active_generation}" + ) + } + SledKind::NexusZoneGenerationImageSourceMismatch { + zone1, + zone2, + generation, + } => { + write!( + f, + "Nexus zones {} and {} both have generation {generation} but \ + different image sources ({:?} vs {:?})", + zone1.id, zone2.id, zone1.image_source, zone2.image_source, + ) + } } } } @@ -473,6 +534,14 @@ impl<'a> Blippy<'a> { self.blueprint } + pub(crate) fn push_blueprint_note( + &mut self, + severity: Severity, + kind: BlueprintKind, + ) { + self.notes.push(Note { severity, kind: Kind::Blueprint(kind) }); + } + pub(crate) fn push_sled_note( &mut self, sled_id: SledUuid, diff --git a/nexus/reconfigurator/blippy/src/checks.rs b/nexus/reconfigurator/blippy/src/checks.rs index 3894ae335b3..66e69dcacd9 100644 --- a/nexus/reconfigurator/blippy/src/checks.rs +++ b/nexus/reconfigurator/blippy/src/checks.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::blippy::Blippy; +use crate::blippy::BlueprintKind; use crate::blippy::Severity; use crate::blippy::SledKind; use nexus_sled_agent_shared::inventory::ZoneKind; @@ -21,6 +22,7 @@ use nexus_types::deployment::blueprint_zone_type; use omicron_common::address::DnsSubnet; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; +use omicron_common::api::external::Generation; use omicron_common::disk::DatasetKind; use omicron_common::disk::M2Slot; use omicron_uuid_kinds::MupdateOverrideUuid; @@ -37,6 +39,7 @@ pub(crate) fn perform_all_blueprint_only_checks(blippy: &mut Blippy<'_>) { check_dataset_zpool_uniqueness(blippy); check_datasets(blippy); check_mupdate_override(blippy); + check_nexus_generation_consistency(blippy); } fn check_underlay_ips(blippy: &mut Blippy<'_>) { @@ -632,6 +635,78 @@ fn check_mupdate_override_host_phase_2_contents( } } +fn check_nexus_generation_consistency(blippy: &mut Blippy<'_>) { + use std::collections::HashMap; + + // Map from generation -> (sled_id, image_source, zone) + let mut generation_info: HashMap< + Generation, + Vec<(SledUuid, BlueprintZoneImageSource, &BlueprintZoneConfig)>, + > = HashMap::new(); + + // Collect all Nexus zones and their generations + for (sled_id, zone) in blippy + .blueprint() + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + { + if let BlueprintZoneType::Nexus(nexus) = &zone.zone_type { + generation_info.entry(nexus.nexus_generation).or_default().push(( + sled_id, + zone.image_source.clone(), + zone, + )); + } + } + + // Check that the top-level Nexus generation is consistent with the images + let active_gen = blippy.blueprint().nexus_generation; + if generation_info.get(&active_gen).is_none() { + blippy.push_blueprint_note( + Severity::Fatal, + BlueprintKind::NoZonesWithActiveNexusGeneration(active_gen), + ); + return; + }; + + // Check each generation for image source consistency + for (generation, zones_with_gen) in &generation_info { + // Take the first zone as the reference + let (ref_sled_id, ref_image_source, ref_zone) = &zones_with_gen[0]; + + if *generation > active_gen.next() { + blippy.push_sled_note( + *ref_sled_id, + Severity::Fatal, + SledKind::NexusZoneGenerationTooNew { + active_generation: active_gen, + zone_generation: *generation, + id: ref_zone.id, + }, + ); + } + + if zones_with_gen.len() < 2 { + // Only one zone with this generation, no consistency issue + continue; + } + + // Compare all other zones to the reference + for (_sled_id, image_source, zone) in &zones_with_gen[1..] { + if image_source != ref_image_source { + blippy.push_sled_note( + *ref_sled_id, + Severity::Fatal, + SledKind::NexusZoneGenerationImageSourceMismatch { + zone1: (*ref_zone).clone(), + zone2: (*zone).clone(), + generation: *generation, + }, + ); + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -1311,6 +1386,7 @@ mod tests { } _ => (), }, + _ => panic!("Unexpected note: {note:?}"), } } @@ -1812,4 +1888,132 @@ mod tests { logctx.cleanup_successful(); } + + #[test] + fn test_nexus_generation_no_nexus() { + static TEST_NAME: &str = "test_nexus_generation_no_nexus"; + let logctx = test_setup_log(TEST_NAME); + let (_, blueprint) = ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(1) + .nexus_count(0) + .build(); + + // Run blippy checks + let expected_notes = [Note { + severity: Severity::Fatal, + kind: Kind::Blueprint( + BlueprintKind::NoZonesWithActiveNexusGeneration( + blueprint.nexus_generation, + ), + ), + }]; + + let report = + Blippy::new(&blueprint).into_report(BlippyReportSortKey::Kind); + eprintln!("{}", report.display()); + assert_eq!(report.notes(), &expected_notes); + + logctx.cleanup_successful(); + } + + #[test] + fn test_nexus_generation_image_consistency() { + static TEST_NAME: &str = "test_nexus_generation_image_consistency"; + let logctx = test_setup_log(TEST_NAME); + let (_, mut blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(3) + .nexus_count(3) + .build(); + + // Find the Nexus zones + let ((sled1, zone1_id), (sled2, zone2_id)) = { + let nexus_zones: Vec<_> = blueprint + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .filter_map(|(sled_id, zone)| { + if matches!(zone.zone_type, BlueprintZoneType::Nexus(_)) { + Some((sled_id, zone)) + } else { + None + } + }) + .collect(); + + // Should have exactly 3 Nexus zones + assert_eq!(nexus_zones.len(), 3); + + // Modify two zones to have the same generation but different image sources + let (sled1, zone1) = nexus_zones[0]; + let (sled2, zone2) = nexus_zones[1]; + + ((sled1, zone1.id), (sled2, zone2.id)) + }; + + let generation = Generation::new(); + + let zone1 = { + // Find the zones in the blueprint and modify them + let mut zone1_config = blueprint + .sleds + .get_mut(&sled1) + .unwrap() + .zones + .get_mut(&zone1_id) + .unwrap(); + + match &mut zone1_config.zone_type { + BlueprintZoneType::Nexus(nexus) => { + nexus.nexus_generation = generation; + } + _ => unreachable!("this is a Nexus zone"), + } + zone1_config.image_source = + BlueprintZoneImageSource::InstallDataset; + zone1_config.clone() + }; + + let zone2 = { + let mut zone2_config = blueprint + .sleds + .get_mut(&sled2) + .unwrap() + .zones + .get_mut(&zone2_id) + .unwrap(); + + match &mut zone2_config.zone_type { + BlueprintZoneType::Nexus(nexus) => { + nexus.nexus_generation = generation; + } + _ => unreachable!("this is a Nexus zone"), + } + zone2_config.image_source = BlueprintZoneImageSource::Artifact { + version: BlueprintArtifactVersion::Available { + version: "1.0.0".parse().unwrap(), + }, + hash: ArtifactHash([0; 32]), + }; + zone2_config.clone() + }; + + // Run blippy checks + let expected_notes = [Note { + severity: Severity::Fatal, + kind: Kind::Sled { + sled_id: sled1, + kind: SledKind::NexusZoneGenerationImageSourceMismatch { + zone1, + zone2, + generation, + }, + }, + }]; + + let report = + Blippy::new(&blueprint).into_report(BlippyReportSortKey::Kind); + eprintln!("{}", report.display()); + assert_eq!(report.notes(), &expected_notes); + + logctx.cleanup_successful(); + } } diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index 43f92b80113..1ed22ffa34d 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -55,6 +55,7 @@ async-bb8-diesel.workspace = true diesel.workspace = true httptest.workspace = true ipnet.workspace = true +ipnetwork.workspace = true nexus-db-queries = { workspace = true, features = ["testing"] } nexus-db-schema.workspace = true nexus-reconfigurator-planning.workspace = true diff --git a/nexus/reconfigurator/execution/src/database.rs b/nexus/reconfigurator/execution/src/database.rs index 9652e53ec1a..97b1dd7da7d 100644 --- a/nexus/reconfigurator/execution/src/database.rs +++ b/nexus/reconfigurator/execution/src/database.rs @@ -5,9 +5,12 @@ //! Manages deployment of records into the database. use anyhow::anyhow; +use nexus_db_model::DbMetadataNexusState; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintZoneDisposition; +use nexus_types::deployment::BlueprintZoneType; /// Idempotently ensure that the Nexus records for the zones are populated /// in the database. @@ -16,9 +19,579 @@ pub(crate) async fn deploy_db_metadata_nexus_records( datastore: &DataStore, blueprint: &Blueprint, ) -> Result<(), anyhow::Error> { + // We need to determine the generation of the currently running set of + // Nexuses. This is usually the same as "blueprint.nexus_generation", but + // can lag behind it if we are one of those Nexuses running after quiescing + // has started. + let active_nexus_zones = datastore + .get_db_metadata_nexus_in_state(opctx, &[DbMetadataNexusState::Active]) + .await? + .into_iter() + .map(|z| z.nexus_id()) + .collect::>(); + let active_generation = blueprint + .find_generation_for_nexus(&active_nexus_zones) + .ok_or_else(|| { + anyhow!("No Nexuses with active db_metadata_nexus records found in blueprint") + })?; + + let mut active = vec![]; + let mut not_yet = vec![]; + for (_, zone) in + blueprint.all_omicron_zones(BlueprintZoneDisposition::is_in_service) + { + if let BlueprintZoneType::Nexus(ref nexus) = zone.zone_type { + if nexus.nexus_generation == active_generation { + active.push(zone.id); + } else if nexus.nexus_generation > active_generation { + not_yet.push(zone.id); + } + } + } + datastore - .database_nexus_access_create(opctx, blueprint) + .database_nexus_access_create(opctx, blueprint.id, &active, ¬_yet) .await .map_err(|err| anyhow!(err))?; Ok(()) } + +#[cfg(test)] +mod test { + use super::*; + use id_map::IdMap; + use nexus_db_model::DbMetadataNexusState; + use nexus_db_queries::db::pub_test_utils::TestDatabase; + use nexus_inventory::now_db_precision; + use nexus_types::deployment::Blueprint; + use nexus_types::deployment::BlueprintHostPhase2DesiredSlots; + use nexus_types::deployment::BlueprintSledConfig; + use nexus_types::deployment::BlueprintTarget; + use nexus_types::deployment::BlueprintZoneConfig; + use nexus_types::deployment::BlueprintZoneDisposition; + use nexus_types::deployment::BlueprintZoneImageSource; + use nexus_types::deployment::BlueprintZoneType; + use nexus_types::deployment::CockroachDbPreserveDowngrade; + use nexus_types::deployment::OximeterReadMode; + use nexus_types::deployment::PendingMgsUpdates; + use nexus_types::deployment::PlanningReport; + use nexus_types::deployment::blueprint_zone_type; + use nexus_types::external_api::views::SledState; + use nexus_types::inventory::NetworkInterface; + use nexus_types::inventory::NetworkInterfaceKind; + use omicron_common::api::external::Generation; + use omicron_common::api::external::MacAddr; + use omicron_common::api::external::Vni; + use omicron_common::zpool_name::ZpoolName; + use omicron_test_utils::dev; + use omicron_uuid_kinds::BlueprintUuid; + use omicron_uuid_kinds::ExternalIpUuid; + use omicron_uuid_kinds::GenericUuid; + use omicron_uuid_kinds::OmicronZoneUuid; + use omicron_uuid_kinds::SledUuid; + use omicron_uuid_kinds::ZpoolUuid; + use std::collections::BTreeMap; + + fn create_test_blueprint( + top_level_nexus_generation: Generation, + nexus_zones: Vec<( + OmicronZoneUuid, + BlueprintZoneDisposition, + Generation, + )>, + ) -> Blueprint { + let blueprint_id = BlueprintUuid::new_v4(); + let sled_id = SledUuid::new_v4(); + + let zones: IdMap = nexus_zones + .into_iter() + .map(|(zone_id, disposition, nexus_generation)| BlueprintZoneConfig { + disposition, + id: zone_id, + filesystem_pool: ZpoolName::new_external(ZpoolUuid::new_v4()), + zone_type: BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { + internal_address: "[::1]:0".parse().unwrap(), + external_dns_servers: Vec::new(), + external_ip: nexus_types::deployment::OmicronZoneExternalFloatingIp { + id: ExternalIpUuid::new_v4(), + ip: std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), + }, + external_tls: true, + nic: NetworkInterface { + id: uuid::Uuid::new_v4(), + kind: NetworkInterfaceKind::Service { + id: zone_id.into_untyped_uuid(), + }, + name: "test-nic".parse().unwrap(), + ip: "192.168.1.1".parse().unwrap(), + mac: MacAddr::random_system(), + subnet: ipnetwork::IpNetwork::V4( + "192.168.1.0/24".parse().unwrap() + ).into(), + vni: Vni::try_from(100).unwrap(), + primary: true, + slot: 0, + transit_ips: Vec::new(), + }, + nexus_generation, + }), + image_source: BlueprintZoneImageSource::InstallDataset, + }) + .collect(); + + let mut sleds = BTreeMap::new(); + sleds.insert( + sled_id, + BlueprintSledConfig { + state: SledState::Active, + sled_agent_generation: Generation::new(), + zones, + disks: IdMap::new(), + datasets: IdMap::new(), + remove_mupdate_override: None, + host_phase_2: BlueprintHostPhase2DesiredSlots::current_contents( + ), + }, + ); + + Blueprint { + id: blueprint_id, + sleds, + pending_mgs_updates: PendingMgsUpdates::new(), + parent_blueprint_id: None, + internal_dns_version: Generation::new(), + external_dns_version: Generation::new(), + target_release_minimum_generation: Generation::new(), + nexus_generation: top_level_nexus_generation, + cockroachdb_fingerprint: String::new(), + cockroachdb_setting_preserve_downgrade: + CockroachDbPreserveDowngrade::DoNotModify, + clickhouse_cluster_config: None, + oximeter_read_mode: OximeterReadMode::SingleNode, + oximeter_read_version: Generation::new(), + time_created: now_db_precision(), + creator: "test suite".to_string(), + comment: "test blueprint".to_string(), + report: PlanningReport::new(blueprint_id), + } + } + + #[tokio::test] + async fn test_database_nexus_access_create() { + let logctx = dev::test_setup_log("test_database_nexus_access_create"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let datastore = db.datastore(); + let opctx = db.opctx(); + + // Create a blueprint with in-service Nexus zones, and one expunged + // Nexus. + let nexus1_id = OmicronZoneUuid::new_v4(); + let nexus2_id = OmicronZoneUuid::new_v4(); + let expunged_nexus = OmicronZoneUuid::new_v4(); + + // Our currently-running Nexus must already have a record + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .initialize_nexus_access_from_blueprint_on_connection( + &conn, + vec![nexus1_id], + ) + .await + .unwrap(); + + let blueprint = create_test_blueprint( + Generation::new(), + vec![ + // This nexus matches the top-level generation, and already + // exists as "active". + ( + nexus1_id, + BlueprintZoneDisposition::InService, + Generation::new(), + ), + // This nexus is ahead of the the top-level nexus generation, + // and will be created as "not yet". + ( + nexus2_id, + BlueprintZoneDisposition::InService, + Generation::new().next(), + ), + ( + expunged_nexus, + BlueprintZoneDisposition::Expunged { + as_of_generation: Generation::new(), + ready_for_cleanup: true, + }, + Generation::new(), + ), + ], + ); + + // Insert the blueprint and make it the target + datastore + .blueprint_insert(&opctx, &blueprint) + .await + .expect("Failed to insert blueprint"); + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: blueprint.id, + enabled: false, + time_made_target: chrono::Utc::now(), + }, + ) + .await + .expect("Failed to set blueprint target"); + + // Create nexus access records + deploy_db_metadata_nexus_records(&opctx, datastore, &blueprint) + .await + .expect("Failed to create nexus access"); + + // Verify records were created for in-service Nexuses. + let nexus1_access = datastore + .database_nexus_access(nexus1_id) + .await + .expect("Failed to get nexus1 access"); + let nexus2_access = datastore + .database_nexus_access(nexus2_id) + .await + .expect("Failed to get nexus2 access"); + let expunged_access = datastore + .database_nexus_access(expunged_nexus) + .await + .expect("Failed to get expunged access"); + + assert!(nexus1_access.is_some(), "nexus1 should have access record"); + assert!(nexus2_access.is_some(), "nexus2 should have access record"); + assert!( + expunged_access.is_none(), + "expunged nexus should not have access record" + ); + + // See above for the rationale here: + // + // Nexus 1 already existed, and was active. + // Nexus 2 has a higher generation number (e.g., it represents + // a new deployment that has not yet been activated). + // The expunged Nexus was ignored. + let nexus1_record = nexus1_access.unwrap(); + let nexus2_record = nexus2_access.unwrap(); + assert_eq!(nexus1_record.state(), DbMetadataNexusState::Active); + assert_eq!(nexus2_record.state(), DbMetadataNexusState::NotYet); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_database_nexus_access_create_during_quiesce() { + let logctx = dev::test_setup_log( + "test_database_nexus_access_create_during_quiesce", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let datastore = db.datastore(); + let opctx = db.opctx(); + + // Create a blueprint with in-service Nexus zones, and one expunged + // Nexus. + let nexus1_id = OmicronZoneUuid::new_v4(); + let nexus2_id = OmicronZoneUuid::new_v4(); + let nexus3_id = OmicronZoneUuid::new_v4(); + + // Our currently-running Nexus must already have a record + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .initialize_nexus_access_from_blueprint_on_connection( + &conn, + vec![nexus1_id], + ) + .await + .unwrap(); + + let blueprint = create_test_blueprint( + // NOTE: This is using a "Generation = 2", implying that all + // nexuses using "Generation = 1" should start quiescing. + Generation::new().next(), + vec![ + // This Nexus already exists as active - even though it's + // quiescing currently. + ( + nexus1_id, + BlueprintZoneDisposition::InService, + Generation::new(), + ), + // This Nexus matches the the top-level nexus generation, + // and will be created as "not yet", because "nexus1" is still + // running. + ( + nexus2_id, + BlueprintZoneDisposition::InService, + Generation::new().next(), + ), + // This Nexus will quiesce soon after starting, but can still be + // created as active. + ( + nexus3_id, + BlueprintZoneDisposition::InService, + Generation::new(), + ), + ], + ); + + // Insert the blueprint and make it the target + datastore + .blueprint_insert(&opctx, &blueprint) + .await + .expect("Failed to insert blueprint"); + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: blueprint.id, + enabled: false, + time_made_target: chrono::Utc::now(), + }, + ) + .await + .expect("Failed to set blueprint target"); + + // Create nexus access records + deploy_db_metadata_nexus_records(&opctx, datastore, &blueprint) + .await + .expect("Failed to create nexus access"); + + // Verify records were created for in-service Nexuses. + let nexus1_access = datastore + .database_nexus_access(nexus1_id) + .await + .expect("Failed to get nexus1 access"); + let nexus2_access = datastore + .database_nexus_access(nexus2_id) + .await + .expect("Failed to get nexus2 access"); + let nexus3_access = datastore + .database_nexus_access(nexus3_id) + .await + .expect("Failed to get nexus3 access"); + + assert!(nexus1_access.is_some(), "nexus1 should have access record"); + assert!(nexus2_access.is_some(), "nexus2 should have access record"); + assert!(nexus2_access.is_some(), "nexus3 should have access record"); + + // See above for the rationale here: + // + // Nexus 1 already existed, and was active. + // Nexus 2 has a higher generation number (e.g., it represents + // a new deployment that has not yet been activated). + // Nexus 3 is getting a new record, but using the old generation number. + // It'll be treated as active. + let nexus1_record = nexus1_access.unwrap(); + let nexus2_record = nexus2_access.unwrap(); + let nexus3_record = nexus3_access.unwrap(); + assert_eq!(nexus1_record.state(), DbMetadataNexusState::Active); + assert_eq!(nexus2_record.state(), DbMetadataNexusState::NotYet); + assert_eq!(nexus3_record.state(), DbMetadataNexusState::Active); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_database_nexus_access_create_idempotent() { + let logctx = + dev::test_setup_log("test_database_nexus_access_create_idempotent"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let datastore = db.datastore(); + let opctx = db.opctx(); + + // Create a blueprint with a couple Nexus zones + let nexus1_id = OmicronZoneUuid::new_v4(); + let nexus2_id = OmicronZoneUuid::new_v4(); + let blueprint = create_test_blueprint( + Generation::new(), + vec![ + ( + nexus1_id, + BlueprintZoneDisposition::InService, + Generation::new(), + ), + ( + nexus2_id, + BlueprintZoneDisposition::InService, + Generation::new(), + ), + ], + ); + + // Insert the blueprint and make it the target + datastore + .blueprint_insert(&opctx, &blueprint) + .await + .expect("Failed to insert blueprint"); + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: blueprint.id, + enabled: false, + time_made_target: chrono::Utc::now(), + }, + ) + .await + .expect("Failed to set blueprint target"); + + // Create nexus access records (first time) + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .initialize_nexus_access_from_blueprint_on_connection( + &conn, + vec![nexus1_id, nexus2_id], + ) + .await + .unwrap(); + + // Verify record was created + async fn confirm_state( + datastore: &DataStore, + nexus_id: OmicronZoneUuid, + expected_state: DbMetadataNexusState, + ) { + let state = datastore + .database_nexus_access(nexus_id) + .await + .expect("Failed to get nexus access after first create") + .expect("Entry for Nexus should have been inserted"); + assert_eq!(state.state(), expected_state); + } + + confirm_state(datastore, nexus1_id, DbMetadataNexusState::Active).await; + confirm_state(datastore, nexus2_id, DbMetadataNexusState::Active).await; + + // Creating the record again: not an error. + deploy_db_metadata_nexus_records(&opctx, datastore, &blueprint) + .await + .expect("Failed to create nexus access"); + confirm_state(datastore, nexus1_id, DbMetadataNexusState::Active).await; + confirm_state(datastore, nexus2_id, DbMetadataNexusState::Active).await; + + // Manually make the record "Quiesced". + datastore.database_nexus_access_quiesce(nexus1_id).await.unwrap(); + confirm_state(datastore, nexus1_id, DbMetadataNexusState::Quiesced) + .await; + confirm_state(datastore, nexus2_id, DbMetadataNexusState::Active).await; + + // Create nexus access records another time - should be idempotent, + // but should be "on-conflict, ignore". + deploy_db_metadata_nexus_records(&opctx, datastore, &blueprint) + .await + .expect("Failed to create nexus access"); + confirm_state(datastore, nexus1_id, DbMetadataNexusState::Quiesced) + .await; + confirm_state(datastore, nexus2_id, DbMetadataNexusState::Active).await; + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_database_nexus_access_create_fails_wrong_target_blueprint() { + let logctx = dev::test_setup_log( + "test_database_nexus_access_create_fails_wrong_target_blueprint", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let datastore = db.datastore(); + let opctx = db.opctx(); + + // Create two different blueprints, each with two Nexuses. + // + // One of these Nexuses will have a "db_metadata_nexus" record + // for bootstrapping, the other won't exist (yet). + let nexus1_id = OmicronZoneUuid::new_v4(); + let nexus2_id = OmicronZoneUuid::new_v4(); + let both_nexuses = vec![ + (nexus1_id, BlueprintZoneDisposition::InService, Generation::new()), + (nexus2_id, BlueprintZoneDisposition::InService, Generation::new()), + ]; + + let target_blueprint = + create_test_blueprint(Generation::new(), both_nexuses.clone()); + let non_target_blueprint = + create_test_blueprint(Generation::new(), both_nexuses); + + // Initialize the "db_metadata_nexus" record for one of the Nexuses + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .initialize_nexus_access_from_blueprint_on_connection( + &conn, + vec![nexus1_id], + ) + .await + .unwrap(); + + // Insert both blueprints + datastore + .blueprint_insert(&opctx, &target_blueprint) + .await + .expect("Failed to insert target blueprint"); + datastore + .blueprint_insert(&opctx, &non_target_blueprint) + .await + .expect("Failed to insert non-target blueprint"); + + // Set the first blueprint as the target + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: target_blueprint.id, + enabled: false, + time_made_target: chrono::Utc::now(), + }, + ) + .await + .expect("Failed to set target blueprint"); + + // Try to create nexus access records using the non-target blueprint. + // This should fail because the transaction should check if the + // blueprint is the current target + let result = deploy_db_metadata_nexus_records( + &opctx, + datastore, + &non_target_blueprint, + ) + .await; + assert!( + result.is_err(), + "Creating nexus access with wrong target blueprint should fail" + ); + + // Verify no records were created for the second nexus + let access = datastore + .database_nexus_access(nexus2_id) + .await + .expect("Failed to get nexus access"); + assert!( + access.is_none(), + "No access record should exist when wrong blueprint is used" + ); + + // Verify that using the correct target blueprint works + deploy_db_metadata_nexus_records(&opctx, datastore, &target_blueprint) + .await + .expect("Failed to create nexus access"); + + let access_after_correct = datastore + .database_nexus_access(nexus2_id) + .await + .expect("Failed to get nexus access after correct blueprint"); + assert!( + access_after_correct.is_some(), + "Access record should exist after using correct target blueprint" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 4a259ce3873..4ae3117827b 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -313,6 +313,7 @@ mod test { use internal_dns_types::names::BOUNDARY_NTP_DNS_NAME; use internal_dns_types::names::DNS_ZONE; use internal_dns_types::names::ServiceName; + use nexus_db_model::DbMetadataNexusState; use nexus_db_model::DnsGroup; use nexus_db_model::Silo; use nexus_db_queries::authn; @@ -1502,6 +1503,17 @@ mod test { datastore.zpool_list_all_external_batched(&opctx).await.unwrap(); let ip_pool_range_rows = fetch_all_service_ip_pool_ranges(&datastore, &opctx).await; + let active_nexus_zones = datastore + .get_db_metadata_nexus_in_state( + &opctx, + &[DbMetadataNexusState::Active], + ) + .await + .internal_context("fetching active nexuses") + .unwrap() + .into_iter() + .map(|z| z.nexus_id()) + .collect(); let planning_input = { let mut builder = PlanningInputFromDb { sled_rows: &sled_rows, @@ -1527,6 +1539,8 @@ mod test { tuf_repo: TufRepoPolicy::initial(), old_repo: TufRepoPolicy::initial(), planner_config: PlannerConfig::default(), + active_nexus_zones, + not_yet_nexus_zones: Vec::new(), log, } .build() diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 90bf09fe1ba..4f1a5ab432b 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -139,6 +139,17 @@ pub enum Error { AllocateInternalDnsSubnet(#[from] NoAvailableDnsSubnets), #[error("error allocating external networking resources")] AllocateExternalNetworking(#[from] ExternalNetworkingError), + #[error( + "mismatch while setting nexus_generation for a zone with an old image, \ + expected current value is {expected} but actual value is {actual}" + )] + OldImageNexusGenerationMismatch { expected: Generation, actual: Generation }, + #[error( + "mismatch while setting nexus_generation for a zone with a new image, \ + expected current value is {expected} (or that +1) but actual value is \ + {actual}" + )] + NewImageNexusGenerationMismatch { expected: Generation, actual: Generation }, #[error("can only have {INTERNAL_DNS_REDUNDANCY} internal DNS servers")] PolicySpecifiesTooManyInternalDnsServers, #[error("zone is already up-to-date and should not be updated")] @@ -1545,6 +1556,92 @@ impl<'a> BlueprintBuilder<'a> { Ok(Ensure::Added) } + // Determines TLS and DNS server configuration from existing Nexus zones. + // + // Returns `Some((external_tls, external_dns_servers))` if existing Nexus + // zones are found, or `None` if no existing Nexus zones exist. + fn determine_nexus_tls_dns_config(&self) -> Option<(bool, Vec)> { + self.parent_blueprint + .all_omicron_zones(BlueprintZoneDisposition::any) + .find_map(|(_, z)| match &z.zone_type { + BlueprintZoneType::Nexus(nexus) => Some(( + nexus.external_tls, + nexus.external_dns_servers.clone(), + )), + _ => None, + }) + } + + // Determines the appropriate generation number for a new Nexus zone. + // This generation is based on the generation number used by existing + // Nexus zones. + // + // The logic is: + // - If any existing Nexus zone has the same image source, reuse its generation + // - Otherwise, use the highest existing generation + 1 + // - If no existing zones exist, return an error + // + // This function also validates that the determined generation matches the + // top-level current blueprint generation. + fn determine_nexus_generation( + &self, + image_source: &BlueprintZoneImageSource, + ) -> Result { + // If any other Nexus in the blueprint has the same image source, + // use it. Otherwise, use the highest generation number + 1. + // + // TODO: This will check the parent blueprint, but perhaps should + // also be checking all "pending" updates in "sled_editors". + // If we are adding "multiple new nexus zones" in a blueprint, + // they'll all happen to get a generation number equal to "the previous + // highest generation, plus 1". But if, for some weird reason, + // we added multiple Nexuses with different new image sources in a single + // blueprint, they'd also get assigned the same generation (which should + // be a bug). + // + // In the meantime: There is a blippy check to verify that all Nexus + // zones with the same generation have the same image source. + let mut highest_seen_generation = None; + let mut same_image_nexus_generation = None; + + for (zone, nexus) in self + .parent_blueprint + .all_omicron_zones(BlueprintZoneDisposition::any) + .filter_map(|(_, z)| match &z.zone_type { + BlueprintZoneType::Nexus(nexus) => Some((z, nexus)), + _ => None, + }) + { + if zone.image_source == *image_source { + // If the image matches exactly, use it. + same_image_nexus_generation = Some(nexus.nexus_generation); + break; + } else if let Some(gen) = highest_seen_generation { + // Otherwise, use the generation number if it's the highest + // we've seen + if nexus.nexus_generation > gen { + highest_seen_generation = Some(nexus.nexus_generation); + } + } else { + // Use it regardless if it's the first generation number we've + // seen + highest_seen_generation = Some(nexus.nexus_generation); + } + } + + let determined_generation = match same_image_nexus_generation { + Some(gen) => Some(gen), + None => highest_seen_generation.map(|gen| gen.next()), + }; + + let Some(gen) = determined_generation else { + return Err(Error::NoNexusZonesInParentBlueprint); + }; + + Ok(gen) + } + + /// Adds a nexus zone on this sled. pub fn sled_add_zone_nexus( &mut self, sled_id: SledUuid, @@ -1561,31 +1658,36 @@ impl<'a> BlueprintBuilder<'a> { // check that we're if this builder is being used to make such a change, // that change is also reflected here in a new zone. Perhaps these // settings should be part of `Policy` instead? - let (external_tls, external_dns_servers) = self - .parent_blueprint - .all_omicron_zones(BlueprintZoneDisposition::any) - .find_map(|(_, z)| match &z.zone_type { - BlueprintZoneType::Nexus(nexus) => Some(( - nexus.external_tls, - nexus.external_dns_servers.clone(), - )), - _ => None, - }) - .ok_or(Error::NoNexusZonesInParentBlueprint)?; + let (external_tls, external_dns_servers) = + match self.determine_nexus_tls_dns_config() { + Some(config) => config, + None => { + return Err(Error::NoNexusZonesInParentBlueprint); + } + }; + + let nexus_generation = + self.determine_nexus_generation(&image_source)?; + self.sled_add_zone_nexus_with_config( sled_id, external_tls, external_dns_servers, image_source, + nexus_generation, ) } + /// Add a Nexus zone on this sled with a specific configuration. + /// + /// If possible, callers should prefer to use [Self::sled_add_zone_nexus] pub fn sled_add_zone_nexus_with_config( &mut self, sled_id: SledUuid, external_tls: bool, external_dns_servers: Vec, image_source: BlueprintZoneImageSource, + nexus_generation: Generation, ) -> Result<(), Error> { let nexus_id = self.rng.sled_rng(sled_id).next_zone(); let ExternalNetworkingChoice { @@ -1623,7 +1725,7 @@ impl<'a> BlueprintBuilder<'a> { nic, external_tls, external_dns_servers: external_dns_servers.clone(), - nexus_generation: Generation::new(), + nexus_generation, }); let filesystem_pool = self.sled_select_zpool(sled_id, zone_type.kind())?; @@ -3714,4 +3816,463 @@ pub mod test { logctx.cleanup_successful(); } + + /// Test nexus generation assignment logic for new zones + #[test] + fn test_nexus_generation_assignment_new_generation() { + static TEST_NAME: &str = + "test_nexus_generation_assignment_new_generation"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system that has no Nexus zones + let (example_system, blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nexus_count(0) + .build(); + verify_blueprint(&blueprint); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + // Get first sled + let sled_id = example_system + .input + .all_sled_ids(SledFilter::Commissioned) + .next() + .unwrap(); + let image_source = BlueprintZoneImageSource::InstallDataset; + + // Add first Nexus zone - should get generation 1 + builder + .sled_add_zone_nexus_with_config( + sled_id, + false, + vec![], + image_source.clone(), + builder.parent_blueprint().nexus_generation, + ) + .expect("failed to add nexus zone"); + + let blueprint1 = builder.build(); + verify_blueprint(&blueprint1); + + // Find the nexus zone and verify it has generation 1 + let nexus_zones: Vec<_> = blueprint1 + .all_omicron_zones(BlueprintZoneDisposition::any) + .filter_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => Some(nexus), + _ => None, + }) + .collect(); + + assert_eq!(nexus_zones.len(), 1); + assert_eq!(nexus_zones[0].nexus_generation, Generation::new()); + + logctx.cleanup_successful(); + } + + /// Test that adding a Nexus zone with the same image source as an existing + /// Nexus zone re-uses the same generation number + #[test] + fn test_nexus_generation_assignment_same_image_reuse() { + static TEST_NAME: &str = + "test_nexus_generation_assignment_same_image_reuse"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system that has one Nexus zone + let (example_system, blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nexus_count(1) + .build(); + verify_blueprint(&blueprint); + + // Get the generation of the existing nexus zone + let existing_nexus_gen = blueprint + .all_omicron_zones(BlueprintZoneDisposition::any) + .find_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => { + // We're gonna add a new Nexus with this source in a moment + // - we want to be sure this image_source matches. + assert_eq!( + zone.image_source, + BlueprintZoneImageSource::InstallDataset + ); + Some(nexus.nexus_generation) + } + _ => None, + }) + .expect("should have found existing nexus"); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + // Get a different sled + let sled_ids: Vec<_> = example_system + .input + .all_sled_ids(SledFilter::Commissioned) + .collect(); + let second_sled_id = sled_ids[1]; + let image_source = BlueprintZoneImageSource::InstallDataset; + + // Add another Nexus zone with same image source - should reuse generation + builder + .sled_add_zone_nexus_with_config( + second_sled_id, + false, + vec![], + image_source.clone(), + builder.parent_blueprint().nexus_generation, + ) + .expect("failed to add nexus zone"); + + let blueprint2 = builder.build(); + verify_blueprint(&blueprint2); + + // Find all nexus zones and verify they have the same generation + let nexus_zones: Vec<_> = blueprint2 + .all_omicron_zones(BlueprintZoneDisposition::any) + .filter_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => Some(nexus), + _ => None, + }) + .collect(); + + assert_eq!(nexus_zones.len(), 2); + assert_eq!(nexus_zones[0].nexus_generation, existing_nexus_gen); + assert_eq!(nexus_zones[1].nexus_generation, existing_nexus_gen); + + logctx.cleanup_successful(); + } + + /// Test nexus generation assignment logic for different image sources + #[test] + fn test_nexus_generation_assignment_different_image_increment() { + static TEST_NAME: &str = + "test_nexus_generation_assignment_different_image_increment"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system that has one Nexus zone + let (example_system, blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nexus_count(1) + .build(); + verify_blueprint(&blueprint); + + // Get the generation of the existing nexus zone + let existing_nexus_gen = blueprint + .all_omicron_zones(BlueprintZoneDisposition::any) + .find_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => { + assert_eq!( + zone.image_source, + BlueprintZoneImageSource::InstallDataset + ); + Some(nexus.nexus_generation) + } + _ => None, + }) + .expect("should have found existing nexus"); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + // Get a different sled + let sled_ids: Vec<_> = example_system + .input + .all_sled_ids(SledFilter::Commissioned) + .collect(); + let second_sled_id = sled_ids[1]; + + // Use a different image source (artifact vs install dataset) + let different_image_source = BlueprintZoneImageSource::Artifact { + version: BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("1.2.3.4"), + }, + hash: ArtifactHash([0x42; 32]), + }; + + // Add another Nexus zone with different image source - should increment generation + builder + .sled_add_zone_nexus(second_sled_id, different_image_source.clone()) + .expect("failed to add nexus zone"); + + let blueprint2 = builder.build(); + verify_blueprint(&blueprint2); + + // Find all nexus zones and verify generations + let mut nexus_zones: Vec<_> = blueprint2 + .all_omicron_zones(BlueprintZoneDisposition::any) + .filter_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => Some((zone, nexus)), + _ => None, + }) + .collect(); + + // Sort by generation to ensure predictable ordering + nexus_zones.sort_by_key(|(_, nexus)| nexus.nexus_generation); + + assert_eq!(nexus_zones.len(), 2); + assert_eq!(nexus_zones[0].1.nexus_generation, existing_nexus_gen); + assert_eq!( + nexus_zones[1].1.nexus_generation, + existing_nexus_gen.next() + ); + + // Verify image sources are different + assert_eq!( + nexus_zones[0].0.image_source, + BlueprintZoneImageSource::InstallDataset + ); + assert_eq!(nexus_zones[1].0.image_source, different_image_source); + + logctx.cleanup_successful(); + } + + /// Test nexus generation assignment logic with mixed old/new image sources + /// + /// Tests a scenario where we restore redundancy with existing image source + /// while also adding zones with new image source for upgrade. + #[test] + fn test_nexus_generation_assignment_multiple_generations() { + static TEST_NAME: &str = + "test_nexus_generation_assignment_multiple_generations"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system with one Nexus zone using the install dataset as an image source + let (example_system, blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(3) + .nexus_count(1) + .build(); + verify_blueprint(&blueprint); + + // Get the existing nexus zone's generation (should be generation 1) + let existing_nexus_gen = blueprint + .all_omicron_zones(BlueprintZoneDisposition::any) + .find_map(|(_, zone)| match &zone.zone_type { + BlueprintZoneType::Nexus(nexus) => Some(nexus.nexus_generation), + _ => None, + }) + .expect("should have found existing nexus"); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + let sled_ids: Vec<_> = example_system + .input + .all_sled_ids(SledFilter::Commissioned) + .collect(); + + // Define image sources: A (same as existing Nexus) and B (new) + let image_source_a = BlueprintZoneImageSource::InstallDataset; + let image_source_b = BlueprintZoneImageSource::Artifact { + version: BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("2.0.0"), + }, + hash: ArtifactHash([0x11; 32]), + }; + + // In a single BlueprintBuilder step, add: + // 1. One zone with image source A (should reuse existing generation) + // 2. One zone with image source B (should get existing generation + 1) + builder + .sled_add_zone_nexus(sled_ids[1], image_source_a.clone()) + .expect("failed to add nexus zone with image source A"); + builder + .sled_add_zone_nexus(sled_ids[2], image_source_b.clone()) + .expect("failed to add nexus zone with image source B"); + + let blueprint2 = builder.build(); + verify_blueprint(&blueprint2); + + // Collect all nexus zones and organize by image source + let mut nexus_by_image: std::collections::HashMap< + BlueprintZoneImageSource, + Vec, + > = std::collections::HashMap::new(); + + for (_, zone) in + blueprint2.all_omicron_zones(BlueprintZoneDisposition::any) + { + if let BlueprintZoneType::Nexus(nexus) = &zone.zone_type { + nexus_by_image + .entry(zone.image_source.clone()) + .or_insert_with(Vec::new) + .push(nexus.nexus_generation); + } + } + + // Should have 2 image sources now + assert_eq!(nexus_by_image.len(), 2); + + // Image source A should have 2 zones (original + new) with same generation + let image_a_gens = nexus_by_image.get(&image_source_a).unwrap(); + assert_eq!(image_a_gens.len(), 2); + assert_eq!(image_a_gens[0], existing_nexus_gen); + assert_eq!(image_a_gens[1], existing_nexus_gen); + + // Image source B should have 1 zone with next generation + let image_b_gens = nexus_by_image.get(&image_source_b).unwrap(); + assert_eq!(image_b_gens.len(), 1); + assert_eq!(image_b_gens[0], existing_nexus_gen.next()); + + logctx.cleanup_successful(); + } + + /// Test that the validation which normally occurs as a part of + /// "sled_add_zone_nexus" - namely, the invocation of + /// "determine_nexus_generation" - throws expected errors when the + /// "next Nexus zone" generation does not match the parent blueprint's + /// value of "nexus generation". + #[test] + fn test_nexus_generation_blueprint_validation() { + static TEST_NAME: &str = "test_nexus_generation_blueprint_validation"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system that has one Nexus zone + let (example_system, mut blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nexus_count(1) + .build(); + verify_blueprint(&blueprint); + + // Manually modify the blueprint to create a mismatch: + // Set the top-level nexus_generation to 2, but keep the zone generation + // at 1 + blueprint.nexus_generation = Generation::new().next(); + + let builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + // Same as existing + let image_source = BlueprintZoneImageSource::InstallDataset; + + // Try to add another Nexus zone with same image source This should fail + // because existing zone has generation 1 but blueprint has generation 2 + let result = builder.determine_nexus_generation(&image_source); + + match result { + Err(Error::OldImageNexusGenerationMismatch { + expected, + actual, + }) => { + // Blueprint generation + assert_eq!(expected, Generation::new().next()); + // Zone generation + assert_eq!(actual, Generation::new()); + } + other => panic!( + "Expected OldImageNexusGenerationMismatch error, got: {:?}", + other + ), + } + + logctx.cleanup_successful(); + } + + /// Test nexus generation validation for new image source + #[test] + fn test_nexus_generation_blueprint_validation_new_image() { + static TEST_NAME: &str = + "test_nexus_generation_blueprint_validation_new_image"; + let logctx = test_setup_log(TEST_NAME); + let mut rng = SimRngState::from_seed(TEST_NAME); + + // Start with a system that has one Nexus zone + let (example_system, mut blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nexus_count(1) + .build(); + verify_blueprint(&blueprint); + + // The zone has generation 1 and blueprint has generation 1 + // Now modify the blueprint generation to be different from what + // the new image source logic would expect + // + // Set to generation 3 + blueprint.nexus_generation = Generation::new().next().next(); + + let builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &example_system.input, + &example_system.collection, + "test", + rng.next_planner_rng(), + ) + .expect("failed to create builder"); + + // Use a different image source (this should get existing generation + 1 + // = 2) + let different_image_source = BlueprintZoneImageSource::Artifact { + version: BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("2.0.0"), + }, + hash: ArtifactHash([0x42; 32]), + }; + + // Try to add a Nexus zone with different image source This should fail + // because the calculated generation (2) doesn't match blueprint + // generation + 1 (4) + let result = + builder.determine_nexus_generation(&different_image_source); + + match result { + Err(Error::NewImageNexusGenerationMismatch { + expected, + actual, + }) => { + // Blueprint generation + 1 = 4 + assert_eq!(expected, Generation::new().next().next().next()); + // Calculated generation = 2 + assert_eq!(actual, Generation::new().next()); + } + other => panic!( + "Expected NewImageNexusGenerationMismatch error, got: {:?}", + other + ), + } + + logctx.cleanup_successful(); + } } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index ce793980b2c..796be8c5449 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -4,6 +4,7 @@ //! Example blueprints +use std::collections::BTreeSet; use std::fmt; use std::hash::Hash; use std::net::IpAddr; @@ -480,12 +481,17 @@ impl ExampleSystemBuilder { for _ in 0..nexus_count .on(discretionary_ix, discretionary_sled_count) { + let external_tls = false; + let external_dns_servers = vec![]; + let nexus_generation = + builder.parent_blueprint().nexus_generation; builder .sled_add_zone_nexus_with_config( sled_id, - false, - vec![], + external_tls, + external_dns_servers, image_source.clone(), + nexus_generation, ) .unwrap(); } @@ -547,6 +553,22 @@ impl ExampleSystemBuilder { } let blueprint = builder.build(); + + // Find and set the set of active Nexuses + let active_nexus_zone_ids: BTreeSet<_> = blueprint + .sleds + .values() + .flat_map(|sled_cfg| sled_cfg.zones.iter()) + .filter_map(|zone| match &zone.zone_type { + nexus_types::deployment::BlueprintZoneType::Nexus(_) => { + Some(zone.id) + } + _ => None, + }) + .collect(); + input_builder.set_active_nexus_zones(active_nexus_zone_ids.clone()); + system.set_active_nexus_zones(active_nexus_zone_ids); + for sled_cfg in blueprint.sleds.values() { for zone in sled_cfg.zones.iter() { let service_id = zone.id; diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 4f98013e5d6..e34885577b4 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -31,6 +31,7 @@ use nexus_types::deployment::BlueprintPhysicalDiskDisposition; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; +use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::CockroachDbSettings; @@ -42,9 +43,10 @@ use nexus_types::deployment::SledFilter; use nexus_types::deployment::TufRepoContentsError; use nexus_types::deployment::ZpoolFilter; use nexus_types::deployment::{ - CockroachdbUnsafeToShutdown, PlanningAddStepReport, - PlanningCockroachdbSettingsStepReport, PlanningDecommissionStepReport, - PlanningExpungeStepReport, PlanningMgsUpdatesStepReport, + CockroachdbUnsafeToShutdown, NexusGenerationBumpWaitingOn, + PlanningAddStepReport, PlanningCockroachdbSettingsStepReport, + PlanningDecommissionStepReport, PlanningExpungeStepReport, + PlanningMgsUpdatesStepReport, PlanningNexusGenerationBumpReport, PlanningNoopImageSourceStepReport, PlanningReport, PlanningZoneUpdatesStepReport, ZoneAddWaitingOn, ZoneUnsafeToShutdown, ZoneUpdatesWaitingOn, @@ -53,6 +55,7 @@ use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledState; use nexus_types::inventory::Collection; +use omicron_common::api::external::Generation; use omicron_common::disk::M2Slot; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; @@ -111,6 +114,40 @@ const NUM_CONCURRENT_MGS_UPDATES: usize = 1; /// A receipt that `check_input_validity` has been run prior to planning. struct InputChecked; +// Details of why a zone has not yet propagated from blueprint to sled inventory +#[derive(Debug)] +#[expect(dead_code)] +struct ZonePropagationIncomplete<'a> { + zone_id: OmicronZoneUuid, + zone_kind: ZoneKind, + reason: ZonePropagationStatus<'a>, +} + +#[derive(Debug)] +#[expect(dead_code)] +enum ZonePropagationStatus<'a> { + // The current blueprint and the sled inventory disagree + // about the image source for a zone. + // + // This can mean that the sled inventory is out-of-date, or + // that a different blueprint has been applied. + ImageSourceMismatch { + bp_image_source: &'a BlueprintZoneImageSource, + inv_image_source: &'a OmicronZoneImageSource, + }, + // Although this zone appears in the blueprint, it does + // not exist on the sled's inventory. + MissingInInventory { + bp_image_source: &'a BlueprintZoneImageSource, + }, + // The last reconciliation attempt for this zone failed + ReconciliationError { + bp_image_source: &'a BlueprintZoneImageSource, + inv_image_source: &'a OmicronZoneImageSource, + message: &'a str, + }, +} + pub struct Planner<'a> { log: Logger, input: &'a PlanningInput, @@ -232,6 +269,10 @@ impl<'a> Planner<'a> { self.do_plan_zone_updates(&mgs_updates)? }; + // We may need to bump the top-level Nexus generation number + // to update Nexus zones. + let nexus_generation_bump = self.do_plan_nexus_generation_update()?; + // CockroachDB settings aren't dependent on zones, so they can be // planned independently of the rest of the system. let cockroachdb_settings = self.do_plan_cockroachdb_settings(); @@ -245,6 +286,7 @@ impl<'a> Planner<'a> { add, mgs_updates, zone_updates, + nexus_generation_bump, cockroachdb_settings, }) } @@ -975,55 +1017,98 @@ impl<'a> Planner<'a> { DiscretionaryOmicronZone::Nexus, DiscretionaryOmicronZone::Oximeter, ] { - let num_zones_to_add = - self.num_additional_zones_needed(zone_kind, report); - if num_zones_to_add == 0 { - continue; - } - // We need to add at least one zone; construct our `zone_placement` - // (or reuse the existing one if a previous loop iteration already - // created it). - let zone_placement = zone_placement.get_or_insert_with(|| { - // This constructs a picture of the sleds as we currently - // understand them, as far as which sleds have discretionary - // zones. This will remain valid as we loop through the - // `zone_kind`s in this function, as any zone additions will - // update the `zone_placement` heap in-place. - let current_discretionary_zones = self - .input - .all_sled_resources(SledFilter::Discretionary) - .filter(|(sled_id, _)| { - !report.sleds_waiting_for_ntp_zone.contains(&sled_id) - }) - .map(|(sled_id, sled_resources)| { - OmicronZonePlacementSledState { - sled_id, - num_zpools: sled_resources - .all_zpools(ZpoolFilter::InService) - .count(), - discretionary_zones: self - .blueprint - .current_sled_zones( - sled_id, - BlueprintZoneDisposition::is_in_service, - ) - .filter_map(|zone| { - DiscretionaryOmicronZone::from_zone_type( - &zone.zone_type, - ) - }) - .collect(), + // Our goal here is to make sure that if we have less redundancy for + // discretionary zones than needed, we deploy additional zones. + // + // For most zone types, we only care about the total count of that + // kind of zone, regardless of image. In contrast, for Nexus, we may + // need to reach a minimum redundancy count for multiple zone images + // (new and old) during a handoff. + let image_sources = match zone_kind { + DiscretionaryOmicronZone::Nexus => { + let new_image = self + .input + .tuf_repo() + .description() + .zone_image_source(zone_kind.into())?; + + let mut images = vec![]; + if let Some(nexus_in_charge_image) = + self.lookup_current_nexus_image() + { + // We may still want to deploy the old image alongside + // the new image: if we're running the "old version of a + // Nexus" currently, we need to ensure we have + // redundancy before the handoff completes. + if nexus_in_charge_image != new_image { + images.push(nexus_in_charge_image); } - }); - OmicronZonePlacement::new(current_discretionary_zones) - }); - self.add_discretionary_zones( - zone_placement, - zone_kind, - num_zones_to_add, - mgs_updates, - report, - )?; + } + images.push(new_image); + images + } + _ => { + vec![self.image_source_for_new_zone( + zone_kind.into(), + mgs_updates, + )?] + } + }; + + for image_source in image_sources { + let num_zones_to_add = self.num_additional_zones_needed( + zone_kind, + &image_source, + report, + ); + if num_zones_to_add == 0 { + continue; + } + // We need to add at least one zone; construct our `zone_placement` + // (or reuse the existing one if a previous loop iteration already + // created it). + let zone_placement = zone_placement.get_or_insert_with(|| { + // This constructs a picture of the sleds as we currently + // understand them, as far as which sleds have discretionary + // zones. This will remain valid as we loop through the + // `zone_kind`s in this function, as any zone additions will + // update the `zone_placement` heap in-place. + let current_discretionary_zones = self + .input + .all_sled_resources(SledFilter::Discretionary) + .filter(|(sled_id, _)| { + !report.sleds_waiting_for_ntp_zone.contains(&sled_id) + }) + .map(|(sled_id, sled_resources)| { + OmicronZonePlacementSledState { + sled_id, + num_zpools: sled_resources + .all_zpools(ZpoolFilter::InService) + .count(), + discretionary_zones: self + .blueprint + .current_sled_zones( + sled_id, + BlueprintZoneDisposition::is_in_service, + ) + .filter_map(|zone| { + DiscretionaryOmicronZone::from_zone_type( + &zone.zone_type, + ) + }) + .collect(), + } + }); + OmicronZonePlacement::new(current_discretionary_zones) + }); + self.add_discretionary_zones( + zone_placement, + zone_kind, + num_zones_to_add, + image_source, + report, + )?; + } } Ok(()) @@ -1033,7 +1118,8 @@ impl<'a> Planner<'a> { /// additional zones needed of the given `zone_kind` to satisfy the policy. fn num_additional_zones_needed( &mut self, - zone_kind: DiscretionaryOmicronZone, + discretionary_zone_kind: DiscretionaryOmicronZone, + image_source: &BlueprintZoneImageSource, report: &mut PlanningAddStepReport, ) -> usize { // Count the number of `kind` zones on all in-service sleds. This @@ -1042,7 +1128,7 @@ impl<'a> Planner<'a> { // decommissioned. let mut num_existing_kind_zones = 0; for sled_id in self.input.all_sled_ids(SledFilter::InService) { - let zone_kind = ZoneKind::from(zone_kind); + let zone_kind = ZoneKind::from(discretionary_zone_kind); // Internal DNS is special: if we have an expunged internal DNS zone // that might still be running, we want to count it here: we can't @@ -1057,11 +1143,20 @@ impl<'a> Planner<'a> { num_existing_kind_zones += self .blueprint .current_sled_zones(sled_id, disposition_filter) - .filter(|z| z.zone_type.kind() == zone_kind) + .filter(|z| { + let matches_kind = z.zone_type.kind() == zone_kind; + let matches_image = z.image_source == *image_source; + match discretionary_zone_kind { + DiscretionaryOmicronZone::Nexus => { + matches_kind && matches_image + } + _ => matches_kind, + } + }) .count(); } - let target_count = match zone_kind { + let target_count = match discretionary_zone_kind { DiscretionaryOmicronZone::BoundaryNtp => { self.input.target_boundary_ntp_zone_count() } @@ -1103,7 +1198,7 @@ impl<'a> Planner<'a> { target_count.saturating_sub(num_existing_kind_zones); if num_zones_to_add == 0 { report.sufficient_zones_exist( - ZoneKind::from(zone_kind).report_str(), + ZoneKind::from(discretionary_zone_kind).report_str(), target_count, num_existing_kind_zones, ); @@ -1121,7 +1216,7 @@ impl<'a> Planner<'a> { zone_placement: &mut OmicronZonePlacement, kind: DiscretionaryOmicronZone, num_zones_to_add: usize, - mgs_updates: &PlanningMgsUpdatesStepReport, + image_source: BlueprintZoneImageSource, report: &mut PlanningAddStepReport, ) -> Result<(), Error> { for i in 0..num_zones_to_add { @@ -1141,46 +1236,45 @@ impl<'a> Planner<'a> { } }; - let image_source = - self.image_source_for_new_zone(kind.into(), mgs_updates)?; + let image = image_source.clone(); match kind { DiscretionaryOmicronZone::BoundaryNtp => { self.blueprint.sled_promote_internal_ntp_to_boundary_ntp( - sled_id, - image_source, + sled_id, image, )? } - DiscretionaryOmicronZone::Clickhouse => self - .blueprint - .sled_add_zone_clickhouse(sled_id, image_source)?, + DiscretionaryOmicronZone::Clickhouse => { + self.blueprint.sled_add_zone_clickhouse(sled_id, image)? + } DiscretionaryOmicronZone::ClickhouseKeeper => self .blueprint - .sled_add_zone_clickhouse_keeper(sled_id, image_source)?, + .sled_add_zone_clickhouse_keeper(sled_id, image)?, DiscretionaryOmicronZone::ClickhouseServer => self .blueprint - .sled_add_zone_clickhouse_server(sled_id, image_source)?, - DiscretionaryOmicronZone::CockroachDb => self - .blueprint - .sled_add_zone_cockroachdb(sled_id, image_source)?, + .sled_add_zone_clickhouse_server(sled_id, image)?, + DiscretionaryOmicronZone::CockroachDb => { + self.blueprint.sled_add_zone_cockroachdb(sled_id, image)? + } DiscretionaryOmicronZone::CruciblePantry => self .blueprint - .sled_add_zone_crucible_pantry(sled_id, image_source)?, - DiscretionaryOmicronZone::InternalDns => self - .blueprint - .sled_add_zone_internal_dns(sled_id, image_source)?, - DiscretionaryOmicronZone::ExternalDns => self - .blueprint - .sled_add_zone_external_dns(sled_id, image_source)?, + .sled_add_zone_crucible_pantry(sled_id, image)?, + DiscretionaryOmicronZone::InternalDns => { + self.blueprint.sled_add_zone_internal_dns(sled_id, image)? + } + DiscretionaryOmicronZone::ExternalDns => { + self.blueprint.sled_add_zone_external_dns(sled_id, image)? + } DiscretionaryOmicronZone::Nexus => { - self.blueprint.sled_add_zone_nexus(sled_id, image_source)? + self.blueprint.sled_add_zone_nexus(sled_id, image)? + } + DiscretionaryOmicronZone::Oximeter => { + self.blueprint.sled_add_zone_oximeter(sled_id, image)? } - DiscretionaryOmicronZone::Oximeter => self - .blueprint - .sled_add_zone_oximeter(sled_id, image_source)?, }; report.discretionary_zone_placed( sled_id, ZoneKind::from(kind).report_str(), + &image_source, ); } @@ -1262,13 +1356,12 @@ impl<'a> Planner<'a> { Ok(PlanningMgsUpdatesStepReport::new(pending_updates)) } - /// Update at most one existing zone to use a new image source. - fn do_plan_zone_updates( - &mut self, - mgs_updates: &PlanningMgsUpdatesStepReport, - ) -> Result { - let mut report = PlanningZoneUpdatesStepReport::new(); - + // Returns the zones which appear in the blueprint on commissioned sleds, + // but which have not been reported by the latest reconciliation result from + // inventory. + fn get_zones_not_yet_propagated_to_inventory( + &self, + ) -> Vec> { // We are only interested in non-decommissioned sleds. let sleds = self .input @@ -1283,31 +1376,7 @@ impl<'a> Planner<'a> { .map(|(z, sa_result)| (z.id, (&z.image_source, sa_result))) .collect::>(); - #[derive(Debug)] - #[expect(dead_code)] - struct ZoneCurrentlyUpdating<'a> { - zone_id: OmicronZoneUuid, - zone_kind: ZoneKind, - reason: UpdatingReason<'a>, - } - - #[derive(Debug)] - #[expect(dead_code)] - enum UpdatingReason<'a> { - ImageSourceMismatch { - bp_image_source: &'a BlueprintZoneImageSource, - inv_image_source: &'a OmicronZoneImageSource, - }, - MissingInInventory { - bp_image_source: &'a BlueprintZoneImageSource, - }, - ReconciliationError { - bp_image_source: &'a BlueprintZoneImageSource, - inv_image_source: &'a OmicronZoneImageSource, - message: &'a str, - }, - } - + let mut updating = vec![]; for &sled_id in &sleds { // Build a list of zones currently in the blueprint but where // inventory has a mismatch or does not know about the zone. @@ -1315,7 +1384,7 @@ impl<'a> Planner<'a> { // What about the case where a zone is in inventory but not in the // blueprint? See // https://github.com/oxidecomputer/omicron/issues/8589. - let zones_currently_updating = self + let mut zones_currently_updating = self .blueprint .current_sled_zones( sled_id, @@ -1338,13 +1407,14 @@ impl<'a> Planner<'a> { ConfigReconcilerInventoryResult::Ok, )) => { // The inventory and blueprint image sources differ. - Some(ZoneCurrentlyUpdating { + Some(ZonePropagationIncomplete { zone_id: zone.id, zone_kind: zone.kind(), - reason: UpdatingReason::ImageSourceMismatch { - bp_image_source: &zone.image_source, - inv_image_source, - }, + reason: + ZonePropagationStatus::ImageSourceMismatch { + bp_image_source: &zone.image_source, + inv_image_source, + }, }) } Some(( @@ -1354,40 +1424,60 @@ impl<'a> Planner<'a> { // The inventory reports this zone but there was an // error reconciling it (most likely an error // starting the zone). - Some(ZoneCurrentlyUpdating { + Some(ZonePropagationIncomplete { zone_id: zone.id, zone_kind: zone.kind(), - reason: UpdatingReason::ReconciliationError { - bp_image_source: &zone.image_source, - inv_image_source, - message, - }, + reason: + ZonePropagationStatus::ReconciliationError { + bp_image_source: &zone.image_source, + inv_image_source, + message, + }, }) } None => { // The blueprint has a zone that inventory does not have. - Some(ZoneCurrentlyUpdating { + Some(ZonePropagationIncomplete { zone_id: zone.id, zone_kind: zone.kind(), - reason: UpdatingReason::MissingInInventory { - bp_image_source: &zone.image_source, - }, + reason: + ZonePropagationStatus::MissingInInventory { + bp_image_source: &zone.image_source, + }, }) } } }) .collect::>(); + updating.append(&mut zones_currently_updating); + } + updating + } - if !zones_currently_updating.is_empty() { - info!( - self.log, "some zones not yet up-to-date"; - "sled_id" => %sled_id, - "zones_currently_updating" => ?zones_currently_updating, - ); - return Ok(report); - } + /// Update at most one existing zone to use a new image source. + fn do_plan_zone_updates( + &mut self, + mgs_updates: &PlanningMgsUpdatesStepReport, + ) -> Result { + let mut report = PlanningZoneUpdatesStepReport::new(); + + let zones_currently_updating = + self.get_zones_not_yet_propagated_to_inventory(); + if !zones_currently_updating.is_empty() { + info!( + self.log, "some zones not yet up-to-date"; + "zones_currently_updating" => ?zones_currently_updating, + ); + return Ok(report); } + // We are only interested in non-decommissioned sleds. + let sleds = self + .input + .all_sleds(SledFilter::Commissioned) + .map(|(id, _details)| id) + .collect::>(); + // Find out of date zones, as defined by zones whose image source does // not match what it should be based on our current target release. let target_release = self.input.tuf_repo().description(); @@ -1439,10 +1529,7 @@ impl<'a> Planner<'a> { if !self.can_zone_be_shut_down_safely(zone, &mut report) { return false; } - match self.is_zone_ready_for_update( - zone.zone_type.kind(), - mgs_updates, - ) { + match self.is_zone_ready_for_update(mgs_updates) { Ok(true) => true, Ok(false) => false, Err(err) => { @@ -1761,6 +1848,136 @@ impl<'a> Planner<'a> { Ok(reasons) } + // Determines whether or not the top-level "nexus_generation" + // value should be increased. + // + // Doing so will be a signal for all running Nexus instances at + // lower versions to start quiescing, and to perform handoff. + fn do_plan_nexus_generation_update( + &mut self, + ) -> Result { + let mut report = PlanningNexusGenerationBumpReport::new(); + + // Nexus can only be updated if all non-Nexus zones have been + // updated, i.e., their image source is an artifact from the new + // repo. + let new_repo = self.input.tuf_repo().description(); + + // If we don't actually have a TUF repo here, we can't do + // updates anyway; any return value is fine. + if new_repo.tuf_repo().is_none() { + return Ok(report); + } + + // Check that all in-service zones (other than Nexus) on all + // sleds have an image source consistent with `new_repo`. + for sled_id in self.blueprint.sled_ids_with_zones() { + for z in self.blueprint.current_sled_zones( + sled_id, + BlueprintZoneDisposition::is_in_service, + ) { + let kind = z.zone_type.kind(); + if kind != ZoneKind::Nexus + && z.image_source != new_repo.zone_image_source(kind)? + { + report.set_waiting_on( + NexusGenerationBumpWaitingOn::NonNexusZoneUpdate, + ); + return Ok(report); + } + } + } + + // In order to do a handoff, there must be Nexus instances at the + // proposed generation number. Tese Nexuses must also have records in + // "db_metadata_nexus" (this is verified by checking that new Nexuses + // have entries in "self.input.not_yet_nexus_zones"). + let current_generation = self.blueprint.nexus_generation(); + let proposed_generation = self.blueprint.nexus_generation().next(); + let mut old_nexuses_at_current_gen = 0; + let mut nexuses_at_proposed_gen = 0; + let mut nexuses_at_proposed_gen_missing_metadata_record = 0; + for sled_id in self.blueprint.sled_ids_with_zones() { + for z in self.blueprint.current_sled_zones( + sled_id, + BlueprintZoneDisposition::is_in_service, + ) { + if let BlueprintZoneType::Nexus(nexus_zone) = &z.zone_type { + if nexus_zone.nexus_generation == proposed_generation { + nexuses_at_proposed_gen += 1; + if !self.input.not_yet_nexus_zones().contains(&z.id) { + nexuses_at_proposed_gen_missing_metadata_record += + 1; + } + } + + if nexus_zone.nexus_generation == current_generation + && z.image_source + != new_repo.zone_image_source(z.zone_type.kind())? + { + old_nexuses_at_current_gen += 1; + } + } + } + } + + if old_nexuses_at_current_gen == 0 { + // If all the current-generation Nexuses are "up-to-date", then we may have + // just completed handoff successfully. In this case, there's nothing to report. + return Ok(report); + } else if nexuses_at_proposed_gen < self.input.target_nexus_zone_count() + { + // If there aren't enough Nexuses at the next generation, quiescing could + // be a dangerous operation. Blueprint execution should be able to continue + // even if the new Nexuses haven't started, but to be conservative, we'll wait + // for the target count. + report + .set_waiting_on(NexusGenerationBumpWaitingOn::NewNexusBringup); + return Ok(report); + } else if nexuses_at_proposed_gen_missing_metadata_record > 0 { + // There are enough Nexuses at the target generation, but not all of + // them have records yet. Blueprint execution should fix this, by + // creating these records. + report.set_waiting_on( + NexusGenerationBumpWaitingOn::NexusDatabasePropagation, + ); + return Ok(report); + } + + // Confirm that all blueprint zones have propagated to inventory + let zones_currently_updating = + self.get_zones_not_yet_propagated_to_inventory(); + if !zones_currently_updating.is_empty() { + info!( + self.log, "some zones not yet up-to-date"; + "zones_currently_updating" => ?zones_currently_updating, + ); + report + .set_waiting_on(NexusGenerationBumpWaitingOn::ZonePropagation); + return Ok(report); + } + + // If we're here: + // - There's a new repo + // - The current generation of Nexuses are considered "out-of-date" + // - There are Nexuses running with "current generation + 1" + // - All non-Nexus zones have updated + // - All other blueprint zones have propagated to inventory + // + // If all of these are true, the "zone update" portion of the planner + // has completed, aside from Nexus, and we're ready for old Nexuses + // to start quiescing. + // + // Blueprint planning and execution will be able to continue past this + // point, for the purposes of restoring redundancy, expunging sleds, + // etc. However, making this committment will also halt the creation of + // new sagas temporarily, as handoff from old to new Nexuses occurs. + self.blueprint.set_nexus_generation(proposed_generation); + report.set_next_generation(proposed_generation); + + Ok(report) + } + fn do_plan_cockroachdb_settings( &mut self, ) -> PlanningCockroachdbSettingsStepReport { @@ -1861,63 +2078,50 @@ impl<'a> Planner<'a> { zone_kind: ZoneKind, mgs_updates: &PlanningMgsUpdatesStepReport, ) -> Result { - let source_repo = - if self.is_zone_ready_for_update(zone_kind, mgs_updates)? { - self.input.tuf_repo().description() - } else { - self.input.old_repo().description() - }; + let source_repo = if self.is_zone_ready_for_update(mgs_updates)? { + self.input.tuf_repo().description() + } else { + self.input.old_repo().description() + }; source_repo.zone_image_source(zone_kind) } - /// Return `true` iff a zone of the given kind is ready to be updated; - /// i.e., its dependencies have been updated. + /// Return `true` iff a zone is ready to be updated; i.e., its dependencies + /// have been updated. fn is_zone_ready_for_update( &self, - zone_kind: ZoneKind, mgs_updates: &PlanningMgsUpdatesStepReport, ) -> Result { - // We return false regardless of `zone_kind` if there are still + // We return false for all zone kinds if there are still // pending updates for components earlier in the update ordering // than zones: RoT bootloader / RoT / SP / Host OS. if !mgs_updates.is_empty() { return Ok(false); } - match zone_kind { - ZoneKind::Nexus => { - // Nexus can only be updated if all non-Nexus zones have been - // updated, i.e., their image source is an artifact from the new - // repo. - let new_repo = self.input.tuf_repo().description(); - - // If we don't actually have a TUF repo here, we can't do - // updates anyway; any return value is fine. - if new_repo.tuf_repo().is_none() { - return Ok(false); - } + Ok(true) + } - // Check that all in-service zones (other than Nexus) on all - // sleds have an image source consistent with `new_repo`. - for sled_id in self.blueprint.sled_ids_with_zones() { - for z in self.blueprint.current_sled_zones( - sled_id, - BlueprintZoneDisposition::is_in_service, - ) { - let kind = z.zone_type.kind(); - if kind != ZoneKind::Nexus - && z.image_source - != new_repo.zone_image_source(kind)? - { - return Ok(false); - } - } + fn lookup_current_nexus_image(&self) -> Option { + // Look up the active Nexus zone in the blueprint to get its image + self.blueprint + .parent_blueprint() + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .find_map(|(_, blueprint_zone)| { + if self.input.active_nexus_zones().contains(&blueprint_zone.id) + { + Some(blueprint_zone.image_source.clone()) + } else { + None } + }) + } - Ok(true) - } - _ => Ok(true), // other zone kinds have no special dependencies - } + fn lookup_current_nexus_generation(&self) -> Option { + // Look up the active Nexus zone in the blueprint to get its generation + self.blueprint + .parent_blueprint() + .find_generation_for_nexus(self.input.active_nexus_zones()) } /// Return `true` iff we believe a zone can safely be shut down; e.g., any @@ -2098,6 +2302,53 @@ impl<'a> Planner<'a> { false } } + ZoneKind::Nexus => { + // Get the nexus_generation of the zone being considered for shutdown + let zone_nexus_generation = match &zone.zone_type { + BlueprintZoneType::Nexus(nexus_zone) => { + nexus_zone.nexus_generation + } + _ => unreachable!("zone kind is Nexus but type is not"), + }; + + // Get the generation of the currently-executing Nexus zones. + // + // This presumably includes the currently-executing Nexus where + // this logic is being considered. + let Some(current_gen) = self.lookup_current_nexus_generation() + else { + // If we don't know the current Nexus zone ID, or its + // generation, we can't perform the handoff safety check. + report.unsafe_zone( + zone, + Nexus { + zone_generation: zone_nexus_generation, + current_nexus_generation: None, + }, + ); + return false; + }; + + // We need to prevent old Nexus zones from shutting themselves + // down. In other words: it's only safe to shut down if handoff + // has occurred. + // + // That only happens when the current generation of Nexus (the + // one running right now) is greater than the zone we're + // considering expunging. + if current_gen <= zone_nexus_generation { + report.unsafe_zone( + zone, + Nexus { + zone_generation: zone_nexus_generation, + current_nexus_generation: Some(current_gen), + }, + ); + return false; + } + + true + } _ => true, // other zone kinds have no special safety checks } } @@ -2189,6 +2440,7 @@ pub(crate) mod test { ClickhousePolicy { version: 0, mode, time_created: Utc::now() } } + #[track_caller] pub(crate) fn assert_planning_makes_no_changes( log: &Logger, blueprint: &Blueprint, @@ -2205,7 +2457,8 @@ pub(crate) mod test { PlannerRng::from_entropy(), ) .expect("created planner"); - let child_blueprint = planner.plan().expect("planning succeeded"); + let child_blueprint = + planner.plan().expect("planning should have succeded"); verify_blueprint(&child_blueprint); let summary = child_blueprint.diff_since_blueprint(&blueprint); eprintln!( @@ -3238,7 +3491,10 @@ pub(crate) mod test { .resources .zpools .iter_mut() - .next() + // Skip over the first disk - this is the one which hosts + // many of our zones, like Nexus, and is more complicated + // to expunge. + .nth(1) .unwrap() .1; expunged_disk.policy = PhysicalDiskPolicy::Expunged; @@ -3621,7 +3877,7 @@ pub(crate) mod test { .unwrap(); disk.policy = PhysicalDiskPolicy::Expunged; - let input = builder.build(); + let mut input = builder.build(); let blueprint2 = Planner::new_based_on( logctx.log.clone(), @@ -3679,6 +3935,12 @@ pub(crate) mod test { } assert_eq!(zone_kinds_on_pool, zone_kinds_added); + // This operation expunged the old Nexus, and added a new one. + // + // Update our input to "become the newer Nexus" to observe these + // changes. + set_current_nexus_to_highest_generation(&mut input, &blueprint2); + // Test a no-op planning iteration. assert_planning_makes_no_changes( &logctx.log, @@ -4185,7 +4447,7 @@ pub(crate) mod test { let logctx = test_setup_log(TEST_NAME); let (example, bp1) = - ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(0).build(); + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(1).build(); let collection = example.collection; let input = example.input; let mut builder = input.into_builder(); @@ -5628,8 +5890,8 @@ pub(crate) mod test { /// Ensure that dependent zones (here just Crucible Pantry) are updated /// before Nexus. #[test] - fn test_update_crucible_pantry() { - static TEST_NAME: &str = "update_crucible_pantry"; + fn test_update_crucible_pantry_before_nexus() { + static TEST_NAME: &str = "update_crucible_pantry_before_nexus"; let logctx = test_setup_log(TEST_NAME); let log = logctx.log.clone(); @@ -5736,18 +5998,18 @@ pub(crate) mod test { }; } - // Request another Nexus zone. - input_builder.policy_mut().target_nexus_zone_count = - input_builder.policy_mut().target_nexus_zone_count + 1; - let input = input_builder.build(); + // Nexus should deploy new zones, but keep the old ones running. + let expected_new_nexus_zones = + input_builder.policy_mut().target_nexus_zone_count; + example.input = input_builder.build(); - // Check that there is a new nexus zone that does *not* use the new - // artifact (since not all of its dependencies are updated yet). + // Check that there are new nexus zones deployed, though handoff is + // incomplete (since not all of its dependencies are updated yet). update_collection_from_blueprint(&mut example, &blueprint1); let blueprint2 = Planner::new_based_on( log.clone(), &blueprint1, - &input, + &example.input, "test_blueprint3", &example.collection, PlannerRng::from_seed((TEST_NAME, "bp3")), @@ -5757,6 +6019,7 @@ pub(crate) mod test { .expect("can't re-plan for new Nexus zone"); { let summary = blueprint2.diff_since_blueprint(&blueprint1); + let mut modified_sleds = 0; for sled in summary.diff.sleds.modified_values_diff() { assert!(sled.zones.removed.is_empty()); assert_eq!(sled.zones.added.len(), 1); @@ -5765,11 +6028,10 @@ pub(crate) mod test { &added.zone_type, BlueprintZoneType::Nexus(_) )); - assert!(matches!( - &added.image_source, - BlueprintZoneImageSource::InstallDataset - )); + assert_eq!(&added.image_source, &image_source); + modified_sleds += 1; } + assert_eq!(modified_sleds, expected_new_nexus_zones); } // We should now have three sets of expunge/add iterations for the @@ -5781,7 +6043,7 @@ pub(crate) mod test { let blueprint = Planner::new_based_on( log.clone(), &parent, - &input, + &example.input, &blueprint_name, &example.collection, PlannerRng::from_seed((TEST_NAME, &blueprint_name)), @@ -5853,17 +6115,32 @@ pub(crate) mod test { .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_old_nexus(z)) .count(), - NEXUS_REDUNDANCY + 1, + NEXUS_REDUNDANCY, + ); + assert_eq!( + blueprint8 + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .filter(|(_, z)| is_up_to_date_nexus(z)) + .count(), + NEXUS_REDUNDANCY, + ); + + // We have to pretend that we're running the "Newer Nexus" to shut down + // the old Nexuses. If we don't do this: it's as if handoff has not + // happened, and the old Nexuses cannot shut down. + set_current_nexus_to_highest_generation( + &mut example.input, + &blueprint8, ); let mut parent = blueprint8; - for i in 9..=16 { - update_collection_from_blueprint(&mut example, &parent); + for i in 9..=12 { + update_collection_from_blueprint(&mut example, &parent); let blueprint_name = format!("blueprint{i}"); let blueprint = Planner::new_based_on( log.clone(), &parent, - &input, + &example.input, &blueprint_name, &example.collection, PlannerRng::from_seed((TEST_NAME, &blueprint_name)), @@ -5874,41 +6151,72 @@ pub(crate) mod test { { let summary = blueprint.diff_since_blueprint(&parent); + assert!(summary.has_changes(), "No changes at iteration {i}"); for sled in summary.diff.sleds.modified_values_diff() { - if i % 2 == 1 { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); - } else { - assert!(sled.zones.removed.is_empty()); - assert_eq!(sled.zones.added.len(), 1); - let added = sled.zones.added.values().next().unwrap(); + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); + for modified_zone in sled.zones.modified_values_diff() { + // We're only modifying Nexus zones on the old image assert!(matches!( - &added.zone_type, + *modified_zone.zone_type.before, BlueprintZoneType::Nexus(_) )); - assert_eq!(added.image_source, image_source); + assert_eq!( + *modified_zone.image_source.before, + BlueprintZoneImageSource::InstallDataset + ); + + // If the zone was previously in-service, it gets + // expunged. + if modified_zone.disposition.before.is_in_service() { + assert!( + modified_zone.disposition.after.is_expunged(), + ); + } + + // If the zone was previously expunged and not ready for + // cleanup, it should be marked ready-for-cleanup + if modified_zone.disposition.before.is_expunged() + && !modified_zone + .disposition + .before + .is_ready_for_cleanup() + { + assert!( + modified_zone + .disposition + .after + .is_ready_for_cleanup(), + ); + } } } } - parent = blueprint; } // Everything's up-to-date in Kansas City! - let blueprint16 = parent; + let blueprint12 = parent; assert_eq!( - blueprint16 + blueprint12 .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_up_to_date_nexus(z)) .count(), - NEXUS_REDUNDANCY + 1, + NEXUS_REDUNDANCY, + ); + assert_eq!( + blueprint12 + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .filter(|(_, z)| is_old_nexus(z)) + .count(), + 0, ); - update_collection_from_blueprint(&mut example, &blueprint16); + update_collection_from_blueprint(&mut example, &blueprint12); assert_planning_makes_no_changes( &logctx.log, - &blueprint16, - &input, + &blueprint12, + &example.input, &example.collection, TEST_NAME, ); @@ -7073,6 +7381,62 @@ pub(crate) mod test { logctx.cleanup_successful(); } + // Updates the PlanningInput to pretend like we're running + // from whichever Nexuses have the highest "nexus_generation" value. + fn set_current_nexus_to_highest_generation( + input: &mut PlanningInput, + blueprint: &Blueprint, + ) { + // Grab the current generation from the input / blueprint + let mut current_gen = blueprint + .sleds + .values() + .find_map(|sled| { + for zone in &sled.zones { + if input.active_nexus_zones().contains(&zone.id) { + if let BlueprintZoneType::Nexus(nexus_config) = + &zone.zone_type + { + return Some(nexus_config.nexus_generation); + } + } + } + None + }) + .unwrap_or(Generation::new()); + + // Look for Nexuses with a higher generation anywhere in the blueprint + for (_, zone) in + blueprint.all_omicron_zones(BlueprintZoneDisposition::is_in_service) + { + if let BlueprintZoneType::Nexus(nexus_config) = &zone.zone_type { + if nexus_config.nexus_generation > current_gen { + current_gen = nexus_config.nexus_generation; + } + } + } + + // Set the active nexus zones as "the ones the highest generation" + input.set_active_nexus_zones( + blueprint + .sleds + .values() + .flat_map(|sled_config| { + sled_config.zones.iter().filter_map(|zone| { + if let BlueprintZoneType::Nexus(nexus_config) = + &zone.zone_type + { + if nexus_config.nexus_generation == current_gen { + return Some(zone.id); + } + } + None + }) + }) + .collect(), + ); + } + /// Ensure that planning to update all zones terminates. #[test] fn test_update_all_zones() { @@ -7130,13 +7494,13 @@ pub(crate) mod test { ), }; input_builder.policy_mut().tuf_repo = tuf_repo; - let input = input_builder.build(); + let mut input = input_builder.build(); /// Expected number of planner iterations required to converge. /// If incidental planner work changes this value occasionally, /// that's fine; but if we find we're changing it all the time, /// we should probably drop it and keep just the maximum below. - const EXP_PLANNING_ITERATIONS: usize = 57; + const EXP_PLANNING_ITERATIONS: usize = 55; /// Planning must not take more than this number of iterations. const MAX_PLANNING_ITERATIONS: usize = 100; @@ -7157,7 +7521,9 @@ pub(crate) mod test { ) .expect("can't create planner") .plan() - .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); + .unwrap_or_else(|err| { + panic!("can't re-plan after {i} iterations: {err}") + }); assert_eq!(blueprint.report.blueprint_id, blueprint.id); eprintln!("{}\n", blueprint.report); @@ -7189,9 +7555,503 @@ pub(crate) mod test { } } + // If there is a newer Nexus, we must jump to it to expunge + // the older Nexus zones. + set_current_nexus_to_highest_generation(&mut input, &blueprint); + parent = blueprint; } panic!("did not converge after {MAX_PLANNING_ITERATIONS} iterations"); } + + struct BlueprintGenerator { + log: Logger, + example: ExampleSystem, + blueprint: Blueprint, + rng: SimRngState, + target_release_generation: Generation, + } + + impl BlueprintGenerator { + fn new( + log: Logger, + example: ExampleSystem, + blueprint: Blueprint, + rng: SimRngState, + ) -> Self { + Self { + log, + example, + blueprint, + rng, + target_release_generation: Generation::new(), + } + } + + fn create_image_at_version( + version: &ArtifactVersion, + ) -> BlueprintZoneImageSource { + let fake_hash = ArtifactHash([0; 32]); + BlueprintZoneImageSource::Artifact { + version: BlueprintArtifactVersion::Available { + version: version.clone(), + }, + hash: fake_hash, + } + } + + // - Bumps the target_release_generation + // - Sets a new "tuf_repo" as part of the "example.input" + // - The system version is hard-coded as "2.0.0" + // - Sets artifacts in the repo to `artifacts` + fn set_new_tuf_repo_with_artifacts( + &mut self, + artifacts: Vec, + ) { + let mut input_builder = self.example.input.clone().into_builder(); + let fake_hash = ArtifactHash([0; 32]); + self.target_release_generation = + self.target_release_generation.next(); + + let tuf_repo = TufRepoPolicy { + target_release_generation: self.target_release_generation, + description: TargetReleaseDescription::TufRepo( + TufRepoDescription { + repo: TufRepoMeta { + hash: fake_hash, + targets_role_version: 0, + valid_until: Utc::now(), + system_version: Version::new(2, 0, 0), + file_name: String::from(""), + }, + artifacts, + }, + ), + }; + + input_builder.policy_mut().tuf_repo = tuf_repo; + self.example.input = input_builder.build(); + } + + fn set_old_tuf_repo_to_target(&mut self) { + let mut input_builder = self.example.input.clone().into_builder(); + input_builder.policy_mut().old_repo = + self.example.input.tuf_repo().clone(); + self.example.input = input_builder.build(); + } + + // Plans a new blueprint, validates it, and returns it + // + // Does not set the current blueprint to this new value + #[track_caller] + fn plan_new_blueprint(&mut self, name: &str) -> Blueprint { + let planner = Planner::new_based_on( + self.log.clone(), + &self.blueprint, + &self.example.input, + name, + &self.example.collection, + self.rng.next_planner_rng(), + ) + .expect("can't create planner"); + let bp = planner.plan().expect("planning succeeded"); + verify_blueprint(&bp); + bp + } + + // Asserts that a new blueprint, if generated, will make no changes + #[track_caller] + fn assert_child_bp_makes_no_changes( + &self, + child_blueprint: &Blueprint, + ) { + verify_blueprint(&child_blueprint); + let summary = child_blueprint.diff_since_blueprint(&self.blueprint); + assert_eq!( + summary.diff.sleds.added.len(), + 0, + "{}", + summary.display() + ); + assert_eq!( + summary.diff.sleds.removed.len(), + 0, + "{}", + summary.display() + ); + assert_eq!( + summary.diff.sleds.modified().count(), + 0, + "{}", + summary.display() + ); + } + + // Asserts that a new blueprint, if generated, will have no report. + // + // This function explicitly ignores the "noop_image_source" report. + // + // NOTE: More reports can be added, but we aren't using + // "PlanningReport::is_empty()", because some checks (e.g. + // noop_image_source) are almost always non-empty. + #[track_caller] + fn assert_child_bp_has_no_report(&self, child_blueprint: &Blueprint) { + verify_blueprint(&child_blueprint); + let summary = child_blueprint.diff_since_blueprint(&self.blueprint); + + assert!( + child_blueprint.report.expunge.is_empty() + && child_blueprint.report.decommission.is_empty() + && child_blueprint.report.mgs_updates.is_empty() + && child_blueprint.report.add.is_empty() + && child_blueprint.report.zone_updates.is_empty() + && child_blueprint.report.nexus_generation_bump.is_empty() + && child_blueprint.report.cockroachdb_settings.is_empty(), + "Blueprint Summary: {}\n + Planning report is not empty: {}", + summary.display(), + child_blueprint.report, + ); + } + + // Updates the input inventory to reflect changes from the blueprint + fn update_inventory_from_blueprint(&mut self) { + update_collection_from_blueprint( + &mut self.example, + &self.blueprint, + ); + } + + // Use the "highest generation Nexus". + // + // This effectively changes "which Nexus is trying to perform planning". + fn set_current_nexus_to_highest_generation(&mut self) { + set_current_nexus_to_highest_generation( + &mut self.example.input, + &self.blueprint, + ); + } + } + + #[test] + fn test_nexus_generation_update() { + static TEST_NAME: &str = "test_nexus_generation_update"; + let logctx = test_setup_log(TEST_NAME); + + // Use our example system with multiple Nexus zones + let mut rng = SimRngState::from_seed(TEST_NAME); + let (example, blueprint) = ExampleSystemBuilder::new_with_rng( + &logctx.log, + rng.next_system_rng(), + ) + .nexus_count(3) // Ensure we have multiple Nexus zones + .build(); + verify_blueprint(&blueprint); + + let mut bp_generator = BlueprintGenerator::new( + logctx.log.clone(), + example, + blueprint, + rng, + ); + + // We shouldn't try to bump the generation number without a new TUF + // repo. + let new_bp = bp_generator.plan_new_blueprint("no-op"); + bp_generator.assert_child_bp_makes_no_changes(&new_bp); + bp_generator.assert_child_bp_has_no_report(&new_bp); + + // Initially, all zones should be sourced from the install dataset + assert!( + bp_generator + .blueprint + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .all(|(_, z)| matches!( + z.image_source, + BlueprintZoneImageSource::InstallDataset + )) + ); + + // Set up a TUF repo with new artifacts + let artifact_version = + ArtifactVersion::new_static("2.0.0-nexus-gen-test") + .expect("can't parse artifact version"); + bp_generator.set_new_tuf_repo_with_artifacts( + create_artifacts_at_version(&artifact_version), + ); + let image_source = + BlueprintGenerator::create_image_at_version(&artifact_version); + + // Check: Initially, nexus generation update should be blocked because + // non-Nexus zones haven't been updated yet + { + let new_bp = + bp_generator.plan_new_blueprint("test_blocked_by_non_nexus"); + // The blueprint should have a report showing what's blocked + assert!( + matches!( + new_bp.report.nexus_generation_bump, + PlanningNexusGenerationBumpReport::WaitingOn( + NexusGenerationBumpWaitingOn::NonNexusZoneUpdate + ), + ), + "Unexpected Nexus Generation report: {:?}", + new_bp.report.nexus_generation_bump + ); + } + + // Manually update all non-Nexus zones to the new image source + for sled_config in bp_generator.blueprint.sleds.values_mut() { + for mut zone in &mut sled_config.zones { + if zone.zone_type.kind() != ZoneKind::Nexus { + zone.image_source = image_source.clone(); + } + } + } + bp_generator.update_inventory_from_blueprint(); + + // Check: Now nexus generation update should be blocked by lack of new + // Nexus zones. + // + // When we create the new blueprint, we'll plan for the new zones, but: + // ✘ The new Nexus Zones are missing db records (we see this first) + // ✘ The new Nexus Zones aren't in inventory. + let old_generation = bp_generator.blueprint.nexus_generation; + let new_generation = old_generation.next(); + let new_bp = + bp_generator.plan_new_blueprint("test_blocked_by_new_nexus_db"); + { + assert_eq!(new_bp.nexus_generation, old_generation); + + let summary = new_bp.diff_since_blueprint(&bp_generator.blueprint); + + // This new blueprint does do *something* - it adds new Nexus zones, + // even though they aren't sufficiently "up" for the nexus + // generation bump. + assert_eq!( + summary.total_zones_added(), + bp_generator.example.input.target_nexus_zone_count() + ); + assert_eq!(summary.total_zones_removed(), 0); + assert_eq!(summary.total_zones_modified(), 0); + assert!( + matches!( + new_bp.report.nexus_generation_bump, + PlanningNexusGenerationBumpReport::WaitingOn( + NexusGenerationBumpWaitingOn::NexusDatabasePropagation + ), + ), + "Unexpected Nexus Generation report: {:?}", + new_bp.report.nexus_generation_bump + ); + } + bp_generator.blueprint = new_bp; + + // "Propagate the DB records to database", by creating records for them. + let new_nexus_zones: BTreeSet<_> = bp_generator + .blueprint + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .filter_map(|(_, zone)| { + match &zone.zone_type { + BlueprintZoneType::Nexus(nexus_zone) => { + if nexus_zone.nexus_generation == new_generation { + return Some(zone.id); + } + } + _ => (), + } + None + }) + .collect(); + bp_generator + .example + .input + .set_not_yet_nexus_zones(new_nexus_zones.clone()); + + // Check: If we try generating a new blueprint, we're still stuck behind + // propagation to inventory. + // + // ✔ The new Nexus Zones have db records + // ✘ The new Nexus Zones aren't in inventory. + // + // We'll refuse to bump the top-level generation number (which would + // begin quiescing old Nexuses) until we've seen that the new nexus + // zones are up. + { + let new_bp = + bp_generator.plan_new_blueprint("wait_for_propagation"); + assert_eq!(new_bp.nexus_generation, old_generation); + + let summary = new_bp.diff_since_blueprint(&bp_generator.blueprint); + assert_eq!(summary.total_zones_added(), 0); + assert_eq!(summary.total_zones_removed(), 0); + assert_eq!(summary.total_zones_modified(), 0); + assert!( + matches!( + new_bp.report.nexus_generation_bump, + PlanningNexusGenerationBumpReport::WaitingOn( + NexusGenerationBumpWaitingOn::ZonePropagation + ), + ), + "Unexpected Nexus Generation report: {:?}", + new_bp.report.nexus_generation_bump + ); + } + + // Make the new Nexus zones appear in inventory + bp_generator.update_inventory_from_blueprint(); + + // If the Nexus zones appear in inventory, but we're missing db records + // for any of these new zones, we shouldn't proceed. + // + // ✘ The new Nexus Zones are missing db records + // ✔ The new Nexus Zones are in inventory. + bp_generator.example.input.set_not_yet_nexus_zones(BTreeSet::new()); + { + let new_bp = + bp_generator.plan_new_blueprint("test_blocked_by_db_records"); + assert_eq!(new_bp.nexus_generation, old_generation); + + let summary = new_bp.diff_since_blueprint(&bp_generator.blueprint); + assert_eq!(summary.total_zones_added(), 0); + assert_eq!(summary.total_zones_removed(), 0); + assert_eq!(summary.total_zones_modified(), 0); + + assert!( + matches!( + new_bp.report.nexus_generation_bump, + PlanningNexusGenerationBumpReport::WaitingOn( + NexusGenerationBumpWaitingOn::NexusDatabasePropagation + ), + ), + "Unexpected Nexus Generation report: {:?}", + new_bp.report.nexus_generation_bump + ); + } + bp_generator.example.input.set_not_yet_nexus_zones(new_nexus_zones); + + // Check: Now nexus generation update should succeed. + // + // Finally: + // ✔ The new Nexus Zones have db records + // ✔ The new Nexus Zones are in inventory. + let new_bp = bp_generator.plan_new_blueprint("update_generation"); + // Finally, the top-level Nexus generation should get bumped. + let PlanningNexusGenerationBumpReport::BumpingGeneration( + observed_next_gen, + ) = &new_bp.report.nexus_generation_bump + else { + panic!( + "Unexpected nexus generation report: {:?}", + new_bp.report.nexus_generation_bump, + ); + }; + + assert_eq!(new_bp.nexus_generation, new_generation); + assert_eq!(*observed_next_gen, new_generation); + bp_generator.blueprint = new_bp; + + // Check: After the generation bump, further planning should make no changes + bp_generator.update_inventory_from_blueprint(); + let new_bp = bp_generator.plan_new_blueprint("no-op"); + bp_generator.assert_child_bp_makes_no_changes(&new_bp); + + // However, there will be a report of "three Nexus zones that aren't + // ready to shut down". The blueprint generator still thinks it's + // running from one of these "old Nexuses". + let unsafe_to_shutdown_zones = &new_bp.report.zone_updates.unsafe_zones; + assert_eq!( + unsafe_to_shutdown_zones.len(), + bp_generator.example.input.target_nexus_zone_count() + ); + for why in unsafe_to_shutdown_zones.values() { + use nexus_types::deployment::ZoneUnsafeToShutdown; + match why { + ZoneUnsafeToShutdown::Nexus { + zone_generation, + current_nexus_generation, + } => { + assert_eq!(zone_generation, &Generation::new()); + assert_eq!( + current_nexus_generation, + &Some(Generation::new()) + ); + } + _ => panic!("Unexpected unsafe-to-shutdown zone: {why}"), + } + } + assert_eq!( + unsafe_to_shutdown_zones.len(), + bp_generator.example.input.target_nexus_zone_count() + ); + + // Move ourselves to a "new Nexus". Now observe: we expunge the old + // Nexus zones. + bp_generator.set_current_nexus_to_highest_generation(); + + // Old Nexuses which are in-service + let mut old_nexuses = + bp_generator.example.input.target_nexus_zone_count(); + // Old Nexuses which were expunged, but which still need propagation + let mut expunging_nexuses = 0; + + while old_nexuses > 0 || expunging_nexuses > 0 { + let new_bp = bp_generator.plan_new_blueprint("removal"); + + // We expect to expunge one old nexus at a time, if any exist, and + // also to finalize the expungement of old nexuses that were removed + // in prior iterations. + let expected_modified_nexuses = + expunging_nexuses + if old_nexuses > 0 { 1 } else { 0 }; + + { + let summary = + new_bp.diff_since_blueprint(&bp_generator.blueprint); + assert_eq!( + summary.total_zones_added(), + 0, + "{}", + summary.display() + ); + assert_eq!( + summary.total_zones_removed(), + 0, + "{}", + summary.display() + ); + assert_eq!( + summary.total_zones_modified(), + expected_modified_nexuses, + "{}", + summary.display() + ); + } + if old_nexuses > 0 { + old_nexuses -= 1; + expunging_nexuses = 1; + } else { + expunging_nexuses = 0; + } + + bp_generator.blueprint = new_bp; + bp_generator.update_inventory_from_blueprint(); + } + + let new_bp = bp_generator.plan_new_blueprint("no-op"); + bp_generator.assert_child_bp_makes_no_changes(&new_bp); + bp_generator.assert_child_bp_has_no_report(&new_bp); + + // Check: If the "old TUF repo = new TUF repo", we'll still make no changes + bp_generator.set_old_tuf_repo_to_target(); + let new_bp = bp_generator.plan_new_blueprint("repo-update"); + bp_generator.assert_child_bp_makes_no_changes(&new_bp); + bp_generator.assert_child_bp_has_no_report(&new_bp); + + // After all this, the Nexus generation number has still been updated + // exactly once. + assert_eq!(new_bp.nexus_generation, new_generation); + + logctx.cleanup_successful(); + } } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 44cfb3249fb..8f6dcd8173d 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -67,6 +67,7 @@ use omicron_common::policy::CRUCIBLE_PANTRY_REDUNDANCY; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; use omicron_uuid_kinds::MupdateOverrideUuid; +use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeMap; @@ -125,6 +126,8 @@ pub struct SystemDescription { old_repo: TufRepoPolicy, planner_config: PlannerConfig, ignore_impossible_mgs_updates_since: DateTime, + active_nexus_zones: BTreeSet, + not_yet_nexus_zones: BTreeSet, } impl SystemDescription { @@ -208,6 +211,8 @@ impl SystemDescription { old_repo: TufRepoPolicy::initial(), planner_config: PlannerConfig::default(), ignore_impossible_mgs_updates_since: Utc::now(), + active_nexus_zones: BTreeSet::new(), + not_yet_nexus_zones: BTreeSet::new(), } } @@ -807,6 +812,22 @@ impl SystemDescription { self } + pub fn set_active_nexus_zones( + &mut self, + active_nexus_zones: BTreeSet, + ) -> &mut Self { + self.active_nexus_zones = active_nexus_zones; + self + } + + pub fn set_not_yet_nexus_zones( + &mut self, + not_yet_nexus_zones: BTreeSet, + ) -> &mut Self { + self.not_yet_nexus_zones = not_yet_nexus_zones; + self + } + pub fn target_release(&self) -> &TufRepoPolicy { &self.tuf_repo } @@ -1029,6 +1050,8 @@ impl SystemDescription { self.external_dns_version, CockroachDbSettings::empty(), ); + builder.set_active_nexus_zones(self.active_nexus_zones.clone()); + builder.set_not_yet_nexus_zones(self.not_yet_nexus_zones.clone()); builder.set_ignore_impossible_mgs_updates_since( self.ignore_impossible_mgs_updates_since, ); diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index 0ba58c47e2d..7abd79ac081 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -328,7 +328,9 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd planning report for blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e: * discretionary zones placed: - * 2 zones on sled d67ce8f0-a691-4010-b414-420d82e80527: crucible_pantry, nexus - * 2 zones on sled fefcf4cf-f7e7-46b3-b629-058526ce440e: clickhouse, internal_dns + * crucible_pantry zone on sled d67ce8f0-a691-4010-b414-420d82e80527 from source install dataset + * nexus zone on sled d67ce8f0-a691-4010-b414-420d82e80527 from source install dataset + * clickhouse zone on sled fefcf4cf-f7e7-46b3-b629-058526ce440e from source install dataset + * internal_dns zone on sled fefcf4cf-f7e7-46b3-b629-058526ce440e from source install dataset * zone updates waiting on discretionary zones diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 4a34e4cb9e0..a98c9208439 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -516,7 +516,11 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b planning report for blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6: * discretionary zones placed: - * 3 zones on sled 75bc286f-2b4b-482c-9431-59272af529da: nexus, nexus, nexus - * 3 zones on sled affab35f-600a-4109-8ea0-34a067a4e0bc: nexus, nexus, nexus + * nexus zone on sled 75bc286f-2b4b-482c-9431-59272af529da from source install dataset + * nexus zone on sled 75bc286f-2b4b-482c-9431-59272af529da from source install dataset + * nexus zone on sled 75bc286f-2b4b-482c-9431-59272af529da from source install dataset + * nexus zone on sled affab35f-600a-4109-8ea0-34a067a4e0bc from source install dataset + * nexus zone on sled affab35f-600a-4109-8ea0-34a067a4e0bc from source install dataset + * nexus zone on sled affab35f-600a-4109-8ea0-34a067a4e0bc from source install dataset * zone updates waiting on discretionary zones diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs index 736fba3dfa0..a224a20d8b9 100644 --- a/nexus/reconfigurator/preparation/src/lib.rs +++ b/nexus/reconfigurator/preparation/src/lib.rs @@ -6,6 +6,7 @@ use anyhow::Context; use futures::StreamExt; +use nexus_db_model::DbMetadataNexusState; use nexus_db_model::DnsGroup; use nexus_db_model::Generation; use nexus_db_queries::context::OpContext; @@ -84,6 +85,8 @@ pub struct PlanningInputFromDb<'a> { pub tuf_repo: TufRepoPolicy, pub old_repo: TufRepoPolicy, pub planner_config: PlannerConfig, + pub active_nexus_zones: Vec, + pub not_yet_nexus_zones: Vec, pub log: &'a Logger, } @@ -200,6 +203,27 @@ impl PlanningInputFromDb<'_> { .await .internal_context("fetching oximeter read policy")?; + let (active_nexus_zones, not_yet_nexus_zones): (Vec<_>, Vec<_>) = + datastore + .get_db_metadata_nexus_in_state( + opctx, + &[ + DbMetadataNexusState::Active, + DbMetadataNexusState::NotYet, + ], + ) + .await + .internal_context("fetching db_metdata_nexus records")? + .into_iter() + .partition(|nexus| { + nexus.state() == DbMetadataNexusState::Active + }); + + let active_nexus_zones = + active_nexus_zones.into_iter().map(|n| n.nexus_id()).collect(); + let not_yet_nexus_zones = + not_yet_nexus_zones.into_iter().map(|n| n.nexus_id()).collect(); + let planning_input = PlanningInputFromDb { sled_rows: &sled_rows, zpool_rows: &zpool_rows, @@ -223,6 +247,8 @@ impl PlanningInputFromDb<'_> { tuf_repo, old_repo, planner_config, + active_nexus_zones, + not_yet_nexus_zones, } .build() .internal_context("assembling planning_input")?; @@ -256,6 +282,12 @@ impl PlanningInputFromDb<'_> { self.external_dns_version.into(), self.cockroachdb_settings.clone(), ); + builder.set_active_nexuses( + self.active_nexus_zones.clone().into_iter().collect(), + ); + builder.set_not_yet_nexuses( + self.not_yet_nexus_zones.clone().into_iter().collect(), + ); let mut zpools_by_sled_id = { // Iterate over all Zpools, identifying their disks and datasets diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 3b103a7b6c4..5b49eebd73b 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -56,6 +56,7 @@ use serde::Deserialize; use serde::Serialize; use slog::Key; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::fmt; use std::net::Ipv6Addr; use std::net::SocketAddrV6; @@ -116,12 +117,14 @@ pub use planning_input::TufRepoContentsError; pub use planning_input::TufRepoPolicy; pub use planning_input::ZpoolFilter; pub use planning_report::CockroachdbUnsafeToShutdown; +pub use planning_report::NexusGenerationBumpWaitingOn; pub use planning_report::PlanningAddStepReport; pub use planning_report::PlanningCockroachdbSettingsStepReport; pub use planning_report::PlanningDecommissionStepReport; pub use planning_report::PlanningExpungeStepReport; pub use planning_report::PlanningMgsUpdatesStepReport; pub use planning_report::PlanningMupdateOverrideStepReport; +pub use planning_report::PlanningNexusGenerationBumpReport; pub use planning_report::PlanningNoopImageSourceSkipSledHostPhase2Reason; pub use planning_report::PlanningNoopImageSourceSkipSledZonesReason; pub use planning_report::PlanningNoopImageSourceSkipZoneReason; @@ -405,6 +408,27 @@ impl Blueprint { Ok(zone_config.nexus_generation < self.nexus_generation) } + + /// Given a set of Nexus zone UUIDs, returns the "nexus generation" + /// of the first one found in the blueprint. + pub fn find_generation_for_nexus( + &self, + nexus_zones: &BTreeSet, + ) -> Option { + self.all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .find_map(|(_, blueprint_zone)| { + if nexus_zones.contains(&blueprint_zone.id) { + match &blueprint_zone.zone_type { + BlueprintZoneType::Nexus(nexus_zone) => { + Some(nexus_zone.nexus_generation) + } + _ => None, + } + } else { + None + } + }) + } } /// Wrapper to display a table of a `BlueprintSledConfig`'s host phase 2 diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index bcf23f355d6..1332f2285d1 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -41,6 +41,7 @@ use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::btree_map::Entry; use std::error; use std::fmt; @@ -123,6 +124,19 @@ pub struct PlanningInput { /// mark under the assumption that they may appear to be impossible because /// they're currently in progress. ignore_impossible_mgs_updates_since: DateTime, + + /// IDs of the currently running Nexus zones + /// + /// This is used to determine which Nexus instances are currently in + /// control, which is needed for safe shutdown decisions during handoff. + active_nexus_zones: BTreeSet, + + /// IDs of the about-to-be-running Nexus zones + /// + /// This is used to identify when it is safe to bump the top-level Nexus + /// Generation number in the blueprint, which triggers active Nexuses to + /// quiesce. + not_yet_nexus_zones: BTreeSet, } impl PlanningInput { @@ -240,6 +254,24 @@ impl PlanningInput { self.policy.oximeter_read_policy.mode.single_node_enabled() } + /// ID of the currently running Nexus zones + pub fn active_nexus_zones(&self) -> &BTreeSet { + &self.active_nexus_zones + } + + pub fn set_active_nexus_zones(&mut self, ids: BTreeSet) { + self.active_nexus_zones = ids; + } + + /// ID of the soon-to-be-running Nexus zones + pub fn not_yet_nexus_zones(&self) -> &BTreeSet { + &self.not_yet_nexus_zones + } + + pub fn set_not_yet_nexus_zones(&mut self, ids: BTreeSet) { + self.not_yet_nexus_zones = ids; + } + pub fn all_sleds( &self, filter: SledFilter, @@ -318,6 +350,8 @@ impl PlanningInput { network_resources: self.network_resources, ignore_impossible_mgs_updates_since: self .ignore_impossible_mgs_updates_since, + active_nexus_zones: self.active_nexus_zones, + not_yet_nexus_zones: self.not_yet_nexus_zones, } } } @@ -1260,6 +1294,8 @@ pub struct PlanningInputBuilder { sleds: BTreeMap, network_resources: OmicronZoneNetworkResources, ignore_impossible_mgs_updates_since: DateTime, + active_nexus_zones: BTreeSet, + not_yet_nexus_zones: BTreeSet, } impl PlanningInputBuilder { @@ -1288,6 +1324,8 @@ impl PlanningInputBuilder { sleds: BTreeMap::new(), network_resources: OmicronZoneNetworkResources::new(), ignore_impossible_mgs_updates_since: Utc::now(), + active_nexus_zones: BTreeSet::new(), + not_yet_nexus_zones: BTreeSet::new(), } } @@ -1306,9 +1344,25 @@ impl PlanningInputBuilder { network_resources: OmicronZoneNetworkResources::new(), ignore_impossible_mgs_updates_since: Utc::now() - MGS_UPDATE_SETTLE_TIMEOUT, + active_nexus_zones: BTreeSet::new(), + not_yet_nexus_zones: BTreeSet::new(), } } + pub fn set_active_nexuses( + &mut self, + active_nexus_zones: BTreeSet, + ) { + self.active_nexus_zones = active_nexus_zones; + } + + pub fn set_not_yet_nexuses( + &mut self, + not_yet_nexus_zones: BTreeSet, + ) { + self.not_yet_nexus_zones = not_yet_nexus_zones; + } + pub fn add_sled( &mut self, sled_id: SledUuid, @@ -1401,6 +1455,20 @@ impl PlanningInputBuilder { self.cockroachdb_settings = cockroachdb_settings; } + pub fn set_active_nexus_zones( + &mut self, + active_nexus_zones: BTreeSet, + ) { + self.active_nexus_zones = active_nexus_zones; + } + + pub fn set_not_yet_nexus_zones( + &mut self, + not_yet_nexus_zones: BTreeSet, + ) { + self.not_yet_nexus_zones = not_yet_nexus_zones; + } + pub fn build(self) -> PlanningInput { PlanningInput { policy: self.policy, @@ -1411,6 +1479,8 @@ impl PlanningInputBuilder { network_resources: self.network_resources, ignore_impossible_mgs_updates_since: self .ignore_impossible_mgs_updates_since, + active_nexus_zones: self.active_nexus_zones, + not_yet_nexus_zones: self.not_yet_nexus_zones, } } } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 0a408727283..4b1c7001c6f 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -13,6 +13,7 @@ use super::PlannerConfig; use daft::Diffable; use indent_write::fmt::IndentWriter; +use omicron_common::api::external::Generation; use omicron_common::policy::COCKROACHDB_REDUNDANCY; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::MupdateOverrideUuid; @@ -63,6 +64,7 @@ pub struct PlanningReport { pub mgs_updates: PlanningMgsUpdatesStepReport, pub add: PlanningAddStepReport, pub zone_updates: PlanningZoneUpdatesStepReport, + pub nexus_generation_bump: PlanningNexusGenerationBumpReport, pub cockroachdb_settings: PlanningCockroachdbSettingsStepReport, } @@ -79,6 +81,7 @@ impl PlanningReport { ), add: PlanningAddStepReport::new(), zone_updates: PlanningZoneUpdatesStepReport::new(), + nexus_generation_bump: PlanningNexusGenerationBumpReport::new(), cockroachdb_settings: PlanningCockroachdbSettingsStepReport::new(), } } @@ -90,6 +93,7 @@ impl PlanningReport { && self.mgs_updates.is_empty() && self.add.is_empty() && self.zone_updates.is_empty() + && self.nexus_generation_bump.is_empty() && self.cockroachdb_settings.is_empty() } } @@ -112,6 +116,7 @@ impl fmt::Display for PlanningReport { mgs_updates, add, zone_updates, + nexus_generation_bump, cockroachdb_settings, } = self; writeln!(f, "planning report for blueprint {blueprint_id}:")?; @@ -124,6 +129,7 @@ impl fmt::Display for PlanningReport { mgs_updates.fmt(f)?; add.fmt(f)?; zone_updates.fmt(f)?; + nexus_generation_bump.fmt(f)?; cockroachdb_settings.fmt(f)?; } Ok(()) @@ -524,6 +530,14 @@ pub struct PlanningAddSufficientZonesExist { pub num_existing: usize, } +#[derive( + Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, +)] +pub struct DiscretionaryZonePlacement { + kind: String, + source: String, +} + #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] @@ -576,7 +590,8 @@ pub struct PlanningAddStepReport { /// Sled ID → kinds of discretionary zones placed there // TODO: make `sled_add_zone_*` methods return the added zone config // so that we can report it here. - pub discretionary_zones_placed: BTreeMap>, + pub discretionary_zones_placed: + BTreeMap>, } impl PlanningAddStepReport { @@ -659,11 +674,22 @@ impl PlanningAddStepReport { &mut self, sled_id: SledUuid, zone_kind: &str, + image_source: &BlueprintZoneImageSource, ) { self.discretionary_zones_placed .entry(sled_id) - .and_modify(|kinds| kinds.push(zone_kind.to_owned())) - .or_insert_with(|| vec![zone_kind.to_owned()]); + .and_modify(|kinds| { + kinds.push(DiscretionaryZonePlacement { + kind: zone_kind.to_owned(), + source: image_source.to_string(), + }) + }) + .or_insert_with(|| { + vec![DiscretionaryZonePlacement { + kind: zone_kind.to_owned(), + source: image_source.to_string(), + }] + }); } } @@ -784,13 +810,13 @@ impl fmt::Display for PlanningAddStepReport { if !discretionary_zones_placed.is_empty() { writeln!(f, "* discretionary zones placed:")?; - for (sled_id, kinds) in discretionary_zones_placed.iter() { - let (n, s) = plural_vec(kinds); - writeln!( - f, - " * {n} zone{s} on sled {sled_id}: {}", - kinds.join(", ") - )?; + for (sled_id, placements) in discretionary_zones_placed.iter() { + for DiscretionaryZonePlacement { kind, source } in placements { + writeln!( + f, + " * {kind} zone on sled {sled_id} from source {source}", + )?; + } } } @@ -993,9 +1019,21 @@ impl ZoneUpdatesWaitingOn { )] #[serde(rename_all = "snake_case", tag = "type")] pub enum ZoneUnsafeToShutdown { - Cockroachdb { reason: CockroachdbUnsafeToShutdown }, - BoundaryNtp { total_boundary_ntp_zones: usize, synchronized_count: usize }, - InternalDns { total_internal_dns_zones: usize, synchronized_count: usize }, + Cockroachdb { + reason: CockroachdbUnsafeToShutdown, + }, + BoundaryNtp { + total_boundary_ntp_zones: usize, + synchronized_count: usize, + }, + InternalDns { + total_internal_dns_zones: usize, + synchronized_count: usize, + }, + Nexus { + zone_generation: Generation, + current_nexus_generation: Option, + }, } impl fmt::Display for ZoneUnsafeToShutdown { @@ -1010,6 +1048,103 @@ impl fmt::Display for ZoneUnsafeToShutdown { total_internal_dns_zones: t, synchronized_count: s, } => write!(f, "only {s}/{t} internal DNS zones are synchronized"), + Self::Nexus { zone_generation, current_nexus_generation } => { + match current_nexus_generation { + Some(current) => write!( + f, + "zone gen ({zone_generation}) >= currently-running \ + Nexus gen ({current})" + ), + None => write!( + f, + "zone gen is {zone_generation}, but currently-running \ + Nexus generation is unknown" + ), + } + } + } + } +} + +#[derive( + Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, +)] +#[serde(tag = "component", rename_all = "snake_case", content = "value")] +pub enum PlanningNexusGenerationBumpReport { + /// We have no reason to bump the Nexus generation number. + NothingToReport, + + /// We are waiting on some condition before we can bump the + /// Nexus generation. + WaitingOn(NexusGenerationBumpWaitingOn), + + /// We are bumping the Nexus generation number to this value. + BumpingGeneration(Generation), +} + +impl PlanningNexusGenerationBumpReport { + pub fn new() -> Self { + Self::NothingToReport + } + + pub fn is_empty(&self) -> bool { + matches!(self, Self::NothingToReport) + } + + pub fn set_waiting_on(&mut self, why: NexusGenerationBumpWaitingOn) { + *self = Self::WaitingOn(why); + } + + pub fn set_next_generation(&mut self, next_generation: Generation) { + *self = Self::BumpingGeneration(next_generation); + } +} + +impl fmt::Display for PlanningNexusGenerationBumpReport { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::WaitingOn(why) => { + writeln!( + f, + "* waiting to update top-level nexus_generation: {}", + why.as_str() + )?; + } + Self::BumpingGeneration(gen) => { + writeln!(f, "* updating top-level nexus_generation to: {gen}")?; + } + // Nothing to report + Self::NothingToReport => (), + } + Ok(()) + } +} + +#[derive( + Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, +)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum NexusGenerationBumpWaitingOn { + /// Waiting for non-Nexus zones to finish updating + NonNexusZoneUpdate, + + /// Waiting for enough new Nexus zones to appear + NewNexusBringup, + + /// Waiting for Nexus database records to get populated + NexusDatabasePropagation, + + /// Waiting for zones to propagate to inventory + ZonePropagation, +} + +impl NexusGenerationBumpWaitingOn { + pub fn as_str(&self) -> &'static str { + match self { + Self::NonNexusZoneUpdate => "pending non-nexus zone updates", + Self::NewNexusBringup => "waiting for new nexus zones", + Self::NexusDatabasePropagation => "pending database records", + Self::ZonePropagation => "pending zone reconciliation", } } } diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 9e8e9904520..37682c9ec5e 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -4219,6 +4219,21 @@ "saga_id" ] }, + "DiscretionaryZonePlacement": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "required": [ + "kind", + "source" + ] + }, "DiskIdentity": { "description": "Uniquely identifies a disk.", "type": "object", @@ -5852,6 +5867,70 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "NexusGenerationBumpWaitingOn": { + "oneOf": [ + { + "description": "Waiting for non-Nexus zones to finish updating", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "non_nexus_zone_update" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Waiting for enough new Nexus zones to appear", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "new_nexus_bringup" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Waiting for Nexus database records to get populated", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "nexus_database_propagation" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Waiting for zones to propagate to inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zone_propagation" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, "NodeName": { "description": "Unique name for a saga [`Node`]\n\nEach node requires a string name that's unique within its DAG. The name is used to identify its output. Nodes that depend on a given node (either directly or indirectly) can access the node's output using its name.", "type": "string" @@ -6782,7 +6861,7 @@ "additionalProperties": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/DiscretionaryZonePlacement" } } }, @@ -6940,6 +7019,63 @@ "pending_mgs_updates" ] }, + "PlanningNexusGenerationBumpReport": { + "oneOf": [ + { + "description": "We have no reason to bump the Nexus generation number.", + "type": "object", + "properties": { + "component": { + "type": "string", + "enum": [ + "nothing_to_report" + ] + } + }, + "required": [ + "component" + ] + }, + { + "description": "We are waiting on some condition before we can bump the Nexus generation.", + "type": "object", + "properties": { + "component": { + "type": "string", + "enum": [ + "waiting_on" + ] + }, + "value": { + "$ref": "#/components/schemas/NexusGenerationBumpWaitingOn" + } + }, + "required": [ + "component", + "value" + ] + }, + { + "description": "We are bumping the Nexus generation number to this value.", + "type": "object", + "properties": { + "component": { + "type": "string", + "enum": [ + "bumping_generation" + ] + }, + "value": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "component", + "value" + ] + } + ] + }, "PlanningNoopImageSourceConverted": { "description": "How many of the total install-dataset zones and/or host phase 2 slots were noop-converted to use the artifact store on a particular sled.", "type": "object", @@ -7235,6 +7371,9 @@ "mgs_updates": { "$ref": "#/components/schemas/PlanningMgsUpdatesStepReport" }, + "nexus_generation_bump": { + "$ref": "#/components/schemas/PlanningNexusGenerationBumpReport" + }, "noop_image_source": { "$ref": "#/components/schemas/PlanningNoopImageSourceStepReport" }, @@ -7257,6 +7396,7 @@ "decommission", "expunge", "mgs_updates", + "nexus_generation_bump", "noop_image_source", "planner_config", "zone_updates" @@ -9678,6 +9818,32 @@ "total_internal_dns_zones", "type" ] + }, + { + "type": "object", + "properties": { + "current_nexus_generation": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "nexus" + ] + }, + "zone_generation": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "type", + "zone_generation" + ] } ] },