diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-mupdate-stale-inventory-set.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-mupdate-stale-inventory-set.txt new file mode 100644 index 00000000000..8de57d0b033 --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-mupdate-stale-inventory-set.txt @@ -0,0 +1,47 @@ +# This test exercises stale inventory with `BpSetOverride`. +# +# After the planner has cleared the "will remove mupdate override" field on a +# sled, it should not set that field again if a subsequent planner run is +# performed against a stale inventory that still has the override visible. + +load-example --nsleds 1 --ndisks-per-sled 3 +sled-list + +# Create a TUF repository, set as target release, and update the install dataset. +tuf-assemble ../../update-common/manifests/fake.toml +set target-release repo-1.0.0.zip +sled-update-install-dataset serial0 --to-target-release + +# Simulate a mupdate on serial0. +sled-set serial0 mupdate-override 11111111-1111-1111-1111-111111111111 + +# Generate an inventory that captures the post-MUPdate state (mupdate override +# visible, sled-agent generation still 2). We'll come back to plan against +# this collection later, by which time it will be stale. +inventory-generate +inventory-list + +# First plan: sets remove_mupdate_override on serial0 and bumps the target +# release minimum generation. +blueprint-plan latest latest +blueprint-diff latest + +# Simulate the sled processing the recovery blueprint: it clears the +# override marker AND advances its reconciled config generation. +sled-set serial0 mupdate-override unset +sled-set serial0 omicron-config latest + +# Generate a fresh inventory. This inventory would not have a mupdate override, +# and would advance the sled agent generation. +inventory-generate + +# This planner run clears the "will remove mupdate override" field and +# noop-convert zones to Artifact image sources. +blueprint-plan latest latest +blueprint-diff latest + +# Now plan against the stale inventory from earlier. The expected behavior +# is that the planner detects inventory staleness on the BpSetOverride path and +# does nothing. As a result, the blueprint diff should be empty. +blueprint-plan latest eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 +blueprint-diff latest diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-stale-inventory-set-stderr b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-stale-inventory-set-stderr new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-stale-inventory-set-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-stale-inventory-set-stdout new file mode 100644 index 00000000000..1c475523844 --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-stale-inventory-set-stdout @@ -0,0 +1,486 @@ +using provided RNG seed: reconfigurator-cli-test +> # This test exercises stale inventory with `BpSetOverride`. +> # +> # After the planner has cleared the "will remove mupdate override" field on a +> # sled, it should not set that field again if a subsequent planner run is +> # performed against a stale inventory that still has the override visible. + +> load-example --nsleds 1 --ndisks-per-sled 3 +loaded example system with: +- collection: f45ba181-4b56-42cc-a762-874d90184a43 +- blueprint: dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 + +> sled-list +ID SERIAL NZPOOLS SUBNET +98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 serial0 3 fd00:1122:3344:101::/64 + + +> # Create a TUF repository, set as target release, and update the install dataset. +> tuf-assemble ../../update-common/manifests/fake.toml +INFO assembling repository in +INFO artifacts assembled and archived to `repo-1.0.0.zip`, component: OmicronRepoAssembler +created repo-1.0.0.zip for system version 1.0.0 + +> set target-release repo-1.0.0.zip +INFO extracting uploaded archive to +INFO created directory to store extracted artifacts, path: +INFO added artifact, name: fake-gimlet-sp, kind: gimlet_sp, version: 1.0.0, hash: 68465b8e3f808f475510b525cfd62086d37ddd57688bd854184fdafb2b2198a4, length: 732 +INFO added artifact, name: fake-rot, kind: gimlet_rot_image_a, version: 1.0.0, hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, length: 783 +INFO added artifact, name: fake-rot, kind: gimlet_rot_image_b, version: 1.0.0, hash: d11e65f934bf0de51df2e5b484f61ee72072417b43ac87f33e958008428e7b02, length: 783 +INFO added artifact, name: fake-rot-bootloader, kind: gimlet_rot_bootloader, version: 1.0.0, hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, length: 797 +INFO added artifact, name: fake-host, kind: gimlet_host_phase_1, version: 1.0.0, hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, length: 524288 +INFO added artifact, name: fake-host, kind: cosmo_host_phase_1, version: 1.0.0, hash: 9525f567106549a3fc32df870a74803d77e51dcc44190b218e227a2c5d444f58, length: 524288 +INFO added artifact, name: fake-host, kind: host_phase_2, version: 1.0.0, hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, length: 1048576 +INFO added artifact, name: fake-trampoline, kind: gimlet_trampoline_phase_1, version: 1.0.0, hash: bcb27520ee5a56e19f6df9662c66d69ac681fbd873a97547be5f6a5ae3d250c4, length: 524288 +INFO added artifact, name: fake-trampoline, kind: cosmo_trampoline_phase_1, version: 1.0.0, hash: e235b8afb58ee69d966853bd5efe7c7e904da84b9035a332b3e691dc1d5cdbd0, length: 524288 +INFO added artifact, name: fake-trampoline, kind: trampoline_phase_2, version: 1.0.0, hash: a5dfcc4bc69b791f1c509df499e9e72cce844cb2b53a56d8bb357b264bdf13b6, length: 1048576 +INFO added artifact, name: clickhouse, kind: zone, version: 1.0.0, hash: 52b1eb4daff6f9140491d547b11248392920230db3db0eef5f5fa5333fe9e659, length: 1686 +INFO added artifact, name: clickhouse_keeper, kind: zone, version: 1.0.0, hash: cda702919449d86663be97295043aeca0ead69ae5db3bbdb20053972254a27a3, length: 1690 +INFO added artifact, name: clickhouse_server, kind: zone, version: 1.0.0, hash: 5f9ae6a9821bbe8ff0bf60feddf8b167902fe5f3e2c98bd21edd1ec9d969a001, length: 1690 +INFO added artifact, name: cockroachdb, kind: zone, version: 1.0.0, hash: f3a1a3c0b3469367b005ee78665d982059d5e14e93a479412426bf941c4ed291, length: 1689 +INFO added artifact, name: crucible-zone, kind: zone, version: 1.0.0, hash: 6f17cf65fb5a5bec5542dd07c03cd0acc01e59130f02c532c8d848ecae810047, length: 1690 +INFO added artifact, name: crucible-pantry-zone, kind: zone, version: 1.0.0, hash: 21f0ada306859c23917361f2e0b9235806c32607ec689c7e8cf16bb898bc5a02, length: 1695 +INFO added artifact, name: external-dns, kind: zone, version: 1.0.0, hash: ccca13ed19b8731f9adaf0d6203b02ea3b9ede4fa426b9fac0a07ce95440046d, length: 1689 +INFO added artifact, name: internal-dns, kind: zone, version: 1.0.0, hash: ffbf1373f7ee08dddd74c53ed2a94e7c4c572a982d3a9bc94000c6956b700c6a, length: 1689 +INFO added artifact, name: ntp, kind: zone, version: 1.0.0, hash: 67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439, length: 1681 +INFO added artifact, name: nexus, kind: zone, version: 1.0.0, hash: 0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388, length: 1682 +INFO added artifact, name: oximeter, kind: zone, version: 1.0.0, hash: 048d8fe8cdef5b175aad714d0f148aa80ce36c9114ac15ce9d02ed3d37877a77, length: 1682 +INFO added artifact, name: fake-corpus, kind: measurement_corpus, version: 1.0.0, hash: 8a0e23157bae655fceec7376926c9758efee6511c7b7ff8355bbb49545a2257f, length: 1048576 +INFO added artifact, name: fake-psc-sp, kind: psc_sp, version: 1.0.0, hash: 89245fe2ac7e6a2ac8dfa4e7d6891a6e6df95e4141395c07c64026778f6d76d7, length: 721 +INFO added artifact, name: fake-psc-rot, kind: psc_rot_image_a, version: 1.0.0, hash: 5d8ea834dd6d42d386f1eb8a2c5f6e99b697c9958bb4ab8edf63e56003e25d8d, length: 775 +INFO added artifact, name: fake-psc-rot, kind: psc_rot_image_b, version: 1.0.0, hash: 5d8ea834dd6d42d386f1eb8a2c5f6e99b697c9958bb4ab8edf63e56003e25d8d, length: 775 +INFO added artifact, name: fake-psc-rot-bootloader, kind: psc_rot_bootloader, version: 1.0.0, hash: 18c9c774bfe4bb086e869509dcccacee8476fd87670692b765aee216f2c7f003, length: 805 +INFO added artifact, name: fake-switch-sp, kind: switch_sp, version: 1.0.0, hash: bf1bc1da5059f76182c3007c3049941f8898abede2f3765b106c6e7f7c42d44c, length: 740 +INFO added artifact, name: fake-switch-rot, kind: switch_rot_image_a, version: 1.0.0, hash: 32307d6d75c9707e8499ba4a4d379f99c0358237b6e190ff6a8024b470f62342, length: 774 +INFO added artifact, name: fake-switch-rot, kind: switch_rot_image_b, version: 1.0.0, hash: 32307d6d75c9707e8499ba4a4d379f99c0358237b6e190ff6a8024b470f62342, length: 774 +INFO added artifact, name: fake-switch-rot-bootloader, kind: switch_rot_bootloader, version: 1.0.0, hash: 70836d170abd5621f95bb4225987b27b3d3dd6168e73cd60e44309bdfeb94e98, length: 804 +INFO added artifact, name: installinator_document, kind: installinator_document, version: 1.0.0, hash: 16717c0caa420c7811666e720936c7ca913cd59a4d376331cc79a90544e5e2e8, length: 526 +set target release based on repo-1.0.0.zip + +> sled-update-install-dataset serial0 --to-target-release +sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: install dataset updated: to target release (system version 1.0.0) + + +> # Simulate a mupdate on serial0. +> sled-set serial0 mupdate-override 11111111-1111-1111-1111-111111111111 +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 mupdate override: unset -> 11111111-1111-1111-1111-111111111111 + + +> # Generate an inventory that captures the post-MUPdate state (mupdate override +> # visible, sled-agent generation still 2). We'll come back to plan against +> # this collection later, by which time it will be stale. +> inventory-generate +generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds + +> inventory-list +ID NERRORS TIME_DONE +f45ba181-4b56-42cc-a762-874d90184a43 0 +eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 0 + + +> # First plan: sets remove_mupdate_override on serial0 and bumps the target +> # release minimum generation. +> blueprint-plan latest latest +INFO blueprint mupdate override updated to match inventory, phase: do_plan_mupdate_override, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, new_bp_override: 11111111-1111-1111-1111-111111111111, prev_bp_override: None, zones: + - zone 005548be-c5e4-49ff-a27b-88f314f1bc51 (ExternalDns) left unchanged, image source: install dataset + - zone 10676bfe-b61f-40e1-bc07-a4cb76ea1f30 (Clickhouse) left unchanged, image source: install dataset + - zone 2205353a-e1d2-48ff-863b-9d6b1487d474 (ExternalDns) left unchanged, image source: install dataset + - zone 2bc0b0c4-63a9-44cc-afff-76ce645ef1d4 (Nexus) left unchanged, image source: install dataset + - zone 2de1c525-4e04-4ba6-ac6b-377aaa542f96 (CruciblePantry) left unchanged, image source: install dataset + - zone 31b26053-8b94-4d8c-9e4a-d7d720afe265 (CruciblePantry) left unchanged, image source: install dataset + - zone 3c2cd97f-7b4a-4ff3-a9d5-6ce141fcdbbe (Nexus) left unchanged, image source: install dataset + - zone 3d9b7487-d7b9-4e25-960f-f2086f3e2919 (Crucible) left unchanged, image source: install dataset + - zone 3fd06852-06cb-4d8a-b4b2-eb88ff5a6035 (InternalDns) left unchanged, image source: install dataset + - zone 45ce130e-c5ac-4e26-ab73-7589ba634418 (InternalDns) left unchanged, image source: install dataset + - zone 4b19d194-8b25-4396-88da-3df1b3788601 (Crucible) left unchanged, image source: install dataset + - zone 70aab480-3d6c-47d5-aaf9-8d2ddab2931c (ExternalDns) left unchanged, image source: install dataset + - zone 9b135c74-c09a-4dcc-ba19-f6f8deae135a (InternalNtp) left unchanged, image source: install dataset + - zone a7b7bfbe-0588-4781-9a5e-fba63584e5d2 (InternalDns) left unchanged, image source: install dataset + - zone c204730a-0946-4793-a470-64c88e89da96 (Crucible) left unchanged, image source: install dataset + - zone c2cbbf34-b852-4164-a572-01d4d79445a1 (Nexus) left unchanged, image source: install dataset + - zone f87ada51-4419-4144-86a8-e5e4ff0f64d3 (CruciblePantry) left unchanged, image source: install dataset +, host_phase_2: + - host phase 2 slot A: current contents (unchanged) + - host phase 2 slot B: current contents (unchanged) +, measurements: (unknown) +INFO no previous MGS update found as part of updating blueprint mupdate override to match inventory, phase: do_plan_mupdate_override, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +INFO updating target release minimum generation based on new set-override actions, phase: do_plan_mupdate_override, current_generation: 1, new_generation: 3 +INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, reason: remove_mupdate_override is set in the blueprint (11111111-1111-1111-1111-111111111111) +generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 +blueprint source: planner with report: +planning report: +* zone adds waiting on blockers +* zone adds and updates are blocked: + - current target release generation (2) is lower than minimum required by blueprint (3) + - sleds have remove mupdate override set in blueprint: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 + - sleds have deployment units with image sources not set to Artifact: + - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: 17 zones + + - sleds have measurements with image sources not set to Artifact: + - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 set to install dataset + +* zone updates waiting on zone add blockers +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated +Measurement updates: +Waiting on zone add/update blockers + + + +> blueprint-diff latest +from: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 +to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 + + MODIFIED SLEDS: + + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 2 -> 3): ++ will remove mupdate override: (none) -> 11111111-1111-1111-1111-111111111111 + + measurements: + ----------------------------------- + hash version + ----------------------------------- +- unknown (unknown version) ++ install dataset (no version) + + + host phase 2 contents: + ------------------------ + slot boot image source + ------------------------ + A current contents + B current contents + + + physical disks: + ------------------------------------------------------------------------------------ + vendor model serial disposition + ------------------------------------------------------------------------------------ + fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service + fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service + fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service + + + datasets: + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 42bd3fb3-2aae-4754-9c41-0f9b50a77322 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible b7813bc0-d5ee-487d-95ad-6c5cce6fdcd6 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 6dd64de1-7dd3-492c-830f-d4a4e4c3dda7 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/clickhouse b1007e3d-7aaf-4ccb-9773-241dcbf79b21 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns d3cd3ec4-703f-4e73-8cc2-b4bc19c7d596 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/external_dns c7e90d9c-22db-43ca-a602-2a295bac7eec in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/external_dns bef875a9-3cf1-4b02-9682-c0286fb23d4d in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns 66788e15-bb3a-4369-a5cb-3357f7b85f64 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/internal_dns d88eb02f-b66d-4ae7-91a1-23154417c1f1 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/internal_dns ebfd425c-4e5d-4385-a105-8f2ce32fa9ec in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone e4b83186-c6e2-4d33-ae1b-803c32d1a86e in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 20cb5d98-bcd7-4a11-9190-7570a0aa6fe0 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone aa860565-2bbb-4a70-ae8e-7d8f29f1a536 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_clickhouse_10676bfe-b61f-40e1-bc07-a4cb76ea1f30 3523db58-10ab-4e2c-95d9-fec43036c0f5 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_3d9b7487-d7b9-4e25-960f-f2086f3e2919 32bb65ae-855f-461b-9203-32b2ddfe5a64 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_4b19d194-8b25-4396-88da-3df1b3788601 dd1954a2-7848-4950-a694-38870c339d3e in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_c204730a-0946-4793-a470-64c88e89da96 f3ddb3ca-388d-4462-9fc8-e552cf7de668 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_pantry_2de1c525-4e04-4ba6-ac6b-377aaa542f96 c6d7ecb4-135c-4f09-8948-b73fec13db76 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_31b26053-8b94-4d8c-9e4a-d7d720afe265 d4f66eab-2870-4a74-b4d5-ae5da3276682 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_pantry_f87ada51-4419-4144-86a8-e5e4ff0f64d3 f822a7e2-28aa-4ae2-ba86-d4e552a15bcb in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_external_dns_005548be-c5e4-49ff-a27b-88f314f1bc51 0c7e4b2a-502a-4f4c-8ac4-4068db25252e in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_external_dns_2205353a-e1d2-48ff-863b-9d6b1487d474 8355e747-b375-4d9e-b5da-03d9540ab5cd in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_70aab480-3d6c-47d5-aaf9-8d2ddab2931c 49dc7c86-c865-457f-8d58-37b92b002691 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_internal_dns_3fd06852-06cb-4d8a-b4b2-eb88ff5a6035 506ddf0e-8cbf-44a9-93d7-130a550fd42d in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_45ce130e-c5ac-4e26-ab73-7589ba634418 d1611973-b22d-4097-b6fd-0a480f2199a5 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_internal_dns_a7b7bfbe-0588-4781-9a5e-fba63584e5d2 382cbbf2-8b08-4388-839a-b43b8bc82999 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_nexus_2bc0b0c4-63a9-44cc-afff-76ce645ef1d4 6b672182-6d17-41d5-81d7-19debe7d3f9b in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_nexus_3c2cd97f-7b4a-4ff3-a9d5-6ce141fcdbbe 8b89525f-0764-4937-9fa8-022242647c0f in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_c2cbbf34-b852-4164-a572-01d4d79445a1 ac802759-2ac3-4c5c-b8e7-6d51689e1537 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_9b135c74-c09a-4dcc-ba19-f6f8deae135a 8144f1ee-dbef-45c1-9397-2031a9ca8b38 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 38fafd69-0275-4f4d-885d-3bbf2ea77551 in service 100 GiB none gzip-9 + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug ba445402-2775-4f97-87e9-640be3a00c6f in service 100 GiB none gzip-9 + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 4d322f8a-d461-4ead-995b-9ba7d76becad in service 100 GiB none gzip-9 + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/local_storage cb8d5371-640d-4364-8a5e-0fde125d28af in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/local_storage f4ea4a15-b3b3-4a67-b61a-8fba7c7ab61f in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/local_storage 647f344c-b4c5-4788-a732-40658e852f63 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/local_storage_unencrypted f306ebde-464c-49ef-94eb-1d72cf9afc7e in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/local_storage_unencrypted 2aeeb036-0e23-419f-82f7-9ab04c06d7ec in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/local_storage_unencrypted 53e5b466-aac5-4cf0-a93b-9f68f58ac755 in service none none off + + + omicron zones: + --------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + --------------------------------------------------------------------------------------------------------------- + clickhouse 10676bfe-b61f-40e1-bc07-a4cb76ea1f30 install dataset in service fd00:1122:3344:101::7 + crucible 3d9b7487-d7b9-4e25-960f-f2086f3e2919 install dataset in service fd00:1122:3344:101::10 + crucible 4b19d194-8b25-4396-88da-3df1b3788601 install dataset in service fd00:1122:3344:101::f + crucible c204730a-0946-4793-a470-64c88e89da96 install dataset in service fd00:1122:3344:101::e + crucible_pantry 2de1c525-4e04-4ba6-ac6b-377aaa542f96 install dataset in service fd00:1122:3344:101::d + crucible_pantry 31b26053-8b94-4d8c-9e4a-d7d720afe265 install dataset in service fd00:1122:3344:101::b + crucible_pantry f87ada51-4419-4144-86a8-e5e4ff0f64d3 install dataset in service fd00:1122:3344:101::c + external_dns 005548be-c5e4-49ff-a27b-88f314f1bc51 install dataset in service fd00:1122:3344:101::9 + external_dns 2205353a-e1d2-48ff-863b-9d6b1487d474 install dataset in service fd00:1122:3344:101::a + external_dns 70aab480-3d6c-47d5-aaf9-8d2ddab2931c install dataset in service fd00:1122:3344:101::8 + internal_dns 3fd06852-06cb-4d8a-b4b2-eb88ff5a6035 install dataset in service fd00:1122:3344:2::1 + internal_dns 45ce130e-c5ac-4e26-ab73-7589ba634418 install dataset in service fd00:1122:3344:1::1 + internal_dns a7b7bfbe-0588-4781-9a5e-fba63584e5d2 install dataset in service fd00:1122:3344:3::1 + internal_ntp 9b135c74-c09a-4dcc-ba19-f6f8deae135a install dataset in service fd00:1122:3344:101::3 + nexus 2bc0b0c4-63a9-44cc-afff-76ce645ef1d4 install dataset in service fd00:1122:3344:101::6 + nexus 3c2cd97f-7b4a-4ff3-a9d5-6ce141fcdbbe install dataset in service fd00:1122:3344:101::5 + nexus c2cbbf34-b852-4164-a572-01d4d79445a1 install dataset in service fd00:1122:3344:101::4 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) +* target release min gen: 1 -> 3 + nexus gen:::::::::::::: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 36 (records: 48) + +external DNS: + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 9) + + + + +> # Simulate the sled processing the recovery blueprint: it clears the +> # override marker AND advances its reconciled config generation. +> sled-set serial0 mupdate-override unset +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 mupdate override: 11111111-1111-1111-1111-111111111111 -> unset + +> sled-set serial0 omicron-config latest +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest blueprint (8da82a8e-bf97-4fbd-8ddd-9f6462732cf1) + + +> # Generate a fresh inventory. This inventory would not have a mupdate override, +> # and would advance the sled agent generation. +> inventory-generate +generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds + + +> # This planner run clears the "will remove mupdate override" field and +> # noop-convert zones to Artifact image sources. +> blueprint-plan latest latest +INFO inventory override no longer exists, blueprint override cleared, phase: do_plan_mupdate_override, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, prev_bp_override: 11111111-1111-1111-1111-111111111111 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 17, num_already_artifact: 0, num_eligible: 17, num_ineligible: 0 +INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a +INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b +INFO measurement is eligible for noop image source conversion, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, new_measurement_source: 8a0e23157bae655fceec7376926c9758efee6511c7b7ff8355bbb49545a2257f version 1.0.0 + +generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 +blueprint source: planner with report: +planning report: +* noop converting 17/17 install-dataset zones to artifact store on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +* noop converting current measurements to Artifact on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +* zone adds waiting on blockers +* zone adds and updates are blocked: + - current target release generation (2) is lower than minimum required by blueprint (3) +* zone updates waiting on zone add blockers +Measurement updates: +Waiting on zone add/update blockers + + + +> blueprint-diff latest +from: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 +to: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 + + MODIFIED SLEDS: + + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 3 -> 4): +- will remove mupdate override: 11111111-1111-1111-1111-111111111111 -> (none) + + measurements: + -------------------------------------------------------------------------------- + hash version + -------------------------------------------------------------------------------- +- install dataset (no version) ++ 8a0e23157bae655fceec7376926c9758efee6511c7b7ff8355bbb49545a2257f version 1.0.0 + + + host phase 2 contents: + ------------------------ + slot boot image source + ------------------------ + A current contents + B current contents + + + physical disks: + ------------------------------------------------------------------------------------ + vendor model serial disposition + ------------------------------------------------------------------------------------ + fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service + fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service + fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service + + + datasets: + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + dataset name dataset id disposition quota reservation compression + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 42bd3fb3-2aae-4754-9c41-0f9b50a77322 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible b7813bc0-d5ee-487d-95ad-6c5cce6fdcd6 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 6dd64de1-7dd3-492c-830f-d4a4e4c3dda7 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/clickhouse b1007e3d-7aaf-4ccb-9773-241dcbf79b21 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns d3cd3ec4-703f-4e73-8cc2-b4bc19c7d596 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/external_dns c7e90d9c-22db-43ca-a602-2a295bac7eec in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/external_dns bef875a9-3cf1-4b02-9682-c0286fb23d4d in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns 66788e15-bb3a-4369-a5cb-3357f7b85f64 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/internal_dns d88eb02f-b66d-4ae7-91a1-23154417c1f1 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/internal_dns ebfd425c-4e5d-4385-a105-8f2ce32fa9ec in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone e4b83186-c6e2-4d33-ae1b-803c32d1a86e in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 20cb5d98-bcd7-4a11-9190-7570a0aa6fe0 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone aa860565-2bbb-4a70-ae8e-7d8f29f1a536 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_clickhouse_10676bfe-b61f-40e1-bc07-a4cb76ea1f30 3523db58-10ab-4e2c-95d9-fec43036c0f5 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_3d9b7487-d7b9-4e25-960f-f2086f3e2919 32bb65ae-855f-461b-9203-32b2ddfe5a64 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_4b19d194-8b25-4396-88da-3df1b3788601 dd1954a2-7848-4950-a694-38870c339d3e in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_c204730a-0946-4793-a470-64c88e89da96 f3ddb3ca-388d-4462-9fc8-e552cf7de668 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_pantry_2de1c525-4e04-4ba6-ac6b-377aaa542f96 c6d7ecb4-135c-4f09-8948-b73fec13db76 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_31b26053-8b94-4d8c-9e4a-d7d720afe265 d4f66eab-2870-4a74-b4d5-ae5da3276682 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_pantry_f87ada51-4419-4144-86a8-e5e4ff0f64d3 f822a7e2-28aa-4ae2-ba86-d4e552a15bcb in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_external_dns_005548be-c5e4-49ff-a27b-88f314f1bc51 0c7e4b2a-502a-4f4c-8ac4-4068db25252e in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_external_dns_2205353a-e1d2-48ff-863b-9d6b1487d474 8355e747-b375-4d9e-b5da-03d9540ab5cd in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_70aab480-3d6c-47d5-aaf9-8d2ddab2931c 49dc7c86-c865-457f-8d58-37b92b002691 in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_internal_dns_3fd06852-06cb-4d8a-b4b2-eb88ff5a6035 506ddf0e-8cbf-44a9-93d7-130a550fd42d in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_45ce130e-c5ac-4e26-ab73-7589ba634418 d1611973-b22d-4097-b6fd-0a480f2199a5 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_internal_dns_a7b7bfbe-0588-4781-9a5e-fba63584e5d2 382cbbf2-8b08-4388-839a-b43b8bc82999 in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_nexus_2bc0b0c4-63a9-44cc-afff-76ce645ef1d4 6b672182-6d17-41d5-81d7-19debe7d3f9b in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_nexus_3c2cd97f-7b4a-4ff3-a9d5-6ce141fcdbbe 8b89525f-0764-4937-9fa8-022242647c0f in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_c2cbbf34-b852-4164-a572-01d4d79445a1 ac802759-2ac3-4c5c-b8e7-6d51689e1537 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_9b135c74-c09a-4dcc-ba19-f6f8deae135a 8144f1ee-dbef-45c1-9397-2031a9ca8b38 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 38fafd69-0275-4f4d-885d-3bbf2ea77551 in service 100 GiB none gzip-9 + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug ba445402-2775-4f97-87e9-640be3a00c6f in service 100 GiB none gzip-9 + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 4d322f8a-d461-4ead-995b-9ba7d76becad in service 100 GiB none gzip-9 + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/local_storage cb8d5371-640d-4364-8a5e-0fde125d28af in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/local_storage f4ea4a15-b3b3-4a67-b61a-8fba7c7ab61f in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/local_storage 647f344c-b4c5-4788-a732-40658e852f63 in service none none off + oxp_073979dd-3248-44a5-9fa1-cc72a140d682/local_storage_unencrypted f306ebde-464c-49ef-94eb-1d72cf9afc7e in service none none off + oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/local_storage_unencrypted 2aeeb036-0e23-419f-82f7-9ab04c06d7ec in service none none off + oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/local_storage_unencrypted 53e5b466-aac5-4cf0-a93b-9f68f58ac755 in service none none off + + + omicron zones: + ------------------------------------------------------------------------------------------------------------------------- + zone type zone id image source disposition underlay IP + ------------------------------------------------------------------------------------------------------------------------- +* clickhouse 10676bfe-b61f-40e1-bc07-a4cb76ea1f30 - install dataset in service fd00:1122:3344:101::7 + └─ + artifact: version 1.0.0 +* crucible 3d9b7487-d7b9-4e25-960f-f2086f3e2919 - install dataset in service fd00:1122:3344:101::10 + └─ + artifact: version 1.0.0 +* crucible 4b19d194-8b25-4396-88da-3df1b3788601 - install dataset in service fd00:1122:3344:101::f + └─ + artifact: version 1.0.0 +* crucible c204730a-0946-4793-a470-64c88e89da96 - install dataset in service fd00:1122:3344:101::e + └─ + artifact: version 1.0.0 +* crucible_pantry 2de1c525-4e04-4ba6-ac6b-377aaa542f96 - install dataset in service fd00:1122:3344:101::d + └─ + artifact: version 1.0.0 +* crucible_pantry 31b26053-8b94-4d8c-9e4a-d7d720afe265 - install dataset in service fd00:1122:3344:101::b + └─ + artifact: version 1.0.0 +* crucible_pantry f87ada51-4419-4144-86a8-e5e4ff0f64d3 - install dataset in service fd00:1122:3344:101::c + └─ + artifact: version 1.0.0 +* external_dns 005548be-c5e4-49ff-a27b-88f314f1bc51 - install dataset in service fd00:1122:3344:101::9 + └─ + artifact: version 1.0.0 +* external_dns 2205353a-e1d2-48ff-863b-9d6b1487d474 - install dataset in service fd00:1122:3344:101::a + └─ + artifact: version 1.0.0 +* external_dns 70aab480-3d6c-47d5-aaf9-8d2ddab2931c - install dataset in service fd00:1122:3344:101::8 + └─ + artifact: version 1.0.0 +* internal_dns 3fd06852-06cb-4d8a-b4b2-eb88ff5a6035 - install dataset in service fd00:1122:3344:2::1 + └─ + artifact: version 1.0.0 +* internal_dns 45ce130e-c5ac-4e26-ab73-7589ba634418 - install dataset in service fd00:1122:3344:1::1 + └─ + artifact: version 1.0.0 +* internal_dns a7b7bfbe-0588-4781-9a5e-fba63584e5d2 - install dataset in service fd00:1122:3344:3::1 + └─ + artifact: version 1.0.0 +* internal_ntp 9b135c74-c09a-4dcc-ba19-f6f8deae135a - install dataset in service fd00:1122:3344:101::3 + └─ + artifact: version 1.0.0 +* nexus 2bc0b0c4-63a9-44cc-afff-76ce645ef1d4 - install dataset in service fd00:1122:3344:101::6 + └─ + artifact: version 1.0.0 +* nexus 3c2cd97f-7b4a-4ff3-a9d5-6ce141fcdbbe - install dataset in service fd00:1122:3344:101::5 + └─ + artifact: version 1.0.0 +* nexus c2cbbf34-b852-4164-a572-01d4d79445a1 - install dataset in service fd00:1122:3344:101::4 + └─ + artifact: version 1.0.0 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 3 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 36 (records: 48) + +external DNS: + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 9) + + + + +> # Now plan against the stale inventory from earlier. The expected behavior +> # is that the planner detects inventory staleness on the BpSetOverride path and +> # does nothing. As a result, the blueprint diff should be empty. +> blueprint-plan latest eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 +INFO inventory override observed, but blueprint override was not set to match it, phase: do_plan_mupdate_override, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, inv_override: 11111111-1111-1111-1111-111111111111, bp_override: None, reason: inventory stale: inventory reported generation (2) that is older than the generation in the parent blueprint (4) +INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, reason: inventory stale: inventory reported generation (2) that is older than the generation in the parent blueprint (4) +generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 +blueprint source: planner with report: +planning report: +* zone adds waiting on blockers +* zone adds and updates are blocked: + - current target release generation (2) is lower than minimum required by blueprint (3) +* zone updates waiting on zone add blockers +Measurement updates: +Waiting on zone add/update blockers + + + +> blueprint-diff latest +from: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 +to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 3 (unchanged) + nexus gen:::::::::::::: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 36 (records: 48) + +external DNS: + DNS zone: "oxide.example" (unchanged) + unchanged names: 5 (records: 9) + + + diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 059825b5621..89aad17a547 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -6,6 +6,7 @@ use crate::blueprint_editor::DiskExpungeDetails; use crate::blueprint_editor::EditedSled; +use crate::blueprint_editor::EnsureMupdateOverrideError; use crate::blueprint_editor::ExternalNetworkingChoice; use crate::blueprint_editor::ExternalNetworkingError; use crate::blueprint_editor::ExternalSnatNetworkingChoice; @@ -1481,7 +1482,16 @@ impl<'a> BlueprintBuilder<'a> { pending_mgs_update, noop_sled_info, ) - .map_err(|err| Error::SledEditError { sled_id, err }) + .map_err(|err| match err { + EnsureMupdateOverrideError::SledEdit(err) => { + Error::SledEditError { sled_id, err } + } + EnsureMupdateOverrideError::Planner(err) => { + Error::Planner(err.context(format!( + "for sled {sled_id}, programming error ensuring mupdate override" + ))) + } + }) } fn next_internal_dns_gz_address_index(&self, sled_id: SledUuid) -> u32 { @@ -2467,6 +2477,19 @@ pub(crate) enum EnsureMupdateOverrideAction { /// The reason the blueprint override was not cleared. reason: BpMupdateOverrideNotClearedReason, }, + /// The inventory had an override, but the blueprint's + /// `remove_mupdate_override` field was not set to match it. This happens + /// when the sled's inventory is stale (older than the parent blueprint), so + /// acting on it would potentially overwrite a previously acknowledged + /// mupdate override. + BpOverrideNotSet { + /// The override observed in (stale) inventory. + inv_override: MupdateOverrideUuid, + /// The current blueprint override value, left unchanged. + bp_override: Option, + /// The reason the blueprint override was not set. + reason: BpMupdateOverrideNotSetReason, + }, /// Sled Agent encountered an error retrieving the mupdate override from the /// inventory. /// @@ -2557,6 +2580,20 @@ impl EnsureMupdateOverrideAction { "reason" => %reason, ); } + EnsureMupdateOverrideAction::BpOverrideNotSet { + inv_override, + bp_override, + reason, + } => { + info!( + log, + "inventory override observed, but blueprint override \ + was not set to match it"; + "inv_override" => %inv_override, + "bp_override" => ?bp_override, + "reason" => %reason, + ); + } EnsureMupdateOverrideAction::GetOverrideError { message, bp_override, @@ -2722,6 +2759,38 @@ impl fmt::Display for BpMupdateOverrideNotClearedReason { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum BpMupdateOverrideNotSetReason { + /// The sled's inventory is stale relative to the parent blueprint, so we + /// can't trust an override observed in inventory to reflect current + /// reality. + InventoryStale { parent_bp_gen: Generation, inventory_gen: Generation }, + /// The sled has not yet successfully reconciled any config, so we have no + /// generation to compare against. + NoLastReconciliation, +} + +impl fmt::Display for BpMupdateOverrideNotSetReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BpMupdateOverrideNotSetReason::InventoryStale { + parent_bp_gen, + inventory_gen, + } => { + write!( + f, + "inventory stale: inventory reported generation \ + ({inventory_gen}) that is older than the generation in \ + the parent blueprint ({parent_bp_gen})", + ) + } + BpMupdateOverrideNotSetReason::NoLastReconciliation => { + write!(f, "sled doesn't have a last reconciled config") + } + } + } +} + fn is_external_networking_config_different( blueprint1: &Blueprint, blueprint2: &Blueprint, diff --git a/nexus/reconfigurator/planning/src/blueprint_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor.rs index b70f2e44b31..a4fabc060db 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor.rs @@ -15,6 +15,7 @@ pub use allocators::ExternalNetworkingError; pub use allocators::ExternalSnatNetworkingChoice; pub use sled_editor::DatasetsEditError; pub use sled_editor::DisksEditError; +pub use sled_editor::EnsureMupdateOverrideError; pub use sled_editor::MultipleDatasetsOfKind; pub use sled_editor::SledEditError; pub use sled_editor::SledInputError; diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs index d4787202d26..1622323dced 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs @@ -5,6 +5,7 @@ //! Support for editing the blueprint details of a single sled. use crate::blueprint_builder::BpMupdateOverrideNotClearedReason; +use crate::blueprint_builder::BpMupdateOverrideNotSetReason; use crate::blueprint_builder::EditedSledScalarEdits; use crate::blueprint_builder::EnsureMupdateOverrideAction; use crate::blueprint_builder::EnsureMupdateOverrideUpdatedZone; @@ -15,6 +16,7 @@ use crate::planner::NoopConvertSledIneligibleReason; use crate::planner::NoopConvertSledInfoMut; use crate::planner::NoopConvertSledStatus; use crate::planner::SledPlannerRng; +use anyhow::anyhow; use host_phase_2::HostPhase2Editor; use iddqd::IdOrdMap; use iddqd::id_ord_map::Entry; @@ -148,6 +150,18 @@ pub enum SledEditError { }, } +/// Error returned by `SledEditor::ensure_mupdate_override`. +/// +/// Distinct from [`SledEditError`] so that the caller can route the `Planner` +/// arm to `Error::Planner`. +#[derive(Debug, thiserror::Error)] +pub enum EnsureMupdateOverrideError { + #[error(transparent)] + SledEdit(#[from] SledEditError), + #[error("programming error in planner")] + Planner(#[source] anyhow::Error), +} + #[derive(Debug)] pub struct SledEditor { underlay_ip_allocator: SledUnderlayIpAllocator, @@ -581,7 +595,7 @@ impl SledEditor { >, pending_mgs_update: Entry<'_, PendingMgsUpdate>, noop_sled_info: NoopConvertSledInfoMut<'_>, - ) -> Result { + ) -> Result { match (inv_mupdate_override_info, *self.remove_mupdate_override.value()) { (Ok(Some(inv_override)), Some(bp_override)) @@ -597,9 +611,94 @@ impl SledEditor { (Ok(Some(inv_override)), bp_override) => { // Inventory says there's an override in place, but the // blueprint doesn't (or has a different override in place). - // This means that a MUPdate happened since we last did - // blueprint planning. + // This normally means that a MUPdate happened since we last + // did blueprint planning. // + // However, if the sled's inventory is stale or the sled has no + // reconciled config, we can't trust the inventory's claim that + // the override exists. Acting on stale inventory here would + // rewind the mupdate/update state machine to an earlier point + // and bump `target_release_minimum_generation` a second time. + if let NoopConvertSledInfoMut::Ok(info) = &noop_sled_info { + use NoopConvertSledIneligibleReason::*; + let stale_reason = match &info.status { + NoopConvertSledStatus::Ineligible(InventoryStale { + parent_bp_gen, + inventory_gen, + }) => Some( + BpMupdateOverrideNotSetReason::InventoryStale { + parent_bp_gen: *parent_bp_gen, + inventory_gen: *inventory_gen, + }, + ), + NoopConvertSledStatus::Ineligible( + NoLastReconciliation, + ) => Some( + BpMupdateOverrideNotSetReason::NoLastReconciliation, + ), + + // ManifestError: the mupdate override is read from + // boot-partition info, which is independent of the + // zone manifest, so manifest errors don't invalidate + // us observing the override. + // + // MupdateOverride: the equal-id case is handled by + // the prior match arm above, so we're here because + // the inventory and blueprint override IDs differ; + // we want to swap in the new one. + // + // Eligible: the standard "MUPdate happened" path. + NoopConvertSledStatus::Ineligible( + ManifestError { .. } | MupdateOverride { .. }, + ) + | NoopConvertSledStatus::Eligible(_) => None, + + // The remaining variants represent invariant + // violations: + // + // * NotInInventory: do_plan_mupdate_override skips + // sleds missing from inventory before calling + // ensure_mupdate_override, and + // inv_mupdate_override_info was read from the same + // inventory entry. Ok(Some(_)) implies the sled is + // in inventory. + // * MupdateOverrideError: never constructed by + // NoopConvertInfo::new; only assigned as a side + // effect of the Err arm of this same method, and + // this method runs at most once per sled per + // planning round. + NoopConvertSledStatus::Ineligible(NotInInventory) => { + return Err(EnsureMupdateOverrideError::Planner( + anyhow!( + "inventory reported a mupdate \ + override, but noop_sled_info \ + reports NotInInventory" + ), + )); + } + NoopConvertSledStatus::Ineligible( + MupdateOverrideError { .. }, + ) => { + return Err(EnsureMupdateOverrideError::Planner( + anyhow!( + "inventory reported a mupdate \ + override, but noop_sled_info \ + reports MupdateOverrideError" + ), + )); + } + }; + if let Some(reason) = stale_reason { + return Ok( + EnsureMupdateOverrideAction::BpOverrideNotSet { + inv_override: inv_override.mupdate_override_id, + bp_override, + reason, + }, + ); + } + } + // Set the blueprint's remove_mupdate_override. self.set_remove_mupdate_override(Some( inv_override.mupdate_override_id, @@ -645,10 +744,13 @@ impl SledEditor { let mut zones = IdOrdMap::with_capacity(zone_ids.len()); for (zone_id, kind) in zone_ids { - let old_image_source = self.zones.set_zone_image_source( - &zone_id, - BlueprintZoneImageSource::InstallDataset, - )?; + let old_image_source = self + .zones + .set_zone_image_source( + &zone_id, + BlueprintZoneImageSource::InstallDataset, + ) + .map_err(SledEditError::from)?; let item = EnsureMupdateOverrideUpdatedZone { zone_id, kind, @@ -733,7 +835,8 @@ impl SledEditor { SledEditError::NoopMupdateOverrideMismatch { noop_id: Some(*mupdate_override_id), blueprint_id: bp_override, - }, + } + .into(), ) } } @@ -749,7 +852,8 @@ impl SledEditor { Err(SledEditError::NoopMupdateOverrideMismatch { noop_id: None, blueprint_id: bp_override, - }) + } + .into()) } }, NoopConvertSledInfoMut::GlobalIneligible(