kubevirt: keep domain bookkeeping until Cleanup

evetest/README.md

@@ -399,7 +399,11 @@ Use the CLI to inspect state, then run `evetest continue` to resume.
 - **Reuse existing package-level helpers** before writing new ones. Each test package
   has shared helpers for common patterns — for example the networking package has
   `getDevicePort`, `getCurrentDPC`, `appHasError`, `niHasError`. Check the other
-  `_test.go` files in the package before duplicating logic.
+  `_test.go` files in the package before duplicating logic. When a package
+  accumulates enough shared helpers to need files of their own, name each file for
+  the state it observes — not for the test that first needed it — and record the
+  layout in the package comment in that package's `testsuite_test.go`; see
+  `tests/apps/` for a worked example. Check that comment before adding a helper.
 
 - **Do not mutate shared global state.** If a test needs to modify a package-level
   variable (e.g. a network model defined in `evetest/netmodels/`), operate on a deep

evetest/edgecluster.go

@@ -164,13 +164,27 @@ func clusterNodeReady(info *eveinfo.ZInfoKubeCluster, nodeName string) bool {
 // It watches ZInfoKubeCluster updates from all devices and returns the device
 // whose cluster info reports the app (matched by display name) in EveApps
 // or EveVmApps with a non-empty NodeName.
-func (ec *EdgeCluster) FindDeviceHostingApp(
-	appUUID uuid.UUID, timeout time.Duration) *EdgeDevice {
+//
+// Pass excludeDevNames to wait for the app to move OFF the named devices, which
+// is what a failover test needs. Without it, this can return a device that no
+// longer hosts the app: each device's last published cluster info is consulted
+// first, and that snapshot can predate the event the caller is waiting for -
+// most visibly when the excluded device is powered off, since its own stale
+// snapshot still names it as the host and it publishes nothing further.
+// Excluded devices are skipped as a source of cluster info and are never
+// returned, so the answer can only come from a device that is still up.
+func (ec *EdgeCluster) FindDeviceHostingApp(appUUID uuid.UUID,
+	timeout time.Duration, excludeDevNames ...string) *EdgeDevice {
 	ec.checkDevices("FindDeviceHostingApp")
 	ctx, cancel := context.WithTimeout(ec.th.ctx, timeout)
 	defer cancel()
 	appUUIDStr := appUUID.String()
 
+	excluded := make(map[string]bool, len(excludeDevNames))
+	for _, name := range excludeDevNames {
+		excluded[name] = true
+	}
+
 	// Look up the app display name from the first device's config.
 	var appDisplayName string
 	for _, dev := range ec.devices {
@@ -190,24 +204,29 @@ func (ec *EdgeCluster) FindDeviceHostingApp(
 			appUUID)
 	}
 
-	// First check already published cluster info from all devices.
-	for _, dev := range ec.devices {
-		if info := dev.GetClusterInfo(); info != nil {
-			if nodeName := findAppNodeName(info, appDisplayName); nodeName != "" {
-				for _, d := range ec.devices {
-					if d.devName == nodeName {
-						return d
-					}
-				}
-				ec.th.t.Fatalf("Node %q reports hosting app %q, but no matching "+
-					"device was found in cluster %q",
-					nodeName, appUUID, ec.clusterName)
+	// hostingNode returns the node the info reports as hosting the app, unless
+	// that node is excluded - an excluded node is what the caller is waiting for
+	// the app to leave, so reporting it is never the answer.
+	hostingNode := func(info *eveinfo.ZInfoKubeCluster) string {
+		nodeName := findAppNodeName(info, appDisplayName)
+		if nodeName == "" || excluded[nodeName] {
+			return ""
+		}
+		return nodeName
+	}
+
+	// deviceByName maps a reported node name back to a cluster device.
+	deviceByName := func(nodeName string) *EdgeDevice {
+		for _, dev := range ec.devices {
+			if dev.devName == nodeName {
+				return dev
 			}
 		}
+		ec.th.t.Fatalf("Node %q reports hosting app %q, but no matching device "+
+			"was found in cluster %q", nodeName, appUUID, ec.clusterName)
+		return nil
 	}
 
-	// Subscribe to cluster info from all devices and wait for the app
-	// to appear with a node name.
 	type result struct {
 		nodeName string
 	}
@@ -216,18 +235,25 @@ func (ec *EdgeCluster) FindDeviceHostingApp(
 	subCtx, subCancel := context.WithCancel(ctx)
 	defer subCancel()
 
+	// Subscribe before taking the cached snapshot below, so an update landing
+	// between the two is not missed.
 	for _, dev := range ec.devices {
+		if excluded[dev.devName] {
+			// A device the app must move off is not a trustworthy source: if it
+			// is powered off it publishes nothing, and its last snapshot is
+			// stale by definition.
+			continue
+		}
 		updates, stop := dev.WatchClusterInfo()
-		go func(dev *EdgeDevice, updates <-chan *eveinfo.ZInfoKubeCluster, stop func()) {
+		go func(updates <-chan *eveinfo.ZInfoKubeCluster, stop func()) {
 			defer stop()
 			for {
 				select {
 				case info, ok := <-updates:
 					if !ok {
 						return
 					}
-					nodeName := findAppNodeName(info, appDisplayName)
-					if nodeName != "" {
+					if nodeName := hostingNode(info); nodeName != "" {
 						select {
 						case resultCh <- result{nodeName: nodeName}:
 						default:
@@ -238,21 +264,31 @@ func (ec *EdgeCluster) FindDeviceHostingApp(
 					return
 				}
 			}
-		}(dev, updates, stop)
+		}(updates, stop)
+	}
+
+	// Then check the cluster info each still-eligible device has already
+	// published, so the common case does not have to wait for a fresh message.
+	for _, dev := range ec.devices {
+		if excluded[dev.devName] {
+			continue
+		}
+		if info := dev.GetClusterInfo(); info != nil {
+			if nodeName := hostingNode(info); nodeName != "" {
+				return deviceByName(nodeName)
+			}
+		}
 	}
 
 	select {
 	case res := <-resultCh:
 		subCancel()
-		// Map node name back to an EdgeDevice.
-		for _, dev := range ec.devices {
-			if dev.devName == res.nodeName {
-				return dev
-			}
-		}
-		ec.th.t.Fatalf("Node %q reports hosting app %q, but no matching device "+
-			"was found in cluster %q", res.nodeName, appUUID, ec.clusterName)
+		return deviceByName(res.nodeName)
 	case <-ctx.Done():
+		if len(excludeDevNames) > 0 {
+			ec.th.t.Fatalf("Timed out waiting for app %q to move off %v in cluster %q",
+				appUUID, excludeDevNames, ec.clusterName)
+		}
 		ec.th.t.Fatalf("Timed out waiting for app %q to be scheduled in cluster %q",
 			appUUID, ec.clusterName)
 	}
@@ -261,7 +297,7 @@ func (ec *EdgeCluster) FindDeviceHostingApp(
 
 // findAppNodeName checks if the given cluster info contains the app
 // (by display name) in EveApps or EveVmApps with a non-empty NodeName.
-// Kubernetes adds a hash suffix to the display name (e.g. "my-app-584dbd8fnx"),
+// Kubernetes adds a hash suffix to the display name (e.g. "my-app-abcde12345"),
 // so we match by prefix with a "-" separator.
 func findAppNodeName(info *eveinfo.ZInfoKubeCluster, appDisplayName string) string {
 	prefix := appDisplayName + "-"

evetest/edgedevice.go

@@ -50,6 +50,11 @@ func GetEdgeDevice(devName string) *EdgeDevice {
 	return &EdgeDevice{th: th, devName: devName}
 }
 
+// Name returns the device name.
+func (d *EdgeDevice) Name() string {
+	return d.devName
+}
+
 // GetAllEdgeDevices returns handles for all EdgeDevices currently known to the
 // test th.
 func GetAllEdgeDevices() (devices []*EdgeDevice) {

evetest/tests/apps/appstate_test.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2026 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Pillar's own view of the app instance: pubsub publications and persisted state
+// keyed by app UUID.
+//
+// Rule for this file: one reader per (agent, topic) question. Tests do not call
+// evetest.ReadPublication or ReadAllPublications directly - they call a named
+// reader here, so the knowledge of which agent publishes what, and how a
+// transient read failure is reported, lives in one place.
+
+package apps_test
+
+import (
+	"strconv"
+
+	"github.com/lf-edge/eve/evetest"
+	"github.com/lf-edge/eve/pkg/pillar/types"
+	uuid "github.com/satori/go.uuid"
+)
+
+// appPublication returns the publication of type T that belongs to appUUID.
+//
+// Every pillar status type keyed by app instance has a Key() returning the app
+// UUID string (types.AppInstanceStatus.Key, types.DomainStatus.Key), which is
+// what the constraint expresses. evetest.ReadAllPublications derives the pubsub
+// topic name from T itself, so instantiating this is all a new reader needs.
+// Note that types.VolumeStatus does NOT satisfy the intent here even though it
+// has a Key(): its key is "<volume-uuid>#<generation>", not an app UUID - see
+// soleVolumeStatus in appvolumes_test.go for why volumes cannot be attributed to
+// an app this way at all.
+func appPublication[T interface{ Key() string }](
+	dev *evetest.EdgeDevice, agent string, appUUID uuid.UUID) (item T, found bool) {
+	pubs, err := evetest.ReadAllPublications[T](dev, agent, false)
+	if err != nil {
+		// Transient: a publication can vanish between being listed and being
+		// copied. Report not-found and let the caller's Eventually retry.
+		evetest.Logger().Warnf("appPublication: reading %s publications: %v", agent, err)
+		return item, false
+	}
+	for _, pub := range pubs {
+		if pub.Key() == appUUID.String() {
+			return pub, true
+		}
+	}
+	return item, false
+}
+
+// appPurgePhase returns the app's state and purge phase as zedmanager itself
+// publishes them. This is what distinguishes "the purge finished" from "the purge
+// is wedged": a purge parked on a VolumeRefStatus removal volumemgr will never
+// confirm stays in DownloadAndVerify indefinitely and never even requests the new
+// volume. That code is in zedmanager, so the failure mode is not specific to a
+// hypervisor.
+func appPurgePhase(dev *evetest.EdgeDevice, appUUID uuid.UUID) (
+	state types.SwState, purge types.Inprogress, found bool) {
+	status, found := appPublication[types.AppInstanceStatus](dev, "zedmanager", appUUID)
+	if !found {
+		return state, purge, false
+	}
+	return status.State, status.PurgeInprogress, true
+}
+
+// appDomainStatus returns domainmgr's published DomainStatus for the app. There
+// is at most one, because DomainStatus is keyed by app UUID - which is exactly
+// why it cannot be used to count workload generations (see listAppVMIRS and
+// listKVMDomainDirs in appworkload_test.go). It is authoritative for the
+// domain's id, name and attached disks.
+func appDomainStatus(
+	dev *evetest.EdgeDevice, appUUID uuid.UUID) (types.DomainStatus, bool) {
+	return appPublication[types.DomainStatus](dev, "domainmgr", appUUID)
+}
+
+// purgeCounter reads the persisted purge counter zedmanager keeps for appUUID
+// (pkg/pillar/types.UuidToNum, NumType "purgeCmdCounter"). This counter is
+// exactly what a reboot mid-purge can corrupt - advancing the purge phase
+// before the old generation is actually gone - and it is not republished
+// anywhere in the EVE API, so it is read from the persisted pubsub state.
+//
+// found is false while the file does not exist, which is the expected state
+// before an app's first purge.
+func purgeCounter(
+	dev *evetest.EdgeDevice, appUUID uuid.UUID) (counter uint32, found bool) {
+	var rec types.UuidToNum
+	if err := evetest.ReadPublication(
+		dev, "zedmanager", true, appUUID.String(), &rec); err != nil {
+		// Absent before the app's first purge, which is expected; a transient
+		// read failure lands here too and the caller's retry absorbs it.
+		return 0, false
+	}
+	return uint32(rec.Number), true
+}
+
+// inprogressName gives types.Inprogress a readable form for failure messages.
+// The type has no String method of its own.
+func inprogressName(p types.Inprogress) string {
+	switch p {
+	case types.NotInprogress:
+		return "NotInprogress"
+	case types.DownloadAndVerify:
+		return "DownloadAndVerify"
+	case types.BringDown:
+		return "BringDown"
+	case types.RecreateVolumes:
+		return "RecreateVolumes"
+	case types.BringUp:
+		return "BringUp"
+	}
+	return "Inprogress(" + strconv.Itoa(int(p)) + ")"
+}