From 89ee25abc5c5f4b6f51a66a30acb112d7857e3de Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Thu, 30 Jul 2026 14:47:52 +0200 Subject: [PATCH 1/4] zedrouter, zedagent: rename NI status field to CombinedNTPServers ZInfoApp was not reporting all NTP servers for an application's network instance: NetworkInstanceConfig.NtpServers (the NI's own raw configured list) and NetworkInstanceStatus.NTPServers (the combined NI+port list zedrouter actually maintains) differed only by case, so the reporting code in zedagent silently picked up the wrong one via field promotion through the embedded config struct. Renamed the status field to CombinedNTPServers to make the two unambiguous by name, and fixed zedagent to publish it. Signed-off-by: Milan Lenco Co-Authored-By: Claude Sonnet 5 --- pkg/pillar/cmd/zedagent/handlemetrics.go | 2 +- pkg/pillar/cmd/zedrouter/networkinstance.go | 4 ++-- pkg/pillar/types/zedroutertypes.go | 12 ++++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/pillar/cmd/zedagent/handlemetrics.go b/pkg/pillar/cmd/zedagent/handlemetrics.go index 35f3461623a..2259bc44923 100644 --- a/pkg/pillar/cmd/zedagent/handlemetrics.go +++ b/pkg/pillar/cmd/zedagent/handlemetrics.go @@ -1232,7 +1232,7 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string, networkInfo.DevName = *proto.String(name) niStatus := appIfnameToNetworkInstance(ctx, aiStatus, ifname) if niStatus != nil { - networkInfo.NtpServers = utils.ToStrings(niStatus.NtpServers) + networkInfo.NtpServers = utils.ToStrings(niStatus.CombinedNTPServers) networkInfo.DefaultRouters = []string{niStatus.Gateway.String()} networkInfo.Dns = &info.ZInfoDNS{ DNSservers: []string{}, diff --git a/pkg/pillar/cmd/zedrouter/networkinstance.go b/pkg/pillar/cmd/zedrouter/networkinstance.go index fb5bfe61372..c6b3ee058a3 100644 --- a/pkg/pillar/cmd/zedrouter/networkinstance.go +++ b/pkg/pillar/cmd/zedrouter/networkinstance.go @@ -256,9 +256,9 @@ func (z *zedrouter) updateNIPorts(niConfig types.NetworkInstanceConfig, newNTPServers = generics.FilterDuplicatesFn(newNTPServers, netutils.EqualHostnameOrIPs) changed = changed || !generics.EqualSets(niStatus.Ports, validatedPortLLs) niStatus.Ports = validatedPortLLs - changed = changed || !generics.EqualSetsFn(niStatus.NTPServers, newNTPServers, + changed = changed || !generics.EqualSetsFn(niStatus.CombinedNTPServers, newNTPServers, netutils.EqualHostnameOrIPs) - niStatus.NTPServers = newNTPServers + niStatus.CombinedNTPServers = newNTPServers // Update BridgeMac for Switch NI bridge created by NIM. if z.niBridgeIsCreatedByNIM(niConfig) { // Only switch NI with single port may have the bridge created by NIM. diff --git a/pkg/pillar/types/zedroutertypes.go b/pkg/pillar/types/zedroutertypes.go index 7a66105a35b..9c29c6d80b6 100644 --- a/pkg/pillar/types/zedroutertypes.go +++ b/pkg/pillar/types/zedroutertypes.go @@ -1090,10 +1090,14 @@ type NetworkInstanceStatus struct { // Labels of device ports used for external connectivity. // The list is empty for air-gapped network instances. Ports []string - // List of NTP servers published to applications connected to this network instance. - // This includes the NTP server from the NI config (if any) and all NTP servers - // associated with ports used by the network instance for external connectivity. - NTPServers []netutils.HostnameOrIP + // CombinedNTPServers is the list of NTP servers published to applications + // connected to this network instance. It combines the NTP server from the + // NI config (if any) with all NTP servers associated with ports used by + // the network instance for external connectivity. Named distinctly from + // the embedded NetworkInstanceConfig.NtpServers (the NI-only raw config) + // since the two previously differed only by case, which let a caller + // silently read the wrong one through field promotion. + CombinedNTPServers []netutils.HostnameOrIP // The intended state of the routing table. // Includes user-configured static routes and potentially also automatically // generated default route. From bf6012ceea4281f2bde2a50ae282d0d12beb007d Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Fri, 31 Jul 2026 16:01:16 +0200 Subject: [PATCH 2/4] evetest/sdn: fix dnsmasq router option not being suppressed When a DHCP server's GatewayIPv4 is left nil (the documented way to make clients not install a default IPv4 route), the config renderer simply omitted the dhcp-option=option:router line instead of explicitly suppressing it. dnsmasq's own default, absent an explicit override, is to advertise its own listening address on that subnet as the router, so clients still received one -- defeating the purpose of leaving GatewayIPv4 unset. Signed-off-by: Milan Lenco Co-Authored-By: Claude Sonnet 5 Signed-off-by: Milan Lenco --- evetest/sdn/VERSION | 2 +- evetest/sdn/vm/pkg/configitems/dhcpSrv.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/evetest/sdn/VERSION b/evetest/sdn/VERSION index d5ab32c46f5..bc0a981c008 100644 --- a/evetest/sdn/VERSION +++ b/evetest/sdn/VERSION @@ -1,2 +1,2 @@ # Evetest-SDN version. Increment this manually whenever changes are made to evetest/sdn/vm. -1.0 +1.1 diff --git a/evetest/sdn/vm/pkg/configitems/dhcpSrv.go b/evetest/sdn/vm/pkg/configitems/dhcpSrv.go index d9309e4c370..237f26b562e 100644 --- a/evetest/sdn/vm/pkg/configitems/dhcpSrv.go +++ b/evetest/sdn/vm/pkg/configitems/dhcpSrv.go @@ -271,6 +271,13 @@ func (c *DhcpServerConfigurator) createDnsmasqConfFile(server DhcpServer) error if err := writeLine("dhcp-option=option:router,%s\n", server.GatewayIPv4.String()); err != nil { return err } + } else { + // Explicitly suppress option 3. Without this, dnsmasq falls back + // to advertising its own listening address on this subnet as the + // router. + if err := writeLine("dhcp-option=option:router\n"); err != nil { + return err + } } if server.DomainName != "" { // DHCP option 15. From 1c02f7827874d2b933d24c067e6badbc0e1ddca2 Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Fri, 31 Jul 2026 16:03:03 +0200 Subject: [PATCH 3/4] evetest: framework/library enhancements Add the EdgeDevice/EdgeCluster/harness capabilities needed by the eden-to-evetest test rewrites: - Flow-log streaming: AdamClient.IterateDeviceFlowLogs/ SubscribeToDeviceFlowLogs, the corresponding gRPC server iterator, and the `evetest eve flow-logs` CLI subcommand. - EdgeDevice.GetArch, PowerOff, PowerOn and WaitForClusterNodeIsReady. - EdgeDevice.DialViaSSH (an SSH-tunneled net.Conn, used for reaching the Kubevirt VNC proxy which only binds to the device's loopback). Also removes the now-unused SkipIfHypervisorKubevirt test-parameter helper; its call sites move to an inline `if hypervisor == evetest.HypervisorKubevirt` check in the test-suite rewrite commit that follows. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Milan Lenco --- evetest/README.md | 2 +- evetest/VERSION | 2 +- evetest/cli/evecmd.go | 17 +- evetest/constants/config.go | 4 +- evetest/controller/adam.go | 225 ++++++++++++ evetest/devconfig.go | 33 +- evetest/edgecluster.go | 39 +- evetest/edgedevice.go | 699 ++++++++++++++++++++++++++++-------- evetest/go.mod | 1 + evetest/go.sum | 3 + evetest/grpcserver.go | 52 ++- evetest/harness.go | 5 + evetest/ssh.go | 68 +++- evetest/testparam.go | 13 - 14 files changed, 976 insertions(+), 187 deletions(-) diff --git a/evetest/README.md b/evetest/README.md index 531e08c2362..e690e63e215 100644 --- a/evetest/README.md +++ b/evetest/README.md @@ -751,7 +751,7 @@ same terminal session beforehand. | `EVETEST_ORG` | Docker Hub organization for evetest and evetest-broker images | `lfedge` | | `EVETEST_EVE_REPO` | EVE image repository | `lfedge/eve` | | `EVETEST_ADAM_VERSION` | Adam controller version *(build-time only, see note below)* | `0.0.75` | -| `EVETEST_SDN_VERSION` | SDN emulator version | `1.0` | +| `EVETEST_SDN_VERSION` | SDN emulator version | `1.1` | > **`EVETEST_ADAM_VERSION` requires evetest container rebuild.** Unlike the other > variables above, Adam's binary is baked into the evetest image at build time diff --git a/evetest/VERSION b/evetest/VERSION index 7d9bfdfb779..d7cec1d7df4 100644 --- a/evetest/VERSION +++ b/evetest/VERSION @@ -1,2 +1,2 @@ # Evetest version. Increment this manually whenever changes are made to the evetest framework. -1.0 +1.1 diff --git a/evetest/cli/evecmd.go b/evetest/cli/evecmd.go index 17081d8d61a..f2b475695f9 100644 --- a/evetest/cli/evecmd.go +++ b/evetest/cli/evecmd.go @@ -468,22 +468,27 @@ func eveAppFlowLogsCmd() *cobra.Command { if err != nil { return fmt.Errorf("stream error: %w", err) } - var lines []string + var newEntries []string for _, ipFlow := range resp.IpFlows { - lines = append(lines, fmt.Sprintf("IP flow: %s", ipFlow.String())) + newEntries = append(newEntries, + fmt.Sprintf("IP flow: %s", ipFlow.String())) } for _, dnsReq := range resp.DnsRequests { - lines = append(lines, fmt.Sprintf("DNS request: %s", dnsReq.String())) + newEntries = append(newEntries, + fmt.Sprintf("DNS request: %s", dnsReq.String())) } - entry := strings.Join(lines, "\n") if tail > 0 { - entries = append(entries, entry) + entries = append(entries, newEntries...) } else { - fmt.Println(entry) + for _, entry := range newEntries { + fmt.Println(entry) + fmt.Println() + } } } for _, e := range tailEntries(entries, tail) { fmt.Println(e) + fmt.Println() } return nil }, diff --git a/evetest/constants/config.go b/evetest/constants/config.go index 05b79577ec8..c6c46d1d121 100644 --- a/evetest/constants/config.go +++ b/evetest/constants/config.go @@ -276,10 +276,10 @@ const ( DefaultSDNRepo = "lfedge/evetest-sdn" // DefaultAdamVersion specifies the Adam version to use by default. - DefaultAdamVersion = "0.0.75" + DefaultAdamVersion = "0.0.81" // DefaultSDNVersion specifies the SDN version to use by default. - DefaultSDNVersion = "1.0" + DefaultSDNVersion = "1.1" // DefaultSDNUplinkIPv4Subnet species the IPv4 subnet used for SDN uplink // interfaces by default. diff --git a/evetest/controller/adam.go b/evetest/controller/adam.go index 769243607c2..1017075536e 100644 --- a/evetest/controller/adam.go +++ b/evetest/controller/adam.go @@ -31,6 +31,7 @@ import ( evecerts "github.com/lf-edge/eve-api/go/certs" eveconfig "github.com/lf-edge/eve-api/go/config" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evelogs "github.com/lf-edge/eve-api/go/logs" evemetrics "github.com/lf-edge/eve-api/go/metrics" @@ -168,6 +169,16 @@ type MetricMsgIterator interface { Iterate(msg *evemetrics.ZMetricMsg) (stop bool, err error) } +// FlowMsgIterator iterates over device flow log messages (FlowMessage), each +// of which carries both flow records and DNS request records for one +// application VIF. Iterate is called for each message that passes the match +// filter. Returning stop=true signals that no further messages are needed +// and iteration should stop cleanly. Returning a non-nil error aborts +// iteration and propagates the error to the caller. +type FlowMsgIterator interface { + Iterate(msg *eveflowlog.FlowMessage) (stop bool, err error) +} + // NewAdamClient creates a new AdamClient. // The caller is responsible for providing a CA certificate and key used // to sign all Adam server certificates. @@ -1684,6 +1695,220 @@ func (ac *AdamClient) SubscribeToDeviceMetrics(devUUID uuid.UUID, return unsubscribe, nil } +// IterateDeviceFlowLogs retrieves flow log messages (FlowMessage) published +// by the specified device and passes matching messages to iterator. Flow +// messages are stored and served per-device, not per-app (each one carries +// a Scope identifying which app/VIF it belongs to), so callers that only +// care about one application filter by msg.GetScope().GetUuid() in match (or +// inside iterator) themselves. +// +// It first performs a one-shot GET request to fetch all currently available +// messages. If follow is true, it then subscribes to the streaming endpoint +// and continues delivering new messages until ctx is canceled. +// +// If match is non-nil, only messages for which match(msg) returns true are +// iterated. If match is nil, all messages are iterated. +func (ac *AdamClient) IterateDeviceFlowLogs(ctx context.Context, devUUID uuid.UUID, + match func(msg *eveflowlog.FlowMessage) bool, iterator FlowMsgIterator, + follow bool) error { + if err := ac.checkAdamRunning(); err != nil { + return err + } + + ac.mutex.Lock() + _, known := ac.knownDevices[devUUID] + ac.mutex.Unlock() + if !known { + return fmt.Errorf("unknown device UUID %q", devUUID) + } + + // -------- Initial GET -------- + + url := ac.adminURL("device/" + devUUID.String() + "/flowlogs") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to create GET %s request: %w", url, err) + } + + resp, err := ac.httpClient().Do(req) + if err != nil { + return fmt.Errorf("GET %s failed: %w", url, err) + } + defer resp.Body.Close() + + // No flow logs recorded yet for this device. + if resp.StatusCode != http.StatusNotFound { + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status from GET %s: %d", url, resp.StatusCode) + } + + dec := json.NewDecoder(resp.Body) + for { + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("failed to decode flow message JSON: %w", err) + } + + msg := &eveflowlog.FlowMessage{} + if err := protojson.Unmarshal(raw, msg); err != nil { + return fmt.Errorf("failed to proto-unmarshal flow message: %w", err) + } + if match == nil || match(msg) { + stop, iterErr := iterator.Iterate(msg) + if iterErr != nil { + return fmt.Errorf("failed to iterate flow message: %w", iterErr) + } + if stop { + return nil + } + } + } + } + + // -------- Follow mode -------- + + if !follow { + return nil + } + + flowMsgCh := make(chan *eveflowlog.FlowMessage, 100) + unsubscribe, err := ac.SubscribeToDeviceFlowLogs(devUUID, match, flowMsgCh) + if err != nil { + return err + } + defer unsubscribe() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case msg := <-flowMsgCh: + stop, iterErr := iterator.Iterate(msg) + if iterErr != nil { + return fmt.Errorf("failed to iterate flow message: %w", iterErr) + } + if stop { + return nil + } + } + } +} + +// SubscribeToDeviceFlowLogs subscribes to flow log messages (FlowMessage) +// emitted by the specified device and delivers matching messages to +// channel. Flow messages are stored and served per-device, not per-app; see +// IterateDeviceFlowLogs. +// +// If match is non-nil, only messages for which match(msg) returns true are +// forwarded. If match is nil, all messages are delivered. +// +// The streaming connection is opened synchronously: by the time this method +// returns, Adam has accepted the request and any subsequent flow messages +// for the device will be delivered. On transient failures after the initial +// connection, a background goroutine reconnects with a fixed retry delay. +// +// The returned unsubscribe function stops the background stream and waits +// for it to exit. It is safe to call multiple times. The channel is closed +// when the subscription ends. +func (ac *AdamClient) SubscribeToDeviceFlowLogs(devUUID uuid.UUID, + match func(msg *eveflowlog.FlowMessage) bool, + channel chan<- *eveflowlog.FlowMessage) (unsubscribe func(), err error) { + const retryDelay = 3 * time.Second + + if err = ac.checkAdamRunning(); err != nil { + return nil, err + } + + ac.mutex.Lock() + _, known := ac.knownDevices[devUUID] + ac.mutex.Unlock() + if !known { + return nil, fmt.Errorf("unknown device UUID %q", devUUID) + } + + streamCtx, cancel := context.WithCancel(context.Background()) + url := ac.adminURL("device/" + devUUID.String() + "/flowlogs") + + resp, err := ac.openStream(streamCtx, url) + if err != nil { + cancel() + return nil, err + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + defer close(channel) + + current := resp + for { + if current == nil { + select { + case <-time.After(retryDelay): + case <-streamCtx.Done(): + return + } + r, err := ac.openStream(streamCtx, url) + if err != nil { + if streamCtx.Err() != nil { + return + } + ac.log.Errorf("failed to reopen flow log stream: %v", err) + continue + } + current = r + } + + func() { + defer current.Body.Close() + dec := json.NewDecoder(current.Body) + for { + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + if streamCtx.Err() != nil { + return + } + if errors.Is(err, io.EOF) { + ac.log.Warn("flow log stream closed by server") + return + } + ac.log.Errorf("failed to decode streamed flow message: %v", err) + return + } + msg := &eveflowlog.FlowMessage{} + if err := protojson.Unmarshal(raw, msg); err != nil { + ac.log.Errorf( + "failed to proto-unmarshal streamed flow message: %v", err) + continue + } + if match != nil && !match(msg) { + continue + } + select { + case channel <- msg: + case <-streamCtx.Done(): + return + } + } + }() + current = nil + } + }() + + var once sync.Once + unsubscribe = func() { + once.Do(func() { + cancel() + wg.Wait() + }) + } + return unsubscribe, nil +} + // findDeviceUUID searches Adam for a device with certificates/serial matching // the given callback and returns its UUID if found. func (ac *AdamClient) findDeviceUUID(ctx context.Context, httpClient *http.Client, diff --git a/evetest/devconfig.go b/evetest/devconfig.go index 5a2a752c43d..6335147cdac 100644 --- a/evetest/devconfig.go +++ b/evetest/devconfig.go @@ -871,17 +871,27 @@ func (config SwitchNetworkInstanceConfig) toProto(th *TestHarness, // ApplicationInstanceConfig wraps configuration for a single application deployed on EVE. type ApplicationInstanceConfig struct { - DisplayName string - Activate bool - ProfileList []string - Image ApplicationImageStorage - VirtualizationMode eveconfig.VmMode - CPUs uint - MemoryBytes uint64 - DiskBytes uint64 - EnableVNC bool - VNCDisplay uint - VNCPassword string + DisplayName string + Activate bool + ProfileList []string + Image ApplicationImageStorage + VirtualizationMode eveconfig.VmMode + CPUs uint + MemoryBytes uint64 + DiskBytes uint64 + EnableVNC bool + VNCDisplay uint + VNCPassword string + // RemoteConsole gates VNC access under the Kubevirt hypervisor: unlike + // KVM (where EnableVNC/VNCDisplay/VNCPassword directly configure a raw + // QEMU VNC socket reachable on the device's uplink IP), Kubevirt exposes + // VNC via zedkube's virtctl-based proxy, which only binds the VNC port + // on 127.0.0.1 and is gated by this field, not by EnableVNC. VNCDisplay + // still selects the port (5900+VNCDisplay); VNCPassword is not enforced + // -- see pkg/pillar/docs/vnc-workflows.md. zedkube only starts/stops the + // proxy on a *change* of this field, and only one remote-console session + // is allowed on the device at a time. + RemoteConsole bool DisableLogs bool UserData string NetworkAdapters []AppNetworkAdapter @@ -912,6 +922,7 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, Fixedresources: vmConfig, Activate: config.Activate, ProfileList: config.ProfileList, + RemoteConsole: config.RemoteConsole, } if volumeUUID != NilUUID { appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, diff --git a/evetest/edgecluster.go b/evetest/edgecluster.go index 3ebb30da72c..10582d9709c 100644 --- a/evetest/edgecluster.go +++ b/evetest/edgecluster.go @@ -11,7 +11,6 @@ import ( "time" eveinfo "github.com/lf-edge/eve-api/go/info" - "github.com/lf-edge/eve/pkg/pillar/utils/generics" uuid "github.com/satori/go.uuid" ) @@ -86,8 +85,16 @@ func (ec *EdgeCluster) WaitUntilNodesAreReady(timeout time.Duration) { closeOnce := sync.Once{} RunParallel(len(ec.devices), func(i int) { dev := ec.devices[i] + // Subscribe before taking the initial snapshot, so a transition + // landing between the two calls can never be missed. updates, stop := dev.WatchClusterInfo() defer stop() + if info := dev.GetClusterInfo(); info != nil && allNodesReady(info, expectedNodes) { + ec.th.log.Infof("All cluster nodes already ready per device %q", + dev.devName) + closeOnce.Do(func() { close(doneCh) }) + return + } var tickerCh <-chan time.Time if i == 0 { ticker := time.NewTicker(1 * time.Minute) @@ -121,20 +128,36 @@ func (ec *EdgeCluster) WaitUntilNodesAreReady(timeout time.Duration) { }) } -// allNodesReady returns true if the cluster info reports every node in -// expectedNodes as Ready. +// allNodesReady returns true if the cluster storage is healthy and every +// node in expectedNodes is reported as Ready. func allNodesReady(info *eveinfo.ZInfoKubeCluster, expectedNodes []string) bool { + for _, name := range expectedNodes { + if !clusterNodeReady(info, name) { + return false + } + } + return true +} + +// clusterNodeReady returns true if the given cluster info reports the named +// node as Ready and the cluster's storage as healthy. +func clusterNodeReady(info *eveinfo.ZInfoKubeCluster, nodeName string) bool { const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY - var readyNodes []string + if info.GetStorage().GetHealth() != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { + return false + } for _, node := range info.GetNodes() { + if node.GetName() != nodeName { + continue + } for _, cond := range node.GetConditions() { - if cond.GetType() == nodeReadyCond && cond.GetSet() { - readyNodes = append(readyNodes, node.GetName()) - break + if cond.GetType() == nodeReadyCond { + return cond.GetSet() } } + return false } - return generics.EqualSets(readyNodes, expectedNodes) + return false } // FindDeviceHostingApp finds the cluster device that hosts the given application. diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 45ac71dffb7..7dcecef897c 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -350,15 +350,85 @@ func (d *EdgeDevice) GetDeviceIPAddress(netAdapterLogicalLabel string) []net.IP return ips } +// GetArch returns the CPU architecture of the device ("amd64" or "arm64"), +// as determined during Setup (see TestHarness.selectArch) -- not merely the +// preferred architecture requested via EVETEST_PREFERRED_ARCH, which can +// differ from the device's actual one on a broker that does not support it +// (selectArch falls back to whatever the broker does support). +func (d *EdgeDevice) GetArch() string { + d.th.devicesM.Lock() + devState, found := d.th.devices[d.devName] + d.th.devicesM.Unlock() + if !found { + d.th.t.Fatalf("Unknown device %q", d.devName) + } + switch devState.imageRef.Arch { + case api.ArchType_ARCH_AMD64: + return "amd64" + case api.ArchType_ARCH_ARM64: + return "arm64" + default: + d.th.t.Fatalf("Device %q has unknown architecture: %v", + d.devName, devState.imageRef.Arch) + return "" + } +} + +// BaseOSDatastoreType selects how EdgeDevice.UpgradeEVE delivers the target +// EVE rootfs to the device. +type BaseOSDatastoreType int + +const ( + // BaseOSDatastoreHTTP has evetest extract the raw rootfs image from the + // locally-pulled EVE docker image and serve it over evetest's own + // embedded HTTP image server. This is the traditional/default path and + // works even when the target image is not (or cannot be) hosted on a + // registry reachable from the device. + BaseOSDatastoreHTTP BaseOSDatastoreType = iota + // BaseOSDatastoreOCI has EVE pull the target rootfs directly from the + // same OCI registry the target image was tagged in (e.g. Docker Hub), + // using a container-registry Datastore. evetest still pulls the image + // locally first to determine the target short version, but skips + // extracting the raw rootfs and serving it over HTTP. + BaseOSDatastoreOCI +) + +func (t BaseOSDatastoreType) String() string { + switch t { + case BaseOSDatastoreOCI: + return "oci" + case BaseOSDatastoreHTTP: + fallthrough + default: + return "http" + } +} + +// FromString parses a datastore type name string and sets the +// BaseOSDatastoreType value. +func (t *BaseOSDatastoreType) FromString(s string) error { + switch strings.ToLower(s) { + case "", "http": + *t = BaseOSDatastoreHTTP + case "oci": + *t = BaseOSDatastoreOCI + default: + return fmt.Errorf("invalid BaseOSDatastoreType: %q", s) + } + return nil +} + // UpgradeEVE upgrades the EVE OS to the specified version and optionally // waits until the upgrade completes or reverts. +// datastoreType selects how the target rootfs is delivered to the device -- +// see BaseOSDatastoreType. // When expectRevert is true, the upgrade is expected to fail and EVE to revert // to the previous version -- the function then waits for the target version to // show a FAILED status instead of waiting for it to become active. // A reverted upgrade causes two reboots (one to try the new version, one to // revert), so the expected reboot count is incremented accordingly. func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hypervisor, - waitUntilUpgraded bool, expectRevert bool) { + datastoreType BaseOSDatastoreType, waitUntilUpgraded bool, expectRevert bool) { // Read current device arch (set during Setup). d.th.devicesM.Lock() @@ -407,43 +477,61 @@ func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hyp shortVersion := strings.TrimSpace(versionOut) d.th.log.Debugf("Target EVE short version is %q", shortVersion) - // Extract rootfs (cache by short version to avoid re-extraction on reuse). - rootfsFilename := "rootfs-" + shortVersion + ".img" - rootfsPath := filepath.Join(d.th.imgServerDir, rootfsFilename) - if _, statErr := os.Stat(rootfsPath); os.IsNotExist(statErr) { - d.th.log.Infof("Extracting EVE rootfs from %s", imageName) - _, err = utils.RunDockerCommand(ctx, logger, imageName, - "-f raw rootfs", - map[string]string{"/out": d.th.imgServerDir}, - platform) - if err != nil { - d.th.t.Fatalf("Failed to extract EVE rootfs from %s: %v", - imageName, err) + // Build upgrade device config from a clone of the current config. + config := d.GetConfig() + switch datastoreType { + case BaseOSDatastoreOCI: + // Let EVE pull the rootfs directly from the same OCI registry the + // target image was tagged in -- no local extraction or HTTP hosting + // needed. imageName is ":" (see utils.EVEDockerImageName). + idx := strings.LastIndex(imageName, ":") + if idx < 0 { + d.th.t.Fatalf("UpgradeEVE: could not split image reference %q into repo:tag", + imageName) + } + repo, tag := imageName[:idx], imageName[idx+1:] + d.th.log.Infof("Configuring EVE to pull rootfs %s directly (OCI datastore)", imageName) + config.SetBaseOS(DockerContainer{ + ImageName: repo, + Tag: tag, + }, shortVersion) + default: + // Extract rootfs (cache by short version to avoid re-extraction on reuse). + rootfsFilename := "rootfs-" + shortVersion + ".img" + rootfsPath := filepath.Join(d.th.imgServerDir, rootfsFilename) + if _, statErr := os.Stat(rootfsPath); os.IsNotExist(statErr) { + d.th.log.Infof("Extracting EVE rootfs from %s", imageName) + _, err = utils.RunDockerCommand(ctx, logger, imageName, + "-f raw rootfs", + map[string]string{"/out": d.th.imgServerDir}, + platform) + if err != nil { + d.th.t.Fatalf("Failed to extract EVE rootfs from %s: %v", + imageName, err) + } + defaultOut := filepath.Join(d.th.imgServerDir, "rootfs.img") + if renErr := os.Rename(defaultOut, rootfsPath); renErr != nil { + d.th.t.Fatalf("Failed to rename EVE rootfs: %v", renErr) + } + } else { + d.th.log.Infof("Reusing cached rootfs %s", rootfsFilename) } - defaultOut := filepath.Join(d.th.imgServerDir, "rootfs.img") - if renErr := os.Rename(defaultOut, rootfsPath); renErr != nil { - d.th.t.Fatalf("Failed to rename EVE rootfs: %v", renErr) + + sha256hex, fileSize, err := utils.FileHashAndSize(rootfsPath) + if err != nil { + d.th.t.Fatalf("UpgradeEVE: failed to hash rootfs %s: %v", rootfsPath, err) } - } else { - d.th.log.Infof("Reusing cached rootfs %s", rootfsFilename) - } - sha256hex, fileSize, err := utils.FileHashAndSize(rootfsPath) - if err != nil { - d.th.t.Fatalf("UpgradeEVE: failed to hash rootfs %s: %v", rootfsPath, err) + config.SetBaseOS(HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageSHA256: sha256hex, + MaxDownloadBytes: uint64(fileSize), + ImageRelativePath: rootfsFilename, + ServerAddress: GetImageServerIPv4().String(), + ServerPort: GetImageServerPort(), + }, shortVersion) } - // Build upgrade device config from a clone of the current config. - config := d.GetConfig() - config.SetBaseOS(HTTPStorage{ - ImageFormat: eveconfig.Format_RAW, - ImageSHA256: sha256hex, - MaxDownloadBytes: uint64(fileSize), - ImageRelativePath: rootfsFilename, - ServerAddress: GetImageServerIPv4().String(), - ServerPort: GetImageServerPort(), - }, shortVersion) - d.th.log.Infof("Applying EVE upgrade config (target=%s)", shortVersion) // A successful upgrade reboots once; a reverted upgrade reboots twice // (once to try the new version, once to revert to the previous one). @@ -626,6 +714,51 @@ func (d *EdgeDevice) HardReboot(waitUntilRebooted bool) { }) } +// PowerOff hard-powers off the device through the broker (bypassing any +// graceful ACPI shutdown). The broker RPC blocks until the provider confirms +// the VM is stopped, so no separate wait parameter is needed. +func (d *EdgeDevice) PowerOff() { + d.th.collectCoverageFromDevice(d.th.ctx, d.devName) + devCtrlReq := &api.DeviceControlRequest{ + ClientId: d.th.brokerClientID, + DeviceName: d.devName, + } + ctx, cancel := context.WithTimeout(d.th.ctx, brokerPowerOffEVEDeviceTimeout) + defer cancel() + _, err := d.th.brokerClient.PowerOffDevice(ctx, devCtrlReq) + if err != nil { + d.th.t.Fatalf("PowerOff: broker failed to power off device %q: %v", + d.devName, err) + } +} + +// PowerOn powers the device back on through the broker and optionally waits +// until it boots and reports back to the controller. +// +// TODO: following a true hard power-off, nodeagent does not reliably +// republish an updated ZInfoDevice.LastRebootTime (the reboot-reason +// detection that rebootAndWait's wait relies on appears to assume a +// cooperative in-place OS reboot, not an external power-cycle). +// Until that's root-caused on the EVE side, pass waitUntilOnline=false here +// and confirm recovery some other way (e.g. via WaitUntilAppIsRunning +// on the relevant apps). +func (d *EdgeDevice) PowerOn(waitUntilOnline bool) { + d.th.incExpectedRebootCount(d.devName) + d.rebootAndWait(waitUntilOnline, func() { + devCtrlReq := &api.DeviceControlRequest{ + ClientId: d.th.brokerClientID, + DeviceName: d.devName, + } + ctx, cancel := context.WithTimeout(d.th.ctx, brokerPowerOnEVEDeviceTimeout) + defer cancel() + _, err := d.th.brokerClient.PowerOnDevice(ctx, devCtrlReq) + if err != nil { + d.th.t.Fatalf("PowerOn: broker failed to power on device %q: %v", + d.devName, err) + } + }) +} + // rebootAndWait executes triggerFn to initiate a device reboot and, if // wait is true, blocks until the device confirms the reboot by reporting // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called. @@ -748,67 +881,202 @@ func (d *EdgeDevice) GetAppLogs(appUUID uuid.UUID, match LogMsgMatch) []LogMsg { // GetAppFlowLogs returns flow records for the specified application // matching the provided criteria. func (d *EdgeDevice) GetAppFlowLogs( - appUUID uuid.UUID, match FlowLogMatch) []eveflowlog.FlowRecord { - // TODO: implement AdamClient.IterateAppFlowLogs first - d.th.t.Fatalf("GetAppFlowLogs is not implemented") - return nil + appUUID uuid.UUID, match FlowLogMatch) []*eveflowlog.FlowRecord { + devUUID := d.getDevUUID() + scopeMatch := flowScopeMatcher(appUUID, match.VirtualNetAdapter, match.NetworkInstance) + + var records []*eveflowlog.FlowRecord + ctx, cancel := context.WithTimeout(d.th.ctx, gatherLogsTimeout) + err := d.th.adamClient.IterateDeviceFlowLogs(ctx, devUUID, scopeMatch, + flowMsgIterFn(func(msg *eveflowlog.FlowMessage) (bool, error) { + for _, rec := range msg.GetFlows() { + if flowRecordMatches(rec, match) { + records = append(records, rec) + } + } + return false, nil + }), false) + cancel() + if err != nil { + d.th.t.Fatalf("Failed to retrieve app flow logs for device %q app %q: %v", + d.devName, appUUID, err) + } + return records +} + +// flowScopeMatcher returns a match function for IterateDeviceFlowLogs / +// SubscribeToDeviceFlowLogs that selects FlowMessages belonging to the given +// application and, if non-empty/non-zero, the given VIF logical label and +// network instance. +func flowScopeMatcher(appUUID uuid.UUID, virtualNetAdapter string, + networkInstance uuid.UUID) func(*eveflowlog.FlowMessage) bool { + appUUIDStr := appUUID.String() + niUUIDStr := networkInstance.String() + return func(msg *eveflowlog.FlowMessage) bool { + scope := msg.GetScope() + if scope.GetUuid() != appUUIDStr { + return false + } + if virtualNetAdapter != "" && scope.GetIntf() != virtualNetAdapter { + return false + } + if networkInstance != NilUUID && scope.GetNetInstUUID() != niUUIDStr { + return false + } + return true + } +} + +// flowRecordMatches reports whether rec satisfies match's flow-record-level +// criteria. Scope-level criteria (VirtualNetAdapter, NetworkInstance) are +// checked by the caller against the enclosing FlowMessage's Scope instead, +// since FlowRecord itself carries no scope information. +func flowRecordMatches(rec *eveflowlog.FlowRecord, match FlowLogMatch) bool { + if rec.GetInbound() != match.Inbound { + return false + } + if match.Flow != nil { + flow := rec.GetFlow() + if match.Flow.GetSrc() != "" && flow.GetSrc() != match.Flow.GetSrc() { + return false + } + if match.Flow.GetSrcPort() != 0 && flow.GetSrcPort() != match.Flow.GetSrcPort() { + return false + } + if match.Flow.GetDest() != "" && flow.GetDest() != match.Flow.GetDest() { + return false + } + if match.Flow.GetDestPort() != 0 && flow.GetDestPort() != match.Flow.GetDestPort() { + return false + } + if match.Flow.GetProtocol() != 0 && flow.GetProtocol() != match.Flow.GetProtocol() { + return false + } + } + ts := rec.GetStartTime().AsTime() + if !match.NotBefore.IsZero() && ts.Before(match.NotBefore) { + return false + } + if !match.NotAfter.IsZero() && ts.After(match.NotAfter) { + return false + } + return true } // GetAppDNSLogs returns DNS request logs for the specified application // matching the provided criteria. func (d *EdgeDevice) GetAppDNSLogs( - appUUID uuid.UUID, match DNSLogMatch) []eveflowlog.DnsRequest { - // TODO: implement AdamClient.IterateAppFlowLogs first - d.th.t.Fatalf("GetAppDNSLogs is not implemented") - return nil + appUUID uuid.UUID, match DNSLogMatch) []*eveflowlog.DnsRequest { + devUUID := d.getDevUUID() + scopeMatch := flowScopeMatcher(appUUID, match.VirtualNetAdapter, match.NetworkInstance) + + var records []*eveflowlog.DnsRequest + ctx, cancel := context.WithTimeout(d.th.ctx, gatherLogsTimeout) + err := d.th.adamClient.IterateDeviceFlowLogs(ctx, devUUID, scopeMatch, + flowMsgIterFn(func(msg *eveflowlog.FlowMessage) (bool, error) { + for _, req := range msg.GetDnsReqs() { + ts := req.GetRequestTime().AsTime() + if !match.NotBefore.IsZero() && ts.Before(match.NotBefore) { + continue + } + if !match.NotAfter.IsZero() && ts.After(match.NotAfter) { + continue + } + records = append(records, req) + } + return false, nil + }), false) + cancel() + if err != nil { + d.th.t.Fatalf("Failed to retrieve app DNS logs for device %q app %q: %v", + d.devName, appUUID, err) + } + return records } // waitUntilAppState waits until the app reaches one of targetStates, // logging every state transition along the way. +// +// Safe to call even if the app already has matching state history from an +// earlier phase (e.g. it was already RUNNING before): it subscribes to live +// updates before checking the current snapshot, so a stale historical +// record can never be mistaken for a new transition. +// // ctx controls the deadline; callers must derive it from d.th.ctx. // Calls t.Fatalf on timeout or error. func (d *EdgeDevice) waitUntilAppState( ctx context.Context, appUUID uuid.UUID, targetStates ...eveinfo.ZSwState) { devUUID := d.getDevUUID() appUUIDStr := appUUID.String() - - var lastState = eveinfo.ZSwState_INVALID + filter := func(msg *eveinfo.ZInfoMsg) bool { + if msg.GetZtype() != eveinfo.ZInfoTypes_ZiApp { + return false + } + ainfo := msg.GetAinfo() + return ainfo != nil && ainfo.GetAppID() == appUUIDStr + } d.th.log.Infof("Waiting for app %q on device %q to reach state(s) %v", appUUID, d.devName, targetStates) - err := d.th.adamClient.IterateDeviceInfoMsgs(ctx, devUUID, - func(msg *eveinfo.ZInfoMsg) bool { - if msg.GetZtype() != eveinfo.ZInfoTypes_ZiApp { - return false - } - ainfo := msg.GetAinfo() - return ainfo != nil && ainfo.GetAppID() == appUUIDStr - }, - infoMsgIterFn(func(msg *eveinfo.ZInfoMsg) (bool, error) { - ainfo := msg.GetAinfo() - state := ainfo.GetState() - if state != lastState { - lastState = state - d.th.log.Infof("App %q (%s) on device %q state changed to %s", - appUUID, ainfo.GetAppName(), d.devName, state) - } - if generics.ContainsItem(targetStates, state) { - return true, nil - } - return false, nil - }), - true, - ) + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + ch := make(chan *eveinfo.ZInfoMsg, watchChannelBufSize) + unsub, err := d.th.adamClient.SubscribeToDeviceInfoMsgs(devUUID, filter, ch) if err != nil { - d.th.t.Fatalf("Waiting for app %q on device %q to reach state(s) %v: %v", - appUUID, d.devName, targetStates, err) + d.th.t.Fatalf("Failed to subscribe to info messages for app %q on device %q: %v", + appUUID, d.devName, err) + } + defer unsub() + + var lastState = eveinfo.ZSwState_INVALID + logIfChanged := func(ainfo *eveinfo.ZInfoApp) { + state := ainfo.GetState() + if state != lastState { + lastState = state + d.th.log.Infof("App %q (%s) on device %q state changed to %s", + appUUID, ainfo.GetAppName(), d.devName, state) + } + } + + // Snapshot: the app may already be in one of the target states right now. + // Logged as the current state, not a "change", since nothing just + // happened -- this is a fact we already knew, not a new event. + if info := d.GetAppInfo(appUUID); info != nil { + lastState = info.GetState() + d.th.log.Infof("App %q (%s) on device %q currently in state %s", + appUUID, info.GetAppName(), d.devName, lastState) + if generics.ContainsItem(targetStates, lastState) { + return + } + } + + for { + select { + case <-ctx.Done(): + d.th.t.Fatalf("Waiting for app %q on device %q to reach state(s) %v: %v", + appUUID, d.devName, targetStates, ctx.Err()) + case msg, ok := <-ch: + if !ok { + d.th.t.Fatalf("Info subscription closed while waiting for app %q "+ + "on device %q to reach state(s) %v", appUUID, d.devName, targetStates) + } + logIfChanged(msg.GetAinfo()) + if generics.ContainsItem(targetStates, lastState) { + return + } + } } } // WaitUntilAppIsRunning waits until the specified application reaches // the running state or fails. // +// Safe to call even if the app already has RUNNING history from an earlier +// phase: it subscribes to live updates before checking the current +// snapshot, so a stale historical record can never be mistaken for a new +// transition. +// // timeoutExcludingDownload is the maximum time to wait excluding any // period spent actively downloading (i.e. in DOWNLOAD_STARTED state with // advancing progress). If a download stalls for downloadStalledTimeout the @@ -833,28 +1101,12 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( lastAppErrs string // concatenated error descriptions for change detection // Keyed by volume UUID; accumulates the latest ZInfoVolume for each volume. volumes = make(map[string]*eveinfo.ZInfoVolume) + // timer is nil during the initial snapshot replay (see below), so + // iterCb's Reset calls are guarded and become no-ops until the live + // phase arms it. + timer *time.Timer ) - // ctx is canceled either by the timer below (timeout) or by d.th.ctx (test end). - ctx, cancel := context.WithCancel(d.th.ctx) - defer cancel() - - // The timer drives timeouts when no info messages arrive: - // - non-download phase: fires after the remaining non-download budget - // - download phase: fires after downloadStalledTimeout with no progress - // iterCb resets it on each relevant transition or progress update. - timer := time.NewTimer(timeoutExcludingDownload) - defer timer.Stop() - - // Cancel the context when the timer fires so IterateDeviceInfoMsgs unblocks. - go func() { - select { - case <-timer.C: - cancel() - case <-ctx.Done(): - } - }() - // Accept ZiApp messages for this app and all ZiVolume messages. // Volume messages are further filtered in the iterator once the app's // VolumeRefs are known. @@ -869,7 +1121,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( return false } - iterCb := func(msg *eveinfo.ZInfoMsg) (bool, error) { + // iterCb processes a single info message, updating all the tracking state + // above. live distinguishes the two contexts it's called from: false + // during the initial snapshot replay (state is still established, but + // nothing is logged -- these are historical facts, not new events, and + // logging each one would misleadingly read as if they just happened); + // true during the live phase (normal logging of genuinely new events). + iterCb := func(msg *eveinfo.ZInfoMsg, live bool) (bool, error) { // Handle volume updates: store the latest state for each volume // and re-evaluate download progress if the app is currently downloading. if msg.GetZtype() == eveinfo.ZInfoTypes_ZiVolume { @@ -884,9 +1142,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( pct := appDownloadProgress(volumeRefs, volumes) if pct != lastDownloadPct { lastDownloadPct = pct - timer.Reset(downloadStalledTimeout) - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, lastState, pct) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, lastState, pct) + } } } return false, nil @@ -904,20 +1166,26 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( nowInDownload := state == eveinfo.ZSwState_DOWNLOAD_STARTED if inDownload && !nowInDownload { // Leaving download: resume non-download clock and set timer to - // the remaining non-download budget. + // the remaining non-download budget. Only meaningful once the + // live timer is armed -- during the initial snapshot replay, + // elapsed wall-clock time is negligible and this check is skipped. nonDownloadStart = time.Now() - remaining := timeoutExcludingDownload - nonDownloadElapsed - if remaining <= 0 { - return true, fmt.Errorf( - "timed out after %s (excluding download) waiting for app %q (%s) "+ - "on device %q to reach RUNNING state (last state: %s)", - timeoutExcludingDownload, appUUID, appName, d.devName, state) + if timer != nil { + remaining := timeoutExcludingDownload - nonDownloadElapsed + if remaining <= 0 { + return true, fmt.Errorf( + "timed out after %s (excluding download) waiting for app %q (%s) "+ + "on device %q to reach RUNNING state (last state: %s)", + timeoutExcludingDownload, appUUID, appName, d.devName, state) + } + timer.Reset(remaining) } - timer.Reset(remaining) } else if !inDownload && nowInDownload { // Entering download: freeze non-download clock and arm stall timer. nonDownloadElapsed += time.Since(nonDownloadStart) - timer.Reset(downloadStalledTimeout) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } } inDownload = nowInDownload @@ -927,9 +1195,11 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( if state == eveinfo.ZSwState_DOWNLOAD_STARTED { pct := appDownloadProgress(volumeRefs, volumes) lastDownloadPct = pct - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, state, pct) - } else { + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, state, pct) + } + } else if live { d.th.log.Infof("App %q (%s) on device %q state changed to %s", appUUID, appName, d.devName, state) } @@ -937,9 +1207,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( pct := appDownloadProgress(volumeRefs, volumes) if pct != lastDownloadPct { lastDownloadPct = pct - timer.Reset(downloadStalledTimeout) - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, state, pct) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, state, pct) + } } } @@ -953,12 +1227,14 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( currentAppErrs := strings.Join(errDescs, "; ") if currentAppErrs != lastAppErrs { lastAppErrs = currentAppErrs - if currentAppErrs != "" { - d.th.log.Warnf("App %q (%s) on device %q errors: %s", - appUUID, appName, d.devName, currentAppErrs) - } else { - d.th.log.Infof("App %q (%s) on device %q errors cleared", - appUUID, appName, d.devName) + if live { + if currentAppErrs != "" { + d.th.log.Warnf("App %q (%s) on device %q errors: %s", + appUUID, appName, d.devName, currentAppErrs) + } else { + d.th.log.Infof("App %q (%s) on device %q errors cleared", + appUUID, appName, d.devName) + } } } @@ -976,41 +1252,107 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( // Success. if state == eveinfo.ZSwState_RUNNING { - d.th.log.Infof("App %q (%s) on device %q is RUNNING", - appUUID, appName, d.devName) + if live { + d.th.log.Infof("App %q (%s) on device %q is RUNNING", + appUUID, appName, d.devName) + } return true, nil } return false, nil } - err := d.th.adamClient.IterateDeviceInfoMsgs(ctx, devUUID, filter, - infoMsgIterFn(iterCb), true) - + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + ch := make(chan *eveinfo.ZInfoMsg, watchChannelBufSize) + unsub, err := d.th.adamClient.SubscribeToDeviceInfoMsgs(devUUID, filter, ch) if err != nil { - // If the test framework context was canceled, propagate the error. - if d.th.ctx.Err() != nil { - d.th.t.Fatalf("%v", err) + d.th.t.Fatalf("Failed to subscribe to info messages for app %q on device %q: %v", + appUUID, d.devName, err) + } + defer unsub() + + // Snapshot: replay every already-known message (app and volume) through + // iterCb with no timer armed (its Reset calls are then no-ops), keeping + // only the LAST app-state evaluation -- i.e. the app's current state, + // not the first historical occurrence of any target condition. + var ( + snapshotDone bool + snapshotErr error + ) + d.iterateInfoMsgs(devUUID, filter, func(msg *eveinfo.ZInfoMsg) { + done, cbErr := iterCb(msg, false) + if msg.GetZtype() == eveinfo.ZInfoTypes_ZiApp { + snapshotDone, snapshotErr = done, cbErr + } + }) + if snapshotDone { + if snapshotErr != nil { + d.th.t.Fatalf("%v", snapshotErr) } + d.th.log.Infof("App %q (%s) on device %q is already RUNNING", + appUUID, appName, d.devName) + return + } + d.th.log.Infof("App %q (%s) on device %q currently in state %s; "+ + "waiting for it to reach RUNNING", appUUID, appName, d.devName, lastState) + + // Live phase: arm the real timer -- matching the app's current phase, + // established during the snapshot above -- and wait for a genuinely + // new transition. + nonDownloadStart = time.Now() + initialTimeout := timeoutExcludingDownload + if inDownload { + initialTimeout = downloadStalledTimeout + } + timer = time.NewTimer(initialTimeout) + defer timer.Stop() - // If our context was not canceled, the error came from iterCb - // (e.g. ZSwState_ERROR or explicit failure). - if ctx.Err() == nil { - d.th.t.Fatalf("%v", err) + // ctx is canceled either by the timer above (timeout) or by d.th.ctx (test end). + ctx, cancel := context.WithCancel(d.th.ctx) + defer cancel() + go func() { + select { + case <-timer.C: + cancel() + case <-ctx.Done(): } + }() + + for { + select { + case <-ctx.Done(): + // If the test framework context was canceled, propagate that. + if d.th.ctx.Err() != nil { + d.th.t.Fatalf("Waiting for app %q (%s) on device %q to reach "+ + "RUNNING state: %v", appUUID, appName, d.devName, d.th.ctx.Err()) + } - // Otherwise our timer fired — determine which timeout occurred. - if inDownload { + // Otherwise our own timer fired -- determine which timeout occurred. + if inDownload { + d.th.t.Fatalf( + "app %q (%s) on device %q download stalled at %d%% for more than %s", + appUUID, appName, d.devName, lastDownloadPct, downloadStalledTimeout) + } + nonDownloadTotal := nonDownloadElapsed + time.Since(nonDownloadStart) d.th.t.Fatalf( - "app %q (%s) on device %q download stalled at %d%% for more than %s", - appUUID, appName, d.devName, lastDownloadPct, downloadStalledTimeout) + "timed out after %s (excluding download) waiting for app %q (%s) "+ + "on device %q to reach RUNNING state (last state: %s)", + nonDownloadTotal, appUUID, appName, d.devName, lastState) + case msg, ok := <-ch: + if !ok { + d.th.t.Fatalf("Info subscription closed while waiting for app %q "+ + "(%s) on device %q to reach RUNNING state", appUUID, appName, d.devName) + } + done, cbErr := iterCb(msg, true) + if !done { + continue + } + if cbErr != nil { + d.th.t.Fatalf("%v", cbErr) + } + return } - - nonDownloadTotal := nonDownloadElapsed + time.Since(nonDownloadStart) - d.th.t.Fatalf( - "timed out after %s (excluding download) waiting for app %q (%s) "+ - "on device %q to reach RUNNING state (last state: %s)", - nonDownloadTotal, appUUID, appName, d.devName, lastState) } } @@ -1063,6 +1405,15 @@ func (d *EdgeDevice) PurgeApplication(appUUID uuid.UUID, waitUntilPurged bool, } else { app.Purge = &eveconfig.InstanceOpsCmd{Counter: purge.GetCounter() + 1} } + for _, volRef := range app.GetVolumeRefList() { + volRef.GenerationCount++ + for _, vol := range config.GetVolumes() { + if vol.GetUuid() == volRef.GetUuid() { + vol.GenerationCount++ + break + } + } + } found = true break } @@ -1080,6 +1431,15 @@ func (d *EdgeDevice) PurgeApplication(appUUID uuid.UUID, waitUntilPurged bool, } } +// DialViaSSH opens a TCP connection to address, tunneled through an SSH +// connection to this device (an SSH direct-tcpip channel), as if address is +// dialed from the device itself. Useful for reaching services that only +// listen on the device's own loopback interface, e.g. the Kubevirt VNC proxy +// gated by ApplicationInstanceConfig.RemoteConsole. +func (d *EdgeDevice) DialViaSSH(network, address string) (net.Conn, error) { + return d.th.dialViaSSH(d.th.ctx, d.devName, network, address) +} + // ActivateApplication activates the specified application instance. func (d *EdgeDevice) ActivateApplication(appUUID uuid.UUID, waitUntilActivated bool, timeout time.Duration) { @@ -1350,8 +1710,13 @@ func getAdaptersByLabel(config *EdgeDeviceConfig, label string) []string { // FileExists checks whether a file exists on the device. func (d *EdgeDevice) FileExists(fileName string) bool { + // "; true" forces the overall exit status to 0 regardless of whether + // the file exists, so a missing file (test -f exits nonzero) can't be + // conflated with a genuine SSH/transport failure -- err is asserted nil + // for the latter, and only the "EXISTS" marker in stdout answers the + // actual question. stdout, _, err := d.RunShellScript( - "test -f "+shellEscape(fileName)+" && echo EXISTS", + "test -f "+shellEscape(fileName)+" && echo EXISTS; true", quickSSHCommandTimeout, 0) if err != nil { d.th.t.Fatalf("FileExists: SSH command failed: %v", err) @@ -2000,6 +2365,47 @@ func (d *EdgeDevice) WatchClusterInfo() ( return ch, d.trackWatcherUnsub(unsub) } +// WaitForClusterNodeIsReady waits until this device's own ZInfoKubeCluster +// report shows this device as a Ready node with healthy cluster storage. +// +// Only the elected leader node publishes cluster info, so this is only +// meaningful when called on a single-node cluster (where the sole device is +// necessarily the leader) or on a device already known to be the leader; for +// a multi-node cluster where the leader isn't known in advance, use +// EdgeCluster.WaitUntilNodesAreReady instead. +func (d *EdgeDevice) WaitForClusterNodeIsReady(timeout time.Duration) { + d.th.log.Infof("Waiting for cluster node %q to become ready...", d.devName) + + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + updates, stop := d.WatchClusterInfo() + defer stop() + + if info := d.GetClusterInfo(); info != nil && clusterNodeReady(info, d.devName) { + d.th.log.Infof("Cluster node %q is already ready", d.devName) + return + } + + ctx, cancel := context.WithTimeout(d.th.ctx, timeout) + defer cancel() + for { + select { + case info, ok := <-updates: + if !ok { + d.th.t.Fatalf("Cluster info subscription closed while waiting "+ + "for node %q to become ready", d.devName) + } + if clusterNodeReady(info, d.devName) { + d.th.log.Infof("Cluster node %q is now ready", d.devName) + return + } + case <-ctx.Done(): + d.th.t.Fatalf("Timed out waiting for cluster node %q to become ready", + d.devName) + } + } +} + // GetClusterUpdateInfo returns the last recorded information regarding the Kubernetes // cluster update, or nil if no such info message has been received yet. func (d *EdgeDevice) GetClusterUpdateInfo() *eveinfo.ZInfoKubeClusterUpdateStatus { @@ -2271,23 +2677,29 @@ func (d *EdgeDevice) WatchClusterMetrics() ( // - key: identifies the specific message within the topic to fetch // - output: pointer to a value of type T to unmarshal the message into // -// Returns an error if the topic or message does not exist, cannot be read, or -// fails to unmarshal into the provided output type. +// Returns false if the topic or message does not exist yet (e.g. before the +// agent has first published it) -- callers that need to wait for it should +// poll on the returned bool instead of treating absence as an error. Calls +// t.Fatalf on any other read or unmarshal failure. func ReadPublication[T any](d *EdgeDevice, fromAgent string, persistent bool, - key string, output *T) { + key string, output *T) bool { fullName := fmt.Sprintf("%T", *new(T)) typeName := fullName[strings.LastIndex(fullName, ".")+1:] var path string if persistent { - path = fmt.Sprintf("/persistent/status/%s/%s/%s.json", fromAgent, typeName, key) + path = fmt.Sprintf("/persist/status/%s/%s/%s.json", fromAgent, typeName, key) } else { path = fmt.Sprintf("/run/%s/%s/%s.json", fromAgent, typeName, key) } + if !d.FileExists(path) { + return false + } data := d.ReadFile(path) if err := json.Unmarshal(data, output); err != nil { d.th.t.Fatalf("ReadPublication: failed to unmarshal %q from device %q: %v", path, d.devName, err) } + return true } // ReadAllPublications retrieves all messages from a pub-sub topic published by @@ -2304,7 +2716,7 @@ func ReadAllPublications[T any](d *EdgeDevice, fromAgent string, persistent bool typeName := fullName[strings.LastIndex(fullName, ".")+1:] var dir string if persistent { - dir = fmt.Sprintf("/persistent/status/%s/%s", fromAgent, typeName) + dir = fmt.Sprintf("/persist/status/%s/%s", fromAgent, typeName) } else { dir = fmt.Sprintf("/run/%s/%s", fromAgent, typeName) } @@ -2387,6 +2799,11 @@ type metricMsgIterFn func(*evemetrics.ZMetricMsg) (bool, error) func (f metricMsgIterFn) Iterate(msg *evemetrics.ZMetricMsg) (bool, error) { return f(msg) } +// flowMsgIterFn adapts a function to the controller.FlowMsgIterator interface. +type flowMsgIterFn func(*eveflowlog.FlowMessage) (bool, error) + +func (f flowMsgIterFn) Iterate(msg *eveflowlog.FlowMessage) (bool, error) { return f(msg) } + // appDownloadProgress returns the average download progress (0–100) across // the app's volumes. For each volume UUID listed in volumeRefs the progress // is taken from the latest ZInfoVolume in volumes: diff --git a/evetest/go.mod b/evetest/go.mod index 5e04a1d3993..0c8c0c18eed 100644 --- a/evetest/go.mod +++ b/evetest/go.mod @@ -5,6 +5,7 @@ go 1.25.2 toolchain go1.25.11 require ( + github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913 github.com/containerd/errdefs v1.0.0 github.com/distribution/reference v0.6.0 github.com/docker/cli v29.2.0+incompatible diff --git a/evetest/go.sum b/evetest/go.sum index 709d60deb05..ae439d5d21e 100644 --- a/evetest/go.sum +++ b/evetest/go.sum @@ -4,6 +4,8 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913 h1:gQl0n269O2lxtDz6QBXtqEjiH2auPaG8GbTu3tByA3w= +github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913/go.mod h1:HfBAAYdSeX18f2nwbuMIcA12RhvgYolx0XDbbhusDXY= github.com/anchore/go-lzo v0.1.0 h1:NgAacnzqPeGH49Ky19QKLBZEuFRqtTG9cdaucc3Vncs= github.com/anchore/go-lzo v0.1.0/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= @@ -208,6 +210,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/evetest/grpcserver.go b/evetest/grpcserver.go index bfa7ca63bb7..7f88a89dbd3 100644 --- a/evetest/grpcserver.go +++ b/evetest/grpcserver.go @@ -15,6 +15,7 @@ import ( "sync" "time" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evemetrics "github.com/lf-edge/eve-api/go/metrics" "github.com/lf-edge/eve/evetest/constants" @@ -71,6 +72,23 @@ func (w *metricMsgGrpcIterator[T]) Iterate(msg *evemetrics.ZMetricMsg) (bool, er return false, w.stream.Send(resp) } +type flowMsgGrpcIterator[T any] struct { + stream grpc.ServerStreamingServer[T] + // mapper extracts a response from a flow message; returns nil to skip. + mapper func(*eveflowlog.FlowMessage) (*T, error) +} + +func (w *flowMsgGrpcIterator[T]) Iterate(msg *eveflowlog.FlowMessage) (bool, error) { + resp, err := w.mapper(msg) + if err != nil { + return false, err + } + if resp == nil { + return false, nil + } + return false, w.stream.Send(resp) +} + func (th *TestHarness) errIfBrokerNotReady() error { if th.brokerClientID == "" { return errors.New("broker is not yet connected") @@ -396,8 +414,38 @@ func (th *TestHarness) GetAppLogs( // GetAppFlowLogs streams flow logs and DNS request logs for an application. func (th *TestHarness) GetAppFlowLogs( req *api.AppRequest, stream api.Evetest_GetAppFlowLogsServer) error { - // TODO - return errors.New("not implemented") + if err := th.errIfAdamNotReady(); err != nil { + return err + } + devName, devUUID, err := th.resolveEVEDeviceName(req.GetDeviceName()) + if err != nil { + return err + } + if devUUID == uuid.Nil { + return fmt.Errorf("device %q is not onboarded", devName) + } + appUUID, err := th.resolveAppUUID(stream.Context(), devUUID, req.GetAppNameOrUuid()) + if err != nil { + return err + } + appUUIDStr := appUUID.String() + match := func(msg *eveflowlog.FlowMessage) bool { + return msg.GetScope().GetUuid() == appUUIDStr + } + iterator := &flowMsgGrpcIterator[api.AppFlowLogsResponse]{ + stream: stream, + mapper: func(msg *eveflowlog.FlowMessage) (*api.AppFlowLogsResponse, error) { + if len(msg.GetFlows()) == 0 && len(msg.GetDnsReqs()) == 0 { + return nil, nil + } + return &api.AppFlowLogsResponse{ + IpFlows: msg.GetFlows(), + DnsRequests: msg.GetDnsReqs(), + }, nil + }, + } + return th.adamClient.IterateDeviceFlowLogs( + stream.Context(), devUUID, match, iterator, req.GetFollow()) } // GetNIInfo streams info (ZInfoNetworkInstance) about a network instance (NI). diff --git a/evetest/harness.go b/evetest/harness.go index a4fc3c2b0c8..2d78a12f17e 100644 --- a/evetest/harness.go +++ b/evetest/harness.go @@ -70,6 +70,11 @@ const ( // Timeout for powering on an EVE VM (not for waiting for it to boot). brokerPowerOnEVEDeviceTimeout = 20 * time.Second + // Timeout for powering off an EVE VM. The broker RPC blocks until the + // provider confirms the VM is stopped, so this must accommodate a + // graceful ACPI-less hard power-off, not just issuing the request. + brokerPowerOffEVEDeviceTimeout = 20 * time.Second + // Timeout for triggering an EVE VM reboot (not for waiting for it to boot). brokerRebootEVEDeviceTimeout = 20 * time.Second diff --git a/evetest/ssh.go b/evetest/ssh.go index 8f22c7d09aa..6f4d812bfda 100644 --- a/evetest/ssh.go +++ b/evetest/ssh.go @@ -54,6 +54,64 @@ func (th *TestHarness) runScriptOnEVEOverSSH(ctx context.Context, devName string stdout, stderr, stdoutWatchdogTimeout) } +// dialViaSSH opens an SSH connection to the given EVE device and tunnels a +// TCP connection to remoteAddr through it (an SSH direct-tcpip channel), as +// if network is dialed from the device itself. Useful for reaching services +// that only listen on the device's own loopback interface, e.g. the Kubevirt +// VNC proxy (see pkg/pillar/docs/vnc-workflows.md). +// +// The returned net.Conn owns the underlying SSH client: closing it also +// closes the SSH connection. +func (th *TestHarness) dialViaSSH(ctx context.Context, devName string, + network, remoteAddr string) (net.Conn, error) { + + eveIP, err := th.getReachableEVEAddr(ctx, devName, 22, "") + if err != nil { + return nil, err + } + + keyPEM, err := os.ReadFile("/root/.ssh/eve_rsa") + if err != nil { + return nil, fmt.Errorf("failed to read EVE SSH key: %w", err) + } + signer, err := ssh.ParsePrivateKey(keyPEM) + if err != nil { + return nil, fmt.Errorf("failed to parse client key: %w", err) + } + + addr := net.JoinHostPort(eveIP, "22") + sshConfig := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 5 * time.Second, + } + client, err := ssh.Dial("tcp", addr, sshConfig) + if err != nil { + return nil, fmt.Errorf("SSH dial to %s failed: %w", addr, err) + } + + conn, err := client.Dial(network, remoteAddr) + if err != nil { + _ = client.Close() + return nil, fmt.Errorf("SSH tunnel dial to %s failed: %w", remoteAddr, err) + } + return &sshTunnelConn{Conn: conn, client: client}, nil +} + +// sshTunnelConn wraps an SSH-tunneled net.Conn so that closing it also closes +// the underlying SSH client connection that carries it. +type sshTunnelConn struct { + net.Conn + client *ssh.Client +} + +func (c *sshTunnelConn) Close() error { + err := c.Conn.Close() + _ = c.client.Close() + return err +} + // runScriptOverSSH executes a shell script on a remote host over SSH // using the Go crypto/ssh client library. It supports username/password and // client-certificate authentication methods (see AuthMethod). @@ -152,9 +210,15 @@ func (th *TestHarness) scpFromEVE(ctx context.Context, if recursive { scpArgs = append(scpArgs, "-r") } + // The path after the colon is interpreted by a remote shell, so it needs + // its own shell quoting independent of how this argv element is split + // locally (scp itself is invoked directly via exec, with no local shell + // involved) -- otherwise a remote path containing spaces (e.g. a pubsub + // key like "Application Data Store") gets split into multiple arguments + // remotely. scpArgs = append(scpArgs, "-i", "/root/.ssh/eve_rsa", - "root@"+eveIP+":"+remotePath, + "root@"+eveIP+":"+shellEscape(remotePath), localPath, ) cmd := exec.CommandContext(ctx, "scp", scpArgs...) @@ -178,7 +242,7 @@ func (th *TestHarness) scpToEVE(ctx context.Context, scpArgs = append(scpArgs, "-i", "/root/.ssh/eve_rsa", localPath, - "root@"+eveIP+":"+remotePath, + "root@"+eveIP+":"+shellEscape(remotePath), ) cmd := exec.CommandContext(ctx, "scp", scpArgs...) return cmd.Run() diff --git a/evetest/testparam.go b/evetest/testparam.go index 64d666b40d8..09f1a4252c0 100644 --- a/evetest/testparam.go +++ b/evetest/testparam.go @@ -285,19 +285,6 @@ func GetDiskSizeMiBParameterValue() uint32 { return GetTestParameter[uint32](DiskSizeMiBParameterKey) } -// SkipIfHypervisorKubevirt skips the current test if the resolved HYPERVISOR -// parameter is HypervisorKubevirt. Kubevirt is only supported by tests under -// `evetest/tests/cluster`; non-cluster tests should call this helper right -// after defining the HypervisorParameter to ensure they are not accidentally -// exercised on a Kubevirt-flavored EVE build. -func SkipIfHypervisorKubevirt() { - th := getTestHarness() - if GetHypervisorParameterValue() == HypervisorKubevirt { - th.t.Skipf("Kubevirt hypervisor is only supported by cluster tests " + - "(under evetest/tests/cluster); use kvm or xen") - } -} - // FilesystemParameterKey is the key used for the Filesystem parameter. const FilesystemParameterKey = "FILESYSTEM" From 09567f04fce1c8979a0b3c3ba7db982bbe825662 Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Fri, 31 Jul 2026 16:03:20 +0200 Subject: [PATCH 4/4] evetest: rewrite eden test suites (networking, security, apps, lps, upgrade) Port the corresponding eden E2E scenarios to the evetest Go framework across networking (flow logs, DPC fallback/failover, intermittent connectivity, mgmt traffic routed via an app-based NAT gateway, and several smaller fixes), security (AppArmor status, vcomlink TPM-over-vsock), apps (VNC console access, app purge), LPS and upgrade. Adds the TwoMgmtPortsWithPublicNTP and MgmtViaAppTopology network models used by the rewritten tests. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Milan Lenco --- evetest/Dockerfile.evetest | 2 +- evetest/Makefile | 2 +- evetest/README.md | 2 +- evetest/netmodels/multi-eth.go | 221 ++ evetest/tests/apps/purge_test.go | 242 +++ evetest/tests/apps/testsuite_test.go | 41 + evetest/tests/apps/vnc_test.go | 414 ++++ evetest/tests/cluster/cluster_test.go | 40 +- evetest/tests/lps/app_local_info_test.go | 319 +++ evetest/tests/lps/dev_local_info_test.go | 271 +++ evetest/tests/lps/helpers_test.go | 241 +++ evetest/tests/lps/network_test.go | 213 +- evetest/tests/lps/profile_test.go | 277 +++ evetest/tests/lps/radio_silence_test.go | 193 ++ evetest/tests/lps/testsuite_test.go | 35 +- evetest/tests/networking/acls_test.go | 14 +- evetest/tests/networking/bonds_test.go | 36 +- evetest/tests/networking/bootstrap_test.go | 50 +- evetest/tests/networking/dns_test.go | 20 +- evetest/tests/networking/failover_test.go | 693 ++++++- evetest/tests/networking/ipv6_test.go | 22 +- evetest/tests/networking/net_adapter_test.go | 32 +- evetest/tests/networking/netinst_test.go | 1840 ++++++++++++++++- evetest/tests/networking/ntp_test.go | 561 +++-- .../tests/networking/pciback_error_test.go | 111 +- evetest/tests/networking/routing_test.go | 445 +++- evetest/tests/networking/stp_test.go | 8 +- evetest/tests/networking/testsuite_test.go | 84 +- evetest/tests/networking/vlans_test.go | 21 +- evetest/tests/security/apparmor_test.go | 50 + evetest/tests/security/testsuite_test.go | 34 + evetest/tests/security/vcom_test.go | 315 +++ evetest/tests/storage/testsuite_test.go | 36 + evetest/tests/storage/vault_trim_test.go | 35 +- .../storage/zvol_provisioned_size_test.go | 18 +- evetest/tests/upgrade/testsuite_test.go | 16 +- evetest/tests/upgrade/upgrade_test.go | 18 +- 37 files changed, 6324 insertions(+), 648 deletions(-) create mode 100644 evetest/tests/apps/purge_test.go create mode 100644 evetest/tests/apps/testsuite_test.go create mode 100644 evetest/tests/apps/vnc_test.go create mode 100644 evetest/tests/lps/app_local_info_test.go create mode 100644 evetest/tests/lps/dev_local_info_test.go create mode 100644 evetest/tests/lps/helpers_test.go create mode 100644 evetest/tests/lps/profile_test.go create mode 100644 evetest/tests/lps/radio_silence_test.go create mode 100644 evetest/tests/security/apparmor_test.go create mode 100644 evetest/tests/security/testsuite_test.go create mode 100644 evetest/tests/security/vcom_test.go create mode 100644 evetest/tests/storage/testsuite_test.go diff --git a/evetest/Dockerfile.evetest b/evetest/Dockerfile.evetest index c45ab2276d1..dca7c26ff41 100644 --- a/evetest/Dockerfile.evetest +++ b/evetest/Dockerfile.evetest @@ -5,7 +5,7 @@ ARG EVETEST_VERSION=dev ARG ALPINE_VERSION=3.21 ARG GOLANG_VERSION=1.25 ARG EVETEST_ADAM_REPO=lfedge/adam -ARG EVETEST_ADAM_VERSION=0.0.75 +ARG EVETEST_ADAM_VERSION=0.0.81 ########################### # Pull adam binary stage diff --git a/evetest/Makefile b/evetest/Makefile index d4dbccf4b80..45979ed2244 100644 --- a/evetest/Makefile +++ b/evetest/Makefile @@ -4,7 +4,7 @@ EVETEST_VERSION := $(shell grep -v '^\#' VERSION | head -n1) EVETEST_ORG ?= lfedge -EVETEST_ADAM_VERSION ?= 0.0.75 +EVETEST_ADAM_VERSION ?= 0.0.81 EVETEST_ADAM_REPO ?= lfedge/adam EVETEST_IMAGE := $(EVETEST_ORG)/evetest:$(EVETEST_VERSION) diff --git a/evetest/README.md b/evetest/README.md index e690e63e215..9d5902993c3 100644 --- a/evetest/README.md +++ b/evetest/README.md @@ -750,7 +750,7 @@ same terminal session beforehand. |----------|-------------|---------| | `EVETEST_ORG` | Docker Hub organization for evetest and evetest-broker images | `lfedge` | | `EVETEST_EVE_REPO` | EVE image repository | `lfedge/eve` | -| `EVETEST_ADAM_VERSION` | Adam controller version *(build-time only, see note below)* | `0.0.75` | +| `EVETEST_ADAM_VERSION` | Adam controller version *(build-time only, see note below)* | `0.0.81` | | `EVETEST_SDN_VERSION` | SDN emulator version | `1.1` | > **`EVETEST_ADAM_VERSION` requires evetest container rebuild.** Unlike the other diff --git a/evetest/netmodels/multi-eth.go b/evetest/netmodels/multi-eth.go index 505bb40d8db..a491fdc9161 100644 --- a/evetest/netmodels/multi-eth.go +++ b/evetest/netmodels/multi-eth.go @@ -180,6 +180,121 @@ var TwoMgmtPorts = &api.NetworkModel{ }, } +// TwoMgmtPortsWithPublicNTP is a network model with two ethernet ports, each +// on its own bridge/network with DHCP and access to the controller (same +// layout as TwoMgmtPorts), plus a distinct real public NTP server IP +// advertised via DHCP option 42 on each network (api.DHCP.PublicNtp, +// dnsmasq's "public_ntp" -- a real address, not an SDN-hosted endpoint). This +// lets NTP tests exercise EVE's DHCP+static NTP server merging (and actually +// observe chronyd syncing) without SDN having to run an NTP daemon of its own. +// +// network0 (eth0) advertises Cloudflare's primary NTP anycast address +// (162.159.200.1); network1 (eth1) advertises Google's time1.google.com +// address (216.239.35.0). Both are long-stable, single, documented IPs (not +// pool.ntp.org-style rotating addresses), so tests can assert on them exactly. +var TwoMgmtPortsWithPublicNTP = &api.NetworkModel{ + Ports: []*api.Port{ + { + LogicalLabel: "eth0", + AdminUp: true, + }, + { + LogicalLabel: "eth1", + AdminUp: true, + }, + }, + Bridges: []*api.Bridge{ + { + LogicalLabel: "bridge0", + Ports: []string{"eth0"}, + }, + { + LogicalLabel: "bridge1", + Ports: []string{"eth1"}, + }, + }, + Networks: []*api.Network{ + { + LogicalLabel: "network0", + Bridge: "bridge0", + Ipv4: &api.NetworkIPConfig{ + Subnet: "172.20.20.0/24", + GwIp: "172.20.20.1", + Dhcp: &api.DHCP{ + Enable: true, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server0"}, + }, + NtpSource: &api.DHCP_PublicNtp{PublicNtp: "162.159.200.1"}, + }, + }, + }, + { + LogicalLabel: "network1", + Bridge: "bridge1", + Ipv4: &api.NetworkIPConfig{ + Subnet: "172.20.21.0/24", + GwIp: "172.20.21.1", + Dhcp: &api.DHCP{ + Enable: true, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server1"}, + }, + NtpSource: &api.DHCP_PublicNtp{PublicNtp: "216.239.35.0"}, + }, + }, + }, + }, + Endpoints: &api.Endpoints{ + DnsServers: []*api.DNSServer{ + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server0", + Fqdn: "dns-server0.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.16.0/24", + Ip: "10.16.16.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server1", + Fqdn: "dns-server1.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.17.0/24", + Ip: "10.16.17.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + }, + }, +} + // TwoMgmtPortsOneBridge is a network model with two ethernet ports on a single // bridge and network with DHCP and access to the controller. It is intended // for bond (link aggregation) tests where both ports must reach the same network. @@ -1813,6 +1928,112 @@ var AppGatewayTopology = &api.NetworkModel{ }, } +// MgmtViaAppTopology is a two-port network model for routing EVE's own +// device-management traffic through an application acting as a NAT gateway. +// +// - eth0 ("wan-network", 10.60.10.0/24): app-shared Switch NI port, fully +// reachable (controller, dns-server). DHCP with a static reservation: +// MAC 02:16:3e:02:00:00 -> 10.60.10.150. The gateway app's WAN VIF must +// use this MAC to receive the deterministic IP. +// - eth1 ("lan-network", 10.60.20.0/24): management port, but the network +// itself has no outside reachability and (WithoutDefaultRoute) hands out +// no router option -- it only provides L2 connectivity between EVE's own +// static IP and the gateway app's LAN VIF. DHCP with a static +// reservation: MAC 02:16:3e:02:00:01 -> 10.60.20.150. The gateway app's +// LAN VIF must use this MAC so EVE can target it as a fixed gateway IP. +// +// The dns-server (10.16.16.25) is reachable only via eth0/wan-network and +// resolves the controller hostname. +var MgmtViaAppTopology = &api.NetworkModel{ + Ports: []*api.Port{ + {LogicalLabel: "eth0", AdminUp: true}, + {LogicalLabel: "eth1", AdminUp: true}, + }, + Bridges: []*api.Bridge{ + {LogicalLabel: "bridge0", Ports: []string{"eth0"}}, + {LogicalLabel: "bridge1", Ports: []string{"eth1"}}, + }, + Networks: []*api.Network{ + { + // WAN leg: the app's outbound (MASQUERADE'd) traffic exits here. + LogicalLabel: "wan-network", + Bridge: "bridge0", + Ipv4: &api.NetworkIPConfig{ + Subnet: "10.60.10.0/24", + GwIp: "10.60.10.1", + Dhcp: &api.DHCP{ + Enable: true, + IpRange: &api.IPRange{ + FromIp: "10.60.10.100", ToIp: "10.60.10.140"}, + StaticEntries: []*api.MACToIP{ + {Mac: "02:16:3e:02:00:00", Ip: "10.60.10.150"}, + }, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server"}, + }, + }, + }, + Router: &api.Router{ + OutsideReachability: true, + ReachableEndpoints: []string{"dns-server"}, + }, + }, + { + // LAN leg: isolated at the SDN level -- EVE's own static IP and the + // app's LAN VIF are directly L2-adjacent on this bridge, with no + // router-provided path to anywhere else. WithoutDefaultRoute keeps + // the app's own DHCP-assigned VIF from picking up a bogus default + // route via this segment (its real default route must go via the + // WAN leg instead). + LogicalLabel: "lan-network", + Bridge: "bridge1", + Ipv4: &api.NetworkIPConfig{ + Subnet: "10.60.20.0/24", + GwIp: "10.60.20.1", + Dhcp: &api.DHCP{ + Enable: true, + IpRange: &api.IPRange{ + FromIp: "10.60.20.100", ToIp: "10.60.20.140"}, + DomainName: "test", + WithoutDefaultRoute: true, + StaticEntries: []*api.MACToIP{ + {Mac: "02:16:3e:02:00:01", Ip: "10.60.20.150"}, + }, + }, + }, + Router: &api.Router{ + OutsideReachability: false, + }, + }, + }, + Endpoints: &api.Endpoints{ + DnsServers: []*api.DNSServer{ + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server", + Fqdn: "dns-server.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.16.0/24", + Ip: "10.16.16.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + }, + }, +} + // SeparateClusterPort is a multi-Ethernet network model with a dedicated cluster port per device. var SeparateClusterPort = &api.NetworkModel{ Ports: []*api.Port{ diff --git a/evetest/tests/apps/purge_test.go b/evetest/tests/apps/purge_test.go new file mode 100644 index 00000000000..d6f3fb810b6 --- /dev/null +++ b/evetest/tests/apps/purge_test.go @@ -0,0 +1,242 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + uuid "github.com/satori/go.uuid" + "google.golang.org/protobuf/proto" +) + +// TestPurgeNeverActivatedApp is a regression test for EVE commit a1582bb40 +// ("zedmanager: fix purge stuck when app was never activated"). +// +// The bug: when a purge was triggered for an app whose image had failed to +// download (so no domain was ever created for it), the old code left +// PurgeInprogress=BringDown without calling purgeCmdDone. This caused the +// app to get stuck indefinitely (at LOADED with VerifyOnly=true) because +// volumemgr was never told to create the volume. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability +// and to pull the container images; the app under test needs no network +// adapter of its own for this test. +// +// Phases +// ------ +// 1. Deploy "bad-image-app" referencing a nonexistent image tag +// (docker://nginx:purge-test-nonexistent-99999). Tag resolution fails, +// so the app never activates (no domain is ever created). Wait for the +// app to report a non-empty AppErr while not RUNNING -- this confirms +// the broken precondition without assuming any particular stuck +// SwState (the exact state the app settles into isn't the point of this +// test, and asserting on the wrong one would make the test as fragile +// as the bug it's meant to catch). +// 2. fixImageAndPurge applies, in a single EdgeDevConfig mutation / +// ApplyConfig call: +// a. a fresh ContentTree (new UUID, same datastore, URL fixed to the +// working tag docker://nginx:stable, Sha256 cleared), +// b. a fresh Volume (new UUID) whose origin points at that fresh +// ContentTree, +// c. the app's VolumeRefList rewired to the fresh Volume, +// d. the app's purge counter incremented. +// This combination -- fixing the image while simultaneously purging -- +// is exactly what triggers the bug: EVE must tear down the +// never-activated app and reprocess it from scratch instead of getting +// stuck. +// 3. With the fix, the app eventually reaches RUNNING. (Without the fix, +// this step is where the test would time out -- the app would remain +// stuck indefinitely instead.) +// 4. Cleanup: delete the app, wait for ZSwState_INVALID. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestAppsSuite. +func TestPurgeNeverActivatedApp(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + // Step 1: deploy an app whose image tag does not exist, so it never + // activates. + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "bad-image-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "nginx", + Tag: "purge-test-nonexistent-99999", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + }) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + device.ApplyConfig(devConfig, false, false) + + log := evetest.Logger() + log.Infof("Waiting for the app to fail to activate (nonexistent image tag)") + timeout := 5 * time.Minute + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app never activates (bad image tag) and reports an error", + func(info *eveinfo.ZInfoApp) bool { + return info.State != eveinfo.ZSwState_RUNNING && len(info.AppErr) > 0 + }))) + evetest.Checkpoint("app-never-activated") + + // Step 2: fix the image and purge, atomically. + log.Infof("Fixing the image and purging the app in a single config apply") + fixImageAndPurge(t, devConfig, appUUID, "nginx:stable") + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("app-fixed-and-purged") + + // Step 3: with the fix, the app recovers and reaches RUNNING. Without it, + // this is where the test would time out. + log.Infof("Waiting for the app to reach RUNNING after the purge") + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app reaches RUNNING after purge", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_RUNNING + }))) + evetest.Checkpoint("app-running-after-purge") + + // Cleanup. + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopAppWatch() +} + +// fixImageAndPurge replicates, as a single EdgeDevConfig mutation, what a +// real app purge does (fresh Volume + ContentTree identities) while also +// fixing the app's image reference to a working one -- the exact +// combination that reproduces EVE commit a1582bb40. goodImageURL is the +// ":" reference to switch the app's ContentTree to (same +// Datastore, only the URL and identity change). +func fixImageAndPurge(t *WithT, devConfig *evetest.EdgeDeviceConfig, + appUUID uuid.UUID, goodImageURL string) { + appUUIDStr := appUUID.String() + + var app *eveconfig.AppInstanceConfig + for _, a := range devConfig.Apps { + if a.GetUuidandversion().GetUuid() == appUUIDStr { + app = a + break + } + } + t.Expect(app).ToNot(BeNil(), "app %s not found in config", appUUIDStr) + t.Expect(app.VolumeRefList).To(HaveLen(1)) + + oldVolUUID := app.VolumeRefList[0].Uuid + oldVolIdx := -1 + for i, v := range devConfig.Volumes { + if v.Uuid == oldVolUUID { + oldVolIdx = i + break + } + } + t.Expect(oldVolIdx).To(BeNumerically(">=", 0), + "volume %s not found in config", oldVolUUID) + oldVolume := devConfig.Volumes[oldVolIdx] + + oldCTUUID := oldVolume.GetOrigin().GetDownloadContentTreeID() + oldCTIdx := -1 + for i, ct := range devConfig.ContentInfo { + if ct.Uuid == oldCTUUID { + oldCTIdx = i + break + } + } + t.Expect(oldCTIdx).To(BeNumerically(">=", 0), + "content tree %s not found in config", oldCTUUID) + oldCT := devConfig.ContentInfo[oldCTIdx] + + newCTUUID, err := uuid.NewV4() + t.Expect(err).ToNot(HaveOccurred()) + newVolUUID, err := uuid.NewV4() + t.Expect(err).ToNot(HaveOccurred()) + + // Fresh ContentTree: same Datastore reference, fixed (working) URL. + newCT := proto.CloneOf(oldCT) + newCT.Uuid = newCTUUID.String() + newCT.URL = goodImageURL + newCT.Sha256 = "" + devConfig.ContentInfo[oldCTIdx] = newCT + + // Fresh Volume pointing at the fresh ContentTree. + newVolume := proto.CloneOf(oldVolume) + newVolume.Uuid = newVolUUID.String() + newVolume.Origin.DownloadContentTreeID = newCTUUID.String() + devConfig.Volumes[oldVolIdx] = newVolume + + // Rewire the app's volume reference and increment the purge counter -- + // together, this is exactly what a real purge does. + app.VolumeRefList[0].Uuid = newVolUUID.String() + purgeCounter := uint32(0) + if app.Purge != nil { + purgeCounter = app.Purge.Counter + } + app.Purge = &eveconfig.InstanceOpsCmd{Counter: purgeCounter + 1} +} + +func appHasError(info *eveinfo.ZInfoApp) (string, bool) { + stop := info.State == eveinfo.ZSwState_ERROR + if stop { + return "Application instance is in error state", true + } + return "", false +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go new file mode 100644 index 00000000000..a589b996af3 --- /dev/null +++ b/evetest/tests/apps/testsuite_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestAppsSuite drives application-lifecycle scenarios that are not +// specifically about networking (regression tests for zedmanager/volumemgr +// bugs, VNC console access, etc.) -- kept separate from +// evetest/tests/networking's TestApplicationConnectivitySuite, which is +// already large and focused on network connectivity. +// +// Subtests +// -------- +// - TestPurgeNeverActivatedApp -- regression test for a zedmanager bug +// where purging an app that never activated (failed image download) +// would leave it stuck instead of recovering. +// - TestVNC -- VNC access to a VM app, a container app, and the container +// app's shim VM console. +func TestAppsSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestPurgeNeverActivatedApp, + }, + evetest.TestCase{ + Test: TestVNC, + }, + ) +} diff --git a/evetest/tests/apps/vnc_test.go b/evetest/tests/apps/vnc_test.go new file mode 100644 index 00000000000..9407b768b68 --- /dev/null +++ b/evetest/tests/apps/vnc_test.go @@ -0,0 +1,414 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "fmt" + "net" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/amitbet/vncproxy/client" + "github.com/amitbet/vncproxy/logger" + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// X11/RFB keysym constants used to drive the shim-VM console switch +// documented in docs/VNC.md ("Switching between the container and the shim +// VM session"): Ctrl+Alt+2 then Enter. +const ( + keysymControlL = 0xffe3 + keysymAltL = 0xffe9 + keysym2 = 0x0032 + keysymReturn = 0xff0d +) + +// alpineCloudImage describes the arch-specific pinned Alpine Linux +// cloud-init qcow2 image used to boot the VM app in this test. +type alpineCloudImage struct { + relativePath string + sha256 string + sizeBytes uint64 +} + +// Alpine 3.24.1 cloud images, pinned by release version (not a rolling +// "latest" alias) so the SHA256 below stays valid indefinitely. See +// https://alpinelinux.org/cloud/ for the full image list. +var alpineCloudImages = map[string]alpineCloudImage{ + "amd64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2", + sha256: "6e2e6fe0572b6632527f268d3659e8fccebda4e1ee470fafe2c4d7b85b6a4df6", + sizeBytes: 183697408, + }, + "arm64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-aarch64-uefi-cloudinit-r0.qcow2", + sha256: "3059a6280977c2122982632e0317c5ddbd39069d46ca1e60480de283091f720f", + sizeBytes: 239271936, + }, +} + +// TestVNC verifies VNC access to EVE application consoles: a VM app's own +// display, a container app's display, and the container app's underlying +// shim VM console (reached via the documented key-combo switch once shim-VM +// VNC access is enabled). +// +// Under KVM the VNC server is a raw QEMU socket reachable directly on the +// device's uplink IP. Under Kubevirt there is no such socket: zedkube starts +// a `virtctl vnc --proxy-only` process (see pkg/pillar/docs/ +// vnc-workflows.md) that bridges the VMI's console onto a TCP port bound to +// 127.0.0.1 only, gated by AppInstanceConfig.RemoteConsole rather than +// EnableVNC. This test reaches that port through an SSH tunnel to the device +// (EdgeDevice.DialViaSSH) instead of a direct dial, and falls back to VNC's +// "none" security type when the server doesn't offer password auth (virtctl +// relies on Kubernetes RBAC for access control, not a VNC password) -- see +// connectVNC. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability +// and Internet access (to pull the Alpine cloud image and the +// evetest-ubuntu-ctr container image). VNC itself is never reached +// through an app network, so neither app in this test is given a network +// adapter of its own. +// +// Phases +// ------ +// 1. Global config: enable app.allow.vnc (opens the device's uplink +// firewall for TCP ports 5900-5999 -- see +// pkg/pillar/dpcreconciler/linux.go) and debug.enable.vnc.shim.vm (a +// per-node flag which, combined with a VM's own EnableVnc, additionally +// allows switching into the shim VM console -- see +// pkg/pillar/hypervisor/kvm.go, isVncShimVMEnabled). Both are KVM-only +// knobs (harmless, but ineffective, under Kubevirt). +// 2. VM app: deploy an Alpine Linux VM (VirtualizationMode=HVM, image from +// alpineCloudImages matching the device's actual arch (device.GetArch()), +// EnableVNC=true, VNCDisplay=1, a fixed VNCPassword). No UserData/ +// cloud-init is configured -- VNC reaches the QEMU console as soon as +// the VM starts, independent of what the guest OS is doing. +// WaitUntilAppIsRunning, then (under Kubevirt) enable RemoteConsole on +// the app config and re-apply, connect a real VNC/RFB client to port +// 5901 (directly under KVM, through an SSH tunnel under Kubevirt), +// authenticate, and assert the handshake succeeds. Delete the app +// afterwards (disabling RemoteConsole first under Kubevirt, since only +// one remote-console session is allowed on the device at a time). +// 3. Container app: deploy lfedge/evetest-ubuntu-ctr:1.0 with EnableVNC=true, +// VNCDisplay=2, the same VNCPassword. WaitUntilAppIsRunning, then connect +// to port 5902 (same per-hypervisor path as above) and assert the same +// successful handshake -- proving VNC also works for container apps +// (viewing the shim VM's console showing the container's entry point). +// 4. Shim VM console switch: on that same VNC connection, send the +// documented key combo (Ctrl+Alt+2, then Enter -- see docs/VNC.md, +// "Switching between the container and the shim VM session") via RFB +// KeyEvent messages, then issue a fresh FramebufferUpdateRequest and a +// benign follow-up call, asserting neither errors -- i.e. the session +// stays healthy across the console switch. Note: asserting that the +// *displayed content* actually changed to the shim VM's login prompt +// would need pixel-level framebuffer decoding, which the available Go +// VNC client library does not expose at a level convenient enough to +// verify robustly here; left as a known scope limit. +// 5. Cleanup: delete the container app. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestAppsSuite. +func TestVNC(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Phase 1: allow external VNC access to the device's uplink ports and + // enable node-wide shim-VM VNC access. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueBool(pillartypes.AllowAppVnc, true) + cfgProps.SetGlobalValueBool(pillartypes.VncShimVMAccess, true) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + const vncPassword = "12345678" // classic VNC (DES) auth is limited to 8 chars + timeout := 3 * time.Minute + timeoutExcludingDownload := 8 * time.Minute + log := evetest.Logger() + + // arch selects which pinned Alpine image matches the device's actual + // CPU architecture. + arch := device.GetArch() + image, ok := alpineCloudImages[arch] + t.Expect(ok).To(BeTrue(), "no pinned Alpine cloud image for arch %q", arch) + + var deviceIP []net.IP + if hypervisor != evetest.HypervisorKubevirt { + deviceIP = device.GetDeviceIPAddress("ethernet0") + t.Expect(deviceIP).ToNot(BeEmpty()) + } + + // vncDialer returns a description (for log/error messages) and a dial + // function for the VNC endpoint at 5900+vncDisplay: a direct dial to the + // device's uplink IP under KVM/Xen, or a dial tunneled over SSH to the + // device's own loopback under Kubevirt (see the doc comment above). + vncDialer := func(vncDisplay uint) (string, func() (net.Conn, error)) { + port := fmt.Sprintf("%d", 5900+vncDisplay) + if hypervisor == evetest.HypervisorKubevirt { + addr := net.JoinHostPort("127.0.0.1", port) + return addr + " (via SSH tunnel)", func() (net.Conn, error) { + return device.DialViaSSH("tcp", addr) + } + } + addr := net.JoinHostPort(deviceIP[0].String(), port) + return addr, func() (net.Conn, error) { + return net.DialTimeout("tcp", addr, 5*time.Second) + } + } + + // setRemoteConsole toggles AppInstanceConfig.RemoteConsole for the app + // identified by appUUIDStr and re-applies the device config. This is the + // field that gates Kubevirt's virtctl-based VNC proxy (see + // ApplicationInstanceConfig.RemoteConsole). + setRemoteConsole := func(appUUIDStr string, enable bool) { + found := false + for _, app := range devConfig.Apps { + if app.GetUuidandversion().GetUuid() == appUUIDStr { + app.RemoteConsole = enable + found = true + break + } + } + t.Expect(found).To(BeTrue(), "app %q not found in device config", appUUIDStr) + device.ApplyConfig(devConfig, false, false) + } + + // waitRemoteConsoleReleased polls (via the same SSH-tunneled dial used to + // reach it) until the Kubevirt VNC proxy for vncDisplay has stopped + // listening, i.e. zedkube has torn it down after RemoteConsole was + // disabled. Only one remote-console session is allowed on the device at + // a time (see pkg/pillar/docs/vnc-workflows.md, canClaimVNCFile), so the + // next app's setRemoteConsole(true) would otherwise be silently ignored + // while the previous proxy is still up. + waitRemoteConsoleReleased := func(vncDisplay uint) { + _, dial := vncDialer(vncDisplay) + t.Eventually(func() error { + conn, err := dial() + if err == nil { + _ = conn.Close() + return fmt.Errorf("VNC proxy for display %d is still listening", vncDisplay) + } + return nil + }, 30*time.Second, 2*time.Second).Should(Succeed()) + } + + // Phase 2: VM app VNC access. + vmAppUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vnc-vm-app", + Activate: true, + Image: evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW2, + ImageSHA256: image.sha256, + MaxDownloadBytes: image.sizeBytes, + ImageRelativePath: image.relativePath, + ServerAddress: "dl-cdn.alpinelinux.org", + UseHTTPS: true, + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + EnableVNC: true, + VNCDisplay: 1, + VNCPassword: vncPassword, + }) + vmAppUpdates, stopVMAppWatch := device.WatchAppInfo(vmAppUUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(vmAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("vm-app-running") + + vmAppUUIDStr := vmAppUUID.String() + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(vmAppUUIDStr, true) + } + log.Infof("Testing VNC access to the VM app") + vmVNCDesc, vmVNCDial := vncDialer(1) + vmConn := connectVNC(t, vmVNCDesc, vmVNCDial, vncPassword) + log.Infof("Connected to VM app VNC desktop %q", vmConn.DesktopName) + t.Expect(vmConn.Close()).To(Succeed()) + evetest.Checkpoint("vm-app-vnc-verified") + + if hypervisor == evetest.HypervisorKubevirt { + // Release the remote-console session before deleting the app: only + // one is allowed on the device at a time, and the container app's + // session below would otherwise be silently blocked. + setRemoteConsole(vmAppUUIDStr, false) + waitRemoteConsoleReleased(1) + } + + devConfig.DeleteApplication(vmAppUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(vmAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "VM app state is UNSPECIFIED", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopVMAppWatch() + + // Phase 3: container app VNC access. + ctrAppUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vnc-ctr-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnableVNC: true, + VNCDisplay: 2, + VNCPassword: vncPassword, + }) + ctrAppUUIDStr := ctrAppUUID.String() + ctrAppUpdates, stopCtrAppWatch := device.WatchAppInfo(ctrAppUUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(ctrAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("container-app-running") + + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(ctrAppUUIDStr, true) + } + log.Infof("Testing VNC access to the container app") + ctrVNCDesc, ctrVNCDial := vncDialer(2) + ctrConn := connectVNC(t, ctrVNCDesc, ctrVNCDial, vncPassword) + log.Infof("Connected to container app VNC desktop %q", ctrConn.DesktopName) + evetest.Checkpoint("container-app-vnc-verified") + + // Phase 4: switch to the shim VM console (Ctrl+Alt+2, then Enter -- see + // docs/VNC.md) and confirm the session stays healthy across the switch. + log.Infof("Switching to the shim VM console") + t.Expect(pressKeyCombo(ctrConn, keysymControlL, keysymAltL, keysym2)).To(Succeed()) + t.Expect(pressKeyCombo(ctrConn, keysymReturn)).To(Succeed()) + t.Expect(ctrConn.FramebufferUpdateRequest( + false, 0, 0, ctrConn.FrameBufferWidth, ctrConn.FrameBufferHeight)).To(Succeed()) + time.Sleep(2 * time.Second) + // A benign follow-up call: if the console switch broke the session, the + // underlying connection would already be closed and this would error. + t.Expect(ctrConn.FramebufferUpdateRequest( + true, 0, 0, ctrConn.FrameBufferWidth, ctrConn.FrameBufferHeight)).To(Succeed()) + t.Expect(ctrConn.Close()).To(Succeed()) + evetest.Checkpoint("shim-vm-switch-verified") + + // Cleanup. + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(ctrAppUUIDStr, false) + } + devConfig.DeleteApplication(ctrAppUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(ctrAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "Container app state is UNSPECIFIED", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopCtrAppWatch() +} + +// connectVNC calls dial (retrying while the endpoint isn't reachable yet -- +// under Kubevirt the virtctl proxy takes a few seconds to start after +// RemoteConsole is enabled), then performs the full RFB handshake and +// returns the connected client. dialDesc identifies the endpoint in log/error +// messages. It fails the test (via t) rather than returning an error, since a +// broken VNC connection always indicates a test failure at the call sites +// above. +func connectVNC(t *WithT, dialDesc string, dial func() (net.Conn, error), + password string) *client.ClientConn { + // The client's background mainLoop goroutine logs an Error-level message + // whenever its blocking read unblocks because the connection was closed + // -- including on an intentional Close() from our side. Silence it here + // (matching eden's own pkg/utils/vnc.go, which does the same) so a clean + // shutdown doesn't look like a failure in the test output. + logger.SetLogLevel("fatal") + + var conn net.Conn + t.Eventually(func() error { + var err error + conn, err = dial() + return err + }, 90*time.Second, 3*time.Second).Should(Succeed(), + "failed to dial VNC endpoint %s", dialDesc) + + clientConn, err := client.NewClientConn(conn, &client.ClientConfig{ + // A raw QEMU VNC socket (KVM) offers VNC password auth; Kubevirt's + // virtctl proxy offers only the "none" security type, relying on + // Kubernetes RBAC instead of a VNC password. Offering both lets the + // client pick whichever the server actually supports. + Auth: []client.ClientAuth{&client.PasswordAuth{Password: password}, new(client.ClientAuthNone)}, + Exclusive: false, + }) + t.Expect(err).ToNot(HaveOccurred()) + + t.Expect(clientConn.Connect()).To(Succeed(), + "VNC RFB handshake with %s failed", dialDesc) + return clientConn +} + +// pressKeyCombo sends a key-down event for every keysym in order, then a +// key-up event in reverse order, emulating a user holding several keys +// together (e.g. Ctrl+Alt+2). +func pressKeyCombo(conn *client.ClientConn, keysyms ...uint32) error { + for _, keysym := range keysyms { + if err := conn.KeyEvent(keysym, true); err != nil { + return fmt.Errorf("key-down for keysym 0x%x: %w", keysym, err) + } + } + for i := len(keysyms) - 1; i >= 0; i-- { + if err := conn.KeyEvent(keysyms[i], false); err != nil { + return fmt.Errorf("key-up for keysym 0x%x: %w", keysyms[i], err) + } + } + return nil +} diff --git a/evetest/tests/cluster/cluster_test.go b/evetest/tests/cluster/cluster_test.go index f3f6a983b71..df9520de302 100644 --- a/evetest/tests/cluster/cluster_test.go +++ b/evetest/tests/cluster/cluster_test.go @@ -13,9 +13,7 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" - eveinfo "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/evetest" - "github.com/lf-edge/eve/evetest/matchers" "github.com/lf-edge/eve/evetest/netmodels" "github.com/lf-edge/eve/pkg/pillar/types" ) @@ -74,13 +72,13 @@ func clusterDeviceRequirements( // Phases // ------ // 1. setup-done -> initial-config-applied: apply the bare device config -// (no app yet) and start watching ClusterInfo. -// 2. k3s-is-ready: ZInfoKubeCluster eventually reports a single node -// whose NodeReady condition is true AND -// Storage.Health=SERVICE_STATUS_HEALTHY. Then assert ClusterId is -// non-empty, the node is RoleServer + Schedulable, and that no -// EveApps / EveVmApps / PodNameSpaces have been created yet -// (clean-slate cluster, no workload). +// (no app yet). +// 2. k3s-is-ready: device.WaitForClusterNodeIsReady waits until this +// device reports itself as a Ready node with healthy cluster storage. +// Then assert there is exactly one node, ClusterId is non-empty, the +// node is RoleServer + Schedulable, and that no EveApps / EveVmApps / +// PodNameSpaces have been created yet (clean-slate cluster, no +// workload). // 3. app-config-is-submitted: add the local NI + the container app to the // config and re-apply. // 4. app-is-deployed: WaitUntilAppIsRunning (the helper tracks @@ -135,31 +133,13 @@ func TestSingleNodeCluster(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device := evetest.GetEdgeDevice(devName) - clusterUpdates, stopClusterWatch := device.WatchClusterInfo() - defer stopClusterWatch() device.ApplyConfig(devConfig, true, true) evetest.Checkpoint("initial-config-applied") timeout := 20 * time.Minute - var clusterInfo *eveinfo.ZInfoKubeCluster - const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY - t.Eventually(clusterUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( - "K3s is ready", - func(info *eveinfo.ZInfoKubeCluster) bool { - clusterInfo = info - if len(info.Nodes) != 1 { - return false - } - if clusterInfo.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { - return false - } - for _, cond := range info.Nodes[0].GetConditions() { - if cond.GetType() == nodeReadyCond { - return cond.GetSet() - } - } - return false - }))) + device.WaitForClusterNodeIsReady(timeout) + clusterInfo := device.GetClusterInfo() + t.Expect(clusterInfo.Nodes).To(HaveLen(1)) t.Expect(clusterInfo.ClusterId).NotTo(BeEmpty()) t.Expect(clusterInfo.Nodes[0].RoleServer).To(BeTrue()) t.Expect(clusterInfo.Nodes[0].Schedulable).To(BeTrue()) diff --git a/evetest/tests/lps/app_local_info_test.go b/evetest/tests/lps/app_local_info_test.go new file mode 100644 index 00000000000..f8a23e6a800 --- /dev/null +++ b/evetest/tests/lps/app_local_info_test.go @@ -0,0 +1,319 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/utils/generics" +) + +// TestAppLocalInfo verifies the Local Profile Server (LPS) app-info / +// app-command exchange: EVE reports every deployed app's state to the LPS +// (GET /manage/v1/appinfo), and the LPS can request a purge or restart of a +// specific app (PUT /manage/v1/app-command) independently of the +// controller. EVE sums LPS-driven and controller-driven purge/restart +// counters -- both paths must work, and only a purge (LPS-driven or +// controller-driven) recreates the app's volume; a restart does not. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. Local NI "local-ni" +// hosts both the LPS app (port-fwd for its management API + SSH) and +// "app1" (port-fwd for direct SSH, used to create/check files that +// prove whether a purge recreated the volume). +// +// Phases +// ------ +// 1. Deploy the LPS app. WaitUntilAppIsRunning, configure the LPS token, +// and configure EVE to use it via SetLPS. +// 2. Confirm the LPS's app-info list includes "lps-app" but not "app1" +// (app1 does not exist yet). +// 3. Deploy "app1" (plain container, no ProfileList). WaitUntilAppIsRunning, +// wait for SSH, then confirm the LPS's app-info list now includes both +// apps. +// 4. LPS-driven purge: create /root/purge_test in app1, confirm app1's +// LPS-reported LastCmdTimestamp is still 0, then PUT an app-command +// {app1, timestamp=123, COMMAND_PURGE}. Wait for app1 to pass through +// PURGING/HALTING and back to RUNNING, confirm the LPS now reports +// LastCmdTimestamp=123, and confirm /root/purge_test is gone (purge +// recreated the volume). +// 5. Controller-driven purge: create /root/purge_test again, call +// PurgeApplication directly (bypassing the LPS). Wait for the same +// PURGING/HALTING -> RUNNING cycle, confirm the LPS-reported +// LastCmdTimestamp is *unchanged* at 123 (a controller-driven purge +// does not touch the LPS-tracked command timestamp), and confirm +// /root/purge_test is gone again. +// 6. LPS-driven restart: create both /tmp/restart_test (tmpfs -- wiped by +// any VM reboot) and /root/purge_test (persistent -- survives a +// restart, unlike a purge) in app1, then PUT an app-command {app1, +// timestamp=456, COMMAND_RESTART}. Wait for app1 to pass through +// RESTARTING/HALTING and back to RUNNING, confirm LastCmdTimestamp=456, +// confirm /tmp/restart_test is gone but /root/purge_test still exists +// (a restart does not recreate the volume). +// 7. Controller-driven restart: same file setup, call RebootApplication +// directly. Confirm LastCmdTimestamp is *unchanged* at 456, and the +// same file-survival pattern. +// 8. Delete app1, wait for it to be gone, then confirm the LPS's app-info +// list no longer includes "app1" (but still includes "lps-app"). +// 9. Cleanup: delete the LPS app and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestAppLocalInfo(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 5 * time.Minute + polling := 3 * time.Second + + // Step 2: appinfo includes lps-app but not app1 yet. + log.Infof("Waiting for the LPS to report appinfo for lps-app only") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).To(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-before-app1") + + // Step 3: deploy app1. + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2224, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + sshTimeout := 20 * time.Second + waitApp1SSHReachable := func() { + log.Infof("Waiting for app1 SSH to become reachable...") + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + app1UUID, lpsAppAuth, "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitApp1SSHReachable() + + log.Infof("Waiting for the LPS to report appinfo for both apps") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).ToNot(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-with-app1") + + // waitAppTransientThenRunning drains app1Updates until it reports one of + // the given transient states (no exported helper targets these), then + // waits for RUNNING again via WaitUntilAppIsRunning directly, then for + // SSH to become reachable again. + waitAppTransientThenRunning := func(transient ...eveinfo.ZSwState) { + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 enters a transient state", + func(info *eveinfo.ZInfoApp) bool { + return generics.ContainsItem(transient, info.State) + }))) + device.WaitUntilAppIsRunning(app1UUID, timeout) + waitApp1SSHReachable() + } + touchApp1 := func(path string) { + _, _, err := device.RunShellScriptInsideApp(app1UUID, lpsAppAuth, + "touch "+path, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + } + app1FileExists := func(path string) bool { + output, _, err := device.RunShellScriptInsideApp(app1UUID, lpsAppAuth, + "test -f "+path+" && echo EXISTS; true", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + return strings.Contains(output, "EXISTS") + } + waitLastCmdTimestamp := func(want uint64, desc string) { + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + info := appInfoByName(list, "app1") + t.Expect(info).ToNot(BeNil()) + t.Expect(info.GetLastCmdTimestamp()).To(Equal(want), desc) + }, timeout, polling).Should(Succeed()) + } + + // Step 4: LPS-driven purge. + log.Infof("Testing LPS-driven purge") + touchApp1("/root/purge_test") + waitLastCmdTimestamp(0, "app1 has no LPS command applied yet") + putLPSAppCommand(t, device, lpsAppUUID, "app1", 123, "COMMAND_PURGE") + waitAppTransientThenRunning(eveinfo.ZSwState_PURGING, eveinfo.ZSwState_HALTING) + waitLastCmdTimestamp(123, "app1 should report the LPS purge command timestamp") + t.Expect(app1FileExists("/root/purge_test")).To(BeFalse(), + "purge should have recreated app1's volume") + evetest.Checkpoint("lps-purge-done") + + // Step 5: controller-driven purge. EVE sums local and remote purge + // counters, but the LPS-visible LastCmdTimestamp must not change. + log.Infof("Testing controller-driven purge") + touchApp1("/root/purge_test") + device.PurgeApplication(app1UUID, true, timeout) + waitApp1SSHReachable() + waitLastCmdTimestamp(123, "controller-driven purge must not change the LPS timestamp") + t.Expect(app1FileExists("/root/purge_test")).To(BeFalse(), + "controller-driven purge should also recreate app1's volume") + evetest.Checkpoint("controller-purge-done") + + // Step 6: LPS-driven restart. Unlike a purge, a restart does not + // recreate the volume. + log.Infof("Testing LPS-driven restart") + touchApp1("/tmp/restart_test") + touchApp1("/root/purge_test") + putLPSAppCommand(t, device, lpsAppUUID, "app1", 456, "COMMAND_RESTART") + waitAppTransientThenRunning(eveinfo.ZSwState_RESTARTING, eveinfo.ZSwState_HALTING) + waitLastCmdTimestamp(456, "app1 should report the LPS restart command timestamp") + t.Expect(app1FileExists("/tmp/restart_test")).To(BeFalse(), + "restart should have wiped the tmpfs /tmp") + t.Expect(app1FileExists("/root/purge_test")).To(BeTrue(), + "restart should not have recreated app1's volume") + evetest.Checkpoint("lps-restart-done") + + // Step 7: controller-driven restart. + log.Infof("Testing controller-driven restart") + touchApp1("/tmp/restart_test") + touchApp1("/root/purge_test") + device.RebootApplication(app1UUID, true, timeout) + waitApp1SSHReachable() + waitLastCmdTimestamp(456, "controller-driven restart must not change the LPS timestamp") + t.Expect(app1FileExists("/tmp/restart_test")).To(BeFalse(), + "controller-driven restart should have wiped the tmpfs /tmp") + t.Expect(app1FileExists("/root/purge_test")).To(BeTrue(), + "controller-driven restart should not have recreated app1's volume") + evetest.Checkpoint("controller-restart-done") + + // Step 8: remove app1 and confirm the LPS stops seeing it. + devConfig.DeleteApplication(app1UUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + + log.Infof("Waiting for the LPS to stop reporting app1") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).To(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-after-app1-removed") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(lpsAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "lps-app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopLPSAppWatch() +} diff --git a/evetest/tests/lps/dev_local_info_test.go b/evetest/tests/lps/dev_local_info_test.go new file mode 100644 index 00000000000..0fbec1a361e --- /dev/null +++ b/evetest/tests/lps/dev_local_info_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestDevLocalInfo verifies the Local Profile Server (LPS) device-info / +// device-command exchange: EVE reports its own device state to the LPS (GET +// /manage/v1/devinfo), and the LPS can request a graceful shutdown or a +// graceful shutdown-and-poweroff of the whole device (PUT +// /manage/v1/dev-command) independently of the controller. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. Local NI "local-ni" +// hosts both the LPS app and "app1" (port-fwd for direct SSH, used only +// to confirm it is reachable/unreachable at the expected points). +// +// Phases +// ------ +// 1. Deploy the LPS app. WaitUntilAppIsRunning, configure the LPS token, +// and configure EVE to use it via SetLPS. +// 2. Deploy "app1" (plain container, no ProfileList). WaitUntilAppIsRunning, +// wait for SSH, then confirm the LPS's devinfo reports device state +// ONLINE. +// 3. LPS-driven graceful shutdown: PUT a dev-command {timestamp=100, +// COMMAND_SHUTDOWN}. Confirm the LPS-reported devinfo reaches state +// PREPARING_POWEROFF with LastCmdTimestamp=100 (checked while the LPS +// app is still up), then wait for both app1 and lps-app to reach +// HALTED (via their WatchAppInfo channels). +// 4. Recovery: reboot the device via RequestReboot (a controller-driven +// reboot). Wait for both apps to reach RUNNING again, wait for app1's +// SSH to become reachable again, then re-run waitLPSAppReady +// (the LPS app's own in-memory token is wiped by its restart) and confirm +// devinfo reports ONLINE again. +// 5. LPS-driven graceful shutdown-and-poweroff: PUT a dev-command +// {timestamp=200, COMMAND_GRACEFUL_POWEROFF}. Confirm the LPS-reported +// devinfo reaches state POWERING_OFF with LastCmdTimestamp=200 (again +// checked while the LPS app is still up), then wait for both app1 and +// lps-app to reach HALTED. +// 6. Call device.PowerOff() to force the VM fully off (this both confirms +// and, if needed, completes the poweroff EVE initiated in step 5), then +// device.PowerOn(false) to bring it back -- waitUntilOnline=false since +// LastRebootTime isn't reliable after a true hard power-off (see +// PowerOn's doc comment); recovery is confirmed via WaitUntilAppIsRunning +// on both apps below instead. Wait for both apps to reach RUNNING again +// and for app1's SSH to become reachable. +// 7. Cleanup: delete both apps and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestDevLocalInfo(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 5 * time.Minute + polling := 3 * time.Second + + // Step 2: deploy app1. + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2224, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + sshTimeout := 20 * time.Second + log.Infof("Waiting for app1 SSH to become reachable...") + waitApp1SSHReachable := func() { + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + app1UUID, lpsAppAuth, "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitApp1SSHReachable() + + // waitDevState polls the LPS-reported devinfo until it reaches the given + // state, and confirms the LastCmdTimestamp last applied by a dev-command + // (0 if none has been applied yet). Only valid while the LPS app itself + // is still up and reachable. + waitDevState := func(want eveinfo.ZDeviceState, wantLastCmdTimestamp uint64, desc string) { + log.Infof("Waiting for device state %s (%s)", want, desc) + t.Eventually(func(t Gomega) { + info := getLPSDevInfo(t, device, lpsAppUUID) + t.Expect(info.GetState()).To(Equal(want), desc) + t.Expect(info.GetLastCmdTimestamp()).To(Equal(wantLastCmdTimestamp), desc) + }, timeout, polling).Should(Succeed()) + } + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_ONLINE, 0, + "device online after app1 deployed") + evetest.Checkpoint("device-online") + + // waitAppState waits for the given app's WatchAppInfo channel to report + // the given state. Used for HALTED/INVALID waits below (RUNNING waits + // use WaitUntilAppIsRunning directly instead). + waitAppState := func(updates <-chan *eveinfo.ZInfoApp, appName string, want eveinfo.ZSwState) { + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + appName+" reaches "+want.String(), + func(info *eveinfo.ZInfoApp) bool { + return info.State == want + }))) + } + + // Step 3: LPS-driven graceful shutdown. + log.Infof("Testing LPS-driven COMMAND_SHUTDOWN") + putLPSDevCommand(t, device, lpsAppUUID, 100, "COMMAND_SHUTDOWN") + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_PREPARING_POWEROFF, 100, + "device shutting down after COMMAND_SHUTDOWN") + waitAppState(app1Updates, "app1", eveinfo.ZSwState_HALTED) + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_HALTED) + evetest.Checkpoint("shutdown-done") + + // Step 4: recovery via a controller-driven reboot. + log.Infof("Rebooting the device via the controller to recover from the shutdown") + device.RequestReboot(true) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + waitApp1SSHReachable() + + // The LPS app's in-memory token was wiped by its own restart; re-set it + // and re-confirm EVE has reconnected to it. + lpsIP = waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_ONLINE, 100, + "device online again after reboot") + evetest.Checkpoint("shutdown-recovered") + + // Step 5: LPS-driven graceful shutdown-and-poweroff. + log.Infof("Testing LPS-driven COMMAND_GRACEFUL_POWEROFF") + putLPSDevCommand(t, device, lpsAppUUID, 200, "COMMAND_GRACEFUL_POWEROFF") + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_POWERING_OFF, 200, + "device powering off after COMMAND_GRACEFUL_POWEROFF") + waitAppState(app1Updates, "app1", eveinfo.ZSwState_HALTED) + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_HALTED) + evetest.Checkpoint("poweroff-done") + + // Step 6: force the VM off (confirming/completing the poweroff EVE + // initiated above) and power it back on. + log.Infof("Powering the device off (via the broker) and back on") + device.PowerOff() + // waitUntilOnline=false: see PowerOn's doc comment -- LastRebootTime + // isn't reliable after a true hard power-off, so recovery is confirmed + // below via WaitUntilAppIsRunning instead. + device.PowerOn(false) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + waitApp1SSHReachable() + evetest.Checkpoint("poweroff-recovered") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + waitAppState(app1Updates, "app1", eveinfo.ZSwState_INVALID) + stopApp1Watch() + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_INVALID) + stopLPSAppWatch() +} diff --git a/evetest/tests/lps/helpers_test.go b/evetest/tests/lps/helpers_test.go new file mode 100644 index 00000000000..ac30572ad0f --- /dev/null +++ b/evetest/tests/lps/helpers_test.go @@ -0,0 +1,241 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "fmt" + "strings" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + "google.golang.org/protobuf/encoding/protojson" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/profile" + "github.com/lf-edge/eve/evetest" +) + +// LPS token used by every LPS test in this package. +const lpsServerToken = "server_token_123" + +// Shared management-API paths, reused by every LPS test in this package. +const ( + lpsLocalBaseURL = "http://localhost:8888" + lpsManageURL = lpsLocalBaseURL + "/manage/v1" + lpsManageTokenURL = lpsManageURL + "/token" + lpsManageProfileURL = lpsManageURL + "/profile" + lpsManageRadioStatusURL = lpsManageURL + "/radio-status" + lpsManageRadioConfigURL = lpsManageURL + "/radio-config" + lpsManageAppInfoURL = lpsManageURL + "/appinfo" + lpsManageAppCommandURL = lpsManageURL + "/app-command" + lpsManageDevInfoURL = lpsManageURL + "/devinfo" + lpsManageDevCommandURL = lpsManageURL + "/dev-command" + lpsManageNetConfigURL = lpsManageURL + "/network-config" +) + +// lpsAppAuth is the fixed SSH credential baked into the evetest-lps image. +var lpsAppAuth = evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", +} + +// newLPSAppConfig returns the ApplicationInstanceConfig for the evetest-lps +// container app, wired to niUUID with SSH (port sshPort) and management-API +// (port 8888) port-forwarding. The caller adds it to a devConfig and is +// responsible for calling ApplyConfig / WaitUntilAppIsRunning. +func newLPSAppConfig(displayName string, niUUID uuid.UUID, sshPort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-lps", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: sshPort, + AppPort: 22, + }, + { + // For developers troubleshooting LPS who need access to the + // UI: pause the test once the app is running, then run + // `evetest eve portfwd 8888:8888` and open + // http://localhost:8888 in a browser. + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 8888, + AppPort: 8888, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + } +} + +// waitLPSAppReady waits for the LPS app to become reachable over SSH, +// configures the LPS server token via its management API, and returns the +// app's IP address (used to build LPSConfig.Address for SetLPS). +func waitLPSAppReady( + t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, token string) string { + sshTimeout := 20 * time.Second + polling := 5 * time.Second + timeout := 3 * time.Minute + log := evetest.Logger() + + log.Infof("Waiting for LPS app SSH to become reachable...") + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("hello")) + }, timeout, polling).Should(Succeed()) + + _, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + fmt.Sprintf(`curl -sS -X PUT -d '{"token":"%s"}' `+lpsManageTokenURL, token), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + "hostname -I | awk '{print $1}'", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + ip := strings.TrimSpace(output) + log.Infof("LPS app IP: %s", ip) + return ip +} + +// runInLPSApp runs a shell command inside the LPS app over SSH and fails the +// assertion (via t) on any transport-level error. +func runInLPSApp(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, script string) string { + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + script, 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + return output +} + +// getLPSRadioStatus retrieves and parses the radio status that EVE posted +// to the LPS (GET /manage/v1/radio-status). +func getLPSRadioStatus( + t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.RadioStatus { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageRadioStatusURL) + var status profile.RadioStatus + t.Expect(protojson.Unmarshal([]byte(output), &status)).To(Succeed()) + return &status +} + +// getLPSAppInfo retrieves and parses the app info list that EVE posted to +// the LPS (GET /manage/v1/appinfo). +func getLPSAppInfo(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.LocalAppInfoList { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageAppInfoURL) + var list profile.LocalAppInfoList + t.Expect(protojson.Unmarshal([]byte(output), &list)).To(Succeed()) + return &list +} + +// getLPSDevInfo retrieves and parses the device info that EVE posted to the +// LPS (GET /manage/v1/devinfo). +func getLPSDevInfo(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.LocalDevInfo { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageDevInfoURL) + var info profile.LocalDevInfo + t.Expect(protojson.Unmarshal([]byte(output), &info)).To(Succeed()) + return &info +} + +// getLPSNetworkInfo retrieves and parses the network info that EVE posted to the LPS. +func getLPSNetworkInfo(t Gomega, device *evetest.EdgeDevice, + appUUID uuid.UUID) *profile.NetworkInfo { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageURL+"/network") + var netInfo profile.NetworkInfo + t.Expect(protojson.Unmarshal([]byte(output), &netInfo)).To(Succeed()) + return &netInfo +} + +// putLPSProfile sets the local profile the LPS reports back to EVE +// (PUT /manage/v1/profile). +func putLPSProfile(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, profileName string) { + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '{"profile":"%s"}' `+lpsManageProfileURL, profileName)) +} + +// putLPSRadioSilence sets the radio-silence config the LPS reports back to +// EVE (PUT /manage/v1/radio-config). +func putLPSRadioSilence(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, silence bool) { + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '{"radioSilence":%t}' `+lpsManageRadioConfigURL, silence)) +} + +// putLPSAppCommand submits a single-element AppCommand list via +// PUT /manage/v1/app-command (the endpoint always takes a JSON array). +// command must be one of the org.lfedge.eve.profile.AppCommand_Command +// symbolic names (e.g. "COMMAND_PURGE", "COMMAND_RESTART"). +func putLPSAppCommand(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + appDisplayName string, timestamp uint64, command string) { + body := fmt.Sprintf(`[{"displayname":"%s","timestamp":%d,"command":"%s"}]`, + appDisplayName, timestamp, command) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' `+lpsManageAppCommandURL, + body)) +} + +// putLPSDevCommand submits a device command via PUT /manage/v1/dev-command. +// command must be one of the org.lfedge.eve.profile.LocalDevCmd_Command +// symbolic names (e.g. "COMMAND_SHUTDOWN", "COMMAND_GRACEFUL_POWEROFF"). +func putLPSDevCommand(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + timestamp uint64, command string) { + body := fmt.Sprintf(`{"timestamp":%d,"command":"%s"}`, timestamp, command) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '%s' `+lpsManageDevCommandURL, body)) +} + +// putLPSNetworkConfig sets the local network config the LPS reports back to +// EVE (PUT /manage/v1/network-config). portsJSON is the raw JSON array for +// the "ports" field (e.g. "[]" to revert to the controller-supplied config). +func putLPSNetworkConfig(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, portsJSON string) { + body := fmt.Sprintf(`{"ports":%s}`, portsJSON) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' `+lpsManageNetConfigURL, + body)) +} + +// appInfoByName returns the LocalAppInfo entry matching the given +// displayname, or nil if the LPS has no entry for it yet. +func appInfoByName(list *profile.LocalAppInfoList, name string) *profile.LocalAppInfo { + for _, info := range list.GetAppsInfo() { + if info.GetName() == name { + return info + } + } + return nil +} + +// portStatusByLabel returns the NetworkPortStatus entry matching the given +// logical label. Fails the assertion if no such entry is present in the +// NetworkInfo published by EVE. +func portStatusByLabel(t Gomega, netInfo *profile.NetworkInfo, + label string) *profile.NetworkPortStatus { + for _, ps := range netInfo.PortStatus { + if ps.LogicalLabel == label { + return ps + } + } + t.Expect(netInfo.PortStatus).To(ContainElement( + HaveField("LogicalLabel", label)), + "NetworkInfo.PortStatus should include "+label) + return nil +} diff --git a/evetest/tests/lps/network_test.go b/evetest/tests/lps/network_test.go index 07ea199a8bb..60da64b15e4 100644 --- a/evetest/tests/lps/network_test.go +++ b/evetest/tests/lps/network_test.go @@ -12,22 +12,11 @@ import ( // revive:disable:dot-imports . "github.com/onsi/gomega" - uuid "github.com/satori/go.uuid" - "google.golang.org/protobuf/encoding/protojson" - "github.com/lf-edge/eve-api/go/evecommon" - "github.com/lf-edge/eve-api/go/profile" "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/netmodels" ) -const ( - lpsServerToken = "evetest-lps-token" - lpsLocalBaseURL = "http://localhost:8888" - lpsManageURL = lpsLocalBaseURL + "/manage/v1" - lpsManageNetConfigURL = lpsManageURL + "/network-config" -) - // TestNetworkLocalChanges verifies that EVE's Local Profile Server (LPS) // integration honors the per-port AllowLocalModifications flag when an // app deployed on the device submits network-config overrides via the @@ -54,27 +43,25 @@ const ( // - SystemAdapter for eth1 (DHCP, mgmt+app) with // AllowLocalModifications=true. // - Local NI "local-ni" (10.11.12.0/24, MTU=1500) on eth0. -// - LPS application "lps-app" (lfedge/evetest-lps:1.0) on the NI -// with two port-fwd rules: -// - 2222->22 for the test framework to drive curl-against-LPS commands -// via SSH inside the app, -// - 8888->8888 to let a developer expose the LPS UI through +// - LPS application "lps-app" (lfedge/evetest-lps:1.0, see +// newLPSAppConfig in helpers_test.go) on the NI, with SSH port-fwd +// 2222->22 for the test framework to drive curl-against-LPS commands, +// and 8888->8888 to let a developer expose the LPS UI through // `evetest eve portfwd 8888:8888` while a checkpoint is paused. -// - After the LPS app is reachable over SSH, the test configures the LPS -// server token via the LPS management API, reads the app's IP, and -// pushes evetest.LPSConfig{Address: :8888, AuthToken: token} -// into the device config so EVE actually talks to the LPS. +// - waitLPSAppReady (helpers_test.go) waits for the LPS app to become +// reachable over SSH, configures the LPS server token via the LPS +// management API, and returns the app's IP; the test then pushes +// evetest.LPSConfig{Address: :8888, AuthToken: token} into the +// device config so EVE actually talks to the LPS. // // Phases / assertions // ------------------- // 1. setup-done -> initial-config-applied -> lps-app-is-running: // the LPS container is up. -// 2. lps-app-ssh-reachable: the framework can SSH into the app over the -// port-fwd; a hello probe succeeds. -// 3. lps-configured -> lps-receiving-network-info: EVE picks up the LPS +// 2. lps-configured -> lps-receiving-network-info: EVE picks up the LPS // config and starts posting NetworkInfo (HTTP 200 on // /manage/v1/network). -// 4. Submit a localNetworkConfig via the LPS management API that +// 3. Submit a localNetworkConfig via the LPS management API that // overrides DNS for eth0 (dns-server0-alt, 10.16.18.25) and MTU for // eth1 (9000). Assert via `Eventually` (configChangeTimeout): // - NetworkInfo.LocalConfig.Ports has entries for both adapters. @@ -87,13 +74,13 @@ const ( // - Runtime PortStatus for eth1: Mtu=9000. // - On EVE itself: /run/nim/dnsmasq.mgmt.servers does NOT contain // 10.16.18.25, /sys/class/net/eth1/mtu == "9000". -// 5. Enable AllowLocalModifications=true on eth0 via UpdateNetworkAdapter +// 4. Enable AllowLocalModifications=true on eth0 via UpdateNetworkAdapter // and re-ApplyConfig. Assert: // - LocalConfig.Ports[eth0]: no "not permitted" error, // ConfigApplied=true. // - PortStatus for eth0: DNS now includes 10.16.18.25. // - On EVE: /run/nim/dnsmasq.mgmt.servers now contains 10.16.18.25. -// 6. Push an empty config via the LPS management API +// 5. Push an empty config via the LPS management API // ({"serverToken":..., "ports":[]}). Assert that both ports revert // to the controller-supplied config: // - LatestConfig.ConfigApplied=true for both ports; eth1.Mtu is @@ -103,31 +90,25 @@ const ( // - On EVE: dnsmasq.mgmt.servers no longer contains 10.16.18.25; // /sys/class/net/eth1/mtu == "1500". // -// Helpers used -// ------------ -// - getLPSNetworkInfo (defined below): curls /manage/v1/network from -// inside the LPS app and unmarshals the protobuf-json into a -// profile.NetworkInfo. -// - portStatusByLabel (defined below): walks NetworkInfo.PortStatus -// looking up a port by its LogicalLabel, with a failing assertion -// if not found. -// // Hypervisor / suite placement // ---------------------------- -// - Hardcoded HypervisorKVM. The TODO in TestLPSSuite notes that the -// HypervisorParameter will be added once additional LPS tests exist -// that depend on app virtualization. +// - HYPERVISOR (defaults to KVM). func TestNetworkLocalChanges(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -173,84 +154,25 @@ func TestNetworkLocalChanges(test *testing.T) { Gateway: evetest.IPAddress("10.11.12.1"), MTU: 1500, }) - appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ - DisplayName: "lps-app", - Activate: true, - Image: evetest.DockerContainer{ - ImageName: "lfedge/evetest-lps", - Tag: "1.0", - }, - CPUs: 1, - MemoryBytes: 512 * evetest.MiB, - NetworkAdapters: []evetest.AppNetworkAdapter{ - evetest.VirtualNetworkAdapter{ - LogicalLabel: "vif0", - NetworkInstanceUUID: niUUID, - PortFwdRules: []evetest.PortFwdRule{ - { - // SSH access - Protocol: evetest.NetworkProtocolTCP, - EdgeNodePort: 2222, - AppPort: 22, - }, - { - // For developers troubleshooting LPS who need access to the UI: - // Pause test after LPS is deployed (at checkpoint - // "lps-app-is-running" or later), then run: - // $ evetest eve portfwd 8888:8888 - // And open http://localhost:8888 in your browser. - Protocol: evetest.NetworkProtocolTCP, - EdgeNodePort: 8888, - AppPort: 8888, - }, - }, - ACLAllowRules: []evetest.ACLAllowRule{ - { - Protocol: evetest.NetworkProtocolAny, - RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), - }, - }, - }, - }, - }) + appUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) device := evetest.GetEdgeDevice(devName) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("initial-config-applied") device.WaitUntilAppIsRunning(appUUID, 10*time.Minute) evetest.Checkpoint("lps-app-is-running") - // Wait for the LPS app to become reachable via SSH. - appAuth := evetest.UsernamePasswordAuth{ - Username: "root", - Password: "testpassword", - } log := evetest.Logger() sshTimeout := 20 * time.Second polling := 5 * time.Second - timeout := 3 * time.Minute - log.Infof("Waiting for LPS app SSH to become reachable...") - t.Eventually(func(t Gomega) { - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "echo hello", sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - t.Expect(output).To(ContainSubstring("hello")) - }, timeout, polling).Should(Succeed()) - evetest.Checkpoint("lps-app-ssh-reachable") - - // Configure the server token via the LPS management API. - _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -d '{"token":"%s"}' `+lpsManageURL+`/token`, - lpsServerToken), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + var output string + var err error - // Get the application's IP (LPS is reachable at this IP from EVE). - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "hostname -I | awk '{print $1}'", sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - lpsIP := strings.TrimSpace(output) - log.Infof("LPS app IP: %s", lpsIP) + lpsIP := waitLPSAppReady(t, device, appUUID, lpsServerToken) // Configure EVE to use the LPS. devConfig.SetLPS(evetest.LPSConfig{ @@ -264,35 +186,26 @@ func TestNetworkLocalChanges(test *testing.T) { configChangeTimeout := 2 * time.Minute log.Infof("Waiting for LPS to receive network info from EVE...") t.Eventually(func(t Gomega) { - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "curl -sS -o /dev/null -w '%{http_code}' "+lpsManageURL+"/network", - sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + output := runInLPSApp(t, device, appUUID, + "curl -sS -o /dev/null -w '%{http_code}' "+lpsManageURL+"/network") t.Expect(output).To(Equal("200")) }, configChangeTimeout, polling).Should(Succeed()) evetest.Checkpoint("lps-receiving-network-info") // Apply local config: DNS override for eth0, MTU override for eth1. - localNetworkConfig := fmt.Sprintf(`{ - "serverToken": "%s", - "ports": [ - { - "logicalLabel": "ethernet0", - "useDhcp": true, - "dnsServers": ["10.16.18.25"] - }, - { - "logicalLabel": "ethernet1", - "useDhcp": true, - "mtu": 9000 - } - ] - }`, lpsServerToken) log.Infof("Submitting local network config via LPS management API") - _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' %s`, - localNetworkConfig, lpsManageNetConfigURL), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + putLPSNetworkConfig(t, device, appUUID, `[ + { + "logicalLabel": "ethernet0", + "useDhcp": true, + "dnsServers": ["10.16.18.25"] + }, + { + "logicalLabel": "ethernet1", + "useDhcp": true, + "mtu": 9000 + } + ]`) evetest.Checkpoint("local-config-submitted") // Verify eth0 changes are rejected, eth1 changes are applied. @@ -300,7 +213,7 @@ func TestNetworkLocalChanges(test *testing.T) { // for eth1 was applied (MTU=9000) and eth0 was rejected (not permitted). log.Infof("Verifying eth1 local changes are applied and eth0 is rejected...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) t.Expect(netInfo.LocalConfig).ToNot(BeNil()) for _, port := range netInfo.LocalConfig.Ports { switch port.LogicalLabel { @@ -367,7 +280,7 @@ func TestNetworkLocalChanges(test *testing.T) { // Verify eth0 changes are now applied log.Infof("Verifying eth0 local changes are now applied...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) t.Expect(netInfo.LocalConfig).ToNot(BeNil()) for _, port := range netInfo.LocalConfig.Ports { if port.LogicalLabel == "ethernet0" { @@ -395,20 +308,13 @@ func TestNetworkLocalChanges(test *testing.T) { // Revert local changes by submitting empty config log.Infof("Reverting local network config by submitting empty config...") - emptyConfig := fmt.Sprintf(`{ - "serverToken": "%s", - "ports": [] - }`, lpsServerToken) - _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' %s`, - emptyConfig, lpsManageNetConfigURL), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + putLPSNetworkConfig(t, device, appUUID, "[]") evetest.Checkpoint("local-changes-reverted") // Verify both ports revert to controller config log.Infof("Verifying both ports reverted to controller config...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) // After submitting empty config, LocalConfig should have no ports // or all ports should show controller config applied. for _, port := range netInfo.LatestConfig { @@ -447,32 +353,3 @@ func TestNetworkLocalChanges(test *testing.T) { "eth1 MTU should have reverted to 1500") }, configChangeTimeout, polling).Should(Succeed()) } - -// portStatusByLabel returns the NetworkPortStatus entry matching the given -// logical label. Fails the assertion if no such entry is present in the -// NetworkInfo published by EVE. -func portStatusByLabel(t Gomega, netInfo *profile.NetworkInfo, - label string) *profile.NetworkPortStatus { - for _, ps := range netInfo.PortStatus { - if ps.LogicalLabel == label { - return ps - } - } - t.Expect(netInfo.PortStatus).To(ContainElement( - HaveField("LogicalLabel", label)), - "NetworkInfo.PortStatus should include "+label) - return nil -} - -// getLPSNetworkInfo retrieves and parses the network info that EVE posted to the LPS. -func getLPSNetworkInfo(t Gomega, device *evetest.EdgeDevice, - appUUID uuid.UUID, auth evetest.AuthMethod, - timeout time.Duration) *profile.NetworkInfo { - output, _, err := device.RunShellScriptInsideApp(appUUID, auth, - "curl -sS "+lpsManageURL+"/network", timeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - var netInfo profile.NetworkInfo - err = protojson.Unmarshal([]byte(output), &netInfo) - t.Expect(err).ToNot(HaveOccurred()) - return &netInfo -} diff --git a/evetest/tests/lps/profile_test.go b/evetest/tests/lps/profile_test.go new file mode 100644 index 00000000000..9562a81150b --- /dev/null +++ b/evetest/tests/lps/profile_test.go @@ -0,0 +1,277 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestProfile verifies that EVE filters which applications run based on the +// currently active profile -- both a device-config-only GlobalProfile +// override and a Local Profile Server (LPS)-reported local profile -- by +// matching it against each app's ProfileList (an app with an empty +// ProfileList always runs, regardless of the active profile). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port, used only for +// controller reachability and the LPS app's Local NI. The three +// ProfileList apps under test have no network adapter of their own +// (only their RUNNING/HALTED state matters here). +// +// Phases +// ------ +// 1. Deploy four apps on Local NI "local-ni": "lps-app" (the evetest-lps +// app, ProfileList empty --> always runs), "app-profile-1" +// (ProfileList=[profile-1]), "app-profile-2" (ProfileList=[profile-2]), +// "app-profile-1-2" (ProfileList=[profile-1, profile-2]). +// WaitUntilAppIsRunning for all four -- with no profile active yet +// (GlobalProfile and LocalProfileServer both empty), every app runs. +// 2. GlobalProfile-only phase (no LPS involved yet): SetLPS with only +// GlobalProfile set, re-applying for "profile-1", "profile-2" and +// "profile-3" in turn. After each apply, assert exactly the apps whose +// ProfileList does not contain the active profile are HALTED and the +// rest (including lps-app, always) are RUNNING. +// 3. Manual activate/deactivate sanity check: with GlobalProfile still +// "profile-3" (so app-profile-* are all HALTED), DeactivateApplication +// then ActivateApplication lps-app (whose empty ProfileList makes it +// immune to profile filtering) to confirm ordinary controller-driven +// activation still works independently of the profile mechanism. +// 4. LPS-driven phase: configure the LPS address/token via SetLPS +// (GlobalProfile is deliberately left at "profile-3" -- this also +// proves that once a LocalProfileServer is configured, EVE is driven +// entirely by whatever profile the LPS reports, not by GlobalProfile). +// Submit "profile-1", "profile-2", "profile-3" via PUT /manage/v1/profile in +// turn, asserting the same HALTED/RUNNING pattern as phase 2 -- but +// this time driven by the LPS instead of the static device config. +// 5. Revert: SetLPS with all fields empty (clears GlobalProfile, +// LocalProfileServer and ProfileServerToken). Assert all four apps +// return to RUNNING. +// 6. Cleanup: delete all four apps and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestProfile(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app plus three plain apps distinguished only + // by their ProfileList. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-1"}, + }) + app2UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-2", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-2"}, + }) + app12UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-1-2", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-1", "profile-2"}, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app12UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running-no-profile") + + log := evetest.Logger() + timeout := 10 * time.Minute + polling := 3 * time.Second + + // waitAppState polls EdgeDevice.GetAppInfo -- a synchronous snapshot of + // the app's latest known state -- until it reports the expected SwState. + // Used for HALTED/INVALID waits below (WaitUntilAppIsRunning only + // targets RUNNING); a WatchAppInfo channel wouldn't work here either: + // several apps have overlapping ProfileLists (e.g. app-profile-1-2 + // matches both "profile-1" and "profile-2"), so switching between two + // profiles it both matches causes no state transition and thus no new + // channel event at all. + waitAppState := func(appUUID uuid.UUID, desc string, expected eveinfo.ZSwState) { + log.Infof("Waiting for: %s...", desc) + t.Eventually(func() eveinfo.ZSwState { + if info := device.GetAppInfo(appUUID); info != nil { + return info.GetState() + } + return eveinfo.ZSwState_INVALID + }, timeout, polling).Should(Equal(expected), desc) + } + + // Step 2: GlobalProfile-only phase (no LPS configured yet). + log.Infof("Setting GlobalProfile=profile-1") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-1"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app2UUID, "app-profile-2 is HALTED (profile-1 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("global-profile-1") + + log.Infof("Setting GlobalProfile=profile-2") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-2"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app1UUID, "app-profile-1 is HALTED (profile-2 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("global-profile-2") + + log.Infof("Setting GlobalProfile=profile-3") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-3"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app1UUID, "app-profile-1 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + waitAppState(app2UUID, "app-profile-2 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + waitAppState(app12UUID, "app-profile-1-2 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + evetest.Checkpoint("global-profile-3") + + // Step 3: manual activate/deactivate still works independently of the + // profile mechanism (lps-app has an empty ProfileList, so it is immune + // to the currently-active "profile-3"). + log.Infof("Manually deactivating and reactivating lps-app") + device.DeactivateApplication(lpsAppUUID, true, timeout) + device.ActivateApplication(lpsAppUUID, true, timeout) + evetest.Checkpoint("lps-app-manually-cycled") + + // Step 4: LPS-driven phase. GlobalProfile is deliberately left at + // "profile-3" -- configuring a LocalProfileServer must make EVE defer + // entirely to whatever profile the LPS reports, ignoring GlobalProfile. + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + GlobalProfile: "profile-3", + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log.Infof("Submitting local profile 'profile-1' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-1") + waitAppState(app2UUID, "app-profile-2 is HALTED (LPS profile-1)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("lps-profile-1") + + log.Infof("Submitting local profile 'profile-2' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-2") + waitAppState(app1UUID, "app-profile-1 is HALTED (LPS profile-2)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("lps-profile-2") + + log.Infof("Submitting local profile 'profile-3' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-3") + waitAppState(app1UUID, "app-profile-1 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + waitAppState(app2UUID, "app-profile-2 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + waitAppState(app12UUID, "app-profile-1-2 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + evetest.Checkpoint("lps-profile-3") + + // Step 5: revert to empty profiles -- all apps come back RUNNING. + log.Infof("Reverting to empty GlobalProfile/LocalProfileServer") + devConfig.SetLPS(evetest.LPSConfig{}) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("profiles-cleared") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteApplication(app12UUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + waitAppState(lpsAppUUID, "lps-app is gone", eveinfo.ZSwState_INVALID) + waitAppState(app1UUID, "app-profile-1 is gone", eveinfo.ZSwState_INVALID) + waitAppState(app2UUID, "app-profile-2 is gone", eveinfo.ZSwState_INVALID) + waitAppState(app12UUID, "app-profile-1-2 is gone", eveinfo.ZSwState_INVALID) +} diff --git a/evetest/tests/lps/radio_silence_test.go b/evetest/tests/lps/radio_silence_test.go new file mode 100644 index 00000000000..31108af88ed --- /dev/null +++ b/evetest/tests/lps/radio_silence_test.go @@ -0,0 +1,193 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestRadioSilence verifies EVE's radio-silence message exchange with the +// Local Profile Server (LPS): the LPS can request radio silence be imposed +// or lifted, EVE reports the resulting state back to both the LPS and its +// own internal ZedAgentStatus, and the imposed state survives a device +// reboot. +// +// Note: none of evetest's device models currently define a wireless (cellular/WiFi) +// network adapter, so this only exercises the message-passing and persistence +// paths between the LPS, zedagent and nim -- not an actual radio transmitter being +// switched off. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port, used for controller +// reachability and the LPS app's Local NI. +// +// Phases +// ------ +// 1. Deploy the evetest-lps app on Local NI "local-ni". WaitUntilAppIsRunning, +// then configure EVE to use it via SetLPS. +// 2. Confirm the initial radio status is unsilenced, per both the LPS +// (GET /manage/v1/radio-status) and EVE's own ZedAgentStatus. +// 3. Toggle radio silence ON (PUT /manage/v1/radio-config +// {"radioSilence":true}). Assert both: +// - the LPS's radio-status eventually reports RadioSilence=true, and +// - EVE's own /run/zedagent/ZedAgentStatus/zedagent.json (read via +// `eve exec pillar jq ...` over SSH) reports RadioSilence.Imposed=true. +// 4. Toggle radio silence OFF and assert both views report false again. +// 5. Toggle radio silence back ON (assert both views report true again), +// then reboot the device via RequestReboot and wait for it to come back online. +// 6. After the reboot, assert -- reading EVE's own ZedAgentStatus only, +// deliberately without any further LPS interaction -- that +// RadioSilence.Imposed is still true. zedagent persists the +// last-applied radio-silence state across reboots and re-publishes it +// on startup. +// 7. Cleanup: delete the app and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestRadioSilence(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + device.WaitUntilAppIsRunning(lpsAppUUID, 5*time.Minute) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 3 * time.Minute + polling := 3 * time.Second + + // waitRadioSilence asserts that both the LPS-reported radio status and + // EVE's own ZedAgentStatus agree on the given RadioSilence value. + waitRadioSilence := func(want bool, desc string) { + log.Infof("Waiting for radio-silence=%t (%s)", want, desc) + t.Eventually(func(t Gomega) { + status := getLPSRadioStatus(t, device, lpsAppUUID) + t.Expect(status.RadioSilence).To(Equal(want), "LPS-reported radio status") + t.Expect(getEVERadioSilenceImposed(t, device)).To(Equal(want), + "EVE-side ZedAgentStatus.RadioSilence.Imposed") + }, timeout, polling).Should(Succeed()) + } + + // Step 2: confirm the initial (unsilenced) radio status. + waitRadioSilence(false, "initial state") + evetest.Checkpoint("initial-radio-status-confirmed") + + // Step 3: toggle radio silence ON. + putLPSRadioSilence(t, device, lpsAppUUID, true) + waitRadioSilence(true, "toggled ON") + evetest.Checkpoint("radio-silence-on") + + // Step 4: toggle radio silence OFF. + putLPSRadioSilence(t, device, lpsAppUUID, false) + waitRadioSilence(false, "toggled OFF") + evetest.Checkpoint("radio-silence-off") + + // Step 5: toggle back ON, then reboot. + putLPSRadioSilence(t, device, lpsAppUUID, true) + waitRadioSilence(true, "toggled ON again, pre-reboot") + evetest.Checkpoint("radio-silence-on-pre-reboot") + + log.Infof("Rebooting the device via the controller") + device.RequestReboot(true) + evetest.Checkpoint("device-rebooted") + + // Step 6: persistence -- EVE's own state only, no LPS interaction. + log.Infof("Confirming radio-silence=true persisted across the reboot " + + "(EVE-side check only)") + t.Eventually(func(t Gomega) { + t.Expect(getEVERadioSilenceImposed(t, device)).To(BeTrue(), + "EVE should still report RadioSilence.Imposed=true after reboot") + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("radio-silence-persisted") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(lpsAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "lps-app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopLPSAppWatch() +} + +// getEVERadioSilenceImposed reads RadioSilence.Imposed directly from EVE's +// own /run/zedagent/ZedAgentStatus/zedagent.json, bypassing the LPS +// entirely. Used to confirm EVE's internal state independent of whether the +// LPS app has (re)connected. +func getEVERadioSilenceImposed(t Gomega, device *evetest.EdgeDevice) bool { + output, _, err := device.RunShellScript( + `eve exec pillar jq -r '.RadioSilence.Imposed' /run/zedagent/ZedAgentStatus/zedagent.json`, + 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + return strings.TrimSpace(output) == "true" +} diff --git a/evetest/tests/lps/testsuite_test.go b/evetest/tests/lps/testsuite_test.go index 282a09f1a0c..28e5c19c8a8 100644 --- a/evetest/tests/lps/testsuite_test.go +++ b/evetest/tests/lps/testsuite_test.go @@ -10,21 +10,44 @@ import ( ) // TestLPSSuite is the entry point for Local Profile Server (LPS) tests. -// Currently it contains only TestNetworkLocalChanges. The TODO above -// indicates we will enable HypervisorParameter once the suite grows to -// include app-related LPS scenarios that depend on hypervisor choice; -// for now the single subtest hardcodes its own hypervisor. +// See https://github.com/lf-edge/eve-api/blob/main/PROFILE.md +// +// Subtests +// -------- +// - TestProfile -- device-level (GlobalProfile) and LPS-driven +// (local profile) app filtering by ProfileList. +// - TestRadioSilence -- LPS-driven radio-silence toggling, verified via +// both the LPS and EVE's own ZedAgentStatus, including persistence of +// the imposed state across a controller-driven reboot. +// - TestAppLocalInfo -- LPS appinfo reporting and LPS-driven +// COMMAND_PURGE/COMMAND_RESTART, alongside the equivalent +// controller-driven operations (whose counters EVE sums with the +// LPS-driven ones). +// - TestDevLocalInfo -- LPS devinfo reporting and LPS-driven +// COMMAND_SHUTDOWN/COMMAND_GRACEFUL_POWEROFF device commands. +// - TestNetworkLocalChanges -- per-port AllowLocalModifications / +// LPS network-config override behavior. func TestLPSSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() - /* TODO: re-enable if there are any app-related LPS tests evetest.DefineTestParameters( evetest.HypervisorParameter(), ) - */ evetest.RunTestSuite( + evetest.TestCase{ + Test: TestProfile, + }, + evetest.TestCase{ + Test: TestRadioSilence, + }, + evetest.TestCase{ + Test: TestAppLocalInfo, + }, + evetest.TestCase{ + Test: TestDevLocalInfo, + }, evetest.TestCase{ Test: TestNetworkLocalChanges, }, diff --git a/evetest/tests/networking/acls_test.go b/evetest/tests/networking/acls_test.go index 43f75878098..90db2e4c994 100644 --- a/evetest/tests/networking/acls_test.go +++ b/evetest/tests/networking/acls_test.go @@ -75,8 +75,7 @@ const enableFlowlogParamKey = "ENABLE_FLOWLOG" // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() -- Kubevirt -// is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // - ENABLE_FLOWLOG: enable flow logging on the Local NI (default: false). func TestLocalNetInstanceACLs(test *testing.T) { evetestT := evetest.Init(test) @@ -96,7 +95,6 @@ func TestLocalNetInstanceACLs(test *testing.T) { ) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() enableFlowlog := evetest.GetTestParameter[bool](enableFlowlogParamKey) devName := "edge-dev" @@ -125,6 +123,9 @@ func TestLocalNetInstanceACLs(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } const ( niGateway = "10.11.12.1" @@ -490,8 +491,7 @@ func TestLocalNetInstanceACLs(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() -- Kubevirt is -// reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // - ENABLE_FLOWLOG: enable flow logging on the Switch NI (default: false). func TestSwitchNetInstanceACLs(test *testing.T) { evetestT := evetest.Init(test) @@ -511,7 +511,6 @@ func TestSwitchNetInstanceACLs(test *testing.T) { ) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() enableFlowlog := evetest.GetTestParameter[bool](enableFlowlogParamKey) devName := "edge-dev" @@ -540,6 +539,9 @@ func TestSwitchNetInstanceACLs(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } const ( app1MACAddr = "02:16:3e:00:00:03" diff --git a/evetest/tests/networking/bonds_test.go b/evetest/tests/networking/bonds_test.go index f87785b92a4..21d837c4fb2 100644 --- a/evetest/tests/networking/bonds_test.go +++ b/evetest/tests/networking/bonds_test.go @@ -70,20 +70,23 @@ import ( // failed member. // - Restore the SDN model (link back up). // -// Hypervisor -// ---------- -// - Hardcoded WithHypervisor=HypervisorKVM in RequireEdgeDevice -- this -// test lives in TestDeviceConnectivitySuite and does not parameterize -// the hypervisor. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestActiveBackupBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } // Active-backup bond is transparent to the network switch -- only one @@ -143,6 +146,9 @@ func TestActiveBackupBond(test *testing.T) { devMetrics, stopDevMetricsWatch := device.WatchDeviceMetrics() defer stopDevMetricsWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Wait for device info to report the bond interface with an IP address, @@ -339,20 +345,23 @@ func TestActiveBackupBond(test *testing.T) { // ethernet0 and ethernet1 in its Members list, and every member has // LACP sub-metrics populated. // -// Hypervisor -// ---------- -// - Hardcoded WithHypervisor=HypervisorKVM in RequireEdgeDevice -- this -// test lives in TestDeviceConnectivitySuite and does not parameterize -// the hypervisor. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestLACPBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } // Start with individual ports on a bridge so that EVE can onboard @@ -413,6 +422,9 @@ func TestLACPBond(test *testing.T) { // waitUntilConfirmed=false: after the model switch below, EVE may temporarily // lose controller connectivity while the LACP bond negotiates. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to process the bond config before switching the SDN side. time.Sleep(10 * time.Second) evetest.Checkpoint("config-applied") diff --git a/evetest/tests/networking/bootstrap_test.go b/evetest/tests/networking/bootstrap_test.go index 2bde79fe4bf..a72347478a3 100644 --- a/evetest/tests/networking/bootstrap_test.go +++ b/evetest/tests/networking/bootstrap_test.go @@ -49,14 +49,15 @@ var ( ) func deviceRequirementsForBootstrap( - devName string, useInstaller bool) evetest.RequireEdgeDevice { + devName string, useInstaller bool, + hypervisor evetest.Hypervisor) evetest.RequireEdgeDevice { reusePolicy := evetest.CreateFromScratchWithLiveImage if useInstaller { reusePolicy = evetest.CreateFromScratchWithInstaller } return evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, MinCPUs: 4, WithGrubOptions: []string{ // No applications are deployed in network bootstrapping tests. @@ -119,7 +120,7 @@ func deviceRequirementsForBootstrap( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test; not parameterized). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithLastResort(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -127,17 +128,19 @@ func TestBootstrapWithLastResort(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), lastResortParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() lastResortExplicitlyEnabled := evetest.GetTestParameter[bool](lastResortParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCP, } @@ -171,6 +174,9 @@ func TestBootstrapWithLastResort(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Wait for device info to report the expected DevicePortStatus list. @@ -251,7 +257,7 @@ var ( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test; not parameterized). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithStaticIP(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -259,11 +265,13 @@ func TestBootstrapWithStaticIP(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -290,7 +298,7 @@ func TestBootstrapWithStaticIP(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -328,6 +336,9 @@ func TestBootstrapWithStaticIP(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -460,7 +471,7 @@ var ( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithProxy(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -468,12 +479,14 @@ func TestBootstrapWithProxy(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, proxyConfigTypeParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) proxyConfigType := evetest.GetTestParameter[ProxyConfigType](proxyConfigTypeParamKey) @@ -526,7 +539,7 @@ func TestBootstrapWithProxy(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { var proxyConfig pillartypes.ProxyConfig switch proxyConfigType { @@ -614,6 +627,9 @@ func TestBootstrapWithProxy(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -672,7 +688,7 @@ func TestBootstrapWithProxy(test *testing.T) { // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithMgmtVLAN(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -680,11 +696,13 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -712,7 +730,7 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -761,6 +779,9 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -818,7 +839,7 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithLACPBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -826,11 +847,13 @@ func TestBootstrapWithLACPBond(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -874,7 +897,7 @@ func TestBootstrapWithLACPBond(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -940,6 +963,9 @@ func TestBootstrapWithLACPBond(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after diff --git a/evetest/tests/networking/dns_test.go b/evetest/tests/networking/dns_test.go index 112d887a3a8..4a400955506 100644 --- a/evetest/tests/networking/dns_test.go +++ b/evetest/tests/networking/dns_test.go @@ -12,6 +12,7 @@ import ( // revive:disable:dot-imports . "github.com/onsi/gomega" + eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" eveinfo "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/evetest" @@ -101,6 +102,10 @@ import ( // - nslookup http-server2.test: succeeds (ethernet2's exclusive DNS // server knows it), confirming dnsmasq rebuilt its upstream list after // the uplink change. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDNSFunctionality(test *testing.T) { // DNS server IPs and FQDNs as defined in netmodels.ManyDNSServers. const ( @@ -127,11 +132,16 @@ func TestDNSFunctionality(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify test prerequisites. devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } requiredNetModel := evetest.RequireNetworkModel{ @@ -217,6 +227,9 @@ func TestDNSFunctionality(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // ------------------------------------------------------------------ @@ -486,8 +499,9 @@ func TestDNSFunctionality(test *testing.T) { ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0", }, - CPUs: 1, - MemoryBytes: 500 * evetest.MiB, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, NetworkAdapters: []evetest.AppNetworkAdapter{ evetest.VirtualNetworkAdapter{ LogicalLabel: "vif0", diff --git a/evetest/tests/networking/failover_test.go b/evetest/tests/networking/failover_test.go index 4e005a329db..7b036dd7602 100644 --- a/evetest/tests/networking/failover_test.go +++ b/evetest/tests/networking/failover_test.go @@ -4,6 +4,10 @@ package networking_test import ( + "fmt" + "net" + "regexp" + "strconv" "testing" "time" @@ -69,19 +73,22 @@ import ( // // Test params // ----------- -// - None. WithHypervisor=HypervisorKVM is hardcoded in RequireEdgeDevice -// because this test lives in TestDeviceConnectivitySuite and -// Device-suite tests do not parameterize the hypervisor. +// - HYPERVISOR (defaults to KVM). func TestPortFailover(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -122,6 +129,9 @@ func TestPortFailover(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("port-config-applied") // Local NI on "uplink" (predefined shared label matching every mgmt port). @@ -387,54 +397,56 @@ func TestPortFailover(test *testing.T) { // TestNetworkConfigFallback verifies that EVE rolls back to the previously // working DevicePortConfig (DPC) when a newly applied configuration cannot -// reach the controller, and that it re-applies the new config once the +// reach the controller at all, and that it re-adopts the new config once the // network actually matches it. // +// ethernet1 is configured once at the start and never touched again: it is +// what keeps the *first* DPC genuinely working (so there is something to +// fall back to), but it deliberately has no role in the broken/recovered DPC +// under test in phases 2-3, which only ever contains ethernet0. A DPC is +// only considered failed as a whole once none of its ports can reach the +// controller (EVE's connectivity test tries every management port in the +// active DPC and succeeds if any one of them works -- see +// ControllerConnectivityTester.TestConnectivity), so leaving a second, +// unrelated working port in the broken DPC would have masked the failure +// this test needs to trigger. +// // Network model -// - Start with netmodels.TwoMgmtPorts. The "second" port (eth1) is initially -// used as a backup with cost=10. +// ------------- +// - netmodels.TwoMgmtPorts -- two management ports. // // Device configuration -// - Initial config: SystemAdapter for eth0 (mgmt) DHCP, SystemAdapter for -// eth1 (mgmt) DHCP. Apply, wait until SystemAdapterInfo (in published -// device info) reports currentIndex=0 and exactly one DevicePortStatus -// entry with key="zedagent" -- same pattern as bootstrap_test.go uses -// via its matchSystemAdapterInfo helper. No raw pubsub readback is -// needed; the SystemAdapterInfo embedded in ZInfoDevice carries the -// full DPC list, the currentIndex pointer, and per-DPC lastError / -// lastFailed / lastSucceeded timestamps. -// -// Phase 1 — induce a broken-config rollback -// - Apply a NEW device config that intentionally does NOT match the SDN -// network (so it cannot reach the controller): -// -> Switch eth0 to StaticNetworkConfig with a wrong subnet/gateway -// (e.g., 10.99.99.0/24 / 10.99.99.1). -// - Wait for EVE to test the new config and fall back. All assertions -// read SystemAdapterInfo from WatchDeviceInfo: -// - SystemAdapterInfo.Status grows by one entry (the just-submitted DPC). -// The new DPC -- the one at index 0 by priority -- must have -// LastError set to a description mentioning the connectivity test -// failure, and LastFailed timestamp populated. -// - SystemAdapterInfo.CurrentIndex points at the OLDER (working) DPC, -// not the one we just submitted (i.e. CurrentIndex > 0). -// - The older DPC referenced by CurrentIndex must have LastSucceeded -// advancing (it is still working). -// - The device must REMAIN online — controller still receives info -// messages, RunShellScript still works. +// -------------------- +// - Baseline: SystemAdapter for eth0 (mgmt) DHCP, SystemAdapter for eth1 +// (mgmt) DHCP. timer.port.testduration is lowered to 10s (fast per-DPC +// connectivity test) and timer.port.testbetterinterval to 60s (fast +// retest of a higher-priority DPC once it might have become usable +// again); both via SetConfigProperties. // -// Phase 2 — recovery -// - UpdateNetworkModel (or update SDN router config) to make the network -// actually match the broken config. For variant (a), change the SDN -// network's subnet/gateway from 172.20.20.0/24 to 10.99.99.0/24 -// (clone netmodels.TwoMgmtPorts and rewrite Networks[0].Ipv4 — note -// evetest.UpdateNetworkModel allows changing subnets but not the set -// of ports). -// - EVE periodically retests higher-priority DPCs (timer.port.testbetterinterval, -// default 10 min — set it lower via SetConfigProperties for the test, -// e.g. 60 s). Watching SystemAdapterInfo, eventually: -// - CurrentIndex returns to 0 (the latest DPC works again). -// - DevicePortStatus[0].LastSucceeded advances to a timestamp newer than -// the recovery moment, and LastError is cleared. +// Phases +// ------ +// 1. Baseline: WatchDeviceInfo until SystemAdapterInfo reports currentIndex=0 +// with exactly one DPC entry keyed "zedagent" and no error (same pattern +// as bootstrap_test.go's matchSystemAdapterInfo helper). +// 2. Broken-config rollback: applies a brand-new EdgeDeviceConfig containing +// only ethernet0, switched from DHCP to a StaticNetworkConfig with a +// subnet/gateway (10.99.99.0/24 / 10.99.99.1) that does not exist on the +// SDN network (plus a static DNS server pointing at the real SDN DNS +// endpoint, since a StaticNetworkConfig has no DHCP to supply one). +// Since this DPC has no other management port to fall back on, EVE's +// connectivity test for it fails outright. Eventually SystemAdapterInfo +// reports: two DPC entries; CurrentIndex=1 (the older, still-working +// two-port DPC); the newest entry (index 0) has a non-empty LastError +// and a populated LastFailed. The device stays online throughout +// (verified via EdgeDevice.GetState()) since it never actually lost +// controller connectivity -- eth1 in the older DPC kept working the +// whole time. +// 3. Recovery: UpdateNetworkModel clones netmodels.TwoMgmtPorts and rewrites +// ethernet0's SDN subnet/gateway to 10.99.99.0/24 / 10.99.99.1, matching +// the broken config -- so it is not actually broken anymore. Once EVE's +// periodic testbetterinterval retest picks this up, SystemAdapterInfo +// eventually reports CurrentIndex=0 again, with LastSucceeded advancing +// past the recovery instant and LastError cleared. // // Future extension // ---------------- @@ -443,70 +455,553 @@ func TestPortFailover(test *testing.T) { // that a brief, "remote" failure (server cert expired) does NOT trigger // a fallback (per DEVICE-CONNECTIVITY.md "Handling remote (temporary) // failures"). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestNetworkConfigFallback(test *testing.T) { - test.Skip("not yet implemented") + // DNS server endpoint reachable from ethernet0's bridge, as defined in + // netmodels.TwoMgmtPorts (its own subnet, unaffected by network0's + // client-facing subnet being rewritten below). + const dnsServer0IP = "10.16.16.25" + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPorts, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.NetworkTestDuration, 10) + cfgProps.SetGlobalValueInt(pillartypes.NetworkTestBetterInterval, 60) + + devConfig := evetest.NewEdgeDeviceConfig(devName) + devConfig.SetConfigProperties(cfgProps) + + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + eth1Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("baseline-applied") + + log := evetest.Logger() + + // Phase 1: baseline. + log.Infof("Phase 1: waiting for the baseline DPC to become active...") + baselineTimeout := 3 * time.Minute + t.Eventually(devUpdates, baselineTimeout).Should(Receive(matchers.SatisfyPredicate( + "Baseline DPC (zedagent) is active with no error", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 0, []string{"zedagent"}) { + return false + } + return sa.GetStatus()[0].GetLastError() == "" + }))) + evetest.Checkpoint("phase1-baseline-complete") + + // Phase 2: apply a broken, ethernet0-only config. With no other + // management port in this DPC, EVE's connectivity test for it fails + // outright and NIM must roll back to the still-working two-port DPC. + log.Infof("Phase 2: applying a broken ethernet0-only config...") + brokenConfig := evetest.NewEdgeDeviceConfig(devName) + brokenConfig.SetConfigProperties(cfgProps) + brokenNet := brokenConfig.AddNetwork(evetest.StaticNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + Subnet: evetest.IPSubnet("10.99.99.0/24"), + Gateway: evetest.IPAddress("10.99.99.1"), + DNSServers: []net.IP{evetest.IPAddress(dnsServer0IP)}, + }) + brokenConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: brokenNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + StaticIP: evetest.IPAddress("10.99.99.5"), + }) + device.ApplyConfig(brokenConfig, false, false) + evetest.Checkpoint("phase2-broken-config-applied") + + phase2Timeout := 5 * time.Minute + var brokenDPCLastFailed time.Time + t.Eventually(devUpdates, phase2Timeout).Should(Receive(matchers.SatisfyPredicate( + "New DPC fails outright; EVE falls back to the previous DPC", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 1, []string{"zedagent", "zedagent"}) { + return false + } + newDPC := sa.GetStatus()[0] + if newDPC.GetLastError() == "" || newDPC.GetLastFailed() == nil { + return false + } + brokenDPCLastFailed = newDPC.GetLastFailed().AsTime() + return true + }))) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE), + "device must stay online via the still-working older DPC") + evetest.Checkpoint("phase2-fallback-complete") + + // Phase 3: fix the SDN network to match the broken config's subnet. + // Once EVE's periodic retest of the higher-priority DPC succeeds, it + // becomes active again. + log.Infof("Phase 3: updating the SDN network to match ethernet0's static config...") + fixedModel := proto.Clone(netmodels.TwoMgmtPorts).(*api.NetworkModel) + for _, n := range fixedModel.Networks { + if n.LogicalLabel == "network0" { + n.Ipv4.Subnet = "10.99.99.0/24" + n.Ipv4.GwIp = "10.99.99.1" + } + } + recoveryStart := time.Now() + evetest.UpdateNetworkModel(fixedModel) + // Always restore the model on exit so a mid-test failure does not leave + // the SDN in an altered state for subsequent suite tests. + defer evetest.UpdateNetworkModel(netmodels.TwoMgmtPorts) + evetest.Checkpoint("phase3-network-fixed") + + // DevicePortConfig.IsDPCTestable (pkg/pillar/types/dpc.go) refuses to + // retest a previously-failed DPC until DpcMinTimeSinceFailure has passed + // since its own LastFailed instant -- a hardcoded 5-minute constant in + // pkg/pillar/dpcmanager/dpcmanager.go with no controller-config override + // (unlike timer.port.testduration/testbetterinterval, which we do lower + // above). Budget from the broken DPC's LastFailed, not from when we fix + // the network here, plus one testbetterinterval tick and a safety margin + // -- otherwise the retest can still be within its cooldown by the time a + // timeout measured from recoveryStart elapses. + const dpcMinTimeSinceFailure = 5 * time.Minute + const testBetterInterval = 60 * time.Second + phase3Timeout := time.Until(brokenDPCLastFailed.Add( + dpcMinTimeSinceFailure + testBetterInterval + 2*time.Minute)) + if phase3Timeout < 3*time.Minute { + phase3Timeout = 3 * time.Minute + } + // TODO: this can still time out even with the generous budget above. + // The candidate (index 0) DPC is static on eth0 (10.99.99.5) while the + // currently-active DPC (index 1) uses DHCP on the *same* eth0/subnet + // (post-fix, leasing e.g. 10.99.99.123): each retest of the candidate + // makes NIM flip eth0's address between the two, and the mgmt dnsmasq + // (pkg/pillar/dpcreconciler/genericitems/mgmtdnsmasq.go) forwards to + // 10.16.16.25@eth0 for both DPCs -- so it can get caught with eth0 + // mid-reconfiguration and time out ("read udp 127.0.0.1:53: i/o + // timeout"), even though the network is genuinely fine moments before + // and after. This was observed live: eth0 briefly held the static + // candidate's address (10.99.99.5) while SystemAdapter.CurrentIndex + // still reported the DHCP DPC (index 1) as active. DpcManager's + // DNSCacheClearCounter (mgmtdnsmasq.go) only flushes dnsmasq's cached + // *answers* on a DPC transition; it does not address a transiently + // unavailable/inconsistent eth0 address+route during the swap, and + // verifyDPC's AsyncInProgress wait (dpcmanager/verify.go) is meant for + // slow reconcile operations (DHCP negotiation, etc.), not the brief + // settling window after a plain address change. Needs a real fix in + // dpcmanager/dpcreconciler (e.g. an explicit dependency/ordering so the + // interface is confirmed stable before the connectivity test runs, or + // before mgmt dnsmasq is told to reload) rather than a test-side + // workaround. Tracked as a follow-up; not fixed by this test. + t.Eventually(devUpdates, phase3Timeout).Should(Receive(matchers.SatisfyPredicate( + "The newer DPC becomes active again once the network matches it", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 0, []string{"zedagent", "zedagent"}) { + return false + } + dpc := sa.GetStatus()[0] + if dpc.GetLastError() != "" { + return false + } + ts := dpc.GetLastSucceeded() + return ts != nil && !ts.AsTime().Before(recoveryStart) + }))) + evetest.Checkpoint("phase3-recovery-complete") } // TestIntermittentConnectivity verifies that EVE remains (or eventually -// becomes) ONLINE when the network exhibits significant impairments such as -// high latency, packet loss, low bandwidth and intermittent outages. +// becomes) ONLINE when the network exhibits significant impairments -- +// packet loss, high latency and jitter, narrow bandwidth, and full outages -- +// on its only management uplink. +// +// Connectivity under each impairment is confirmed directly: +// timer.deviceinfo.interval is lowered to its allowed minimum (30s) so zedagent +// publishes a fresh ZInfoDevice periodically even absent any real change +// (see zedagent/handleconfig.go's configTimerTask), and after every impairment +// change the test asserts (via WatchDeviceInfo) that such an update still +// arrives within a bounded timeout. Actually getting a message through is a +// direct proof of connectivity. // // Network model -// - Single management port (netmodels.SingleEthWithDHCP). The interesting -// dimension is the per-port TrafficControl, not topology. We don't need -// multi-port to exercise resilience to a flaky single uplink. +// ------------- +// - netmodels.SingleEthWithDHCP -- a single management port. The +// interesting dimension here is per-port TrafficControl, not topology; +// multi-port fail-over is already covered by TestPortFailover. // // Device configuration -// - Plain DHCP-on-eth0 mgmt config. -// - Add a Local NI + a small ICMP-only test app to also exercise app -// connectivity under degraded network conditions. -// -// Phase 1 — baseline -// - Apply config and confirm device is ONLINE, app is RUNNING, app can curl -// http-server.test. -// -// Phase 2 — high-loss link -// - UpdateNetworkModel: set TrafficControl on eth0 with loss_probability=20. -// - Consistently for, say, 3 minutes (longer than EVE's default test -// interval), poll device.GetState() / DeviceInfo: device must stay -// ONLINE. SystemAdapterInfo.currentIndex must remain 0 (no fallback — -// EVE retries succeed often enough). -// - The app's `ping http-server.test` should mostly succeed (>50% -// success rate) — assert >= 50% success over 100 pings. -// -// Phase 3 — high latency + jitter -// - UpdateNetworkModel: TrafficControl{delay=500, delay_jitter=300, loss=0}. -// - Device must stay ONLINE; HTTP request from app must still succeed -// within a reasonable timeout (e.g. 30s). -// -// Phase 4 — narrow bandwidth -// - UpdateNetworkModel: TrafficControl{rate_limit=64 KB/s, queue_limit=32 KB, -// burst_limit=8 KB}. -// - Device must stay ONLINE (controller traffic is small). -// - The app's HTTP fetch of "/helloworld" must still succeed (it's a few -// bytes of payload). -// -// Phase 5 — full outage windows -// - For three iterations, alternate: -// a) UpdateNetworkModel: AdminUp=false on eth0 -> hold for 90s. -// b) UpdateNetworkModel: AdminUp=true -> hold for 90s. -// - During AdminUp=false windows, device may transiently report an error -// for the port; this is acceptable. The hard requirement is that the -// device returns to ONLINE within X seconds (e.g. 60s) of every -// AdminUp=true transition. -// - lastSucceeded timestamp on the active DPC must keep advancing across -// the test duration. +// -------------------- +// - ethernet0 (mgmt+app, DHCP). +// - One Local NI on ethernet0 with one container app (port-fwd 2222->22, +// default-allow ACL) to also exercise app connectivity under degraded +// network conditions. // -// Phase 6 — restore and verify steady state -// - UpdateNetworkModel back to TrafficControl-less; verify ONLINE, -// latency-free behavior. +// Phases +// ------ +// 1. Baseline: apply config (including the lowered timer.deviceinfo.interval), +// wait for the NI ONLINE and the app RUNNING, confirm the device is +// ONLINE and the app can curl http-server.test. +// 2. High-loss link: UpdateNetworkModel sets TrafficControl{loss_probability: +// 20} on eth0. Two consecutive fresh-device-info waits (~3 minutes total) +// confirm EVE keeps getting through repeatedly, not just once, despite +// the loss. From the app, `ping -c 100 http-server.test` must show a +// packet-loss percentage no higher than 50% (i.e. at least half of the +// pings get through). +// 3. High latency + jitter: TrafficControl{delay: 500, delay_jitter: 300}. +// A fresh device-info update still arrives within the timeout, and an +// HTTP request from the app still succeeds within 30s. +// 4. Narrow bandwidth: TrafficControl{rate_limit: 64, queue_limit: 32, +// burst_limit: 8} (KB/s and KB -- a few bytes of controller/HTTP traffic +// still fit easily). A fresh device-info update still arrives, and the +// app's HTTP fetch of /helloworld still succeeds. +// 5. Full outage window: AdminUp=false (90s) followed by AdminUp=true. +// After the AdminUp=true transition, WatchDeviceInfo eventually reports +// the active DPC's LastSucceeded newer than the transition instant, then +// a fresh-device-info wait holds at the recovered state for a window +// equal to the outage. +// 6. Restore and verify steady state: UpdateNetworkModel back to the +// TrafficControl-less model; a fresh device-info update arrives and the +// app's HTTP fetch succeeds promptly (no latency/loss left to mask a +// regression). // // Notes // ----- -// - This test is non-trivially time-sensitive. Generous timeouts are -// necessary; the focus is on EVE's eventual recovery, not strict timing. -// - If a CI run becomes too long, individual phases can be split into -// separate test functions (each phase already maps cleanly to a sub-test). +// - This test is non-trivially time-sensitive; timeouts are generous +// since the focus is on EVE's eventual recovery, not strict timing. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestIntermittentConnectivity(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Lower the periodic (unconditional, even absent any real change) + // device-info publish interval to its allowed minimum, so the test can + // use "a fresh ZInfoDevice arrives within a bounded timeout" as a + // direct, real-time signal that EVE is still getting through to the + // controller -- see zedagent/handleconfig.go's configTimerTask + // ("ticker for periodical info publish when no real change"). + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, 30) + devConfig.SetConfigProperties(cfgProps) + + eth0Net := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress("10.50.0.1"), + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "intermittent-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + niUpdates, stopNIWatch := device.WatchNetworkInstanceInfo(niUUID) + defer stopNIWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + timeout := 3 * time.Minute + + // Phase 1: baseline. + log.Infof("Phase 1: verifying baseline connectivity...") + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "NI is ONLINE", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.GetState() == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("phase1-app-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("phase1-baseline-complete") + + // infoTimeout bounds how long a fresh ZInfoDevice update may take to + // arrive after an impairment is applied: with timer.deviceinfo.interval + // lowered to 30s above, 90s gives 3x margin for retries under packet + // loss/high latency before treating an actual delivery failure as such. + infoTimeout := 90 * time.Second + waitForFreshInfo := func(reason string) { + // Discard anything already buffered on the channel first, so this + // only accepts a message that arrives after the check starts -- + // otherwise a backlog from before the impairment was applied could + // satisfy Receive() immediately without proving anything new. + drainBacklog: + for { + select { + case <-devUpdates: + default: + break drainBacklog + } + } + log.Infof("Waiting for a fresh device info update (%s)...", reason) + t.Eventually(devUpdates, infoTimeout).Should(Receive(), + "EVE should still get periodic device info through "+ + "to the controller (%s)", reason) + } + + // Always restore the model on exit so a mid-test failure does not leave + // the SDN in an altered state for subsequent suite tests. + restoreModel := func() { + evetest.UpdateNetworkModel(netmodels.SingleEthWithDHCP) + } + defer restoreModel() + + setTrafficControl := func(tc *api.TrafficControl) { + model := proto.Clone(netmodels.SingleEthWithDHCP).(*api.NetworkModel) + for _, p := range model.Ports { + if p.LogicalLabel == "eth0" { + p.TrafficControl = tc + } + } + evetest.UpdateNetworkModel(model) + } + + // Phase 2: high-loss link. + log.Infof("Phase 2: applying 20%% packet loss on eth0...") + setTrafficControl(&api.TrafficControl{LossProbability: 20}) + evetest.Checkpoint("phase2-loss-applied") + + // Give the lossy link a sustained period (longer than a single probe + // cycle): two consecutive fresh-info waits cover ~3 minutes under loss, + // confirming EVE keeps getting through repeatedly, not just once. + waitForFreshInfo("20% packet loss, check 1/2") + waitForFreshInfo("20% packet loss, check 2/2") + + log.Infof("Phase 2: pinging http-server.test through the lossy link...") + pingOut, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "ping -c 50 -w 75 http-server.test", 80*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + lossPct, err := pingPacketLossPercent(pingOut) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(lossPct).To(BeNumerically("<=", 50), + "at least half of the pings must get through a 20%% -loss link:\n%s", pingOut) + evetest.Checkpoint("phase2-complete") + + // Phase 3: high latency + jitter. + log.Infof("Phase 3: applying 500ms +/- 300ms latency on eth0...") + setTrafficControl(&api.TrafficControl{Delay: 500, DelayJitter: 300}) + evetest.Checkpoint("phase3-latency-applied") + + waitForFreshInfo("500ms +/- 300ms latency") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 30 http://http-server.test/helloworld", 35*time.Second, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase3-complete") + + // Phase 4: narrow bandwidth. + log.Infof("Phase 4: applying a 64 KB/s rate limit on eth0...") + setTrafficControl(&api.TrafficControl{ + RateLimit: 64, + QueueLimit: 32, + BurstLimit: 8, + }) + evetest.Checkpoint("phase4-bandwidth-limited") + + waitForFreshInfo("64 KB/s rate limit") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 20 http://http-server.test/helloworld", 25*time.Second, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase4-complete") + + // Phase 5: full outage window. + log.Infof("Phase 5: taking eth0 down, then restoring it...") + const outageWindow = 90 * time.Second + recoveryTimeout := 5 * time.Minute + downModel := proto.Clone(netmodels.SingleEthWithDHCP).(*api.NetworkModel) + for _, p := range downModel.Ports { + if p.LogicalLabel == "eth0" { + p.AdminUp = false + } + } + evetest.UpdateNetworkModel(downModel) + time.Sleep(outageWindow) + + log.Infof("Phase 5: restoring eth0...") + recoveryStart := time.Now() + restoreModel() + + t.Eventually(devUpdates, recoveryTimeout).Should(Receive(matchers.SatisfyPredicate( + "DPC LastSucceeded advances past the AdminUp=true transition", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if sa == nil { + return false + } + statusList := sa.GetStatus() + idx := int(sa.GetCurrentIndex()) + if idx < 0 || idx >= len(statusList) { + return false + } + ts := statusList[idx].GetLastSucceeded() + return ts != nil && !ts.AsTime().Before(recoveryStart) + }))) + // Hold at the recovered state for a window equal to the outage above, + // actively confirming (rather than just sleeping) that fresh device + // info keeps arriving throughout. + waitForFreshInfo("recovered after outage") + evetest.Checkpoint("phase5-complete") + + // Phase 6: restore and verify steady state. + log.Infof("Phase 6: verifying steady state after restoring the clean network model...") + restoreModel() + waitForFreshInfo("steady state restored") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase6-complete") +} + +// pingPacketLossPercent extracts the packet-loss percentage from the summary +// line of `ping` output (e.g. "100 packets transmitted, 82 received, 18% +// packet loss, time 99231ms"). The percentage is fractional whenever the +// loss ratio isn't a whole number (e.g. "14.5299%"), so it must be parsed as +// a float rather than truncated to the digits right before the '%'. +func pingPacketLossPercent(output string) (float64, error) { + re := regexp.MustCompile(`([\d.]+)% packet loss`) + m := re.FindStringSubmatch(output) + if len(m) != 2 { + return 0, fmt.Errorf("could not parse packet loss from ping output: %q", output) + } + return strconv.ParseFloat(m[1], 64) } diff --git a/evetest/tests/networking/ipv6_test.go b/evetest/tests/networking/ipv6_test.go index f0985a64475..68384af5f09 100644 --- a/evetest/tests/networking/ipv6_test.go +++ b/evetest/tests/networking/ipv6_test.go @@ -61,11 +61,20 @@ import ( // - ip -6 route show contains a default route via a fe80:: link-local // address (RA-derived routes always use the router's link-local address). // - ip -4 addr show dev eth0 contains no "inet" lines (no IPv4). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDeviceIPv6Connectivity(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" // Clone the shared model so we can modify it without side effects. // Clear upstream DNS servers: everything this test needs (controller, DNS @@ -78,7 +87,7 @@ func TestDeviceIPv6Connectivity(test *testing.T) { evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -102,6 +111,9 @@ func TestDeviceIPv6Connectivity(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -225,8 +237,7 @@ func TestDeviceIPv6Connectivity(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. evetest.SkipIfHypervisorKubevirt() is called after reading -// the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestApplicationIPv6Connectivity(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -236,8 +247,6 @@ func TestApplicationIPv6Connectivity(test *testing.T) { evetest.HypervisorParameter(), ) hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // IPv6 address of the SDN HTTP server and DNS server defined in netmodels.SingleEthIPv6Only. const httpServerIPv6 = "fdde:55a:74d4::7" @@ -317,6 +326,9 @@ func TestApplicationIPv6Connectivity(test *testing.T) { appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) defer stopAppWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } timeoutExcludingDownload := 5 * time.Minute device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) diff --git a/evetest/tests/networking/net_adapter_test.go b/evetest/tests/networking/net_adapter_test.go index e2a4dc2c822..3851da88340 100644 --- a/evetest/tests/networking/net_adapter_test.go +++ b/evetest/tests/networking/net_adapter_test.go @@ -25,10 +25,11 @@ import ( "github.com/lf-edge/eve/pkg/pillar/utils/netutils" ) -func deviceRequirementsForNetAdapterTests(devName string) evetest.RequireEdgeDevice { +func deviceRequirementsForNetAdapterTests( + devName string, hypervisor evetest.Hypervisor) evetest.RequireEdgeDevice { return evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, MinCPUs: 4, WithGrubOptions: []string{ // No applications are deployed in these network adapter tests. @@ -64,9 +65,14 @@ func TestDHCPIPv4Only(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCPAndIPv6, } @@ -89,6 +95,9 @@ func TestDHCPIPv4Only(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -131,9 +140,14 @@ func TestStaticIPv4Only(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCPAndIPv6, } @@ -160,6 +174,9 @@ func TestStaticIPv4Only(test *testing.T) { StaticIP: evetest.IPAddress("172.20.20.100"), }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -253,15 +270,17 @@ func TestPNAC(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), requireSCEPProxyParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() requireSCEPProxy := evetest.GetTestParameter[bool](requireSCEPProxyParamKey) // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithPNAC(requireSCEPProxy), } @@ -329,6 +348,9 @@ func TestPNAC(test *testing.T) { defer stopDevMetricsWatch() configAppliedAt := time.Now() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") timeout := 3 * time.Minute diff --git a/evetest/tests/networking/netinst_test.go b/evetest/tests/networking/netinst_test.go index e756c5492c7..e0e956b03b9 100644 --- a/evetest/tests/networking/netinst_test.go +++ b/evetest/tests/networking/netinst_test.go @@ -6,7 +6,9 @@ package networking_test import ( + "fmt" "net" + "strings" "testing" "time" @@ -15,6 +17,7 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evemetrics "github.com/lf-edge/eve-api/go/metrics" "github.com/lf-edge/eve/evetest" @@ -22,6 +25,7 @@ import ( "github.com/lf-edge/eve/evetest/netmodels" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/lf-edge/eve/pkg/pillar/utils/generics" + uuid "github.com/satori/go.uuid" ) // TestLocalNI is the canonical end-to-end exercise of a Local (L3/NAT) @@ -78,20 +82,16 @@ import ( // 6. NI metrics: ZMetricNetworkInstance for the NI eventually has // non-zero RX and TX TotalPackets, proving the per-NI dataplane // counters track the traffic generated above. -// 7. Flow / DNS log assertions are commented out -- GetAppFlowLogs / -// GetAppDNSLogs are not yet implemented in evetest (see -// edgedevice.go). The placeholders document the intended check -// (with flowlog disabled the lists must be empty) and will be enabled -// once the framework support lands. +// 7. Flow / DNS log check: flow logging is off by default (it is enabled +// and exercised in TestFlowLog), so GetAppFlowLogs / GetAppDNSLogs must +// both return empty for this app's VIF. // 8. App teardown: delete the app, wait until ZSwState_INVALID, then // assert NetworkInstance.Vifs is empty and the bridge-IP assignment // persists. Finally delete the NI and wait for UNSPECIFIED. // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter -- Kubevirt is reserved for cluster -// tests. +// - HYPERVISOR (defaults to KVM). // // Suite placement // --------------- @@ -109,8 +109,6 @@ func TestLocalNI(test *testing.T) { // Get parameter values set for this test execution. hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // Set up the test harness and specify the test prerequisites. devName := "edge-dev" @@ -142,6 +140,9 @@ func TestLocalNI(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Try to create local network instance. niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ @@ -392,7 +393,6 @@ func TestLocalNI(test *testing.T) { stopNIMetricsWatch() // Flowlog is disabled by default (it is enabled and tested in TestFlowLog). - /* TODO: GetAppFlowLogs is not yet implemented t.Expect(device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ VirtualNetAdapter: "vif0", NetworkInstance: niUUID, @@ -401,7 +401,6 @@ func TestLocalNI(test *testing.T) { VirtualNetAdapter: "vif0", NetworkInstance: niUUID, })).To(BeEmpty()) - */ // Undeploy app and check that VIF was disconnected from the network instance. devConfig.DeleteApplication(appUUID) @@ -502,8 +501,7 @@ func TestLocalNI(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter. +// - HYPERVISOR (defaults to KVM). // // Suite placement // --------------- @@ -520,8 +518,6 @@ func TestSwitchNI(test *testing.T) { // Get parameter values set for this test execution. hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // Set up the test harness and specify the test prerequisites. devName := "edge-dev" @@ -553,6 +549,9 @@ func TestSwitchNI(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Try to create switch network instance. niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ @@ -783,7 +782,6 @@ func TestSwitchNI(test *testing.T) { stopNIMetricsWatch() // Flowlog is disabled by default (it is enabled and tested in TestFlowLog). - /* TODO: GetAppFlowLogs is not yet implemented t.Expect(device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ VirtualNetAdapter: "vif0", NetworkInstance: niUUID, @@ -792,7 +790,6 @@ func TestSwitchNI(test *testing.T) { VirtualNetAdapter: "vif0", NetworkInstance: niUUID, })).To(BeEmpty()) - */ // Undeploy app and check that VIF was disconnected from the network instance. devConfig.DeleteApplication(appUUID) @@ -827,69 +824,1766 @@ func TestSwitchNI(test *testing.T) { stopNIWatch() } -// TestFlowLog verifies that EVE produces flow log records and DNS request log -// records for application traffic when flow logging is enabled on a Local -// Network Instance, and that those records correctly attribute flows to ACE -// IDs (allowed flows -> matching ACE; dropped flows -> ACE id 0 = implicit -// reject-all). +// TestNIReplace verifies that EVE correctly handles Network Instances being +// rapidly replaced -- deleted and recreated (possibly reusing the same +// subnet, or splitting/merging across a different number of NIs) -- all +// within a single config apply. This exercises internal number/resource +// reuse (bridge number, IPAM state, iptables chains, etc.) inside zedrouter, +// which could otherwise conflict if stale state from the deleted NI is not +// fully torn down before the replacement comes up. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port; no application is +// deployed, only Local Network Instances are created/replaced. +// +// Phases +// ------ +// 1. Create n1 (Local NI, subnet 10.11.12.0/24). Wait for ONLINE. +// 2. Replace n1 with n2 using a *different* subnet (10.11.13.0/24), deleting +// n1 and adding n2 in one config apply. Wait for n1 -> UNSPECIFIED and +// n2 -> ONLINE with the new subnet's bridge IP. +// 3. Replace n2 with n3, reusing the *same* subnet (10.11.13.0/24) that n2 +// just released, again in one apply. Wait for n2 -> UNSPECIFIED and +// n3 -> ONLINE with BridgeIPAddr unchanged (10.11.13.1), proving the +// just-freed subnet can be immediately reused by a different NI. +// 4. Replace one NI (n3) with two NIs in one apply: n4 (reusing the +// 10.11.12.0/24 subnet that has been free since step 2) and n5 (a fresh +// subnet). Both come up ONLINE while n3 goes UNSPECIFIED. +// 5. Move the subnet used by n4 to a new one (10.11.14.0/24) and, in the very +// same apply, add n6 that takes over n4's *former* subnet +// (10.11.12.0/24). Wait for n6 -> ONLINE with the taken-over subnet's +// bridge IP and for n4 to keep running with its new bridge IP.. +// 6. Cleanup: delete n4, n5 and n6, and wait for all three to report +// ZNETINST_STATE_UNSPECIFIED. // -// SKIPPED: app flow logs are not yet supported in evetest. The framework -// provides the API (EdgeDevice.GetAppFlowLogs / GetAppDNSLogs) but its -// implementation depends on AdamClient.IterateAppFlowLogs which is not -// implemented (see edgedevice.go). +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). // -// When implemented later, the scenario: +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestNIReplace(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + // localNI builds a Local NI config for a /24 subnet whose gateway is the + // ".1" address; the DHCP range spans the rest of the subnet (.2-.254). + localNI := func(displayName, subnetPrefix string) evetest.LocalNetworkInstanceConfig { + return evetest.LocalNetworkInstanceConfig{ + DisplayName: displayName, + Port: "ethernet0", + Subnet: evetest.IPSubnet(subnetPrefix + ".0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress(subnetPrefix + ".2"), + End: evetest.IPAddress(subnetPrefix + ".254"), + }, + Gateway: evetest.IPAddress(subnetPrefix + ".1"), + MTU: 1500, + } + } + + timeout := 3 * time.Minute + waitOnline := func(niUUID uuid.UUID, updates <-chan *eveinfo.ZInfoNetworkInstance, + expectBridgeIP string) *eveinfo.ZInfoNetworkInstance { + var niInfo *eveinfo.ZInfoNetworkInstance + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("NI %s is ONLINE with bridge IP %s", niUUID, expectBridgeIP), + func(info *eveinfo.ZInfoNetworkInstance) bool { + niInfo = info + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE && + info.BridgeIPAddr == expectBridgeIP + }))) + t.Expect(niInfo.NetworkID).To(Equal(niUUID.String())) + t.Expect(niInfo.NetworkErr).To(BeEmpty()) + return niInfo + } + waitGone := func(updates <-chan *eveinfo.ZInfoNetworkInstance) { + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "NI state is UNSPECIFIED", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + } + + // Step 1: create n1. + n1UUID := devConfig.AddNetworkInstance(localNI("n1", "10.11.12")) + n1Updates, stopN1Watch := device.WatchNetworkInstanceInfo(n1UUID) + device.ApplyConfig(devConfig, false, false) + waitOnline(n1UUID, n1Updates, "10.11.12.1") + evetest.Checkpoint("n1-created") + + // Step 2: replace n1 with n2, using a different subnet, in a single apply. + devConfig.DeleteNetworkInstance(n1UUID) + n2UUID := devConfig.AddNetworkInstance(localNI("n2", "10.11.13")) + n2Updates, stopN2Watch := device.WatchNetworkInstanceInfo(n2UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n1Updates) + stopN1Watch() + waitOnline(n2UUID, n2Updates, "10.11.13.1") + evetest.Checkpoint("n1-replaced-by-n2") + + // Step 3: replace n2 with n3, reusing the exact same subnet n2 just released. + devConfig.DeleteNetworkInstance(n2UUID) + n3UUID := devConfig.AddNetworkInstance(localNI("n3", "10.11.13")) + n3Updates, stopN3Watch := device.WatchNetworkInstanceInfo(n3UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n2Updates) + stopN2Watch() + waitOnline(n3UUID, n3Updates, "10.11.13.1") + evetest.Checkpoint("n2-replaced-by-n3") + + // Step 4: replace n3 with two NIs -- n4 (reusing the 10.11.12.0/24 subnet + // that has been free since step 2) and n5 (a brand new subnet). + devConfig.DeleteNetworkInstance(n3UUID) + n4UUID := devConfig.AddNetworkInstance(localNI("n4", "10.11.12")) + n5UUID := devConfig.AddNetworkInstance(localNI("n5", "10.11.15")) + n4Updates, stopN4Watch := device.WatchNetworkInstanceInfo(n4UUID) + n5Updates, stopN5Watch := device.WatchNetworkInstanceInfo(n5UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n3Updates) + stopN3Watch() + waitOnline(n4UUID, n4Updates, "10.11.12.1") + waitOnline(n5UUID, n5Updates, "10.11.15.1") + evetest.Checkpoint("n3-replaced-by-n4-and-n5") + + // Step 5: move n4's subnet elsewhere and let n6 take over the subnet that + // n4 just vacated -- both changes submitted in the very same apply. + devConfig.UpdateNetworkInstance(n4UUID, localNI("n4", "10.11.14")) + n6UUID := devConfig.AddNetworkInstance(localNI("n6", "10.11.12")) + n6Updates, stopN6Watch := device.WatchNetworkInstanceInfo(n6UUID) + device.ApplyConfig(devConfig, false, false) + waitOnline(n4UUID, n4Updates, "10.11.14.1") + waitOnline(n6UUID, n6Updates, "10.11.12.1") + evetest.Checkpoint("n4-subnet-moved-to-n6") + + // Cleanup. + devConfig.DeleteNetworkInstance(n4UUID) + devConfig.DeleteNetworkInstance(n5UUID) + devConfig.DeleteNetworkInstance(n6UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n4Updates) + waitGone(n5Updates) + waitGone(n6Updates) + stopN4Watch() + stopN5Watch() + stopN6Watch() +} + +// TestMoveAppBetweenNIs verifies that moving an application's network +// adapter from one Network Instance to another (i.e. changing which NI a +// VIF is attached to, within an otherwise-unchanged app config) takes +// effect at runtime: the app is redeployed onto the new NI, gets a fresh IP +// from its subnet, loses connectivity with peers on the NI it left, and +// gains connectivity with peers on the NI it joined. // // Network model -// - netmodels.SingleEthWithDHCP, plus a second SDN HTTP server -// (alt-server.test) for an additional differentiable target. +// ------------- +// - netmodels.SingleEthWithDHCP -- a single mgmt+app port (ethernet0), +// shared as the uplink by both Local NIs below (a port can back more +// than one Local NI at once). // -// Device configuration -// - One Local NI with EnableFlowlog=true and a Subnet/DHCPRange. -// - One container app on the NI with three ACLs: -// - allow IP+TCP+fport=80 to http-server.test (specific ACE id, e.g. 100) -// - allow IP+ICMP to NI bridge IP (ACE id 200) -// - default-deny is implicit (ACE id 0) -// - Port-fwd 2222->22 on a separate ACE (id 300) to enable test SSH. -// -// Phase 1 — generate distinguishable traffic -// - From inside the app: curl http-server.test/helloworld (allowed by ACE 100). -// - From inside the app: curl --max-time 5 alt-server.test/helloworld -// (must fail; matches the implicit deny -> ACE id 0). -// - From inside the app: ping -c 3 (allowed by ACE 200). -// - From inside the app: nslookup http-server.test (DNS request log entry). -// -// Phase 2 — flow log assertions (via GetAppFlowLogs) -// - Wait for the flow log batch (default 2-min interval per -// APP-CONNECTIVITY.md). Use a generous timeout (3 min). -// - For ACE id 100: at least one outbound flow with dst IP = -// http-server.test's IP, dst port 80, proto TCP, packet count > 0. -// - For ACE id 200: at least one outbound ICMP flow toward the NI -// bridge IP. -// - For ACE id 0: at least one outbound flow toward alt-server's IP, dst -// port 80, proto TCP. The flow record is created because the app sent -// the packet even though it was dropped (flow logging on Local NI uses -// the conntrack-based mark-and-blackhole pattern; see APP-CONNECTIVITY.md). -// - The reverse direction of each flow (Inbound) for the allowed flows is -// also logged once the response packet is observed. -// -// Phase 3 — DNS log assertions (via GetAppDNSLogs) -// - At least one DNSRequest record with hostname "http-server.test" and the -// resolved IP equal to that of the SDN http-server endpoint. -// - The request time is within the test window. -// -// Phase 4 — flow logging disabled -// - UpdateNetworkInstance to set EnableFlowlog=false; re-apply. -// - Generate fresh traffic; after the next reporting interval, assert that -// no NEW flow records appear (timestamps strictly older than the -// reapply time). This confirms the runtime toggle works. +// Phases +// ------ +// 1. Create two Local NIs on ethernet0: "n1" (10.11.12.0/24) and "n2" +// (10.11.13.0/24). Wait for both ONLINE. +// 2. Deploy three container apps (lfedge/evetest-ubuntu-ctr:1.0, +// VirtualizationMode=HVM): "ping1" on n1 (port-fwd 2223->22, ICMP +// allow-all ACL), "ping2" on n2 (port-fwd 2224->22, ICMP allow-all ACL) +// and "pong" on n1 (ICMP allow-all ACL, no port-fwd -- nothing ever SSHes +// into it). WaitUntilAppIsRunning for all three, then read pong's NI IP +// from its ZInfoApp. +// 3. Same-NI reachability: from ping1 (on n1, same NI as pong), `ping -c 3 +// -W 1 ` succeeds. From ping2 (on n2), the same ping fails -- +// pong's private /24 is not reachable from a different, NAT-isolated +// Local NI. +// 4. Move pong from n1 to n2: UpdateApplication with pong's +// VirtualNetworkAdapter now pointing at n2's UUID (same MAC, same ACL). +// Changing the NI reference changes the app's Interfaces, so EVE purges +// and redeploys pong. WaitUntilAppIsRunning again, then read pong's new +// (n2-subnet) IP. +// 5. Switched reachability: ping2 can now reach pong; ping1 can no longer +// reach it (mirrors step 3, with the roles of ping1/ping2 reversed). +// 6. Move pong back to n1 and repeat the step-3 assertions, confirming the +// switch is fully reversible. +// 7. Cleanup: delete all three apps and both NIs, waiting for each to be +// gone. // // Test params // ----------- -// - HYPERVISOR. The test must call evetest.SkipIfHypervisorKubevirt() -// after reading the parameter -- Kubevirt is reserved for cluster tests. -func TestFlowLog(test *testing.T) { - test.Skip("not yet implemented") +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestMoveAppBetweenNIs(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + n1UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "n1", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + n2UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "n2", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.13.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.13.2"), + End: evetest.IPAddress("10.11.13.254"), + }, + Gateway: evetest.IPAddress("10.11.13.1"), + MTU: 1500, + }) + + n1Updates, stopN1Watch := device.WatchNetworkInstanceInfo(n1UUID) + n2Updates, stopN2Watch := device.WatchNetworkInstanceInfo(n2UUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(n1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n1 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(n2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n2 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + icmpAllowAny := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + const ( + ping1MAC = "02:16:3e:00:00:01" + ping2MAC = "02:16:3e:00:00:02" + pongMAC = "02:16:3e:00:00:03" + ) + newApp := func(displayName string, niUUID uuid.UUID, mac string, + portFwd []evetest.PortFwdRule) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress(mac), + PortFwdRules: portFwd, + ACLAllowRules: icmpAllowAny, + }, + }, + } + } + ping1UUID := devConfig.AddApplication(newApp("ping1", n1UUID, ping1MAC, + []evetest.PortFwdRule{{Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2223, AppPort: 22}})) + ping2UUID := devConfig.AddApplication(newApp("ping2", n2UUID, ping2MAC, + []evetest.PortFwdRule{{Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2224, AppPort: 22}})) + pongUUID := devConfig.AddApplication(newApp("pong", n1UUID, pongMAC, nil)) + + ping1Updates, stopPing1Watch := device.WatchAppInfo(ping1UUID) + ping2Updates, stopPing2Watch := device.WatchAppInfo(ping2UUID) + pongUpdates, stopPongWatch := device.WatchAppInfo(pongUUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(ping1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(ping2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(pongUUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + // getVifIPInSubnet drains updates until the app reports a VIF IP address + // that belongs to the given subnet. Requiring subnet membership (rather + // than accepting any IP) makes this robust against a stale, previously + // drained IP (from before an NI move) still sitting in the channel. + getVifIPInSubnet := func(updates <-chan *eveinfo.ZInfoApp, subnet *net.IPNet) string { + var ip string + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("App has a VIF IP address from %s", subnet), + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) == 0 { + return false + } + for _, addr := range info.Network[0].IPAddrs { + if subnet.Contains(evetest.IPAddress(addr)) { + ip = addr + return true + } + } + return false + }).StopIf(appHasError))) + return ip + } + n1Subnet := evetest.IPSubnet("10.11.12.0/24") + n2Subnet := evetest.IPSubnet("10.11.13.0/24") + pongIP := getVifIPInSubnet(pongUpdates, n1Subnet) + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(ping1UUID) + waitSSHReady(ping2UUID) + + pingFrom := func(appUUID uuid.UUID, targetIP string) error { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 %s", targetIP), sshTimeout, 0) + return err + } + + // Phase 1: pong is on n1, same as ping1. + log.Infof("Testing connectivity before the move: pong is on n1 (with ping1)") + t.Expect(pingFrom(ping1UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping2UUID, pongIP)).To(HaveOccurred()) + evetest.Checkpoint("pong-on-n1-verified") + + // Move pong from n1 to n2. + devConfig.UpdateApplication(pongUUID, newApp("pong", n2UUID, pongMAC, nil)) + device.ApplyConfig(devConfig, false, false) + pongIP = getVifIPInSubnet(pongUpdates, n2Subnet) + evetest.Checkpoint("pong-moved-to-n2") + + // Phase 2: pong is now on n2, same as ping2. + log.Infof("Testing connectivity after the move: pong is on n2 (with ping2)") + t.Expect(pingFrom(ping2UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping1UUID, pongIP)).To(HaveOccurred()) + evetest.Checkpoint("pong-on-n2-verified") + + // Move pong back from n2 to n1. + devConfig.UpdateApplication(pongUUID, newApp("pong", n1UUID, pongMAC, nil)) + device.ApplyConfig(devConfig, false, false) + pongIP = getVifIPInSubnet(pongUpdates, n1Subnet) + evetest.Checkpoint("pong-moved-back-to-n1") + + // Phase 3: pong is back on n1. + log.Infof("Testing connectivity after moving back: pong is on n1 again") + t.Expect(pingFrom(ping1UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping2UUID, pongIP)).To(HaveOccurred()) + + // Cleanup. + devConfig.DeleteApplication(ping1UUID) + devConfig.DeleteApplication(ping2UUID) + devConfig.DeleteApplication(pongUUID) + devConfig.DeleteNetworkInstance(n1UUID) + devConfig.DeleteNetworkInstance(n2UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(ping1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ping1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(ping2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ping2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(pongUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "pong is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopPing1Watch() + stopPing2Watch() + stopPongWatch() + + t.Eventually(n1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n1 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(n2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n2 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopN1Watch() + stopN2Watch() +} + +// TestPortForwarding exercises port-forwarding (D-NAT) rules of a Local +// Network Instance: hairpin connectivity between two apps reached through +// the edge node's *external* port-forwarded address (rather than directly +// via their NI-internal IPs), across both a shared uplink and two different +// uplink adapters, plus changing a port-forwarding rule's external port at +// runtime. +// +// Network model +// ------------- +// - netmodels.TwoMgmtPorts -- two independent mgmt+app ports (ethernet0, +// ethernet1), each with its own DHCP subnet and its own external IP. +// Needed for the cross-adapter hairpin scenario below; the same-adapter +// scenarios simply ignore ethernet1's NI. +// +// Phases +// ------ +// 1. Create two Local NIs: "ni1" on ethernet0 (10.11.12.0/24) and "ni2" on +// ethernet1 (10.11.13.0/24). Wait for both ONLINE. +// 2. Deploy three container apps (lfedge/evetest-ubuntu-ctr:1.0, +// VirtualizationMode=HVM, allow-all ACL on every VIF): "app1" and "app2" +// on ni1 (port-fwd 2223->22 and 2224->22, respectively) and "app3" on +// ni2 (port-fwd 2226->22). WaitUntilAppIsRunning for all three. +// 3. Same-adapter hairpin: from app1 (reached via its own port-fwd), +// open a raw TCP connection to :2224 -- app2's port-fwd +// address -- and read app2's sshd banner. Repeat in the opposite +// direction (app2 -> app1's port-fwd). Both must succeed, proving hairpin +// NAT works when the initiator and the target share both the NI and +// the uplink adapter. +// 4. Runtime port-fwd rule change: change app1's port-fwd rule from 2223->22 +// to 2225->22 via UpdateApplication (which purges/redeploys app1). +// After WaitUntilAppIsRunning: +// - the old external port (2223) must no longer forward (probed from +// app2: raw TCP connect times out / errors); +// - the new external port (2225) must forward to app1 (probed from +// app2: app1's sshd banner is read back); +// - RunShellScriptInsideApp(app1, ...) -- which re-derives the SSH +// endpoint from the *current* device config -- keeps working +// transparently through the port change. +// Then the rule is switched back to 2223->22 and the same two-sided +// check (old port dead, new/original port alive) is repeated, proving +// the change is fully reversible. +// 5. Cross-adapter hairpin: from app1 (on ni1/ethernet0), open a raw TCP +// connection to :2226 -- app3's port-fwd address on the +// *other* uplink adapter -- and read app3's sshd banner. This proves +// D-NAT is applied correctly per-adapter (the two uplinks have different +// external IPs) even when initiator and target sit on different Network +// Instances bound to different ports. +// 6. Cleanup: delete all three apps and both NIs. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestPortForwarding(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPorts, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + ni1UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "ni1", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + ni2UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "ni2", + Port: "ethernet1", + Subnet: evetest.IPSubnet("10.11.13.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.13.2"), + End: evetest.IPAddress("10.11.13.254"), + }, + Gateway: evetest.IPAddress("10.11.13.1"), + MTU: 1500, + }) + ni1Updates, stopNI1Watch := device.WatchNetworkInstanceInfo(ni1UUID) + ni2Updates, stopNI2Watch := device.WatchNetworkInstanceInfo(ni2UUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(ni1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni1 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(ni2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni2 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + newPortFwdApp := func(displayName string, niUUID uuid.UUID, mac string, + edgeNodePort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress(mac), + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: edgeNodePort, + AppPort: 22, + }, + }, + ACLAllowRules: allowAll, + }, + }, + } + } + const ( + app1MAC = "02:16:3e:00:01:01" + app2MAC = "02:16:3e:00:01:02" + app3MAC = "02:16:3e:00:01:03" + ) + app1UUID := devConfig.AddApplication(newPortFwdApp("app1", ni1UUID, app1MAC, 2223)) + app2UUID := devConfig.AddApplication(newPortFwdApp("app2", ni1UUID, app2MAC, 2224)) + app3UUID := devConfig.AddApplication(newPortFwdApp("app3", ni2UUID, app3MAC, 2226)) + + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + app3Updates, stopApp3Watch := device.WatchAppInfo(app3UUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app3UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(app1UUID) + waitSSHReady(app2UUID) + waitSSHReady(app3UUID) + + // hairpinProbe opens a raw TCP connection from inside srcApp to + // targetIP:targetPort and returns the first bytes read back (expected to + // be the target sshd's banner, "SSH-2.0-...", when the port-fwd works). + hairpinProbe := func(srcApp uuid.UUID, targetIP net.IP, targetPort uint16) (string, error) { + script := fmt.Sprintf( + "timeout 5 bash -c 'exec 3<>/dev/tcp/%s/%d; head -c 4 <&3'", + targetIP, targetPort) + out, _, err := device.RunShellScriptInsideApp( + srcApp, appAuth, script, sshTimeout, 0) + return out, err + } + + eth0IP := device.GetDeviceIPAddress("ethernet0") + t.Expect(eth0IP).ToNot(BeEmpty()) + eth1IP := device.GetDeviceIPAddress("ethernet1") + t.Expect(eth1IP).ToNot(BeEmpty()) + + // Phase 1: same-adapter hairpin between app1 and app2 (both on ni1/ethernet0). + log.Infof("Testing same-adapter hairpin: app1 -> app2's port-fwd") + out, err := hairpinProbe(app1UUID, eth0IP[0], 2224) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + log.Infof("Testing same-adapter hairpin: app2 -> app1's port-fwd") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + evetest.Checkpoint("same-adapter-hairpin-verified") + + // Phase 2: change app1's port-fwd rule at runtime (2223 -> 2225). + devConfig.UpdateApplication(app1UUID, newPortFwdApp("app1", ni1UUID, app1MAC, 2225)) + device.ApplyConfig(devConfig, false, false) + waitSSHReady(app1UUID) // re-derives the endpoint from the updated config + evetest.Checkpoint("app1-portfwd-switched-to-2225") + + log.Infof("Testing that the old port-fwd (2223) no longer forwards to app1") + _, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).To(HaveOccurred()) + + log.Infof("Testing that the new port-fwd (2225) forwards to app1") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2225) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + // Switch app1's port-fwd rule back to its original port (2225 -> 2223). + devConfig.UpdateApplication(app1UUID, newPortFwdApp("app1", ni1UUID, app1MAC, 2223)) + device.ApplyConfig(devConfig, false, false) + waitSSHReady(app1UUID) + evetest.Checkpoint("app1-portfwd-switched-back-to-2223") + + log.Infof("Testing that the temporary port-fwd (2225) no longer forwards to app1") + _, err = hairpinProbe(app2UUID, eth0IP[0], 2225) + t.Expect(err).To(HaveOccurred()) + + log.Infof("Testing that the original port-fwd (2223) forwards to app1 again") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + // Phase 3: cross-adapter hairpin -- app1 (ni1/ethernet0) reaches app3 + // (ni2/ethernet1) via ethernet1's own external IP. + log.Infof("Testing cross-adapter hairpin: app1 (ethernet0) -> app3's port-fwd (ethernet1)") + out, err = hairpinProbe(app1UUID, eth1IP[0], 2226) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + evetest.Checkpoint("cross-adapter-hairpin-verified") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteApplication(app3UUID) + devConfig.DeleteNetworkInstance(ni1UUID) + devConfig.DeleteNetworkInstance(ni2UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app3Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app3 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + stopApp2Watch() + stopApp3Watch() + + t.Eventually(ni1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni1 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(ni2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni2 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopNI1Watch() + stopNI2Watch() +} + +// TestAirGapSwitchNI verifies that EVE can bridge applications over an +// air-gapped (portless) Switch Network Instance and correctly learns, via +// passive packet snooping (there is no internal DHCP server nor an external +// one here -- the NI has no port at all), the (statically assigned) IP address(es) +// each application VIF is using. See APP-CONNECTIVITY.md, "Switch Network +// Instance" / "IP address detection". +// +// This test additionally covers detecting *multiple* IP addresses recorded +// under the same VIF MAC address, by assigning a second IP directly on the +// same air-gapped interface (no new sub-interface). +// +// Note: EVE is also documented to detect multiple IPs per MAC when an +// application places VLAN sub-interfaces on top of its VIF (they share the +// parent interface's MAC) -- but that path was confirmed NOT to work on a +// Switch NI without VLAN-aware bridge config: the switch-NI ARP-snooping BPF +// filter in pkg/pillar/nistate/linux_flow.go (sniffDNSandDHCP) is a +// hand-maintained raw BPF program compiled from a filter string that never +// accounts for an 802.1Q tag, so tagged ARP frames are dropped by the kernel +// filter before EVE's collector ever sees them (verified with tcpdump on the +// EVE bridge: the tagged ARP requests do arrive, EVE just never reports +// them). Left as a known finding rather than exercised here. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port (ethernet0), used +// only to give each app an SSH-reachable VIF; the actual scenario under +// test runs entirely over the second, air-gapped VIF. +// +// Phases +// ------ +// 1. Config: a Local NI "local-ni" on ethernet0 (10.11.10.0/24, for SSH +// reachability only) and a Switch NI "switch-ni" with Port="" (air-gapped). +// Two container apps (lfedge/evetest-ubuntu-ctr:1.0, VirtualizationMode= +// HVM, EnforceNetIntfOrder=true so vif0=eth0/vif1=eth1 deterministically): +// "app1" and "app2", each with vif0 on "local-ni" (port-fwd +// 2223/2224->22) and vif1 on "switch-ni" (fixed MAC, ICMP allow-all ACL -- +// Switch NI ACLs are still enforced even though the NI itself is L2-only). +// WaitUntilAppIsRunning for both. +// 2. Static IP assignment + real traffic: from inside each app, +// `ip addr add /24 dev eth1` assigns a static IP on the air-gapped +// VIF (11.12.13.11 for app1, .12 for app2) -- no DHCP or +// controller involvement, exactly as APP-CONNECTIVITY.md describes for +// Switch NIs. `ping -c 3 -W 1 -I eth1 ` between the two +// confirms real L2 connectivity over the bridge and generates the ARP +// traffic EVE needs to snoop. +// 3. IP detection: NetworkInstanceInfo for "switch-ni" eventually reports an +// IpAssignment for each app's MAC address containing the IP it just +// configured -- proving EVE learned both addresses purely from ARP +// snooping (no DHCP was ever involved on this NI). +// 4. Multiple IPs per MAC: on app2 only, two more static IPs are added +// directly on eth1 (`ip addr add /24 dev eth1`, same interface, same +// MAC, no sub-interface). A best-effort ping from each new address (the +// target need not answer -- an ARP request alone is enough for EVE to +// learn the Sender IP+MAC pair) generates the ARP traffic. +// NetworkInstanceInfo eventually reports app2's single IpAssignment +// (still keyed by the one shared VIF MAC) now listing all three IP +// addresses. +// 5. Cleanup: delete both apps, then both NIs, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestAirGapSwitchNI(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + localNIUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.10.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.10.2"), + End: evetest.IPAddress("10.11.10.254"), + }, + Gateway: evetest.IPAddress("10.11.10.1"), + MTU: 1500, + }) + switchNIUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "", // air-gapped + MTU: 1500, + }) + localNIUpdates, stopLocalNIWatch := device.WatchNetworkInstanceInfo(localNIUUID) + switchNIUpdates, stopSwitchNIWatch := device.WatchNetworkInstanceInfo(switchNIUUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(localNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "local-ni NI is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + icmpAllowAny := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + const ( + app1MAC = "02:16:3e:00:02:01" + app2MAC = "02:16:3e:00:02:02" + ) + newApp := func(displayName, mac string, + edgeNodePort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnforceNetIntfOrder: true, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: localNIUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: edgeNodePort, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif1", + NetworkInstanceUUID: switchNIUUID, + MAC: evetest.MACAddress(mac), + ACLAllowRules: icmpAllowAny, + }, + }, + } + } + app1UUID := devConfig.AddApplication(newApp("app1", app1MAC, 2223)) + app2UUID := devConfig.AddApplication(newApp("app2", app2MAC, 2224)) + + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(app1UUID) + waitSSHReady(app2UUID) + + const ( + app1IP = "11.12.13.11" + app2IP = "11.12.13.12" + ) + + log.Infof("Assigning static IP %s to app1's air-gapped VIF", app1IP) + _, _, err := device.RunShellScriptInsideApp(app1UUID, appAuth, + fmt.Sprintf("ip addr add %s/24 dev eth1 && ip link set eth1 up", app1IP), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + log.Infof("Assigning static IP %s to app2's air-gapped VIF", app2IP) + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf("ip addr add %s/24 dev eth1 && ip link set eth1 up", app2IP), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + evetest.Checkpoint("static-ips-assigned") + + log.Infof("Testing connectivity over the air-gapped switch NI") + _, _, err = device.RunShellScriptInsideApp(app1UUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 -I eth1 %s", app2IP), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 -I eth1 %s", app1IP), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + // EVE learns statically-assigned IPs on a Switch NI purely from ARP + // snooping (see APP-CONNECTIVITY.md, "IP address detection"). The pings + // above generated the necessary ARP traffic. + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI reports both static IP assignments", + func(info *eveinfo.ZInfoNetworkInstance) bool { + var app1Found, app2Found bool + for _, a := range info.IpAssignments { + switch a.MacAddress { + case app1MAC: + app1Found = generics.ContainsItem(a.IpAddress, app1IP) + case app2MAC: + app2Found = generics.ContainsItem(a.IpAddress, app2IP) + } + } + return app1Found && app2Found + }).StopIf(niHasError))) + evetest.Checkpoint("static-ips-detected") + + // Phase 4: multiple IPs under the same MAC. Two more static IPs are + // added directly on eth1 -- same interface, same MAC, no sub-interface + // (see the doc comment above for why VLAN sub-interfaces are not used + // here). + const ( + extraIP1 = "11.12.16.12" + extraIP2 = "11.12.17.12" + ) + log.Infof("Adding two more static IPs on app2's air-gapped VIF") + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf(`set -e +ip addr add %s/24 dev eth1 +ip addr add %s/24 dev eth1 +set +e +# Best-effort: the targets below need not respond, an outbound ARP request +# alone is enough for EVE to learn the Sender IP+MAC assignment. +ping -c 3 -W 1 -I eth1 11.12.16.99 +ping -c 3 -W 1 -I eth1 11.12.17.99 +true +`, extraIP1, extraIP2), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("extra-ips-added") + + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI reports all three IPs for app2's MAC", + func(info *eveinfo.ZInfoNetworkInstance) bool { + for _, a := range info.IpAssignments { + if a.MacAddress != app2MAC { + continue + } + return generics.ContainsItem(a.IpAddress, app2IP) && + generics.ContainsItem(a.IpAddress, extraIP1) && + generics.ContainsItem(a.IpAddress, extraIP2) + } + return false + }).StopIf(niHasError))) + evetest.Checkpoint("multiple-ips-per-mac-detected") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteNetworkInstance(localNIUUID) + devConfig.DeleteNetworkInstance(switchNIUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + stopApp2Watch() + + t.Eventually(localNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "local-ni NI is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopLocalNIWatch() + stopSwitchNIWatch() +} + +// TestLimitedIPSpace verifies two related IP-allocation edge cases on a +// Local Network Instance whose DHCP range holds only a single free address: +// +// 1. An application deployed onto a NI with no free IP left must fail +// (report an error), not be silently skipped or crash other apps, and +// must be automatically redeployed the moment an IP frees up (i.e. once +// the other application holding it is deleted) -- with no config +// re-apply needed beyond the one that removed the IP-holding app. +// 2. Replacing one application with another *within a single config +// apply* (same NI, competing for the same single free IP) must work: +// EVE has to tear down the obsolete app instance before bringing up the +// replacement, otherwise the replacement would itself fail to get an IP +// (exactly the failure mode from point 1, self-inflicted). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. +// +// Phases +// ------ +// 1. Create a Local NI ("limited-ni") on ethernet0 with subnet +// 10.11.12.0/30. One address is the bridge/gateway (.1) and the DHCP +// range is deliberately restricted to the single remaining host address, +// 10.11.12.2 (Start == End) -- exactly one IP is available for +// applications at any given time. Wait for ONLINE. +// 2. Deploy "app1" on limited-ni (container, no static IP, allow-all ACL). +// It takes the only free IP. WaitUntilAppIsRunning. +// 3. Deploy "app2" on the same NI. There is no free IP left, so zedrouter's +// IPAM must report the allocation failure as an AppErr with a +// description containing "no free IP addresses in DHCP range" -- +// the app stays in whatever SwState it reached (INSTALLED here; this is +// a retryable/pending condition, not a terminal one, so EVE does not +// move the app to ZSwState_ERROR for it). The app2 watch is opened +// *before* applying this config change, since WaitUntilAppIsRunning +// must not be used here -- it treats any ZSwState_ERROR as fatal, and +// more importantly would never observe this particular failure since +// the app's SwState never becomes ZSwState_ERROR. +// 4. Delete app1 (freeing its IP) and re-apply. app1's watch confirms it +// reaches ZSwState_INVALID. Without any further app2-specific config +// change, app2 must automatically pick up the now-free IP and reach +// RUNNING -- proving EVE retries pending IP allocations as soon as one +// becomes available, not just on the next explicit config edit for that +// app. +// 5. Replace app2 with a new application instance within a single config +// apply: DeleteApplication(app2) and AddApplication(same DisplayName, +// same NI) are both applied to the EdgeDeviceConfig before the next +// ApplyConfig call. The replacement gets a fresh UUID with no prior history, +// so WaitUntilAppIsRunning is safe to use and confirms it reaches +// RUNNING -- proving EVE tears down the obsolete instance before deploying +// the replacement, rather than attempting both at once and failing to +// allocate a second IP. +// 6. Cleanup: delete the replacement app instance and the NI, waiting for +// each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestLimitedIPSpace(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "limited-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/30"), + DHCPRange: types.IPRange{ + // Only one host address is available for allocation: .0 is the + // network address, .1 is taken by the bridge, .3 is the + // broadcast address. + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.2"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + niUpdates, stopNIWatch := device.WatchNetworkInstanceInfo(niUUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "limited-ni is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("ni-created") + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + newApp := func(displayName string) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + ACLAllowRules: allowAll, + }, + }, + } + } + + // Step 2: app1 takes the only free IP. + app1UUID := devConfig.AddApplication(newApp("app1")) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + // Step 3: app2 has no free IP to allocate. This surfaces as a persistent + // AppErr on the app while it remains in whatever SwState it reached + // (INSTALLED here, since the image/volume are fine -- only network + // activation is blocked) -- it is a retryable/pending condition, not a + // terminal one, so EVE does not move the app to ZSwState_ERROR for it. + // The watch is opened before applying, and WaitUntilAppIsRunning is + // deliberately not used here -- the no-free-IP AppErr is the expected + // outcome, not a fatal condition. + app2UUID := devConfig.AddApplication(newApp("app2")) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + device.ApplyConfig(devConfig, false, false) + + const expectedErrMsg = "no free IP addresses in DHCP range" + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 fails with a no-free-IP error", + func(info *eveinfo.ZInfoApp) bool { + for _, appErr := range info.AppErr { + if strings.Contains(appErr.GetDescription(), expectedErrMsg) { + return true + } + } + return false + }))) + evetest.Checkpoint("app2-out-of-ip-error") + + // Step 4: delete app1, freeing its IP. app2 must automatically pick it + // up and reach RUNNING, without any app2-specific config change. + devConfig.DeleteApplication(app1UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is RUNNING", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_RUNNING + }).StopIf(appHasError))) + stopApp2Watch() + evetest.Checkpoint("app2-recovered") + + // Step 5: replace app2 with a new application instance within a single + // config apply -- delete the old one and add the replacement before the + // next ApplyConfig call. + devConfig.DeleteApplication(app2UUID) + newApp2UUID := devConfig.AddApplication(newApp("app2")) + newApp2Updates, stopNewApp2Watch := device.WatchAppInfo(newApp2UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(newApp2UUID, timeoutExcludingDownload) + evetest.Checkpoint("app2-replaced") + + // Cleanup. + devConfig.DeleteApplication(newApp2UUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(newApp2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "replacement app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopNewApp2Watch() + + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "limited-ni is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopNIWatch() +} + +// TestFlowLog verifies that EVE produces flow log records and DNS request log +// records for application traffic when flow logging is enabled on a Local +// Network Instance, and that those records correctly attribute flows to ACE +// IDs (allowed flows -> matching ACE; dropped flows -> ACE id 0, the implicit +// reject-all). +// +// ACE numbering: EVE assigns ACE ids sequentially in the order rules are +// listed on the VIF, starting at 1 (id 0 is reserved for the implicit +// reject-all) -- see EdgeDeviceConfig's ACL-building code in devconfig.go. +// The app's single VIF below lists, in order: the port-fwd rule (-> ACE 1), +// the HTTP allow rule (-> ACE 2), the ICMP allow rule (-> ACE 3). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- already has a second HTTP endpoint, +// http-server2.test:8080, used here as the target that is NOT covered +// by any allow rule (so it falls through to the implicit reject-all). +// +// Device configuration +// -------------------- +// - ethernet0 (mgmt+app, DHCP). +// - One Local NI ("flow-ni") with EnableFlowlog=true. +// - One container app on the NI with one VIF and three ACEs: port-fwd +// 2222->22 (ACE 1), allow TCP:80 to http-server.test (ACE 2), allow ICMP +// to the NI's own gateway IP (ACE 3). Everything else (in particular +// http-server2.test:8080) falls through to the implicit reject-all +// (ACE 0). +// +// Phases +// ------ +// 1. Generate distinguishable traffic from inside the app: curl +// http-server.test/helloworld (allowed, ACE 2); curl --max-time 5 +// http-server2.test:8080/helloworld (must fail, ACE 0); ping -c 3 against +// the NI gateway IP (allowed, ACE 3); nslookup http-server.test (DNS +// request log entry). +// 2. Flow log assertions (GetAppFlowLogs, NotBefore the traffic above): EVE +// tracks one flow record per connection (conntrack-based), not one per +// packet direction -- Flow.Src is always the app's own VIF IP and +// Flow.Dest the remote endpoint regardless of direction, so a flow's +// Tx/Rx counters together cover the whole connection. Eventually there is +// an outbound (app-initiated) record matching each of: +// - ACE 2: dst httpServerIP:80/TCP, with TxPkts and RxPkts both > 0 +// (request sent, response received). +// - ACE 3: dst NI gateway/ICMP, with TxPkts and RxPkts both > 0 (ping +// requests sent, replies received). +// - ACE 0 (implicit reject-all): dst httpServer2IP:8080/TCP, with +// TxPkts > 0 (the app did send it) but RxPkts == 0 (blackholed -- +// see APP-CONNECTIVITY.md -- so nothing ever came back). +// Plus one inbound (externally-initiated) record for ACE 1, the +// port-forwarded SSH session already in use to drive the app (src port +// 22 on the app side), with both Tx and Rx counters > 0. +// 3. DNS log assertions (GetAppDNSLogs): eventually there is a DNSRequest +// record for hostname "http-server.test" whose resolved address matches +// the SDN http-server endpoint's IP, with a request time within the +// test window. +// 4. Flow logging disabled: UpdateNetworkInstance sets EnableFlowlog=false; +// re-apply. Fresh traffic is generated, then GetAppFlowLogs (NotBefore +// the reapply instant) consistently returns empty over the reporting +// interval, confirming the runtime toggle actually stops new records +// from being produced (not just that old ones happen to still be there). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestFlowLog(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + // ACE ids, per the numbering scheme documented above. + const ( + portFwdAceID = int32(1) + allowHTTPAceID = int32(2) + allowICMPAceID = int32(3) + denyAceID = int32(0) + ) + // IANA IP protocol numbers, as reported in FlowRecord.Flow.Protocol. + const ( + ipProtoICMP = int32(1) + ipProtoTCP = int32(6) + ) + // SDN endpoint IPs, as defined in netmodels.SingleEthWithDHCP. + const ( + httpServerIP = "10.17.17.25" // http-server.test:80 (allowed) + httpServer2IP = "10.18.18.25" // http-server2.test:8080 (not allowed -> denied) + ) + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + eth0Net := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + const niGatewayIP = "10.50.0.1" + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "flow-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress(niGatewayIP), + EnableFlowlog: true, + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "flowlog-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolTCP, + RemoteHostname: "http-server.test", + RemotePort: 80, + }, + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet(niGatewayIP + "/32"), + }, + }, + }, + }, + }) + + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("app-running") + + log := evetest.Logger() + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "hostname", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(appUUID.String())) + }, 3*time.Minute, polling).Should(Succeed()) + + // Phase 1: generate distinguishable traffic. + log.Infof("Phase 1: generating traffic for each ACE...") + trafficStart := time.Now() + + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("Hello world!")) + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "curl --max-time 5 http://http-server2.test:8080/helloworld", 10*time.Second, 0) + t.Expect(err).To(HaveOccurred(), + "http-server2.test is not covered by any allow ACE and must be blocked") + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "ping -c 3 "+niGatewayIP, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "nslookup http-server.test", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("phase1-traffic-generated") + + // Phase 2: flow log assertions. + // + // EVE tracks one flow record per connection (conntrack-based), not one + // per packet direction: FlowRecord.Inbound says who *initiated* the + // connection, while Flow.Src is always the app's own VIF IP and Flow.Dest + // the remote endpoint, regardless of Inbound -- see + // protoEncodeAppFlowMonitorProto / nistate/linux_flow.go. So the + // app-initiated flows (HTTP, ICMP, the denied HTTP) are all Inbound=false + // with a single record each (already carrying both Tx and Rx counters for + // the whole connection); the port-forwarded SSH session is the only + // Inbound=true flow here, since it was initiated from outside the app. + log.Infof("Phase 2: waiting for flow log records to be reported...") + appInfo := device.GetAppInfo(appUUID) + t.Expect(appInfo.GetNetwork()).To(HaveLen(1)) + t.Expect(appInfo.GetNetwork()[0].GetIPAddrs()).ToNot(BeEmpty()) + appIP := appInfo.GetNetwork()[0].GetIPAddrs()[0] + + // pkg/pillar/nistate/linux.go's flowCollectInterval (~108-120s, randomized, + // hardcoded -- no controller-config override) governs how often the + // conntrack table is swept and a connection's flow record published, so + // give it comfortable room for at least one full cycle plus margin. + flowLogTimeout := 3 * time.Minute + t.Eventually(func(g Gomega) { + outbound := device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: false, + NotBefore: trafficStart, + }) + + httpRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: allowHTTPAceID, srcIP: appIP, dstIP: httpServerIP, + dstPort: 80, protocol: ipProtoTCP, + }) + g.Expect(httpRec).ToNot(BeNil(), + "expected an outbound flow record for the allowed HTTP ACE (%d) "+ + "from %s to %s:80", allowHTTPAceID, appIP, httpServerIP) + g.Expect(httpRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent the HTTP request") + g.Expect(httpRec.GetRxPkts()).To(BeNumerically(">", 0), + "app must have received the HTTP response") + + icmpRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: allowICMPAceID, srcIP: appIP, dstIP: niGatewayIP, protocol: ipProtoICMP, + }) + g.Expect(icmpRec).ToNot(BeNil(), + "expected an outbound flow record for the allowed ICMP ACE (%d) "+ + "from %s to %s", allowICMPAceID, appIP, niGatewayIP) + g.Expect(icmpRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent ping requests") + g.Expect(icmpRec.GetRxPkts()).To(BeNumerically(">", 0), + "app must have received ping replies") + + deniedRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: denyAceID, srcIP: appIP, dstIP: httpServer2IP, + dstPort: 8080, protocol: ipProtoTCP, + }) + g.Expect(deniedRec).ToNot(BeNil(), + "expected an outbound flow record for the denied traffic "+ + "(implicit reject-all, ACE %d) from %s to %s:8080 -- EVE logs "+ + "the flow even though the packet was blackholed", + denyAceID, appIP, httpServer2IP) + g.Expect(deniedRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent connection attempts") + g.Expect(deniedRec.GetRxPkts()).To(BeNumerically("==", 0), + "no reply should ever come back for a blackholed connection") + + inbound := device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: true, + NotBefore: trafficStart, + }) + sshRec := findFlowRecord(inbound, flowRecordMatch{ + aclID: portFwdAceID, inbound: true, srcIP: appIP, protocol: ipProtoTCP, + }) + g.Expect(sshRec).ToNot(BeNil(), + "expected an inbound flow record for the port-forwarded SSH "+ + "session (ACE %d) to app %s", portFwdAceID, appIP) + g.Expect(sshRec.GetFlow().GetSrcPort()).To(Equal(int32(22)), + "the app's own listening port for the port-forwarded connection must be 22") + g.Expect(sshRec.GetTxPkts()).To(BeNumerically(">", 0)) + g.Expect(sshRec.GetRxPkts()).To(BeNumerically(">", 0)) + }, flowLogTimeout, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("phase2-flow-logs-verified") + + // Phase 3: DNS log assertions. + log.Infof("Phase 3: waiting for the DNS request log record...") + t.Eventually(func(g Gomega) { + dnsLogs := device.GetAppDNSLogs(appUUID, evetest.DNSLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + NotBefore: trafficStart, + }) + req := findDNSRequest(dnsLogs, "http-server.test") + g.Expect(req).ToNot(BeNil(), + "expected a DNS request log record for http-server.test") + g.Expect(req.GetAddrs()).To(ContainElement(httpServerIP), + "resolved address must match the SDN http-server endpoint") + }, flowLogTimeout, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("phase3-dns-logs-verified") + + // Phase 4: disable flow logging and confirm no new records appear. + log.Infof("Phase 4: disabling flow logging and generating fresh traffic...") + devConfig.UpdateNetworkInstance(niUUID, evetest.LocalNetworkInstanceConfig{ + DisplayName: "flow-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress(niGatewayIP), + EnableFlowlog: false, + MTU: 1500, + }) + device.ApplyConfig(devConfig, false, false) + reapplyTime := time.Now() + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("phase4-flowlog-disabled") + + t.Consistently(func() []*eveflowlog.FlowRecord { + return device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: false, + NotBefore: reapplyTime, + }) + }, 3*time.Minute, 10*time.Second).Should(BeEmpty(), + "no new flow records should be produced after EnableFlowlog is set to false") + evetest.Checkpoint("phase4-complete") +} + +// flowRecordMatch describes the identifying criteria for a single flow log +// record. aclID and inbound are always checked; srcIP, dstIP, dstPort and +// protocol are only checked when non-zero (dstPort and protocol are left +// unset for ICMP, which has neither a meaningful port nor is worth +// double-checking the protocol number for here). +type flowRecordMatch struct { + aclID int32 + inbound bool + srcIP string // Flow.Src is always the app's own VIF IP, regardless of direction. + dstIP string // Flow.Dest is always the remote endpoint. + dstPort int32 + protocol int32 +} + +// findFlowRecord returns the first record in records matching every +// non-zero-valued field of want, or nil if there is none. +func findFlowRecord(records []*eveflowlog.FlowRecord, want flowRecordMatch) *eveflowlog.FlowRecord { + for _, rec := range records { + if rec.GetAclId() != want.aclID || rec.GetInbound() != want.inbound { + continue + } + flow := rec.GetFlow() + if want.srcIP != "" && flow.GetSrc() != want.srcIP { + continue + } + if want.dstIP != "" && flow.GetDest() != want.dstIP { + continue + } + if want.dstPort != 0 && flow.GetDestPort() != want.dstPort { + continue + } + if want.protocol != 0 && flow.GetProtocol() != want.protocol { + continue + } + return rec + } + return nil +} + +// findDNSRequest returns the first record in records with the given +// hostname, or nil if there is none. +func findDNSRequest(records []*eveflowlog.DnsRequest, hostname string) *eveflowlog.DnsRequest { + for _, rec := range records { + if rec.GetHostName() == hostname { + return rec + } + } + return nil } func niHasError(info *eveinfo.ZInfoNetworkInstance) (string, bool) { diff --git a/evetest/tests/networking/ntp_test.go b/evetest/tests/networking/ntp_test.go index eb2260ab89a..d5dbe5a5f41 100644 --- a/evetest/tests/networking/ntp_test.go +++ b/evetest/tests/networking/ntp_test.go @@ -3,171 +3,464 @@ package networking_test -import "testing" +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/lf-edge/eve/pkg/pillar/utils/generics" +) // TestDeviceNTPConfig verifies how EVE assembles per-port NTP server lists -// from DHCP and statically-configured sources, that chronyd actually -// synchronizes against the configured peers, and that the configuration -// surfaces correctly in published EVE state. +// from DHCP and statically-configured sources, that a per-port exclusive +// override correctly discards the DHCP-provided entry, and that chronyd +// actually synchronizes against the resulting device-wide server set. // // Scope: device side only. Propagation of NTP servers to applications via -// DHCP option 42 is covered separately by TestApplicationNTPConfig (which -// lives in TestApplicationConnectivitySuite and parameterizes the hypervisor). +// DHCP option 42 is covered separately by TestApplicationNTPConfig. // -// SDN framework prerequisite (NOT yet implemented) -// ------------------------------------------------ -// SDN exposes the api.NTPServer endpoint type in grpcapi/proto/sdn.proto -// and the sdnagent parses/validates it (parse.go around the GetNtpServers -// loop), but it does NOT actually deploy an NTP daemon: there is no -// `ntpSrv.go` config item under evetest/sdn/vm/pkg/configitems (compare -// with the existing dnsSrv.go / httpSrv.go / scepSrv.go). The DHCP server -// (dhcpSrv.go) is wired to ADVERTISE an NTP server IP via DHCP option 42 -// / 56, but nothing actually listens on that IP. Before this test can be -// implemented, the SDN side needs an "NTP server endpoint" config item -// that runs a real NTP daemon (chronyd, ntpd, or a small in-process -// stratum-1-like responder) bound to the endpoint's IP. The scenario -// below assumes that work is complete. +// NTP servers used +// ----------------- +// Real, long-stable, single-IP public NTP servers rather than an SDN-hosted +// fake one, so the test needs no SDN-side NTP daemon and can still assert on +// exact addresses (unlike pool.ntp.org-style rotating addresses): +// - 162.159.200.1 (Cloudflare's primary anycast NTP address) -- advertised +// via DHCP option 42 on eth0 (see netmodels.TwoMgmtPortsWithPublicNTP). +// - 216.239.35.0 (Google's time1.google.com address) -- advertised via +// DHCP option 42 on eth1; must be excluded (see below). +// - 162.159.200.123 (Cloudflare's secondary anycast NTP address) -- +// statically configured on both ports (device side). // // Network model // ------------- -// Add netmodel `MultiPortWithNTPServers` (proposed: evetest/netmodels/multi-eth.go) -// extending TwoMgmtPorts with three SDN NTPServer endpoints: -// -// eth0 -- DHCP, DHCP advertises NTP server "ntp0" (private endpoint inside SDN) -// eth1 -- DHCP, DHCP advertises NTP server "ntp1" (different SDN endpoint) -// + "ntp-static" reachable via SDN routing for the static-override case. -// -// All NTP endpoints (api.NTPServer in evetest/grpcapi/proto/sdn.proto) run -// inside SDN and answer NTP queries with a stable clock. They have distinct -// IPs so the test can tell which sources EVE actually uses. +// - netmodels.TwoMgmtPortsWithPublicNTP -- two management ports, each +// advertising a different public NTP server via DHCP option 42. // // Device configuration // -------------------- -// - SystemAdapter for eth0 (DHCP, mgmt) and eth1 (DHCP, mgmt). -// - On eth0, also add a STATIC NTP server in NetworkConfig.NTPServers, e.g. -// "static-ntp.test" (resolved via SDN DNS to the SDN NTPServer endpoint -// "ntp-static"). Leave IgnoreNTPFromDHCP=false so it is appended to -// DHCP-provided servers. -// - On eth1, set IgnoreNTPFromDHCP=true and add a different static entry -// ("static-ntp-2.test" -> NTPServer "ntp-static-2"). This exercises the -// DhcpOptionsIgnore.ntp_server_exclusively code path: only ntp-static-2 -// should appear for eth1, NOT ntp1. -// - Hypervisor: hardcode WithHypervisor=HypervisorKVM in RequireEdgeDevice. -// Device-suite tests do not parameterize the hypervisor (the device-level -// plumbing under test does not depend on app virtualization). -// -// Assertions -// ---------- -// - WatchDeviceInfo / DevicePortStatus.ntpServer + more_ntp_servers: -// - eth0 reports BOTH ntp0 (DHCP) and ntp-static (static append). -// - eth1 reports ONLY ntp-static-2 (the DHCP-supplied ntp1 must be -// filtered out by the exclusive flag). -// - WatchNTPSources (uses the ZInfoNTPSources publication; see -// edgedevice.go GetNTPSources / WatchNTPSources and -// proto/info/ntpsources.proto): -// - Eventually at least one source reaches state SYNC ('*'). -// - The set of source addresses contains every configured NTP IP EVE -// should be using (ntp0, ntp-static, ntp-static-2). ntp1 must NOT -// appear (it was excluded by the exclusive flag). -// - Each tracked source has mode=CLIENT and reachability ramps up -// (consistently within a few minutes). -// - SSH check (acceptable -- chronyd is the canonical timekeeping -// daemon on EVE per CLOCK-SYNCHRONIZATION.md): `chronyc tracking` -// reports a non-zero Reference ID matching one of the configured SDN -// NTPServer IPs. +// - ethernet0 (eth0, mgmt, cost=0): DHCP with the static server appended +// (IgnoreNTPFromDHCP=false). Effective set: {DHCP-provided Cloudflare +// primary, static Cloudflare secondary}. +// - ethernet1 (eth1, mgmt, cost=1): DHCP with IgnoreNTPFromDHCP=true and +// the same static server. The DHCP-provided Google address must be +// discarded. Effective set: {static Cloudflare secondary} only. // -// Runtime update -// -------------- -// - UpdateNetworkAdapter to remove ntp-static from eth0; re-apply. -// - Eventually: -// - DevicePort NTP list shrinks accordingly. -// - WatchNTPSources stops including ntp-static (state may transition -// to UNREACH first; the eventual condition is "source disappears -// or stops being used"). +// Phases +// ------ +// 1. Per-port NTP state: waits (WatchDeviceInfo) until both ports have +// acquired DHCP addresses and DevicePort.ntpServer + more_ntp_servers +// match the expected sets exactly (set equality, catching both missing +// entries and the exclusive-override leaking the DHCP entry through). +// 2. Device-wide chrony synchronization: chronyd has a single global source +// list built from the deduplicated union of every port's NTP servers +// (see pkg/pillar/scripts/device-steps.sh, get_ntp_servers_from_nim), so +// the Google address excluded on eth1 must never reach chronyd either. +// WatchNTPSources (ZInfoNTPSources) eventually reports at least one +// source in SYNC state and a source-address set equal to exactly the two +// Cloudflare addresses. +// 3. SSH cross-check: `eve exec pillar chronyc sources` (chronyd runs inside +// the pillar service container) shows at least one of the two Cloudflare +// addresses marked as the current sync source ('*'). // -// Notes -// ----- -// - The test does not depend on real-world Internet NTP (pool.ntp.org). -// All NTP traffic stays inside SDN. -// - DHCPNetworkConfig already exposes NTPServers + IgnoreNTPFromDHCP and -// the evecommon.DhcpOptionsIgnore plumbing -- no framework additions -// are needed. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDeviceNTPConfig(test *testing.T) { - test.Skip("not yet implemented") + const ( + ntpCloudflarePrimary = "162.159.200.1" // DHCP-advertised on eth0 + ntpCloudflareSecondary = "162.159.200.123" // statically configured on both ports + ) + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPortsWithPublicNTP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + net0 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: net0, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + Cost: 0, + }) + + net1 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + IgnoreNTPFromDHCP: true, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: net1, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + Cost: 1, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + + // Phase 1: per-port NTP state. + log.Infof("Phase 1: waiting for per-port NTP state to settle...") + phase1Timeout := 3 * time.Minute + t.Eventually(devUpdates, phase1Timeout).Should(Receive(matchers.SatisfyPredicate( + "Both ports have expected NTP servers", + func(dinfo *eveinfo.ZInfoDevice) bool { + eth0 := getDevicePort("ethernet0", dinfo) + eth1 := getDevicePort("ethernet1", dinfo) + if eth0 == nil || eth1 == nil { + return false + } + if len(eth0.GetIPAddrs()) == 0 || len(eth1.GetIPAddrs()) == 0 { + return false + } + // eth1's expected set has only the static entry, so this already + // requires the DHCP-provided Google address to be absent (a set + // containing it, or an empty set, would both fail EqualSets here). + return generics.EqualSets(portNtpServers(eth0), + []string{ntpCloudflarePrimary, ntpCloudflareSecondary}) && + generics.EqualSets(portNtpServers(eth1), []string{ntpCloudflareSecondary}) + }))) + evetest.Checkpoint("phase1-complete") + + // Phase 2: device-wide chrony synchronization. + log.Infof("Phase 2: waiting for chronyd to synchronize with the expected NTP servers...") + expectedSources := []string{ntpCloudflarePrimary, ntpCloudflareSecondary} + ntpUpdates, stopNTPWatch := device.WatchNTPSources() + defer stopNTPWatch() + if ntpSourcesSynced(device.GetNTPSources(), expectedSources) { + log.Infof("chronyd is already synchronized with the expected NTP servers") + } else { + phase2Timeout := 5 * time.Minute + t.Eventually(ntpUpdates, phase2Timeout).Should(Receive(matchers.SatisfyPredicate( + "chronyd synchronized with exactly the expected NTP servers", + func(sources *eveinfo.ZInfoNTPSources) bool { + return ntpSourcesSynced(sources, expectedSources) + }))) + } + evetest.Checkpoint("phase2-complete") + + // Phase 3: SSH cross-check via chronyc (chronyd runs inside the pillar + // service container). "-n" disables reverse-DNS lookups so the address + // column shows the numeric IP we configured, not a resolved hostname. + log.Infof("Phase 3: cross-checking via chronyc sources over SSH...") + output, _, err := device.RunShellScript( + "eve exec pillar chronyc -n sources", 15*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + foundSyncedLine := false + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "^*") && + (strings.Contains(line, ntpCloudflarePrimary) || + strings.Contains(line, ntpCloudflareSecondary)) { + foundSyncedLine = true + break + } + } + t.Expect(foundSyncedLine).To(BeTrue(), + "chronyc sources must show one of the expected Cloudflare addresses "+ + "as the current sync source ('^*'):\n%s", output) + evetest.Checkpoint("phase3-complete") +} + +// portNtpServers combines DevicePort.ntpServer and more_ntp_servers into a +// single slice (or nil if the port has no NTP servers configured). +func portNtpServers(port *eveinfo.DevicePort) []string { + if port.GetNtpServer() == "" { + return nil + } + return append([]string{port.GetNtpServer()}, port.GetMoreNtpServers()...) +} + +// ntpSourcesSynced reports whether sources contains at least one source in +// SYNC state and its set of source addresses equals exactly expectedAddrs. +func ntpSourcesSynced(sources *eveinfo.ZInfoNTPSources, expectedAddrs []string) bool { + if sources == nil { + return false + } + var addrs []string + synced := false + for _, src := range sources.GetSources() { + addrs = append(addrs, src.GetDstAddr()) + if src.GetState() == eveinfo.NTPSourceState_NTP_SOURCE_STATE_SYNC { + synced = true + } + } + return synced && generics.EqualSets(addrs, expectedAddrs) } // TestApplicationNTPConfig verifies that the per-NI DHCP server propagates -// the correct NTP server list (port-NTP ∪ NI-NTP) to applications via DHCP -// option 42 / 56, and that the application's NI VIF status reflects this in -// published EVE state. +// the correct NTP server list (port-NTP union NI-NTP) to an application, and +// that the application's NI VIF status reflects this in published EVE state. // // Scope: application side only. Device-side NTP plumbing is covered by // TestDeviceNTPConfig. // -// SDN framework prerequisite (NOT yet implemented) -// ------------------------------------------------ -// Same prerequisite as TestDeviceNTPConfig: SDN must actually run an NTP -// daemon bound to each `api.NTPServer` endpoint's IP. The proto/API exists -// and the sdnagent validates the config, but there is no NTP server config -// item that actually listens on the wire (no `ntpSrv.go` under -// evetest/sdn/vm/pkg/configitems). The DHCP server can only advertise an -// NTP server IP via DHCP option 42; nothing answers on that IP today. -// In-app chronyc would therefore never see a SYNC source. This test cannot -// be implemented until that gap is filled. +// Known limitation: EVE's per-NI dnsmasq does advertise the NTP list via +// DHCP option 42 (the same mechanism that carries DNS servers), but the +// shim-VM's DHCP client script (pkg/xen-tools/initrd/udhcpc_script.sh) only +// acts on the DNS/IP/route options it receives -- it never writes the NTP +// option (udhcpc's "$ntpsrv") anywhere inside the guest. So unlike DNS +// servers (verifiable via nslookup from inside the app), there is currently +// no guest-visible effect of the DHCP-advertised NTP list to check over SSH; +// this test can only verify what EVE itself reports it configured +// (ZInfoNetwork.ntp_servers), which is exactly the per-NI DHCP server's +// computed advertisement. +// +// NTP servers used +// ----------------- +// Reuses the same real public NTP servers as TestDeviceNTPConfig, plus one +// more for the NI-level override: +// - 162.159.200.1 (Cloudflare primary) -- DHCP-advertised on eth0. +// - 162.159.200.123 (Cloudflare secondary) -- statically configured on +// eth0. +// - 216.239.35.0 (Google time1) -- DHCP-advertised on eth1 (unrelated port; +// must not leak into the app). +// - 216.239.35.4 (Google time2) -- configured on the Local NI itself. // // Network model // ------------- -// Reuse the same `MultiPortWithNTPServers` netmodel proposed for -// TestDeviceNTPConfig. Add one more SDN NTPServer endpoint "ntp-local-ni" -// that the NI itself will advertise to apps in addition to the port-level -// NTP servers. +// - netmodels.TwoMgmtPortsWithPublicNTP (same model as TestDeviceNTPConfig). // // Device configuration // -------------------- -// - SystemAdapter for eth0 (DHCP, mgmt+app, with static NTP "ntp-static" -// appended) and eth1 (DHCP, mgmt, with exclusive override -- only -// ntp-static-2 used). Same plumbing as TestDeviceNTPConfig. -// - One Local NI ("local-ni") on eth0 with its own NIConfig.NTPServers -// entry pointing to "ntp-local-ni". The NI's dnsmasq must merge port -// NTP servers and the NI NTP server and advertise the union to apps. -// - One container app on the NI (default-allow + port-fwd 2222->22). -// Use the existing lfedge/evetest-ubuntu-ctr image; it ships -// with chronyd which will sync against whatever DHCP delivers. +// - ethernet0 (eth0, mgmt+app, cost=0): DHCP with the Cloudflare secondary +// server appended. Effective port set: {Cloudflare primary (DHCP), +// Cloudflare secondary (static)}. +// - ethernet1 (eth1, mgmt only): plain DHCP, left unexcluded (Google time1 +// stays active on this port), but not part of any NI -- present purely +// to prove a port's NTP servers don't leak into an app bound to a +// different port. +// - One Local NI ("local-ni") on eth0 with its own NTPServers entry +// pointing at the Google time2 address. +// - One container app on the NI (default-allow ACL + port-fwd 2222->22), +// using the lfedge/evetest-ubuntu-ctr image. // -// Assertions -// ---------- -// - WaitUntilAppIsRunning(appUUID). -// - WatchAppInfo: the app VIF's ZInfoNetwork.ntp_servers reports the -// EXPECTED set: ntp0, ntp-static (from the port), plus ntp-local-ni -// (from the NI itself). ntp1 / ntp-static-2 must NOT appear (they -// belong to eth1, which is not used by this NI). Use set equality so -// accidental over-reporting is caught. -// - SSH inside the app: `chronyc sources` reports the same set of -// servers. Optionally `cat /etc/chrony.conf` to confirm the DHCP-driven -// configuration was applied. (chronyd-in-container is supported on the -// evetest-ubuntu-ctr image; see CLOCK-SYNCHRONIZATION.md -// "Recommendations when NTP is not available on the Guests".) -// -// Runtime update -// -------------- -// - UpdateNetworkInstance to remove ntp-local-ni from the NI's NTP list. -// To force the app to pick up the new DHCP option without redeploying, -// either DeactivateApplication + ActivateApplication to trigger a fresh -// DHCP cycle, or wait for the DHCP lease renewal. The first approach -// is faster and is what the eden tests effectively do. -// - After re-acquire: ZInfoNetwork.ntp_servers no longer includes -// ntp-local-ni; the in-app chronyc sources reflects the same change. +// Phases +// ------ +// 1. WaitUntilAppIsRunning, then wait (WatchAppInfo) until the app's VIF +// receives an IP from the NI subnet and ZInfoNetwork.ntp_servers equals +// exactly the port union NI set: {Cloudflare primary, Cloudflare +// secondary, Google time2}. Set equality (not subset) also catches +// eth1's Google time1 address leaking through, which must not happen +// since the NI doesn't use eth1. +// 2. Runtime update: removes the NI's NTPServers entry (UpdateNetworkInstance) +// and deactivates/reactivates the app to force a fresh DHCP cycle. +// Eventually ZInfoNetwork.ntp_servers drops the Google time2 address, +// leaving only the two eth0 port-level servers. // // Test params // ----------- -// - HYPERVISOR. The test must call evetest.SkipIfHypervisorKubevirt() -// after reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // -// Notes -// ----- -// - All NTP traffic stays inside SDN; no public pool.ntp.org dependency. -// - LocalNetworkInstanceConfig already exposes NTPServers in the framework, -// so no devconfig changes are required. +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. func TestApplicationNTPConfig(test *testing.T) { - test.Skip("not yet implemented") + const ( + ntpCloudflarePrimary = "162.159.200.1" // DHCP-advertised on eth0 + ntpCloudflareSecondary = "162.159.200.123" // statically configured on eth0 + ntpGoogleTime2 = "216.239.35.4" // configured on the NI itself + ) + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPortsWithPublicNTP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + net0 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: net0, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + // ethernet1 is not used by the NI below; it is configured (with its own + // DHCP-advertised NTP server left active, unexcluded) purely to prove + // that a port's NTP servers do not leak into an app whose NI is bound to + // a different port. + net1 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: net1, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niSubnet := evetest.IPSubnet("10.11.12.0/24") + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: niSubnet, + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + NTPServers: []string{ntpGoogleTime2}, + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "ntp-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + defer stopAppWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + timeout := 5 * time.Minute + + log.Infof("Waiting for the app to become running...") + device.WaitUntilAppIsRunning(appUUID, timeout) + evetest.Checkpoint("app-running") + + // Phase 1: app VIF NTP server set. + log.Infof("Phase 1: waiting for the app VIF to report the expected NTP servers...") + expectedNTPServers := []string{ntpCloudflarePrimary, ntpCloudflareSecondary, ntpGoogleTime2} + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "App VIF has an IP from the NI subnet and the expected NTP server set", + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) != 1 || len(info.Network[0].IPAddrs) == 0 { + return false + } + return generics.EqualSets(info.Network[0].GetNtpServers(), expectedNTPServers) + }).StopIf(appHasError))) + evetest.Checkpoint("phase1-complete") + + // Phase 2: runtime update -- remove the NI-level NTP server and force a + // fresh DHCP cycle by deactivating and reactivating the app. + log.Infof("Phase 2: removing the NI's NTP server and forcing a fresh DHCP cycle...") + devConfig.UpdateNetworkInstance(niUUID, evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: niSubnet, + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + device.ApplyConfig(devConfig, false, false) + device.DeactivateApplication(appUUID, true, timeout) + device.ActivateApplication(appUUID, true, timeout) + + expectedNTPServersAfterUpdate := []string{ntpCloudflarePrimary, ntpCloudflareSecondary} + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "App VIF NTP servers no longer include the removed NI-level server", + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) != 1 || len(info.Network[0].IPAddrs) == 0 { + return false + } + return generics.EqualSets(info.Network[0].GetNtpServers(), expectedNTPServersAfterUpdate) + }).StopIf(appHasError))) + evetest.Checkpoint("phase2-complete") } diff --git a/evetest/tests/networking/pciback_error_test.go b/evetest/tests/networking/pciback_error_test.go index f94446431a1..b2cd68252b8 100644 --- a/evetest/tests/networking/pciback_error_test.go +++ b/evetest/tests/networking/pciback_error_test.go @@ -4,7 +4,6 @@ package networking_test import ( - "encoding/json" "strings" "testing" "time" @@ -17,6 +16,7 @@ import ( "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/matchers" "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" ) // This suite verifies that device-model inconsistencies in the assignable-I/O @@ -73,23 +73,8 @@ func newBaseNetConfig(devName string) *evetest.EdgeDeviceConfig { func portPciLong(t *WithT, device *evetest.EdgeDevice, phylabel string) string { var pci string t.Eventually(func() string { - stdout, _, err := device.RunShellScript( - "eve exec pillar cat /run/domainmgr/AssignableAdapters/global.json 2>/dev/null", - 60*time.Second, 0) - if err != nil { - return "" - } - i := strings.IndexByte(stdout, '{') - if i < 0 { - return "" - } - var aa struct { - IoBundleList []struct { - Phylabel string - PciLong string - } - } - if json.Unmarshal([]byte(stdout[i:]), &aa) != nil { + var aa pillartypes.AssignableAdapters + if !evetest.ReadPublication(device, "domainmgr", false, "global", &aa) { return "" } for _, b := range aa.IoBundleList { @@ -124,10 +109,15 @@ func TestReportMissingDevice(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -146,6 +136,9 @@ func TestReportMissingDevice(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "phantom device with a non-existent PCI address is reported as an error", @@ -164,10 +157,15 @@ func TestReportParentAssigngrp(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -187,6 +185,9 @@ func TestReportParentAssigngrp(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "self-parent assignment group is reported as an error", @@ -209,10 +210,15 @@ func TestReportCycleDetected(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -240,6 +246,9 @@ func TestReportCycleDetected(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "parentassigngrp cycle is reported as an error", @@ -258,10 +267,15 @@ func TestReportCollision(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -283,6 +297,9 @@ func TestReportCollision(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "USB collision is reported once as an error for the group", @@ -303,10 +320,15 @@ func TestReportIfnameMismatch(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -318,6 +340,9 @@ func TestReportIfnameMismatch(test *testing.T) { // Phase 1: correct config; let EVE resolve eth1's real PCI address. device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") // Phase 2: give eth1 a bogus model interface name but the real PCI address. @@ -340,10 +365,15 @@ func TestReportAssignmentGroupConflict(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -353,6 +383,9 @@ func TestReportAssignmentGroupConflict(test *testing.T) { defer stopDevWatch() device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") cfg := newBaseNetConfig(pcibackErrDevName) @@ -384,10 +417,15 @@ func TestReportWarningPlusError(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -397,6 +435,9 @@ func TestReportWarningPlusError(test *testing.T) { defer stopDevWatch() device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") // eth1 gets a wrong ifname (warning) and a phantom shares its PCI in another @@ -434,10 +475,15 @@ func TestReportClearsOnFix(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -463,6 +509,9 @@ func TestReportClearsOnFix(test *testing.T) { // Break it: self-parent -> error reported. device.ApplyConfig(phantomParentConfig("grpx"), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "self-parent assignment group is reported as an error", func(info *eveinfo.ZInfoDevice) bool { @@ -524,10 +573,15 @@ func TestReportWarningsOnly(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.ManyDNSServers}, @@ -538,6 +592,9 @@ func TestReportWarningsOnly(test *testing.T) { // Phase 1: correct config; resolve eth1 and eth2 real PCI addresses. device.ApplyConfig(manyPortsConfig(pcibackErrDevName, "eth1", "", "eth2", ""), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci1 := portPciLong(t, device, "eth1") pci2 := portPciLong(t, device, "eth2") diff --git a/evetest/tests/networking/routing_test.go b/evetest/tests/networking/routing_test.go index c838fefa218..48398ddd22c 100644 --- a/evetest/tests/networking/routing_test.go +++ b/evetest/tests/networking/routing_test.go @@ -96,8 +96,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestPropagatedRoutes(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -105,7 +104,6 @@ func TestPropagatedRoutes(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -164,6 +162,9 @@ func TestPropagatedRoutes(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // ni-eth0: PropagateConnectedRoutes=true so the eth0 port subnet (172.22.12.0/24) // is delivered to the app. Static route to http-server-0's subnet. @@ -474,8 +475,7 @@ func TestPropagatedRoutes(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestLocalNIWithMultiplePorts(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -483,7 +483,6 @@ func TestLocalNIWithMultiplePorts(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -568,6 +567,9 @@ func TestLocalNIWithMultiplePorts(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Local NI spanning all 4 ports (port="all"). // Static routes use shared labels with probing: @@ -916,8 +918,7 @@ func findRoute(routes []*eveinfo.IPRoute, dst string) *eveinfo.IPRoute { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestApplicationGateway(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -925,7 +926,6 @@ func TestApplicationGateway(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -966,6 +966,9 @@ func TestApplicationGateway(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // ni-eth0: Local NI on eth0 — used by app-client1 for its default route // and SSH access (portfwd 2222→22). @@ -1376,7 +1379,427 @@ func TestApplicationGateway(test *testing.T) { t.Expect(out).To(ContainSubstring("MASQUERADE_ACTIVE:")) } -// TestMgmtTrafficRoutedViaApp : TODO replicate github.com/lf-edge/eden/examples/mgmt-over-app +// TestMgmtTrafficRoutedViaApp verifies that EVE's own device-management +// traffic (not just application traffic) can be routed through an +// application acting as a NAT gateway, and that this arrangement survives a +// full device reboot. +// +// Topology +// -------- +// +// +------------+ +// | controller | +// +------------+ +// | +// | +// +--------------------+ +-----------+ +---------------------+ +// | eth0 (app-shared) |---| ni-wan |---| mgmt-gw-app | +// | Switch NI, WAN | | (Switch) | | (vif0) 10.60.10.150 | +// +--------------------+ +-----------+ | ^ MASQUERADE | +// | | + forwarding | +// | v | +// +--------------------+ +-----------+ | (vif1) | +// | eth1 (management) |---| ni-lan |---| 10.60.20.150 | +// | static IP, gw=app | | (Switch) | +---------------------+ +// +--------------------+ +-----------+ +// EVE: 10.60.20.5 +// +// Network model +// ------------- +// - netmodels.MgmtViaAppTopology -- two ports, each hosting a Switch NI: +// eth0/wan-network (10.60.10.0/24) is fully reachable (controller, +// dns-server) and hands out a DHCP static reservation for the app's WAN +// VIF MAC -> 10.60.10.150; eth1/lan-network (10.60.20.0/24) has no +// outside reachability and suppresses the DHCP router option +// (WithoutDefaultRoute) -- it only provides L2 connectivity between +// EVE's own static IP and the app's LAN VIF (reserved -> 10.60.20.150). +// +// Phases +// ------ +// 1. Device config (safe state): both eth0 and eth1 start out as ordinary +// PhyIoUsageMgmtAndApps DHCP ports, so EVE keeps a normal, working +// controller path via eth0 while the gateway app is brought up. +// Two Switch NIs are created: ni-wan on ethernet0, ni-lan on ethernet1, +// and one container app with vif0 on ni-wan (MAC 02:16:3e:02:00:00) +// and vif1 on ni-lan (MAC 02:16:3e:02:00:01), both with default-allow ACLs. +// 2. NI/app readiness: both NIs reach ONLINE, WaitUntilAppIsRunning +// succeeds, and WatchAppInfo confirms vif0=10.60.10.150 and +// vif1=10.60.20.150 (the SDN's static MAC reservations). +// 3. Gateway setup: `ip route` must already show a single default route +// via ni-wan (10.60.10.1) -- ni-lan's WithoutDefaultRoute keeps vif1 +// from contributing one. IP forwarding and +// `POSTROUTING -o eth0 MASQUERADE` are then enabled on the WAN VIF. +// 4. Migrate EVE's management path onto the app: eth0 is reconfigured to +// PhyIoUsageShared with no EVE-side IP (the app's vif0 is now the only +// address on that segment), and eth1 is reconfigured from DHCP to a +// static IP (10.60.20.5/24) whose gateway is set to the app's LAN VIF +// (10.60.20.150) instead of the SDN's own router address -- so all of +// EVE's own outbound traffic (DNS, controller) that isn't for a +// directly-connected subnet is sent to the app, NATed, and exits via +// eth0. WatchDeviceInfo must report a fresh ZInfoDevice update within a +// bounded timeout (proving the new path works), and GetState() must be +// ONLINE. +// 5. Firewall restriction: UpdateNetworkModel adds a Firewall rule set that +// allows both controller and dns-server (10.16.16.25) access only from +// the app's WAN IP (10.60.10.150) and drops each from every other +// source. Two consecutive fresh-info waits (each well under the ~5-minute +// threshold before EVE would report itself SUSPECT) confirm sustained, +// not just momentary, connectivity -- the decisive proof that all of this +// traffic really is sourced from the app's IP, since any other path would +// now be dropped by the SDN firewall. +// 6. Reboot via the controller: RequestReboot is issued without waiting +// (waiting would deadlock on the very SSH-driven step needed to bring +// the path back up), since a full device reboot also restarts the +// gateway app's container, discarding its MASQUERADE setup. The test +// polls SSH connectivity to the app's WAN VIF directly (independent of +// EVE's own management path) until the fresh container responds, then +// reapplies the same IP-forwarding + MASQUERADE commands. Only then +// does WatchDeviceInfo wait for a ZInfoDevice with LastRebootTime newer +// than the reboot request, confirming EVE is back online via the +// app-routed management path after a full reboot. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestMgmtTrafficRoutedViaApp(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters(evetest.HypervisorParameter()) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.MgmtViaAppTopology, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Lower the periodic device-info publish interval so a bounded "a fresh + // ZInfoDevice arrives" wait can serve as a direct, real-time signal that + // EVE is still getting through to the controller (see + // TestIntermittentConnectivity for the same rationale). + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, 30) + devConfig.SetConfigProperties(cfgProps) + + const ( + wanAppIP = "10.60.10.150" // SDN static reservation, wan-network + lanAppIP = "10.60.20.150" // SDN static reservation, lan-network + wanGateway = "10.60.10.1" + lanDeviceIP = "10.60.20.5" + vifWanMAC = "02:16:3e:02:00:00" + vifLanMAC = "02:16:3e:02:00:01" + dnsServerIP = "10.16.16.25" // netmodels.MgmtViaAppTopology's dns-server endpoint + ) + + // Phase 1: safe starting state -- both ports are ordinary + // PhyIoUsageMgmtAndApps DHCP ports. eth0 gives EVE a normal, working + // controller path while the gateway app is brought up; eth1's DHCP + // attempt never succeeds as a DPC (lan-network has no outside + // reachability), which is harmless. + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + eth1Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("safe-port-config-applied") + + // Two Switch NIs -- one per port -- and the gateway app connected to both. + niWanUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "ni-wan", + Port: "ethernet0", + MTU: 1500, + }) + niLanUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "ni-lan", + Port: "ethernet1", + MTU: 1500, + }) + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "mgmt-gw-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnforceNetIntfOrder: true, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niWanUUID, + MAC: evetest.MACAddress(vifWanMAC), + ACLAllowRules: allowAll, + }, + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif1", + NetworkInstanceUUID: niLanUUID, + MAC: evetest.MACAddress(vifLanMAC), + ACLAllowRules: allowAll, + }, + }, + }) + + niWanUpdates, stopNIWanWatch := device.WatchNetworkInstanceInfo(niWanUUID) + niLanUpdates, stopNILanWatch := device.WatchNetworkInstanceInfo(niLanUUID) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + device.ApplyConfig(devConfig, false, false) + + niTimeout := 3 * time.Minute + t.Eventually(niWanUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "ni-wan is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + stopNIWanWatch() + + t.Eventually(niLanUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "ni-lan is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + stopNILanWatch() + + evetest.Checkpoint("nis-online") + + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("app-running") + + var appInfo *eveinfo.ZInfoApp + t.Eventually(appUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "app reports 2 VIFs with the reserved IPs", + func(info *eveinfo.ZInfoApp) bool { + appInfo = info + if len(info.Network) != 2 { + return false + } + for _, vif := range info.Network { + if len(vif.IPAddrs) == 0 { + return false + } + } + return true + }).StopIf(appHasError))) + stopAppWatch() + + t.Expect(appInfo.Network[0].IPAddrs).To(ContainElement(wanAppIP)) + t.Expect(appInfo.Network[1].IPAddrs).To(ContainElement(lanAppIP)) + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + // Phase 3: gateway setup, over SSH via ni-wan's IP directly (Switch NI + // VIFs are reachable from the evetest host without a port-forward). + log.Infof("Waiting for gateway app SSH...") + var routes string + t.Eventually(func(gt Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "ip route", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(out).To(ContainSubstring("default via " + wanGateway)) + routes = out + }, 5*time.Minute, polling).Should(Succeed()) + // ni-lan's WithoutDefaultRoute keeps vif1 from also contributing a + // default route -- there must be exactly one, via the WAN leg. + t.Expect(routes).To(ContainSubstring("default via " + wanGateway)) + + evetest.Checkpoint("app-ssh-ready") + + configureGateway := func() { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "sysctl -w net.ipv4.ip_forward=1; "+ + "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE", + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + } + log.Infof("Configuring gateway app: IP forwarding + MASQUERADE on eth0 (WAN)") + configureGateway() + evetest.Checkpoint("app-gateway-configured") + + // Phase 4: migrate EVE's own management path onto the app. eth0 loses + // its EVE-side IP (app-shared only); eth1 switches from DHCP to a + // static IP whose gateway is the app's LAN VIF instead of the SDN's own + // router address. + log.Infof("Phase 4: migrating EVE's management path via the gateway app...") + devConfig.UpdateNetwork(eth0Net, evetest.NoIPNetworkConfig{}) + devConfig.UpdateNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageShared, + }) + devConfig.UpdateNetwork(eth1Net, evetest.StaticNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + Subnet: evetest.IPSubnet("10.60.20.0/24"), + Gateway: evetest.IPAddress(lanAppIP), + DNSServers: []net.IP{evetest.IPAddress(dnsServerIP)}, + }) + devConfig.UpdateNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + StaticIP: evetest.IPAddress(lanDeviceIP), + }) + // waitUntilConfirmed is deliberately left false: this config changes the + // management port, and EVE may not be able to publish metrics again + // until the app-routed path above is actually working. + device.ApplyConfig(devConfig, true, false) + + // infoTimeout bounds how long a fresh ZInfoDevice update may take once + // the app-routed path takes over (DevInfoInterval was lowered to 30s + // above). + infoTimeout := 3 * time.Minute + waitForFreshInfo := func(reason string) { + drainBacklog: + for { + select { + case <-devUpdates: + default: + break drainBacklog + } + } + log.Infof("Waiting for a fresh device info update (%s)...", reason) + t.Eventually(devUpdates, infoTimeout).Should(Receive(), + "EVE should get device info through to the controller via the "+ + "app-routed management path (%s)", reason) + } + waitForFreshInfo("mgmt path migrated to the gateway app") + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("mgmt-via-app-active") + + // Phase 5: restrict the SDN firewall so both the controller and the DNS + // server are reachable only from the app's WAN IP -- covering DNS too + // closes off the obvious "cheat" of resolving the controller hostname + // (or anything else) via some other, non-app-routed path while still + // routing the actual controller connection through the app. Sustained + // (not just momentary) connectivity from here on is the decisive proof + // that all of this traffic is genuinely sourced from the app, since any + // other path is now dropped. + log.Infof("Phase 5: restricting controller and DNS access to the app's WAN IP...") + restrictedModel := proto.Clone(netmodels.MgmtViaAppTopology).(*api.NetworkModel) + restrictedModel.Firewall = &api.Firewall{ + Rules: []*api.FwRule{ + { + SrcSubnet: wanAppIP + "/32", + DstSubnet: evetest.GetControllerIPv4().String() + "/32", + Action: api.FwAction_FW_ALLOW, + }, + { + SrcSubnet: "0.0.0.0/0", + DstSubnet: evetest.GetControllerIPv4().String() + "/32", + Action: api.FwAction_FW_DROP, + }, + { + SrcSubnet: wanAppIP + "/32", + DstSubnet: dnsServerIP + "/32", + Action: api.FwAction_FW_ALLOW, + }, + { + SrcSubnet: "0.0.0.0/0", + DstSubnet: dnsServerIP + "/32", + Action: api.FwAction_FW_DROP, + }, + }, + } + evetest.UpdateNetworkModel(restrictedModel) + defer evetest.UpdateNetworkModel(netmodels.MgmtViaAppTopology) + + waitForFreshInfo("firewall restricted, check 1/2") + waitForFreshInfo("firewall restricted, check 2/2") + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("firewall-restricted") + + // Phase 6: reboot via the controller. RequestReboot is issued without + // waiting: a full device reboot also restarts the gateway app's + // container, discarding its MASQUERADE setup, so waiting here would + // deadlock on the very SSH-driven step below that brings the + // management path back up. + // + // /proc/sys/kernel/random/boot_id is captured beforehand so the + // post-reboot poll can tell a genuinely fresh container (running under + // a rebooted shim VM/kernel) apart from the pre-reboot instance still + // answering SSH: RequestReboot only requests a reboot -- it does not + // wait for it -- so an early poll attempt could otherwise reach the + // old, not-yet-rebooted container and report success prematurely. + preRebootBootID, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "cat /proc/sys/kernel/random/boot_id", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + log.Infof("Phase 6: triggering reboot via the controller...") + rebootIssuedAt := time.Now() + device.RequestReboot(false) + + // Poll SSH connectivity to the app's WAN VIF directly -- independent of + // EVE's own management path -- until boot_id differs from the + // pre-reboot value, proving the container is genuinely running under a + // fresh boot. Then reconfigure it (container state does not survive a + // full device reboot). + log.Infof("Waiting for the gateway app to come back up after reboot...") + t.Eventually(func(gt Gomega) { + bootID, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "cat /proc/sys/kernel/random/boot_id", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(bootID).ToNot(Equal(preRebootBootID), + "still answering as the pre-reboot instance") + }, 10*time.Minute, polling).Should(Succeed()) + configureGateway() + evetest.Checkpoint("app-gateway-reconfigured-post-reboot") + + rebootTimeout := 10 * time.Minute + t.Eventually(devUpdates, rebootTimeout).Should(Receive(matchers.SatisfyPredicate( + "device reports a fresh reboot via the app-routed management path", + func(info *eveinfo.ZInfoDevice) bool { + ts := info.GetLastRebootTime() + return ts != nil && ts.AsTime().After(rebootIssuedAt) + }))) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("device-back-online-post-reboot") } diff --git a/evetest/tests/networking/stp_test.go b/evetest/tests/networking/stp_test.go index f73c8d502ab..e71685dd8ab 100644 --- a/evetest/tests/networking/stp_test.go +++ b/evetest/tests/networking/stp_test.go @@ -119,8 +119,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestSwitchNIWithMultiplePorts(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -130,8 +129,6 @@ func TestSwitchNIWithMultiplePorts(test *testing.T) { evetest.HypervisorParameter(), ) hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ @@ -191,6 +188,9 @@ func TestSwitchNIWithMultiplePorts(test *testing.T) { // Apply the base adapter configuration. device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Add Switch NI with all three app-shared ports. // BPDU guard is enabled on eth3 ("edge-port" label). diff --git a/evetest/tests/networking/testsuite_test.go b/evetest/tests/networking/testsuite_test.go index a3a94b3a2a9..08c16d83346 100644 --- a/evetest/tests/networking/testsuite_test.go +++ b/evetest/tests/networking/testsuite_test.go @@ -34,12 +34,15 @@ import ( // SDN-side LACP peer requires the bond to be configured before EVE // ever transmits, hence bootstrap-only path. // -// All bootstrap tests hardcode WithHypervisor=HypervisorKVM and do not -// parameterize the hypervisor. +// All bootstrap tests share the HYPERVISOR parameter (defaults to KVM). func TestBootstrapSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + // This below will be implemented using t.Run() // Note that evetest.Close needs to behave differently when test is part of // a test suite and there are more tests to execute. @@ -172,11 +175,8 @@ func TestBootstrapSuite(test *testing.T) { // TestDeviceConnectivitySuite drives every device-side networking // scenario: how EVE itself manages its physical / L2 / IP adapters and -// keeps controller connectivity alive. None of the subtests deploy an -// application -- they focus on the EVE control plane only -- and therefore -// none parameterize the hypervisor (all hardcode HypervisorKVM via the -// shared deviceRequirementsForNetAdapterTests / deviceRequirementsForBootstrap -// helpers or directly). +// keeps controller connectivity alive. +// All subtests share the HYPERVISOR parameter (defaults to KVM). // // Subtests // -------- @@ -190,9 +190,12 @@ func TestBootstrapSuite(test *testing.T) { // - TestPortFailover / TestNetworkConfigFallback / // TestIntermittentConnectivity -- fail-over / fallback resilience // (currently stub scenarios). +// - TestMgmtTrafficRoutedViaApp -- EVE's own device-management traffic +// routed through an app-based NAT gateway, surviving a full reboot. // - TestDeviceIPv6Connectivity -- IPv6-only device side (stub scenario). -// - TestDeviceNTPConfig -- per-port NTP server propagation to EVE's -// chrony (stub scenario; needs SDN-side NTP daemon). +// - TestDeviceNTPConfig -- per-port NTP server aggregation (DHCP + static, +// exclusive-override) and chronyd synchronization against real public +// NTP servers. // - TestActiveBackupBond / TestLACPBond -- bond status, failover, // LACP negotiation. // - TestVLANSubinterfaces / TestVLANSubinterfacesOnTopOfLAGs -- VLAN @@ -203,6 +206,10 @@ func TestDeviceConnectivitySuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{ Test: TestPNAC, @@ -239,6 +246,9 @@ func TestDeviceConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestIntermittentConnectivity, }, + evetest.TestCase{ + Test: TestMgmtTrafficRoutedViaApp, + }, evetest.TestCase{ Test: TestDeviceIPv6Connectivity, }, @@ -270,14 +280,21 @@ func TestDeviceConnectivitySuite(test *testing.T) { // networking scenario. All subtests deploy at least one application and // therefore share the HYPERVISOR parameter -- the suite declares // evetest.HypervisorParameter() once and every subtest reads it via -// evetest.GetHypervisorParameterValue(). Each subtest also calls -// evetest.SkipIfHypervisorKubevirt() right after reading the value: -// Kubevirt is reserved for cluster tests under evetest/tests/cluster. +// evetest.GetHypervisorParameterValue() (defaults to KVM). // // Subtests // -------- // - TestLocalNI / TestSwitchNI -- canonical Local-NI / Switch-NI // life-cycles plus connected-app smoke tests. +// - TestNIReplace -- rapid NI delete+recreate (and subnet reuse across +// NIs) within single config applies. +// - TestMoveAppBetweenNIs -- moving an app's VIF from one Local NI to +// another at runtime. +// - TestAirGapSwitchNI -- portless Switch NI IP-detection via packet +// snooping: statically-assigned IPs and multiple IPs per VIF MAC +// (e.g. via VLAN sub-interfaces). +// - TestLimitedIPSpace -- app deployment/replacement on a NI with only a +// single free IP address available. // - TestFlowLog -- per-app flow log + DNS log assertions // (skipped; depends on GetAppFlowLogs / GetAppDNSLogs which are not // yet wired up in evetest). @@ -285,11 +302,14 @@ func TestDeviceConnectivitySuite(test *testing.T) { // filtering on Local / Switch NIs (variants: ENABLE_FLOWLOG=false / =true). // - TestApplicationIPv6Connectivity -- app on a Switch NI in an // IPv6-only segment (stub scenario). -// - TestApplicationNTPConfig -- DHCP-propagated NTP server set -// reaching the application (stub scenario; needs SDN NTP daemon). +// - TestApplicationNTPConfig -- per-NI DHCP-advertised NTP server set +// (port union NI) reaching the application's published VIF status. // - TestPropagatedRoutes / TestLocalNIWithMultiplePorts / // TestApplicationGateway -- IP-routing-related scenarios mirroring // the eden app-routing examples (stub scenarios). +// - TestPortForwarding -- port-forwarding (D-NAT) hairpin connectivity, +// including across two different uplink adapters, and changing a +// port-fwd rule's external port at runtime. // - TestSwitchNIWithMultiplePorts -- STP / BPDU-guard on a Switch NI // with redundant L2 links (stub scenario). // - TestAccessVLANs -- VLAN-aware Switch NI (stub scenario). @@ -311,6 +331,18 @@ func TestApplicationConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestSwitchNI, }, + evetest.TestCase{ + Test: TestNIReplace, + }, + evetest.TestCase{ + Test: TestMoveAppBetweenNIs, + }, + evetest.TestCase{ + Test: TestAirGapSwitchNI, + }, + evetest.TestCase{ + Test: TestLimitedIPSpace, + }, evetest.TestCase{ Test: TestFlowLog, }, @@ -363,6 +395,9 @@ func TestApplicationConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestApplicationGateway, }, + evetest.TestCase{ + Test: TestPortForwarding, + }, evetest.TestCase{ Test: TestSwitchNIWithMultiplePorts, }, @@ -378,11 +413,9 @@ func TestApplicationConnectivitySuite(test *testing.T) { // TestDatastoreSuite drives every datastore-pull scenario: EVE downloads // an application content tree from a backend (HTTP, HTTPS, S3, SFTP, // Azure, container registry), verifies the SHA, and brings the resulting -// app up. The suite does not parameterize the hypervisor -- datastore -// tests deploy a tiny "consumer" app but the test value is in the -// download/verification plumbing, not in the app runtime, so the -// hypervisor is hardcoded to KVM per the same rule applied to Device-suite -// tests. +// app up. Datastore tests deploy a tiny "consumer" app but the test value +// is in the download/verification plumbing, not in the app runtime; the +// suite still shares the HYPERVISOR parameter (defaults to KVM). // // Subtests // -------- @@ -398,6 +431,10 @@ func TestDatastoreSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{ Test: TestHTTPDatastore, @@ -425,9 +462,8 @@ func TestDatastoreSuite(test *testing.T) { // (phantom PCI address, self-parent assign-group, USB address collision, // interface-name mismatch, cross-group PCI conflict, warning+error bundle) // and reports it to the controller with the correct severity, then clears -// it once the model is fixed. The suite does not parameterize the -// hypervisor -- no application is deployed, so the hypervisor is hardcoded -// to KVM like the other Device-suite tests. +// it once the model is fixed. No application is deployed, but the suite +// still shares the HYPERVISOR parameter (defaults to KVM). // // Every scenario but the last runs on the TwoMgmtPorts model and reuses // the same device via ResetDeviceConfig. TestReportWarningsOnly needs the @@ -457,6 +493,10 @@ func TestPcibackErrorSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{Test: TestReportMissingDevice}, evetest.TestCase{Test: TestReportParentAssigngrp}, diff --git a/evetest/tests/networking/vlans_test.go b/evetest/tests/networking/vlans_test.go index 00889ce805a..c6c001afc0d 100644 --- a/evetest/tests/networking/vlans_test.go +++ b/evetest/tests/networking/vlans_test.go @@ -105,8 +105,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestAccessVLANs(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -114,7 +113,6 @@ func TestAccessVLANs(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -171,6 +169,9 @@ func TestAccessVLANs(test *testing.T) { SharedLabels: []string{"switch-ports"}, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Switch NI: eth1 is trunk (carries VLAN 100 and 200 tagged via the SDN router); // eth2 is access port for VLAN 100; eth3 is access port for VLAN 200. @@ -568,8 +569,7 @@ func TestAccessVLANs(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestVLANSubinterfaces(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -577,7 +577,6 @@ func TestVLANSubinterfaces(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" // Clone the bootstrap model and shorten the DHCP lease on the management network. @@ -645,6 +644,9 @@ func TestVLANSubinterfaces(test *testing.T) { // waitUntilConfirmed=false: after the model switch below, EVE may temporarily // lose controller connectivity, delaying the LastProcessedConfig metric publish. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to process the port config (activate vlan interfaces) // before switching the SDN side, so the new interfaces are ready to carry // traffic as soon as VLAN tagging is enabled. @@ -913,8 +915,7 @@ func TestVLANSubinterfaces(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -922,7 +923,6 @@ func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" @@ -1025,6 +1025,9 @@ func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { // because after the SDN model switch below, EVE may temporarily lose controller // connectivity while LACP negotiates. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to create the bond and VLAN sub-interfaces before switching // the SDN side, so the interfaces are ready to carry traffic immediately. time.Sleep(10 * time.Second) diff --git a/evetest/tests/security/apparmor_test.go b/evetest/tests/security/apparmor_test.go new file mode 100644 index 00000000000..5e1c27a68f5 --- /dev/null +++ b/evetest/tests/security/apparmor_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve/evetest" +) + +// TestAppArmorEnabled verifies that EVE's kernel has AppArmor compiled in +// and enabled, by reading the kernel's own status flag directly over the +// device's management SSH. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestAppArmorEnabled(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + WithTPM: true, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + out, _, err := device.RunShellScript( + "cat /sys/module/apparmor/parameters/enabled", 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.TrimSpace(out)).To(Equal("Y"), "AppArmor is not enabled") +} diff --git a/evetest/tests/security/testsuite_test.go b/evetest/tests/security/testsuite_test.go new file mode 100644 index 00000000000..957f1ff7598 --- /dev/null +++ b/evetest/tests/security/testsuite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestSecuritySuite drives every device-security scenario in this package. +// +// Subtests +// -------- +// - TestAppArmorEnabled -- kernel AppArmor status flag. +// - TestVCom -- vcomlink (TPM-over-vsock) request/response from inside a VM app. +func TestSecuritySuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestAppArmorEnabled, + }, + evetest.TestCase{ + Test: TestVCom, + }, + ) +} diff --git a/evetest/tests/security/vcom_test.go b/evetest/tests/security/vcom_test.go new file mode 100644 index 00000000000..5295d17beca --- /dev/null +++ b/evetest/tests/security/vcom_test.go @@ -0,0 +1,315 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "encoding/base64" + "fmt" + "regexp" + "strconv" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// vcomCheckScript is a self-contained (stdlib-only) Python script that +// exercises vcomlink's TPM ActivateCredParams endpoint over vsock from +// inside the guest. +// +// vcomlink (pkg/pillar/cmd/vcomlink) serves a plain HTTP/1.1 API tunneled +// over an AF_VSOCK connection to the host (CID=VMADDR_CID_HOST, port 2000; +// see vsocksrv.go), with protobuf-encoded request/response bodies (see +// pkg/pillar/vcom/api/proto/messages.proto). vcomRequestBody below is the +// precomputed protobuf encoding of +// vcom.TpmRequestActivateCredParams{Index: 0x81000003} -- 0x81000003 is +// evetpm.TpmAIKHdl, the well-known permanent handle for EVE's AIK -- so this +// script needs no protobuf library. The response is parsed just far enough +// (a minimal length-delimited-field walk) to read the length of the "ek" +// field (field 1), which is all this test needs to assert. +const vcomCheckScript = `#!/usr/bin/env python3 +import socket +import sys + +# protobuf encoding of TpmRequestActivateCredParams{Index: 0x81000003} +REQUEST_BODY = bytes([0x08, 0x83, 0x80, 0x80, 0x88, 0x08]) + +def read_varint(data, pos): + result = 0 + shift = 0 + while True: + b = data[pos] + pos += 1 + result |= (b & 0x7f) << shift + if not (b & 0x80): + break + shift += 7 + return result, pos + +def bytes_field_len(data, field_num): + pos = 0 + while pos < len(data): + tag, pos = read_varint(data, pos) + field, wire_type = tag >> 3, tag & 0x7 + if wire_type == 0: + _, pos = read_varint(data, pos) + elif wire_type == 2: + length, pos = read_varint(data, pos) + if field == field_num: + return length + pos += length + elif wire_type == 5: + pos += 4 + elif wire_type == 1: + pos += 8 + else: + raise ValueError("unsupported wire type %d" % wire_type) + return None + +s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) +s.connect((socket.VMADDR_CID_HOST, 2000)) +request = ( + b"POST /tpm/activatecredparams HTTP/1.1\r\n" + b"Host: vcom\r\n" + b"Content-Type: application/x-proto-binary\r\n" + b"Content-Length: " + str(len(REQUEST_BODY)).encode() + b"\r\n" + b"Connection: close\r\n\r\n" +) + REQUEST_BODY +s.sendall(request) + +data = b"" +while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk +s.close() + +header_end = data.index(b"\r\n\r\n") +status_line = data[:header_end].split(b"\r\n")[0].decode() +body = data[header_end + 4:] + +if " 200 " not in status_line: + print("STATUS=%s BODY=%s" % (status_line, body.decode(errors="replace"))) + sys.exit(1) + +print("STATUS=200 EK_LEN=%d" % bytes_field_len(body, 1)) +` + +var vcomEKLenRegexp = regexp.MustCompile(`EK_LEN=(\d+)`) + +// alpineCloudImage describes the arch-specific pinned Alpine Linux +// cloud-init qcow2 image used to boot the VM app in this test. +type alpineCloudImage struct { + relativePath string + sha256 string + sizeBytes uint64 +} + +// Alpine 3.24.1 cloud images, pinned by release version (not a rolling +// "latest" alias) so the SHA256 below stays valid indefinitely. See +// https://alpinelinux.org/cloud/ for the full image list. +// +// amd64 uses the "bios-cloudinit" variant, not "uefi-cloudinit": pillar only +// attaches OVMF/UEFI firmware automatically for VmMode_FML on amd64 (see +// handledomainmgr.go), so a VmMode_HVM VM on amd64 gets legacy SeaBIOS, which +// cannot boot a UEFI-only image. arm64 has no legacy BIOS at all, so pillar +// always attaches OVMF there regardless of VirtualizationMode, and the +// "uefi-cloudinit" variant is the correct (only) choice. +var alpineCloudImages = map[string]alpineCloudImage{ + "amd64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2", + sha256: "6e2e6fe0572b6632527f268d3659e8fccebda4e1ee470fafe2c4d7b85b6a4df6", + sizeBytes: 183697408, + }, + "arm64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-aarch64-uefi-cloudinit-r0.qcow2", + sha256: "3059a6280977c2122982632e0317c5ddbd39069d46ca1e60480de283091f720f", + sizeBytes: 239271936, + }, +} + +// TestVCom verifies that vcomlink (pkg/pillar/cmd/vcomlink), EVE's +// vsock-based host<->VM communication agent, is running and correctly +// serves a TPM request from inside a guest VM. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- controller reachability plus Internet +// access, needed to pull the Alpine cloud image below. +// +// Phases +// ------ +// 1. Confirm vcomlink is actually listening on the host side (the device +// always has TPM emulation enabled -- vcomlink's TPM handlers depend +// on one being present): `eve exec pillar ss -l --vsock` must report +// a listener on CID:port "2:2000" (VMADDR_CID_HOST, vcomlink's fixed +// port -- see vsocksrv.go). +// 2. Deploy a VM app (VirtualizationMode=HVM) from the pinned Alpine +// Linux cloud image (matching the device's actual arch, device.GetArch()), +// on a Local NI with a 2222->22 port-forward. UserData is a cloud-config +// enabling password SSH login for root (disabled by default). +// Wait for the app to reach RUNNING, then for SSH to become reachable. +// 3. Write vcomCheckScript into the guest over SSH and run it with +// python3 (present by default on the Alpine cloud image; AF_VSOCK +// needs Python >= 3.9, satisfied by Alpine 3.24's default 3.12). +// Assert the script's own output reports HTTP 200 and a non-zero EK +// length -- i.e. vcomlink actually returned real TPM-backed data to +// the VM over vsock, not just accepted the connection. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestVCom(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters(evetest.HypervisorParameter()) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + WithTPM: true, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + log := evetest.Logger() + log.Infof("Checking if vComLink is running on EVE...") + t.Eventually(func(gt Gomega) { + out, _, err := device.RunShellScript( + "eve exec pillar ss -l --vsock", 20*time.Second, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(out).To( + ContainSubstring("2:2000"), "vComLink is not listening on vsock") + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("vcomlink-listening") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress("10.50.0.1"), + MTU: 1500, + }) + + image := alpineCloudImages[device.GetArch()] + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + cloudConfig := fmt.Sprintf(`#cloud-config +ssh_pwauth: true +chpasswd: + list: | + root:%s + expire: false +write_files: + - path: /etc/ssh/sshd_config.d/99-allow-root-password.conf + content: | + PermitRootLogin yes +runcmd: + - rc-service sshd restart +`, appAuth.Password) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vcom-test-vm", + Activate: true, + Image: evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW2, + ImageSHA256: image.sha256, + MaxDownloadBytes: image.sizeBytes, + ImageRelativePath: image.relativePath, + ServerAddress: "dl-cdn.alpinelinux.org", + UseHTTPS: true, + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + UserData: base64.StdEncoding.EncodeToString([]byte(cloudConfig)), + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 8*time.Minute) + evetest.Checkpoint("vm-running") + + sshTimeout := 20 * time.Second + polling := 5 * time.Second + log.Infof("Waiting for VM SSH to become reachable...") + t.Eventually(func(gt Gomega) { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "echo ok", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + }, 5*time.Minute, polling).Should(Succeed()) + evetest.Checkpoint("vm-ssh-ready") + + log.Infof("Running the vComLink TPM check script inside the VM...") + encoded := base64.StdEncoding.EncodeToString([]byte(vcomCheckScript)) + script := "echo " + encoded + " | base64 -d > vcomcheck.py && python3 vcomcheck.py" + checkOut, checkErr, err := device.RunShellScriptInsideApp( + appUUID, appAuth, script, 60*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred(), "vComLink check script failed: %s", checkErr) + t.Expect(checkOut).To( + ContainSubstring("STATUS=200"), "unexpected vComLink response: %s", checkOut) + + match := vcomEKLenRegexp.FindStringSubmatch(checkOut) + t.Expect(match).To(HaveLen(2), "could not find EK_LEN in script output: %s", checkOut) + ekLen, err := strconv.Atoi(match[1]) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(ekLen).To(BeNumerically(">", 0), "vComLink returned an empty EK") + evetest.Checkpoint("vcom-tpm-request-verified") +} diff --git a/evetest/tests/storage/testsuite_test.go b/evetest/tests/storage/testsuite_test.go new file mode 100644 index 00000000000..db461435e3d --- /dev/null +++ b/evetest/tests/storage/testsuite_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestStorageSuite drives storage-layer regression tests. +// +// Subtests +// -------- +// - TestZVolProvisionedSizeReported -- regression test for a volumemgr bug +// where a ZFS zvol-backed volume always reported a provisioned size of 0. +// - TestVaultZvolTrimReclaimsBlocks -- verifies fstrim on /persist/vault +// reclaims ghost blocks on a ZFS node. +func TestStorageSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestZVolProvisionedSizeReported, + }, + evetest.TestCase{ + Test: TestVaultZvolTrimReclaimsBlocks, + }, + ) +} diff --git a/evetest/tests/storage/vault_trim_test.go b/evetest/tests/storage/vault_trim_test.go index 60bdd2695d5..4eb25634c19 100644 --- a/evetest/tests/storage/vault_trim_test.go +++ b/evetest/tests/storage/vault_trim_test.go @@ -28,22 +28,37 @@ import ( // // The test writes 256 MiB of incompressible data (/dev/urandom bypasses ZFS // zstd compression), deletes it to create ghost blocks, then verifies that -// fstrim causes logicalused to drop. Skipped on non-kubevirt or non-ZFS nodes. +// fstrim causes logicalused to drop. Requires a ZFS node; the underlying bug +// was found on EVE-K (Longhorn replica churn), but the mechanism is purely a +// ZFS/fstrim concern and applies equally to EVE-KVM, so the hypervisor is a +// parameter rather than hard-coded. +// +// Parameters +// ---------- +// - HYPERVISOR (defaults to KVM). func TestVaultZvolTrimReclaimsBlocks(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKubevirt, + WithHypervisor: hypervisor, WithFilesystem: evetest.FilesystemZFS, DeviceReusePolicy: evetest.UseAsIs, }, ) device := evetest.GetEdgeDevice(devName) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // evetest.Setup returns once the device is onboarded and has fetched its // config; it does NOT wait for the vault to be unlocked/mounted. Gate the @@ -51,20 +66,14 @@ func TestVaultZvolTrimReclaimsBlocks(test *testing.T) { // write lands on the parent persist dataset's mountpoint directory (the // ext4-on-zvol is not mounted yet) and logicalused on the zvol never moves. - // Wait for vaultmgr to report the default vault ConversionComplete. Read - // the VaultStatus pubsub JSON on-device via a shell (not ReadPublication): - // ReadPublication/ReadFile Fatalf on a not-yet-published file, and the - // pubsub key "Application Data Store" contains spaces that break scp's - // remote path. A failed cat just fails the poll and we retry. - vaultStatusPath := `/run/vaultmgr/VaultStatus/` + pillartypes.DefaultVaultName + `.json` + // Wait for vaultmgr to report the default vault ConversionComplete. t.Eventually(func() bool { - out, _, err := device.RunShellScript( - `eve exec pillar cat "`+vaultStatusPath+`"`, 15*time.Second, 0) - if err != nil { + var status pillartypes.VaultStatus + if !evetest.ReadPublication(device, "vaultmgr", false, + pillartypes.DefaultVaultName, &status) { return false // status not published yet } - return strings.Contains(strings.ReplaceAll(out, " ", ""), - `"ConversionComplete":true`) + return status.ConversionComplete }, 5*time.Minute, 5*time.Second).Should(BeTrue(), "vaultmgr must report the default vault ConversionComplete before writing") diff --git a/evetest/tests/storage/zvol_provisioned_size_test.go b/evetest/tests/storage/zvol_provisioned_size_test.go index 74912b7f407..aaa50abd6b5 100644 --- a/evetest/tests/storage/zvol_provisioned_size_test.go +++ b/evetest/tests/storage/zvol_provisioned_size_test.go @@ -41,13 +41,17 @@ import ( // it. volumemgr creates standalone (app-unreferenced) volumes on its own, so no // application is needed to trigger creation. // -// Requires a KVM device whose /persist is ZFS: a non-container, non-ISO volume +// Requires a device whose /persist is ZFS: a non-container, non-ISO volume // is only backed by a zvol when /persist is ZFS. On EXT4 there is no zvol and // the code path under test does not run, so the test skips. // +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// // Phases // ------ -// 1. Set up a KVM + ZFS device with a single DHCP mgmt port. +// 1. Set up a ZFS device with a single DHCP mgmt port. // 2. Apply a config that declares one standalone empty 1 GiB block-device // volume (no app, no network instance). // 3. Wait for volumemgr to create the volume (state CREATED_VOLUME) and, @@ -60,6 +64,11 @@ func TestZVolProvisionedSizeReported(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // A 1 GiB volume is an exact multiple of the ZFS volblocksize (16 KiB), so // the resulting zvol volsize -- and therefore the reported provisioned size // -- is exactly 1 GiB with no block-size rounding. @@ -71,7 +80,7 @@ func TestZVolProvisionedSizeReported(test *testing.T) { evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, WithFilesystem: evetest.FilesystemZFS, DeviceReusePolicy: evetest.ResetDeviceConfig, }, @@ -106,6 +115,9 @@ func TestZVolProvisionedSizeReported(test *testing.T) { volInfoUpdates, stopVolInfoWatch := device.WatchVolumeInfo(volUUID) defer stopVolInfoWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("blank-volume-config-applied") // Wait for the volume to be created. CREATED_VOLUME is the volume-equivalent diff --git a/evetest/tests/upgrade/testsuite_test.go b/evetest/tests/upgrade/testsuite_test.go index 0518022f0ba..f63cf101435 100644 --- a/evetest/tests/upgrade/testsuite_test.go +++ b/evetest/tests/upgrade/testsuite_test.go @@ -26,8 +26,7 @@ func TestUpgradeSuite(test *testing.T) { initialEVEVersionForKVM = "16.0.0-lts" // EVE-K (k3s/kubevirt-based EVE) is officially supported starting from 17.0.0. - // TODO: strip the rc suffix once 17.0.0 is released. - initialEVEVersionForKubevirt = "17.0.0-rc2" + initialEVEVersionForKubevirt = "17.0.0-lts" // Enough for the pre-10GB partition layout, but not enough for EVE 17.0.0+, // which is why *WithSmallDisk variants expect revert. @@ -54,6 +53,19 @@ func TestUpgradeSuite(test *testing.T) { {Key: evetest.HypervisorParameterKey, Value: evetest.HypervisorKVM}, }, }, + { + Name: "TestEVEUpgradeKVMtoKVMWithOCIDatastore", + Parameters: []evetest.TestParameterValue{ + // Initial + {Key: initialEVEVersionParamKey, Value: initialEVEVersionForKVM}, + {Key: initialHypervisorParamKey, Value: evetest.HypervisorKVM}, + // Target + {Key: evetest.HypervisorParameterKey, Value: evetest.HypervisorKVM}, + // Have EVE pull the target rootfs directly from its OCI registry + // instead of the default evetest-hosted HTTP datastore. + {Key: datastoreTypeParamKey, Value: evetest.BaseOSDatastoreOCI}, + }, + }, { Name: "TestEVEUpgradeKubevirtToKubevirt", Parameters: []evetest.TestParameterValue{ diff --git a/evetest/tests/upgrade/upgrade_test.go b/evetest/tests/upgrade/upgrade_test.go index 2b8f6f4f23d..c7f4342ebc3 100644 --- a/evetest/tests/upgrade/upgrade_test.go +++ b/evetest/tests/upgrade/upgrade_test.go @@ -25,6 +25,7 @@ const ( initialEVEVersionParamKey = "INITIAL_EVE_VERSION" initialHypervisorParamKey = "INITIAL_HYPERVISOR" expectRevertParamKey = "EXPECT_REVERT" + datastoreTypeParamKey = "DATASTORE_TYPE" appSSHUser = "root" appSSHPassword = "testpassword" @@ -51,6 +52,10 @@ const ( // - DISK_SIZE_MB: device disk size in MiB (0 = framework default 65536 MiB) // - EXPECT_REVERT: if true, the upgrade is expected to fail and EVE to revert // to the previous version (default: false) +// - DATASTORE_TYPE: how the target rootfs is delivered to the device -- +// "http" (evetest extracts the rootfs and serves it over its own embedded +// HTTP image server) or "oci" (EVE pulls the target image directly from +// its OCI registry, e.g. Docker Hub) (default: "http") // // A container app (evetest-ubuntu-ctr) is deployed before the upgrade and verified // to be healthy both before and after the upgrade (or revert). @@ -90,6 +95,16 @@ func TestEVEUpgrade(test *testing.T) { Default: "false", }, }, + evetest.TestParameterDefinition{ + Key: datastoreTypeParamKey, + DefaultValue: evetest.BaseOSDatastoreHTTP, + Description: evetest.TestParameterDescription{ + Summary: "How the target rootfs is delivered to the device: evetest-hosted " + + "HTTP, or an OCI registry pull performed directly by EVE", + Default: "http", + AllowedValues: "http|oci", + }, + }, ) // Get parameter values set for this test execution. @@ -104,6 +119,7 @@ func TestEVEUpgrade(test *testing.T) { targetVersion := evetest.GetEVEVersionParameterValue() targetHypervisor := evetest.GetHypervisorParameterValue() expectRevert := evetest.GetTestParameter[bool](expectRevertParamKey) + datastoreType := evetest.GetTestParameter[evetest.BaseOSDatastoreType](datastoreTypeParamKey) const devName = "edge-dev" evetest.Setup( @@ -218,7 +234,7 @@ func TestEVEUpgrade(test *testing.T) { evetest.Checkpoint("pre-upgrade") - device.UpgradeEVE(targetVersion, targetHypervisor, true, expectRevert) + device.UpgradeEVE(targetVersion, targetHypervisor, datastoreType, true, expectRevert) if expectRevert { evetest.Checkpoint("upgrade-reverted")