diff --git a/evetest/Dockerfile.evetest b/evetest/Dockerfile.evetest index 1984b9f5304..49b7923956b 100644 --- a/evetest/Dockerfile.evetest +++ b/evetest/Dockerfile.evetest @@ -5,7 +5,7 @@ ARG EVETEST_VERSION=dev ARG ALPINE_VERSION=3.21 ARG GOLANG_VERSION=1.25 ARG EVETEST_ADAM_REPO=lfedge/adam -ARG EVETEST_ADAM_VERSION=0.0.75 +ARG EVETEST_ADAM_VERSION=0.0.81 ########################### # Pull adam binary stage diff --git a/evetest/Makefile b/evetest/Makefile index 4f7504f3691..b9a94b6f134 100644 --- a/evetest/Makefile +++ b/evetest/Makefile @@ -4,7 +4,7 @@ EVETEST_VERSION := $(shell grep -v '^\#' VERSION | head -n1) EVETEST_ORG ?= lfedge -EVETEST_ADAM_VERSION ?= 0.0.75 +EVETEST_ADAM_VERSION ?= 0.0.81 EVETEST_ADAM_REPO ?= lfedge/adam EVETEST_IMAGE := $(EVETEST_ORG)/evetest:$(EVETEST_VERSION) diff --git a/evetest/README.md b/evetest/README.md index 5f731f38a38..ab59426a68e 100644 --- a/evetest/README.md +++ b/evetest/README.md @@ -857,8 +857,8 @@ same terminal session beforehand. |----------|-------------|---------| | `EVETEST_ORG` | Docker Hub organization for evetest and evetest-broker images | `lfedge` | | `EVETEST_EVE_REPO` | EVE image repository | `lfedge/eve` | -| `EVETEST_ADAM_VERSION` | Adam controller version *(build-time only, see note below)* | `0.0.75` | -| `EVETEST_SDN_VERSION` | SDN emulator version | `1.0` | +| `EVETEST_ADAM_VERSION` | Adam controller version *(build-time only, see note below)* | `0.0.81` | +| `EVETEST_SDN_VERSION` | SDN emulator version | `1.1` | > **`EVETEST_ADAM_VERSION` requires evetest container rebuild.** Unlike the other > variables above, Adam's binary is baked into the evetest image at build time diff --git a/evetest/VERSION b/evetest/VERSION index 7d9bfdfb779..d7cec1d7df4 100644 --- a/evetest/VERSION +++ b/evetest/VERSION @@ -1,2 +1,2 @@ # Evetest version. Increment this manually whenever changes are made to the evetest framework. -1.0 +1.1 diff --git a/evetest/broker/broker.go b/evetest/broker/broker.go index 1cd5efd80f8..7c7f923655b 100644 --- a/evetest/broker/broker.go +++ b/evetest/broker/broker.go @@ -645,6 +645,7 @@ func (b *broker) BuildImage( softSerial: softSerial, diskSize: req.DiskBytes, installer: req.MakeInstaller, + extraDiskBytes: req.ExtraDiskBytes, }) } else { // No EVE container image need exist on this broker at all on the live @@ -671,6 +672,7 @@ func (b *broker) BuildImage( liveImageSHA256: req.GetLiveImage().GetSha256(), liveTarPath: liveUploadPath(b.imageDir, req.GetLiveImage().GetSha256()), liveSource: liveSource, + extraDiskBytes: req.ExtraDiskBytes, }) } } diff --git a/evetest/broker/image.go b/evetest/broker/image.go index 059475c5151..f9588f86b6e 100644 --- a/evetest/broker/image.go +++ b/evetest/broker/image.go @@ -122,6 +122,10 @@ type buildEVEImageParams struct { // softSerial is the device's soft serial number. Always non-empty; see // resolveSoftSerial. softSerial string + // extraDiskBytes are sizes (in bytes) of additional blank disks to create + // and append to the result, beyond the main boot/target disk -- e.g. for + // tests that exercise EVE-level disk layout/RAID configuration. + extraDiskBytes []uint64 } // buildEVEImageResult holds the outputs of either device-image producer @@ -134,9 +138,10 @@ type buildEVEImageResult struct { // RAW from the legacy buildEVEImage path, QCOW2 from the template-backed // makeDeviceImage path, where it must be QCOW2 to be a template overlay. installerImage *provider.DiskImage - // disks is the list of persistent disk images for the device. Currently always - // a single disk (live QCOW2 for live builds, blank target QCOW2 for installer - // builds), but structured as a slice to accommodate multiple disks in the future. + // disks is the list of persistent disk images for the device: the main + // boot/target disk (live QCOW2 for live builds, blank target QCOW2 for + // installer builds) followed by any extra blank disks requested via + // extraDiskBytes. disks []provider.DiskImage // firmwareDir is the path to the directory containing the extracted UEFI firmware // (OVMF_CODE.fd, OVMF_VARS.fd). @@ -251,34 +256,40 @@ func buildEVEImage(ctx context.Context, log *logrus.Entry, result.disks = []provider.DiskImage{ {Format: provider.DiskImageFormatQcow2, Path: builtImagePath}, } - return result, nil - } + } else { + // For installer mode, create a blank target disk that EVE will be + // installed onto. The installer image is prepended to disks for the + // first boot only. + if params.diskSize == 0 { + err = fmt.Errorf("diskSize must be non-zero for installer builds") + return result, err + } + targetDiskPath := filepath.Join(params.imageDirPath, "installed.qcow2") + targetDisk, err2 := createBlankDisk( + ctx, log, targetDiskPath, provider.DiskImageFormatQcow2, params.diskSize) + if err2 != nil { + err = fmt.Errorf("failed to create installer target disk: %w", err2) + return result, err + } + log.Infof("Created blank target disk for EVE installation: %s", targetDiskPath) - // For installer mode, create a blank target disk that EVE will be installed onto. - // The installer image is prepended to disks for the first boot only. - if params.diskSize == 0 { - err = fmt.Errorf("diskSize must be non-zero for installer builds") - return result, err - } - targetDiskPath := filepath.Join(params.imageDirPath, "installed.qcow2") - diskSizeMiB := params.diskSize >> 20 - log.Infof("Creating blank target disk for EVE installation: %s (%d MiB)", - targetDiskPath, diskSizeMiB) - out, err2 := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", - targetDiskPath, fmt.Sprintf("%dM", diskSizeMiB)).CombinedOutput() - if err2 != nil { - err = fmt.Errorf("failed to create installer target disk %q: %v: %s", - targetDiskPath, err2, out) - return result, err + installerImage := provider.DiskImage{ + Format: provider.DiskImageFormatRaw, Path: builtImagePath} + result.installerImage = &installerImage + result.disks = []provider.DiskImage{targetDisk} } - log.Infof("Created blank target disk for EVE installation: %s", targetDiskPath) - installerImage := provider.DiskImage{ - Format: provider.DiskImageFormatRaw, Path: builtImagePath} - result.installerImage = &installerImage - result.disks = []provider.DiskImage{ - {Format: provider.DiskImageFormatQcow2, Path: targetDiskPath}, + // Append any extra (non-boot) blank disks requested, e.g. for tests that + // exercise EVE-level disk layout/RAID configuration. These live in + // result.disks, not result.installerImage, so they persist across the + // installer's post-install disk reconfiguration (see + // ReconfigureDeviceDisks in broker.go). + extraDisks, err := createBlankDisks(ctx, log, params.imageDirPath, params.extraDiskBytes) + if err != nil { + err = fmt.Errorf("failed to create extra disks: %w", err) + return result, err } + result.disks = append(result.disks, extraDisks...) return result, nil } @@ -673,6 +684,10 @@ type makeDeviceImageParams struct { // this broker can read directly; the template is installed from those and // liveTarPath is never touched. liveSource *api.LocalLiveImageSource + // extraDiskBytes are sizes (in bytes) of additional blank disks to create + // and append to the result, beyond the main boot/target disk -- e.g. for + // tests that exercise EVE-level disk layout/RAID configuration. + extraDiskBytes []uint64 } // makeDeviceImage derives a device's disk image from a cached template: it @@ -805,27 +820,85 @@ func makeDeviceImage(ctx context.Context, log *logrus.Entry, cache *templateCach result.disks = []provider.DiskImage{ {Format: provider.DiskImageFormatQcow2, Path: diskPath}, } - return result, tmpl.Key, nil + } else { + if params.diskSize == 0 { + err = fmt.Errorf("diskSize must be non-zero for installer builds") + return result, "", err + } + targetDiskPath := filepath.Join(params.imageDirPath, "installed.qcow2") + targetDisk, err2 := createBlankDisk( + ctx, log, targetDiskPath, provider.DiskImageFormatQcow2, params.diskSize) + if err2 != nil { + err = fmt.Errorf("failed to create installer target disk: %w", err2) + return result, "", err + } + installerImage := provider.DiskImage{ + Format: provider.DiskImageFormatQcow2, Path: diskPath} + result.installerImage = &installerImage + result.disks = []provider.DiskImage{targetDisk} + } + + // Append any extra (non-boot) blank disks requested, e.g. for tests that + // exercise EVE-level disk layout/RAID configuration. These live in + // result.disks, not result.installerImage, so they persist across the + // installer's post-install disk reconfiguration (see + // ReconfigureDeviceDisks in broker.go). + extraDisks, err := createBlankDisks(ctx, log, params.imageDirPath, params.extraDiskBytes) + if err != nil { + err = fmt.Errorf("failed to create extra disks: %w", err) + return result, "", err } + result.disks = append(result.disks, extraDisks...) + return result, tmpl.Key, nil +} - if params.diskSize == 0 { - err = fmt.Errorf("diskSize must be non-zero for installer builds") - return result, "", err +// diskImageFormatName returns the "-f" format name qemu-img expects for the +// given DiskImageFormat. +func diskImageFormatName(format provider.DiskImageFormat) string { + if format == provider.DiskImageFormatQcow2 { + return "qcow2" } - targetDiskPath := filepath.Join(params.imageDirPath, "installed.qcow2") - diskSizeMiB := params.diskSize >> 20 - out, cmdErr := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", - targetDiskPath, fmt.Sprintf("%dM", diskSizeMiB)).CombinedOutput() - if cmdErr != nil { - err = fmt.Errorf("failed to create installer target disk %q: %v: %s", - targetDiskPath, cmdErr, out) - return result, "", err + return "raw" +} + +// createBlankDisk creates a single blank disk image file of the given format +// and size at diskPath, returning it as a DiskImage. +func createBlankDisk(ctx context.Context, log *logrus.Entry, diskPath string, + format provider.DiskImageFormat, sizeBytes uint64) (provider.DiskImage, error) { + sizeMiB := sizeBytes >> 20 + log.Infof("Creating blank disk %q (%d MiB, format %s)", + diskPath, sizeMiB, diskImageFormatName(format)) + out, err := exec.CommandContext(ctx, "qemu-img", "create", "-f", + diskImageFormatName(format), diskPath, fmt.Sprintf("%dM", sizeMiB)).CombinedOutput() + if err != nil { + return provider.DiskImage{}, fmt.Errorf( + "failed to create blank disk %q: %w: %s", diskPath, err, out) } - installerImage := provider.DiskImage{ - Format: provider.DiskImageFormatQcow2, Path: diskPath} - result.installerImage = &installerImage - result.disks = []provider.DiskImage{ - {Format: provider.DiskImageFormatQcow2, Path: targetDiskPath}, + return provider.DiskImage{Format: format, Path: diskPath}, nil +} + +// createBlankDisks creates one blank raw disk image file per entry in +// sizesBytes, under imageDirPath, and returns them as DiskImage entries in +// the same order. Used to provision extra (non-boot) disks for a device, +// e.g. for tests that exercise EVE-level disk layout/RAID configuration. +func createBlankDisks(ctx context.Context, log *logrus.Entry, + imageDirPath string, sizesBytes []uint64) ([]provider.DiskImage, error) { + if len(sizesBytes) == 0 { + return nil, nil + } + if err := os.MkdirAll(imageDirPath, 0o755); err != nil { + return nil, fmt.Errorf( + "failed to create image directory %q: %w", imageDirPath, err) + } + disks := make([]provider.DiskImage, 0, len(sizesBytes)) + for i, sizeBytes := range sizesBytes { + diskPath := filepath.Join(imageDirPath, fmt.Sprintf("extra-disk-%d.img", i)) + disk, err := createBlankDisk( + ctx, log, diskPath, provider.DiskImageFormatRaw, sizeBytes) + if err != nil { + return nil, err + } + disks = append(disks, disk) } - return result, tmpl.Key, nil + return disks, nil } diff --git a/evetest/cli/evecmd.go b/evetest/cli/evecmd.go index 17081d8d61a..f2b475695f9 100644 --- a/evetest/cli/evecmd.go +++ b/evetest/cli/evecmd.go @@ -468,22 +468,27 @@ func eveAppFlowLogsCmd() *cobra.Command { if err != nil { return fmt.Errorf("stream error: %w", err) } - var lines []string + var newEntries []string for _, ipFlow := range resp.IpFlows { - lines = append(lines, fmt.Sprintf("IP flow: %s", ipFlow.String())) + newEntries = append(newEntries, + fmt.Sprintf("IP flow: %s", ipFlow.String())) } for _, dnsReq := range resp.DnsRequests { - lines = append(lines, fmt.Sprintf("DNS request: %s", dnsReq.String())) + newEntries = append(newEntries, + fmt.Sprintf("DNS request: %s", dnsReq.String())) } - entry := strings.Join(lines, "\n") if tail > 0 { - entries = append(entries, entry) + entries = append(entries, newEntries...) } else { - fmt.Println(entry) + for _, entry := range newEntries { + fmt.Println(entry) + fmt.Println() + } } } for _, e := range tailEntries(entries, tail) { fmt.Println(e) + fmt.Println() } return nil }, diff --git a/evetest/constants/config.go b/evetest/constants/config.go index bc7dfa1b322..4ec20d466da 100644 --- a/evetest/constants/config.go +++ b/evetest/constants/config.go @@ -317,10 +317,10 @@ const ( DefaultSDNRepo = "lfedge/evetest-sdn" // DefaultAdamVersion specifies the Adam version to use by default. - DefaultAdamVersion = "0.0.75" + DefaultAdamVersion = "0.0.81" // DefaultSDNVersion specifies the SDN version to use by default. - DefaultSDNVersion = "1.0" + DefaultSDNVersion = "1.1" // DefaultSDNUplinkIPv4Subnet species the IPv4 subnet used for SDN uplink // interfaces by default. diff --git a/evetest/controller/adam.go b/evetest/controller/adam.go index 769243607c2..13922a5aef8 100644 --- a/evetest/controller/adam.go +++ b/evetest/controller/adam.go @@ -31,6 +31,7 @@ import ( evecerts "github.com/lf-edge/eve-api/go/certs" eveconfig "github.com/lf-edge/eve-api/go/config" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evelogs "github.com/lf-edge/eve-api/go/logs" evemetrics "github.com/lf-edge/eve-api/go/metrics" @@ -168,6 +169,16 @@ type MetricMsgIterator interface { Iterate(msg *evemetrics.ZMetricMsg) (stop bool, err error) } +// FlowMsgIterator iterates over device flow log messages (FlowMessage), each +// of which carries both flow records and DNS request records for one +// application VIF. Iterate is called for each message that passes the match +// filter. Returning stop=true signals that no further messages are needed +// and iteration should stop cleanly. Returning a non-nil error aborts +// iteration and propagates the error to the caller. +type FlowMsgIterator interface { + Iterate(msg *eveflowlog.FlowMessage) (stop bool, err error) +} + // NewAdamClient creates a new AdamClient. // The caller is responsible for providing a CA certificate and key used // to sign all Adam server certificates. @@ -819,7 +830,11 @@ func (ac *AdamClient) SubscribeToDeviceRequests( } func() { - defer current.Body.Close() + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() dec := json.NewDecoder(current.Body) for { var event ReqEvent @@ -928,7 +943,11 @@ func (ac *AdamClient) SubscribeToDeviceLogs( } func() { - defer current.Body.Close() + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() dec := json.NewDecoder(current.Body) for { var raw json.RawMessage @@ -1239,7 +1258,11 @@ func (ac *AdamClient) SubscribeToAppLogs( } func() { - defer current.Body.Close() + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() dec := json.NewDecoder(current.Body) for { var raw json.RawMessage @@ -1447,7 +1470,11 @@ func (ac *AdamClient) SubscribeToDeviceInfoMsgs(devUUID uuid.UUID, } func() { - defer current.Body.Close() + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() dec := json.NewDecoder(current.Body) for { var raw json.RawMessage @@ -1642,7 +1669,11 @@ func (ac *AdamClient) SubscribeToDeviceMetrics(devUUID uuid.UUID, } func() { - defer current.Body.Close() + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() dec := json.NewDecoder(current.Body) for { var raw json.RawMessage @@ -1684,6 +1715,224 @@ func (ac *AdamClient) SubscribeToDeviceMetrics(devUUID uuid.UUID, return unsubscribe, nil } +// IterateDeviceFlowLogs retrieves flow log messages (FlowMessage) published +// by the specified device and passes matching messages to iterator. Flow +// messages are stored and served per-device, not per-app (each one carries +// a Scope identifying which app/VIF it belongs to), so callers that only +// care about one application filter by msg.GetScope().GetUuid() in match (or +// inside iterator) themselves. +// +// It first performs a one-shot GET request to fetch all currently available +// messages. If follow is true, it then subscribes to the streaming endpoint +// and continues delivering new messages until ctx is canceled. +// +// If match is non-nil, only messages for which match(msg) returns true are +// iterated. If match is nil, all messages are iterated. +func (ac *AdamClient) IterateDeviceFlowLogs(ctx context.Context, devUUID uuid.UUID, + match func(msg *eveflowlog.FlowMessage) bool, iterator FlowMsgIterator, + follow bool) error { + if err := ac.checkAdamRunning(); err != nil { + return err + } + + ac.mutex.Lock() + _, known := ac.knownDevices[devUUID] + ac.mutex.Unlock() + if !known { + return fmt.Errorf("unknown device UUID %q", devUUID) + } + + // -------- Initial GET -------- + + url := ac.adminURL("device/" + devUUID.String() + "/flowlogs") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to create GET %s request: %w", url, err) + } + + resp, err := ac.httpClient().Do(req) + if err != nil { + return fmt.Errorf("GET %s failed: %w", url, err) + } + defer resp.Body.Close() + + // No flow logs recorded yet for this device. + if resp.StatusCode != http.StatusNotFound { + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status from GET %s: %d", url, resp.StatusCode) + } + + dec := json.NewDecoder(resp.Body) + for { + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("failed to decode flow message JSON: %w", err) + } + + msg := &eveflowlog.FlowMessage{} + if err := protojson.Unmarshal(raw, msg); err != nil { + return fmt.Errorf("failed to proto-unmarshal flow message: %w", err) + } + if match == nil || match(msg) { + stop, iterErr := iterator.Iterate(msg) + if iterErr != nil { + return fmt.Errorf("failed to iterate flow message: %w", iterErr) + } + if stop { + return nil + } + } + } + } + + // -------- Follow mode -------- + + if !follow { + return nil + } + + flowMsgCh := make(chan *eveflowlog.FlowMessage, 100) + unsubscribe, err := ac.SubscribeToDeviceFlowLogs(devUUID, match, flowMsgCh) + if err != nil { + return err + } + defer unsubscribe() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case msg := <-flowMsgCh: + stop, iterErr := iterator.Iterate(msg) + if iterErr != nil { + return fmt.Errorf("failed to iterate flow message: %w", iterErr) + } + if stop { + return nil + } + } + } +} + +// SubscribeToDeviceFlowLogs subscribes to flow log messages (FlowMessage) +// emitted by the specified device and delivers matching messages to +// channel. Flow messages are stored and served per-device, not per-app; see +// IterateDeviceFlowLogs. +// +// If match is non-nil, only messages for which match(msg) returns true are +// forwarded. If match is nil, all messages are delivered. +// +// The streaming connection is opened synchronously: by the time this method +// returns, Adam has accepted the request and any subsequent flow messages +// for the device will be delivered. On transient failures after the initial +// connection, a background goroutine reconnects with a fixed retry delay. +// +// The returned unsubscribe function stops the background stream and waits +// for it to exit. It is safe to call multiple times. The channel is closed +// when the subscription ends. +func (ac *AdamClient) SubscribeToDeviceFlowLogs(devUUID uuid.UUID, + match func(msg *eveflowlog.FlowMessage) bool, + channel chan<- *eveflowlog.FlowMessage) (unsubscribe func(), err error) { + const retryDelay = 3 * time.Second + + if err = ac.checkAdamRunning(); err != nil { + return nil, err + } + + ac.mutex.Lock() + _, known := ac.knownDevices[devUUID] + ac.mutex.Unlock() + if !known { + return nil, fmt.Errorf("unknown device UUID %q", devUUID) + } + + streamCtx, cancel := context.WithCancel(context.Background()) + url := ac.adminURL("device/" + devUUID.String() + "/flowlogs") + + resp, err := ac.openStream(streamCtx, url) + if err != nil { + cancel() + return nil, err + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + defer close(channel) + + current := resp + for { + if current == nil { + select { + case <-time.After(retryDelay): + case <-streamCtx.Done(): + return + } + r, err := ac.openStream(streamCtx, url) + if err != nil { + if streamCtx.Err() != nil { + return + } + ac.log.Errorf("failed to reopen flow log stream: %v", err) + continue + } + current = r + } + + func() { + defer func() { + if err := current.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() + dec := json.NewDecoder(current.Body) + for { + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + if streamCtx.Err() != nil { + return + } + if errors.Is(err, io.EOF) { + ac.log.Warn("flow log stream closed by server") + return + } + ac.log.Errorf("failed to decode streamed flow message: %v", err) + return + } + msg := &eveflowlog.FlowMessage{} + if err := protojson.Unmarshal(raw, msg); err != nil { + ac.log.Errorf( + "failed to proto-unmarshal streamed flow message: %v", err) + continue + } + if match != nil && !match(msg) { + continue + } + select { + case channel <- msg: + case <-streamCtx.Done(): + return + } + } + }() + current = nil + } + }() + + var once sync.Once + unsubscribe = func() { + once.Do(func() { + cancel() + wg.Wait() + }) + } + return unsubscribe, nil +} + // findDeviceUUID searches Adam for a device with certificates/serial matching // the given callback and returns its UUID if found. func (ac *AdamClient) findDeviceUUID(ctx context.Context, httpClient *http.Client, @@ -1700,7 +1949,11 @@ func (ac *AdamClient) findDeviceUUID(ctx context.Context, httpClient *http.Clien if err != nil { return uuid.Nil, false, fmt.Errorf("GET %s failed: %w", url, err) } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + ac.log.Warnf("failed to close response body: %v", err) + } + }() if resp.StatusCode != http.StatusOK { return uuid.Nil, false, fmt.Errorf( diff --git a/evetest/devconfig.go b/evetest/devconfig.go index 5a2a752c43d..553b17838ed 100644 --- a/evetest/devconfig.go +++ b/evetest/devconfig.go @@ -871,24 +871,67 @@ func (config SwitchNetworkInstanceConfig) toProto(th *TestHarness, // ApplicationInstanceConfig wraps configuration for a single application deployed on EVE. type ApplicationInstanceConfig struct { - DisplayName string - Activate bool - ProfileList []string - Image ApplicationImageStorage - VirtualizationMode eveconfig.VmMode - CPUs uint - MemoryBytes uint64 - DiskBytes uint64 - EnableVNC bool - VNCDisplay uint - VNCPassword string + DisplayName string + Activate bool + ProfileList []string + Image ApplicationImageStorage + VirtualizationMode eveconfig.VmMode + CPUs uint + MemoryBytes uint64 + DiskBytes uint64 + EnableVNC bool + VNCDisplay uint + VNCPassword string + // RemoteConsole gates VNC access under the Kubevirt hypervisor: unlike + // KVM (where EnableVNC/VNCDisplay/VNCPassword directly configure a raw + // QEMU VNC socket reachable on the device's uplink IP), Kubevirt exposes + // VNC via zedkube's virtctl-based proxy, which only binds the VNC port + // on 127.0.0.1 and is gated by this field, not by EnableVNC. VNCDisplay + // still selects the port (5900+VNCDisplay); VNCPassword is not enforced + // -- see pkg/pillar/docs/vnc-workflows.md. zedkube only starts/stops the + // proxy on a *change* of this field, and only one remote-console session + // is allowed on the device at a time. + RemoteConsole bool DisableLogs bool UserData string NetworkAdapters []AppNetworkAdapter EnforceNetIntfOrder bool + // Mounts are additional volumes attached to the application alongside its + // root disk (built from Image/DiskBytes). Each entry references an + // existing, independently created volume (see EdgeDeviceConfig.AddVolume + // / AddBlankVolume) -- AddApplication/UpdateApplication only wire up (or + // rewire) the app's VolumeRefList to point at it; they never create or + // remove the volume itself. + Mounts []MountConfig // Many more parameters can be configured; they will be added later as needed. } +// MountConfig attaches an existing volume to an application, in addition to +// its root disk. The volume must already exist in the device configuration +// (created via EdgeDeviceConfig.AddVolume or AddBlankVolume) -- +// AddApplication/UpdateApplication fail if VolumeUUID does not reference one. +// The volume's lifecycle is independent of any application that mounts it: +// deleting the application does not delete it (use DeleteVolume explicitly), +// and DeleteVolume itself fails while any application still mounts it. +type MountConfig struct { + // VolumeUUID is the UUID of an existing volume, as returned by AddVolume + // or AddBlankVolume. + VolumeUUID uuid.UUID + // MountDir is the path where the volume is mounted inside the guest. + // Empty means the volume is attached as a raw block device instead: no + // guest-side mount is performed, and the disk shows up as a plain block + // device (e.g. /dev/sdX for a VM, or a device node under + // /dev/eve/volumes/by-id for a container) -- see VolumeRefConfig.MountDir + // handling in pillar's zedmanager/domainmgr/containerd. + MountDir string + // ReadOnly marks the volume as read-only inside the guest. This sets + // Volume.Readonly, a property of the volume itself (the EVE API has no + // separate per-reference read-only flag) -- so mounting the same volume + // read-write from one application and read-only from another is not + // possible; the last MountConfig applied wins. + ReadOnly bool +} + func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, appUUID, volumeUUID uuid.UUID) *eveconfig.AppInstanceConfig { vmConfig := &eveconfig.VmConfig{ @@ -912,6 +955,7 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, Fixedresources: vmConfig, Activate: config.Activate, ProfileList: config.ProfileList, + RemoteConsole: config.RemoteConsole, } if volumeUUID != NilUUID { appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, @@ -1052,6 +1096,15 @@ type DockerContainer struct { // Username and password are not configurable here. // Instead, evetest pulls credentials from the docker client running on the host // (docker socket will be mounted to evetest container). + + // TrustedCACertsPEM lists CA certificates EVE should trust when connecting + // to Domain over HTTPS, in addition to its normal trust store -- e.g. the + // harness's own CA (GetCACertPEM) when Domain points at evetest's embedded + // OCI registry (see PushDockerImageToLocalRegistry), or an enterprise + // registry's own internal CA. Mirrors HTTPStorage.HTTPSTrustedCACertsPEM; + // leave unset for a registry already trusted via the normal chain (e.g. + // Docker Hub). + TrustedCACertsPEM []string } func (container DockerContainer) toProto(th *TestHarness, log *logrus.Entry, @@ -1094,6 +1147,9 @@ func (container DockerContainer) toProto(th *TestHarness, log *logrus.Entry, } else { dsConfig.Fqdn = fmt.Sprintf("docker://%s", domain) } + for _, cert := range container.TrustedCACertsPEM { + dsConfig.DsCertPEM = append(dsConfig.DsCertPEM, []byte(cert)) + } username, password, err := utils.GetDockerAuthPlain(log, dsConfig.Fqdn) if err != nil { // Just log warning, container will be pulled without authentication. @@ -2272,13 +2328,65 @@ func (dc *EdgeDeviceConfig) addApplicationWithUUIDs( contentTreeUUID, datastoreUUID, config.DisplayName) dc.ContentInfo = append(dc.ContentInfo, contentTree) dc.Datastores = append(dc.Datastores, dsConfig) + + // appInstConfig.VolumeRefList currently holds only the root disk (index + // 0); append one VolumeRef per mount, referencing already-existing + // volumes (see MountConfig / buildMountRefs). + appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, + dc.buildMountRefs(config.DisplayName, config.Mounts)...) +} + +// findVolume returns the Volume with the given UUID, or nil if not found. +func (dc *EdgeDeviceConfig) findVolume(volUUID string) *eveconfig.Volume { + for _, v := range dc.Volumes { + if v.Uuid == volUUID { + return v + } + } + return nil +} + +// buildMountRefs validates config.Mounts against the device's existing +// volumes (see AddVolume/AddBlankVolume) and returns the corresponding +// VolumeRefs. It fails the test if a mount references a volume that does not +// exist, or if the same volume is mounted more than once by appDisplayName. +// It does not create or remove any volume -- that is the caller's +// responsibility via AddVolume/AddBlankVolume/DeleteVolume. +func (dc *EdgeDeviceConfig) buildMountRefs( + appDisplayName string, mounts []MountConfig) []*eveconfig.VolumeRef { + seen := make(map[string]bool, len(mounts)) + refs := make([]*eveconfig.VolumeRef, 0, len(mounts)) + for _, mount := range mounts { + volUUIDStr := mount.VolumeUUID.String() + volume := dc.findVolume(volUUIDStr) + if volume == nil { + dc.th.t.Fatalf( + "Application %q mount references non-existent volume %q "+ + "(create it first with AddVolume/AddBlankVolume)", + appDisplayName, volUUIDStr) + continue + } + if seen[volUUIDStr] { + dc.th.t.Fatalf("Application %q mounts volume %q more than once", + appDisplayName, volUUIDStr) + continue + } + seen[volUUIDStr] = true + volume.Readonly = mount.ReadOnly + refs = append(refs, &eveconfig.VolumeRef{ + Uuid: volUUIDStr, + MountDir: mount.MountDir, + }) + } + return refs } // UpdateApplication updates an existing application instance identified // by its UUID. func (dc *EdgeDeviceConfig) UpdateApplication( appUUID uuid.UUID, newConfig ApplicationInstanceConfig) { - // For now, we will only allow to change Activation flag, profile list and adapters. + // For now, we will only allow to change Activation flag, profile list, + // adapters, and mounts. for i, app := range dc.Apps { if app.Uuidandversion.Uuid == appUUID.String() { newProtoConfig := newConfig.toProto(dc.th, dc.DeviceName, appUUID, NilUUID) @@ -2301,6 +2409,17 @@ func (dc *EdgeDeviceConfig) UpdateApplication( if !generics.EqualSetsFn(app.Interfaces, newProtoConfig.Interfaces, equalNetAdapter) { needPurge = true } + // The root ref (VolumeRefList[0]) is always left untouched; + // buildMountRefs only ever references existing volumes, it does + // not create or remove any. Any change to the mount refs + // (add/remove/move) requires a purge. + newMountRefs := dc.buildMountRefs(newConfig.DisplayName, newConfig.Mounts) + equalVolumeRef := func(a1, a2 *eveconfig.VolumeRef) bool { + return proto.Equal(a1, a2) + } + if !generics.EqualSetsFn(app.VolumeRefList[1:], newMountRefs, equalVolumeRef) { + needPurge = true + } if needPurge { if app.Purge == nil { app.Purge = &eveconfig.InstanceOpsCmd{Counter: 0} @@ -2311,6 +2430,7 @@ func (dc *EdgeDeviceConfig) UpdateApplication( dc.Apps[i].ProfileList = newProtoConfig.ProfileList dc.Apps[i].Adapters = newProtoConfig.Adapters dc.Apps[i].Interfaces = newProtoConfig.Interfaces + dc.Apps[i].VolumeRefList = append(app.VolumeRefList[:1:1], newMountRefs...) return } } @@ -2319,14 +2439,19 @@ func (dc *EdgeDeviceConfig) UpdateApplication( } // DeleteApplication removes an application instance identified by its UUID -// and cleans up all associated resources. +// and its root volume (the one created together with it from +// ApplicationInstanceConfig.Image/DiskBytes). Any volumes referenced through +// Mounts are left untouched -- their lifecycle is independent of the +// application (see MountConfig); remove them explicitly with DeleteVolume. func (dc *EdgeDeviceConfig) DeleteApplication(appUUID uuid.UUID) { var found bool - var volumeRefs []*eveconfig.VolumeRef + var rootVolUUID string uuidStr := appUUID.String() for i, app := range dc.Apps { if app.Uuidandversion.Uuid == uuidStr { - volumeRefs = app.VolumeRefList + if len(app.VolumeRefList) > 0 { + rootVolUUID = app.VolumeRefList[0].Uuid + } // Remove the application instance from the slice. dc.Apps = append(dc.Apps[:i], dc.Apps[i+1:]...) found = true @@ -2336,46 +2461,49 @@ func (dc *EdgeDeviceConfig) DeleteApplication(appUUID uuid.UUID) { if !found { dc.th.t.Fatalf("Application instance with UUID %q was not found", uuidStr) } + if rootVolUUID != "" { + dc.deleteVolumeAndDeps(rootVolUUID) + } +} - // Remove volumes for the app. - var contentTreeIDs []string - for _, volumeRef := range volumeRefs { - for i, volume := range dc.Volumes { - if volume.Uuid == volumeRef.Uuid { - if volume.Origin.Type == eveconfig.VolumeContentOriginType_VCOT_DOWNLOAD { - contentTreeIDs = append(contentTreeIDs, - volume.Origin.DownloadContentTreeID) - } - // Remove the volume from the slice. - dc.Volumes = append(dc.Volumes[:i], dc.Volumes[i+1:]...) - break +// deleteVolumeAndDeps removes the Volume with the given UUID from the device +// configuration and, if it is a VCOT_DOWNLOAD volume, its ContentTree and the +// Datastores that ContentTree referenced. Returns whether the volume was +// found. +func (dc *EdgeDeviceConfig) deleteVolumeAndDeps(volUUID string) bool { + var contentTreeID string + var found bool + for i, volume := range dc.Volumes { + if volume.Uuid == volUUID { + if volume.Origin.Type == eveconfig.VolumeContentOriginType_VCOT_DOWNLOAD { + contentTreeID = volume.Origin.DownloadContentTreeID } + dc.Volumes = append(dc.Volumes[:i], dc.Volumes[i+1:]...) + found = true + break } } + if !found || contentTreeID == "" { + return found + } - // Remove content trees created for the app. var datastoreIDs []string - for _, contentTreeID := range contentTreeIDs { - for i, contentTree := range dc.ContentInfo { - if contentTree.Uuid == contentTreeID { - datastoreIDs = append(datastoreIDs, contentTree.DsIdsList...) - // Remove the content tree from the slice. - dc.ContentInfo = append(dc.ContentInfo[:i], dc.ContentInfo[i+1:]...) - break - } + for i, contentTree := range dc.ContentInfo { + if contentTree.Uuid == contentTreeID { + datastoreIDs = append(datastoreIDs, contentTree.DsIdsList...) + dc.ContentInfo = append(dc.ContentInfo[:i], dc.ContentInfo[i+1:]...) + break } } - - // Remove datastore configs created for the app. for _, datastoreID := range datastoreIDs { for i, datastore := range dc.Datastores { if datastore.Id == datastoreID { - // Remove the datastore config from the slice. dc.Datastores = append(dc.Datastores[:i], dc.Datastores[i+1:]...) break } } } + return true } // AddBlankVolume adds a standalone empty (VCOT_BLANK) volume of the requested @@ -2409,6 +2537,180 @@ func (dc *EdgeDeviceConfig) AddBlankVolume( return volumeUUID } +// AddVolume adds a standalone (app-unreferenced) VCOT_DOWNLOAD volume of the +// requested size, with content sourced from image, and returns its UUID. +// +// Unlike AddBlankVolume, the volume's content is populated by a real +// download (e.g. from a docker registry, or evetest's built-in HTTP/SFTP +// image server -- see CreateBlankImageFile), so this exercises the +// download and (for non-raw formats) disk-format conversion path in +// volumemgr, rather than the empty zero-content path. +func (dc *EdgeDeviceConfig) AddVolume( + displayName string, image ApplicationImageStorage, sizeBytes uint64) uuid.UUID { + volumeUUID := dc.th.newUUID("volume") + contentTreeUUID := dc.th.newUUID("volume content tree") + datastoreUUID := dc.th.newUUID("volume datastore") + dc.Volumes = append(dc.Volumes, &eveconfig.Volume{ + Uuid: volumeUUID.String(), + Origin: &eveconfig.VolumeContentOrigin{ + Type: eveconfig.VolumeContentOriginType_VCOT_DOWNLOAD, + DownloadContentTreeID: contentTreeUUID.String(), + }, + Maxsizebytes: int64(sizeBytes), + DisplayName: displayName, + }) + contentTree, dsConfig := image.toProto(dc.th, dc.log, dc.DeviceName, + contentTreeUUID, datastoreUUID, displayName) + dc.ContentInfo = append(dc.ContentInfo, contentTree) + dc.Datastores = append(dc.Datastores, dsConfig) + return volumeUUID +} + +// DeleteVolume removes a standalone volume (created via AddVolume or +// AddBlankVolume) and its associated ContentTree/Datastore, if any, from the +// device configuration. The volume must not currently be referenced by any +// application; detach it first with DetachVolume. +func (dc *EdgeDeviceConfig) DeleteVolume(volumeUUID uuid.UUID) { + uuidStr := volumeUUID.String() + for _, app := range dc.Apps { + for _, ref := range app.VolumeRefList { + if ref.Uuid == uuidStr { + dc.th.t.Fatalf( + "Cannot delete volume %q: still attached to application %q", + uuidStr, app.Displayname) + } + } + } + if !dc.deleteVolumeAndDeps(uuidStr) { + dc.th.t.Fatalf("Volume with UUID %q was not found", uuidStr) + } +} + +// DiskLayoutType is the desired ZFS/RAID disk-array layout for +// EdgeDeviceConfig.SetDisksLayout. +type DiskLayoutType int + +const ( + // DiskLayoutUnspecified leaves EVE's automatic (default) disk selection + // in place -- no explicit layout is configured. + DiskLayoutUnspecified DiskLayoutType = iota + // DiskLayoutRAID1 mirrors 2 disks. + DiskLayoutRAID1 + // DiskLayoutRAID10 stripes 2 RAID1 mirrors (4 disks total). + DiskLayoutRAID10 +) + +// DisksLayout describes the desired ZFS/RAID disk-array layout of a device's +// extra (non-boot) virtio disks (see RequireEdgeDevice.ExtraDisks), mirroring +// eden's own disks-layout model (pkg/device/disksLayout.go in the eden +// repo, adapted here since evetest's broker only ever attaches virtio +// disks -- see DiskName). Disk slot indices below (0..3 for RAID10, 0..1 for +// RAID1) refer to positions within the layout, not the device's overall +// extra-disk list. +type DisksLayout struct { + LayoutType DiskLayoutType + // OfflineDisks marks disk slots as ZFS_OFFLINE. + OfflineDisks []uint + // UnusedDisks marks disk slots as UNUSED (removed from the array). + UnusedDisks []uint + // ReplaceDisks requests replacing the disk currently at the given slot + // with a spare disk, selected in order from the device's extra disks + // beyond what the layout itself occupies (e.g. for RAID1, slot 0's + // replacement is the device's 3rd extra disk). + ReplaceDisks []uint +} + +// DiskName returns the virtio device name (e.g. "/dev/vdc") of the +// zero-based extraDiskIdx-th extra (non-boot) disk requested via +// RequireEdgeDevice.ExtraDisks: extraDiskIdx 0 is /dev/vdb, since /dev/vda is +// always the device's boot disk. +func DiskName(extraDiskIdx uint) string { + return fmt.Sprintf("/dev/vd%c", rune('b'+extraDiskIdx)) +} + +// maxDisks returns how many disk slots this layout occupies. +func (layout DisksLayout) maxDisks() uint { + switch layout.LayoutType { + case DiskLayoutRAID1: + return 2 + case DiskLayoutRAID10: + return 4 + default: + return 0 + } +} + +func (layout DisksLayout) diskState(slot uint) eveconfig.DiskConfigType { + for _, s := range layout.OfflineDisks { + if s == slot { + return eveconfig.DiskConfigType_DISK_CONFIG_TYPE_ZFS_OFFLINE + } + } + for _, s := range layout.UnusedDisks { + if s == slot { + return eveconfig.DiskConfigType_DISK_CONFIG_TYPE_UNUSED + } + } + return eveconfig.DiskConfigType_DISK_CONFIG_TYPE_ZFS_ONLINE +} + +func (layout DisksLayout) diskConfig(slot uint) *eveconfig.DiskConfig { + cfg := &eveconfig.DiskConfig{ + Disk: &evecommon.DiskDescription{Name: DiskName(slot)}, + DiskConfig: layout.diskState(slot), + } + for i, s := range layout.ReplaceDisks { + if s == slot { + cfg.OldDisk = cfg.Disk + cfg.Disk = &evecommon.DiskDescription{ + Name: DiskName(layout.maxDisks() + uint(i)), + } + break + } + } + return cfg +} + +// SetDisksLayout configures the device's ZFS/RAID disk-array layout for its +// extra (non-boot) virtio disks (see RequireEdgeDevice.ExtraDisks). +// +// Both DiskLayoutRAID1 and DiskLayoutRAID10 are represented, at the EVE API +// level, as a top-level DisksConfig with ArrayType RAID0 (stripe) whose +// Children are the individual RAID1 mirrors -- a single mirror for RAID1, two +// mirrors for RAID10 -- exactly matching how eden itself builds this +// structure (pkg/device/disksLayout.go, GetDisksConfig). +func (dc *EdgeDeviceConfig) SetDisksLayout(layout DisksLayout) { + var cfg eveconfig.DisksConfig + switch layout.LayoutType { + case DiskLayoutUnspecified: + // Nothing to configure. + case DiskLayoutRAID1: + cfg.ArrayType = eveconfig.DisksArrayType_DISKS_ARRAY_TYPE_RAID0 + cfg.Children = append(cfg.Children, &eveconfig.DisksConfig{ + ArrayType: eveconfig.DisksArrayType_DISKS_ARRAY_TYPE_RAID1, + Disks: []*eveconfig.DiskConfig{ + layout.diskConfig(0), layout.diskConfig(1)}, + }) + case DiskLayoutRAID10: + cfg.ArrayType = eveconfig.DisksArrayType_DISKS_ARRAY_TYPE_RAID0 + cfg.Children = append(cfg.Children, + &eveconfig.DisksConfig{ + ArrayType: eveconfig.DisksArrayType_DISKS_ARRAY_TYPE_RAID1, + Disks: []*eveconfig.DiskConfig{ + layout.diskConfig(0), layout.diskConfig(1)}, + }, + &eveconfig.DisksConfig{ + ArrayType: eveconfig.DisksArrayType_DISKS_ARRAY_TYPE_RAID1, + Disks: []*eveconfig.DiskConfig{ + layout.diskConfig(2), layout.diskConfig(3)}, + }, + ) + default: + dc.th.t.Fatalf("Unsupported disk layout type: %v", layout.LayoutType) + } + dc.Disks = &cfg +} + // SetLPS configures the Local Profile Server (LPS) settings for the device. func (dc *EdgeDeviceConfig) SetLPS(config LPSConfig) { dc.GlobalProfile = config.GlobalProfile diff --git a/evetest/diskimages.go b/evetest/diskimages.go new file mode 100644 index 00000000000..6615416f2c6 --- /dev/null +++ b/evetest/diskimages.go @@ -0,0 +1,100 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package evetest + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strconv" + + eveconfig "github.com/lf-edge/eve-api/go/config" +) + +// CreateBlankImageFile creates an empty disk image of the given format and +// size, served by evetest's built-in image server (both over HTTP and +// SFTP), and returns its filename (for ImageRelativePath) together with the +// hex-encoded SHA256 of the resulting file's content (for ImageSHA256). +// +// Requires qemu-img to be available inside the evetest container (already a +// build dependency of the broker). +func CreateBlankImageFile( + name string, format eveconfig.Format, sizeBytes uint64) (relativePath, sha256Hex string) { + th := getTestHarness() + formatStr, ok := qemuImgFormat(format) + if !ok { + th.t.Fatalf( + "unsupported disk image format for CreateBlankImageFile: %v", format) + } + path := filepath.Join(th.imgServerDir, name) + cmd := exec.Command("qemu-img", "create", "-f", formatStr, + path, strconv.FormatUint(sizeBytes, 10)) + if out, err := cmd.CombinedOutput(); err != nil { + th.t.Fatalf("qemu-img create -f %s %s %d failed: %v: %s", + formatStr, path, sizeBytes, err, out) + } + content, err := os.ReadFile(path) + if err != nil { + th.t.Fatalf("failed to read back created disk image %s: %v", path, err) + } + sum := sha256.Sum256(content) + return name, hex.EncodeToString(sum[:]) +} + +// CreateRandomImageFile creates a file of sizeBytes random bytes, served by +// evetest's built-in image server (see AddImageServerFile), and returns its +// filename (for ImageRelativePath) together with the hex-encoded SHA256 of +// its content (for ImageSHA256). +// +// Random (non-blank) content makes ImageSHA256 verification meaningful: a +// blank file's checksum can't distinguish "downloaded correctly" from +// "downloaded as all zeros/corrupted-but-still-blank" -- a corrupted +// download of random content is vanishingly unlikely to still match the +// checksum computed here. +func CreateRandomImageFile(name string, sizeBytes uint64) (relativePath, sha256Hex string) { + th := getTestHarness() + content := make([]byte, sizeBytes) + if _, err := rand.Read(content); err != nil { + th.t.Fatalf("failed to generate random content for %s: %v", name, err) + } + sum := sha256.Sum256(content) + relativePath = AddImageServerFile(name, content) + return relativePath, hex.EncodeToString(sum[:]) +} + +// AddImageServerFile writes content to a file served by evetest's built-in +// image server (both over HTTP and SFTP), returning its filename for use as +// HTTPStorage.ImageRelativePath / SFTPStorage.ImageRelativePath. +func AddImageServerFile(name string, content []byte) string { + th := getTestHarness() + path := filepath.Join(th.imgServerDir, name) + if err := os.WriteFile(path, content, 0o644); err != nil { + th.t.Fatalf("failed to write image server file %s: %v", name, err) + } + return name +} + +// qemuImgFormat maps an eveconfig.Format to the "-f" format name accepted by +// qemu-img. +func qemuImgFormat(format eveconfig.Format) (string, bool) { + switch format { + case eveconfig.Format_RAW: + return "raw", true + case eveconfig.Format_QCOW: + return "qcow", true + case eveconfig.Format_QCOW2: + return "qcow2", true + case eveconfig.Format_VHD: + return "vpc", true // qemu-img calls the VHD format "vpc" + case eveconfig.Format_VMDK: + return "vmdk", true + case eveconfig.Format_VHDX: + return "vhdx", true + default: + return "", false + } +} diff --git a/evetest/edgecluster.go b/evetest/edgecluster.go index 3ebb30da72c..10582d9709c 100644 --- a/evetest/edgecluster.go +++ b/evetest/edgecluster.go @@ -11,7 +11,6 @@ import ( "time" eveinfo "github.com/lf-edge/eve-api/go/info" - "github.com/lf-edge/eve/pkg/pillar/utils/generics" uuid "github.com/satori/go.uuid" ) @@ -86,8 +85,16 @@ func (ec *EdgeCluster) WaitUntilNodesAreReady(timeout time.Duration) { closeOnce := sync.Once{} RunParallel(len(ec.devices), func(i int) { dev := ec.devices[i] + // Subscribe before taking the initial snapshot, so a transition + // landing between the two calls can never be missed. updates, stop := dev.WatchClusterInfo() defer stop() + if info := dev.GetClusterInfo(); info != nil && allNodesReady(info, expectedNodes) { + ec.th.log.Infof("All cluster nodes already ready per device %q", + dev.devName) + closeOnce.Do(func() { close(doneCh) }) + return + } var tickerCh <-chan time.Time if i == 0 { ticker := time.NewTicker(1 * time.Minute) @@ -121,20 +128,36 @@ func (ec *EdgeCluster) WaitUntilNodesAreReady(timeout time.Duration) { }) } -// allNodesReady returns true if the cluster info reports every node in -// expectedNodes as Ready. +// allNodesReady returns true if the cluster storage is healthy and every +// node in expectedNodes is reported as Ready. func allNodesReady(info *eveinfo.ZInfoKubeCluster, expectedNodes []string) bool { + for _, name := range expectedNodes { + if !clusterNodeReady(info, name) { + return false + } + } + return true +} + +// clusterNodeReady returns true if the given cluster info reports the named +// node as Ready and the cluster's storage as healthy. +func clusterNodeReady(info *eveinfo.ZInfoKubeCluster, nodeName string) bool { const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY - var readyNodes []string + if info.GetStorage().GetHealth() != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { + return false + } for _, node := range info.GetNodes() { + if node.GetName() != nodeName { + continue + } for _, cond := range node.GetConditions() { - if cond.GetType() == nodeReadyCond && cond.GetSet() { - readyNodes = append(readyNodes, node.GetName()) - break + if cond.GetType() == nodeReadyCond { + return cond.GetSet() } } + return false } - return generics.EqualSets(readyNodes, expectedNodes) + return false } // FindDeviceHostingApp finds the cluster device that hosts the given application. diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 206622299a6..64d8b033646 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -21,6 +21,7 @@ import ( eveinfo "github.com/lf-edge/eve-api/go/info" evelogs "github.com/lf-edge/eve-api/go/logs" evemetrics "github.com/lf-edge/eve-api/go/metrics" + "github.com/lf-edge/eve/evetest/constants" api "github.com/lf-edge/eve/evetest/grpcapi/go" "github.com/lf-edge/eve/evetest/logger" "github.com/lf-edge/eve/evetest/utils" @@ -350,15 +351,85 @@ func (d *EdgeDevice) GetDeviceIPAddress(netAdapterLogicalLabel string) []net.IP return ips } +// GetArch returns the CPU architecture of the device ("amd64" or "arm64"), +// as determined during Setup (see TestHarness.selectArch) -- not merely the +// preferred architecture requested via EVETEST_PREFERRED_ARCH, which can +// differ from the device's actual one on a broker that does not support it +// (selectArch falls back to whatever the broker does support). +func (d *EdgeDevice) GetArch() string { + d.th.devicesM.Lock() + devState, found := d.th.devices[d.devName] + d.th.devicesM.Unlock() + if !found { + d.th.t.Fatalf("Unknown device %q", d.devName) + } + switch devState.imageRef.Arch { + case api.ArchType_ARCH_AMD64: + return "amd64" + case api.ArchType_ARCH_ARM64: + return "arm64" + default: + d.th.t.Fatalf("Device %q has unknown architecture: %v", + d.devName, devState.imageRef.Arch) + return "" + } +} + +// BaseOSDatastoreType selects how EdgeDevice.UpgradeEVE delivers the target +// EVE rootfs to the device. +type BaseOSDatastoreType int + +const ( + // BaseOSDatastoreHTTP has evetest extract the raw rootfs image from the + // locally-pulled EVE docker image and serve it over evetest's own + // embedded HTTP image server. This is the traditional/default path and + // works even when the target image is not (or cannot be) hosted on a + // registry reachable from the device. + BaseOSDatastoreHTTP BaseOSDatastoreType = iota + // BaseOSDatastoreOCI has EVE pull the target rootfs directly from the + // same OCI registry the target image was tagged in (e.g. Docker Hub), + // using a container-registry Datastore. evetest still pulls the image + // locally first to determine the target short version, but skips + // extracting the raw rootfs and serving it over HTTP. + BaseOSDatastoreOCI +) + +func (t BaseOSDatastoreType) String() string { + switch t { + case BaseOSDatastoreOCI: + return "oci" + case BaseOSDatastoreHTTP: + fallthrough + default: + return "http" + } +} + +// FromString parses a datastore type name string and sets the +// BaseOSDatastoreType value. +func (t *BaseOSDatastoreType) FromString(s string) error { + switch strings.ToLower(s) { + case "", "http": + *t = BaseOSDatastoreHTTP + case "oci": + *t = BaseOSDatastoreOCI + default: + return fmt.Errorf("invalid BaseOSDatastoreType: %q", s) + } + return nil +} + // UpgradeEVE upgrades the EVE OS to the specified version and optionally // waits until the upgrade completes or reverts. +// datastoreType selects how the target rootfs is delivered to the device -- +// see BaseOSDatastoreType. // When expectRevert is true, the upgrade is expected to fail and EVE to revert // to the previous version -- the function then waits for the target version to // show a FAILED status instead of waiting for it to become active. // A reverted upgrade causes two reboots (one to try the new version, one to // revert), so the expected reboot count is incremented accordingly. func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hypervisor, - waitUntilUpgraded bool, expectRevert bool) { + datastoreType BaseOSDatastoreType, waitUntilUpgraded bool, expectRevert bool) { // Read current device arch (set during Setup). d.th.devicesM.Lock() @@ -370,12 +441,25 @@ func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hyp currentImageRef := devState.imageRef d.th.devicesM.Unlock() + // BaseOSDatastoreOCI has EVE pull a container image, while the live image transport + // delivers a local disk image instead of a container. + // This combination does not make sense. + if datastoreType == BaseOSDatastoreOCI && LocalLiveImageRequested() { + d.th.t.Fatalf( + "UpgradeEVE: BaseOSDatastoreOCI requires EVE to pull a container image, but "+ + "%s%s selects a local EVE live (disk) image instead of a container image -- "+ + "unset %s%s to upgrade via OCI, or use BaseOSDatastoreHTTP to upgrade with "+ + "the live image", + constants.EnvPrefix, constants.EVELiveImageEnv, + constants.EnvPrefix, constants.EVELiveImageEnv) + } + // The live transport delivers an upgrade as the raw rootfs the local build // already contains, rather than pulling a container image to extract the same // file from. Which build that is comes from the version axis exactly as it // does for a fresh device, so an explicitly requested target version is // honoured (and must be built locally) while an unset one means the newest. - if LocalLiveImageRequested() { + if datastoreType == BaseOSDatastoreHTTP && LocalLiveImageRequested() { d.upgradeEVEFromLocalBuild(targetEVEVersion, currentImageRef.Arch, currentImageRef.Hypervisor, waitUntilUpgraded, expectRevert) return @@ -418,6 +502,20 @@ func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hyp shortVersion := strings.TrimSpace(versionOut) d.th.log.Debugf("Target EVE short version is %q", shortVersion) + if datastoreType == BaseOSDatastoreOCI { + // Publish imageName to evetest's own embedded OCI registry. + dockerContainer, err := PushDockerImageToLocalRegistry(imageName) + if err != nil { + d.th.t.Fatalf("UpgradeEVE: %v", err) + } + d.th.log.Infof( + "Configuring EVE to pull rootfs %s from evetest's local OCI registry", imageName) + config := d.GetConfig() + config.SetBaseOS(dockerContainer, shortVersion) + d.applyUpgradeConfig(config, shortVersion, waitUntilUpgraded, expectRevert) + return + } + // Extract rootfs (cache by short version to avoid re-extraction on reuse). rootfsFilename := "rootfs-" + shortVersion + ".img" rootfsPath := filepath.Join(d.th.imgServerDir, rootfsFilename) @@ -439,7 +537,7 @@ func (d *EdgeDevice) UpgradeEVE(targetEVEVersion string, targetEVEHypervisor Hyp d.th.log.Infof("Reusing cached rootfs %s", rootfsFilename) } - d.applyUpgrade(rootfsPath, rootfsFilename, shortVersion, + d.applyUpgradeOverHTTP(rootfsPath, rootfsFilename, shortVersion, waitUntilUpgraded, expectRevert) } @@ -503,15 +601,15 @@ func (d *EdgeDevice) upgradeEVEFromLocalBuild(targetEVEVersion string, d.th.log.Infof("Reusing staged rootfs %s", rootfsFilename) } - d.applyUpgrade(rootfsPath, rootfsFilename, img.ShortVersion, + d.applyUpgradeOverHTTP(rootfsPath, rootfsFilename, img.ShortVersion, waitUntilUpgraded, expectRevert) } -// applyUpgrade points the device's BaseOS config at a rootfs image already -// staged in the harness's HTTP image server and, optionally, waits for the -// outcome. Shared by both transports: they differ only in how rootfsPath got -// there. -func (d *EdgeDevice) applyUpgrade(rootfsPath, rootfsFilename, shortVersion string, +// applyUpgradeOverHTTP points the device's BaseOS config at a rootfs image +// already staged in the harness's HTTP image server and, optionally, waits +// for the outcome. Shared by the two HTTP-serving transports (registry-pulled +// and local-build): they differ only in how rootfsPath got there. +func (d *EdgeDevice) applyUpgradeOverHTTP(rootfsPath, rootfsFilename, shortVersion string, waitUntilUpgraded, expectRevert bool) { sha256hex, fileSize, err := utils.FileHashAndSize(rootfsPath) @@ -530,6 +628,15 @@ func (d *EdgeDevice) applyUpgrade(rootfsPath, rootfsFilename, shortVersion strin ServerPort: GetImageServerPort(), }, shortVersion) + d.applyUpgradeConfig(config, shortVersion, waitUntilUpgraded, expectRevert) +} + +// applyUpgradeConfig applies an already-built upgrade device config and, +// optionally, waits for the outcome. Shared by all datastore transports: +// they differ only in how the BaseOS config gets built. +func (d *EdgeDevice) applyUpgradeConfig(config *EdgeDeviceConfig, shortVersion string, + waitUntilUpgraded, expectRevert bool) { + d.th.log.Infof("Applying EVE upgrade config (target=%s)", shortVersion) // A successful upgrade reboots once; a reverted upgrade reboots twice // (once to try the new version, once to revert to the previous one). @@ -712,6 +819,51 @@ func (d *EdgeDevice) HardReboot(waitUntilRebooted bool) { }) } +// PowerOff hard-powers off the device through the broker (bypassing any +// graceful ACPI shutdown). The broker RPC blocks until the provider confirms +// the VM is stopped, so no separate wait parameter is needed. +func (d *EdgeDevice) PowerOff() { + d.th.collectCoverageFromDevice(d.th.ctx, d.devName) + devCtrlReq := &api.DeviceControlRequest{ + ClientId: d.th.brokerClientID, + DeviceName: d.devName, + } + ctx, cancel := context.WithTimeout(d.th.ctx, brokerPowerOffEVEDeviceTimeout) + defer cancel() + _, err := d.th.brokerClient.PowerOffDevice(ctx, devCtrlReq) + if err != nil { + d.th.t.Fatalf("PowerOff: broker failed to power off device %q: %v", + d.devName, err) + } +} + +// PowerOn powers the device back on through the broker and optionally waits +// until it boots and reports back to the controller. +// +// TODO: following a true hard power-off, nodeagent does not reliably +// republish an updated ZInfoDevice.LastRebootTime (the reboot-reason +// detection that rebootAndWait's wait relies on appears to assume a +// cooperative in-place OS reboot, not an external power-cycle). +// Until that's root-caused on the EVE side, pass waitUntilOnline=false here +// and confirm recovery some other way (e.g. via WaitUntilAppIsRunning +// on the relevant apps). +func (d *EdgeDevice) PowerOn(waitUntilOnline bool) { + d.th.incExpectedRebootCount(d.devName) + d.rebootAndWait(waitUntilOnline, func() { + devCtrlReq := &api.DeviceControlRequest{ + ClientId: d.th.brokerClientID, + DeviceName: d.devName, + } + ctx, cancel := context.WithTimeout(d.th.ctx, brokerPowerOnEVEDeviceTimeout) + defer cancel() + _, err := d.th.brokerClient.PowerOnDevice(ctx, devCtrlReq) + if err != nil { + d.th.t.Fatalf("PowerOn: broker failed to power on device %q: %v", + d.devName, err) + } + }) +} + // rebootAndWait executes triggerFn to initiate a device reboot and, if // wait is true, blocks until the device confirms the reboot by reporting // a ZInfoDevice.lastRebootTime strictly after the moment triggerFn was called. @@ -834,67 +986,202 @@ func (d *EdgeDevice) GetAppLogs(appUUID uuid.UUID, match LogMsgMatch) []LogMsg { // GetAppFlowLogs returns flow records for the specified application // matching the provided criteria. func (d *EdgeDevice) GetAppFlowLogs( - appUUID uuid.UUID, match FlowLogMatch) []eveflowlog.FlowRecord { - // TODO: implement AdamClient.IterateAppFlowLogs first - d.th.t.Fatalf("GetAppFlowLogs is not implemented") - return nil + appUUID uuid.UUID, match FlowLogMatch) []*eveflowlog.FlowRecord { + devUUID := d.getDevUUID() + scopeMatch := flowScopeMatcher(appUUID, match.VirtualNetAdapter, match.NetworkInstance) + + var records []*eveflowlog.FlowRecord + ctx, cancel := context.WithTimeout(d.th.ctx, gatherLogsTimeout) + err := d.th.adamClient.IterateDeviceFlowLogs(ctx, devUUID, scopeMatch, + flowMsgIterFn(func(msg *eveflowlog.FlowMessage) (bool, error) { + for _, rec := range msg.GetFlows() { + if flowRecordMatches(rec, match) { + records = append(records, rec) + } + } + return false, nil + }), false) + cancel() + if err != nil { + d.th.t.Fatalf("Failed to retrieve app flow logs for device %q app %q: %v", + d.devName, appUUID, err) + } + return records +} + +// flowScopeMatcher returns a match function for IterateDeviceFlowLogs / +// SubscribeToDeviceFlowLogs that selects FlowMessages belonging to the given +// application and, if non-empty/non-zero, the given VIF logical label and +// network instance. +func flowScopeMatcher(appUUID uuid.UUID, virtualNetAdapter string, + networkInstance uuid.UUID) func(*eveflowlog.FlowMessage) bool { + appUUIDStr := appUUID.String() + niUUIDStr := networkInstance.String() + return func(msg *eveflowlog.FlowMessage) bool { + scope := msg.GetScope() + if scope.GetUuid() != appUUIDStr { + return false + } + if virtualNetAdapter != "" && scope.GetIntf() != virtualNetAdapter { + return false + } + if networkInstance != NilUUID && scope.GetNetInstUUID() != niUUIDStr { + return false + } + return true + } +} + +// flowRecordMatches reports whether rec satisfies match's flow-record-level +// criteria. Scope-level criteria (VirtualNetAdapter, NetworkInstance) are +// checked by the caller against the enclosing FlowMessage's Scope instead, +// since FlowRecord itself carries no scope information. +func flowRecordMatches(rec *eveflowlog.FlowRecord, match FlowLogMatch) bool { + if rec.GetInbound() != match.Inbound { + return false + } + if match.Flow != nil { + flow := rec.GetFlow() + if match.Flow.GetSrc() != "" && flow.GetSrc() != match.Flow.GetSrc() { + return false + } + if match.Flow.GetSrcPort() != 0 && flow.GetSrcPort() != match.Flow.GetSrcPort() { + return false + } + if match.Flow.GetDest() != "" && flow.GetDest() != match.Flow.GetDest() { + return false + } + if match.Flow.GetDestPort() != 0 && flow.GetDestPort() != match.Flow.GetDestPort() { + return false + } + if match.Flow.GetProtocol() != 0 && flow.GetProtocol() != match.Flow.GetProtocol() { + return false + } + } + ts := rec.GetStartTime().AsTime() + if !match.NotBefore.IsZero() && ts.Before(match.NotBefore) { + return false + } + if !match.NotAfter.IsZero() && ts.After(match.NotAfter) { + return false + } + return true } // GetAppDNSLogs returns DNS request logs for the specified application // matching the provided criteria. func (d *EdgeDevice) GetAppDNSLogs( - appUUID uuid.UUID, match DNSLogMatch) []eveflowlog.DnsRequest { - // TODO: implement AdamClient.IterateAppFlowLogs first - d.th.t.Fatalf("GetAppDNSLogs is not implemented") - return nil + appUUID uuid.UUID, match DNSLogMatch) []*eveflowlog.DnsRequest { + devUUID := d.getDevUUID() + scopeMatch := flowScopeMatcher(appUUID, match.VirtualNetAdapter, match.NetworkInstance) + + var records []*eveflowlog.DnsRequest + ctx, cancel := context.WithTimeout(d.th.ctx, gatherLogsTimeout) + err := d.th.adamClient.IterateDeviceFlowLogs(ctx, devUUID, scopeMatch, + flowMsgIterFn(func(msg *eveflowlog.FlowMessage) (bool, error) { + for _, req := range msg.GetDnsReqs() { + ts := req.GetRequestTime().AsTime() + if !match.NotBefore.IsZero() && ts.Before(match.NotBefore) { + continue + } + if !match.NotAfter.IsZero() && ts.After(match.NotAfter) { + continue + } + records = append(records, req) + } + return false, nil + }), false) + cancel() + if err != nil { + d.th.t.Fatalf("Failed to retrieve app DNS logs for device %q app %q: %v", + d.devName, appUUID, err) + } + return records } // waitUntilAppState waits until the app reaches one of targetStates, // logging every state transition along the way. +// +// Safe to call even if the app already has matching state history from an +// earlier phase (e.g. it was already RUNNING before): it subscribes to live +// updates before checking the current snapshot, so a stale historical +// record can never be mistaken for a new transition. +// // ctx controls the deadline; callers must derive it from d.th.ctx. // Calls t.Fatalf on timeout or error. func (d *EdgeDevice) waitUntilAppState( ctx context.Context, appUUID uuid.UUID, targetStates ...eveinfo.ZSwState) { devUUID := d.getDevUUID() appUUIDStr := appUUID.String() - - var lastState = eveinfo.ZSwState_INVALID + filter := func(msg *eveinfo.ZInfoMsg) bool { + if msg.GetZtype() != eveinfo.ZInfoTypes_ZiApp { + return false + } + ainfo := msg.GetAinfo() + return ainfo != nil && ainfo.GetAppID() == appUUIDStr + } d.th.log.Infof("Waiting for app %q on device %q to reach state(s) %v", appUUID, d.devName, targetStates) - err := d.th.adamClient.IterateDeviceInfoMsgs(ctx, devUUID, - func(msg *eveinfo.ZInfoMsg) bool { - if msg.GetZtype() != eveinfo.ZInfoTypes_ZiApp { - return false - } - ainfo := msg.GetAinfo() - return ainfo != nil && ainfo.GetAppID() == appUUIDStr - }, - infoMsgIterFn(func(msg *eveinfo.ZInfoMsg) (bool, error) { - ainfo := msg.GetAinfo() - state := ainfo.GetState() - if state != lastState { - lastState = state - d.th.log.Infof("App %q (%s) on device %q state changed to %s", - appUUID, ainfo.GetAppName(), d.devName, state) - } - if generics.ContainsItem(targetStates, state) { - return true, nil - } - return false, nil - }), - true, - ) + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + ch := make(chan *eveinfo.ZInfoMsg, watchChannelBufSize) + unsub, err := d.th.adamClient.SubscribeToDeviceInfoMsgs(devUUID, filter, ch) if err != nil { - d.th.t.Fatalf("Waiting for app %q on device %q to reach state(s) %v: %v", - appUUID, d.devName, targetStates, err) + d.th.t.Fatalf("Failed to subscribe to info messages for app %q on device %q: %v", + appUUID, d.devName, err) + } + defer unsub() + + var lastState = eveinfo.ZSwState_INVALID + logIfChanged := func(ainfo *eveinfo.ZInfoApp) { + state := ainfo.GetState() + if state != lastState { + lastState = state + d.th.log.Infof("App %q (%s) on device %q state changed to %s", + appUUID, ainfo.GetAppName(), d.devName, state) + } + } + + // Snapshot: the app may already be in one of the target states right now. + // Logged as the current state, not a "change", since nothing just + // happened -- this is a fact we already knew, not a new event. + if info := d.GetAppInfo(appUUID); info != nil { + lastState = info.GetState() + d.th.log.Infof("App %q (%s) on device %q currently in state %s", + appUUID, info.GetAppName(), d.devName, lastState) + if generics.ContainsItem(targetStates, lastState) { + return + } + } + + for { + select { + case <-ctx.Done(): + d.th.t.Fatalf("Waiting for app %q on device %q to reach state(s) %v: %v", + appUUID, d.devName, targetStates, ctx.Err()) + case msg, ok := <-ch: + if !ok { + d.th.t.Fatalf("Info subscription closed while waiting for app %q "+ + "on device %q to reach state(s) %v", appUUID, d.devName, targetStates) + } + logIfChanged(msg.GetAinfo()) + if generics.ContainsItem(targetStates, lastState) { + return + } + } } } // WaitUntilAppIsRunning waits until the specified application reaches // the running state or fails. // +// Safe to call even if the app already has RUNNING history from an earlier +// phase: it subscribes to live updates before checking the current +// snapshot, so a stale historical record can never be mistaken for a new +// transition. +// // timeoutExcludingDownload is the maximum time to wait excluding any // period spent actively downloading (i.e. in DOWNLOAD_STARTED state with // advancing progress). If a download stalls for downloadStalledTimeout the @@ -919,28 +1206,12 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( lastAppErrs string // concatenated error descriptions for change detection // Keyed by volume UUID; accumulates the latest ZInfoVolume for each volume. volumes = make(map[string]*eveinfo.ZInfoVolume) + // timer is nil during the initial snapshot replay (see below), so + // iterCb's Reset calls are guarded and become no-ops until the live + // phase arms it. + timer *time.Timer ) - // ctx is canceled either by the timer below (timeout) or by d.th.ctx (test end). - ctx, cancel := context.WithCancel(d.th.ctx) - defer cancel() - - // The timer drives timeouts when no info messages arrive: - // - non-download phase: fires after the remaining non-download budget - // - download phase: fires after downloadStalledTimeout with no progress - // iterCb resets it on each relevant transition or progress update. - timer := time.NewTimer(timeoutExcludingDownload) - defer timer.Stop() - - // Cancel the context when the timer fires so IterateDeviceInfoMsgs unblocks. - go func() { - select { - case <-timer.C: - cancel() - case <-ctx.Done(): - } - }() - // Accept ZiApp messages for this app and all ZiVolume messages. // Volume messages are further filtered in the iterator once the app's // VolumeRefs are known. @@ -955,7 +1226,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( return false } - iterCb := func(msg *eveinfo.ZInfoMsg) (bool, error) { + // iterCb processes a single info message, updating all the tracking state + // above. live distinguishes the two contexts it's called from: false + // during the initial snapshot replay (state is still established, but + // nothing is logged -- these are historical facts, not new events, and + // logging each one would misleadingly read as if they just happened); + // true during the live phase (normal logging of genuinely new events). + iterCb := func(msg *eveinfo.ZInfoMsg, live bool) (bool, error) { // Handle volume updates: store the latest state for each volume // and re-evaluate download progress if the app is currently downloading. if msg.GetZtype() == eveinfo.ZInfoTypes_ZiVolume { @@ -970,9 +1247,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( pct := appDownloadProgress(volumeRefs, volumes) if pct != lastDownloadPct { lastDownloadPct = pct - timer.Reset(downloadStalledTimeout) - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, lastState, pct) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, lastState, pct) + } } } return false, nil @@ -990,20 +1271,26 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( nowInDownload := state == eveinfo.ZSwState_DOWNLOAD_STARTED if inDownload && !nowInDownload { // Leaving download: resume non-download clock and set timer to - // the remaining non-download budget. + // the remaining non-download budget. Only meaningful once the + // live timer is armed -- during the initial snapshot replay, + // elapsed wall-clock time is negligible and this check is skipped. nonDownloadStart = time.Now() - remaining := timeoutExcludingDownload - nonDownloadElapsed - if remaining <= 0 { - return true, fmt.Errorf( - "timed out after %s (excluding download) waiting for app %q (%s) "+ - "on device %q to reach RUNNING state (last state: %s)", - timeoutExcludingDownload, appUUID, appName, d.devName, state) + if timer != nil { + remaining := timeoutExcludingDownload - nonDownloadElapsed + if remaining <= 0 { + return true, fmt.Errorf( + "timed out after %s (excluding download) waiting for app %q (%s) "+ + "on device %q to reach RUNNING state (last state: %s)", + timeoutExcludingDownload, appUUID, appName, d.devName, state) + } + timer.Reset(remaining) } - timer.Reset(remaining) } else if !inDownload && nowInDownload { // Entering download: freeze non-download clock and arm stall timer. nonDownloadElapsed += time.Since(nonDownloadStart) - timer.Reset(downloadStalledTimeout) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } } inDownload = nowInDownload @@ -1013,9 +1300,11 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( if state == eveinfo.ZSwState_DOWNLOAD_STARTED { pct := appDownloadProgress(volumeRefs, volumes) lastDownloadPct = pct - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, state, pct) - } else { + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, state, pct) + } + } else if live { d.th.log.Infof("App %q (%s) on device %q state changed to %s", appUUID, appName, d.devName, state) } @@ -1023,9 +1312,13 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( pct := appDownloadProgress(volumeRefs, volumes) if pct != lastDownloadPct { lastDownloadPct = pct - timer.Reset(downloadStalledTimeout) - d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", - appUUID, appName, d.devName, state, pct) + if timer != nil { + timer.Reset(downloadStalledTimeout) + } + if live { + d.th.log.Infof("App %q (%s) on device %q state changed to %s (%d%%)", + appUUID, appName, d.devName, state, pct) + } } } @@ -1039,12 +1332,14 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( currentAppErrs := strings.Join(errDescs, "; ") if currentAppErrs != lastAppErrs { lastAppErrs = currentAppErrs - if currentAppErrs != "" { - d.th.log.Warnf("App %q (%s) on device %q errors: %s", - appUUID, appName, d.devName, currentAppErrs) - } else { - d.th.log.Infof("App %q (%s) on device %q errors cleared", - appUUID, appName, d.devName) + if live { + if currentAppErrs != "" { + d.th.log.Warnf("App %q (%s) on device %q errors: %s", + appUUID, appName, d.devName, currentAppErrs) + } else { + d.th.log.Infof("App %q (%s) on device %q errors cleared", + appUUID, appName, d.devName) + } } } @@ -1062,41 +1357,107 @@ func (d *EdgeDevice) WaitUntilAppIsRunning( // Success. if state == eveinfo.ZSwState_RUNNING { - d.th.log.Infof("App %q (%s) on device %q is RUNNING", - appUUID, appName, d.devName) + if live { + d.th.log.Infof("App %q (%s) on device %q is RUNNING", + appUUID, appName, d.devName) + } return true, nil } return false, nil } - err := d.th.adamClient.IterateDeviceInfoMsgs(ctx, devUUID, filter, - infoMsgIterFn(iterCb), true) - + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + ch := make(chan *eveinfo.ZInfoMsg, watchChannelBufSize) + unsub, err := d.th.adamClient.SubscribeToDeviceInfoMsgs(devUUID, filter, ch) if err != nil { - // If the test framework context was canceled, propagate the error. - if d.th.ctx.Err() != nil { - d.th.t.Fatalf("%v", err) + d.th.t.Fatalf("Failed to subscribe to info messages for app %q on device %q: %v", + appUUID, d.devName, err) + } + defer unsub() + + // Snapshot: replay every already-known message (app and volume) through + // iterCb with no timer armed (its Reset calls are then no-ops), keeping + // only the LAST app-state evaluation -- i.e. the app's current state, + // not the first historical occurrence of any target condition. + var ( + snapshotDone bool + snapshotErr error + ) + d.iterateInfoMsgs(devUUID, filter, func(msg *eveinfo.ZInfoMsg) { + done, cbErr := iterCb(msg, false) + if msg.GetZtype() == eveinfo.ZInfoTypes_ZiApp { + snapshotDone, snapshotErr = done, cbErr + } + }) + if snapshotDone { + if snapshotErr != nil { + d.th.t.Fatalf("%v", snapshotErr) } + d.th.log.Infof("App %q (%s) on device %q is already RUNNING", + appUUID, appName, d.devName) + return + } + d.th.log.Infof("App %q (%s) on device %q currently in state %s; "+ + "waiting for it to reach RUNNING", appUUID, appName, d.devName, lastState) + + // Live phase: arm the real timer -- matching the app's current phase, + // established during the snapshot above -- and wait for a genuinely + // new transition. + nonDownloadStart = time.Now() + initialTimeout := timeoutExcludingDownload + if inDownload { + initialTimeout = downloadStalledTimeout + } + timer = time.NewTimer(initialTimeout) + defer timer.Stop() - // If our context was not canceled, the error came from iterCb - // (e.g. ZSwState_ERROR or explicit failure). - if ctx.Err() == nil { - d.th.t.Fatalf("%v", err) + // ctx is canceled either by the timer above (timeout) or by d.th.ctx (test end). + ctx, cancel := context.WithCancel(d.th.ctx) + defer cancel() + go func() { + select { + case <-timer.C: + cancel() + case <-ctx.Done(): } + }() - // Otherwise our timer fired — determine which timeout occurred. - if inDownload { + for { + select { + case <-ctx.Done(): + // If the test framework context was canceled, propagate that. + if d.th.ctx.Err() != nil { + d.th.t.Fatalf("Waiting for app %q (%s) on device %q to reach "+ + "RUNNING state: %v", appUUID, appName, d.devName, d.th.ctx.Err()) + } + + // Otherwise our own timer fired -- determine which timeout occurred. + if inDownload { + d.th.t.Fatalf( + "app %q (%s) on device %q download stalled at %d%% for more than %s", + appUUID, appName, d.devName, lastDownloadPct, downloadStalledTimeout) + } + nonDownloadTotal := nonDownloadElapsed + time.Since(nonDownloadStart) d.th.t.Fatalf( - "app %q (%s) on device %q download stalled at %d%% for more than %s", - appUUID, appName, d.devName, lastDownloadPct, downloadStalledTimeout) + "timed out after %s (excluding download) waiting for app %q (%s) "+ + "on device %q to reach RUNNING state (last state: %s)", + nonDownloadTotal, appUUID, appName, d.devName, lastState) + case msg, ok := <-ch: + if !ok { + d.th.t.Fatalf("Info subscription closed while waiting for app %q "+ + "(%s) on device %q to reach RUNNING state", appUUID, appName, d.devName) + } + done, cbErr := iterCb(msg, true) + if !done { + continue + } + if cbErr != nil { + d.th.t.Fatalf("%v", cbErr) + } + return } - - nonDownloadTotal := nonDownloadElapsed + time.Since(nonDownloadStart) - d.th.t.Fatalf( - "timed out after %s (excluding download) waiting for app %q (%s) "+ - "on device %q to reach RUNNING state (last state: %s)", - nonDownloadTotal, appUUID, appName, d.devName, lastState) } } @@ -1149,6 +1510,15 @@ func (d *EdgeDevice) PurgeApplication(appUUID uuid.UUID, waitUntilPurged bool, } else { app.Purge = &eveconfig.InstanceOpsCmd{Counter: purge.GetCounter() + 1} } + for _, volRef := range app.GetVolumeRefList() { + volRef.GenerationCount++ + for _, vol := range config.GetVolumes() { + if vol.GetUuid() == volRef.GetUuid() { + vol.GenerationCount++ + break + } + } + } found = true break } @@ -1166,6 +1536,15 @@ func (d *EdgeDevice) PurgeApplication(appUUID uuid.UUID, waitUntilPurged bool, } } +// DialViaSSH opens a TCP connection to address, tunneled through an SSH +// connection to this device (an SSH direct-tcpip channel), as if address is +// dialed from the device itself. Useful for reaching services that only +// listen on the device's own loopback interface, e.g. the Kubevirt VNC proxy +// gated by ApplicationInstanceConfig.RemoteConsole. +func (d *EdgeDevice) DialViaSSH(network, address string) (net.Conn, error) { + return d.th.dialViaSSH(d.th.ctx, d.devName, network, address) +} + // ActivateApplication activates the specified application instance. func (d *EdgeDevice) ActivateApplication(appUUID uuid.UUID, waitUntilActivated bool, timeout time.Duration) { @@ -1436,8 +1815,13 @@ func getAdaptersByLabel(config *EdgeDeviceConfig, label string) []string { // FileExists checks whether a file exists on the device. func (d *EdgeDevice) FileExists(fileName string) bool { + // "; true" forces the overall exit status to 0 regardless of whether + // the file exists, so a missing file (test -f exits nonzero) can't be + // conflated with a genuine SSH/transport failure -- err is asserted nil + // for the latter, and only the "EXISTS" marker in stdout answers the + // actual question. stdout, _, err := d.RunShellScript( - "test -f "+shellEscape(fileName)+" && echo EXISTS", + "test -f "+shellEscape(fileName)+" && echo EXISTS; true", quickSSHCommandTimeout, 0) if err != nil { d.th.t.Fatalf("FileExists: SSH command failed: %v", err) @@ -2086,6 +2470,47 @@ func (d *EdgeDevice) WatchClusterInfo() ( return ch, d.trackWatcherUnsub(unsub) } +// WaitForClusterNodeIsReady waits until this device's own ZInfoKubeCluster +// report shows this device as a Ready node with healthy cluster storage. +// +// Only the elected leader node publishes cluster info, so this is only +// meaningful when called on a single-node cluster (where the sole device is +// necessarily the leader) or on a device already known to be the leader; for +// a multi-node cluster where the leader isn't known in advance, use +// EdgeCluster.WaitUntilNodesAreReady instead. +func (d *EdgeDevice) WaitForClusterNodeIsReady(timeout time.Duration) { + d.th.log.Infof("Waiting for cluster node %q to become ready...", d.devName) + + // Subscribe before taking the initial snapshot, so a transition landing + // between the two calls can never be missed. + updates, stop := d.WatchClusterInfo() + defer stop() + + if info := d.GetClusterInfo(); info != nil && clusterNodeReady(info, d.devName) { + d.th.log.Infof("Cluster node %q is already ready", d.devName) + return + } + + ctx, cancel := context.WithTimeout(d.th.ctx, timeout) + defer cancel() + for { + select { + case info, ok := <-updates: + if !ok { + d.th.t.Fatalf("Cluster info subscription closed while waiting "+ + "for node %q to become ready", d.devName) + } + if clusterNodeReady(info, d.devName) { + d.th.log.Infof("Cluster node %q is now ready", d.devName) + return + } + case <-ctx.Done(): + d.th.t.Fatalf("Timed out waiting for cluster node %q to become ready", + d.devName) + } + } +} + // GetClusterUpdateInfo returns the last recorded information regarding the Kubernetes // cluster update, or nil if no such info message has been received yet. func (d *EdgeDevice) GetClusterUpdateInfo() *eveinfo.ZInfoKubeClusterUpdateStatus { @@ -2357,23 +2782,29 @@ func (d *EdgeDevice) WatchClusterMetrics() ( // - key: identifies the specific message within the topic to fetch // - output: pointer to a value of type T to unmarshal the message into // -// Returns an error if the topic or message does not exist, cannot be read, or -// fails to unmarshal into the provided output type. +// Returns false if the topic or message does not exist yet (e.g. before the +// agent has first published it) -- callers that need to wait for it should +// poll on the returned bool instead of treating absence as an error. Calls +// t.Fatalf on any other read or unmarshal failure. func ReadPublication[T any](d *EdgeDevice, fromAgent string, persistent bool, - key string, output *T) { + key string, output *T) bool { fullName := fmt.Sprintf("%T", *new(T)) typeName := fullName[strings.LastIndex(fullName, ".")+1:] var path string if persistent { - path = fmt.Sprintf("/persistent/status/%s/%s/%s.json", fromAgent, typeName, key) + path = fmt.Sprintf("/persist/status/%s/%s/%s.json", fromAgent, typeName, key) } else { path = fmt.Sprintf("/run/%s/%s/%s.json", fromAgent, typeName, key) } + if !d.FileExists(path) { + return false + } data := d.ReadFile(path) if err := json.Unmarshal(data, output); err != nil { d.th.t.Fatalf("ReadPublication: failed to unmarshal %q from device %q: %v", path, d.devName, err) } + return true } // ReadAllPublications retrieves all messages from a pub-sub topic published by @@ -2390,7 +2821,7 @@ func ReadAllPublications[T any](d *EdgeDevice, fromAgent string, persistent bool typeName := fullName[strings.LastIndex(fullName, ".")+1:] var dir string if persistent { - dir = fmt.Sprintf("/persistent/status/%s/%s", fromAgent, typeName) + dir = fmt.Sprintf("/persist/status/%s/%s", fromAgent, typeName) } else { dir = fmt.Sprintf("/run/%s/%s", fromAgent, typeName) } @@ -2473,6 +2904,11 @@ type metricMsgIterFn func(*evemetrics.ZMetricMsg) (bool, error) func (f metricMsgIterFn) Iterate(msg *evemetrics.ZMetricMsg) (bool, error) { return f(msg) } +// flowMsgIterFn adapts a function to the controller.FlowMsgIterator interface. +type flowMsgIterFn func(*eveflowlog.FlowMessage) (bool, error) + +func (f flowMsgIterFn) Iterate(msg *eveflowlog.FlowMessage) (bool, error) { return f(msg) } + // appDownloadProgress returns the average download progress (0–100) across // the app's volumes. For each volume UUID listed in volumeRefs the progress // is taken from the latest ZInfoVolume in volumes: diff --git a/evetest/go.mod b/evetest/go.mod index 5e04a1d3993..2c758fb1f2b 100644 --- a/evetest/go.mod +++ b/evetest/go.mod @@ -5,17 +5,20 @@ go 1.25.2 toolchain go1.25.11 require ( + github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913 github.com/containerd/errdefs v1.0.0 github.com/distribution/reference v0.6.0 - github.com/docker/cli v29.2.0+incompatible + github.com/docker/cli v29.6.2+incompatible + github.com/google/go-containerregistry v0.21.9 github.com/google/uuid v1.6.0 github.com/lf-edge/eve-api/go v0.0.0-20260622100545-186e61c68f39 github.com/lf-edge/eve/pkg/pillar v0.0.0-20260421125048-8d3825045e4e github.com/luthermonson/go-proxmox v0.8.0 github.com/moby/moby/api v1.55.0 - github.com/moby/moby/client v0.5.0 + github.com/moby/moby/client v0.5.1 github.com/onsi/gomega v1.39.1 github.com/opencontainers/image-spec v1.1.1 + github.com/pkg/sftp v1.13.9 github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 @@ -38,7 +41,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/diskfs/go-diskfs v1.9.3 // indirect github.com/djherbis/times v1.6.0 // indirect - github.com/docker/docker-credential-helpers v0.8.2 // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7 // indirect @@ -54,11 +57,12 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/go-containerregistry v0.14.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.19.1 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d // indirect github.com/magefile/mage v1.14.0 // indirect @@ -82,6 +86,7 @@ require ( go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/evetest/go.sum b/evetest/go.sum index 709d60deb05..c6e6f837082 100644 --- a/evetest/go.sum +++ b/evetest/go.sum @@ -4,6 +4,8 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913 h1:gQl0n269O2lxtDz6QBXtqEjiH2auPaG8GbTu3tByA3w= +github.com/amitbet/vncproxy v0.0.0-20200118084310-ea8f9b510913/go.mod h1:HfBAAYdSeX18f2nwbuMIcA12RhvgYolx0XDbbhusDXY= github.com/anchore/go-lzo v0.1.0 h1:NgAacnzqPeGH49Ky19QKLBZEuFRqtTG9cdaucc3Vncs= github.com/anchore/go-lzo v0.1.0/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= @@ -25,10 +27,10 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= -github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= -github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/docker/cli v29.6.2+incompatible h1:/bjePvcbbFTnRrMfWJBY7AjfICdsiLVgHn6LwTVOcqw= +github.com/docker/cli v29.6.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -71,10 +73,11 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw= -github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= +github.com/google/go-containerregistry v0.21.9 h1:F+D4uZ3iA3DLMJLfhaqMdHJbzeqm/216WGQq2dokuLs= +github.com/google/go-containerregistry v0.21.9/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= @@ -94,8 +97,10 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -116,8 +121,8 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= -github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= +github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -135,6 +140,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= +github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -183,6 +190,7 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= @@ -202,44 +210,98 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/evetest/grpcapi/go/broker.pb.go b/evetest/grpcapi/go/broker.pb.go index 01bf11d2d7a..3dfbad89daa 100644 --- a/evetest/grpcapi/go/broker.pb.go +++ b/evetest/grpcapi/go/broker.pb.go @@ -565,8 +565,13 @@ type BuildImageRequest struct { // client upload bytes it can already see. Advisory only -- see // LocalLiveImageSource. LiveImageSource *LocalLiveImageSource `protobuf:"bytes,8,opt,name=live_image_source,json=liveImageSource,proto3" json:"live_image_source,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Sizes (in bytes) of additional blank disks to attach to the device's VM, + // beyond its main boot disk (disk_bytes). Each entry becomes one extra + // virtio-blk disk, in the given order, e.g. for device-side disk + // layout/RAID testing (see EdgeDevConfig.disks). + ExtraDiskBytes []uint64 `protobuf:"varint,9,rep,packed,name=extra_disk_bytes,json=extraDiskBytes,proto3" json:"extra_disk_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BuildImageRequest) Reset() { @@ -655,6 +660,13 @@ func (x *BuildImageRequest) GetLiveImageSource() *LocalLiveImageSource { return nil } +func (x *BuildImageRequest) GetExtraDiskBytes() []uint64 { + if x != nil { + return x.ExtraDiskBytes + } + return nil +} + // LocalLiveImageSource points at the files behind a LiveImageRef on the // client's filesystem. It is purely an optimization hint: a broker that cannot // use these paths -- they do not exist, the size disagrees, or the content does @@ -1648,7 +1660,7 @@ const file_broker_proto_rawDesc = "" + "globalJson\x12#\n" + "\roverride_json\x18\n" + " \x01(\tR\foverrideJson\x12.\n" + - "\x13bootstrap_config_pb\x18\v \x01(\fR\x11bootstrapConfigPb\"\x99\x03\n" + + "\x13bootstrap_config_pb\x18\v \x01(\fR\x11bootstrapConfigPb\"\xc3\x03\n" + "\x11BuildImageRequest\x12\x1b\n" + "\tclient_id\x18\x01 \x01(\tR\bclientId\x12\x1f\n" + "\vdevice_name\x18\x02 \x01(\tR\n" + @@ -1660,7 +1672,8 @@ const file_broker_proto_rawDesc = "" + "\x06config\x18\x06 \x01(\v2\x1d.org.lfedge.evetest.EveConfigR\x06config\x12?\n" + "\n" + "live_image\x18\a \x01(\v2 .org.lfedge.evetest.LiveImageRefR\tliveImage\x12T\n" + - "\x11live_image_source\x18\b \x01(\v2(.org.lfedge.evetest.LocalLiveImageSourceR\x0fliveImageSource\"\x9d\x01\n" + + "\x11live_image_source\x18\b \x01(\v2(.org.lfedge.evetest.LocalLiveImageSourceR\x0fliveImageSource\x12(\n" + + "\x10extra_disk_bytes\x18\t \x03(\x04R\x0eextraDiskBytes\"\x9d\x01\n" + "\x14LocalLiveImageSource\x12\x1b\n" + "\tdisk_path\x18\x01 \x01(\tR\bdiskPath\x12\x1d\n" + "\n" + diff --git a/evetest/grpcapi/proto/broker.proto b/evetest/grpcapi/proto/broker.proto index 7c99f42303a..1ddddcac334 100644 --- a/evetest/grpcapi/proto/broker.proto +++ b/evetest/grpcapi/proto/broker.proto @@ -152,6 +152,11 @@ message BuildImageRequest { // client upload bytes it can already see. Advisory only -- see // LocalLiveImageSource. LocalLiveImageSource live_image_source = 8; + // Sizes (in bytes) of additional blank disks to attach to the device's VM, + // beyond its main boot disk (disk_bytes). Each entry becomes one extra + // virtio-blk disk, in the given order, e.g. for device-side disk + // layout/RAID testing (see EdgeDevConfig.disks). + repeated uint64 extra_disk_bytes = 9; } // LocalLiveImageSource points at the files behind a LiveImageRef on the @@ -277,4 +282,4 @@ message ConnectConsoleResponse { // Subsequent messages containing raw output bytes from the device console. bytes data = 2; } -} \ No newline at end of file +} diff --git a/evetest/grpcserver.go b/evetest/grpcserver.go index bfa7ca63bb7..7f88a89dbd3 100644 --- a/evetest/grpcserver.go +++ b/evetest/grpcserver.go @@ -15,6 +15,7 @@ import ( "sync" "time" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evemetrics "github.com/lf-edge/eve-api/go/metrics" "github.com/lf-edge/eve/evetest/constants" @@ -71,6 +72,23 @@ func (w *metricMsgGrpcIterator[T]) Iterate(msg *evemetrics.ZMetricMsg) (bool, er return false, w.stream.Send(resp) } +type flowMsgGrpcIterator[T any] struct { + stream grpc.ServerStreamingServer[T] + // mapper extracts a response from a flow message; returns nil to skip. + mapper func(*eveflowlog.FlowMessage) (*T, error) +} + +func (w *flowMsgGrpcIterator[T]) Iterate(msg *eveflowlog.FlowMessage) (bool, error) { + resp, err := w.mapper(msg) + if err != nil { + return false, err + } + if resp == nil { + return false, nil + } + return false, w.stream.Send(resp) +} + func (th *TestHarness) errIfBrokerNotReady() error { if th.brokerClientID == "" { return errors.New("broker is not yet connected") @@ -396,8 +414,38 @@ func (th *TestHarness) GetAppLogs( // GetAppFlowLogs streams flow logs and DNS request logs for an application. func (th *TestHarness) GetAppFlowLogs( req *api.AppRequest, stream api.Evetest_GetAppFlowLogsServer) error { - // TODO - return errors.New("not implemented") + if err := th.errIfAdamNotReady(); err != nil { + return err + } + devName, devUUID, err := th.resolveEVEDeviceName(req.GetDeviceName()) + if err != nil { + return err + } + if devUUID == uuid.Nil { + return fmt.Errorf("device %q is not onboarded", devName) + } + appUUID, err := th.resolveAppUUID(stream.Context(), devUUID, req.GetAppNameOrUuid()) + if err != nil { + return err + } + appUUIDStr := appUUID.String() + match := func(msg *eveflowlog.FlowMessage) bool { + return msg.GetScope().GetUuid() == appUUIDStr + } + iterator := &flowMsgGrpcIterator[api.AppFlowLogsResponse]{ + stream: stream, + mapper: func(msg *eveflowlog.FlowMessage) (*api.AppFlowLogsResponse, error) { + if len(msg.GetFlows()) == 0 && len(msg.GetDnsReqs()) == 0 { + return nil, nil + } + return &api.AppFlowLogsResponse{ + IpFlows: msg.GetFlows(), + DnsRequests: msg.GetDnsReqs(), + }, nil + }, + } + return th.adamClient.IterateDeviceFlowLogs( + stream.Context(), devUUID, match, iterator, req.GetFollow()) } // GetNIInfo streams info (ZInfoNetworkInstance) about a network instance (NI). diff --git a/evetest/harness.go b/evetest/harness.go index 4c55f707937..5e9e7cc7e28 100644 --- a/evetest/harness.go +++ b/evetest/harness.go @@ -7,9 +7,11 @@ import ( "context" "crypto/ecdsa" "crypto/rsa" + "crypto/tls" "crypto/x509" "fmt" "io" + "math/big" "net" "net/http" "os" @@ -72,6 +74,11 @@ const ( // Timeout for powering on an EVE VM (not for waiting for it to boot). brokerPowerOnEVEDeviceTimeout = 20 * time.Second + // Timeout for powering off an EVE VM. The broker RPC blocks until the + // provider confirms the VM is stopped, so this must accommodate a + // graceful ACPI-less hard power-off, not just issuing the request. + brokerPowerOffEVEDeviceTimeout = 20 * time.Second + // Timeout for triggering an EVE VM reboot (not for waiting for it to boot). brokerRebootEVEDeviceTimeout = 20 * time.Second @@ -146,12 +153,28 @@ const ( // Timeout for an EVE device to complete an OS upgrade (download, install, reboot, // complete the testing period, and mark the new partition as active). eveUpgradeTimeout = 20 * time.Minute + + // staleImageCacheRetention is how long an img-cache-* directory is kept + // before being swept as stale (see removeStaleImageCacheDirs). Each + // harness process gets its own such directory, created in Init and + // removed in Close; a container killed before Close runs (SIGKILL, + // `docker stop` without a graceful shutdown) skips that cleanup and + // leaks it otherwise. A generous margin over any realistic single test + // or suite run, so it never races a run still legitimately in progress + // under a different img-cache dir. + staleImageCacheRetention = 3 * 24 * time.Hour ) const ( controllerIntfName = "controller" imgServerIntfName = "img-server" imgServerPort = 80 + // imgServerSFTPPort is the default SFTP port (22), matching the port + // SFTPStorage assumes when ServerPort is left unset. + imgServerSFTPPort = 22 + // imgServerHTTPSPort is the default HTTPS port (443), matching the port + // HTTPStorage assumes when UseHTTPS is set and ServerPort is left unset. + imgServerHTTPSPort = 443 sdnTunName = "sdn-tun" sdnTunMTU = 1500 @@ -242,6 +265,15 @@ type TestHarness struct { imgServerListener net.Listener imgServerDir string // temp dir under $HOME/.evetest; removed in Close() + // SFTP listener serving the same imgServerDir as an alternate download + // transport (see DsType_DsSFTP / SFTPStorage). + sftpServerListener net.Listener + + // HTTPS listener serving the same imgServerDir as the plain HTTP image + // server, over TLS with a certificate signed by the harness's own CA + // (see DsType_DsHttps / HTTPStorage.UseHTTPS, GetCACertPEM). + imgServerTLSListener net.Listener + // SDN sdnConn *grpc.ClientConn sdnClient api.SDNClient @@ -383,6 +415,37 @@ func getTestHarness() *TestHarness { return _globalTH } +// removeStaleImageCacheDirs removes img-cache-* directories under +// imgCacheParent whose mtime is older than staleImageCacheRetention -- ones a +// prior harness process left behind by being killed before its own Close() +// could remove its img-cache dir. A directory's mtime advances every time a +// file is added inside it (i.e. every download served during that run), so an +// abandoned one simply stops advancing at whenever that run ended. +// Best-effort: failures only leave a stale directory for the next attempt. +func removeStaleImageCacheDirs(log *logrus.Logger, imgCacheParent string) { + entries, err := os.ReadDir(imgCacheParent) + if err != nil { + return + } + cutoff := time.Now().Add(-staleImageCacheRetention) + for _, e := range entries { + if !e.IsDir() || !strings.HasPrefix(e.Name(), "img-cache-") { + continue + } + info, err := e.Info() + if err != nil || info.ModTime().After(cutoff) { + continue + } + path := filepath.Join(imgCacheParent, e.Name()) + if err := os.RemoveAll(path); err != nil { + log.Warnf("Failed to remove stale image cache dir %q: %v", path, err) + continue + } + log.Infof("Removed stale image cache dir %q (older than %s)", + path, staleImageCacheRetention) + } +} + // Init initializes the test harness and must be called exactly once per test. // When used inside a test suite, Init may be called multiple times, once per // test case, but only a single harness instance will be created. @@ -555,6 +618,7 @@ func Init(t *testing.T) *T { // host), which is bind-mounted at the same path inside the container so that // Docker bind-mounts issued via RunDockerCommand resolve correctly on the host. imgCacheParent := viper.GetString(constants.HomeDirEnv) + removeStaleImageCacheDirs(th.log, imgCacheParent) if err = os.MkdirAll(imgCacheParent, 0755); err != nil { th.t.Fatalf("failed to create image cache parent dir: %v", err) } @@ -569,18 +633,67 @@ func Init(t *testing.T) *T { if err != nil { th.t.Fatalf("failed to create image server interface: %v", err) } + imgServerMux := http.NewServeMux() + // Registered before the file-serving "/" handler: EVE and app images + // never happen to be named "v2", so the two handlers never collide, and + // this lets EVE (or evetest itself) pull/push OCI content without it + // having to already exist in a real, reachable registry (see + // PushDockerImageToLocalRegistry). + imgServerMux.Handle("/v2/", newLocalRegistryHandler(th.imgServerDir, th.log)) + imgServerMux.Handle("/", http.FileServer(http.Dir(th.imgServerDir))) + imgListenAddr := net.JoinHostPort(imgServerIPv4.String(), strconv.Itoa(imgServerPort)) th.imgServerListener, err = net.Listen("tcp", imgListenAddr) if err != nil { th.t.Fatalf("failed to listen on image server address %s: %v", imgListenAddr, err) } go func() { - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(th.imgServerDir))) - _ = http.Serve(th.imgServerListener, mux) + _ = http.Serve(th.imgServerListener, imgServerMux) }() th.log.Infof("Image server listening on http://%s/ (serving %s)", imgListenAddr, th.imgServerDir) + // Start an HTTPS listener on the same interface and directory, using a + // certificate signed by the harness's own CA (see GetCACertPEM) -- an + // alternate download transport for tests that exercise DsType_DsHttps + // (see HTTPStorage.UseHTTPS). + imgServerCert, imgServerKey, err := utils.GenServerCertElliptic( + th.caCert, th.caKey, big.NewInt(3), []net.IP{imgServerIPv4}, nil, + "evetest image server") + if err != nil { + th.t.Fatalf("failed to generate image server TLS certificate: %v", err) + } + imgServerTLSConfig := &tls.Config{ + Certificates: []tls.Certificate{{ + Certificate: [][]byte{imgServerCert.Raw}, + PrivateKey: imgServerKey, + }}, + } + imgServerHTTPSAddr := net.JoinHostPort( + imgServerIPv4.String(), strconv.Itoa(imgServerHTTPSPort)) + th.imgServerTLSListener, err = tls.Listen("tcp", imgServerHTTPSAddr, imgServerTLSConfig) + if err != nil { + th.t.Fatalf("failed to listen on image server HTTPS address %s: %v", + imgServerHTTPSAddr, err) + } + go func() { + _ = http.Serve(th.imgServerTLSListener, imgServerMux) + }() + th.log.Infof("Image server listening on https://%s/ (serving %s)", + imgServerHTTPSAddr, th.imgServerDir) + th.log.Infof("OCI registry listening on https://%s/v2/ (serving %s)", + imgServerHTTPSAddr, th.imgServerDir) + + // Start an SFTP server on the same image-server interface, serving the + // same directory, as an alternate download transport for tests that + // exercise DsType_DsSFTP (see SFTPStorage). + sftpListenAddr := net.JoinHostPort(imgServerIPv4.String(), strconv.Itoa(imgServerSFTPPort)) + th.sftpServerListener, err = net.Listen("tcp", sftpListenAddr) + if err != nil { + th.t.Fatalf("failed to listen on SFTP server address %s: %v", sftpListenAddr, err) + } + go th.runSFTPServer(th.sftpServerListener, th.imgServerDir) + th.log.Infof("SFTP server listening on sftp://%s/ (serving %s)", sftpListenAddr, th.imgServerDir) + // Create broker client. brokerAddr := viper.GetString(constants.BrokerAddressEnv) if brokerAddr == "" { @@ -651,7 +764,7 @@ func Init(t *testing.T) *T { // gRPC server, disconnects from the broker (triggering cleanup of all // associated EVE and SDN devices), removes any SDN tunnel interfaces created // by the test, and stops the Adam controller. -func Close() error { +func Close() { panicErr := recover() th := getTestHarness() @@ -706,7 +819,7 @@ func Close() error { // When running as part of a test suite, resource teardown is deferred. // Shared resources (e.g., VMs) may be reused by subsequent test cases // within the same suite and must not be destroyed here. - return nil + return } // Unsubscribe device state watchers. @@ -742,12 +855,24 @@ func Close() error { th.log.Infof("Closed broker connection") } - // Stop the image server and remove its cache directory. + // Stop the SFTP server. + if th.sftpServerListener != nil { + if err := th.sftpServerListener.Close(); err != nil { + th.log.Warnf("Failed to close SFTP server listener: %v", err) + } + } + + // Stop the image server (HTTP and HTTPS) and remove its cache directory. if th.imgServerListener != nil { if err := th.imgServerListener.Close(); err != nil { th.log.Warnf("Failed to close image server listener: %v", err) } } + if th.imgServerTLSListener != nil { + if err := th.imgServerTLSListener.Close(); err != nil { + th.log.Warnf("Failed to close image server HTTPS listener: %v", err) + } + } if th.imgServerDir != "" { if err := os.RemoveAll(th.imgServerDir); err != nil { th.log.Warnf("Failed to remove image cache dir %s: %v", th.imgServerDir, err) @@ -774,7 +899,6 @@ func Close() error { if panicErr != nil { panic(panicErr) } - return nil } // Logger returns the logrus logger associated with the current test harness. @@ -1154,6 +1278,28 @@ func GetImageServerPort() uint16 { return imgServerPort } +// GetImageServerSFTPPort returns the port of the SFTP server that serves the +// same directory as the HTTP image server (see DefaultSFTPUsername/Password). +func GetImageServerSFTPPort() uint16 { + return imgServerSFTPPort +} + +// GetImageServerHTTPSPort returns the port of the HTTPS listener that serves +// the same directory as the HTTP image server (see GetCACertPEM for the CA +// certificate trusting it). +func GetImageServerHTTPSPort() uint16 { + return imgServerHTTPSPort +} + +// GetCACertPEM returns the PEM encoding of the harness's own CA certificate +// -- the one that signed the image server's HTTPS certificate (see +// GetImageServerHTTPSPort). Pass it via HTTPStorage.HTTPSTrustedCACertsPEM +// to trust downloads from the built-in image server over HTTPS. +func GetCACertPEM() []byte { + th := getTestHarness() + return utils.CertToPEM(th.caCert) +} + // GetSrcIPv4ForInternetAccess returns the first non-link-local IPv4 address // of the interface connecting container with the docker network. // This IP should be used as the source IP when tests diff --git a/evetest/localregistry.go b/evetest/localregistry.go new file mode 100644 index 00000000000..35c50f28366 --- /dev/null +++ b/evetest/localregistry.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package evetest + +import ( + "context" + "fmt" + "io" + stdlog "log" + "net" + "net/http" + "os" + "strconv" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/moby/moby/client" + "github.com/sirupsen/logrus" + + "github.com/lf-edge/eve/evetest/utils" +) + +// newLocalRegistryHandler returns an http.Handler implementing the Docker +// Registry HTTP API v2, mounted at "/v2/" on the harness's image-server +// listeners alongside the plain file server. It lets EVE pull a container +// image (an upgrade rootfs, or an application content tree) directly from +// evetest, without that image ever having been published to a real, +// reachable registry -- see PushDockerImageToLocalRegistry. +// +// Blobs are stored under dir (th.imgServerDir) rather than kept in memory -- +// TestUpgradeSuite runs every variant under a single Init, so an in-memory +// registry would keep every pushed image (an EVE rootfs is hundreds of MB) +// resident for the whole suite. Manifest/blob request logging is routed +// through harnessLog (at debug level, since it logs every single request) +// instead of registry.New's default of printing to stderr, outside logrus +// and outside the artifact dir. +func newLocalRegistryHandler(dir string, harnessLog *logrus.Logger) http.Handler { + registryLogger := stdlog.New( + harnessLog.WriterLevel(logrus.DebugLevel), "OCI Registry: ", 0) + return registry.New( + registry.WithBlobHandler(registry.NewDiskBlobHandler(dir)), + registry.Logger(registryLogger), + ) +} + +// localRegistryPullDomain is the registry host:port EVE is pointed at: the +// harness's own HTTPS image-server listener. Its certificate is not in any +// public trust store, so callers must also set DockerContainer. +// TrustedCACertsPEM to GetCACertPEM. +func localRegistryPullDomain() string { + return net.JoinHostPort(imgServerIPv4.String(), strconv.Itoa(imgServerHTTPSPort)) +} + +// localRegistryPushDomain is the same registry, reached over the harness's +// plain-HTTP image-server listener. evetest's own push (unlike EVE's pull) +// is a call this process makes directly, so it can simply ask for the +// insecure endpoint instead of dealing with its own self-signed certificate. +func localRegistryPushDomain() string { + return net.JoinHostPort(imgServerIPv4.String(), strconv.Itoa(imgServerPort)) +} + +// saveDockerImageToTempFile exports imageName from the local Docker daemon +// into a temporary, uncompressed tar file in the same layout `docker save` +// produces (what tarball.ImageFromPath expects), and returns its path. The +// caller must remove it. +func saveDockerImageToTempFile( + ctx context.Context, log *logrus.Entry, imageName string) (path string, err error) { + dockerClient, err := client.New(client.FromEnv) + if err != nil { + return "", fmt.Errorf("failed to create docker client: %w", err) + } + reader, err := dockerClient.ImageSave(ctx, []string{imageName}) + if err != nil { + return "", fmt.Errorf("failed to save docker image %q: %w", imageName, err) + } + defer func() { + if err := reader.Close(); err != nil { + log.Warnf("failed to close docker image save reader: %v", err) + } + }() + + f, err := os.CreateTemp("", "evetest-local-registry-*.tar") + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Warnf("failed to close temp file %q: %v", f.Name(), err) + } + }() + defer func() { + if err != nil { + if rmErr := os.Remove(f.Name()); rmErr != nil { + log.Warnf("failed to remove temp file %q: %v", f.Name(), rmErr) + } + } + }() + + if _, err = io.Copy(f, reader); err != nil { + err = fmt.Errorf("failed to save docker image %q to %q: %w", + imageName, f.Name(), err) + return "", err + } + return f.Name(), nil +} + +// PushDockerImageToLocalRegistry copies a Docker image -- pulling it first if +// not already present locally -- from the local Docker daemon into evetest's +// own embedded OCI registry, and returns the DockerContainer fields that +// point an EVE datastore at that copy. +// +// This is what lets an OCI/container datastore be exercised (an EVE upgrade +// via BaseOSDatastoreOCI, or a DockerContainer volume/app image) without the +// image under test already being published to a real, externally reachable +// registry: evetest re-serves whatever the local Docker daemon has under +// imageName (":", e.g. as returned by utils.EVEDockerImageName) +// as a datastore of its own. +func PushDockerImageToLocalRegistry(imageName string) (DockerContainer, error) { + th := getTestHarness() + log := th.log.WithField("component", "local-registry") + + // Parsed with name.NewTag rather than a naive split on ":", so a + // registry host carrying its own port (e.g. + // "harbor.example.com:5000/lfedge/eve:1.2.3-kvm-amd64") still yields + // the correct repository ("lfedge/eve") and tag. + srcTag, err := name.NewTag(imageName) + if err != nil { + return DockerContainer{}, fmt.Errorf( + "invalid docker image reference %q: expected \":\": %w", imageName, err) + } + repo := srcTag.Context().RepositoryStr() + tag := srcTag.TagStr() + if err := utils.PullDockerImage(th.ctx, log, imageName); err != nil { + return DockerContainer{}, fmt.Errorf( + "failed to obtain docker image %q: %w", imageName, err) + } + + tarPath, err := saveDockerImageToTempFile(th.ctx, log, imageName) + if err != nil { + return DockerContainer{}, fmt.Errorf( + "failed to export docker image %q: %w", imageName, err) + } + defer func() { + if err := os.Remove(tarPath); err != nil { + log.Warnf("failed to remove temp file %q: %v", tarPath, err) + } + }() + + img, err := tarball.ImageFromPath(tarPath, nil) + if err != nil { + return DockerContainer{}, fmt.Errorf( + "failed to read exported docker image %q: %w", imageName, err) + } + + dstRefStr := fmt.Sprintf("%s/%s:%s", localRegistryPushDomain(), repo, tag) + dstRef, err := name.ParseReference(dstRefStr, name.Insecure) + if err != nil { + return DockerContainer{}, fmt.Errorf( + "invalid local registry reference %q: %w", dstRefStr, err) + } + log.Infof("Pushing docker image %q into evetest's local OCI registry as %q", + imageName, dstRefStr) + if err := remote.Write(dstRef, img); err != nil { + err = fmt.Errorf( + "failed to push docker image %q to the local OCI registry: %w", + imageName, err) + return DockerContainer{}, err + } + + return DockerContainer{ + Domain: localRegistryPullDomain(), + ImageName: repo, + Tag: tag, + TrustedCACertsPEM: []string{string(GetCACertPEM())}, + }, nil +} diff --git a/evetest/netmodels/multi-eth.go b/evetest/netmodels/multi-eth.go index 505bb40d8db..a491fdc9161 100644 --- a/evetest/netmodels/multi-eth.go +++ b/evetest/netmodels/multi-eth.go @@ -180,6 +180,121 @@ var TwoMgmtPorts = &api.NetworkModel{ }, } +// TwoMgmtPortsWithPublicNTP is a network model with two ethernet ports, each +// on its own bridge/network with DHCP and access to the controller (same +// layout as TwoMgmtPorts), plus a distinct real public NTP server IP +// advertised via DHCP option 42 on each network (api.DHCP.PublicNtp, +// dnsmasq's "public_ntp" -- a real address, not an SDN-hosted endpoint). This +// lets NTP tests exercise EVE's DHCP+static NTP server merging (and actually +// observe chronyd syncing) without SDN having to run an NTP daemon of its own. +// +// network0 (eth0) advertises Cloudflare's primary NTP anycast address +// (162.159.200.1); network1 (eth1) advertises Google's time1.google.com +// address (216.239.35.0). Both are long-stable, single, documented IPs (not +// pool.ntp.org-style rotating addresses), so tests can assert on them exactly. +var TwoMgmtPortsWithPublicNTP = &api.NetworkModel{ + Ports: []*api.Port{ + { + LogicalLabel: "eth0", + AdminUp: true, + }, + { + LogicalLabel: "eth1", + AdminUp: true, + }, + }, + Bridges: []*api.Bridge{ + { + LogicalLabel: "bridge0", + Ports: []string{"eth0"}, + }, + { + LogicalLabel: "bridge1", + Ports: []string{"eth1"}, + }, + }, + Networks: []*api.Network{ + { + LogicalLabel: "network0", + Bridge: "bridge0", + Ipv4: &api.NetworkIPConfig{ + Subnet: "172.20.20.0/24", + GwIp: "172.20.20.1", + Dhcp: &api.DHCP{ + Enable: true, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server0"}, + }, + NtpSource: &api.DHCP_PublicNtp{PublicNtp: "162.159.200.1"}, + }, + }, + }, + { + LogicalLabel: "network1", + Bridge: "bridge1", + Ipv4: &api.NetworkIPConfig{ + Subnet: "172.20.21.0/24", + GwIp: "172.20.21.1", + Dhcp: &api.DHCP{ + Enable: true, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server1"}, + }, + NtpSource: &api.DHCP_PublicNtp{PublicNtp: "216.239.35.0"}, + }, + }, + }, + }, + Endpoints: &api.Endpoints{ + DnsServers: []*api.DNSServer{ + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server0", + Fqdn: "dns-server0.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.16.0/24", + Ip: "10.16.16.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server1", + Fqdn: "dns-server1.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.17.0/24", + Ip: "10.16.17.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + }, + }, +} + // TwoMgmtPortsOneBridge is a network model with two ethernet ports on a single // bridge and network with DHCP and access to the controller. It is intended // for bond (link aggregation) tests where both ports must reach the same network. @@ -1813,6 +1928,112 @@ var AppGatewayTopology = &api.NetworkModel{ }, } +// MgmtViaAppTopology is a two-port network model for routing EVE's own +// device-management traffic through an application acting as a NAT gateway. +// +// - eth0 ("wan-network", 10.60.10.0/24): app-shared Switch NI port, fully +// reachable (controller, dns-server). DHCP with a static reservation: +// MAC 02:16:3e:02:00:00 -> 10.60.10.150. The gateway app's WAN VIF must +// use this MAC to receive the deterministic IP. +// - eth1 ("lan-network", 10.60.20.0/24): management port, but the network +// itself has no outside reachability and (WithoutDefaultRoute) hands out +// no router option -- it only provides L2 connectivity between EVE's own +// static IP and the gateway app's LAN VIF. DHCP with a static +// reservation: MAC 02:16:3e:02:00:01 -> 10.60.20.150. The gateway app's +// LAN VIF must use this MAC so EVE can target it as a fixed gateway IP. +// +// The dns-server (10.16.16.25) is reachable only via eth0/wan-network and +// resolves the controller hostname. +var MgmtViaAppTopology = &api.NetworkModel{ + Ports: []*api.Port{ + {LogicalLabel: "eth0", AdminUp: true}, + {LogicalLabel: "eth1", AdminUp: true}, + }, + Bridges: []*api.Bridge{ + {LogicalLabel: "bridge0", Ports: []string{"eth0"}}, + {LogicalLabel: "bridge1", Ports: []string{"eth1"}}, + }, + Networks: []*api.Network{ + { + // WAN leg: the app's outbound (MASQUERADE'd) traffic exits here. + LogicalLabel: "wan-network", + Bridge: "bridge0", + Ipv4: &api.NetworkIPConfig{ + Subnet: "10.60.10.0/24", + GwIp: "10.60.10.1", + Dhcp: &api.DHCP{ + Enable: true, + IpRange: &api.IPRange{ + FromIp: "10.60.10.100", ToIp: "10.60.10.140"}, + StaticEntries: []*api.MACToIP{ + {Mac: "02:16:3e:02:00:00", Ip: "10.60.10.150"}, + }, + DomainName: "test", + Dns: &api.DNSClientConfig{ + PrivateDns: []string{"dns-server"}, + }, + }, + }, + Router: &api.Router{ + OutsideReachability: true, + ReachableEndpoints: []string{"dns-server"}, + }, + }, + { + // LAN leg: isolated at the SDN level -- EVE's own static IP and the + // app's LAN VIF are directly L2-adjacent on this bridge, with no + // router-provided path to anywhere else. WithoutDefaultRoute keeps + // the app's own DHCP-assigned VIF from picking up a bogus default + // route via this segment (its real default route must go via the + // WAN leg instead). + LogicalLabel: "lan-network", + Bridge: "bridge1", + Ipv4: &api.NetworkIPConfig{ + Subnet: "10.60.20.0/24", + GwIp: "10.60.20.1", + Dhcp: &api.DHCP{ + Enable: true, + IpRange: &api.IPRange{ + FromIp: "10.60.20.100", ToIp: "10.60.20.140"}, + DomainName: "test", + WithoutDefaultRoute: true, + StaticEntries: []*api.MACToIP{ + {Mac: "02:16:3e:02:00:01", Ip: "10.60.20.150"}, + }, + }, + }, + Router: &api.Router{ + OutsideReachability: false, + }, + }, + }, + Endpoints: &api.Endpoints{ + DnsServers: []*api.DNSServer{ + { + Endpoint: &api.Endpoint{ + LogicalLabel: "dns-server", + Fqdn: "dns-server.test", + Ipv4: &api.EndpointIPConfig{ + Subnet: "10.16.16.0/24", + Ip: "10.16.16.25", + }, + }, + StaticEntries: []*api.DNSEntry{ + { + FqdnSource: &api.DNSEntry_FqdnLiteral{ + FqdnLiteral: evetest.GetControllerHostname(), + }, + IpSource: &api.DNSEntry_IpLiteral{ + IpLiteral: evetest.GetControllerIPv4().String(), + }, + }, + }, + UpstreamServers: []string{"8.8.8.8", "1.1.1.1"}, + }, + }, + }, +} + // SeparateClusterPort is a multi-Ethernet network model with a dedicated cluster port per device. var SeparateClusterPort = &api.NetworkModel{ Ports: []*api.Port{ diff --git a/evetest/requirements.go b/evetest/requirements.go index 77f35e0b2bf..742c724f1a0 100644 --- a/evetest/requirements.go +++ b/evetest/requirements.go @@ -208,6 +208,14 @@ type RequireEdgeDevice struct { WithUSBPassthrough []USBDevice // TODO WithPCIPassthrough []PCIDevice // TODO + // ExtraDisks specifies additional blank disks (size in bytes each) to + // attach to the device's VM, beyond its main boot disk (MinDiskSizeInMiB). + // Each entry becomes one extra virtio-blk disk, in the given order (i.e. + // the first extra disk is the device's second disk overall). Intended for + // tests that configure EVE-level disk layout/RAID via + // EdgeDeviceConfig.SetDisksConfig. + ExtraDisks []uint64 + // What to do if EdgeDevice is already available (and still manageable) // from the previous test: DeviceReusePolicy ExistingEdgeDeviceReusePolicy diff --git a/evetest/sdn/VERSION b/evetest/sdn/VERSION index d5ab32c46f5..bc0a981c008 100644 --- a/evetest/sdn/VERSION +++ b/evetest/sdn/VERSION @@ -1,2 +1,2 @@ # Evetest-SDN version. Increment this manually whenever changes are made to evetest/sdn/vm. -1.0 +1.1 diff --git a/evetest/sdn/vm/pkg/configitems/dhcpSrv.go b/evetest/sdn/vm/pkg/configitems/dhcpSrv.go index d9309e4c370..237f26b562e 100644 --- a/evetest/sdn/vm/pkg/configitems/dhcpSrv.go +++ b/evetest/sdn/vm/pkg/configitems/dhcpSrv.go @@ -271,6 +271,13 @@ func (c *DhcpServerConfigurator) createDnsmasqConfFile(server DhcpServer) error if err := writeLine("dhcp-option=option:router,%s\n", server.GatewayIPv4.String()); err != nil { return err } + } else { + // Explicitly suppress option 3. Without this, dnsmasq falls back + // to advertising its own listening address on this subnet as the + // router. + if err := writeLine("dhcp-option=option:router\n"); err != nil { + return err + } } if server.DomainName != "" { // DHCP option 15. diff --git a/evetest/setup.go b/evetest/setup.go index 881990e393c..89f428cd7c9 100644 --- a/evetest/setup.go +++ b/evetest/setup.go @@ -325,6 +325,7 @@ func (th *TestHarness) prepareImageForEVEDevice(dev *deviceState) { LiveImageSource: dev.liveImageSource, MakeInstaller: dev.requirement.DeviceReusePolicy == CreateFromScratchWithInstaller, DiskBytes: uint64(diskSizeInMiB) << 20, + ExtraDiskBytes: dev.requirement.ExtraDisks, Config: &api.EveConfig{ ServerName: fmt.Sprintf("%s:%d", GetControllerHostname(), GetControllerPort()), SoftSerial: dev.requirement.WithSoftSerial, diff --git a/evetest/sftpserver.go b/evetest/sftpserver.go new file mode 100644 index 00000000000..7f8b67ee0f2 --- /dev/null +++ b/evetest/sftpserver.go @@ -0,0 +1,135 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package evetest + +import ( + "crypto/ed25519" + "crypto/rand" + "fmt" + "io" + "net" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +const ( + // DefaultSFTPUsername is the username accepted by evetest's built-in + // SFTP image server (see GetImageServerIPv4 / GetImageServerSFTPPort). + // Use it (together with DefaultSFTPPassword) when configuring SFTPStorage. + DefaultSFTPUsername = "evetest" + // DefaultSFTPPassword is the password accepted by evetest's built-in + // SFTP image server. + DefaultSFTPPassword = "pass123" +) + +// runSFTPServer serves rootDir read-only over SFTP on every connection +// accepted from listener, authenticating with DefaultSFTPUsername/Password. +// It returns once listener is closed (e.g. by TestHarness.Close), mirroring +// the plain HTTP image server started alongside it in Init(). +func (th *TestHarness) runSFTPServer(listener net.Listener, rootDir string) { + signer, err := generateSSHHostKey() + if err != nil { + th.log.Errorf("Failed to generate SFTP server host key: %v", err) + return + } + config := &ssh.ServerConfig{ + PasswordCallback: func( + conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + if conn.User() == DefaultSFTPUsername && + string(password) == DefaultSFTPPassword { + return nil, nil + } + return nil, fmt.Errorf( + "invalid SFTP credentials for user %q", conn.User()) + }, + } + config.AddHostKey(signer) + + for { + conn, err := listener.Accept() + if err != nil { + // Listener closed (harness shutting down) or accept error. + return + } + go th.handleSFTPConn(conn, config, rootDir) + } +} + +// handleSFTPConn performs the SSH handshake for a single incoming connection +// and serves an SFTP subsystem session (read-only, rooted at rootDir) over +// every session channel the client opens. +func (th *TestHarness) handleSFTPConn( + netConn net.Conn, config *ssh.ServerConfig, rootDir string) { + defer func() { + if err := netConn.Close(); err != nil { + th.log.Warnf("SFTP server: failed to close connection: %v", err) + } + }() + + sshConn, chans, reqs, err := ssh.NewServerConn(netConn, config) + if err != nil { + th.log.Debugf("SFTP server: SSH handshake failed: %v", err) + return + } + defer func() { + if err := sshConn.Close(); err != nil { + th.log.Warnf("SFTP server: failed to close SSH connection: %v", err) + } + }() + go ssh.DiscardRequests(reqs) + + for newChannel := range chans { + if newChannel.ChannelType() != "session" { + _ = newChannel.Reject(ssh.UnknownChannelType, "unsupported channel type") + continue + } + channel, requests, err := newChannel.Accept() + if err != nil { + th.log.Warnf("SFTP server: failed to accept channel: %v", err) + continue + } + go func() { + for req := range requests { + ok := req.Type == "subsystem" && len(req.Payload) >= 4 && + string(req.Payload[4:]) == "sftp" + if req.WantReply { + _ = req.Reply(ok, nil) + } + } + }() + go func() { + defer func() { + if err := channel.Close(); err != nil { + th.log.Warnf("SFTP server: failed to close channel: %v", err) + } + }() + server, err := sftp.NewServer(channel, + sftp.ReadOnly(), + sftp.WithServerWorkingDirectory(rootDir)) + if err != nil { + th.log.Warnf("SFTP server: failed to create session: %v", err) + return + } + defer func() { + if err := server.Close(); err != nil { + th.log.Warnf("SFTP server: failed to close session: %v", err) + } + }() + if err := server.Serve(); err != nil && err != io.EOF { + th.log.Debugf("SFTP server: session ended: %v", err) + } + }() + } +} + +// generateSSHHostKey creates an ephemeral Ed25519 SSH host key, used only +// for the lifetime of a single test-harness run. +func generateSSHHostKey() (ssh.Signer, error) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + return ssh.NewSignerFromKey(priv) +} diff --git a/evetest/ssh.go b/evetest/ssh.go index 8f22c7d09aa..6f4d812bfda 100644 --- a/evetest/ssh.go +++ b/evetest/ssh.go @@ -54,6 +54,64 @@ func (th *TestHarness) runScriptOnEVEOverSSH(ctx context.Context, devName string stdout, stderr, stdoutWatchdogTimeout) } +// dialViaSSH opens an SSH connection to the given EVE device and tunnels a +// TCP connection to remoteAddr through it (an SSH direct-tcpip channel), as +// if network is dialed from the device itself. Useful for reaching services +// that only listen on the device's own loopback interface, e.g. the Kubevirt +// VNC proxy (see pkg/pillar/docs/vnc-workflows.md). +// +// The returned net.Conn owns the underlying SSH client: closing it also +// closes the SSH connection. +func (th *TestHarness) dialViaSSH(ctx context.Context, devName string, + network, remoteAddr string) (net.Conn, error) { + + eveIP, err := th.getReachableEVEAddr(ctx, devName, 22, "") + if err != nil { + return nil, err + } + + keyPEM, err := os.ReadFile("/root/.ssh/eve_rsa") + if err != nil { + return nil, fmt.Errorf("failed to read EVE SSH key: %w", err) + } + signer, err := ssh.ParsePrivateKey(keyPEM) + if err != nil { + return nil, fmt.Errorf("failed to parse client key: %w", err) + } + + addr := net.JoinHostPort(eveIP, "22") + sshConfig := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 5 * time.Second, + } + client, err := ssh.Dial("tcp", addr, sshConfig) + if err != nil { + return nil, fmt.Errorf("SSH dial to %s failed: %w", addr, err) + } + + conn, err := client.Dial(network, remoteAddr) + if err != nil { + _ = client.Close() + return nil, fmt.Errorf("SSH tunnel dial to %s failed: %w", remoteAddr, err) + } + return &sshTunnelConn{Conn: conn, client: client}, nil +} + +// sshTunnelConn wraps an SSH-tunneled net.Conn so that closing it also closes +// the underlying SSH client connection that carries it. +type sshTunnelConn struct { + net.Conn + client *ssh.Client +} + +func (c *sshTunnelConn) Close() error { + err := c.Conn.Close() + _ = c.client.Close() + return err +} + // runScriptOverSSH executes a shell script on a remote host over SSH // using the Go crypto/ssh client library. It supports username/password and // client-certificate authentication methods (see AuthMethod). @@ -152,9 +210,15 @@ func (th *TestHarness) scpFromEVE(ctx context.Context, if recursive { scpArgs = append(scpArgs, "-r") } + // The path after the colon is interpreted by a remote shell, so it needs + // its own shell quoting independent of how this argv element is split + // locally (scp itself is invoked directly via exec, with no local shell + // involved) -- otherwise a remote path containing spaces (e.g. a pubsub + // key like "Application Data Store") gets split into multiple arguments + // remotely. scpArgs = append(scpArgs, "-i", "/root/.ssh/eve_rsa", - "root@"+eveIP+":"+remotePath, + "root@"+eveIP+":"+shellEscape(remotePath), localPath, ) cmd := exec.CommandContext(ctx, "scp", scpArgs...) @@ -178,7 +242,7 @@ func (th *TestHarness) scpToEVE(ctx context.Context, scpArgs = append(scpArgs, "-i", "/root/.ssh/eve_rsa", localPath, - "root@"+eveIP+":"+remotePath, + "root@"+eveIP+":"+shellEscape(remotePath), ) cmd := exec.CommandContext(ctx, "scp", scpArgs...) return cmd.Run() diff --git a/evetest/testparam.go b/evetest/testparam.go index 64d666b40d8..09f1a4252c0 100644 --- a/evetest/testparam.go +++ b/evetest/testparam.go @@ -285,19 +285,6 @@ func GetDiskSizeMiBParameterValue() uint32 { return GetTestParameter[uint32](DiskSizeMiBParameterKey) } -// SkipIfHypervisorKubevirt skips the current test if the resolved HYPERVISOR -// parameter is HypervisorKubevirt. Kubevirt is only supported by tests under -// `evetest/tests/cluster`; non-cluster tests should call this helper right -// after defining the HypervisorParameter to ensure they are not accidentally -// exercised on a Kubevirt-flavored EVE build. -func SkipIfHypervisorKubevirt() { - th := getTestHarness() - if GetHypervisorParameterValue() == HypervisorKubevirt { - th.t.Skipf("Kubevirt hypervisor is only supported by cluster tests " + - "(under evetest/tests/cluster); use kvm or xen") - } -} - // FilesystemParameterKey is the key used for the Filesystem parameter. const FilesystemParameterKey = "FILESYSTEM" diff --git a/evetest/tests/apps/purge_test.go b/evetest/tests/apps/purge_test.go new file mode 100644 index 00000000000..d6f3fb810b6 --- /dev/null +++ b/evetest/tests/apps/purge_test.go @@ -0,0 +1,242 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + uuid "github.com/satori/go.uuid" + "google.golang.org/protobuf/proto" +) + +// TestPurgeNeverActivatedApp is a regression test for EVE commit a1582bb40 +// ("zedmanager: fix purge stuck when app was never activated"). +// +// The bug: when a purge was triggered for an app whose image had failed to +// download (so no domain was ever created for it), the old code left +// PurgeInprogress=BringDown without calling purgeCmdDone. This caused the +// app to get stuck indefinitely (at LOADED with VerifyOnly=true) because +// volumemgr was never told to create the volume. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability +// and to pull the container images; the app under test needs no network +// adapter of its own for this test. +// +// Phases +// ------ +// 1. Deploy "bad-image-app" referencing a nonexistent image tag +// (docker://nginx:purge-test-nonexistent-99999). Tag resolution fails, +// so the app never activates (no domain is ever created). Wait for the +// app to report a non-empty AppErr while not RUNNING -- this confirms +// the broken precondition without assuming any particular stuck +// SwState (the exact state the app settles into isn't the point of this +// test, and asserting on the wrong one would make the test as fragile +// as the bug it's meant to catch). +// 2. fixImageAndPurge applies, in a single EdgeDevConfig mutation / +// ApplyConfig call: +// a. a fresh ContentTree (new UUID, same datastore, URL fixed to the +// working tag docker://nginx:stable, Sha256 cleared), +// b. a fresh Volume (new UUID) whose origin points at that fresh +// ContentTree, +// c. the app's VolumeRefList rewired to the fresh Volume, +// d. the app's purge counter incremented. +// This combination -- fixing the image while simultaneously purging -- +// is exactly what triggers the bug: EVE must tear down the +// never-activated app and reprocess it from scratch instead of getting +// stuck. +// 3. With the fix, the app eventually reaches RUNNING. (Without the fix, +// this step is where the test would time out -- the app would remain +// stuck indefinitely instead.) +// 4. Cleanup: delete the app, wait for ZSwState_INVALID. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestAppsSuite. +func TestPurgeNeverActivatedApp(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + // Step 1: deploy an app whose image tag does not exist, so it never + // activates. + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "bad-image-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "nginx", + Tag: "purge-test-nonexistent-99999", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + }) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + device.ApplyConfig(devConfig, false, false) + + log := evetest.Logger() + log.Infof("Waiting for the app to fail to activate (nonexistent image tag)") + timeout := 5 * time.Minute + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app never activates (bad image tag) and reports an error", + func(info *eveinfo.ZInfoApp) bool { + return info.State != eveinfo.ZSwState_RUNNING && len(info.AppErr) > 0 + }))) + evetest.Checkpoint("app-never-activated") + + // Step 2: fix the image and purge, atomically. + log.Infof("Fixing the image and purging the app in a single config apply") + fixImageAndPurge(t, devConfig, appUUID, "nginx:stable") + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("app-fixed-and-purged") + + // Step 3: with the fix, the app recovers and reaches RUNNING. Without it, + // this is where the test would time out. + log.Infof("Waiting for the app to reach RUNNING after the purge") + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app reaches RUNNING after purge", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_RUNNING + }))) + evetest.Checkpoint("app-running-after-purge") + + // Cleanup. + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopAppWatch() +} + +// fixImageAndPurge replicates, as a single EdgeDevConfig mutation, what a +// real app purge does (fresh Volume + ContentTree identities) while also +// fixing the app's image reference to a working one -- the exact +// combination that reproduces EVE commit a1582bb40. goodImageURL is the +// ":" reference to switch the app's ContentTree to (same +// Datastore, only the URL and identity change). +func fixImageAndPurge(t *WithT, devConfig *evetest.EdgeDeviceConfig, + appUUID uuid.UUID, goodImageURL string) { + appUUIDStr := appUUID.String() + + var app *eveconfig.AppInstanceConfig + for _, a := range devConfig.Apps { + if a.GetUuidandversion().GetUuid() == appUUIDStr { + app = a + break + } + } + t.Expect(app).ToNot(BeNil(), "app %s not found in config", appUUIDStr) + t.Expect(app.VolumeRefList).To(HaveLen(1)) + + oldVolUUID := app.VolumeRefList[0].Uuid + oldVolIdx := -1 + for i, v := range devConfig.Volumes { + if v.Uuid == oldVolUUID { + oldVolIdx = i + break + } + } + t.Expect(oldVolIdx).To(BeNumerically(">=", 0), + "volume %s not found in config", oldVolUUID) + oldVolume := devConfig.Volumes[oldVolIdx] + + oldCTUUID := oldVolume.GetOrigin().GetDownloadContentTreeID() + oldCTIdx := -1 + for i, ct := range devConfig.ContentInfo { + if ct.Uuid == oldCTUUID { + oldCTIdx = i + break + } + } + t.Expect(oldCTIdx).To(BeNumerically(">=", 0), + "content tree %s not found in config", oldCTUUID) + oldCT := devConfig.ContentInfo[oldCTIdx] + + newCTUUID, err := uuid.NewV4() + t.Expect(err).ToNot(HaveOccurred()) + newVolUUID, err := uuid.NewV4() + t.Expect(err).ToNot(HaveOccurred()) + + // Fresh ContentTree: same Datastore reference, fixed (working) URL. + newCT := proto.CloneOf(oldCT) + newCT.Uuid = newCTUUID.String() + newCT.URL = goodImageURL + newCT.Sha256 = "" + devConfig.ContentInfo[oldCTIdx] = newCT + + // Fresh Volume pointing at the fresh ContentTree. + newVolume := proto.CloneOf(oldVolume) + newVolume.Uuid = newVolUUID.String() + newVolume.Origin.DownloadContentTreeID = newCTUUID.String() + devConfig.Volumes[oldVolIdx] = newVolume + + // Rewire the app's volume reference and increment the purge counter -- + // together, this is exactly what a real purge does. + app.VolumeRefList[0].Uuid = newVolUUID.String() + purgeCounter := uint32(0) + if app.Purge != nil { + purgeCounter = app.Purge.Counter + } + app.Purge = &eveconfig.InstanceOpsCmd{Counter: purgeCounter + 1} +} + +func appHasError(info *eveinfo.ZInfoApp) (string, bool) { + stop := info.State == eveinfo.ZSwState_ERROR + if stop { + return "Application instance is in error state", true + } + return "", false +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go new file mode 100644 index 00000000000..a589b996af3 --- /dev/null +++ b/evetest/tests/apps/testsuite_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestAppsSuite drives application-lifecycle scenarios that are not +// specifically about networking (regression tests for zedmanager/volumemgr +// bugs, VNC console access, etc.) -- kept separate from +// evetest/tests/networking's TestApplicationConnectivitySuite, which is +// already large and focused on network connectivity. +// +// Subtests +// -------- +// - TestPurgeNeverActivatedApp -- regression test for a zedmanager bug +// where purging an app that never activated (failed image download) +// would leave it stuck instead of recovering. +// - TestVNC -- VNC access to a VM app, a container app, and the container +// app's shim VM console. +func TestAppsSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestPurgeNeverActivatedApp, + }, + evetest.TestCase{ + Test: TestVNC, + }, + ) +} diff --git a/evetest/tests/apps/vnc_test.go b/evetest/tests/apps/vnc_test.go new file mode 100644 index 00000000000..9407b768b68 --- /dev/null +++ b/evetest/tests/apps/vnc_test.go @@ -0,0 +1,414 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "fmt" + "net" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/amitbet/vncproxy/client" + "github.com/amitbet/vncproxy/logger" + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// X11/RFB keysym constants used to drive the shim-VM console switch +// documented in docs/VNC.md ("Switching between the container and the shim +// VM session"): Ctrl+Alt+2 then Enter. +const ( + keysymControlL = 0xffe3 + keysymAltL = 0xffe9 + keysym2 = 0x0032 + keysymReturn = 0xff0d +) + +// alpineCloudImage describes the arch-specific pinned Alpine Linux +// cloud-init qcow2 image used to boot the VM app in this test. +type alpineCloudImage struct { + relativePath string + sha256 string + sizeBytes uint64 +} + +// Alpine 3.24.1 cloud images, pinned by release version (not a rolling +// "latest" alias) so the SHA256 below stays valid indefinitely. See +// https://alpinelinux.org/cloud/ for the full image list. +var alpineCloudImages = map[string]alpineCloudImage{ + "amd64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2", + sha256: "6e2e6fe0572b6632527f268d3659e8fccebda4e1ee470fafe2c4d7b85b6a4df6", + sizeBytes: 183697408, + }, + "arm64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-aarch64-uefi-cloudinit-r0.qcow2", + sha256: "3059a6280977c2122982632e0317c5ddbd39069d46ca1e60480de283091f720f", + sizeBytes: 239271936, + }, +} + +// TestVNC verifies VNC access to EVE application consoles: a VM app's own +// display, a container app's display, and the container app's underlying +// shim VM console (reached via the documented key-combo switch once shim-VM +// VNC access is enabled). +// +// Under KVM the VNC server is a raw QEMU socket reachable directly on the +// device's uplink IP. Under Kubevirt there is no such socket: zedkube starts +// a `virtctl vnc --proxy-only` process (see pkg/pillar/docs/ +// vnc-workflows.md) that bridges the VMI's console onto a TCP port bound to +// 127.0.0.1 only, gated by AppInstanceConfig.RemoteConsole rather than +// EnableVNC. This test reaches that port through an SSH tunnel to the device +// (EdgeDevice.DialViaSSH) instead of a direct dial, and falls back to VNC's +// "none" security type when the server doesn't offer password auth (virtctl +// relies on Kubernetes RBAC for access control, not a VNC password) -- see +// connectVNC. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability +// and Internet access (to pull the Alpine cloud image and the +// evetest-ubuntu-ctr container image). VNC itself is never reached +// through an app network, so neither app in this test is given a network +// adapter of its own. +// +// Phases +// ------ +// 1. Global config: enable app.allow.vnc (opens the device's uplink +// firewall for TCP ports 5900-5999 -- see +// pkg/pillar/dpcreconciler/linux.go) and debug.enable.vnc.shim.vm (a +// per-node flag which, combined with a VM's own EnableVnc, additionally +// allows switching into the shim VM console -- see +// pkg/pillar/hypervisor/kvm.go, isVncShimVMEnabled). Both are KVM-only +// knobs (harmless, but ineffective, under Kubevirt). +// 2. VM app: deploy an Alpine Linux VM (VirtualizationMode=HVM, image from +// alpineCloudImages matching the device's actual arch (device.GetArch()), +// EnableVNC=true, VNCDisplay=1, a fixed VNCPassword). No UserData/ +// cloud-init is configured -- VNC reaches the QEMU console as soon as +// the VM starts, independent of what the guest OS is doing. +// WaitUntilAppIsRunning, then (under Kubevirt) enable RemoteConsole on +// the app config and re-apply, connect a real VNC/RFB client to port +// 5901 (directly under KVM, through an SSH tunnel under Kubevirt), +// authenticate, and assert the handshake succeeds. Delete the app +// afterwards (disabling RemoteConsole first under Kubevirt, since only +// one remote-console session is allowed on the device at a time). +// 3. Container app: deploy lfedge/evetest-ubuntu-ctr:1.0 with EnableVNC=true, +// VNCDisplay=2, the same VNCPassword. WaitUntilAppIsRunning, then connect +// to port 5902 (same per-hypervisor path as above) and assert the same +// successful handshake -- proving VNC also works for container apps +// (viewing the shim VM's console showing the container's entry point). +// 4. Shim VM console switch: on that same VNC connection, send the +// documented key combo (Ctrl+Alt+2, then Enter -- see docs/VNC.md, +// "Switching between the container and the shim VM session") via RFB +// KeyEvent messages, then issue a fresh FramebufferUpdateRequest and a +// benign follow-up call, asserting neither errors -- i.e. the session +// stays healthy across the console switch. Note: asserting that the +// *displayed content* actually changed to the shim VM's login prompt +// would need pixel-level framebuffer decoding, which the available Go +// VNC client library does not expose at a level convenient enough to +// verify robustly here; left as a known scope limit. +// 5. Cleanup: delete the container app. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestAppsSuite. +func TestVNC(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Phase 1: allow external VNC access to the device's uplink ports and + // enable node-wide shim-VM VNC access. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueBool(pillartypes.AllowAppVnc, true) + cfgProps.SetGlobalValueBool(pillartypes.VncShimVMAccess, true) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + const vncPassword = "12345678" // classic VNC (DES) auth is limited to 8 chars + timeout := 3 * time.Minute + timeoutExcludingDownload := 8 * time.Minute + log := evetest.Logger() + + // arch selects which pinned Alpine image matches the device's actual + // CPU architecture. + arch := device.GetArch() + image, ok := alpineCloudImages[arch] + t.Expect(ok).To(BeTrue(), "no pinned Alpine cloud image for arch %q", arch) + + var deviceIP []net.IP + if hypervisor != evetest.HypervisorKubevirt { + deviceIP = device.GetDeviceIPAddress("ethernet0") + t.Expect(deviceIP).ToNot(BeEmpty()) + } + + // vncDialer returns a description (for log/error messages) and a dial + // function for the VNC endpoint at 5900+vncDisplay: a direct dial to the + // device's uplink IP under KVM/Xen, or a dial tunneled over SSH to the + // device's own loopback under Kubevirt (see the doc comment above). + vncDialer := func(vncDisplay uint) (string, func() (net.Conn, error)) { + port := fmt.Sprintf("%d", 5900+vncDisplay) + if hypervisor == evetest.HypervisorKubevirt { + addr := net.JoinHostPort("127.0.0.1", port) + return addr + " (via SSH tunnel)", func() (net.Conn, error) { + return device.DialViaSSH("tcp", addr) + } + } + addr := net.JoinHostPort(deviceIP[0].String(), port) + return addr, func() (net.Conn, error) { + return net.DialTimeout("tcp", addr, 5*time.Second) + } + } + + // setRemoteConsole toggles AppInstanceConfig.RemoteConsole for the app + // identified by appUUIDStr and re-applies the device config. This is the + // field that gates Kubevirt's virtctl-based VNC proxy (see + // ApplicationInstanceConfig.RemoteConsole). + setRemoteConsole := func(appUUIDStr string, enable bool) { + found := false + for _, app := range devConfig.Apps { + if app.GetUuidandversion().GetUuid() == appUUIDStr { + app.RemoteConsole = enable + found = true + break + } + } + t.Expect(found).To(BeTrue(), "app %q not found in device config", appUUIDStr) + device.ApplyConfig(devConfig, false, false) + } + + // waitRemoteConsoleReleased polls (via the same SSH-tunneled dial used to + // reach it) until the Kubevirt VNC proxy for vncDisplay has stopped + // listening, i.e. zedkube has torn it down after RemoteConsole was + // disabled. Only one remote-console session is allowed on the device at + // a time (see pkg/pillar/docs/vnc-workflows.md, canClaimVNCFile), so the + // next app's setRemoteConsole(true) would otherwise be silently ignored + // while the previous proxy is still up. + waitRemoteConsoleReleased := func(vncDisplay uint) { + _, dial := vncDialer(vncDisplay) + t.Eventually(func() error { + conn, err := dial() + if err == nil { + _ = conn.Close() + return fmt.Errorf("VNC proxy for display %d is still listening", vncDisplay) + } + return nil + }, 30*time.Second, 2*time.Second).Should(Succeed()) + } + + // Phase 2: VM app VNC access. + vmAppUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vnc-vm-app", + Activate: true, + Image: evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW2, + ImageSHA256: image.sha256, + MaxDownloadBytes: image.sizeBytes, + ImageRelativePath: image.relativePath, + ServerAddress: "dl-cdn.alpinelinux.org", + UseHTTPS: true, + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + EnableVNC: true, + VNCDisplay: 1, + VNCPassword: vncPassword, + }) + vmAppUpdates, stopVMAppWatch := device.WatchAppInfo(vmAppUUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(vmAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("vm-app-running") + + vmAppUUIDStr := vmAppUUID.String() + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(vmAppUUIDStr, true) + } + log.Infof("Testing VNC access to the VM app") + vmVNCDesc, vmVNCDial := vncDialer(1) + vmConn := connectVNC(t, vmVNCDesc, vmVNCDial, vncPassword) + log.Infof("Connected to VM app VNC desktop %q", vmConn.DesktopName) + t.Expect(vmConn.Close()).To(Succeed()) + evetest.Checkpoint("vm-app-vnc-verified") + + if hypervisor == evetest.HypervisorKubevirt { + // Release the remote-console session before deleting the app: only + // one is allowed on the device at a time, and the container app's + // session below would otherwise be silently blocked. + setRemoteConsole(vmAppUUIDStr, false) + waitRemoteConsoleReleased(1) + } + + devConfig.DeleteApplication(vmAppUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(vmAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "VM app state is UNSPECIFIED", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopVMAppWatch() + + // Phase 3: container app VNC access. + ctrAppUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vnc-ctr-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnableVNC: true, + VNCDisplay: 2, + VNCPassword: vncPassword, + }) + ctrAppUUIDStr := ctrAppUUID.String() + ctrAppUpdates, stopCtrAppWatch := device.WatchAppInfo(ctrAppUUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(ctrAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("container-app-running") + + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(ctrAppUUIDStr, true) + } + log.Infof("Testing VNC access to the container app") + ctrVNCDesc, ctrVNCDial := vncDialer(2) + ctrConn := connectVNC(t, ctrVNCDesc, ctrVNCDial, vncPassword) + log.Infof("Connected to container app VNC desktop %q", ctrConn.DesktopName) + evetest.Checkpoint("container-app-vnc-verified") + + // Phase 4: switch to the shim VM console (Ctrl+Alt+2, then Enter -- see + // docs/VNC.md) and confirm the session stays healthy across the switch. + log.Infof("Switching to the shim VM console") + t.Expect(pressKeyCombo(ctrConn, keysymControlL, keysymAltL, keysym2)).To(Succeed()) + t.Expect(pressKeyCombo(ctrConn, keysymReturn)).To(Succeed()) + t.Expect(ctrConn.FramebufferUpdateRequest( + false, 0, 0, ctrConn.FrameBufferWidth, ctrConn.FrameBufferHeight)).To(Succeed()) + time.Sleep(2 * time.Second) + // A benign follow-up call: if the console switch broke the session, the + // underlying connection would already be closed and this would error. + t.Expect(ctrConn.FramebufferUpdateRequest( + true, 0, 0, ctrConn.FrameBufferWidth, ctrConn.FrameBufferHeight)).To(Succeed()) + t.Expect(ctrConn.Close()).To(Succeed()) + evetest.Checkpoint("shim-vm-switch-verified") + + // Cleanup. + if hypervisor == evetest.HypervisorKubevirt { + setRemoteConsole(ctrAppUUIDStr, false) + } + devConfig.DeleteApplication(ctrAppUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(ctrAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "Container app state is UNSPECIFIED", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopCtrAppWatch() +} + +// connectVNC calls dial (retrying while the endpoint isn't reachable yet -- +// under Kubevirt the virtctl proxy takes a few seconds to start after +// RemoteConsole is enabled), then performs the full RFB handshake and +// returns the connected client. dialDesc identifies the endpoint in log/error +// messages. It fails the test (via t) rather than returning an error, since a +// broken VNC connection always indicates a test failure at the call sites +// above. +func connectVNC(t *WithT, dialDesc string, dial func() (net.Conn, error), + password string) *client.ClientConn { + // The client's background mainLoop goroutine logs an Error-level message + // whenever its blocking read unblocks because the connection was closed + // -- including on an intentional Close() from our side. Silence it here + // (matching eden's own pkg/utils/vnc.go, which does the same) so a clean + // shutdown doesn't look like a failure in the test output. + logger.SetLogLevel("fatal") + + var conn net.Conn + t.Eventually(func() error { + var err error + conn, err = dial() + return err + }, 90*time.Second, 3*time.Second).Should(Succeed(), + "failed to dial VNC endpoint %s", dialDesc) + + clientConn, err := client.NewClientConn(conn, &client.ClientConfig{ + // A raw QEMU VNC socket (KVM) offers VNC password auth; Kubevirt's + // virtctl proxy offers only the "none" security type, relying on + // Kubernetes RBAC instead of a VNC password. Offering both lets the + // client pick whichever the server actually supports. + Auth: []client.ClientAuth{&client.PasswordAuth{Password: password}, new(client.ClientAuthNone)}, + Exclusive: false, + }) + t.Expect(err).ToNot(HaveOccurred()) + + t.Expect(clientConn.Connect()).To(Succeed(), + "VNC RFB handshake with %s failed", dialDesc) + return clientConn +} + +// pressKeyCombo sends a key-down event for every keysym in order, then a +// key-up event in reverse order, emulating a user holding several keys +// together (e.g. Ctrl+Alt+2). +func pressKeyCombo(conn *client.ClientConn, keysyms ...uint32) error { + for _, keysym := range keysyms { + if err := conn.KeyEvent(keysym, true); err != nil { + return fmt.Errorf("key-down for keysym 0x%x: %w", keysym, err) + } + } + for i := len(keysyms) - 1; i >= 0; i-- { + if err := conn.KeyEvent(keysyms[i], false); err != nil { + return fmt.Errorf("key-up for keysym 0x%x: %w", keysyms[i], err) + } + } + return nil +} diff --git a/evetest/tests/cluster/cluster_test.go b/evetest/tests/cluster/cluster_test.go index f3f6a983b71..df9520de302 100644 --- a/evetest/tests/cluster/cluster_test.go +++ b/evetest/tests/cluster/cluster_test.go @@ -13,9 +13,7 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" - eveinfo "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/evetest" - "github.com/lf-edge/eve/evetest/matchers" "github.com/lf-edge/eve/evetest/netmodels" "github.com/lf-edge/eve/pkg/pillar/types" ) @@ -74,13 +72,13 @@ func clusterDeviceRequirements( // Phases // ------ // 1. setup-done -> initial-config-applied: apply the bare device config -// (no app yet) and start watching ClusterInfo. -// 2. k3s-is-ready: ZInfoKubeCluster eventually reports a single node -// whose NodeReady condition is true AND -// Storage.Health=SERVICE_STATUS_HEALTHY. Then assert ClusterId is -// non-empty, the node is RoleServer + Schedulable, and that no -// EveApps / EveVmApps / PodNameSpaces have been created yet -// (clean-slate cluster, no workload). +// (no app yet). +// 2. k3s-is-ready: device.WaitForClusterNodeIsReady waits until this +// device reports itself as a Ready node with healthy cluster storage. +// Then assert there is exactly one node, ClusterId is non-empty, the +// node is RoleServer + Schedulable, and that no EveApps / EveVmApps / +// PodNameSpaces have been created yet (clean-slate cluster, no +// workload). // 3. app-config-is-submitted: add the local NI + the container app to the // config and re-apply. // 4. app-is-deployed: WaitUntilAppIsRunning (the helper tracks @@ -135,31 +133,13 @@ func TestSingleNodeCluster(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device := evetest.GetEdgeDevice(devName) - clusterUpdates, stopClusterWatch := device.WatchClusterInfo() - defer stopClusterWatch() device.ApplyConfig(devConfig, true, true) evetest.Checkpoint("initial-config-applied") timeout := 20 * time.Minute - var clusterInfo *eveinfo.ZInfoKubeCluster - const nodeReadyCond = eveinfo.KubeNodeConditionType_KUBE_NODE_CONDITION_TYPE_READY - t.Eventually(clusterUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( - "K3s is ready", - func(info *eveinfo.ZInfoKubeCluster) bool { - clusterInfo = info - if len(info.Nodes) != 1 { - return false - } - if clusterInfo.Storage.Health != eveinfo.ServiceStatus_SERVICE_STATUS_HEALTHY { - return false - } - for _, cond := range info.Nodes[0].GetConditions() { - if cond.GetType() == nodeReadyCond { - return cond.GetSet() - } - } - return false - }))) + device.WaitForClusterNodeIsReady(timeout) + clusterInfo := device.GetClusterInfo() + t.Expect(clusterInfo.Nodes).To(HaveLen(1)) t.Expect(clusterInfo.ClusterId).NotTo(BeEmpty()) t.Expect(clusterInfo.Nodes[0].RoleServer).To(BeTrue()) t.Expect(clusterInfo.Nodes[0].Schedulable).To(BeTrue()) diff --git a/evetest/tests/lps/app_local_info_test.go b/evetest/tests/lps/app_local_info_test.go new file mode 100644 index 00000000000..f8a23e6a800 --- /dev/null +++ b/evetest/tests/lps/app_local_info_test.go @@ -0,0 +1,319 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/utils/generics" +) + +// TestAppLocalInfo verifies the Local Profile Server (LPS) app-info / +// app-command exchange: EVE reports every deployed app's state to the LPS +// (GET /manage/v1/appinfo), and the LPS can request a purge or restart of a +// specific app (PUT /manage/v1/app-command) independently of the +// controller. EVE sums LPS-driven and controller-driven purge/restart +// counters -- both paths must work, and only a purge (LPS-driven or +// controller-driven) recreates the app's volume; a restart does not. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. Local NI "local-ni" +// hosts both the LPS app (port-fwd for its management API + SSH) and +// "app1" (port-fwd for direct SSH, used to create/check files that +// prove whether a purge recreated the volume). +// +// Phases +// ------ +// 1. Deploy the LPS app. WaitUntilAppIsRunning, configure the LPS token, +// and configure EVE to use it via SetLPS. +// 2. Confirm the LPS's app-info list includes "lps-app" but not "app1" +// (app1 does not exist yet). +// 3. Deploy "app1" (plain container, no ProfileList). WaitUntilAppIsRunning, +// wait for SSH, then confirm the LPS's app-info list now includes both +// apps. +// 4. LPS-driven purge: create /root/purge_test in app1, confirm app1's +// LPS-reported LastCmdTimestamp is still 0, then PUT an app-command +// {app1, timestamp=123, COMMAND_PURGE}. Wait for app1 to pass through +// PURGING/HALTING and back to RUNNING, confirm the LPS now reports +// LastCmdTimestamp=123, and confirm /root/purge_test is gone (purge +// recreated the volume). +// 5. Controller-driven purge: create /root/purge_test again, call +// PurgeApplication directly (bypassing the LPS). Wait for the same +// PURGING/HALTING -> RUNNING cycle, confirm the LPS-reported +// LastCmdTimestamp is *unchanged* at 123 (a controller-driven purge +// does not touch the LPS-tracked command timestamp), and confirm +// /root/purge_test is gone again. +// 6. LPS-driven restart: create both /tmp/restart_test (tmpfs -- wiped by +// any VM reboot) and /root/purge_test (persistent -- survives a +// restart, unlike a purge) in app1, then PUT an app-command {app1, +// timestamp=456, COMMAND_RESTART}. Wait for app1 to pass through +// RESTARTING/HALTING and back to RUNNING, confirm LastCmdTimestamp=456, +// confirm /tmp/restart_test is gone but /root/purge_test still exists +// (a restart does not recreate the volume). +// 7. Controller-driven restart: same file setup, call RebootApplication +// directly. Confirm LastCmdTimestamp is *unchanged* at 456, and the +// same file-survival pattern. +// 8. Delete app1, wait for it to be gone, then confirm the LPS's app-info +// list no longer includes "app1" (but still includes "lps-app"). +// 9. Cleanup: delete the LPS app and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestAppLocalInfo(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 5 * time.Minute + polling := 3 * time.Second + + // Step 2: appinfo includes lps-app but not app1 yet. + log.Infof("Waiting for the LPS to report appinfo for lps-app only") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).To(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-before-app1") + + // Step 3: deploy app1. + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2224, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + sshTimeout := 20 * time.Second + waitApp1SSHReachable := func() { + log.Infof("Waiting for app1 SSH to become reachable...") + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + app1UUID, lpsAppAuth, "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitApp1SSHReachable() + + log.Infof("Waiting for the LPS to report appinfo for both apps") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).ToNot(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-with-app1") + + // waitAppTransientThenRunning drains app1Updates until it reports one of + // the given transient states (no exported helper targets these), then + // waits for RUNNING again via WaitUntilAppIsRunning directly, then for + // SSH to become reachable again. + waitAppTransientThenRunning := func(transient ...eveinfo.ZSwState) { + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 enters a transient state", + func(info *eveinfo.ZInfoApp) bool { + return generics.ContainsItem(transient, info.State) + }))) + device.WaitUntilAppIsRunning(app1UUID, timeout) + waitApp1SSHReachable() + } + touchApp1 := func(path string) { + _, _, err := device.RunShellScriptInsideApp(app1UUID, lpsAppAuth, + "touch "+path, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + } + app1FileExists := func(path string) bool { + output, _, err := device.RunShellScriptInsideApp(app1UUID, lpsAppAuth, + "test -f "+path+" && echo EXISTS; true", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + return strings.Contains(output, "EXISTS") + } + waitLastCmdTimestamp := func(want uint64, desc string) { + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + info := appInfoByName(list, "app1") + t.Expect(info).ToNot(BeNil()) + t.Expect(info.GetLastCmdTimestamp()).To(Equal(want), desc) + }, timeout, polling).Should(Succeed()) + } + + // Step 4: LPS-driven purge. + log.Infof("Testing LPS-driven purge") + touchApp1("/root/purge_test") + waitLastCmdTimestamp(0, "app1 has no LPS command applied yet") + putLPSAppCommand(t, device, lpsAppUUID, "app1", 123, "COMMAND_PURGE") + waitAppTransientThenRunning(eveinfo.ZSwState_PURGING, eveinfo.ZSwState_HALTING) + waitLastCmdTimestamp(123, "app1 should report the LPS purge command timestamp") + t.Expect(app1FileExists("/root/purge_test")).To(BeFalse(), + "purge should have recreated app1's volume") + evetest.Checkpoint("lps-purge-done") + + // Step 5: controller-driven purge. EVE sums local and remote purge + // counters, but the LPS-visible LastCmdTimestamp must not change. + log.Infof("Testing controller-driven purge") + touchApp1("/root/purge_test") + device.PurgeApplication(app1UUID, true, timeout) + waitApp1SSHReachable() + waitLastCmdTimestamp(123, "controller-driven purge must not change the LPS timestamp") + t.Expect(app1FileExists("/root/purge_test")).To(BeFalse(), + "controller-driven purge should also recreate app1's volume") + evetest.Checkpoint("controller-purge-done") + + // Step 6: LPS-driven restart. Unlike a purge, a restart does not + // recreate the volume. + log.Infof("Testing LPS-driven restart") + touchApp1("/tmp/restart_test") + touchApp1("/root/purge_test") + putLPSAppCommand(t, device, lpsAppUUID, "app1", 456, "COMMAND_RESTART") + waitAppTransientThenRunning(eveinfo.ZSwState_RESTARTING, eveinfo.ZSwState_HALTING) + waitLastCmdTimestamp(456, "app1 should report the LPS restart command timestamp") + t.Expect(app1FileExists("/tmp/restart_test")).To(BeFalse(), + "restart should have wiped the tmpfs /tmp") + t.Expect(app1FileExists("/root/purge_test")).To(BeTrue(), + "restart should not have recreated app1's volume") + evetest.Checkpoint("lps-restart-done") + + // Step 7: controller-driven restart. + log.Infof("Testing controller-driven restart") + touchApp1("/tmp/restart_test") + touchApp1("/root/purge_test") + device.RebootApplication(app1UUID, true, timeout) + waitApp1SSHReachable() + waitLastCmdTimestamp(456, "controller-driven restart must not change the LPS timestamp") + t.Expect(app1FileExists("/tmp/restart_test")).To(BeFalse(), + "controller-driven restart should have wiped the tmpfs /tmp") + t.Expect(app1FileExists("/root/purge_test")).To(BeTrue(), + "controller-driven restart should not have recreated app1's volume") + evetest.Checkpoint("controller-restart-done") + + // Step 8: remove app1 and confirm the LPS stops seeing it. + devConfig.DeleteApplication(app1UUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + + log.Infof("Waiting for the LPS to stop reporting app1") + t.Eventually(func(t Gomega) { + list := getLPSAppInfo(t, device, lpsAppUUID) + t.Expect(appInfoByName(list, "lps-app")).ToNot(BeNil()) + t.Expect(appInfoByName(list, "app1")).To(BeNil()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("appinfo-after-app1-removed") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(lpsAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "lps-app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopLPSAppWatch() +} diff --git a/evetest/tests/lps/dev_local_info_test.go b/evetest/tests/lps/dev_local_info_test.go new file mode 100644 index 00000000000..0fbec1a361e --- /dev/null +++ b/evetest/tests/lps/dev_local_info_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestDevLocalInfo verifies the Local Profile Server (LPS) device-info / +// device-command exchange: EVE reports its own device state to the LPS (GET +// /manage/v1/devinfo), and the LPS can request a graceful shutdown or a +// graceful shutdown-and-poweroff of the whole device (PUT +// /manage/v1/dev-command) independently of the controller. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. Local NI "local-ni" +// hosts both the LPS app and "app1" (port-fwd for direct SSH, used only +// to confirm it is reachable/unreachable at the expected points). +// +// Phases +// ------ +// 1. Deploy the LPS app. WaitUntilAppIsRunning, configure the LPS token, +// and configure EVE to use it via SetLPS. +// 2. Deploy "app1" (plain container, no ProfileList). WaitUntilAppIsRunning, +// wait for SSH, then confirm the LPS's devinfo reports device state +// ONLINE. +// 3. LPS-driven graceful shutdown: PUT a dev-command {timestamp=100, +// COMMAND_SHUTDOWN}. Confirm the LPS-reported devinfo reaches state +// PREPARING_POWEROFF with LastCmdTimestamp=100 (checked while the LPS +// app is still up), then wait for both app1 and lps-app to reach +// HALTED (via their WatchAppInfo channels). +// 4. Recovery: reboot the device via RequestReboot (a controller-driven +// reboot). Wait for both apps to reach RUNNING again, wait for app1's +// SSH to become reachable again, then re-run waitLPSAppReady +// (the LPS app's own in-memory token is wiped by its restart) and confirm +// devinfo reports ONLINE again. +// 5. LPS-driven graceful shutdown-and-poweroff: PUT a dev-command +// {timestamp=200, COMMAND_GRACEFUL_POWEROFF}. Confirm the LPS-reported +// devinfo reaches state POWERING_OFF with LastCmdTimestamp=200 (again +// checked while the LPS app is still up), then wait for both app1 and +// lps-app to reach HALTED. +// 6. Call device.PowerOff() to force the VM fully off (this both confirms +// and, if needed, completes the poweroff EVE initiated in step 5), then +// device.PowerOn(false) to bring it back -- waitUntilOnline=false since +// LastRebootTime isn't reliable after a true hard power-off (see +// PowerOn's doc comment); recovery is confirmed via WaitUntilAppIsRunning +// on both apps below instead. Wait for both apps to reach RUNNING again +// and for app1's SSH to become reachable. +// 7. Cleanup: delete both apps and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestDevLocalInfo(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 5 * time.Minute + polling := 3 * time.Second + + // Step 2: deploy app1. + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2224, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + sshTimeout := 20 * time.Second + log.Infof("Waiting for app1 SSH to become reachable...") + waitApp1SSHReachable := func() { + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + app1UUID, lpsAppAuth, "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitApp1SSHReachable() + + // waitDevState polls the LPS-reported devinfo until it reaches the given + // state, and confirms the LastCmdTimestamp last applied by a dev-command + // (0 if none has been applied yet). Only valid while the LPS app itself + // is still up and reachable. + waitDevState := func(want eveinfo.ZDeviceState, wantLastCmdTimestamp uint64, desc string) { + log.Infof("Waiting for device state %s (%s)", want, desc) + t.Eventually(func(t Gomega) { + info := getLPSDevInfo(t, device, lpsAppUUID) + t.Expect(info.GetState()).To(Equal(want), desc) + t.Expect(info.GetLastCmdTimestamp()).To(Equal(wantLastCmdTimestamp), desc) + }, timeout, polling).Should(Succeed()) + } + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_ONLINE, 0, + "device online after app1 deployed") + evetest.Checkpoint("device-online") + + // waitAppState waits for the given app's WatchAppInfo channel to report + // the given state. Used for HALTED/INVALID waits below (RUNNING waits + // use WaitUntilAppIsRunning directly instead). + waitAppState := func(updates <-chan *eveinfo.ZInfoApp, appName string, want eveinfo.ZSwState) { + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + appName+" reaches "+want.String(), + func(info *eveinfo.ZInfoApp) bool { + return info.State == want + }))) + } + + // Step 3: LPS-driven graceful shutdown. + log.Infof("Testing LPS-driven COMMAND_SHUTDOWN") + putLPSDevCommand(t, device, lpsAppUUID, 100, "COMMAND_SHUTDOWN") + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_PREPARING_POWEROFF, 100, + "device shutting down after COMMAND_SHUTDOWN") + waitAppState(app1Updates, "app1", eveinfo.ZSwState_HALTED) + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_HALTED) + evetest.Checkpoint("shutdown-done") + + // Step 4: recovery via a controller-driven reboot. + log.Infof("Rebooting the device via the controller to recover from the shutdown") + device.RequestReboot(true) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + waitApp1SSHReachable() + + // The LPS app's in-memory token was wiped by its own restart; re-set it + // and re-confirm EVE has reconnected to it. + lpsIP = waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_ONLINE, 100, + "device online again after reboot") + evetest.Checkpoint("shutdown-recovered") + + // Step 5: LPS-driven graceful shutdown-and-poweroff. + log.Infof("Testing LPS-driven COMMAND_GRACEFUL_POWEROFF") + putLPSDevCommand(t, device, lpsAppUUID, 200, "COMMAND_GRACEFUL_POWEROFF") + waitDevState(eveinfo.ZDeviceState_ZDEVICE_STATE_POWERING_OFF, 200, + "device powering off after COMMAND_GRACEFUL_POWEROFF") + waitAppState(app1Updates, "app1", eveinfo.ZSwState_HALTED) + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_HALTED) + evetest.Checkpoint("poweroff-done") + + // Step 6: force the VM off (confirming/completing the poweroff EVE + // initiated above) and power it back on. + log.Infof("Powering the device off (via the broker) and back on") + device.PowerOff() + // waitUntilOnline=false: see PowerOn's doc comment -- LastRebootTime + // isn't reliable after a true hard power-off, so recovery is confirmed + // below via WaitUntilAppIsRunning instead. + device.PowerOn(false) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + waitApp1SSHReachable() + evetest.Checkpoint("poweroff-recovered") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + waitAppState(app1Updates, "app1", eveinfo.ZSwState_INVALID) + stopApp1Watch() + waitAppState(lpsAppUpdates, "lps-app", eveinfo.ZSwState_INVALID) + stopLPSAppWatch() +} diff --git a/evetest/tests/lps/helpers_test.go b/evetest/tests/lps/helpers_test.go new file mode 100644 index 00000000000..ac30572ad0f --- /dev/null +++ b/evetest/tests/lps/helpers_test.go @@ -0,0 +1,241 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "fmt" + "strings" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + "google.golang.org/protobuf/encoding/protojson" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/profile" + "github.com/lf-edge/eve/evetest" +) + +// LPS token used by every LPS test in this package. +const lpsServerToken = "server_token_123" + +// Shared management-API paths, reused by every LPS test in this package. +const ( + lpsLocalBaseURL = "http://localhost:8888" + lpsManageURL = lpsLocalBaseURL + "/manage/v1" + lpsManageTokenURL = lpsManageURL + "/token" + lpsManageProfileURL = lpsManageURL + "/profile" + lpsManageRadioStatusURL = lpsManageURL + "/radio-status" + lpsManageRadioConfigURL = lpsManageURL + "/radio-config" + lpsManageAppInfoURL = lpsManageURL + "/appinfo" + lpsManageAppCommandURL = lpsManageURL + "/app-command" + lpsManageDevInfoURL = lpsManageURL + "/devinfo" + lpsManageDevCommandURL = lpsManageURL + "/dev-command" + lpsManageNetConfigURL = lpsManageURL + "/network-config" +) + +// lpsAppAuth is the fixed SSH credential baked into the evetest-lps image. +var lpsAppAuth = evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", +} + +// newLPSAppConfig returns the ApplicationInstanceConfig for the evetest-lps +// container app, wired to niUUID with SSH (port sshPort) and management-API +// (port 8888) port-forwarding. The caller adds it to a devConfig and is +// responsible for calling ApplyConfig / WaitUntilAppIsRunning. +func newLPSAppConfig(displayName string, niUUID uuid.UUID, sshPort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-lps", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: sshPort, + AppPort: 22, + }, + { + // For developers troubleshooting LPS who need access to the + // UI: pause the test once the app is running, then run + // `evetest eve portfwd 8888:8888` and open + // http://localhost:8888 in a browser. + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 8888, + AppPort: 8888, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + } +} + +// waitLPSAppReady waits for the LPS app to become reachable over SSH, +// configures the LPS server token via its management API, and returns the +// app's IP address (used to build LPSConfig.Address for SetLPS). +func waitLPSAppReady( + t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, token string) string { + sshTimeout := 20 * time.Second + polling := 5 * time.Second + timeout := 3 * time.Minute + log := evetest.Logger() + + log.Infof("Waiting for LPS app SSH to become reachable...") + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("hello")) + }, timeout, polling).Should(Succeed()) + + _, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + fmt.Sprintf(`curl -sS -X PUT -d '{"token":"%s"}' `+lpsManageTokenURL, token), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + "hostname -I | awk '{print $1}'", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + ip := strings.TrimSpace(output) + log.Infof("LPS app IP: %s", ip) + return ip +} + +// runInLPSApp runs a shell command inside the LPS app over SSH and fails the +// assertion (via t) on any transport-level error. +func runInLPSApp(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, script string) string { + output, _, err := device.RunShellScriptInsideApp(appUUID, lpsAppAuth, + script, 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + return output +} + +// getLPSRadioStatus retrieves and parses the radio status that EVE posted +// to the LPS (GET /manage/v1/radio-status). +func getLPSRadioStatus( + t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.RadioStatus { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageRadioStatusURL) + var status profile.RadioStatus + t.Expect(protojson.Unmarshal([]byte(output), &status)).To(Succeed()) + return &status +} + +// getLPSAppInfo retrieves and parses the app info list that EVE posted to +// the LPS (GET /manage/v1/appinfo). +func getLPSAppInfo(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.LocalAppInfoList { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageAppInfoURL) + var list profile.LocalAppInfoList + t.Expect(protojson.Unmarshal([]byte(output), &list)).To(Succeed()) + return &list +} + +// getLPSDevInfo retrieves and parses the device info that EVE posted to the +// LPS (GET /manage/v1/devinfo). +func getLPSDevInfo(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID) *profile.LocalDevInfo { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageDevInfoURL) + var info profile.LocalDevInfo + t.Expect(protojson.Unmarshal([]byte(output), &info)).To(Succeed()) + return &info +} + +// getLPSNetworkInfo retrieves and parses the network info that EVE posted to the LPS. +func getLPSNetworkInfo(t Gomega, device *evetest.EdgeDevice, + appUUID uuid.UUID) *profile.NetworkInfo { + output := runInLPSApp(t, device, appUUID, "curl -sS "+lpsManageURL+"/network") + var netInfo profile.NetworkInfo + t.Expect(protojson.Unmarshal([]byte(output), &netInfo)).To(Succeed()) + return &netInfo +} + +// putLPSProfile sets the local profile the LPS reports back to EVE +// (PUT /manage/v1/profile). +func putLPSProfile(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, profileName string) { + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '{"profile":"%s"}' `+lpsManageProfileURL, profileName)) +} + +// putLPSRadioSilence sets the radio-silence config the LPS reports back to +// EVE (PUT /manage/v1/radio-config). +func putLPSRadioSilence(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, silence bool) { + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '{"radioSilence":%t}' `+lpsManageRadioConfigURL, silence)) +} + +// putLPSAppCommand submits a single-element AppCommand list via +// PUT /manage/v1/app-command (the endpoint always takes a JSON array). +// command must be one of the org.lfedge.eve.profile.AppCommand_Command +// symbolic names (e.g. "COMMAND_PURGE", "COMMAND_RESTART"). +func putLPSAppCommand(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + appDisplayName string, timestamp uint64, command string) { + body := fmt.Sprintf(`[{"displayname":"%s","timestamp":%d,"command":"%s"}]`, + appDisplayName, timestamp, command) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' `+lpsManageAppCommandURL, + body)) +} + +// putLPSDevCommand submits a device command via PUT /manage/v1/dev-command. +// command must be one of the org.lfedge.eve.profile.LocalDevCmd_Command +// symbolic names (e.g. "COMMAND_SHUTDOWN", "COMMAND_GRACEFUL_POWEROFF"). +func putLPSDevCommand(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, + timestamp uint64, command string) { + body := fmt.Sprintf(`{"timestamp":%d,"command":"%s"}`, timestamp, command) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -d '%s' `+lpsManageDevCommandURL, body)) +} + +// putLPSNetworkConfig sets the local network config the LPS reports back to +// EVE (PUT /manage/v1/network-config). portsJSON is the raw JSON array for +// the "ports" field (e.g. "[]" to revert to the controller-supplied config). +func putLPSNetworkConfig(t Gomega, device *evetest.EdgeDevice, appUUID uuid.UUID, portsJSON string) { + body := fmt.Sprintf(`{"ports":%s}`, portsJSON) + runInLPSApp(t, device, appUUID, fmt.Sprintf( + `curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' `+lpsManageNetConfigURL, + body)) +} + +// appInfoByName returns the LocalAppInfo entry matching the given +// displayname, or nil if the LPS has no entry for it yet. +func appInfoByName(list *profile.LocalAppInfoList, name string) *profile.LocalAppInfo { + for _, info := range list.GetAppsInfo() { + if info.GetName() == name { + return info + } + } + return nil +} + +// portStatusByLabel returns the NetworkPortStatus entry matching the given +// logical label. Fails the assertion if no such entry is present in the +// NetworkInfo published by EVE. +func portStatusByLabel(t Gomega, netInfo *profile.NetworkInfo, + label string) *profile.NetworkPortStatus { + for _, ps := range netInfo.PortStatus { + if ps.LogicalLabel == label { + return ps + } + } + t.Expect(netInfo.PortStatus).To(ContainElement( + HaveField("LogicalLabel", label)), + "NetworkInfo.PortStatus should include "+label) + return nil +} diff --git a/evetest/tests/lps/network_test.go b/evetest/tests/lps/network_test.go index 07ea199a8bb..60da64b15e4 100644 --- a/evetest/tests/lps/network_test.go +++ b/evetest/tests/lps/network_test.go @@ -12,22 +12,11 @@ import ( // revive:disable:dot-imports . "github.com/onsi/gomega" - uuid "github.com/satori/go.uuid" - "google.golang.org/protobuf/encoding/protojson" - "github.com/lf-edge/eve-api/go/evecommon" - "github.com/lf-edge/eve-api/go/profile" "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/netmodels" ) -const ( - lpsServerToken = "evetest-lps-token" - lpsLocalBaseURL = "http://localhost:8888" - lpsManageURL = lpsLocalBaseURL + "/manage/v1" - lpsManageNetConfigURL = lpsManageURL + "/network-config" -) - // TestNetworkLocalChanges verifies that EVE's Local Profile Server (LPS) // integration honors the per-port AllowLocalModifications flag when an // app deployed on the device submits network-config overrides via the @@ -54,27 +43,25 @@ const ( // - SystemAdapter for eth1 (DHCP, mgmt+app) with // AllowLocalModifications=true. // - Local NI "local-ni" (10.11.12.0/24, MTU=1500) on eth0. -// - LPS application "lps-app" (lfedge/evetest-lps:1.0) on the NI -// with two port-fwd rules: -// - 2222->22 for the test framework to drive curl-against-LPS commands -// via SSH inside the app, -// - 8888->8888 to let a developer expose the LPS UI through +// - LPS application "lps-app" (lfedge/evetest-lps:1.0, see +// newLPSAppConfig in helpers_test.go) on the NI, with SSH port-fwd +// 2222->22 for the test framework to drive curl-against-LPS commands, +// and 8888->8888 to let a developer expose the LPS UI through // `evetest eve portfwd 8888:8888` while a checkpoint is paused. -// - After the LPS app is reachable over SSH, the test configures the LPS -// server token via the LPS management API, reads the app's IP, and -// pushes evetest.LPSConfig{Address: :8888, AuthToken: token} -// into the device config so EVE actually talks to the LPS. +// - waitLPSAppReady (helpers_test.go) waits for the LPS app to become +// reachable over SSH, configures the LPS server token via the LPS +// management API, and returns the app's IP; the test then pushes +// evetest.LPSConfig{Address: :8888, AuthToken: token} into the +// device config so EVE actually talks to the LPS. // // Phases / assertions // ------------------- // 1. setup-done -> initial-config-applied -> lps-app-is-running: // the LPS container is up. -// 2. lps-app-ssh-reachable: the framework can SSH into the app over the -// port-fwd; a hello probe succeeds. -// 3. lps-configured -> lps-receiving-network-info: EVE picks up the LPS +// 2. lps-configured -> lps-receiving-network-info: EVE picks up the LPS // config and starts posting NetworkInfo (HTTP 200 on // /manage/v1/network). -// 4. Submit a localNetworkConfig via the LPS management API that +// 3. Submit a localNetworkConfig via the LPS management API that // overrides DNS for eth0 (dns-server0-alt, 10.16.18.25) and MTU for // eth1 (9000). Assert via `Eventually` (configChangeTimeout): // - NetworkInfo.LocalConfig.Ports has entries for both adapters. @@ -87,13 +74,13 @@ const ( // - Runtime PortStatus for eth1: Mtu=9000. // - On EVE itself: /run/nim/dnsmasq.mgmt.servers does NOT contain // 10.16.18.25, /sys/class/net/eth1/mtu == "9000". -// 5. Enable AllowLocalModifications=true on eth0 via UpdateNetworkAdapter +// 4. Enable AllowLocalModifications=true on eth0 via UpdateNetworkAdapter // and re-ApplyConfig. Assert: // - LocalConfig.Ports[eth0]: no "not permitted" error, // ConfigApplied=true. // - PortStatus for eth0: DNS now includes 10.16.18.25. // - On EVE: /run/nim/dnsmasq.mgmt.servers now contains 10.16.18.25. -// 6. Push an empty config via the LPS management API +// 5. Push an empty config via the LPS management API // ({"serverToken":..., "ports":[]}). Assert that both ports revert // to the controller-supplied config: // - LatestConfig.ConfigApplied=true for both ports; eth1.Mtu is @@ -103,31 +90,25 @@ const ( // - On EVE: dnsmasq.mgmt.servers no longer contains 10.16.18.25; // /sys/class/net/eth1/mtu == "1500". // -// Helpers used -// ------------ -// - getLPSNetworkInfo (defined below): curls /manage/v1/network from -// inside the LPS app and unmarshals the protobuf-json into a -// profile.NetworkInfo. -// - portStatusByLabel (defined below): walks NetworkInfo.PortStatus -// looking up a port by its LogicalLabel, with a failing assertion -// if not found. -// // Hypervisor / suite placement // ---------------------------- -// - Hardcoded HypervisorKVM. The TODO in TestLPSSuite notes that the -// HypervisorParameter will be added once additional LPS tests exist -// that depend on app virtualization. +// - HYPERVISOR (defaults to KVM). func TestNetworkLocalChanges(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -173,84 +154,25 @@ func TestNetworkLocalChanges(test *testing.T) { Gateway: evetest.IPAddress("10.11.12.1"), MTU: 1500, }) - appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ - DisplayName: "lps-app", - Activate: true, - Image: evetest.DockerContainer{ - ImageName: "lfedge/evetest-lps", - Tag: "1.0", - }, - CPUs: 1, - MemoryBytes: 512 * evetest.MiB, - NetworkAdapters: []evetest.AppNetworkAdapter{ - evetest.VirtualNetworkAdapter{ - LogicalLabel: "vif0", - NetworkInstanceUUID: niUUID, - PortFwdRules: []evetest.PortFwdRule{ - { - // SSH access - Protocol: evetest.NetworkProtocolTCP, - EdgeNodePort: 2222, - AppPort: 22, - }, - { - // For developers troubleshooting LPS who need access to the UI: - // Pause test after LPS is deployed (at checkpoint - // "lps-app-is-running" or later), then run: - // $ evetest eve portfwd 8888:8888 - // And open http://localhost:8888 in your browser. - Protocol: evetest.NetworkProtocolTCP, - EdgeNodePort: 8888, - AppPort: 8888, - }, - }, - ACLAllowRules: []evetest.ACLAllowRule{ - { - Protocol: evetest.NetworkProtocolAny, - RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), - }, - }, - }, - }, - }) + appUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) device := evetest.GetEdgeDevice(devName) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("initial-config-applied") device.WaitUntilAppIsRunning(appUUID, 10*time.Minute) evetest.Checkpoint("lps-app-is-running") - // Wait for the LPS app to become reachable via SSH. - appAuth := evetest.UsernamePasswordAuth{ - Username: "root", - Password: "testpassword", - } log := evetest.Logger() sshTimeout := 20 * time.Second polling := 5 * time.Second - timeout := 3 * time.Minute - log.Infof("Waiting for LPS app SSH to become reachable...") - t.Eventually(func(t Gomega) { - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "echo hello", sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - t.Expect(output).To(ContainSubstring("hello")) - }, timeout, polling).Should(Succeed()) - evetest.Checkpoint("lps-app-ssh-reachable") - - // Configure the server token via the LPS management API. - _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -d '{"token":"%s"}' `+lpsManageURL+`/token`, - lpsServerToken), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + var output string + var err error - // Get the application's IP (LPS is reachable at this IP from EVE). - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "hostname -I | awk '{print $1}'", sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - lpsIP := strings.TrimSpace(output) - log.Infof("LPS app IP: %s", lpsIP) + lpsIP := waitLPSAppReady(t, device, appUUID, lpsServerToken) // Configure EVE to use the LPS. devConfig.SetLPS(evetest.LPSConfig{ @@ -264,35 +186,26 @@ func TestNetworkLocalChanges(test *testing.T) { configChangeTimeout := 2 * time.Minute log.Infof("Waiting for LPS to receive network info from EVE...") t.Eventually(func(t Gomega) { - output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, - "curl -sS -o /dev/null -w '%{http_code}' "+lpsManageURL+"/network", - sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + output := runInLPSApp(t, device, appUUID, + "curl -sS -o /dev/null -w '%{http_code}' "+lpsManageURL+"/network") t.Expect(output).To(Equal("200")) }, configChangeTimeout, polling).Should(Succeed()) evetest.Checkpoint("lps-receiving-network-info") // Apply local config: DNS override for eth0, MTU override for eth1. - localNetworkConfig := fmt.Sprintf(`{ - "serverToken": "%s", - "ports": [ - { - "logicalLabel": "ethernet0", - "useDhcp": true, - "dnsServers": ["10.16.18.25"] - }, - { - "logicalLabel": "ethernet1", - "useDhcp": true, - "mtu": 9000 - } - ] - }`, lpsServerToken) log.Infof("Submitting local network config via LPS management API") - _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' %s`, - localNetworkConfig, lpsManageNetConfigURL), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + putLPSNetworkConfig(t, device, appUUID, `[ + { + "logicalLabel": "ethernet0", + "useDhcp": true, + "dnsServers": ["10.16.18.25"] + }, + { + "logicalLabel": "ethernet1", + "useDhcp": true, + "mtu": 9000 + } + ]`) evetest.Checkpoint("local-config-submitted") // Verify eth0 changes are rejected, eth1 changes are applied. @@ -300,7 +213,7 @@ func TestNetworkLocalChanges(test *testing.T) { // for eth1 was applied (MTU=9000) and eth0 was rejected (not permitted). log.Infof("Verifying eth1 local changes are applied and eth0 is rejected...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) t.Expect(netInfo.LocalConfig).ToNot(BeNil()) for _, port := range netInfo.LocalConfig.Ports { switch port.LogicalLabel { @@ -367,7 +280,7 @@ func TestNetworkLocalChanges(test *testing.T) { // Verify eth0 changes are now applied log.Infof("Verifying eth0 local changes are now applied...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) t.Expect(netInfo.LocalConfig).ToNot(BeNil()) for _, port := range netInfo.LocalConfig.Ports { if port.LogicalLabel == "ethernet0" { @@ -395,20 +308,13 @@ func TestNetworkLocalChanges(test *testing.T) { // Revert local changes by submitting empty config log.Infof("Reverting local network config by submitting empty config...") - emptyConfig := fmt.Sprintf(`{ - "serverToken": "%s", - "ports": [] - }`, lpsServerToken) - _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, - fmt.Sprintf(`curl -sS -X PUT -H 'Content-Type: application/json' -d '%s' %s`, - emptyConfig, lpsManageNetConfigURL), sshTimeout, 0) - t.Expect(err).ToNot(HaveOccurred()) + putLPSNetworkConfig(t, device, appUUID, "[]") evetest.Checkpoint("local-changes-reverted") // Verify both ports revert to controller config log.Infof("Verifying both ports reverted to controller config...") t.Eventually(func(t Gomega) { - netInfo := getLPSNetworkInfo(t, device, appUUID, appAuth, sshTimeout) + netInfo := getLPSNetworkInfo(t, device, appUUID) // After submitting empty config, LocalConfig should have no ports // or all ports should show controller config applied. for _, port := range netInfo.LatestConfig { @@ -447,32 +353,3 @@ func TestNetworkLocalChanges(test *testing.T) { "eth1 MTU should have reverted to 1500") }, configChangeTimeout, polling).Should(Succeed()) } - -// portStatusByLabel returns the NetworkPortStatus entry matching the given -// logical label. Fails the assertion if no such entry is present in the -// NetworkInfo published by EVE. -func portStatusByLabel(t Gomega, netInfo *profile.NetworkInfo, - label string) *profile.NetworkPortStatus { - for _, ps := range netInfo.PortStatus { - if ps.LogicalLabel == label { - return ps - } - } - t.Expect(netInfo.PortStatus).To(ContainElement( - HaveField("LogicalLabel", label)), - "NetworkInfo.PortStatus should include "+label) - return nil -} - -// getLPSNetworkInfo retrieves and parses the network info that EVE posted to the LPS. -func getLPSNetworkInfo(t Gomega, device *evetest.EdgeDevice, - appUUID uuid.UUID, auth evetest.AuthMethod, - timeout time.Duration) *profile.NetworkInfo { - output, _, err := device.RunShellScriptInsideApp(appUUID, auth, - "curl -sS "+lpsManageURL+"/network", timeout, 0) - t.Expect(err).ToNot(HaveOccurred()) - var netInfo profile.NetworkInfo - err = protojson.Unmarshal([]byte(output), &netInfo) - t.Expect(err).ToNot(HaveOccurred()) - return &netInfo -} diff --git a/evetest/tests/lps/profile_test.go b/evetest/tests/lps/profile_test.go new file mode 100644 index 00000000000..9562a81150b --- /dev/null +++ b/evetest/tests/lps/profile_test.go @@ -0,0 +1,277 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestProfile verifies that EVE filters which applications run based on the +// currently active profile -- both a device-config-only GlobalProfile +// override and a Local Profile Server (LPS)-reported local profile -- by +// matching it against each app's ProfileList (an app with an empty +// ProfileList always runs, regardless of the active profile). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port, used only for +// controller reachability and the LPS app's Local NI. The three +// ProfileList apps under test have no network adapter of their own +// (only their RUNNING/HALTED state matters here). +// +// Phases +// ------ +// 1. Deploy four apps on Local NI "local-ni": "lps-app" (the evetest-lps +// app, ProfileList empty --> always runs), "app-profile-1" +// (ProfileList=[profile-1]), "app-profile-2" (ProfileList=[profile-2]), +// "app-profile-1-2" (ProfileList=[profile-1, profile-2]). +// WaitUntilAppIsRunning for all four -- with no profile active yet +// (GlobalProfile and LocalProfileServer both empty), every app runs. +// 2. GlobalProfile-only phase (no LPS involved yet): SetLPS with only +// GlobalProfile set, re-applying for "profile-1", "profile-2" and +// "profile-3" in turn. After each apply, assert exactly the apps whose +// ProfileList does not contain the active profile are HALTED and the +// rest (including lps-app, always) are RUNNING. +// 3. Manual activate/deactivate sanity check: with GlobalProfile still +// "profile-3" (so app-profile-* are all HALTED), DeactivateApplication +// then ActivateApplication lps-app (whose empty ProfileList makes it +// immune to profile filtering) to confirm ordinary controller-driven +// activation still works independently of the profile mechanism. +// 4. LPS-driven phase: configure the LPS address/token via SetLPS +// (GlobalProfile is deliberately left at "profile-3" -- this also +// proves that once a LocalProfileServer is configured, EVE is driven +// entirely by whatever profile the LPS reports, not by GlobalProfile). +// Submit "profile-1", "profile-2", "profile-3" via PUT /manage/v1/profile in +// turn, asserting the same HALTED/RUNNING pattern as phase 2 -- but +// this time driven by the LPS instead of the static device config. +// 5. Revert: SetLPS with all fields empty (clears GlobalProfile, +// LocalProfileServer and ProfileServerToken). Assert all four apps +// return to RUNNING. +// 6. Cleanup: delete all four apps and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestProfile(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + + // Step 1: deploy the LPS app plus three plain apps distinguished only + // by their ProfileList. + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + app1UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-1", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-1"}, + }) + app2UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-2", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-2"}, + }) + app12UUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "app-profile-1-2", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + ProfileList: []string{"profile-1", "profile-2"}, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(lpsAppUUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app12UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running-no-profile") + + log := evetest.Logger() + timeout := 10 * time.Minute + polling := 3 * time.Second + + // waitAppState polls EdgeDevice.GetAppInfo -- a synchronous snapshot of + // the app's latest known state -- until it reports the expected SwState. + // Used for HALTED/INVALID waits below (WaitUntilAppIsRunning only + // targets RUNNING); a WatchAppInfo channel wouldn't work here either: + // several apps have overlapping ProfileLists (e.g. app-profile-1-2 + // matches both "profile-1" and "profile-2"), so switching between two + // profiles it both matches causes no state transition and thus no new + // channel event at all. + waitAppState := func(appUUID uuid.UUID, desc string, expected eveinfo.ZSwState) { + log.Infof("Waiting for: %s...", desc) + t.Eventually(func() eveinfo.ZSwState { + if info := device.GetAppInfo(appUUID); info != nil { + return info.GetState() + } + return eveinfo.ZSwState_INVALID + }, timeout, polling).Should(Equal(expected), desc) + } + + // Step 2: GlobalProfile-only phase (no LPS configured yet). + log.Infof("Setting GlobalProfile=profile-1") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-1"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app2UUID, "app-profile-2 is HALTED (profile-1 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("global-profile-1") + + log.Infof("Setting GlobalProfile=profile-2") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-2"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app1UUID, "app-profile-1 is HALTED (profile-2 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("global-profile-2") + + log.Infof("Setting GlobalProfile=profile-3") + devConfig.SetLPS(evetest.LPSConfig{GlobalProfile: "profile-3"}) + device.ApplyConfig(devConfig, false, false) + waitAppState(app1UUID, "app-profile-1 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + waitAppState(app2UUID, "app-profile-2 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + waitAppState(app12UUID, "app-profile-1-2 is HALTED (profile-3 active)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + evetest.Checkpoint("global-profile-3") + + // Step 3: manual activate/deactivate still works independently of the + // profile mechanism (lps-app has an empty ProfileList, so it is immune + // to the currently-active "profile-3"). + log.Infof("Manually deactivating and reactivating lps-app") + device.DeactivateApplication(lpsAppUUID, true, timeout) + device.ActivateApplication(lpsAppUUID, true, timeout) + evetest.Checkpoint("lps-app-manually-cycled") + + // Step 4: LPS-driven phase. GlobalProfile is deliberately left at + // "profile-3" -- configuring a LocalProfileServer must make EVE defer + // entirely to whatever profile the LPS reports, ignoring GlobalProfile. + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + GlobalProfile: "profile-3", + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log.Infof("Submitting local profile 'profile-1' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-1") + waitAppState(app2UUID, "app-profile-2 is HALTED (LPS profile-1)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("lps-profile-1") + + log.Infof("Submitting local profile 'profile-2' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-2") + waitAppState(app1UUID, "app-profile-1 is HALTED (LPS profile-2)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("lps-profile-2") + + log.Infof("Submitting local profile 'profile-3' via LPS") + putLPSProfile(t, device, lpsAppUUID, "profile-3") + waitAppState(app1UUID, "app-profile-1 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + waitAppState(app2UUID, "app-profile-2 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + waitAppState(app12UUID, "app-profile-1-2 is HALTED (LPS profile-3)", eveinfo.ZSwState_HALTED) + device.WaitUntilAppIsRunning(lpsAppUUID, timeout) + evetest.Checkpoint("lps-profile-3") + + // Step 5: revert to empty profiles -- all apps come back RUNNING. + log.Infof("Reverting to empty GlobalProfile/LocalProfileServer") + devConfig.SetLPS(evetest.LPSConfig{}) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(app1UUID, timeout) + device.WaitUntilAppIsRunning(app2UUID, timeout) + device.WaitUntilAppIsRunning(app12UUID, timeout) + evetest.Checkpoint("profiles-cleared") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteApplication(app12UUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + waitAppState(lpsAppUUID, "lps-app is gone", eveinfo.ZSwState_INVALID) + waitAppState(app1UUID, "app-profile-1 is gone", eveinfo.ZSwState_INVALID) + waitAppState(app2UUID, "app-profile-2 is gone", eveinfo.ZSwState_INVALID) + waitAppState(app12UUID, "app-profile-1-2 is gone", eveinfo.ZSwState_INVALID) +} diff --git a/evetest/tests/lps/radio_silence_test.go b/evetest/tests/lps/radio_silence_test.go new file mode 100644 index 00000000000..31108af88ed --- /dev/null +++ b/evetest/tests/lps/radio_silence_test.go @@ -0,0 +1,193 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package lps_test + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestRadioSilence verifies EVE's radio-silence message exchange with the +// Local Profile Server (LPS): the LPS can request radio silence be imposed +// or lifted, EVE reports the resulting state back to both the LPS and its +// own internal ZedAgentStatus, and the imposed state survives a device +// reboot. +// +// Note: none of evetest's device models currently define a wireless (cellular/WiFi) +// network adapter, so this only exercises the message-passing and persistence +// paths between the LPS, zedagent and nim -- not an actual radio transmitter being +// switched off. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port, used for controller +// reachability and the LPS app's Local NI. +// +// Phases +// ------ +// 1. Deploy the evetest-lps app on Local NI "local-ni". WaitUntilAppIsRunning, +// then configure EVE to use it via SetLPS. +// 2. Confirm the initial radio status is unsilenced, per both the LPS +// (GET /manage/v1/radio-status) and EVE's own ZedAgentStatus. +// 3. Toggle radio silence ON (PUT /manage/v1/radio-config +// {"radioSilence":true}). Assert both: +// - the LPS's radio-status eventually reports RadioSilence=true, and +// - EVE's own /run/zedagent/ZedAgentStatus/zedagent.json (read via +// `eve exec pillar jq ...` over SSH) reports RadioSilence.Imposed=true. +// 4. Toggle radio silence OFF and assert both views report false again. +// 5. Toggle radio silence back ON (assert both views report true again), +// then reboot the device via RequestReboot and wait for it to come back online. +// 6. After the reboot, assert -- reading EVE's own ZedAgentStatus only, +// deliberately without any further LPS interaction -- that +// RadioSilence.Imposed is still true. zedagent persists the +// last-applied radio-silence state across reboots and re-publishes it +// on startup. +// 7. Cleanup: delete the app and the NI, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestLPSSuite. +func TestRadioSilence(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + lpsAppUUID := devConfig.AddApplication(newLPSAppConfig("lps-app", niUUID, 2222)) + lpsAppUpdates, stopLPSAppWatch := device.WatchAppInfo(lpsAppUUID) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + device.WaitUntilAppIsRunning(lpsAppUUID, 5*time.Minute) + evetest.Checkpoint("lps-app-running") + + lpsIP := waitLPSAppReady(t, device, lpsAppUUID, lpsServerToken) + devConfig.SetLPS(evetest.LPSConfig{ + Address: lpsIP + ":8888", + AuthToken: lpsServerToken, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("lps-configured") + + log := evetest.Logger() + timeout := 3 * time.Minute + polling := 3 * time.Second + + // waitRadioSilence asserts that both the LPS-reported radio status and + // EVE's own ZedAgentStatus agree on the given RadioSilence value. + waitRadioSilence := func(want bool, desc string) { + log.Infof("Waiting for radio-silence=%t (%s)", want, desc) + t.Eventually(func(t Gomega) { + status := getLPSRadioStatus(t, device, lpsAppUUID) + t.Expect(status.RadioSilence).To(Equal(want), "LPS-reported radio status") + t.Expect(getEVERadioSilenceImposed(t, device)).To(Equal(want), + "EVE-side ZedAgentStatus.RadioSilence.Imposed") + }, timeout, polling).Should(Succeed()) + } + + // Step 2: confirm the initial (unsilenced) radio status. + waitRadioSilence(false, "initial state") + evetest.Checkpoint("initial-radio-status-confirmed") + + // Step 3: toggle radio silence ON. + putLPSRadioSilence(t, device, lpsAppUUID, true) + waitRadioSilence(true, "toggled ON") + evetest.Checkpoint("radio-silence-on") + + // Step 4: toggle radio silence OFF. + putLPSRadioSilence(t, device, lpsAppUUID, false) + waitRadioSilence(false, "toggled OFF") + evetest.Checkpoint("radio-silence-off") + + // Step 5: toggle back ON, then reboot. + putLPSRadioSilence(t, device, lpsAppUUID, true) + waitRadioSilence(true, "toggled ON again, pre-reboot") + evetest.Checkpoint("radio-silence-on-pre-reboot") + + log.Infof("Rebooting the device via the controller") + device.RequestReboot(true) + evetest.Checkpoint("device-rebooted") + + // Step 6: persistence -- EVE's own state only, no LPS interaction. + log.Infof("Confirming radio-silence=true persisted across the reboot " + + "(EVE-side check only)") + t.Eventually(func(t Gomega) { + t.Expect(getEVERadioSilenceImposed(t, device)).To(BeTrue(), + "EVE should still report RadioSilence.Imposed=true after reboot") + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("radio-silence-persisted") + + // Cleanup. + devConfig.DeleteApplication(lpsAppUUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(lpsAppUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "lps-app is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopLPSAppWatch() +} + +// getEVERadioSilenceImposed reads RadioSilence.Imposed directly from EVE's +// own /run/zedagent/ZedAgentStatus/zedagent.json, bypassing the LPS +// entirely. Used to confirm EVE's internal state independent of whether the +// LPS app has (re)connected. +func getEVERadioSilenceImposed(t Gomega, device *evetest.EdgeDevice) bool { + output, _, err := device.RunShellScript( + `eve exec pillar jq -r '.RadioSilence.Imposed' /run/zedagent/ZedAgentStatus/zedagent.json`, + 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + return strings.TrimSpace(output) == "true" +} diff --git a/evetest/tests/lps/testsuite_test.go b/evetest/tests/lps/testsuite_test.go index 282a09f1a0c..28e5c19c8a8 100644 --- a/evetest/tests/lps/testsuite_test.go +++ b/evetest/tests/lps/testsuite_test.go @@ -10,21 +10,44 @@ import ( ) // TestLPSSuite is the entry point for Local Profile Server (LPS) tests. -// Currently it contains only TestNetworkLocalChanges. The TODO above -// indicates we will enable HypervisorParameter once the suite grows to -// include app-related LPS scenarios that depend on hypervisor choice; -// for now the single subtest hardcodes its own hypervisor. +// See https://github.com/lf-edge/eve-api/blob/main/PROFILE.md +// +// Subtests +// -------- +// - TestProfile -- device-level (GlobalProfile) and LPS-driven +// (local profile) app filtering by ProfileList. +// - TestRadioSilence -- LPS-driven radio-silence toggling, verified via +// both the LPS and EVE's own ZedAgentStatus, including persistence of +// the imposed state across a controller-driven reboot. +// - TestAppLocalInfo -- LPS appinfo reporting and LPS-driven +// COMMAND_PURGE/COMMAND_RESTART, alongside the equivalent +// controller-driven operations (whose counters EVE sums with the +// LPS-driven ones). +// - TestDevLocalInfo -- LPS devinfo reporting and LPS-driven +// COMMAND_SHUTDOWN/COMMAND_GRACEFUL_POWEROFF device commands. +// - TestNetworkLocalChanges -- per-port AllowLocalModifications / +// LPS network-config override behavior. func TestLPSSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() - /* TODO: re-enable if there are any app-related LPS tests evetest.DefineTestParameters( evetest.HypervisorParameter(), ) - */ evetest.RunTestSuite( + evetest.TestCase{ + Test: TestProfile, + }, + evetest.TestCase{ + Test: TestRadioSilence, + }, + evetest.TestCase{ + Test: TestAppLocalInfo, + }, + evetest.TestCase{ + Test: TestDevLocalInfo, + }, evetest.TestCase{ Test: TestNetworkLocalChanges, }, diff --git a/evetest/tests/networking/acls_test.go b/evetest/tests/networking/acls_test.go index 43f75878098..90db2e4c994 100644 --- a/evetest/tests/networking/acls_test.go +++ b/evetest/tests/networking/acls_test.go @@ -75,8 +75,7 @@ const enableFlowlogParamKey = "ENABLE_FLOWLOG" // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() -- Kubevirt -// is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // - ENABLE_FLOWLOG: enable flow logging on the Local NI (default: false). func TestLocalNetInstanceACLs(test *testing.T) { evetestT := evetest.Init(test) @@ -96,7 +95,6 @@ func TestLocalNetInstanceACLs(test *testing.T) { ) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() enableFlowlog := evetest.GetTestParameter[bool](enableFlowlogParamKey) devName := "edge-dev" @@ -125,6 +123,9 @@ func TestLocalNetInstanceACLs(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } const ( niGateway = "10.11.12.1" @@ -490,8 +491,7 @@ func TestLocalNetInstanceACLs(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() -- Kubevirt is -// reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // - ENABLE_FLOWLOG: enable flow logging on the Switch NI (default: false). func TestSwitchNetInstanceACLs(test *testing.T) { evetestT := evetest.Init(test) @@ -511,7 +511,6 @@ func TestSwitchNetInstanceACLs(test *testing.T) { ) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() enableFlowlog := evetest.GetTestParameter[bool](enableFlowlogParamKey) devName := "edge-dev" @@ -540,6 +539,9 @@ func TestSwitchNetInstanceACLs(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } const ( app1MACAddr = "02:16:3e:00:00:03" diff --git a/evetest/tests/networking/bonds_test.go b/evetest/tests/networking/bonds_test.go index f87785b92a4..21d837c4fb2 100644 --- a/evetest/tests/networking/bonds_test.go +++ b/evetest/tests/networking/bonds_test.go @@ -70,20 +70,23 @@ import ( // failed member. // - Restore the SDN model (link back up). // -// Hypervisor -// ---------- -// - Hardcoded WithHypervisor=HypervisorKVM in RequireEdgeDevice -- this -// test lives in TestDeviceConnectivitySuite and does not parameterize -// the hypervisor. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestActiveBackupBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } // Active-backup bond is transparent to the network switch -- only one @@ -143,6 +146,9 @@ func TestActiveBackupBond(test *testing.T) { devMetrics, stopDevMetricsWatch := device.WatchDeviceMetrics() defer stopDevMetricsWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Wait for device info to report the bond interface with an IP address, @@ -339,20 +345,23 @@ func TestActiveBackupBond(test *testing.T) { // ethernet0 and ethernet1 in its Members list, and every member has // LACP sub-metrics populated. // -// Hypervisor -// ---------- -// - Hardcoded WithHypervisor=HypervisorKVM in RequireEdgeDevice -- this -// test lives in TestDeviceConnectivitySuite and does not parameterize -// the hypervisor. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestLACPBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } // Start with individual ports on a bridge so that EVE can onboard @@ -413,6 +422,9 @@ func TestLACPBond(test *testing.T) { // waitUntilConfirmed=false: after the model switch below, EVE may temporarily // lose controller connectivity while the LACP bond negotiates. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to process the bond config before switching the SDN side. time.Sleep(10 * time.Second) evetest.Checkpoint("config-applied") diff --git a/evetest/tests/networking/bootstrap_test.go b/evetest/tests/networking/bootstrap_test.go index 2bde79fe4bf..a72347478a3 100644 --- a/evetest/tests/networking/bootstrap_test.go +++ b/evetest/tests/networking/bootstrap_test.go @@ -49,14 +49,15 @@ var ( ) func deviceRequirementsForBootstrap( - devName string, useInstaller bool) evetest.RequireEdgeDevice { + devName string, useInstaller bool, + hypervisor evetest.Hypervisor) evetest.RequireEdgeDevice { reusePolicy := evetest.CreateFromScratchWithLiveImage if useInstaller { reusePolicy = evetest.CreateFromScratchWithInstaller } return evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, MinCPUs: 4, WithGrubOptions: []string{ // No applications are deployed in network bootstrapping tests. @@ -119,7 +120,7 @@ func deviceRequirementsForBootstrap( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test; not parameterized). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithLastResort(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -127,17 +128,19 @@ func TestBootstrapWithLastResort(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), lastResortParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() lastResortExplicitlyEnabled := evetest.GetTestParameter[bool](lastResortParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCP, } @@ -171,6 +174,9 @@ func TestBootstrapWithLastResort(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Wait for device info to report the expected DevicePortStatus list. @@ -251,7 +257,7 @@ var ( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test; not parameterized). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithStaticIP(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -259,11 +265,13 @@ func TestBootstrapWithStaticIP(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -290,7 +298,7 @@ func TestBootstrapWithStaticIP(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -328,6 +336,9 @@ func TestBootstrapWithStaticIP(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -460,7 +471,7 @@ var ( // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithProxy(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -468,12 +479,14 @@ func TestBootstrapWithProxy(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, proxyConfigTypeParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) proxyConfigType := evetest.GetTestParameter[ProxyConfigType](proxyConfigTypeParamKey) @@ -526,7 +539,7 @@ func TestBootstrapWithProxy(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { var proxyConfig pillartypes.ProxyConfig switch proxyConfigType { @@ -614,6 +627,9 @@ func TestBootstrapWithProxy(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -672,7 +688,7 @@ func TestBootstrapWithProxy(test *testing.T) { // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithMgmtVLAN(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -680,11 +696,13 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -712,7 +730,7 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -761,6 +779,9 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after @@ -818,7 +839,7 @@ func TestBootstrapWithMgmtVLAN(test *testing.T) { // // Hypervisor // ---------- -// - Hardcoded HypervisorKVM (Bootstrap-suite test). +// - HYPERVISOR (defaults to KVM). func TestBootstrapWithLACPBond(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -826,11 +847,13 @@ func TestBootstrapWithLACPBond(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), useOverrideJSONParam, useInstallerParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() useOverrideJSON := evetest.GetTestParameter[bool](useOverrideJSONParamKey) useInstaller := evetest.GetTestParameter[bool](useInstallerParamKey) @@ -874,7 +897,7 @@ func TestBootstrapWithLACPBond(test *testing.T) { }) // Set up the test harness and specify test prerequisites. - requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller) + requiredDevice := deviceRequirementsForBootstrap(devName, useInstaller, hypervisor) if useOverrideJSON { requiredDevice.WithInjectedNetworkOverride = &pillartypes.DevicePortConfig{ Version: 1, @@ -940,6 +963,9 @@ func TestBootstrapWithLACPBond(test *testing.T) { // Apply the same bootstrap configuration also through the controller. device.ApplyConfig(bootstrapConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // Neither bootstrap config nor override.json remain persisted after diff --git a/evetest/tests/networking/datastore_test.go b/evetest/tests/networking/datastore_test.go index eb41d9e4ce3..db3c19ad152 100644 --- a/evetest/tests/networking/datastore_test.go +++ b/evetest/tests/networking/datastore_test.go @@ -1,129 +1,347 @@ // Copyright (c) 2026 Zededa, Inc. // SPDX-License-Identifier: Apache-2.0 -package networking_test - -import "testing" - -// Datastore tests verify that EVE can pull application images from the various -// datastore backends supported by the EVE API (HTTP, HTTPS, AWS S3, SFTP, Azure -// Blob, container registries). They focus on the network/datastore plumbing -// (correct datastore URL construction, authentication, certificate handling, -// download progress reporting, error propagation), not on the application -// runtime — once the image is downloaded and verified, the test can stop. +// Datastore tests verify that EVE can pull volume content from the various +// datastore backends supported by the EVE API (HTTP, HTTPS, AWS S3, SFTP, +// Azure Blob, container registries). They focus on the network/datastore +// plumbing (correct datastore URL construction, authentication, certificate +// handling, download progress reporting, error propagation) -- not on any +// application runtime. +// +// No application is needed. volumemgr fully creates a volume -- including +// the ContentTree download, verification, and (for archive formats) format +// conversion. So these tests are simpler than they might first appear: declare +// a standalone volume via EdgeDeviceConfig.AddVolume with the datastore under +// test as its Image, and watch it reach ZSwState_CREATED_VOLUME (or ZSwState_ERROR +// for negative variants) via device.WatchVolumeInfo. No NI, no app, no +// WaitUntilAppIsRunning. // // Reusable scenario shape // ----------------------- // // All these tests follow the same structure: // -// 1. Setup a single-port mgmt device (netmodels.SingleEthWithDHCP). Internet -// connectivity is required only for tests that talk to a real cloud -// datastore (AWS/Azure); HTTP/HTTPS/SFTP can be fully self-contained inside -// SDN. -// 2. Build a device config with: -// - one DHCP network on eth0 (mgmt+app), -// - a Local NI for the test application, -// - one application referencing a small image stored in the datastore -// under test. -// 3. Drive the test by `device.WatchContentTreeInfo(ctUUID)` and -// `device.WatchVolumeInfo(volUUID)`: -// - Assert that the content tree progresses through DOWNLOAD_STARTED -> -// DOWNLOADED -> VERIFIED -> LOADED, with download progress strictly -// monotonic (this also catches stalled downloads from a misbehaving -// datastore endpoint). -// - Assert that the resulting volume reaches CREATED_VOLUME. -// - Assert that the application reaches RUNNING (use WaitUntilAppIsRunning -// — it already handles download stalls and excludes download time from -// the timeout budget; see edgedevice.go). -// 4. As a sanity check on networking, run a short script inside the app -// (RunShellScriptInsideApp) printing hostname/IP, but the primary -// assertions are about download/verification, not application semantics. -// 5. Negative-path variants where appropriate (see per-test sections): -// - wrong SHA256 -> content tree should reach ERROR with descriptive -// err description. -// - bad credentials -> ditto. -// - server cert not trusted (HTTPS) -> ditto. -// -// Why we still want a deployed app rather than just a content tree + datastore: -// downloading and verifying alone is implemented in volumemgr/downloader, but -// EVE only triggers the download when there is a concrete consumer. The -// simplest way to guarantee that is to declare an application that volume-refs -// the content tree. The app does not need to do anything useful — a tiny -// container image is sufficient. Once support for "datastore + content-tree -// without app" is confirmed possible (volumemgr will pre-download referenced -// content trees), these tests can be simplified to skip the app deployment. +// 1. Setup a single-port mgmt device (netmodels.SingleEthWithDHCP). Add +// RequireInternetConnectivity{} only for tests that talk to a real +// external datastore (AWS/Azure); HTTP/HTTPS/SFTP/container-registry +// against evetest's own built-in servers (see "Test images" below) need +// no Internet access at all. +// 2. Build a device config with just the one DHCP network on eth0 +// (mgmt+app) -- no NI, no application. +// 3. devConfig.AddVolume(displayName, image, sizeBytes) with image set to +// the datastore under test (HTTPStorage/SFTPStorage/DockerContainer/ +// AwsS3Bucket/AzureBlob -- see devconfig.go's ApplicationImageStorage +// implementations). Watch it via device.WatchVolumeInfo(volUUID): +// - Happy path: assert State reaches ZSwState_CREATED_VOLUME. +// ZInfoVolume.ProgressPercentage can be asserted strictly monotonic +// along the way if the test wants to catch a stalled download. +// - Negative-path variants (wrong checksum, bad credentials, server +// returns an error, untrusted cert, ...): assert State reaches +// ZSwState_ERROR (or stays non-CREATED_VOLUME while VolumeErr becomes +// non-empty) with a VolumeErr description matching what's expected +// (matchers.SatisfyPredicate against info.GetVolumeErr().GetDescription()). +// 4. Cleanup: devConfig.DeleteVolume(volUUID); no application, no NI, to +// tear down. // // Test images // ----------- // -// - For HTTP/HTTPS/SFTP: prefer pushing/serving a tiny, fixed Linux image (a -// few-MB Alpine qcow2 or a hand-crafted busybox container tarball) from -// within the SDN environment. This keeps the tests hermetic and fast. -// This requires extending the SDN/HTTPServer endpoint to serve binary -// content (currently it only serves the small "Paths" map) and adding a -// simple SFTPServer endpoint type to evetest's grpcapi/sdn.proto. Both -// enhancements are scoped to evetest and do NOT touch EVE itself. -// Until that exists, point HTTP-only tests at a public, very small, -// versioned image and accept the external dependency. -// -> suggestion: download Tiny core linux (https://gns3.com/tiny-core-linux) -// from inside the test. If it fails, mark the test as skipped. -// Note that both HTTPS and SFTP tests also download this image -- perhaps -// store/reuse it from a fixed /tmp location. -// Then upload to SDN and have it served hy HTTP server inside the SDN. -// (this requires enhancements inside SDN to support uploading and serving binary data) +// - For HTTP/HTTPS/SFTP: evetest already runs its own HTTP, HTTPS, and SFTP +// servers on the same interface and directory (see GetImageServerIPv4/ +// GetImageServerPort/GetImageServerHTTPSPort/GetImageServerSFTPPort/ +// DefaultSFTPUsername/DefaultSFTPPassword/GetCACertPEM in harness.go/ +// sftpserver.go). Use CreateRandomImageFile (a file of random bytes plus +// its SHA256, for exercising checksum verification), AddImageServerFile +// (writes arbitrary bytes), or CreateBlankImageFile (a real +// qcow2/qcow/vmdk/vhdx/raw disk image via qemu-img, for testing +// format-conversion) to serve content, and HTTPStorage/SFTPStorage +// pointing at it. This keeps HTTP/HTTPS/SFTP tests fully self-contained +// and needs no external network access and no SDN changes. // // - For AWS S3 / Azure Blob: parameterize via EVETEST_AWS_* / EVETEST_AZURE_* // environment variables (test parameters). Skip the test if the parameters // are not set, rather than failing -- these tests should be opt-in. // -// - For container registries: use a small public image. lfedge/evetest-* -// test images are already used elsewhere; reuse the smallest one. -// -// TestHTTPDatastore validates that EVE can download a content tree served over -// plain HTTP. -// -// Recommended approach: extend SDN to host a binary file via an HTTPServer -// endpoint (or add a new SDN HTTPFileServer endpoint type), then point the -// device config at "http://http-server.test/" with the matching -// SHA256. With the SDN-internal server the test is fully self-contained. -// -// Variants worth exercising: -// - Happy path: known SHA256, known size -> RUNNING. -// - SHA256 mismatch: content tree reaches ERROR; the error description must -// mention checksum/verification failure (the exact wording can be -// captured in a regexp matcher). -// - Server returns 404: content tree ERROR; description should reference -// the HTTP status. -// - Slow server: configure SDN port TrafficControl with rate_limit (a few -// hundred KB/s) and confirm WaitUntilAppIsRunning's "download stall" -// watchdog still considers progress valid (this exercises the -// downloadStalledTimeout path). +// - For container registries: evetest runs its own embedded OCI registry +// alongside the HTTP/HTTPS/SFTP servers (same interface, see +// PushDockerImageToLocalRegistry in localregistry.go). A small, fixed-tag +// image already used elsewhere (lfedge/evetest-ubuntu-ctr:1.0) is pulled +// into the local Docker daemon if not already present, then republished +// there -- so the test needs no real, externally reachable registry. + +package networking_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestHTTPDatastore verifies that EVE can download and verify a standalone +// volume's content over plain HTTP, using evetest's own built-in HTTP image +// server -- no external network access, no SDN changes. +// +// As with TestSFTPDatastore, the content is a few MiB of random (non-blank) +// bytes rather than a blank disk image (see CreateRandomImageFile), so that +// ImageSHA256 verification is meaningful (a blank file's checksum can't +// distinguish "downloaded correctly" from "downloaded as all +// zeros/corrupted-but-still-blank"). +// +// Phases +// ------ +// 1. Set up a device with a single DHCP mgmt port. No application or +// network instance is needed -- volumemgr creates standalone +// (app-unreferenced) volumes on its own. +// 2. Generate a random-content file (CreateRandomImageFile). Declare a +// standalone volume (AddVolume) downloading it over HTTP (HTTPStorage, +// ImageSHA256 set to the file's checksum). +// 3. Wait for the volume to reach ZSwState_CREATED_VOLUME -- since +// ImageSHA256 was set, this only happens if EVE's downloader verified +// the content against it, i.e. proves the download was not corrupted. +// 4. Delete the volume and wait for it to be reported as ZSwState_INVALID +// (fully removed). +// 5. Negative-path variants, each via expectDownloadError: +// - Wrong ImageSHA256: VolumeErr contains "computed" and "configured" +// (pillar's verifier reports the mismatching hashes in that form). +// - Wrong ImageRelativePath (a file that was never written): the +// built-in HTTP server returns 404, and VolumeErr contains +// "bad response code" and "404" (the HTTP status is propagated +// verbatim by the downloader). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestHTTPDatastore(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + log := evetest.Logger() + + const contentSize = 4 * evetest.MiB + imgFile, sha256Hex := evetest.CreateRandomImageFile( + "http-datastore-test.bin", contentSize) + + volUUID := devConfig.AddVolume("http-datastore-test", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerPort(), + }, contentSize) + + volUpdates, stopVolWatch := device.WatchVolumeInfo(volUUID) + defer stopVolWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("http-volume-config-applied") + + timeout := 10 * time.Minute + t.Eventually(volUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "volume is delivered over HTTP and passes SHA256 verification", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + evetest.Checkpoint("http-volume-delivered") + + // Delete the volume and verify it is fully removed. + devConfig.DeleteVolume(volUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(volUpdates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "volume is gone", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + + // Variant: wrong ImageSHA256 must be detected, not silently accepted. + log.Infof("Verifying a SHA256 mismatch is detected") + expectDownloadError(t, device, devConfig, + "http-datastore-bad-sha256", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: strings.Repeat("0", 64), // deliberately wrong + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerPort(), + }, contentSize, "computed", "configured") + + // Variant: a path that was never written must be reported as missing. + log.Infof("Verifying a nonexistent remote path is reported as missing") + expectDownloadError(t, device, devConfig, + "http-datastore-bad-path", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: "does-not-exist.bin", + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerPort(), + }, contentSize, "bad response code", "404") } -// TestHTTPSDatastore is identical to TestHTTPDatastore except the image is -// served over HTTPS, and the test exercises the certificate trust plumbing. +// TestHTTPSDatastore verifies that EVE can download and verify a standalone +// volume's content over HTTPS, exercising the certificate-trust plumbing. +// evetest runs a built-in HTTPS listener alongside its plain HTTP image +// server (same interface and directory), serving a certificate signed by +// the harness's own CA (see GetCACertPEM, GetImageServerHTTPSPort) -- no +// external network access, no SDN changes. // -// Recommended approach: have SDN host the file behind an HTTPS endpoint with -// a self-signed certificate. The test passes the CA in PEM form to EVE via -// HTTPStorage.HTTPSTrustedCACertsPEM, and verifies a successful download. -// The HTTPServer endpoint type in SDN already supports HTTPS-style serving. +// Phases +// ------ +// 1. Set up a device with a single DHCP mgmt port. No application or +// network instance is needed -- volumemgr creates standalone +// (app-unreferenced) volumes on its own. +// 2. Generate a random-content file (CreateRandomImageFile). Declare a +// standalone volume (AddVolume) downloading it over HTTPS (HTTPStorage +// with UseHTTPS: true and HTTPSTrustedCACertsPEM set to the harness's +// own CA certificate, ImageSHA256 set to the file's checksum). +// 3. Wait for the volume to reach ZSwState_CREATED_VOLUME -- proves both +// that the TLS certificate was trusted and that the download was not +// corrupted. +// 4. Delete the volume and wait for it to be reported as ZSwState_INVALID +// (fully removed). +// 5. Negative-path variant, via expectDownloadError: omitting +// HTTPSTrustedCACertsPEM leaves EVE unable to validate the image +// server's certificate; VolumeErr contains "certificate signed by +// unknown authority" (Go's standard x509 verification error, +// propagated verbatim). // -// Suggestion: download Tiny core linux (https://gns3.com/tiny-core-linux) -// from inside the test. If it fails, mark the test as skipped. -// Note that both HTTP and SFTP tests also download this image -- perhaps -// store/reuse it from a fixed /tmp location. -// Then upload to SDN and have it served hy HTTPS server inside the SDN. -// (this requires enhancements inside SDN to support uploading and serving binary data) -// -// Variants: -// - Happy path with the test-provided CA in HTTPSTrustedCACertsPEM. -// - CA missing / wrong: download must ERROR with an x509-related message. -// - Server cert expired: ERROR with the expected description. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestHTTPSDatastore(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + log := evetest.Logger() + + const contentSize = 4 * evetest.MiB + imgFile, sha256Hex := evetest.CreateRandomImageFile( + "https-datastore-test.bin", contentSize) + caCertPEM := string(evetest.GetCACertPEM()) + + volUUID := devConfig.AddVolume("https-datastore-test", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerHTTPSPort(), + UseHTTPS: true, + HTTPSTrustedCACertsPEM: []string{caCertPEM}, + }, contentSize) + + volUpdates, stopVolWatch := device.WatchVolumeInfo(volUUID) + defer stopVolWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("https-volume-config-applied") + + timeout := 10 * time.Minute + t.Eventually(volUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "volume is delivered over HTTPS and passes SHA256 verification", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + evetest.Checkpoint("https-volume-delivered") + + // Delete the volume and verify it is fully removed. + devConfig.DeleteVolume(volUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(volUpdates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "volume is gone", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + + // Variant: without the trusted CA, the certificate must be rejected. + log.Infof("Verifying an untrusted server certificate is rejected") + expectDownloadError(t, device, devConfig, + "https-datastore-untrusted", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerHTTPSPort(), + UseHTTPS: true, + // HTTPSTrustedCACertsPEM intentionally omitted. + }, contentSize, "certificate signed by unknown authority") } // TestAWSDatastore validates EVE's S3 datastore code path. Because EVE talks @@ -142,37 +360,174 @@ func TestHTTPSDatastore(test *testing.T) { // "set EVETEST_AWS_* to enable" message. // // Variants: -// - Happy path with valid credentials -> RUNNING. -// - Wrong secret access key -> ERROR; description should mention auth/403. -// - Wrong key (object missing) -> ERROR; description should mention 404 / -// NoSuchKey. +// - Happy path with valid credentials -> ZSwState_CREATED_VOLUME. +// - Wrong secret access key -> ZSwState_ERROR; VolumeErr should mention +// auth/403. +// - Wrong key (object missing) -> ZSwState_ERROR; VolumeErr should mention +// 404 / NoSuchKey. // // Network model: SingleEthWithDHCP + RequireInternetConnectivity{}. func TestAWSDatastore(test *testing.T) { test.Skip("not yet implemented") } -// TestSFTPDatastore validates the SFTP datastore code path. +// TestSFTPDatastore validates the SFTP datastore code path: EVE can download +// and verify a standalone volume's content over SFTP, using evetest's own +// built-in SFTP server (no external network access). // -// Recommended approach: add an SFTPServer endpoint to SDN -// (evetest/grpcapi/proto/sdn.proto) hosting a small image with username / -// password authentication. This keeps the test hermetic. Until that exists, -// gate the test on EVETEST_SFTP_* parameters analogous to the AWS test. +// The content is a few MiB of random (non-blank) bytes rather than a blank +// disk image: this makes ImageSHA256 verification meaningful (a blank file's +// checksum can't distinguish "downloaded correctly" from "downloaded as all +// zeros/corrupted-but-still-blank"), giving direct evidence that the +// downloaded content matches byte-for-byte, not just that some volume was +// created. // -// -> suggestion: download Tiny core linux (https://gns3.com/tiny-core-linux) -// from inside the test. If it fails, mark the test as skipped. -// Note that both HTTP and HTTPS tests also download this image -- perhaps -// store/reuse it from a fixed /tmp location. -// Then upload to SDN and have it served hy SFTP server inside the SDN. -// (this requires enhancements inside SDN to support uploading and serving binary data -// and the support for SFTP itself) +// Phases +// ------ +// 1. Set up a device with a single DHCP mgmt port. No application or +// network instance is needed -- volumemgr creates standalone +// (app-unreferenced) volumes on its own. +// 2. Generate a random-content file (CreateRandomImageFile). Declare a +// standalone volume (AddVolume) downloading it over SFTP (SFTPStorage, +// evetest.DefaultSFTPUsername/Password, ImageSHA256 set to the file's +// checksum). +// 3. Wait for the volume to reach ZSwState_CREATED_VOLUME -- since +// ImageSHA256 was set, this only happens if EVE's downloader verified +// the content against it, i.e. proves the download was not corrupted. +// 4. Delete the volume and wait for it to be reported as ZSwState_INVALID +// (fully removed). +// 5. Negative-path variants, each via expectDownloadError (create a +// standalone volume expected to fail, wait for a VolumeErr description +// containing specific substrings, then delete it and wait for it to be +// gone): +// - Wrong ImageSHA256: VolumeErr contains "computed" and "configured" +// (pillar's verifier reports the mismatching hashes in that form). +// - Wrong Username/Password: VolumeErr contains +// "ssh: unable to authenticate" (the SSH handshake failure is +// propagated verbatim). +// - Wrong ImageRelativePath (a file that was never written): VolumeErr +// contains "file does not exist" (pkg/sftp's Open error for the +// server's SSH_FX_NO_SUCH_FILE response, propagated verbatim). // -// Variants: -// - Happy path -> RUNNING. -// - Wrong password -> ERROR with auth-failure description. -// - Wrong path -> ERROR with file-not-found description. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestSFTPDatastore(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + log := evetest.Logger() + + const contentSize = 4 * evetest.MiB + imgFile, sha256Hex := evetest.CreateRandomImageFile( + "sftp-datastore-test.bin", contentSize) + + volUUID := devConfig.AddVolume("sftp-datastore-test", evetest.SFTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerSFTPPort(), + Username: evetest.DefaultSFTPUsername, + Password: evetest.DefaultSFTPPassword, + }, contentSize) + + volUpdates, stopVolWatch := device.WatchVolumeInfo(volUUID) + defer stopVolWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("sftp-volume-config-applied") + + timeout := 10 * time.Minute + t.Eventually(volUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "volume is delivered over SFTP and passes SHA256 verification", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + evetest.Checkpoint("sftp-volume-delivered") + + // Delete the volume and verify it is fully removed. + devConfig.DeleteVolume(volUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(volUpdates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "volume is gone", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + + // Variant: wrong ImageSHA256 must be detected, not silently accepted. + log.Infof("Verifying a SHA256 mismatch is detected") + expectDownloadError(t, device, devConfig, + "sftp-datastore-bad-sha256", evetest.SFTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: strings.Repeat("0", 64), // deliberately wrong + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerSFTPPort(), + Username: evetest.DefaultSFTPUsername, + Password: evetest.DefaultSFTPPassword, + }, contentSize, "computed", "configured") + + // Variant: wrong credentials must be rejected. + log.Infof("Verifying a wrong SFTP password is rejected") + expectDownloadError(t, device, devConfig, + "sftp-datastore-bad-password", evetest.SFTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: imgFile, + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerSFTPPort(), + Username: evetest.DefaultSFTPUsername, + Password: "wrong-password", + }, contentSize, "ssh: unable to authenticate") + + // Variant: a path that was never written must be reported as missing. + log.Infof("Verifying a nonexistent remote path is reported as missing") + expectDownloadError(t, device, devConfig, + "sftp-datastore-bad-path", evetest.SFTPStorage{ + ImageFormat: eveconfig.Format_RAW, + ImageRelativePath: "does-not-exist.bin", + ImageSHA256: sha256Hex, + ServerAddress: evetest.GetImageServerIPv4().String(), + ServerPort: evetest.GetImageServerSFTPPort(), + Username: evetest.DefaultSFTPUsername, + Password: evetest.DefaultSFTPPassword, + }, contentSize, "file does not exist") } // TestAzureDatastore validates Azure Blob storage as a datastore. Like AWS, it @@ -186,34 +541,175 @@ func TestSFTPDatastore(test *testing.T) { // // Variants: // - Happy path. -// - Wrong account key -> ERROR. -// - Missing blob -> ERROR. +// - Wrong account key -> ZSwState_ERROR. +// - Missing blob -> ZSwState_ERROR. // // Network model: SingleEthWithDHCP + RequireInternetConnectivity{}. func TestAzureDatastore(test *testing.T) { test.Skip("not yet implemented") } -// TestContainerRegistry validates that EVE can pull a container image from a -// public registry (Docker Hub by default). +// TestContainerRegistry verifies that EVE can pull a standalone volume's +// content from an OCI container registry. // -// Recommended image: a small, fixed-tag image already used by other evetest -// tests, e.g. lfedge/evetest-ubuntu-ctr:1.0 or an even smaller test -// image (busybox). Keep the tag pinned to avoid reproducibility regressions -// when the registry mutates :latest. +// The registry is evetest's own, not a real external one: evetest pulls +// lfedge/evetest-ubuntu-ctr:1.0 into the local Docker daemon if not already +// present there (from a previous pull or build), then republishes it via +// PushDockerImageToLocalRegistry, and EVE pulls it back from evetest over +// that. This is what a "docker://" pull actually exercises end to end +// (image resolution, auth, layer/manifest download, verification) without +// depending on a real, externally reachable registry or Internet access. // -// Variants: -// - Happy path: deploy a tiny container app, confirm content-tree reaches -// LOADED and app reaches RUNNING. Verify that the configured registry -// mirror (EVETEST_REGISTRY_MIRROR_DOCKER) is honored — when the mirror is -// set, the actual upstream Docker Hub should not be contacted (see the -// mirror plumbing in devconfig.go DockerContainer.toProto). -// - Wrong tag -> content tree ERROR. -// - Wrong (or missing) credentials when pulling a private image: the test -// can be parameterized with EVETEST_DOCKER_PRIVATE_* to additionally -// cover this. +// Unlike the HTTP/HTTPS/SFTP datastore tests, there is no ImageSHA256 check +// here: container images are content-addressed and verified against their +// own manifest/layer digests as part of the pull itself, so a corrupted +// download is already caught by that mechanism, independent of anything +// this test configures. // -// Network model: SingleEthWithDHCP + RequireInternetConnectivity{}. +// Phases +// ------ +// 1. Set up a device with a single DHCP mgmt port. No application or +// network instance is needed -- volumemgr creates standalone +// (app-unreferenced) volumes on its own. +// 2. Push lfedge/evetest-ubuntu-ctr:1.0 to evetest's local OCI registry +// (PushDockerImageToLocalRegistry) -- a small, fixed-tag image already +// used by other evetest tests (the pinned tag avoids reproducibility +// regressions from a mutating :latest). Declare a standalone volume +// (AddVolume) sourced from the returned DockerContainer. +// 3. Wait for the volume to reach ZSwState_CREATED_VOLUME. +// 4. Delete the volume and wait for it to be reported as ZSwState_INVALID +// (fully removed). +// 5. Negative-path variant, via expectDownloadError: a nonexistent tag is +// rejected; VolumeErr contains "MANIFEST_UNKNOWN" (the OCI distribution +// spec's registry API error code for a missing manifest/reference, +// propagated verbatim). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestContainerRegistry(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + log := evetest.Logger() + + registryImage, err := evetest.PushDockerImageToLocalRegistry( + "lfedge/evetest-ubuntu-ctr:1.0") + if err != nil { + test.Fatalf("Failed to publish test image to evetest's local OCI registry: %v", err) + } + volUUID := devConfig.AddVolume("container-registry-test", registryImage, 0) + + volUpdates, stopVolWatch := device.WatchVolumeInfo(volUUID) + defer stopVolWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("registry-volume-config-applied") + + timeout := 10 * time.Minute + t.Eventually(volUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "volume is delivered from the container registry", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + evetest.Checkpoint("registry-volume-delivered") + + // Delete the volume and verify it is fully removed. + devConfig.DeleteVolume(volUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(volUpdates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "volume is gone", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + + // Variant: a nonexistent tag must be rejected. Reuses registryImage's + // Domain/TrustedCACertsPEM (the repo itself exists there) with a tag that + // was never pushed. + log.Infof("Verifying a nonexistent image tag is reported as missing") + badTagImage := registryImage + badTagImage.Tag = "nonexistent-tag-does-not-exist" + expectDownloadError(t, device, devConfig, + "container-registry-bad-tag", badTagImage, 0, "MANIFEST_UNKNOWN") +} + +// expectDownloadError declares a standalone volume expected to fail to +// download, waits for its VolumeErr description to contain every one of +// wantErrSubstrings, then deletes it and waits for it to be gone. +func expectDownloadError(t *WithT, device *evetest.EdgeDevice, + devConfig *evetest.EdgeDeviceConfig, displayName string, + image evetest.ApplicationImageStorage, sizeBytes uint64, + wantErrSubstrings ...string) { + volUUID := devConfig.AddVolume(displayName, image, sizeBytes) + updates, stop := device.WatchVolumeInfo(volUUID) + defer stop() + device.ApplyConfig(devConfig, false, false) + + timeout := 5 * time.Minute + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("volume %s reports the expected error", displayName), + func(info *eveinfo.ZInfoVolume) bool { + desc := info.GetVolumeErr().GetDescription() + if desc == "" { + return false + } + for _, want := range wantErrSubstrings { + if !strings.Contains(desc, want) { + return false + } + } + return true + }))) + + devConfig.DeleteVolume(volUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(updates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("volume %s is gone", displayName), + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) +} + +// volumeHasError reports whether info carries a VolumeErr, for use as a +// StopIf fast-fail condition on Eventually assertions waiting on volume state. +func volumeHasError(info *eveinfo.ZInfoVolume) (string, bool) { + if desc := info.GetVolumeErr().GetDescription(); desc != "" { + return "Volume reports an error: " + desc, true + } + return "", false } diff --git a/evetest/tests/networking/dns_test.go b/evetest/tests/networking/dns_test.go index 112d887a3a8..4a400955506 100644 --- a/evetest/tests/networking/dns_test.go +++ b/evetest/tests/networking/dns_test.go @@ -12,6 +12,7 @@ import ( // revive:disable:dot-imports . "github.com/onsi/gomega" + eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" eveinfo "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/evetest" @@ -101,6 +102,10 @@ import ( // - nslookup http-server2.test: succeeds (ethernet2's exclusive DNS // server knows it), confirming dnsmasq rebuilt its upstream list after // the uplink change. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDNSFunctionality(test *testing.T) { // DNS server IPs and FQDNs as defined in netmodels.ManyDNSServers. const ( @@ -127,11 +132,16 @@ func TestDNSFunctionality(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify test prerequisites. devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, } requiredNetModel := evetest.RequireNetworkModel{ @@ -217,6 +227,9 @@ func TestDNSFunctionality(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") // ------------------------------------------------------------------ @@ -486,8 +499,9 @@ func TestDNSFunctionality(test *testing.T) { ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0", }, - CPUs: 1, - MemoryBytes: 500 * evetest.MiB, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, NetworkAdapters: []evetest.AppNetworkAdapter{ evetest.VirtualNetworkAdapter{ LogicalLabel: "vif0", diff --git a/evetest/tests/networking/failover_test.go b/evetest/tests/networking/failover_test.go index 4e005a329db..7b036dd7602 100644 --- a/evetest/tests/networking/failover_test.go +++ b/evetest/tests/networking/failover_test.go @@ -4,6 +4,10 @@ package networking_test import ( + "fmt" + "net" + "regexp" + "strconv" "testing" "time" @@ -69,19 +73,22 @@ import ( // // Test params // ----------- -// - None. WithHypervisor=HypervisorKVM is hardcoded in RequireEdgeDevice -// because this test lives in TestDeviceConnectivitySuite and -// Device-suite tests do not parameterize the hypervisor. +// - HYPERVISOR (defaults to KVM). func TestPortFailover(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -122,6 +129,9 @@ func TestPortFailover(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("port-config-applied") // Local NI on "uplink" (predefined shared label matching every mgmt port). @@ -387,54 +397,56 @@ func TestPortFailover(test *testing.T) { // TestNetworkConfigFallback verifies that EVE rolls back to the previously // working DevicePortConfig (DPC) when a newly applied configuration cannot -// reach the controller, and that it re-applies the new config once the +// reach the controller at all, and that it re-adopts the new config once the // network actually matches it. // +// ethernet1 is configured once at the start and never touched again: it is +// what keeps the *first* DPC genuinely working (so there is something to +// fall back to), but it deliberately has no role in the broken/recovered DPC +// under test in phases 2-3, which only ever contains ethernet0. A DPC is +// only considered failed as a whole once none of its ports can reach the +// controller (EVE's connectivity test tries every management port in the +// active DPC and succeeds if any one of them works -- see +// ControllerConnectivityTester.TestConnectivity), so leaving a second, +// unrelated working port in the broken DPC would have masked the failure +// this test needs to trigger. +// // Network model -// - Start with netmodels.TwoMgmtPorts. The "second" port (eth1) is initially -// used as a backup with cost=10. +// ------------- +// - netmodels.TwoMgmtPorts -- two management ports. // // Device configuration -// - Initial config: SystemAdapter for eth0 (mgmt) DHCP, SystemAdapter for -// eth1 (mgmt) DHCP. Apply, wait until SystemAdapterInfo (in published -// device info) reports currentIndex=0 and exactly one DevicePortStatus -// entry with key="zedagent" -- same pattern as bootstrap_test.go uses -// via its matchSystemAdapterInfo helper. No raw pubsub readback is -// needed; the SystemAdapterInfo embedded in ZInfoDevice carries the -// full DPC list, the currentIndex pointer, and per-DPC lastError / -// lastFailed / lastSucceeded timestamps. -// -// Phase 1 — induce a broken-config rollback -// - Apply a NEW device config that intentionally does NOT match the SDN -// network (so it cannot reach the controller): -// -> Switch eth0 to StaticNetworkConfig with a wrong subnet/gateway -// (e.g., 10.99.99.0/24 / 10.99.99.1). -// - Wait for EVE to test the new config and fall back. All assertions -// read SystemAdapterInfo from WatchDeviceInfo: -// - SystemAdapterInfo.Status grows by one entry (the just-submitted DPC). -// The new DPC -- the one at index 0 by priority -- must have -// LastError set to a description mentioning the connectivity test -// failure, and LastFailed timestamp populated. -// - SystemAdapterInfo.CurrentIndex points at the OLDER (working) DPC, -// not the one we just submitted (i.e. CurrentIndex > 0). -// - The older DPC referenced by CurrentIndex must have LastSucceeded -// advancing (it is still working). -// - The device must REMAIN online — controller still receives info -// messages, RunShellScript still works. +// -------------------- +// - Baseline: SystemAdapter for eth0 (mgmt) DHCP, SystemAdapter for eth1 +// (mgmt) DHCP. timer.port.testduration is lowered to 10s (fast per-DPC +// connectivity test) and timer.port.testbetterinterval to 60s (fast +// retest of a higher-priority DPC once it might have become usable +// again); both via SetConfigProperties. // -// Phase 2 — recovery -// - UpdateNetworkModel (or update SDN router config) to make the network -// actually match the broken config. For variant (a), change the SDN -// network's subnet/gateway from 172.20.20.0/24 to 10.99.99.0/24 -// (clone netmodels.TwoMgmtPorts and rewrite Networks[0].Ipv4 — note -// evetest.UpdateNetworkModel allows changing subnets but not the set -// of ports). -// - EVE periodically retests higher-priority DPCs (timer.port.testbetterinterval, -// default 10 min — set it lower via SetConfigProperties for the test, -// e.g. 60 s). Watching SystemAdapterInfo, eventually: -// - CurrentIndex returns to 0 (the latest DPC works again). -// - DevicePortStatus[0].LastSucceeded advances to a timestamp newer than -// the recovery moment, and LastError is cleared. +// Phases +// ------ +// 1. Baseline: WatchDeviceInfo until SystemAdapterInfo reports currentIndex=0 +// with exactly one DPC entry keyed "zedagent" and no error (same pattern +// as bootstrap_test.go's matchSystemAdapterInfo helper). +// 2. Broken-config rollback: applies a brand-new EdgeDeviceConfig containing +// only ethernet0, switched from DHCP to a StaticNetworkConfig with a +// subnet/gateway (10.99.99.0/24 / 10.99.99.1) that does not exist on the +// SDN network (plus a static DNS server pointing at the real SDN DNS +// endpoint, since a StaticNetworkConfig has no DHCP to supply one). +// Since this DPC has no other management port to fall back on, EVE's +// connectivity test for it fails outright. Eventually SystemAdapterInfo +// reports: two DPC entries; CurrentIndex=1 (the older, still-working +// two-port DPC); the newest entry (index 0) has a non-empty LastError +// and a populated LastFailed. The device stays online throughout +// (verified via EdgeDevice.GetState()) since it never actually lost +// controller connectivity -- eth1 in the older DPC kept working the +// whole time. +// 3. Recovery: UpdateNetworkModel clones netmodels.TwoMgmtPorts and rewrites +// ethernet0's SDN subnet/gateway to 10.99.99.0/24 / 10.99.99.1, matching +// the broken config -- so it is not actually broken anymore. Once EVE's +// periodic testbetterinterval retest picks this up, SystemAdapterInfo +// eventually reports CurrentIndex=0 again, with LastSucceeded advancing +// past the recovery instant and LastError cleared. // // Future extension // ---------------- @@ -443,70 +455,553 @@ func TestPortFailover(test *testing.T) { // that a brief, "remote" failure (server cert expired) does NOT trigger // a fallback (per DEVICE-CONNECTIVITY.md "Handling remote (temporary) // failures"). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestNetworkConfigFallback(test *testing.T) { - test.Skip("not yet implemented") + // DNS server endpoint reachable from ethernet0's bridge, as defined in + // netmodels.TwoMgmtPorts (its own subnet, unaffected by network0's + // client-facing subnet being rewritten below). + const dnsServer0IP = "10.16.16.25" + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPorts, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.NetworkTestDuration, 10) + cfgProps.SetGlobalValueInt(pillartypes.NetworkTestBetterInterval, 60) + + devConfig := evetest.NewEdgeDeviceConfig(devName) + devConfig.SetConfigProperties(cfgProps) + + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + eth1Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("baseline-applied") + + log := evetest.Logger() + + // Phase 1: baseline. + log.Infof("Phase 1: waiting for the baseline DPC to become active...") + baselineTimeout := 3 * time.Minute + t.Eventually(devUpdates, baselineTimeout).Should(Receive(matchers.SatisfyPredicate( + "Baseline DPC (zedagent) is active with no error", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 0, []string{"zedagent"}) { + return false + } + return sa.GetStatus()[0].GetLastError() == "" + }))) + evetest.Checkpoint("phase1-baseline-complete") + + // Phase 2: apply a broken, ethernet0-only config. With no other + // management port in this DPC, EVE's connectivity test for it fails + // outright and NIM must roll back to the still-working two-port DPC. + log.Infof("Phase 2: applying a broken ethernet0-only config...") + brokenConfig := evetest.NewEdgeDeviceConfig(devName) + brokenConfig.SetConfigProperties(cfgProps) + brokenNet := brokenConfig.AddNetwork(evetest.StaticNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + Subnet: evetest.IPSubnet("10.99.99.0/24"), + Gateway: evetest.IPAddress("10.99.99.1"), + DNSServers: []net.IP{evetest.IPAddress(dnsServer0IP)}, + }) + brokenConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: brokenNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + StaticIP: evetest.IPAddress("10.99.99.5"), + }) + device.ApplyConfig(brokenConfig, false, false) + evetest.Checkpoint("phase2-broken-config-applied") + + phase2Timeout := 5 * time.Minute + var brokenDPCLastFailed time.Time + t.Eventually(devUpdates, phase2Timeout).Should(Receive(matchers.SatisfyPredicate( + "New DPC fails outright; EVE falls back to the previous DPC", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 1, []string{"zedagent", "zedagent"}) { + return false + } + newDPC := sa.GetStatus()[0] + if newDPC.GetLastError() == "" || newDPC.GetLastFailed() == nil { + return false + } + brokenDPCLastFailed = newDPC.GetLastFailed().AsTime() + return true + }))) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE), + "device must stay online via the still-working older DPC") + evetest.Checkpoint("phase2-fallback-complete") + + // Phase 3: fix the SDN network to match the broken config's subnet. + // Once EVE's periodic retest of the higher-priority DPC succeeds, it + // becomes active again. + log.Infof("Phase 3: updating the SDN network to match ethernet0's static config...") + fixedModel := proto.Clone(netmodels.TwoMgmtPorts).(*api.NetworkModel) + for _, n := range fixedModel.Networks { + if n.LogicalLabel == "network0" { + n.Ipv4.Subnet = "10.99.99.0/24" + n.Ipv4.GwIp = "10.99.99.1" + } + } + recoveryStart := time.Now() + evetest.UpdateNetworkModel(fixedModel) + // Always restore the model on exit so a mid-test failure does not leave + // the SDN in an altered state for subsequent suite tests. + defer evetest.UpdateNetworkModel(netmodels.TwoMgmtPorts) + evetest.Checkpoint("phase3-network-fixed") + + // DevicePortConfig.IsDPCTestable (pkg/pillar/types/dpc.go) refuses to + // retest a previously-failed DPC until DpcMinTimeSinceFailure has passed + // since its own LastFailed instant -- a hardcoded 5-minute constant in + // pkg/pillar/dpcmanager/dpcmanager.go with no controller-config override + // (unlike timer.port.testduration/testbetterinterval, which we do lower + // above). Budget from the broken DPC's LastFailed, not from when we fix + // the network here, plus one testbetterinterval tick and a safety margin + // -- otherwise the retest can still be within its cooldown by the time a + // timeout measured from recoveryStart elapses. + const dpcMinTimeSinceFailure = 5 * time.Minute + const testBetterInterval = 60 * time.Second + phase3Timeout := time.Until(brokenDPCLastFailed.Add( + dpcMinTimeSinceFailure + testBetterInterval + 2*time.Minute)) + if phase3Timeout < 3*time.Minute { + phase3Timeout = 3 * time.Minute + } + // TODO: this can still time out even with the generous budget above. + // The candidate (index 0) DPC is static on eth0 (10.99.99.5) while the + // currently-active DPC (index 1) uses DHCP on the *same* eth0/subnet + // (post-fix, leasing e.g. 10.99.99.123): each retest of the candidate + // makes NIM flip eth0's address between the two, and the mgmt dnsmasq + // (pkg/pillar/dpcreconciler/genericitems/mgmtdnsmasq.go) forwards to + // 10.16.16.25@eth0 for both DPCs -- so it can get caught with eth0 + // mid-reconfiguration and time out ("read udp 127.0.0.1:53: i/o + // timeout"), even though the network is genuinely fine moments before + // and after. This was observed live: eth0 briefly held the static + // candidate's address (10.99.99.5) while SystemAdapter.CurrentIndex + // still reported the DHCP DPC (index 1) as active. DpcManager's + // DNSCacheClearCounter (mgmtdnsmasq.go) only flushes dnsmasq's cached + // *answers* on a DPC transition; it does not address a transiently + // unavailable/inconsistent eth0 address+route during the swap, and + // verifyDPC's AsyncInProgress wait (dpcmanager/verify.go) is meant for + // slow reconcile operations (DHCP negotiation, etc.), not the brief + // settling window after a plain address change. Needs a real fix in + // dpcmanager/dpcreconciler (e.g. an explicit dependency/ordering so the + // interface is confirmed stable before the connectivity test runs, or + // before mgmt dnsmasq is told to reload) rather than a test-side + // workaround. Tracked as a follow-up; not fixed by this test. + t.Eventually(devUpdates, phase3Timeout).Should(Receive(matchers.SatisfyPredicate( + "The newer DPC becomes active again once the network matches it", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if !matchSystemAdapterInfo(sa, 0, []string{"zedagent", "zedagent"}) { + return false + } + dpc := sa.GetStatus()[0] + if dpc.GetLastError() != "" { + return false + } + ts := dpc.GetLastSucceeded() + return ts != nil && !ts.AsTime().Before(recoveryStart) + }))) + evetest.Checkpoint("phase3-recovery-complete") } // TestIntermittentConnectivity verifies that EVE remains (or eventually -// becomes) ONLINE when the network exhibits significant impairments such as -// high latency, packet loss, low bandwidth and intermittent outages. +// becomes) ONLINE when the network exhibits significant impairments -- +// packet loss, high latency and jitter, narrow bandwidth, and full outages -- +// on its only management uplink. +// +// Connectivity under each impairment is confirmed directly: +// timer.deviceinfo.interval is lowered to its allowed minimum (30s) so zedagent +// publishes a fresh ZInfoDevice periodically even absent any real change +// (see zedagent/handleconfig.go's configTimerTask), and after every impairment +// change the test asserts (via WatchDeviceInfo) that such an update still +// arrives within a bounded timeout. Actually getting a message through is a +// direct proof of connectivity. // // Network model -// - Single management port (netmodels.SingleEthWithDHCP). The interesting -// dimension is the per-port TrafficControl, not topology. We don't need -// multi-port to exercise resilience to a flaky single uplink. +// ------------- +// - netmodels.SingleEthWithDHCP -- a single management port. The +// interesting dimension here is per-port TrafficControl, not topology; +// multi-port fail-over is already covered by TestPortFailover. // // Device configuration -// - Plain DHCP-on-eth0 mgmt config. -// - Add a Local NI + a small ICMP-only test app to also exercise app -// connectivity under degraded network conditions. -// -// Phase 1 — baseline -// - Apply config and confirm device is ONLINE, app is RUNNING, app can curl -// http-server.test. -// -// Phase 2 — high-loss link -// - UpdateNetworkModel: set TrafficControl on eth0 with loss_probability=20. -// - Consistently for, say, 3 minutes (longer than EVE's default test -// interval), poll device.GetState() / DeviceInfo: device must stay -// ONLINE. SystemAdapterInfo.currentIndex must remain 0 (no fallback — -// EVE retries succeed often enough). -// - The app's `ping http-server.test` should mostly succeed (>50% -// success rate) — assert >= 50% success over 100 pings. -// -// Phase 3 — high latency + jitter -// - UpdateNetworkModel: TrafficControl{delay=500, delay_jitter=300, loss=0}. -// - Device must stay ONLINE; HTTP request from app must still succeed -// within a reasonable timeout (e.g. 30s). -// -// Phase 4 — narrow bandwidth -// - UpdateNetworkModel: TrafficControl{rate_limit=64 KB/s, queue_limit=32 KB, -// burst_limit=8 KB}. -// - Device must stay ONLINE (controller traffic is small). -// - The app's HTTP fetch of "/helloworld" must still succeed (it's a few -// bytes of payload). -// -// Phase 5 — full outage windows -// - For three iterations, alternate: -// a) UpdateNetworkModel: AdminUp=false on eth0 -> hold for 90s. -// b) UpdateNetworkModel: AdminUp=true -> hold for 90s. -// - During AdminUp=false windows, device may transiently report an error -// for the port; this is acceptable. The hard requirement is that the -// device returns to ONLINE within X seconds (e.g. 60s) of every -// AdminUp=true transition. -// - lastSucceeded timestamp on the active DPC must keep advancing across -// the test duration. +// -------------------- +// - ethernet0 (mgmt+app, DHCP). +// - One Local NI on ethernet0 with one container app (port-fwd 2222->22, +// default-allow ACL) to also exercise app connectivity under degraded +// network conditions. // -// Phase 6 — restore and verify steady state -// - UpdateNetworkModel back to TrafficControl-less; verify ONLINE, -// latency-free behavior. +// Phases +// ------ +// 1. Baseline: apply config (including the lowered timer.deviceinfo.interval), +// wait for the NI ONLINE and the app RUNNING, confirm the device is +// ONLINE and the app can curl http-server.test. +// 2. High-loss link: UpdateNetworkModel sets TrafficControl{loss_probability: +// 20} on eth0. Two consecutive fresh-device-info waits (~3 minutes total) +// confirm EVE keeps getting through repeatedly, not just once, despite +// the loss. From the app, `ping -c 100 http-server.test` must show a +// packet-loss percentage no higher than 50% (i.e. at least half of the +// pings get through). +// 3. High latency + jitter: TrafficControl{delay: 500, delay_jitter: 300}. +// A fresh device-info update still arrives within the timeout, and an +// HTTP request from the app still succeeds within 30s. +// 4. Narrow bandwidth: TrafficControl{rate_limit: 64, queue_limit: 32, +// burst_limit: 8} (KB/s and KB -- a few bytes of controller/HTTP traffic +// still fit easily). A fresh device-info update still arrives, and the +// app's HTTP fetch of /helloworld still succeeds. +// 5. Full outage window: AdminUp=false (90s) followed by AdminUp=true. +// After the AdminUp=true transition, WatchDeviceInfo eventually reports +// the active DPC's LastSucceeded newer than the transition instant, then +// a fresh-device-info wait holds at the recovered state for a window +// equal to the outage. +// 6. Restore and verify steady state: UpdateNetworkModel back to the +// TrafficControl-less model; a fresh device-info update arrives and the +// app's HTTP fetch succeeds promptly (no latency/loss left to mask a +// regression). // // Notes // ----- -// - This test is non-trivially time-sensitive. Generous timeouts are -// necessary; the focus is on EVE's eventual recovery, not strict timing. -// - If a CI run becomes too long, individual phases can be split into -// separate test functions (each phase already maps cleanly to a sub-test). +// - This test is non-trivially time-sensitive; timeouts are generous +// since the focus is on EVE's eventual recovery, not strict timing. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestIntermittentConnectivity(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Lower the periodic (unconditional, even absent any real change) + // device-info publish interval to its allowed minimum, so the test can + // use "a fresh ZInfoDevice arrives within a bounded timeout" as a + // direct, real-time signal that EVE is still getting through to the + // controller -- see zedagent/handleconfig.go's configTimerTask + // ("ticker for periodical info publish when no real change"). + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, 30) + devConfig.SetConfigProperties(cfgProps) + + eth0Net := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress("10.50.0.1"), + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "intermittent-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + niUpdates, stopNIWatch := device.WatchNetworkInstanceInfo(niUUID) + defer stopNIWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + timeout := 3 * time.Minute + + // Phase 1: baseline. + log.Infof("Phase 1: verifying baseline connectivity...") + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "NI is ONLINE", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.GetState() == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("phase1-app-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("phase1-baseline-complete") + + // infoTimeout bounds how long a fresh ZInfoDevice update may take to + // arrive after an impairment is applied: with timer.deviceinfo.interval + // lowered to 30s above, 90s gives 3x margin for retries under packet + // loss/high latency before treating an actual delivery failure as such. + infoTimeout := 90 * time.Second + waitForFreshInfo := func(reason string) { + // Discard anything already buffered on the channel first, so this + // only accepts a message that arrives after the check starts -- + // otherwise a backlog from before the impairment was applied could + // satisfy Receive() immediately without proving anything new. + drainBacklog: + for { + select { + case <-devUpdates: + default: + break drainBacklog + } + } + log.Infof("Waiting for a fresh device info update (%s)...", reason) + t.Eventually(devUpdates, infoTimeout).Should(Receive(), + "EVE should still get periodic device info through "+ + "to the controller (%s)", reason) + } + + // Always restore the model on exit so a mid-test failure does not leave + // the SDN in an altered state for subsequent suite tests. + restoreModel := func() { + evetest.UpdateNetworkModel(netmodels.SingleEthWithDHCP) + } + defer restoreModel() + + setTrafficControl := func(tc *api.TrafficControl) { + model := proto.Clone(netmodels.SingleEthWithDHCP).(*api.NetworkModel) + for _, p := range model.Ports { + if p.LogicalLabel == "eth0" { + p.TrafficControl = tc + } + } + evetest.UpdateNetworkModel(model) + } + + // Phase 2: high-loss link. + log.Infof("Phase 2: applying 20%% packet loss on eth0...") + setTrafficControl(&api.TrafficControl{LossProbability: 20}) + evetest.Checkpoint("phase2-loss-applied") + + // Give the lossy link a sustained period (longer than a single probe + // cycle): two consecutive fresh-info waits cover ~3 minutes under loss, + // confirming EVE keeps getting through repeatedly, not just once. + waitForFreshInfo("20% packet loss, check 1/2") + waitForFreshInfo("20% packet loss, check 2/2") + + log.Infof("Phase 2: pinging http-server.test through the lossy link...") + pingOut, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "ping -c 50 -w 75 http-server.test", 80*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + lossPct, err := pingPacketLossPercent(pingOut) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(lossPct).To(BeNumerically("<=", 50), + "at least half of the pings must get through a 20%% -loss link:\n%s", pingOut) + evetest.Checkpoint("phase2-complete") + + // Phase 3: high latency + jitter. + log.Infof("Phase 3: applying 500ms +/- 300ms latency on eth0...") + setTrafficControl(&api.TrafficControl{Delay: 500, DelayJitter: 300}) + evetest.Checkpoint("phase3-latency-applied") + + waitForFreshInfo("500ms +/- 300ms latency") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 30 http://http-server.test/helloworld", 35*time.Second, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase3-complete") + + // Phase 4: narrow bandwidth. + log.Infof("Phase 4: applying a 64 KB/s rate limit on eth0...") + setTrafficControl(&api.TrafficControl{ + RateLimit: 64, + QueueLimit: 32, + BurstLimit: 8, + }) + evetest.Checkpoint("phase4-bandwidth-limited") + + waitForFreshInfo("64 KB/s rate limit") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 20 http://http-server.test/helloworld", 25*time.Second, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase4-complete") + + // Phase 5: full outage window. + log.Infof("Phase 5: taking eth0 down, then restoring it...") + const outageWindow = 90 * time.Second + recoveryTimeout := 5 * time.Minute + downModel := proto.Clone(netmodels.SingleEthWithDHCP).(*api.NetworkModel) + for _, p := range downModel.Ports { + if p.LogicalLabel == "eth0" { + p.AdminUp = false + } + } + evetest.UpdateNetworkModel(downModel) + time.Sleep(outageWindow) + + log.Infof("Phase 5: restoring eth0...") + recoveryStart := time.Now() + restoreModel() + + t.Eventually(devUpdates, recoveryTimeout).Should(Receive(matchers.SatisfyPredicate( + "DPC LastSucceeded advances past the AdminUp=true transition", + func(info *eveinfo.ZInfoDevice) bool { + sa := info.GetSystemAdapter() + if sa == nil { + return false + } + statusList := sa.GetStatus() + idx := int(sa.GetCurrentIndex()) + if idx < 0 || idx >= len(statusList) { + return false + } + ts := statusList[idx].GetLastSucceeded() + return ts != nil && !ts.AsTime().Before(recoveryStart) + }))) + // Hold at the recovered state for a window equal to the outage above, + // actively confirming (rather than just sleeping) that fresh device + // info keeps arriving throughout. + waitForFreshInfo("recovered after outage") + evetest.Checkpoint("phase5-complete") + + // Phase 6: restore and verify steady state. + log.Infof("Phase 6: verifying steady state after restoring the clean network model...") + restoreModel() + waitForFreshInfo("steady state restored") + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Hello world!")) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("phase6-complete") +} + +// pingPacketLossPercent extracts the packet-loss percentage from the summary +// line of `ping` output (e.g. "100 packets transmitted, 82 received, 18% +// packet loss, time 99231ms"). The percentage is fractional whenever the +// loss ratio isn't a whole number (e.g. "14.5299%"), so it must be parsed as +// a float rather than truncated to the digits right before the '%'. +func pingPacketLossPercent(output string) (float64, error) { + re := regexp.MustCompile(`([\d.]+)% packet loss`) + m := re.FindStringSubmatch(output) + if len(m) != 2 { + return 0, fmt.Errorf("could not parse packet loss from ping output: %q", output) + } + return strconv.ParseFloat(m[1], 64) } diff --git a/evetest/tests/networking/ipv6_test.go b/evetest/tests/networking/ipv6_test.go index f0985a64475..68384af5f09 100644 --- a/evetest/tests/networking/ipv6_test.go +++ b/evetest/tests/networking/ipv6_test.go @@ -61,11 +61,20 @@ import ( // - ip -6 route show contains a default route via a fe80:: link-local // address (RA-derived routes always use the router's link-local address). // - ip -4 addr show dev eth0 contains no "inet" lines (no IPv4). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDeviceIPv6Connectivity(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" // Clone the shared model so we can modify it without side effects. // Clear upstream DNS servers: everything this test needs (controller, DNS @@ -78,7 +87,7 @@ func TestDeviceIPv6Connectivity(test *testing.T) { evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{ @@ -102,6 +111,9 @@ func TestDeviceIPv6Connectivity(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -225,8 +237,7 @@ func TestDeviceIPv6Connectivity(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. evetest.SkipIfHypervisorKubevirt() is called after reading -// the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestApplicationIPv6Connectivity(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -236,8 +247,6 @@ func TestApplicationIPv6Connectivity(test *testing.T) { evetest.HypervisorParameter(), ) hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // IPv6 address of the SDN HTTP server and DNS server defined in netmodels.SingleEthIPv6Only. const httpServerIPv6 = "fdde:55a:74d4::7" @@ -317,6 +326,9 @@ func TestApplicationIPv6Connectivity(test *testing.T) { appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) defer stopAppWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } timeoutExcludingDownload := 5 * time.Minute device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) diff --git a/evetest/tests/networking/net_adapter_test.go b/evetest/tests/networking/net_adapter_test.go index e2a4dc2c822..3851da88340 100644 --- a/evetest/tests/networking/net_adapter_test.go +++ b/evetest/tests/networking/net_adapter_test.go @@ -25,10 +25,11 @@ import ( "github.com/lf-edge/eve/pkg/pillar/utils/netutils" ) -func deviceRequirementsForNetAdapterTests(devName string) evetest.RequireEdgeDevice { +func deviceRequirementsForNetAdapterTests( + devName string, hypervisor evetest.Hypervisor) evetest.RequireEdgeDevice { return evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, MinCPUs: 4, WithGrubOptions: []string{ // No applications are deployed in these network adapter tests. @@ -64,9 +65,14 @@ func TestDHCPIPv4Only(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCPAndIPv6, } @@ -89,6 +95,9 @@ func TestDHCPIPv4Only(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -131,9 +140,14 @@ func TestStaticIPv4Only(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithDHCPAndIPv6, } @@ -160,6 +174,9 @@ func TestStaticIPv4Only(test *testing.T) { StaticIP: evetest.IPAddress("172.20.20.100"), }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") log := evetest.Logger() @@ -253,15 +270,17 @@ func TestPNAC(test *testing.T) { // Define configurable parameters available for the test. evetest.DefineTestParameters( + evetest.HypervisorParameter(), requireSCEPProxyParam, ) // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() requireSCEPProxy := evetest.GetTestParameter[bool](requireSCEPProxyParamKey) // Set up the test harness and specify the test prerequisites. devName := "edge-dev" - requiredDevice := deviceRequirementsForNetAdapterTests(devName) + requiredDevice := deviceRequirementsForNetAdapterTests(devName, hypervisor) requiredNetModel := evetest.RequireNetworkModel{ NetworkModel: netmodels.SingleEthWithPNAC(requireSCEPProxy), } @@ -329,6 +348,9 @@ func TestPNAC(test *testing.T) { defer stopDevMetricsWatch() configAppliedAt := time.Now() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("config-applied") timeout := 3 * time.Minute diff --git a/evetest/tests/networking/netinst_test.go b/evetest/tests/networking/netinst_test.go index e756c5492c7..e0e956b03b9 100644 --- a/evetest/tests/networking/netinst_test.go +++ b/evetest/tests/networking/netinst_test.go @@ -6,7 +6,9 @@ package networking_test import ( + "fmt" "net" + "strings" "testing" "time" @@ -15,6 +17,7 @@ import ( eveconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve-api/go/evecommon" + eveflowlog "github.com/lf-edge/eve-api/go/flowlog" eveinfo "github.com/lf-edge/eve-api/go/info" evemetrics "github.com/lf-edge/eve-api/go/metrics" "github.com/lf-edge/eve/evetest" @@ -22,6 +25,7 @@ import ( "github.com/lf-edge/eve/evetest/netmodels" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/lf-edge/eve/pkg/pillar/utils/generics" + uuid "github.com/satori/go.uuid" ) // TestLocalNI is the canonical end-to-end exercise of a Local (L3/NAT) @@ -78,20 +82,16 @@ import ( // 6. NI metrics: ZMetricNetworkInstance for the NI eventually has // non-zero RX and TX TotalPackets, proving the per-NI dataplane // counters track the traffic generated above. -// 7. Flow / DNS log assertions are commented out -- GetAppFlowLogs / -// GetAppDNSLogs are not yet implemented in evetest (see -// edgedevice.go). The placeholders document the intended check -// (with flowlog disabled the lists must be empty) and will be enabled -// once the framework support lands. +// 7. Flow / DNS log check: flow logging is off by default (it is enabled +// and exercised in TestFlowLog), so GetAppFlowLogs / GetAppDNSLogs must +// both return empty for this app's VIF. // 8. App teardown: delete the app, wait until ZSwState_INVALID, then // assert NetworkInstance.Vifs is empty and the bridge-IP assignment // persists. Finally delete the NI and wait for UNSPECIFIED. // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter -- Kubevirt is reserved for cluster -// tests. +// - HYPERVISOR (defaults to KVM). // // Suite placement // --------------- @@ -109,8 +109,6 @@ func TestLocalNI(test *testing.T) { // Get parameter values set for this test execution. hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // Set up the test harness and specify the test prerequisites. devName := "edge-dev" @@ -142,6 +140,9 @@ func TestLocalNI(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Try to create local network instance. niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ @@ -392,7 +393,6 @@ func TestLocalNI(test *testing.T) { stopNIMetricsWatch() // Flowlog is disabled by default (it is enabled and tested in TestFlowLog). - /* TODO: GetAppFlowLogs is not yet implemented t.Expect(device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ VirtualNetAdapter: "vif0", NetworkInstance: niUUID, @@ -401,7 +401,6 @@ func TestLocalNI(test *testing.T) { VirtualNetAdapter: "vif0", NetworkInstance: niUUID, })).To(BeEmpty()) - */ // Undeploy app and check that VIF was disconnected from the network instance. devConfig.DeleteApplication(appUUID) @@ -502,8 +501,7 @@ func TestLocalNI(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter. +// - HYPERVISOR (defaults to KVM). // // Suite placement // --------------- @@ -520,8 +518,6 @@ func TestSwitchNI(test *testing.T) { // Get parameter values set for this test execution. hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() // Set up the test harness and specify the test prerequisites. devName := "edge-dev" @@ -553,6 +549,9 @@ func TestSwitchNI(test *testing.T) { Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Try to create switch network instance. niUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ @@ -783,7 +782,6 @@ func TestSwitchNI(test *testing.T) { stopNIMetricsWatch() // Flowlog is disabled by default (it is enabled and tested in TestFlowLog). - /* TODO: GetAppFlowLogs is not yet implemented t.Expect(device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ VirtualNetAdapter: "vif0", NetworkInstance: niUUID, @@ -792,7 +790,6 @@ func TestSwitchNI(test *testing.T) { VirtualNetAdapter: "vif0", NetworkInstance: niUUID, })).To(BeEmpty()) - */ // Undeploy app and check that VIF was disconnected from the network instance. devConfig.DeleteApplication(appUUID) @@ -827,69 +824,1766 @@ func TestSwitchNI(test *testing.T) { stopNIWatch() } -// TestFlowLog verifies that EVE produces flow log records and DNS request log -// records for application traffic when flow logging is enabled on a Local -// Network Instance, and that those records correctly attribute flows to ACE -// IDs (allowed flows -> matching ACE; dropped flows -> ACE id 0 = implicit -// reject-all). +// TestNIReplace verifies that EVE correctly handles Network Instances being +// rapidly replaced -- deleted and recreated (possibly reusing the same +// subnet, or splitting/merging across a different number of NIs) -- all +// within a single config apply. This exercises internal number/resource +// reuse (bridge number, IPAM state, iptables chains, etc.) inside zedrouter, +// which could otherwise conflict if stale state from the deleted NI is not +// fully torn down before the replacement comes up. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port; no application is +// deployed, only Local Network Instances are created/replaced. +// +// Phases +// ------ +// 1. Create n1 (Local NI, subnet 10.11.12.0/24). Wait for ONLINE. +// 2. Replace n1 with n2 using a *different* subnet (10.11.13.0/24), deleting +// n1 and adding n2 in one config apply. Wait for n1 -> UNSPECIFIED and +// n2 -> ONLINE with the new subnet's bridge IP. +// 3. Replace n2 with n3, reusing the *same* subnet (10.11.13.0/24) that n2 +// just released, again in one apply. Wait for n2 -> UNSPECIFIED and +// n3 -> ONLINE with BridgeIPAddr unchanged (10.11.13.1), proving the +// just-freed subnet can be immediately reused by a different NI. +// 4. Replace one NI (n3) with two NIs in one apply: n4 (reusing the +// 10.11.12.0/24 subnet that has been free since step 2) and n5 (a fresh +// subnet). Both come up ONLINE while n3 goes UNSPECIFIED. +// 5. Move the subnet used by n4 to a new one (10.11.14.0/24) and, in the very +// same apply, add n6 that takes over n4's *former* subnet +// (10.11.12.0/24). Wait for n6 -> ONLINE with the taken-over subnet's +// bridge IP and for n4 to keep running with its new bridge IP.. +// 6. Cleanup: delete n4, n5 and n6, and wait for all three to report +// ZNETINST_STATE_UNSPECIFIED. // -// SKIPPED: app flow logs are not yet supported in evetest. The framework -// provides the API (EdgeDevice.GetAppFlowLogs / GetAppDNSLogs) but its -// implementation depends on AdamClient.IterateAppFlowLogs which is not -// implemented (see edgedevice.go). +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). // -// When implemented later, the scenario: +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestNIReplace(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + // localNI builds a Local NI config for a /24 subnet whose gateway is the + // ".1" address; the DHCP range spans the rest of the subnet (.2-.254). + localNI := func(displayName, subnetPrefix string) evetest.LocalNetworkInstanceConfig { + return evetest.LocalNetworkInstanceConfig{ + DisplayName: displayName, + Port: "ethernet0", + Subnet: evetest.IPSubnet(subnetPrefix + ".0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress(subnetPrefix + ".2"), + End: evetest.IPAddress(subnetPrefix + ".254"), + }, + Gateway: evetest.IPAddress(subnetPrefix + ".1"), + MTU: 1500, + } + } + + timeout := 3 * time.Minute + waitOnline := func(niUUID uuid.UUID, updates <-chan *eveinfo.ZInfoNetworkInstance, + expectBridgeIP string) *eveinfo.ZInfoNetworkInstance { + var niInfo *eveinfo.ZInfoNetworkInstance + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("NI %s is ONLINE with bridge IP %s", niUUID, expectBridgeIP), + func(info *eveinfo.ZInfoNetworkInstance) bool { + niInfo = info + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE && + info.BridgeIPAddr == expectBridgeIP + }))) + t.Expect(niInfo.NetworkID).To(Equal(niUUID.String())) + t.Expect(niInfo.NetworkErr).To(BeEmpty()) + return niInfo + } + waitGone := func(updates <-chan *eveinfo.ZInfoNetworkInstance) { + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "NI state is UNSPECIFIED", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + } + + // Step 1: create n1. + n1UUID := devConfig.AddNetworkInstance(localNI("n1", "10.11.12")) + n1Updates, stopN1Watch := device.WatchNetworkInstanceInfo(n1UUID) + device.ApplyConfig(devConfig, false, false) + waitOnline(n1UUID, n1Updates, "10.11.12.1") + evetest.Checkpoint("n1-created") + + // Step 2: replace n1 with n2, using a different subnet, in a single apply. + devConfig.DeleteNetworkInstance(n1UUID) + n2UUID := devConfig.AddNetworkInstance(localNI("n2", "10.11.13")) + n2Updates, stopN2Watch := device.WatchNetworkInstanceInfo(n2UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n1Updates) + stopN1Watch() + waitOnline(n2UUID, n2Updates, "10.11.13.1") + evetest.Checkpoint("n1-replaced-by-n2") + + // Step 3: replace n2 with n3, reusing the exact same subnet n2 just released. + devConfig.DeleteNetworkInstance(n2UUID) + n3UUID := devConfig.AddNetworkInstance(localNI("n3", "10.11.13")) + n3Updates, stopN3Watch := device.WatchNetworkInstanceInfo(n3UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n2Updates) + stopN2Watch() + waitOnline(n3UUID, n3Updates, "10.11.13.1") + evetest.Checkpoint("n2-replaced-by-n3") + + // Step 4: replace n3 with two NIs -- n4 (reusing the 10.11.12.0/24 subnet + // that has been free since step 2) and n5 (a brand new subnet). + devConfig.DeleteNetworkInstance(n3UUID) + n4UUID := devConfig.AddNetworkInstance(localNI("n4", "10.11.12")) + n5UUID := devConfig.AddNetworkInstance(localNI("n5", "10.11.15")) + n4Updates, stopN4Watch := device.WatchNetworkInstanceInfo(n4UUID) + n5Updates, stopN5Watch := device.WatchNetworkInstanceInfo(n5UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n3Updates) + stopN3Watch() + waitOnline(n4UUID, n4Updates, "10.11.12.1") + waitOnline(n5UUID, n5Updates, "10.11.15.1") + evetest.Checkpoint("n3-replaced-by-n4-and-n5") + + // Step 5: move n4's subnet elsewhere and let n6 take over the subnet that + // n4 just vacated -- both changes submitted in the very same apply. + devConfig.UpdateNetworkInstance(n4UUID, localNI("n4", "10.11.14")) + n6UUID := devConfig.AddNetworkInstance(localNI("n6", "10.11.12")) + n6Updates, stopN6Watch := device.WatchNetworkInstanceInfo(n6UUID) + device.ApplyConfig(devConfig, false, false) + waitOnline(n4UUID, n4Updates, "10.11.14.1") + waitOnline(n6UUID, n6Updates, "10.11.12.1") + evetest.Checkpoint("n4-subnet-moved-to-n6") + + // Cleanup. + devConfig.DeleteNetworkInstance(n4UUID) + devConfig.DeleteNetworkInstance(n5UUID) + devConfig.DeleteNetworkInstance(n6UUID) + device.ApplyConfig(devConfig, false, false) + waitGone(n4Updates) + waitGone(n5Updates) + waitGone(n6Updates) + stopN4Watch() + stopN5Watch() + stopN6Watch() +} + +// TestMoveAppBetweenNIs verifies that moving an application's network +// adapter from one Network Instance to another (i.e. changing which NI a +// VIF is attached to, within an otherwise-unchanged app config) takes +// effect at runtime: the app is redeployed onto the new NI, gets a fresh IP +// from its subnet, loses connectivity with peers on the NI it left, and +// gains connectivity with peers on the NI it joined. // // Network model -// - netmodels.SingleEthWithDHCP, plus a second SDN HTTP server -// (alt-server.test) for an additional differentiable target. +// ------------- +// - netmodels.SingleEthWithDHCP -- a single mgmt+app port (ethernet0), +// shared as the uplink by both Local NIs below (a port can back more +// than one Local NI at once). // -// Device configuration -// - One Local NI with EnableFlowlog=true and a Subnet/DHCPRange. -// - One container app on the NI with three ACLs: -// - allow IP+TCP+fport=80 to http-server.test (specific ACE id, e.g. 100) -// - allow IP+ICMP to NI bridge IP (ACE id 200) -// - default-deny is implicit (ACE id 0) -// - Port-fwd 2222->22 on a separate ACE (id 300) to enable test SSH. -// -// Phase 1 — generate distinguishable traffic -// - From inside the app: curl http-server.test/helloworld (allowed by ACE 100). -// - From inside the app: curl --max-time 5 alt-server.test/helloworld -// (must fail; matches the implicit deny -> ACE id 0). -// - From inside the app: ping -c 3 (allowed by ACE 200). -// - From inside the app: nslookup http-server.test (DNS request log entry). -// -// Phase 2 — flow log assertions (via GetAppFlowLogs) -// - Wait for the flow log batch (default 2-min interval per -// APP-CONNECTIVITY.md). Use a generous timeout (3 min). -// - For ACE id 100: at least one outbound flow with dst IP = -// http-server.test's IP, dst port 80, proto TCP, packet count > 0. -// - For ACE id 200: at least one outbound ICMP flow toward the NI -// bridge IP. -// - For ACE id 0: at least one outbound flow toward alt-server's IP, dst -// port 80, proto TCP. The flow record is created because the app sent -// the packet even though it was dropped (flow logging on Local NI uses -// the conntrack-based mark-and-blackhole pattern; see APP-CONNECTIVITY.md). -// - The reverse direction of each flow (Inbound) for the allowed flows is -// also logged once the response packet is observed. -// -// Phase 3 — DNS log assertions (via GetAppDNSLogs) -// - At least one DNSRequest record with hostname "http-server.test" and the -// resolved IP equal to that of the SDN http-server endpoint. -// - The request time is within the test window. -// -// Phase 4 — flow logging disabled -// - UpdateNetworkInstance to set EnableFlowlog=false; re-apply. -// - Generate fresh traffic; after the next reporting interval, assert that -// no NEW flow records appear (timestamps strictly older than the -// reapply time). This confirms the runtime toggle works. +// Phases +// ------ +// 1. Create two Local NIs on ethernet0: "n1" (10.11.12.0/24) and "n2" +// (10.11.13.0/24). Wait for both ONLINE. +// 2. Deploy three container apps (lfedge/evetest-ubuntu-ctr:1.0, +// VirtualizationMode=HVM): "ping1" on n1 (port-fwd 2223->22, ICMP +// allow-all ACL), "ping2" on n2 (port-fwd 2224->22, ICMP allow-all ACL) +// and "pong" on n1 (ICMP allow-all ACL, no port-fwd -- nothing ever SSHes +// into it). WaitUntilAppIsRunning for all three, then read pong's NI IP +// from its ZInfoApp. +// 3. Same-NI reachability: from ping1 (on n1, same NI as pong), `ping -c 3 +// -W 1 ` succeeds. From ping2 (on n2), the same ping fails -- +// pong's private /24 is not reachable from a different, NAT-isolated +// Local NI. +// 4. Move pong from n1 to n2: UpdateApplication with pong's +// VirtualNetworkAdapter now pointing at n2's UUID (same MAC, same ACL). +// Changing the NI reference changes the app's Interfaces, so EVE purges +// and redeploys pong. WaitUntilAppIsRunning again, then read pong's new +// (n2-subnet) IP. +// 5. Switched reachability: ping2 can now reach pong; ping1 can no longer +// reach it (mirrors step 3, with the roles of ping1/ping2 reversed). +// 6. Move pong back to n1 and repeat the step-3 assertions, confirming the +// switch is fully reversible. +// 7. Cleanup: delete all three apps and both NIs, waiting for each to be +// gone. // // Test params // ----------- -// - HYPERVISOR. The test must call evetest.SkipIfHypervisorKubevirt() -// after reading the parameter -- Kubevirt is reserved for cluster tests. -func TestFlowLog(test *testing.T) { - test.Skip("not yet implemented") +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestMoveAppBetweenNIs(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + n1UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "n1", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + n2UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "n2", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.13.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.13.2"), + End: evetest.IPAddress("10.11.13.254"), + }, + Gateway: evetest.IPAddress("10.11.13.1"), + MTU: 1500, + }) + + n1Updates, stopN1Watch := device.WatchNetworkInstanceInfo(n1UUID) + n2Updates, stopN2Watch := device.WatchNetworkInstanceInfo(n2UUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(n1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n1 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(n2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n2 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + icmpAllowAny := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + const ( + ping1MAC = "02:16:3e:00:00:01" + ping2MAC = "02:16:3e:00:00:02" + pongMAC = "02:16:3e:00:00:03" + ) + newApp := func(displayName string, niUUID uuid.UUID, mac string, + portFwd []evetest.PortFwdRule) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress(mac), + PortFwdRules: portFwd, + ACLAllowRules: icmpAllowAny, + }, + }, + } + } + ping1UUID := devConfig.AddApplication(newApp("ping1", n1UUID, ping1MAC, + []evetest.PortFwdRule{{Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2223, AppPort: 22}})) + ping2UUID := devConfig.AddApplication(newApp("ping2", n2UUID, ping2MAC, + []evetest.PortFwdRule{{Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2224, AppPort: 22}})) + pongUUID := devConfig.AddApplication(newApp("pong", n1UUID, pongMAC, nil)) + + ping1Updates, stopPing1Watch := device.WatchAppInfo(ping1UUID) + ping2Updates, stopPing2Watch := device.WatchAppInfo(ping2UUID) + pongUpdates, stopPongWatch := device.WatchAppInfo(pongUUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(ping1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(ping2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(pongUUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + // getVifIPInSubnet drains updates until the app reports a VIF IP address + // that belongs to the given subnet. Requiring subnet membership (rather + // than accepting any IP) makes this robust against a stale, previously + // drained IP (from before an NI move) still sitting in the channel. + getVifIPInSubnet := func(updates <-chan *eveinfo.ZInfoApp, subnet *net.IPNet) string { + var ip string + t.Eventually(updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("App has a VIF IP address from %s", subnet), + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) == 0 { + return false + } + for _, addr := range info.Network[0].IPAddrs { + if subnet.Contains(evetest.IPAddress(addr)) { + ip = addr + return true + } + } + return false + }).StopIf(appHasError))) + return ip + } + n1Subnet := evetest.IPSubnet("10.11.12.0/24") + n2Subnet := evetest.IPSubnet("10.11.13.0/24") + pongIP := getVifIPInSubnet(pongUpdates, n1Subnet) + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(ping1UUID) + waitSSHReady(ping2UUID) + + pingFrom := func(appUUID uuid.UUID, targetIP string) error { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 %s", targetIP), sshTimeout, 0) + return err + } + + // Phase 1: pong is on n1, same as ping1. + log.Infof("Testing connectivity before the move: pong is on n1 (with ping1)") + t.Expect(pingFrom(ping1UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping2UUID, pongIP)).To(HaveOccurred()) + evetest.Checkpoint("pong-on-n1-verified") + + // Move pong from n1 to n2. + devConfig.UpdateApplication(pongUUID, newApp("pong", n2UUID, pongMAC, nil)) + device.ApplyConfig(devConfig, false, false) + pongIP = getVifIPInSubnet(pongUpdates, n2Subnet) + evetest.Checkpoint("pong-moved-to-n2") + + // Phase 2: pong is now on n2, same as ping2. + log.Infof("Testing connectivity after the move: pong is on n2 (with ping2)") + t.Expect(pingFrom(ping2UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping1UUID, pongIP)).To(HaveOccurred()) + evetest.Checkpoint("pong-on-n2-verified") + + // Move pong back from n2 to n1. + devConfig.UpdateApplication(pongUUID, newApp("pong", n1UUID, pongMAC, nil)) + device.ApplyConfig(devConfig, false, false) + pongIP = getVifIPInSubnet(pongUpdates, n1Subnet) + evetest.Checkpoint("pong-moved-back-to-n1") + + // Phase 3: pong is back on n1. + log.Infof("Testing connectivity after moving back: pong is on n1 again") + t.Expect(pingFrom(ping1UUID, pongIP)).ToNot(HaveOccurred()) + t.Expect(pingFrom(ping2UUID, pongIP)).To(HaveOccurred()) + + // Cleanup. + devConfig.DeleteApplication(ping1UUID) + devConfig.DeleteApplication(ping2UUID) + devConfig.DeleteApplication(pongUUID) + devConfig.DeleteNetworkInstance(n1UUID) + devConfig.DeleteNetworkInstance(n2UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(ping1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ping1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(ping2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ping2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(pongUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "pong is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopPing1Watch() + stopPing2Watch() + stopPongWatch() + + t.Eventually(n1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n1 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(n2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "n2 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopN1Watch() + stopN2Watch() +} + +// TestPortForwarding exercises port-forwarding (D-NAT) rules of a Local +// Network Instance: hairpin connectivity between two apps reached through +// the edge node's *external* port-forwarded address (rather than directly +// via their NI-internal IPs), across both a shared uplink and two different +// uplink adapters, plus changing a port-forwarding rule's external port at +// runtime. +// +// Network model +// ------------- +// - netmodels.TwoMgmtPorts -- two independent mgmt+app ports (ethernet0, +// ethernet1), each with its own DHCP subnet and its own external IP. +// Needed for the cross-adapter hairpin scenario below; the same-adapter +// scenarios simply ignore ethernet1's NI. +// +// Phases +// ------ +// 1. Create two Local NIs: "ni1" on ethernet0 (10.11.12.0/24) and "ni2" on +// ethernet1 (10.11.13.0/24). Wait for both ONLINE. +// 2. Deploy three container apps (lfedge/evetest-ubuntu-ctr:1.0, +// VirtualizationMode=HVM, allow-all ACL on every VIF): "app1" and "app2" +// on ni1 (port-fwd 2223->22 and 2224->22, respectively) and "app3" on +// ni2 (port-fwd 2226->22). WaitUntilAppIsRunning for all three. +// 3. Same-adapter hairpin: from app1 (reached via its own port-fwd), +// open a raw TCP connection to :2224 -- app2's port-fwd +// address -- and read app2's sshd banner. Repeat in the opposite +// direction (app2 -> app1's port-fwd). Both must succeed, proving hairpin +// NAT works when the initiator and the target share both the NI and +// the uplink adapter. +// 4. Runtime port-fwd rule change: change app1's port-fwd rule from 2223->22 +// to 2225->22 via UpdateApplication (which purges/redeploys app1). +// After WaitUntilAppIsRunning: +// - the old external port (2223) must no longer forward (probed from +// app2: raw TCP connect times out / errors); +// - the new external port (2225) must forward to app1 (probed from +// app2: app1's sshd banner is read back); +// - RunShellScriptInsideApp(app1, ...) -- which re-derives the SSH +// endpoint from the *current* device config -- keeps working +// transparently through the port change. +// Then the rule is switched back to 2223->22 and the same two-sided +// check (old port dead, new/original port alive) is repeated, proving +// the change is fully reversible. +// 5. Cross-adapter hairpin: from app1 (on ni1/ethernet0), open a raw TCP +// connection to :2226 -- app3's port-fwd address on the +// *other* uplink adapter -- and read app3's sshd banner. This proves +// D-NAT is applied correctly per-adapter (the two uplinks have different +// external IPs) even when initiator and target sit on different Network +// Instances bound to different ports. +// 6. Cleanup: delete all three apps and both NIs. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestPortForwarding(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPorts, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + ni1UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "ni1", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + ni2UUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "ni2", + Port: "ethernet1", + Subnet: evetest.IPSubnet("10.11.13.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.13.2"), + End: evetest.IPAddress("10.11.13.254"), + }, + Gateway: evetest.IPAddress("10.11.13.1"), + MTU: 1500, + }) + ni1Updates, stopNI1Watch := device.WatchNetworkInstanceInfo(ni1UUID) + ni2Updates, stopNI2Watch := device.WatchNetworkInstanceInfo(ni2UUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(ni1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni1 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(ni2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni2 is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + newPortFwdApp := func(displayName string, niUUID uuid.UUID, mac string, + edgeNodePort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + MAC: evetest.MACAddress(mac), + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: edgeNodePort, + AppPort: 22, + }, + }, + ACLAllowRules: allowAll, + }, + }, + } + } + const ( + app1MAC = "02:16:3e:00:01:01" + app2MAC = "02:16:3e:00:01:02" + app3MAC = "02:16:3e:00:01:03" + ) + app1UUID := devConfig.AddApplication(newPortFwdApp("app1", ni1UUID, app1MAC, 2223)) + app2UUID := devConfig.AddApplication(newPortFwdApp("app2", ni1UUID, app2MAC, 2224)) + app3UUID := devConfig.AddApplication(newPortFwdApp("app3", ni2UUID, app3MAC, 2226)) + + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + app3Updates, stopApp3Watch := device.WatchAppInfo(app3UUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app3UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(app1UUID) + waitSSHReady(app2UUID) + waitSSHReady(app3UUID) + + // hairpinProbe opens a raw TCP connection from inside srcApp to + // targetIP:targetPort and returns the first bytes read back (expected to + // be the target sshd's banner, "SSH-2.0-...", when the port-fwd works). + hairpinProbe := func(srcApp uuid.UUID, targetIP net.IP, targetPort uint16) (string, error) { + script := fmt.Sprintf( + "timeout 5 bash -c 'exec 3<>/dev/tcp/%s/%d; head -c 4 <&3'", + targetIP, targetPort) + out, _, err := device.RunShellScriptInsideApp( + srcApp, appAuth, script, sshTimeout, 0) + return out, err + } + + eth0IP := device.GetDeviceIPAddress("ethernet0") + t.Expect(eth0IP).ToNot(BeEmpty()) + eth1IP := device.GetDeviceIPAddress("ethernet1") + t.Expect(eth1IP).ToNot(BeEmpty()) + + // Phase 1: same-adapter hairpin between app1 and app2 (both on ni1/ethernet0). + log.Infof("Testing same-adapter hairpin: app1 -> app2's port-fwd") + out, err := hairpinProbe(app1UUID, eth0IP[0], 2224) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + log.Infof("Testing same-adapter hairpin: app2 -> app1's port-fwd") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + evetest.Checkpoint("same-adapter-hairpin-verified") + + // Phase 2: change app1's port-fwd rule at runtime (2223 -> 2225). + devConfig.UpdateApplication(app1UUID, newPortFwdApp("app1", ni1UUID, app1MAC, 2225)) + device.ApplyConfig(devConfig, false, false) + waitSSHReady(app1UUID) // re-derives the endpoint from the updated config + evetest.Checkpoint("app1-portfwd-switched-to-2225") + + log.Infof("Testing that the old port-fwd (2223) no longer forwards to app1") + _, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).To(HaveOccurred()) + + log.Infof("Testing that the new port-fwd (2225) forwards to app1") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2225) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + // Switch app1's port-fwd rule back to its original port (2225 -> 2223). + devConfig.UpdateApplication(app1UUID, newPortFwdApp("app1", ni1UUID, app1MAC, 2223)) + device.ApplyConfig(devConfig, false, false) + waitSSHReady(app1UUID) + evetest.Checkpoint("app1-portfwd-switched-back-to-2223") + + log.Infof("Testing that the temporary port-fwd (2225) no longer forwards to app1") + _, err = hairpinProbe(app2UUID, eth0IP[0], 2225) + t.Expect(err).To(HaveOccurred()) + + log.Infof("Testing that the original port-fwd (2223) forwards to app1 again") + out, err = hairpinProbe(app2UUID, eth0IP[0], 2223) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + + // Phase 3: cross-adapter hairpin -- app1 (ni1/ethernet0) reaches app3 + // (ni2/ethernet1) via ethernet1's own external IP. + log.Infof("Testing cross-adapter hairpin: app1 (ethernet0) -> app3's port-fwd (ethernet1)") + out, err = hairpinProbe(app1UUID, eth1IP[0], 2226) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(HavePrefix("SSH-")) + evetest.Checkpoint("cross-adapter-hairpin-verified") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteApplication(app3UUID) + devConfig.DeleteNetworkInstance(ni1UUID) + devConfig.DeleteNetworkInstance(ni2UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app3Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app3 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + stopApp2Watch() + stopApp3Watch() + + t.Eventually(ni1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni1 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(ni2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "ni2 is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopNI1Watch() + stopNI2Watch() +} + +// TestAirGapSwitchNI verifies that EVE can bridge applications over an +// air-gapped (portless) Switch Network Instance and correctly learns, via +// passive packet snooping (there is no internal DHCP server nor an external +// one here -- the NI has no port at all), the (statically assigned) IP address(es) +// each application VIF is using. See APP-CONNECTIVITY.md, "Switch Network +// Instance" / "IP address detection". +// +// This test additionally covers detecting *multiple* IP addresses recorded +// under the same VIF MAC address, by assigning a second IP directly on the +// same air-gapped interface (no new sub-interface). +// +// Note: EVE is also documented to detect multiple IPs per MAC when an +// application places VLAN sub-interfaces on top of its VIF (they share the +// parent interface's MAC) -- but that path was confirmed NOT to work on a +// Switch NI without VLAN-aware bridge config: the switch-NI ARP-snooping BPF +// filter in pkg/pillar/nistate/linux_flow.go (sniffDNSandDHCP) is a +// hand-maintained raw BPF program compiled from a filter string that never +// accounts for an 802.1Q tag, so tagged ARP frames are dropped by the kernel +// filter before EVE's collector ever sees them (verified with tcpdump on the +// EVE bridge: the tagged ARP requests do arrive, EVE just never reports +// them). Left as a known finding rather than exercised here. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port (ethernet0), used +// only to give each app an SSH-reachable VIF; the actual scenario under +// test runs entirely over the second, air-gapped VIF. +// +// Phases +// ------ +// 1. Config: a Local NI "local-ni" on ethernet0 (10.11.10.0/24, for SSH +// reachability only) and a Switch NI "switch-ni" with Port="" (air-gapped). +// Two container apps (lfedge/evetest-ubuntu-ctr:1.0, VirtualizationMode= +// HVM, EnforceNetIntfOrder=true so vif0=eth0/vif1=eth1 deterministically): +// "app1" and "app2", each with vif0 on "local-ni" (port-fwd +// 2223/2224->22) and vif1 on "switch-ni" (fixed MAC, ICMP allow-all ACL -- +// Switch NI ACLs are still enforced even though the NI itself is L2-only). +// WaitUntilAppIsRunning for both. +// 2. Static IP assignment + real traffic: from inside each app, +// `ip addr add /24 dev eth1` assigns a static IP on the air-gapped +// VIF (11.12.13.11 for app1, .12 for app2) -- no DHCP or +// controller involvement, exactly as APP-CONNECTIVITY.md describes for +// Switch NIs. `ping -c 3 -W 1 -I eth1 ` between the two +// confirms real L2 connectivity over the bridge and generates the ARP +// traffic EVE needs to snoop. +// 3. IP detection: NetworkInstanceInfo for "switch-ni" eventually reports an +// IpAssignment for each app's MAC address containing the IP it just +// configured -- proving EVE learned both addresses purely from ARP +// snooping (no DHCP was ever involved on this NI). +// 4. Multiple IPs per MAC: on app2 only, two more static IPs are added +// directly on eth1 (`ip addr add /24 dev eth1`, same interface, same +// MAC, no sub-interface). A best-effort ping from each new address (the +// target need not answer -- an ARP request alone is enough for EVE to +// learn the Sender IP+MAC pair) generates the ARP traffic. +// NetworkInstanceInfo eventually reports app2's single IpAssignment +// (still keyed by the one shared VIF MAC) now listing all three IP +// addresses. +// 5. Cleanup: delete both apps, then both NIs, waiting for each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestAirGapSwitchNI(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + localNIUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.10.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.10.2"), + End: evetest.IPAddress("10.11.10.254"), + }, + Gateway: evetest.IPAddress("10.11.10.1"), + MTU: 1500, + }) + switchNIUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "switch-ni", + Port: "", // air-gapped + MTU: 1500, + }) + localNIUpdates, stopLocalNIWatch := device.WatchNetworkInstanceInfo(localNIUUID) + switchNIUpdates, stopSwitchNIWatch := device.WatchNetworkInstanceInfo(switchNIUUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(localNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "local-ni NI is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("nis-created") + + icmpAllowAny := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + const ( + app1MAC = "02:16:3e:00:02:01" + app2MAC = "02:16:3e:00:02:02" + ) + newApp := func(displayName, mac string, + edgeNodePort uint16) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnforceNetIntfOrder: true, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: localNIUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: edgeNodePort, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif1", + NetworkInstanceUUID: switchNIUUID, + MAC: evetest.MACAddress(mac), + ACLAllowRules: icmpAllowAny, + }, + }, + } + } + app1UUID := devConfig.AddApplication(newApp("app1", app1MAC, 2223)) + app2UUID := devConfig.AddApplication(newApp("app2", app2MAC, 2224)) + + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + device.ApplyConfig(devConfig, false, false) + + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + device.WaitUntilAppIsRunning(app2UUID, timeoutExcludingDownload) + evetest.Checkpoint("apps-running") + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + waitSSHReady := func(appUUID uuid.UUID) { + t.Eventually(func(t Gomega) { + log.Infof("Waiting for app %s SSH daemon to become reachable...", appUUID) + _, _, err := device.RunShellScriptInsideApp( + appUUID, appAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + } + waitSSHReady(app1UUID) + waitSSHReady(app2UUID) + + const ( + app1IP = "11.12.13.11" + app2IP = "11.12.13.12" + ) + + log.Infof("Assigning static IP %s to app1's air-gapped VIF", app1IP) + _, _, err := device.RunShellScriptInsideApp(app1UUID, appAuth, + fmt.Sprintf("ip addr add %s/24 dev eth1 && ip link set eth1 up", app1IP), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + log.Infof("Assigning static IP %s to app2's air-gapped VIF", app2IP) + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf("ip addr add %s/24 dev eth1 && ip link set eth1 up", app2IP), + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + evetest.Checkpoint("static-ips-assigned") + + log.Infof("Testing connectivity over the air-gapped switch NI") + _, _, err = device.RunShellScriptInsideApp(app1UUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 -I eth1 %s", app2IP), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf("ping -c 3 -W 1 -I eth1 %s", app1IP), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + // EVE learns statically-assigned IPs on a Switch NI purely from ARP + // snooping (see APP-CONNECTIVITY.md, "IP address detection"). The pings + // above generated the necessary ARP traffic. + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI reports both static IP assignments", + func(info *eveinfo.ZInfoNetworkInstance) bool { + var app1Found, app2Found bool + for _, a := range info.IpAssignments { + switch a.MacAddress { + case app1MAC: + app1Found = generics.ContainsItem(a.IpAddress, app1IP) + case app2MAC: + app2Found = generics.ContainsItem(a.IpAddress, app2IP) + } + } + return app1Found && app2Found + }).StopIf(niHasError))) + evetest.Checkpoint("static-ips-detected") + + // Phase 4: multiple IPs under the same MAC. Two more static IPs are + // added directly on eth1 -- same interface, same MAC, no sub-interface + // (see the doc comment above for why VLAN sub-interfaces are not used + // here). + const ( + extraIP1 = "11.12.16.12" + extraIP2 = "11.12.17.12" + ) + log.Infof("Adding two more static IPs on app2's air-gapped VIF") + _, _, err = device.RunShellScriptInsideApp(app2UUID, appAuth, + fmt.Sprintf(`set -e +ip addr add %s/24 dev eth1 +ip addr add %s/24 dev eth1 +set +e +# Best-effort: the targets below need not respond, an outbound ARP request +# alone is enough for EVE to learn the Sender IP+MAC assignment. +ping -c 3 -W 1 -I eth1 11.12.16.99 +ping -c 3 -W 1 -I eth1 11.12.17.99 +true +`, extraIP1, extraIP2), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("extra-ips-added") + + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI reports all three IPs for app2's MAC", + func(info *eveinfo.ZInfoNetworkInstance) bool { + for _, a := range info.IpAssignments { + if a.MacAddress != app2MAC { + continue + } + return generics.ContainsItem(a.IpAddress, app2IP) && + generics.ContainsItem(a.IpAddress, extraIP1) && + generics.ContainsItem(a.IpAddress, extraIP2) + } + return false + }).StopIf(niHasError))) + evetest.Checkpoint("multiple-ips-per-mac-detected") + + // Cleanup. + devConfig.DeleteApplication(app1UUID) + devConfig.DeleteApplication(app2UUID) + devConfig.DeleteNetworkInstance(localNIUUID) + devConfig.DeleteNetworkInstance(switchNIUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + stopApp2Watch() + + t.Eventually(localNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "local-ni NI is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + t.Eventually(switchNIUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "switch-ni NI is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopLocalNIWatch() + stopSwitchNIWatch() +} + +// TestLimitedIPSpace verifies two related IP-allocation edge cases on a +// Local Network Instance whose DHCP range holds only a single free address: +// +// 1. An application deployed onto a NI with no free IP left must fail +// (report an error), not be silently skipped or crash other apps, and +// must be automatically redeployed the moment an IP frees up (i.e. once +// the other application holding it is deleted) -- with no config +// re-apply needed beyond the one that removed the IP-holding app. +// 2. Replacing one application with another *within a single config +// apply* (same NI, competing for the same single free IP) must work: +// EVE has to tear down the obsolete app instance before bringing up the +// replacement, otherwise the replacement would itself fail to get an IP +// (exactly the failure mode from point 1, self-inflicted). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port. +// +// Phases +// ------ +// 1. Create a Local NI ("limited-ni") on ethernet0 with subnet +// 10.11.12.0/30. One address is the bridge/gateway (.1) and the DHCP +// range is deliberately restricted to the single remaining host address, +// 10.11.12.2 (Start == End) -- exactly one IP is available for +// applications at any given time. Wait for ONLINE. +// 2. Deploy "app1" on limited-ni (container, no static IP, allow-all ACL). +// It takes the only free IP. WaitUntilAppIsRunning. +// 3. Deploy "app2" on the same NI. There is no free IP left, so zedrouter's +// IPAM must report the allocation failure as an AppErr with a +// description containing "no free IP addresses in DHCP range" -- +// the app stays in whatever SwState it reached (INSTALLED here; this is +// a retryable/pending condition, not a terminal one, so EVE does not +// move the app to ZSwState_ERROR for it). The app2 watch is opened +// *before* applying this config change, since WaitUntilAppIsRunning +// must not be used here -- it treats any ZSwState_ERROR as fatal, and +// more importantly would never observe this particular failure since +// the app's SwState never becomes ZSwState_ERROR. +// 4. Delete app1 (freeing its IP) and re-apply. app1's watch confirms it +// reaches ZSwState_INVALID. Without any further app2-specific config +// change, app2 must automatically pick up the now-free IP and reach +// RUNNING -- proving EVE retries pending IP allocations as soon as one +// becomes available, not just on the next explicit config edit for that +// app. +// 5. Replace app2 with a new application instance within a single config +// apply: DeleteApplication(app2) and AddApplication(same DisplayName, +// same NI) are both applied to the EdgeDeviceConfig before the next +// ApplyConfig call. The replacement gets a fresh UUID with no prior history, +// so WaitUntilAppIsRunning is safe to use and confirms it reaches +// RUNNING -- proving EVE tears down the obsolete instance before deploying +// the replacement, rather than attempting both at once and failing to +// allocate a second IP. +// 6. Cleanup: delete the replacement app instance and the NI, waiting for +// each to be gone. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. +func TestLimitedIPSpace(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "limited-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.12.0/30"), + DHCPRange: types.IPRange{ + // Only one host address is available for allocation: .0 is the + // network address, .1 is taken by the bridge, .3 is the + // broadcast address. + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.2"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + niUpdates, stopNIWatch := device.WatchNetworkInstanceInfo(niUUID) + device.ApplyConfig(devConfig, false, false) + + timeout := 3 * time.Minute + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "limited-ni is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }))) + evetest.Checkpoint("ni-created") + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + newApp := func(displayName string) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + ACLAllowRules: allowAll, + }, + }, + } + } + + // Step 2: app1 takes the only free IP. + app1UUID := devConfig.AddApplication(newApp("app1")) + app1Updates, stopApp1Watch := device.WatchAppInfo(app1UUID) + device.ApplyConfig(devConfig, false, false) + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(app1UUID, timeoutExcludingDownload) + evetest.Checkpoint("app1-running") + + // Step 3: app2 has no free IP to allocate. This surfaces as a persistent + // AppErr on the app while it remains in whatever SwState it reached + // (INSTALLED here, since the image/volume are fine -- only network + // activation is blocked) -- it is a retryable/pending condition, not a + // terminal one, so EVE does not move the app to ZSwState_ERROR for it. + // The watch is opened before applying, and WaitUntilAppIsRunning is + // deliberately not used here -- the no-free-IP AppErr is the expected + // outcome, not a fatal condition. + app2UUID := devConfig.AddApplication(newApp("app2")) + app2Updates, stopApp2Watch := device.WatchAppInfo(app2UUID) + device.ApplyConfig(devConfig, false, false) + + const expectedErrMsg = "no free IP addresses in DHCP range" + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 fails with a no-free-IP error", + func(info *eveinfo.ZInfoApp) bool { + for _, appErr := range info.AppErr { + if strings.Contains(appErr.GetDescription(), expectedErrMsg) { + return true + } + } + return false + }))) + evetest.Checkpoint("app2-out-of-ip-error") + + // Step 4: delete app1, freeing its IP. app2 must automatically pick it + // up and reach RUNNING, without any app2-specific config change. + devConfig.DeleteApplication(app1UUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(app1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app1 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopApp1Watch() + + t.Eventually(app2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app2 is RUNNING", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_RUNNING + }).StopIf(appHasError))) + stopApp2Watch() + evetest.Checkpoint("app2-recovered") + + // Step 5: replace app2 with a new application instance within a single + // config apply -- delete the old one and add the replacement before the + // next ApplyConfig call. + devConfig.DeleteApplication(app2UUID) + newApp2UUID := devConfig.AddApplication(newApp("app2")) + newApp2Updates, stopNewApp2Watch := device.WatchAppInfo(newApp2UUID) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(newApp2UUID, timeoutExcludingDownload) + evetest.Checkpoint("app2-replaced") + + // Cleanup. + devConfig.DeleteApplication(newApp2UUID) + devConfig.DeleteNetworkInstance(niUUID) + device.ApplyConfig(devConfig, false, false) + + t.Eventually(newApp2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "replacement app2 is gone", func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + stopNewApp2Watch() + + t.Eventually(niUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "limited-ni is gone", func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED + }).StopIf(niHasError))) + stopNIWatch() +} + +// TestFlowLog verifies that EVE produces flow log records and DNS request log +// records for application traffic when flow logging is enabled on a Local +// Network Instance, and that those records correctly attribute flows to ACE +// IDs (allowed flows -> matching ACE; dropped flows -> ACE id 0, the implicit +// reject-all). +// +// ACE numbering: EVE assigns ACE ids sequentially in the order rules are +// listed on the VIF, starting at 1 (id 0 is reserved for the implicit +// reject-all) -- see EdgeDeviceConfig's ACL-building code in devconfig.go. +// The app's single VIF below lists, in order: the port-fwd rule (-> ACE 1), +// the HTTP allow rule (-> ACE 2), the ICMP allow rule (-> ACE 3). +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- already has a second HTTP endpoint, +// http-server2.test:8080, used here as the target that is NOT covered +// by any allow rule (so it falls through to the implicit reject-all). +// +// Device configuration +// -------------------- +// - ethernet0 (mgmt+app, DHCP). +// - One Local NI ("flow-ni") with EnableFlowlog=true. +// - One container app on the NI with one VIF and three ACEs: port-fwd +// 2222->22 (ACE 1), allow TCP:80 to http-server.test (ACE 2), allow ICMP +// to the NI's own gateway IP (ACE 3). Everything else (in particular +// http-server2.test:8080) falls through to the implicit reject-all +// (ACE 0). +// +// Phases +// ------ +// 1. Generate distinguishable traffic from inside the app: curl +// http-server.test/helloworld (allowed, ACE 2); curl --max-time 5 +// http-server2.test:8080/helloworld (must fail, ACE 0); ping -c 3 against +// the NI gateway IP (allowed, ACE 3); nslookup http-server.test (DNS +// request log entry). +// 2. Flow log assertions (GetAppFlowLogs, NotBefore the traffic above): EVE +// tracks one flow record per connection (conntrack-based), not one per +// packet direction -- Flow.Src is always the app's own VIF IP and +// Flow.Dest the remote endpoint regardless of direction, so a flow's +// Tx/Rx counters together cover the whole connection. Eventually there is +// an outbound (app-initiated) record matching each of: +// - ACE 2: dst httpServerIP:80/TCP, with TxPkts and RxPkts both > 0 +// (request sent, response received). +// - ACE 3: dst NI gateway/ICMP, with TxPkts and RxPkts both > 0 (ping +// requests sent, replies received). +// - ACE 0 (implicit reject-all): dst httpServer2IP:8080/TCP, with +// TxPkts > 0 (the app did send it) but RxPkts == 0 (blackholed -- +// see APP-CONNECTIVITY.md -- so nothing ever came back). +// Plus one inbound (externally-initiated) record for ACE 1, the +// port-forwarded SSH session already in use to drive the app (src port +// 22 on the app side), with both Tx and Rx counters > 0. +// 3. DNS log assertions (GetAppDNSLogs): eventually there is a DNSRequest +// record for hostname "http-server.test" whose resolved address matches +// the SDN http-server endpoint's IP, with a request time within the +// test window. +// 4. Flow logging disabled: UpdateNetworkInstance sets EnableFlowlog=false; +// re-apply. Fresh traffic is generated, then GetAppFlowLogs (NotBefore +// the reapply instant) consistently returns empty over the reporting +// interval, confirming the runtime toggle actually stops new records +// from being produced (not just that old ones happen to still be there). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestFlowLog(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + // ACE ids, per the numbering scheme documented above. + const ( + portFwdAceID = int32(1) + allowHTTPAceID = int32(2) + allowICMPAceID = int32(3) + denyAceID = int32(0) + ) + // IANA IP protocol numbers, as reported in FlowRecord.Flow.Protocol. + const ( + ipProtoICMP = int32(1) + ipProtoTCP = int32(6) + ) + // SDN endpoint IPs, as defined in netmodels.SingleEthWithDHCP. + const ( + httpServerIP = "10.17.17.25" // http-server.test:80 (allowed) + httpServer2IP = "10.18.18.25" // http-server2.test:8080 (not allowed -> denied) + ) + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + eth0Net := devConfig.AddNetwork( + evetest.DHCPNetworkConfig{NetworkType: evecommon.NetworkType_V4Only}) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + const niGatewayIP = "10.50.0.1" + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "flow-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress(niGatewayIP), + EnableFlowlog: true, + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "flowlog-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolTCP, + RemoteHostname: "http-server.test", + RemotePort: 80, + }, + { + Protocol: evetest.NetworkProtocolICMP, + RemoteSubnet: evetest.IPSubnet(niGatewayIP + "/32"), + }, + }, + }, + }, + }) + + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("app-running") + + log := evetest.Logger() + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + + t.Eventually(func(g Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "hostname", sshTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(appUUID.String())) + }, 3*time.Minute, polling).Should(Succeed()) + + // Phase 1: generate distinguishable traffic. + log.Infof("Phase 1: generating traffic for each ACE...") + trafficStart := time.Now() + + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("Hello world!")) + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "curl --max-time 5 http://http-server2.test:8080/helloworld", 10*time.Second, 0) + t.Expect(err).To(HaveOccurred(), + "http-server2.test is not covered by any allow ACE and must be blocked") + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "ping -c 3 "+niGatewayIP, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "nslookup http-server.test", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("phase1-traffic-generated") + + // Phase 2: flow log assertions. + // + // EVE tracks one flow record per connection (conntrack-based), not one + // per packet direction: FlowRecord.Inbound says who *initiated* the + // connection, while Flow.Src is always the app's own VIF IP and Flow.Dest + // the remote endpoint, regardless of Inbound -- see + // protoEncodeAppFlowMonitorProto / nistate/linux_flow.go. So the + // app-initiated flows (HTTP, ICMP, the denied HTTP) are all Inbound=false + // with a single record each (already carrying both Tx and Rx counters for + // the whole connection); the port-forwarded SSH session is the only + // Inbound=true flow here, since it was initiated from outside the app. + log.Infof("Phase 2: waiting for flow log records to be reported...") + appInfo := device.GetAppInfo(appUUID) + t.Expect(appInfo.GetNetwork()).To(HaveLen(1)) + t.Expect(appInfo.GetNetwork()[0].GetIPAddrs()).ToNot(BeEmpty()) + appIP := appInfo.GetNetwork()[0].GetIPAddrs()[0] + + // pkg/pillar/nistate/linux.go's flowCollectInterval (~108-120s, randomized, + // hardcoded -- no controller-config override) governs how often the + // conntrack table is swept and a connection's flow record published, so + // give it comfortable room for at least one full cycle plus margin. + flowLogTimeout := 3 * time.Minute + t.Eventually(func(g Gomega) { + outbound := device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: false, + NotBefore: trafficStart, + }) + + httpRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: allowHTTPAceID, srcIP: appIP, dstIP: httpServerIP, + dstPort: 80, protocol: ipProtoTCP, + }) + g.Expect(httpRec).ToNot(BeNil(), + "expected an outbound flow record for the allowed HTTP ACE (%d) "+ + "from %s to %s:80", allowHTTPAceID, appIP, httpServerIP) + g.Expect(httpRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent the HTTP request") + g.Expect(httpRec.GetRxPkts()).To(BeNumerically(">", 0), + "app must have received the HTTP response") + + icmpRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: allowICMPAceID, srcIP: appIP, dstIP: niGatewayIP, protocol: ipProtoICMP, + }) + g.Expect(icmpRec).ToNot(BeNil(), + "expected an outbound flow record for the allowed ICMP ACE (%d) "+ + "from %s to %s", allowICMPAceID, appIP, niGatewayIP) + g.Expect(icmpRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent ping requests") + g.Expect(icmpRec.GetRxPkts()).To(BeNumerically(">", 0), + "app must have received ping replies") + + deniedRec := findFlowRecord(outbound, flowRecordMatch{ + aclID: denyAceID, srcIP: appIP, dstIP: httpServer2IP, + dstPort: 8080, protocol: ipProtoTCP, + }) + g.Expect(deniedRec).ToNot(BeNil(), + "expected an outbound flow record for the denied traffic "+ + "(implicit reject-all, ACE %d) from %s to %s:8080 -- EVE logs "+ + "the flow even though the packet was blackholed", + denyAceID, appIP, httpServer2IP) + g.Expect(deniedRec.GetTxPkts()).To(BeNumerically(">", 0), + "app must have sent connection attempts") + g.Expect(deniedRec.GetRxPkts()).To(BeNumerically("==", 0), + "no reply should ever come back for a blackholed connection") + + inbound := device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: true, + NotBefore: trafficStart, + }) + sshRec := findFlowRecord(inbound, flowRecordMatch{ + aclID: portFwdAceID, inbound: true, srcIP: appIP, protocol: ipProtoTCP, + }) + g.Expect(sshRec).ToNot(BeNil(), + "expected an inbound flow record for the port-forwarded SSH "+ + "session (ACE %d) to app %s", portFwdAceID, appIP) + g.Expect(sshRec.GetFlow().GetSrcPort()).To(Equal(int32(22)), + "the app's own listening port for the port-forwarded connection must be 22") + g.Expect(sshRec.GetTxPkts()).To(BeNumerically(">", 0)) + g.Expect(sshRec.GetRxPkts()).To(BeNumerically(">", 0)) + }, flowLogTimeout, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("phase2-flow-logs-verified") + + // Phase 3: DNS log assertions. + log.Infof("Phase 3: waiting for the DNS request log record...") + t.Eventually(func(g Gomega) { + dnsLogs := device.GetAppDNSLogs(appUUID, evetest.DNSLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + NotBefore: trafficStart, + }) + req := findDNSRequest(dnsLogs, "http-server.test") + g.Expect(req).ToNot(BeNil(), + "expected a DNS request log record for http-server.test") + g.Expect(req.GetAddrs()).To(ContainElement(httpServerIP), + "resolved address must match the SDN http-server endpoint") + }, flowLogTimeout, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("phase3-dns-logs-verified") + + // Phase 4: disable flow logging and confirm no new records appear. + log.Infof("Phase 4: disabling flow logging and generating fresh traffic...") + devConfig.UpdateNetworkInstance(niUUID, evetest.LocalNetworkInstanceConfig{ + DisplayName: "flow-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress(niGatewayIP), + EnableFlowlog: false, + MTU: 1500, + }) + device.ApplyConfig(devConfig, false, false) + reapplyTime := time.Now() + + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS --max-time 10 http://http-server.test/helloworld", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Checkpoint("phase4-flowlog-disabled") + + t.Consistently(func() []*eveflowlog.FlowRecord { + return device.GetAppFlowLogs(appUUID, evetest.FlowLogMatch{ + VirtualNetAdapter: "vif0", + NetworkInstance: niUUID, + Inbound: false, + NotBefore: reapplyTime, + }) + }, 3*time.Minute, 10*time.Second).Should(BeEmpty(), + "no new flow records should be produced after EnableFlowlog is set to false") + evetest.Checkpoint("phase4-complete") +} + +// flowRecordMatch describes the identifying criteria for a single flow log +// record. aclID and inbound are always checked; srcIP, dstIP, dstPort and +// protocol are only checked when non-zero (dstPort and protocol are left +// unset for ICMP, which has neither a meaningful port nor is worth +// double-checking the protocol number for here). +type flowRecordMatch struct { + aclID int32 + inbound bool + srcIP string // Flow.Src is always the app's own VIF IP, regardless of direction. + dstIP string // Flow.Dest is always the remote endpoint. + dstPort int32 + protocol int32 +} + +// findFlowRecord returns the first record in records matching every +// non-zero-valued field of want, or nil if there is none. +func findFlowRecord(records []*eveflowlog.FlowRecord, want flowRecordMatch) *eveflowlog.FlowRecord { + for _, rec := range records { + if rec.GetAclId() != want.aclID || rec.GetInbound() != want.inbound { + continue + } + flow := rec.GetFlow() + if want.srcIP != "" && flow.GetSrc() != want.srcIP { + continue + } + if want.dstIP != "" && flow.GetDest() != want.dstIP { + continue + } + if want.dstPort != 0 && flow.GetDestPort() != want.dstPort { + continue + } + if want.protocol != 0 && flow.GetProtocol() != want.protocol { + continue + } + return rec + } + return nil +} + +// findDNSRequest returns the first record in records with the given +// hostname, or nil if there is none. +func findDNSRequest(records []*eveflowlog.DnsRequest, hostname string) *eveflowlog.DnsRequest { + for _, rec := range records { + if rec.GetHostName() == hostname { + return rec + } + } + return nil } func niHasError(info *eveinfo.ZInfoNetworkInstance) (string, bool) { diff --git a/evetest/tests/networking/ntp_test.go b/evetest/tests/networking/ntp_test.go index eb2260ab89a..d5dbe5a5f41 100644 --- a/evetest/tests/networking/ntp_test.go +++ b/evetest/tests/networking/ntp_test.go @@ -3,171 +3,464 @@ package networking_test -import "testing" +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/lf-edge/eve/pkg/pillar/utils/generics" +) // TestDeviceNTPConfig verifies how EVE assembles per-port NTP server lists -// from DHCP and statically-configured sources, that chronyd actually -// synchronizes against the configured peers, and that the configuration -// surfaces correctly in published EVE state. +// from DHCP and statically-configured sources, that a per-port exclusive +// override correctly discards the DHCP-provided entry, and that chronyd +// actually synchronizes against the resulting device-wide server set. // // Scope: device side only. Propagation of NTP servers to applications via -// DHCP option 42 is covered separately by TestApplicationNTPConfig (which -// lives in TestApplicationConnectivitySuite and parameterizes the hypervisor). +// DHCP option 42 is covered separately by TestApplicationNTPConfig. // -// SDN framework prerequisite (NOT yet implemented) -// ------------------------------------------------ -// SDN exposes the api.NTPServer endpoint type in grpcapi/proto/sdn.proto -// and the sdnagent parses/validates it (parse.go around the GetNtpServers -// loop), but it does NOT actually deploy an NTP daemon: there is no -// `ntpSrv.go` config item under evetest/sdn/vm/pkg/configitems (compare -// with the existing dnsSrv.go / httpSrv.go / scepSrv.go). The DHCP server -// (dhcpSrv.go) is wired to ADVERTISE an NTP server IP via DHCP option 42 -// / 56, but nothing actually listens on that IP. Before this test can be -// implemented, the SDN side needs an "NTP server endpoint" config item -// that runs a real NTP daemon (chronyd, ntpd, or a small in-process -// stratum-1-like responder) bound to the endpoint's IP. The scenario -// below assumes that work is complete. +// NTP servers used +// ----------------- +// Real, long-stable, single-IP public NTP servers rather than an SDN-hosted +// fake one, so the test needs no SDN-side NTP daemon and can still assert on +// exact addresses (unlike pool.ntp.org-style rotating addresses): +// - 162.159.200.1 (Cloudflare's primary anycast NTP address) -- advertised +// via DHCP option 42 on eth0 (see netmodels.TwoMgmtPortsWithPublicNTP). +// - 216.239.35.0 (Google's time1.google.com address) -- advertised via +// DHCP option 42 on eth1; must be excluded (see below). +// - 162.159.200.123 (Cloudflare's secondary anycast NTP address) -- +// statically configured on both ports (device side). // // Network model // ------------- -// Add netmodel `MultiPortWithNTPServers` (proposed: evetest/netmodels/multi-eth.go) -// extending TwoMgmtPorts with three SDN NTPServer endpoints: -// -// eth0 -- DHCP, DHCP advertises NTP server "ntp0" (private endpoint inside SDN) -// eth1 -- DHCP, DHCP advertises NTP server "ntp1" (different SDN endpoint) -// + "ntp-static" reachable via SDN routing for the static-override case. -// -// All NTP endpoints (api.NTPServer in evetest/grpcapi/proto/sdn.proto) run -// inside SDN and answer NTP queries with a stable clock. They have distinct -// IPs so the test can tell which sources EVE actually uses. +// - netmodels.TwoMgmtPortsWithPublicNTP -- two management ports, each +// advertising a different public NTP server via DHCP option 42. // // Device configuration // -------------------- -// - SystemAdapter for eth0 (DHCP, mgmt) and eth1 (DHCP, mgmt). -// - On eth0, also add a STATIC NTP server in NetworkConfig.NTPServers, e.g. -// "static-ntp.test" (resolved via SDN DNS to the SDN NTPServer endpoint -// "ntp-static"). Leave IgnoreNTPFromDHCP=false so it is appended to -// DHCP-provided servers. -// - On eth1, set IgnoreNTPFromDHCP=true and add a different static entry -// ("static-ntp-2.test" -> NTPServer "ntp-static-2"). This exercises the -// DhcpOptionsIgnore.ntp_server_exclusively code path: only ntp-static-2 -// should appear for eth1, NOT ntp1. -// - Hypervisor: hardcode WithHypervisor=HypervisorKVM in RequireEdgeDevice. -// Device-suite tests do not parameterize the hypervisor (the device-level -// plumbing under test does not depend on app virtualization). -// -// Assertions -// ---------- -// - WatchDeviceInfo / DevicePortStatus.ntpServer + more_ntp_servers: -// - eth0 reports BOTH ntp0 (DHCP) and ntp-static (static append). -// - eth1 reports ONLY ntp-static-2 (the DHCP-supplied ntp1 must be -// filtered out by the exclusive flag). -// - WatchNTPSources (uses the ZInfoNTPSources publication; see -// edgedevice.go GetNTPSources / WatchNTPSources and -// proto/info/ntpsources.proto): -// - Eventually at least one source reaches state SYNC ('*'). -// - The set of source addresses contains every configured NTP IP EVE -// should be using (ntp0, ntp-static, ntp-static-2). ntp1 must NOT -// appear (it was excluded by the exclusive flag). -// - Each tracked source has mode=CLIENT and reachability ramps up -// (consistently within a few minutes). -// - SSH check (acceptable -- chronyd is the canonical timekeeping -// daemon on EVE per CLOCK-SYNCHRONIZATION.md): `chronyc tracking` -// reports a non-zero Reference ID matching one of the configured SDN -// NTPServer IPs. +// - ethernet0 (eth0, mgmt, cost=0): DHCP with the static server appended +// (IgnoreNTPFromDHCP=false). Effective set: {DHCP-provided Cloudflare +// primary, static Cloudflare secondary}. +// - ethernet1 (eth1, mgmt, cost=1): DHCP with IgnoreNTPFromDHCP=true and +// the same static server. The DHCP-provided Google address must be +// discarded. Effective set: {static Cloudflare secondary} only. // -// Runtime update -// -------------- -// - UpdateNetworkAdapter to remove ntp-static from eth0; re-apply. -// - Eventually: -// - DevicePort NTP list shrinks accordingly. -// - WatchNTPSources stops including ntp-static (state may transition -// to UNREACH first; the eventual condition is "source disappears -// or stops being used"). +// Phases +// ------ +// 1. Per-port NTP state: waits (WatchDeviceInfo) until both ports have +// acquired DHCP addresses and DevicePort.ntpServer + more_ntp_servers +// match the expected sets exactly (set equality, catching both missing +// entries and the exclusive-override leaking the DHCP entry through). +// 2. Device-wide chrony synchronization: chronyd has a single global source +// list built from the deduplicated union of every port's NTP servers +// (see pkg/pillar/scripts/device-steps.sh, get_ntp_servers_from_nim), so +// the Google address excluded on eth1 must never reach chronyd either. +// WatchNTPSources (ZInfoNTPSources) eventually reports at least one +// source in SYNC state and a source-address set equal to exactly the two +// Cloudflare addresses. +// 3. SSH cross-check: `eve exec pillar chronyc sources` (chronyd runs inside +// the pillar service container) shows at least one of the two Cloudflare +// addresses marked as the current sync source ('*'). // -// Notes -// ----- -// - The test does not depend on real-world Internet NTP (pool.ntp.org). -// All NTP traffic stays inside SDN. -// - DHCPNetworkConfig already exposes NTPServers + IgnoreNTPFromDHCP and -// the evecommon.DhcpOptionsIgnore plumbing -- no framework additions -// are needed. +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestDeviceNTPConfig(test *testing.T) { - test.Skip("not yet implemented") + const ( + ntpCloudflarePrimary = "162.159.200.1" // DHCP-advertised on eth0 + ntpCloudflareSecondary = "162.159.200.123" // statically configured on both ports + ) + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPortsWithPublicNTP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + net0 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: net0, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + Cost: 0, + }) + + net1 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + IgnoreNTPFromDHCP: true, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: net1, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + Cost: 1, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + + // Phase 1: per-port NTP state. + log.Infof("Phase 1: waiting for per-port NTP state to settle...") + phase1Timeout := 3 * time.Minute + t.Eventually(devUpdates, phase1Timeout).Should(Receive(matchers.SatisfyPredicate( + "Both ports have expected NTP servers", + func(dinfo *eveinfo.ZInfoDevice) bool { + eth0 := getDevicePort("ethernet0", dinfo) + eth1 := getDevicePort("ethernet1", dinfo) + if eth0 == nil || eth1 == nil { + return false + } + if len(eth0.GetIPAddrs()) == 0 || len(eth1.GetIPAddrs()) == 0 { + return false + } + // eth1's expected set has only the static entry, so this already + // requires the DHCP-provided Google address to be absent (a set + // containing it, or an empty set, would both fail EqualSets here). + return generics.EqualSets(portNtpServers(eth0), + []string{ntpCloudflarePrimary, ntpCloudflareSecondary}) && + generics.EqualSets(portNtpServers(eth1), []string{ntpCloudflareSecondary}) + }))) + evetest.Checkpoint("phase1-complete") + + // Phase 2: device-wide chrony synchronization. + log.Infof("Phase 2: waiting for chronyd to synchronize with the expected NTP servers...") + expectedSources := []string{ntpCloudflarePrimary, ntpCloudflareSecondary} + ntpUpdates, stopNTPWatch := device.WatchNTPSources() + defer stopNTPWatch() + if ntpSourcesSynced(device.GetNTPSources(), expectedSources) { + log.Infof("chronyd is already synchronized with the expected NTP servers") + } else { + phase2Timeout := 5 * time.Minute + t.Eventually(ntpUpdates, phase2Timeout).Should(Receive(matchers.SatisfyPredicate( + "chronyd synchronized with exactly the expected NTP servers", + func(sources *eveinfo.ZInfoNTPSources) bool { + return ntpSourcesSynced(sources, expectedSources) + }))) + } + evetest.Checkpoint("phase2-complete") + + // Phase 3: SSH cross-check via chronyc (chronyd runs inside the pillar + // service container). "-n" disables reverse-DNS lookups so the address + // column shows the numeric IP we configured, not a resolved hostname. + log.Infof("Phase 3: cross-checking via chronyc sources over SSH...") + output, _, err := device.RunShellScript( + "eve exec pillar chronyc -n sources", 15*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + foundSyncedLine := false + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "^*") && + (strings.Contains(line, ntpCloudflarePrimary) || + strings.Contains(line, ntpCloudflareSecondary)) { + foundSyncedLine = true + break + } + } + t.Expect(foundSyncedLine).To(BeTrue(), + "chronyc sources must show one of the expected Cloudflare addresses "+ + "as the current sync source ('^*'):\n%s", output) + evetest.Checkpoint("phase3-complete") +} + +// portNtpServers combines DevicePort.ntpServer and more_ntp_servers into a +// single slice (or nil if the port has no NTP servers configured). +func portNtpServers(port *eveinfo.DevicePort) []string { + if port.GetNtpServer() == "" { + return nil + } + return append([]string{port.GetNtpServer()}, port.GetMoreNtpServers()...) +} + +// ntpSourcesSynced reports whether sources contains at least one source in +// SYNC state and its set of source addresses equals exactly expectedAddrs. +func ntpSourcesSynced(sources *eveinfo.ZInfoNTPSources, expectedAddrs []string) bool { + if sources == nil { + return false + } + var addrs []string + synced := false + for _, src := range sources.GetSources() { + addrs = append(addrs, src.GetDstAddr()) + if src.GetState() == eveinfo.NTPSourceState_NTP_SOURCE_STATE_SYNC { + synced = true + } + } + return synced && generics.EqualSets(addrs, expectedAddrs) } // TestApplicationNTPConfig verifies that the per-NI DHCP server propagates -// the correct NTP server list (port-NTP ∪ NI-NTP) to applications via DHCP -// option 42 / 56, and that the application's NI VIF status reflects this in -// published EVE state. +// the correct NTP server list (port-NTP union NI-NTP) to an application, and +// that the application's NI VIF status reflects this in published EVE state. // // Scope: application side only. Device-side NTP plumbing is covered by // TestDeviceNTPConfig. // -// SDN framework prerequisite (NOT yet implemented) -// ------------------------------------------------ -// Same prerequisite as TestDeviceNTPConfig: SDN must actually run an NTP -// daemon bound to each `api.NTPServer` endpoint's IP. The proto/API exists -// and the sdnagent validates the config, but there is no NTP server config -// item that actually listens on the wire (no `ntpSrv.go` under -// evetest/sdn/vm/pkg/configitems). The DHCP server can only advertise an -// NTP server IP via DHCP option 42; nothing answers on that IP today. -// In-app chronyc would therefore never see a SYNC source. This test cannot -// be implemented until that gap is filled. +// Known limitation: EVE's per-NI dnsmasq does advertise the NTP list via +// DHCP option 42 (the same mechanism that carries DNS servers), but the +// shim-VM's DHCP client script (pkg/xen-tools/initrd/udhcpc_script.sh) only +// acts on the DNS/IP/route options it receives -- it never writes the NTP +// option (udhcpc's "$ntpsrv") anywhere inside the guest. So unlike DNS +// servers (verifiable via nslookup from inside the app), there is currently +// no guest-visible effect of the DHCP-advertised NTP list to check over SSH; +// this test can only verify what EVE itself reports it configured +// (ZInfoNetwork.ntp_servers), which is exactly the per-NI DHCP server's +// computed advertisement. +// +// NTP servers used +// ----------------- +// Reuses the same real public NTP servers as TestDeviceNTPConfig, plus one +// more for the NI-level override: +// - 162.159.200.1 (Cloudflare primary) -- DHCP-advertised on eth0. +// - 162.159.200.123 (Cloudflare secondary) -- statically configured on +// eth0. +// - 216.239.35.0 (Google time1) -- DHCP-advertised on eth1 (unrelated port; +// must not leak into the app). +// - 216.239.35.4 (Google time2) -- configured on the Local NI itself. // // Network model // ------------- -// Reuse the same `MultiPortWithNTPServers` netmodel proposed for -// TestDeviceNTPConfig. Add one more SDN NTPServer endpoint "ntp-local-ni" -// that the NI itself will advertise to apps in addition to the port-level -// NTP servers. +// - netmodels.TwoMgmtPortsWithPublicNTP (same model as TestDeviceNTPConfig). // // Device configuration // -------------------- -// - SystemAdapter for eth0 (DHCP, mgmt+app, with static NTP "ntp-static" -// appended) and eth1 (DHCP, mgmt, with exclusive override -- only -// ntp-static-2 used). Same plumbing as TestDeviceNTPConfig. -// - One Local NI ("local-ni") on eth0 with its own NIConfig.NTPServers -// entry pointing to "ntp-local-ni". The NI's dnsmasq must merge port -// NTP servers and the NI NTP server and advertise the union to apps. -// - One container app on the NI (default-allow + port-fwd 2222->22). -// Use the existing lfedge/evetest-ubuntu-ctr image; it ships -// with chronyd which will sync against whatever DHCP delivers. +// - ethernet0 (eth0, mgmt+app, cost=0): DHCP with the Cloudflare secondary +// server appended. Effective port set: {Cloudflare primary (DHCP), +// Cloudflare secondary (static)}. +// - ethernet1 (eth1, mgmt only): plain DHCP, left unexcluded (Google time1 +// stays active on this port), but not part of any NI -- present purely +// to prove a port's NTP servers don't leak into an app bound to a +// different port. +// - One Local NI ("local-ni") on eth0 with its own NTPServers entry +// pointing at the Google time2 address. +// - One container app on the NI (default-allow ACL + port-fwd 2222->22), +// using the lfedge/evetest-ubuntu-ctr image. // -// Assertions -// ---------- -// - WaitUntilAppIsRunning(appUUID). -// - WatchAppInfo: the app VIF's ZInfoNetwork.ntp_servers reports the -// EXPECTED set: ntp0, ntp-static (from the port), plus ntp-local-ni -// (from the NI itself). ntp1 / ntp-static-2 must NOT appear (they -// belong to eth1, which is not used by this NI). Use set equality so -// accidental over-reporting is caught. -// - SSH inside the app: `chronyc sources` reports the same set of -// servers. Optionally `cat /etc/chrony.conf` to confirm the DHCP-driven -// configuration was applied. (chronyd-in-container is supported on the -// evetest-ubuntu-ctr image; see CLOCK-SYNCHRONIZATION.md -// "Recommendations when NTP is not available on the Guests".) -// -// Runtime update -// -------------- -// - UpdateNetworkInstance to remove ntp-local-ni from the NI's NTP list. -// To force the app to pick up the new DHCP option without redeploying, -// either DeactivateApplication + ActivateApplication to trigger a fresh -// DHCP cycle, or wait for the DHCP lease renewal. The first approach -// is faster and is what the eden tests effectively do. -// - After re-acquire: ZInfoNetwork.ntp_servers no longer includes -// ntp-local-ni; the in-app chronyc sources reflects the same change. +// Phases +// ------ +// 1. WaitUntilAppIsRunning, then wait (WatchAppInfo) until the app's VIF +// receives an IP from the NI subnet and ZInfoNetwork.ntp_servers equals +// exactly the port union NI set: {Cloudflare primary, Cloudflare +// secondary, Google time2}. Set equality (not subset) also catches +// eth1's Google time1 address leaking through, which must not happen +// since the NI doesn't use eth1. +// 2. Runtime update: removes the NI's NTPServers entry (UpdateNetworkInstance) +// and deactivates/reactivates the app to force a fresh DHCP cycle. +// Eventually ZInfoNetwork.ntp_servers drops the Google time2 address, +// leaving only the two eth0 port-level servers. // // Test params // ----------- -// - HYPERVISOR. The test must call evetest.SkipIfHypervisorKubevirt() -// after reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). // -// Notes -// ----- -// - All NTP traffic stays inside SDN; no public pool.ntp.org dependency. -// - LocalNetworkInstanceConfig already exposes NTPServers in the framework, -// so no devconfig changes are required. +// Suite placement +// --------------- +// - TestApplicationConnectivitySuite. func TestApplicationNTPConfig(test *testing.T) { - test.Skip("not yet implemented") + const ( + ntpCloudflarePrimary = "162.159.200.1" // DHCP-advertised on eth0 + ntpCloudflareSecondary = "162.159.200.123" // statically configured on eth0 + ntpGoogleTime2 = "216.239.35.4" // configured on the NI itself + ) + + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.TwoMgmtPortsWithPublicNTP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + net0 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + NTPServers: []string{ntpCloudflareSecondary}, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: net0, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + // ethernet1 is not used by the NI below; it is configured (with its own + // DHCP-advertised NTP server left active, unexcluded) purely to prove + // that a port's NTP servers do not leak into an app whose NI is bound to + // a different port. + net1 := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: net1, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtOnly, + }) + + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niSubnet := evetest.IPSubnet("10.11.12.0/24") + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: niSubnet, + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + NTPServers: []string{ntpGoogleTime2}, + MTU: 1500, + }) + + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "ntp-test-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + defer stopAppWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("config-applied") + + log := evetest.Logger() + timeout := 5 * time.Minute + + log.Infof("Waiting for the app to become running...") + device.WaitUntilAppIsRunning(appUUID, timeout) + evetest.Checkpoint("app-running") + + // Phase 1: app VIF NTP server set. + log.Infof("Phase 1: waiting for the app VIF to report the expected NTP servers...") + expectedNTPServers := []string{ntpCloudflarePrimary, ntpCloudflareSecondary, ntpGoogleTime2} + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "App VIF has an IP from the NI subnet and the expected NTP server set", + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) != 1 || len(info.Network[0].IPAddrs) == 0 { + return false + } + return generics.EqualSets(info.Network[0].GetNtpServers(), expectedNTPServers) + }).StopIf(appHasError))) + evetest.Checkpoint("phase1-complete") + + // Phase 2: runtime update -- remove the NI-level NTP server and force a + // fresh DHCP cycle by deactivating and reactivating the app. + log.Infof("Phase 2: removing the NI's NTP server and forcing a fresh DHCP cycle...") + devConfig.UpdateNetworkInstance(niUUID, evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: niSubnet, + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress("10.11.12.1"), + MTU: 1500, + }) + device.ApplyConfig(devConfig, false, false) + device.DeactivateApplication(appUUID, true, timeout) + device.ActivateApplication(appUUID, true, timeout) + + expectedNTPServersAfterUpdate := []string{ntpCloudflarePrimary, ntpCloudflareSecondary} + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "App VIF NTP servers no longer include the removed NI-level server", + func(info *eveinfo.ZInfoApp) bool { + if len(info.Network) != 1 || len(info.Network[0].IPAddrs) == 0 { + return false + } + return generics.EqualSets(info.Network[0].GetNtpServers(), expectedNTPServersAfterUpdate) + }).StopIf(appHasError))) + evetest.Checkpoint("phase2-complete") } diff --git a/evetest/tests/networking/pciback_error_test.go b/evetest/tests/networking/pciback_error_test.go index f94446431a1..b2cd68252b8 100644 --- a/evetest/tests/networking/pciback_error_test.go +++ b/evetest/tests/networking/pciback_error_test.go @@ -4,7 +4,6 @@ package networking_test import ( - "encoding/json" "strings" "testing" "time" @@ -17,6 +16,7 @@ import ( "github.com/lf-edge/eve/evetest" "github.com/lf-edge/eve/evetest/matchers" "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" ) // This suite verifies that device-model inconsistencies in the assignable-I/O @@ -73,23 +73,8 @@ func newBaseNetConfig(devName string) *evetest.EdgeDeviceConfig { func portPciLong(t *WithT, device *evetest.EdgeDevice, phylabel string) string { var pci string t.Eventually(func() string { - stdout, _, err := device.RunShellScript( - "eve exec pillar cat /run/domainmgr/AssignableAdapters/global.json 2>/dev/null", - 60*time.Second, 0) - if err != nil { - return "" - } - i := strings.IndexByte(stdout, '{') - if i < 0 { - return "" - } - var aa struct { - IoBundleList []struct { - Phylabel string - PciLong string - } - } - if json.Unmarshal([]byte(stdout[i:]), &aa) != nil { + var aa pillartypes.AssignableAdapters + if !evetest.ReadPublication(device, "domainmgr", false, "global", &aa) { return "" } for _, b := range aa.IoBundleList { @@ -124,10 +109,15 @@ func TestReportMissingDevice(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -146,6 +136,9 @@ func TestReportMissingDevice(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "phantom device with a non-existent PCI address is reported as an error", @@ -164,10 +157,15 @@ func TestReportParentAssigngrp(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -187,6 +185,9 @@ func TestReportParentAssigngrp(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "self-parent assignment group is reported as an error", @@ -209,10 +210,15 @@ func TestReportCycleDetected(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -240,6 +246,9 @@ func TestReportCycleDetected(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "parentassigngrp cycle is reported as an error", @@ -258,10 +267,15 @@ func TestReportCollision(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -283,6 +297,9 @@ func TestReportCollision(test *testing.T) { devUpdates, stopDevWatch := device.WatchDeviceInfo() defer stopDevWatch() device.ApplyConfig(cfg, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "USB collision is reported once as an error for the group", @@ -303,10 +320,15 @@ func TestReportIfnameMismatch(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -318,6 +340,9 @@ func TestReportIfnameMismatch(test *testing.T) { // Phase 1: correct config; let EVE resolve eth1's real PCI address. device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") // Phase 2: give eth1 a bogus model interface name but the real PCI address. @@ -340,10 +365,15 @@ func TestReportAssignmentGroupConflict(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -353,6 +383,9 @@ func TestReportAssignmentGroupConflict(test *testing.T) { defer stopDevWatch() device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") cfg := newBaseNetConfig(pcibackErrDevName) @@ -384,10 +417,15 @@ func TestReportWarningPlusError(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -397,6 +435,9 @@ func TestReportWarningPlusError(test *testing.T) { defer stopDevWatch() device.ApplyConfig(newBaseNetConfig(pcibackErrDevName), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci := portPciLong(t, device, "eth1") // eth1 gets a wrong ifname (warning) and a phantom shares its PCI in another @@ -434,10 +475,15 @@ func TestReportClearsOnFix(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.TwoMgmtPorts}, @@ -463,6 +509,9 @@ func TestReportClearsOnFix(test *testing.T) { // Break it: self-parent -> error reported. device.ApplyConfig(phantomParentConfig("grpx"), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } t.Eventually(devUpdates, pcibackReportTimeout).Should(Receive(matchers.SatisfyPredicate( "self-parent assignment group is reported as an error", func(info *eveinfo.ZInfoDevice) bool { @@ -524,10 +573,15 @@ func TestReportWarningsOnly(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + evetest.Setup( evetest.RequireEdgeDevice{ Name: pcibackErrDevName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, DeviceReusePolicy: evetest.ResetDeviceConfig, }, evetest.RequireNetworkModel{NetworkModel: netmodels.ManyDNSServers}, @@ -538,6 +592,9 @@ func TestReportWarningsOnly(test *testing.T) { // Phase 1: correct config; resolve eth1 and eth2 real PCI addresses. device.ApplyConfig(manyPortsConfig(pcibackErrDevName, "eth1", "", "eth2", ""), true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } pci1 := portPciLong(t, device, "eth1") pci2 := portPciLong(t, device, "eth2") diff --git a/evetest/tests/networking/routing_test.go b/evetest/tests/networking/routing_test.go index c838fefa218..48398ddd22c 100644 --- a/evetest/tests/networking/routing_test.go +++ b/evetest/tests/networking/routing_test.go @@ -96,8 +96,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestPropagatedRoutes(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -105,7 +104,6 @@ func TestPropagatedRoutes(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -164,6 +162,9 @@ func TestPropagatedRoutes(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // ni-eth0: PropagateConnectedRoutes=true so the eth0 port subnet (172.22.12.0/24) // is delivered to the app. Static route to http-server-0's subnet. @@ -474,8 +475,7 @@ func TestPropagatedRoutes(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestLocalNIWithMultiplePorts(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -483,7 +483,6 @@ func TestLocalNIWithMultiplePorts(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -568,6 +567,9 @@ func TestLocalNIWithMultiplePorts(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Local NI spanning all 4 ports (port="all"). // Static routes use shared labels with probing: @@ -916,8 +918,7 @@ func findRoute(routes []*eveinfo.IPRoute, dst string) *eveinfo.IPRoute { // // Test params // ----------- -// - HYPERVISOR. SkipIfHypervisorKubevirt() is called immediately after -// reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestApplicationGateway(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -925,7 +926,6 @@ func TestApplicationGateway(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -966,6 +966,9 @@ func TestApplicationGateway(test *testing.T) { }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // ni-eth0: Local NI on eth0 — used by app-client1 for its default route // and SSH access (portfwd 2222→22). @@ -1376,7 +1379,427 @@ func TestApplicationGateway(test *testing.T) { t.Expect(out).To(ContainSubstring("MASQUERADE_ACTIVE:")) } -// TestMgmtTrafficRoutedViaApp : TODO replicate github.com/lf-edge/eden/examples/mgmt-over-app +// TestMgmtTrafficRoutedViaApp verifies that EVE's own device-management +// traffic (not just application traffic) can be routed through an +// application acting as a NAT gateway, and that this arrangement survives a +// full device reboot. +// +// Topology +// -------- +// +// +------------+ +// | controller | +// +------------+ +// | +// | +// +--------------------+ +-----------+ +---------------------+ +// | eth0 (app-shared) |---| ni-wan |---| mgmt-gw-app | +// | Switch NI, WAN | | (Switch) | | (vif0) 10.60.10.150 | +// +--------------------+ +-----------+ | ^ MASQUERADE | +// | | + forwarding | +// | v | +// +--------------------+ +-----------+ | (vif1) | +// | eth1 (management) |---| ni-lan |---| 10.60.20.150 | +// | static IP, gw=app | | (Switch) | +---------------------+ +// +--------------------+ +-----------+ +// EVE: 10.60.20.5 +// +// Network model +// ------------- +// - netmodels.MgmtViaAppTopology -- two ports, each hosting a Switch NI: +// eth0/wan-network (10.60.10.0/24) is fully reachable (controller, +// dns-server) and hands out a DHCP static reservation for the app's WAN +// VIF MAC -> 10.60.10.150; eth1/lan-network (10.60.20.0/24) has no +// outside reachability and suppresses the DHCP router option +// (WithoutDefaultRoute) -- it only provides L2 connectivity between +// EVE's own static IP and the app's LAN VIF (reserved -> 10.60.20.150). +// +// Phases +// ------ +// 1. Device config (safe state): both eth0 and eth1 start out as ordinary +// PhyIoUsageMgmtAndApps DHCP ports, so EVE keeps a normal, working +// controller path via eth0 while the gateway app is brought up. +// Two Switch NIs are created: ni-wan on ethernet0, ni-lan on ethernet1, +// and one container app with vif0 on ni-wan (MAC 02:16:3e:02:00:00) +// and vif1 on ni-lan (MAC 02:16:3e:02:00:01), both with default-allow ACLs. +// 2. NI/app readiness: both NIs reach ONLINE, WaitUntilAppIsRunning +// succeeds, and WatchAppInfo confirms vif0=10.60.10.150 and +// vif1=10.60.20.150 (the SDN's static MAC reservations). +// 3. Gateway setup: `ip route` must already show a single default route +// via ni-wan (10.60.10.1) -- ni-lan's WithoutDefaultRoute keeps vif1 +// from contributing one. IP forwarding and +// `POSTROUTING -o eth0 MASQUERADE` are then enabled on the WAN VIF. +// 4. Migrate EVE's management path onto the app: eth0 is reconfigured to +// PhyIoUsageShared with no EVE-side IP (the app's vif0 is now the only +// address on that segment), and eth1 is reconfigured from DHCP to a +// static IP (10.60.20.5/24) whose gateway is set to the app's LAN VIF +// (10.60.20.150) instead of the SDN's own router address -- so all of +// EVE's own outbound traffic (DNS, controller) that isn't for a +// directly-connected subnet is sent to the app, NATed, and exits via +// eth0. WatchDeviceInfo must report a fresh ZInfoDevice update within a +// bounded timeout (proving the new path works), and GetState() must be +// ONLINE. +// 5. Firewall restriction: UpdateNetworkModel adds a Firewall rule set that +// allows both controller and dns-server (10.16.16.25) access only from +// the app's WAN IP (10.60.10.150) and drops each from every other +// source. Two consecutive fresh-info waits (each well under the ~5-minute +// threshold before EVE would report itself SUSPECT) confirm sustained, +// not just momentary, connectivity -- the decisive proof that all of this +// traffic really is sourced from the app's IP, since any other path would +// now be dropped by the SDN firewall. +// 6. Reboot via the controller: RequestReboot is issued without waiting +// (waiting would deadlock on the very SSH-driven step needed to bring +// the path back up), since a full device reboot also restarts the +// gateway app's container, discarding its MASQUERADE setup. The test +// polls SSH connectivity to the app's WAN VIF directly (independent of +// EVE's own management path) until the fresh container responds, then +// reapplies the same IP-forwarding + MASQUERADE commands. Only then +// does WatchDeviceInfo wait for a ZInfoDevice with LastRebootTime newer +// than the reboot request, confirming EVE is back online via the +// app-routed management path after a full reboot. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). func TestMgmtTrafficRoutedViaApp(test *testing.T) { - test.Skip("not yet implemented") + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters(evetest.HypervisorParameter()) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.MgmtViaAppTopology, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Lower the periodic device-info publish interval so a bounded "a fresh + // ZInfoDevice arrives" wait can serve as a direct, real-time signal that + // EVE is still getting through to the controller (see + // TestIntermittentConnectivity for the same rationale). + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, 30) + devConfig.SetConfigProperties(cfgProps) + + const ( + wanAppIP = "10.60.10.150" // SDN static reservation, wan-network + lanAppIP = "10.60.20.150" // SDN static reservation, lan-network + wanGateway = "10.60.10.1" + lanDeviceIP = "10.60.20.5" + vifWanMAC = "02:16:3e:02:00:00" + vifLanMAC = "02:16:3e:02:00:01" + dnsServerIP = "10.16.16.25" // netmodels.MgmtViaAppTopology's dns-server endpoint + ) + + // Phase 1: safe starting state -- both ports are ordinary + // PhyIoUsageMgmtAndApps DHCP ports. eth0 gives EVE a normal, working + // controller path while the gateway app is brought up; eth1's DHCP + // attempt never succeeds as a DPC (lan-network has no outside + // reachability), which is harmless. + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + eth1Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("safe-port-config-applied") + + // Two Switch NIs -- one per port -- and the gateway app connected to both. + niWanUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "ni-wan", + Port: "ethernet0", + MTU: 1500, + }) + niLanUUID := devConfig.AddNetworkInstance(evetest.SwitchNetworkInstanceConfig{ + DisplayName: "ni-lan", + Port: "ethernet1", + MTU: 1500, + }) + + allowAll := []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + } + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "mgmt-gw-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + EnforceNetIntfOrder: true, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niWanUUID, + MAC: evetest.MACAddress(vifWanMAC), + ACLAllowRules: allowAll, + }, + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif1", + NetworkInstanceUUID: niLanUUID, + MAC: evetest.MACAddress(vifLanMAC), + ACLAllowRules: allowAll, + }, + }, + }) + + niWanUpdates, stopNIWanWatch := device.WatchNetworkInstanceInfo(niWanUUID) + niLanUpdates, stopNILanWatch := device.WatchNetworkInstanceInfo(niLanUUID) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + device.ApplyConfig(devConfig, false, false) + + niTimeout := 3 * time.Minute + t.Eventually(niWanUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "ni-wan is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + stopNIWanWatch() + + t.Eventually(niLanUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "ni-lan is ONLINE", + func(info *eveinfo.ZInfoNetworkInstance) bool { + return info.State == eveinfo.ZNetworkInstanceState_ZNETINST_STATE_ONLINE + }).StopIf(niHasError))) + stopNILanWatch() + + evetest.Checkpoint("nis-online") + + device.WaitUntilAppIsRunning(appUUID, 5*time.Minute) + evetest.Checkpoint("app-running") + + var appInfo *eveinfo.ZInfoApp + t.Eventually(appUpdates, niTimeout).Should(Receive(matchers.SatisfyPredicate( + "app reports 2 VIFs with the reserved IPs", + func(info *eveinfo.ZInfoApp) bool { + appInfo = info + if len(info.Network) != 2 { + return false + } + for _, vif := range info.Network { + if len(vif.IPAddrs) == 0 { + return false + } + } + return true + }).StopIf(appHasError))) + stopAppWatch() + + t.Expect(appInfo.Network[0].IPAddrs).To(ContainElement(wanAppIP)) + t.Expect(appInfo.Network[1].IPAddrs).To(ContainElement(lanAppIP)) + + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + sshTimeout := 20 * time.Second + polling := 3 * time.Second + log := evetest.Logger() + + // Phase 3: gateway setup, over SSH via ni-wan's IP directly (Switch NI + // VIFs are reachable from the evetest host without a port-forward). + log.Infof("Waiting for gateway app SSH...") + var routes string + t.Eventually(func(gt Gomega) { + out, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "ip route", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(out).To(ContainSubstring("default via " + wanGateway)) + routes = out + }, 5*time.Minute, polling).Should(Succeed()) + // ni-lan's WithoutDefaultRoute keeps vif1 from also contributing a + // default route -- there must be exactly one, via the WAN leg. + t.Expect(routes).To(ContainSubstring("default via " + wanGateway)) + + evetest.Checkpoint("app-ssh-ready") + + configureGateway := func() { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "sysctl -w net.ipv4.ip_forward=1; "+ + "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE", + sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + } + log.Infof("Configuring gateway app: IP forwarding + MASQUERADE on eth0 (WAN)") + configureGateway() + evetest.Checkpoint("app-gateway-configured") + + // Phase 4: migrate EVE's own management path onto the app. eth0 loses + // its EVE-side IP (app-shared only); eth1 switches from DHCP to a + // static IP whose gateway is the app's LAN VIF instead of the SDN's own + // router address. + log.Infof("Phase 4: migrating EVE's management path via the gateway app...") + devConfig.UpdateNetwork(eth0Net, evetest.NoIPNetworkConfig{}) + devConfig.UpdateNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageShared, + }) + devConfig.UpdateNetwork(eth1Net, evetest.StaticNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + Subnet: evetest.IPSubnet("10.60.20.0/24"), + Gateway: evetest.IPAddress(lanAppIP), + DNSServers: []net.IP{evetest.IPAddress(dnsServerIP)}, + }) + devConfig.UpdateNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: eth1Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + StaticIP: evetest.IPAddress(lanDeviceIP), + }) + // waitUntilConfirmed is deliberately left false: this config changes the + // management port, and EVE may not be able to publish metrics again + // until the app-routed path above is actually working. + device.ApplyConfig(devConfig, true, false) + + // infoTimeout bounds how long a fresh ZInfoDevice update may take once + // the app-routed path takes over (DevInfoInterval was lowered to 30s + // above). + infoTimeout := 3 * time.Minute + waitForFreshInfo := func(reason string) { + drainBacklog: + for { + select { + case <-devUpdates: + default: + break drainBacklog + } + } + log.Infof("Waiting for a fresh device info update (%s)...", reason) + t.Eventually(devUpdates, infoTimeout).Should(Receive(), + "EVE should get device info through to the controller via the "+ + "app-routed management path (%s)", reason) + } + waitForFreshInfo("mgmt path migrated to the gateway app") + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("mgmt-via-app-active") + + // Phase 5: restrict the SDN firewall so both the controller and the DNS + // server are reachable only from the app's WAN IP -- covering DNS too + // closes off the obvious "cheat" of resolving the controller hostname + // (or anything else) via some other, non-app-routed path while still + // routing the actual controller connection through the app. Sustained + // (not just momentary) connectivity from here on is the decisive proof + // that all of this traffic is genuinely sourced from the app, since any + // other path is now dropped. + log.Infof("Phase 5: restricting controller and DNS access to the app's WAN IP...") + restrictedModel := proto.Clone(netmodels.MgmtViaAppTopology).(*api.NetworkModel) + restrictedModel.Firewall = &api.Firewall{ + Rules: []*api.FwRule{ + { + SrcSubnet: wanAppIP + "/32", + DstSubnet: evetest.GetControllerIPv4().String() + "/32", + Action: api.FwAction_FW_ALLOW, + }, + { + SrcSubnet: "0.0.0.0/0", + DstSubnet: evetest.GetControllerIPv4().String() + "/32", + Action: api.FwAction_FW_DROP, + }, + { + SrcSubnet: wanAppIP + "/32", + DstSubnet: dnsServerIP + "/32", + Action: api.FwAction_FW_ALLOW, + }, + { + SrcSubnet: "0.0.0.0/0", + DstSubnet: dnsServerIP + "/32", + Action: api.FwAction_FW_DROP, + }, + }, + } + evetest.UpdateNetworkModel(restrictedModel) + defer evetest.UpdateNetworkModel(netmodels.MgmtViaAppTopology) + + waitForFreshInfo("firewall restricted, check 1/2") + waitForFreshInfo("firewall restricted, check 2/2") + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("firewall-restricted") + + // Phase 6: reboot via the controller. RequestReboot is issued without + // waiting: a full device reboot also restarts the gateway app's + // container, discarding its MASQUERADE setup, so waiting here would + // deadlock on the very SSH-driven step below that brings the + // management path back up. + // + // /proc/sys/kernel/random/boot_id is captured beforehand so the + // post-reboot poll can tell a genuinely fresh container (running under + // a rebooted shim VM/kernel) apart from the pre-reboot instance still + // answering SSH: RequestReboot only requests a reboot -- it does not + // wait for it -- so an early poll attempt could otherwise reach the + // old, not-yet-rebooted container and report success prematurely. + preRebootBootID, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "cat /proc/sys/kernel/random/boot_id", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + log.Infof("Phase 6: triggering reboot via the controller...") + rebootIssuedAt := time.Now() + device.RequestReboot(false) + + // Poll SSH connectivity to the app's WAN VIF directly -- independent of + // EVE's own management path -- until boot_id differs from the + // pre-reboot value, proving the container is genuinely running under a + // fresh boot. Then reconfigure it (container state does not survive a + // full device reboot). + log.Infof("Waiting for the gateway app to come back up after reboot...") + t.Eventually(func(gt Gomega) { + bootID, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "cat /proc/sys/kernel/random/boot_id", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(bootID).ToNot(Equal(preRebootBootID), + "still answering as the pre-reboot instance") + }, 10*time.Minute, polling).Should(Succeed()) + configureGateway() + evetest.Checkpoint("app-gateway-reconfigured-post-reboot") + + rebootTimeout := 10 * time.Minute + t.Eventually(devUpdates, rebootTimeout).Should(Receive(matchers.SatisfyPredicate( + "device reports a fresh reboot via the app-routed management path", + func(info *eveinfo.ZInfoDevice) bool { + ts := info.GetLastRebootTime() + return ts != nil && ts.AsTime().After(rebootIssuedAt) + }))) + t.Expect(device.GetState()).To(Equal(api.EVEDeviceState_EVE_DEVICE_STATE_ONLINE)) + evetest.Checkpoint("device-back-online-post-reboot") } diff --git a/evetest/tests/networking/stp_test.go b/evetest/tests/networking/stp_test.go index f73c8d502ab..e71685dd8ab 100644 --- a/evetest/tests/networking/stp_test.go +++ b/evetest/tests/networking/stp_test.go @@ -119,8 +119,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter -- Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestSwitchNIWithMultiplePorts(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -130,8 +129,6 @@ func TestSwitchNIWithMultiplePorts(test *testing.T) { evetest.HypervisorParameter(), ) hypervisor := evetest.GetHypervisorParameterValue() - // Kubevirt is only supported by cluster tests. - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" requiredDevice := evetest.RequireEdgeDevice{ @@ -191,6 +188,9 @@ func TestSwitchNIWithMultiplePorts(test *testing.T) { // Apply the base adapter configuration. device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Add Switch NI with all three app-shared ports. // BPDU guard is enabled on eth3 ("edge-port" label). diff --git a/evetest/tests/networking/testsuite_test.go b/evetest/tests/networking/testsuite_test.go index a3a94b3a2a9..08c16d83346 100644 --- a/evetest/tests/networking/testsuite_test.go +++ b/evetest/tests/networking/testsuite_test.go @@ -34,12 +34,15 @@ import ( // SDN-side LACP peer requires the bond to be configured before EVE // ever transmits, hence bootstrap-only path. // -// All bootstrap tests hardcode WithHypervisor=HypervisorKVM and do not -// parameterize the hypervisor. +// All bootstrap tests share the HYPERVISOR parameter (defaults to KVM). func TestBootstrapSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + // This below will be implemented using t.Run() // Note that evetest.Close needs to behave differently when test is part of // a test suite and there are more tests to execute. @@ -172,11 +175,8 @@ func TestBootstrapSuite(test *testing.T) { // TestDeviceConnectivitySuite drives every device-side networking // scenario: how EVE itself manages its physical / L2 / IP adapters and -// keeps controller connectivity alive. None of the subtests deploy an -// application -- they focus on the EVE control plane only -- and therefore -// none parameterize the hypervisor (all hardcode HypervisorKVM via the -// shared deviceRequirementsForNetAdapterTests / deviceRequirementsForBootstrap -// helpers or directly). +// keeps controller connectivity alive. +// All subtests share the HYPERVISOR parameter (defaults to KVM). // // Subtests // -------- @@ -190,9 +190,12 @@ func TestBootstrapSuite(test *testing.T) { // - TestPortFailover / TestNetworkConfigFallback / // TestIntermittentConnectivity -- fail-over / fallback resilience // (currently stub scenarios). +// - TestMgmtTrafficRoutedViaApp -- EVE's own device-management traffic +// routed through an app-based NAT gateway, surviving a full reboot. // - TestDeviceIPv6Connectivity -- IPv6-only device side (stub scenario). -// - TestDeviceNTPConfig -- per-port NTP server propagation to EVE's -// chrony (stub scenario; needs SDN-side NTP daemon). +// - TestDeviceNTPConfig -- per-port NTP server aggregation (DHCP + static, +// exclusive-override) and chronyd synchronization against real public +// NTP servers. // - TestActiveBackupBond / TestLACPBond -- bond status, failover, // LACP negotiation. // - TestVLANSubinterfaces / TestVLANSubinterfacesOnTopOfLAGs -- VLAN @@ -203,6 +206,10 @@ func TestDeviceConnectivitySuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{ Test: TestPNAC, @@ -239,6 +246,9 @@ func TestDeviceConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestIntermittentConnectivity, }, + evetest.TestCase{ + Test: TestMgmtTrafficRoutedViaApp, + }, evetest.TestCase{ Test: TestDeviceIPv6Connectivity, }, @@ -270,14 +280,21 @@ func TestDeviceConnectivitySuite(test *testing.T) { // networking scenario. All subtests deploy at least one application and // therefore share the HYPERVISOR parameter -- the suite declares // evetest.HypervisorParameter() once and every subtest reads it via -// evetest.GetHypervisorParameterValue(). Each subtest also calls -// evetest.SkipIfHypervisorKubevirt() right after reading the value: -// Kubevirt is reserved for cluster tests under evetest/tests/cluster. +// evetest.GetHypervisorParameterValue() (defaults to KVM). // // Subtests // -------- // - TestLocalNI / TestSwitchNI -- canonical Local-NI / Switch-NI // life-cycles plus connected-app smoke tests. +// - TestNIReplace -- rapid NI delete+recreate (and subnet reuse across +// NIs) within single config applies. +// - TestMoveAppBetweenNIs -- moving an app's VIF from one Local NI to +// another at runtime. +// - TestAirGapSwitchNI -- portless Switch NI IP-detection via packet +// snooping: statically-assigned IPs and multiple IPs per VIF MAC +// (e.g. via VLAN sub-interfaces). +// - TestLimitedIPSpace -- app deployment/replacement on a NI with only a +// single free IP address available. // - TestFlowLog -- per-app flow log + DNS log assertions // (skipped; depends on GetAppFlowLogs / GetAppDNSLogs which are not // yet wired up in evetest). @@ -285,11 +302,14 @@ func TestDeviceConnectivitySuite(test *testing.T) { // filtering on Local / Switch NIs (variants: ENABLE_FLOWLOG=false / =true). // - TestApplicationIPv6Connectivity -- app on a Switch NI in an // IPv6-only segment (stub scenario). -// - TestApplicationNTPConfig -- DHCP-propagated NTP server set -// reaching the application (stub scenario; needs SDN NTP daemon). +// - TestApplicationNTPConfig -- per-NI DHCP-advertised NTP server set +// (port union NI) reaching the application's published VIF status. // - TestPropagatedRoutes / TestLocalNIWithMultiplePorts / // TestApplicationGateway -- IP-routing-related scenarios mirroring // the eden app-routing examples (stub scenarios). +// - TestPortForwarding -- port-forwarding (D-NAT) hairpin connectivity, +// including across two different uplink adapters, and changing a +// port-fwd rule's external port at runtime. // - TestSwitchNIWithMultiplePorts -- STP / BPDU-guard on a Switch NI // with redundant L2 links (stub scenario). // - TestAccessVLANs -- VLAN-aware Switch NI (stub scenario). @@ -311,6 +331,18 @@ func TestApplicationConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestSwitchNI, }, + evetest.TestCase{ + Test: TestNIReplace, + }, + evetest.TestCase{ + Test: TestMoveAppBetweenNIs, + }, + evetest.TestCase{ + Test: TestAirGapSwitchNI, + }, + evetest.TestCase{ + Test: TestLimitedIPSpace, + }, evetest.TestCase{ Test: TestFlowLog, }, @@ -363,6 +395,9 @@ func TestApplicationConnectivitySuite(test *testing.T) { evetest.TestCase{ Test: TestApplicationGateway, }, + evetest.TestCase{ + Test: TestPortForwarding, + }, evetest.TestCase{ Test: TestSwitchNIWithMultiplePorts, }, @@ -378,11 +413,9 @@ func TestApplicationConnectivitySuite(test *testing.T) { // TestDatastoreSuite drives every datastore-pull scenario: EVE downloads // an application content tree from a backend (HTTP, HTTPS, S3, SFTP, // Azure, container registry), verifies the SHA, and brings the resulting -// app up. The suite does not parameterize the hypervisor -- datastore -// tests deploy a tiny "consumer" app but the test value is in the -// download/verification plumbing, not in the app runtime, so the -// hypervisor is hardcoded to KVM per the same rule applied to Device-suite -// tests. +// app up. Datastore tests deploy a tiny "consumer" app but the test value +// is in the download/verification plumbing, not in the app runtime; the +// suite still shares the HYPERVISOR parameter (defaults to KVM). // // Subtests // -------- @@ -398,6 +431,10 @@ func TestDatastoreSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{ Test: TestHTTPDatastore, @@ -425,9 +462,8 @@ func TestDatastoreSuite(test *testing.T) { // (phantom PCI address, self-parent assign-group, USB address collision, // interface-name mismatch, cross-group PCI conflict, warning+error bundle) // and reports it to the controller with the correct severity, then clears -// it once the model is fixed. The suite does not parameterize the -// hypervisor -- no application is deployed, so the hypervisor is hardcoded -// to KVM like the other Device-suite tests. +// it once the model is fixed. No application is deployed, but the suite +// still shares the HYPERVISOR parameter (defaults to KVM). // // Every scenario but the last runs on the TwoMgmtPorts model and reuses // the same device via ResetDeviceConfig. TestReportWarningsOnly needs the @@ -457,6 +493,10 @@ func TestPcibackErrorSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + evetest.RunTestSuite( evetest.TestCase{Test: TestReportMissingDevice}, evetest.TestCase{Test: TestReportParentAssigngrp}, diff --git a/evetest/tests/networking/vlans_test.go b/evetest/tests/networking/vlans_test.go index 00889ce805a..c6c001afc0d 100644 --- a/evetest/tests/networking/vlans_test.go +++ b/evetest/tests/networking/vlans_test.go @@ -105,8 +105,7 @@ import ( // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestAccessVLANs(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -114,7 +113,6 @@ func TestAccessVLANs(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" evetest.Setup( @@ -171,6 +169,9 @@ func TestAccessVLANs(test *testing.T) { SharedLabels: []string{"switch-ports"}, }) device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Switch NI: eth1 is trunk (carries VLAN 100 and 200 tagged via the SDN router); // eth2 is access port for VLAN 100; eth3 is access port for VLAN 200. @@ -568,8 +569,7 @@ func TestAccessVLANs(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestVLANSubinterfaces(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -577,7 +577,6 @@ func TestVLANSubinterfaces(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" // Clone the bootstrap model and shorten the DHCP lease on the management network. @@ -645,6 +644,9 @@ func TestVLANSubinterfaces(test *testing.T) { // waitUntilConfirmed=false: after the model switch below, EVE may temporarily // lose controller connectivity, delaying the LastProcessedConfig metric publish. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to process the port config (activate vlan interfaces) // before switching the SDN side, so the new interfaces are ready to carry // traffic as soon as VLAN tagging is enabled. @@ -913,8 +915,7 @@ func TestVLANSubinterfaces(test *testing.T) { // // Test params // ----------- -// - HYPERVISOR. The test calls evetest.SkipIfHypervisorKubevirt() right -// after reading the parameter — Kubevirt is reserved for cluster tests. +// - HYPERVISOR (defaults to KVM). func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) @@ -922,7 +923,6 @@ func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { evetest.DefineTestParameters(evetest.HypervisorParameter()) hypervisor := evetest.GetHypervisorParameterValue() - evetest.SkipIfHypervisorKubevirt() devName := "edge-dev" @@ -1025,6 +1025,9 @@ func TestVLANSubinterfacesOnTopOfLAGs(test *testing.T) { // because after the SDN model switch below, EVE may temporarily lose controller // connectivity while LACP negotiates. device.ApplyConfig(devConfig, true, false) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // Give EVE a moment to create the bond and VLAN sub-interfaces before switching // the SDN side, so the interfaces are ready to carry traffic immediately. time.Sleep(10 * time.Second) diff --git a/evetest/tests/security/apparmor_test.go b/evetest/tests/security/apparmor_test.go new file mode 100644 index 00000000000..5e1c27a68f5 --- /dev/null +++ b/evetest/tests/security/apparmor_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve/evetest" +) + +// TestAppArmorEnabled verifies that EVE's kernel has AppArmor compiled in +// and enabled, by reading the kernel's own status flag directly over the +// device's management SSH. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestAppArmorEnabled(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + WithTPM: true, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + out, _, err := device.RunShellScript( + "cat /sys/module/apparmor/parameters/enabled", 20*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.TrimSpace(out)).To(Equal("Y"), "AppArmor is not enabled") +} diff --git a/evetest/tests/security/testsuite_test.go b/evetest/tests/security/testsuite_test.go new file mode 100644 index 00000000000..957f1ff7598 --- /dev/null +++ b/evetest/tests/security/testsuite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestSecuritySuite drives every device-security scenario in this package. +// +// Subtests +// -------- +// - TestAppArmorEnabled -- kernel AppArmor status flag. +// - TestVCom -- vcomlink (TPM-over-vsock) request/response from inside a VM app. +func TestSecuritySuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestAppArmorEnabled, + }, + evetest.TestCase{ + Test: TestVCom, + }, + ) +} diff --git a/evetest/tests/security/vcom_test.go b/evetest/tests/security/vcom_test.go new file mode 100644 index 00000000000..5295d17beca --- /dev/null +++ b/evetest/tests/security/vcom_test.go @@ -0,0 +1,315 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package security + +import ( + "encoding/base64" + "fmt" + "regexp" + "strconv" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// vcomCheckScript is a self-contained (stdlib-only) Python script that +// exercises vcomlink's TPM ActivateCredParams endpoint over vsock from +// inside the guest. +// +// vcomlink (pkg/pillar/cmd/vcomlink) serves a plain HTTP/1.1 API tunneled +// over an AF_VSOCK connection to the host (CID=VMADDR_CID_HOST, port 2000; +// see vsocksrv.go), with protobuf-encoded request/response bodies (see +// pkg/pillar/vcom/api/proto/messages.proto). vcomRequestBody below is the +// precomputed protobuf encoding of +// vcom.TpmRequestActivateCredParams{Index: 0x81000003} -- 0x81000003 is +// evetpm.TpmAIKHdl, the well-known permanent handle for EVE's AIK -- so this +// script needs no protobuf library. The response is parsed just far enough +// (a minimal length-delimited-field walk) to read the length of the "ek" +// field (field 1), which is all this test needs to assert. +const vcomCheckScript = `#!/usr/bin/env python3 +import socket +import sys + +# protobuf encoding of TpmRequestActivateCredParams{Index: 0x81000003} +REQUEST_BODY = bytes([0x08, 0x83, 0x80, 0x80, 0x88, 0x08]) + +def read_varint(data, pos): + result = 0 + shift = 0 + while True: + b = data[pos] + pos += 1 + result |= (b & 0x7f) << shift + if not (b & 0x80): + break + shift += 7 + return result, pos + +def bytes_field_len(data, field_num): + pos = 0 + while pos < len(data): + tag, pos = read_varint(data, pos) + field, wire_type = tag >> 3, tag & 0x7 + if wire_type == 0: + _, pos = read_varint(data, pos) + elif wire_type == 2: + length, pos = read_varint(data, pos) + if field == field_num: + return length + pos += length + elif wire_type == 5: + pos += 4 + elif wire_type == 1: + pos += 8 + else: + raise ValueError("unsupported wire type %d" % wire_type) + return None + +s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) +s.connect((socket.VMADDR_CID_HOST, 2000)) +request = ( + b"POST /tpm/activatecredparams HTTP/1.1\r\n" + b"Host: vcom\r\n" + b"Content-Type: application/x-proto-binary\r\n" + b"Content-Length: " + str(len(REQUEST_BODY)).encode() + b"\r\n" + b"Connection: close\r\n\r\n" +) + REQUEST_BODY +s.sendall(request) + +data = b"" +while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk +s.close() + +header_end = data.index(b"\r\n\r\n") +status_line = data[:header_end].split(b"\r\n")[0].decode() +body = data[header_end + 4:] + +if " 200 " not in status_line: + print("STATUS=%s BODY=%s" % (status_line, body.decode(errors="replace"))) + sys.exit(1) + +print("STATUS=200 EK_LEN=%d" % bytes_field_len(body, 1)) +` + +var vcomEKLenRegexp = regexp.MustCompile(`EK_LEN=(\d+)`) + +// alpineCloudImage describes the arch-specific pinned Alpine Linux +// cloud-init qcow2 image used to boot the VM app in this test. +type alpineCloudImage struct { + relativePath string + sha256 string + sizeBytes uint64 +} + +// Alpine 3.24.1 cloud images, pinned by release version (not a rolling +// "latest" alias) so the SHA256 below stays valid indefinitely. See +// https://alpinelinux.org/cloud/ for the full image list. +// +// amd64 uses the "bios-cloudinit" variant, not "uefi-cloudinit": pillar only +// attaches OVMF/UEFI firmware automatically for VmMode_FML on amd64 (see +// handledomainmgr.go), so a VmMode_HVM VM on amd64 gets legacy SeaBIOS, which +// cannot boot a UEFI-only image. arm64 has no legacy BIOS at all, so pillar +// always attaches OVMF there regardless of VirtualizationMode, and the +// "uefi-cloudinit" variant is the correct (only) choice. +var alpineCloudImages = map[string]alpineCloudImage{ + "amd64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2", + sha256: "6e2e6fe0572b6632527f268d3659e8fccebda4e1ee470fafe2c4d7b85b6a4df6", + sizeBytes: 183697408, + }, + "arm64": { + relativePath: "/alpine/v3.24/releases/cloud/generic_alpine-3.24.1-aarch64-uefi-cloudinit-r0.qcow2", + sha256: "3059a6280977c2122982632e0317c5ddbd39069d46ca1e60480de283091f720f", + sizeBytes: 239271936, + }, +} + +// TestVCom verifies that vcomlink (pkg/pillar/cmd/vcomlink), EVE's +// vsock-based host<->VM communication agent, is running and correctly +// serves a TPM request from inside a guest VM. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- controller reachability plus Internet +// access, needed to pull the Alpine cloud image below. +// +// Phases +// ------ +// 1. Confirm vcomlink is actually listening on the host side (the device +// always has TPM emulation enabled -- vcomlink's TPM handlers depend +// on one being present): `eve exec pillar ss -l --vsock` must report +// a listener on CID:port "2:2000" (VMADDR_CID_HOST, vcomlink's fixed +// port -- see vsocksrv.go). +// 2. Deploy a VM app (VirtualizationMode=HVM) from the pinned Alpine +// Linux cloud image (matching the device's actual arch, device.GetArch()), +// on a Local NI with a 2222->22 port-forward. UserData is a cloud-config +// enabling password SSH login for root (disabled by default). +// Wait for the app to reach RUNNING, then for SSH to become reachable. +// 3. Write vcomCheckScript into the guest over SSH and run it with +// python3 (present by default on the Alpine cloud image; AF_VSOCK +// needs Python >= 3.9, satisfied by Alpine 3.24's default 3.12). +// Assert the script's own output reports HTTP 200 and a non-zero EK +// length -- i.e. vcomlink actually returned real TPM-backed data to +// the VM over vsock, not just accepted the connection. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestVCom(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters(evetest.HypervisorParameter()) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + WithTPM: true, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + log := evetest.Logger() + log.Infof("Checking if vComLink is running on EVE...") + t.Eventually(func(gt Gomega) { + out, _, err := device.RunShellScript( + "eve exec pillar ss -l --vsock", 20*time.Second, 0) + gt.Expect(err).ToNot(HaveOccurred()) + gt.Expect(out).To( + ContainSubstring("2:2000"), "vComLink is not listening on vsock") + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + evetest.Checkpoint("vcomlink-listening") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + eth0Net := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: eth0Net, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.50.0.0/24"), + DHCPRange: pillartypes.IPRange{ + Start: evetest.IPAddress("10.50.0.2"), + End: evetest.IPAddress("10.50.0.254"), + }, + Gateway: evetest.IPAddress("10.50.0.1"), + MTU: 1500, + }) + + image := alpineCloudImages[device.GetArch()] + appAuth := evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", + } + cloudConfig := fmt.Sprintf(`#cloud-config +ssh_pwauth: true +chpasswd: + list: | + root:%s + expire: false +write_files: + - path: /etc/ssh/sshd_config.d/99-allow-root-password.conf + content: | + PermitRootLogin yes +runcmd: + - rc-service sshd restart +`, appAuth.Password) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vcom-test-vm", + Activate: true, + Image: evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW2, + ImageSHA256: image.sha256, + MaxDownloadBytes: image.sizeBytes, + ImageRelativePath: image.relativePath, + ServerAddress: "dl-cdn.alpinelinux.org", + UseHTTPS: true, + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + UserData: base64.StdEncoding.EncodeToString([]byte(cloudConfig)), + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, 8*time.Minute) + evetest.Checkpoint("vm-running") + + sshTimeout := 20 * time.Second + polling := 5 * time.Second + log.Infof("Waiting for VM SSH to become reachable...") + t.Eventually(func(gt Gomega) { + _, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "echo ok", sshTimeout, 0) + gt.Expect(err).ToNot(HaveOccurred()) + }, 5*time.Minute, polling).Should(Succeed()) + evetest.Checkpoint("vm-ssh-ready") + + log.Infof("Running the vComLink TPM check script inside the VM...") + encoded := base64.StdEncoding.EncodeToString([]byte(vcomCheckScript)) + script := "echo " + encoded + " | base64 -d > vcomcheck.py && python3 vcomcheck.py" + checkOut, checkErr, err := device.RunShellScriptInsideApp( + appUUID, appAuth, script, 60*time.Second, 0) + t.Expect(err).ToNot(HaveOccurred(), "vComLink check script failed: %s", checkErr) + t.Expect(checkOut).To( + ContainSubstring("STATUS=200"), "unexpected vComLink response: %s", checkOut) + + match := vcomEKLenRegexp.FindStringSubmatch(checkOut) + t.Expect(match).To(HaveLen(2), "could not find EK_LEN in script output: %s", checkOut) + ekLen, err := strconv.Atoi(match[1]) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(ekLen).To(BeNumerically(">", 0), "vComLink returned an empty EK") + evetest.Checkpoint("vcom-tpm-request-verified") +} diff --git a/evetest/tests/storage/disk_test.go b/evetest/tests/storage/disk_test.go new file mode 100644 index 00000000000..538acc9a553 --- /dev/null +++ b/evetest/tests/storage/disk_test.go @@ -0,0 +1,219 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "strconv" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// TestExtraDiskAttach verifies that an additional (non-root) volume can be attached +// to a running application as a raw block device -- i.e. with no MountDir, so the +// guest sees it as a plain disk rather than a mounted filesystem -- and shows +// up inside the guest. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port; also provides +// Internet access to pull the app's container image. +// +// Phases +// ------ +// 1. Setup: one Local NI ("disk-ni") on ethernet0. Deploy "disk-app" +// (lfedge/evetest-ubuntu-ctr:1.0, HVM) with an SSH port-forward +// (2222->22) and no extra volumes. Wait for RUNNING and for the app's SSH +// daemon to become reachable. Record the baseline disk count (via +// `lsblk -d -o TYPE`, counting entries of type "disk"). +// 2. Create a standalone blank volume (AddBlankVolume) and mount it on the +// app with an empty MountDir (UpdateApplication). An empty MountDir +// attaches the volume as a raw block device instead of a mounted +// filesystem (see MountConfig.MountDir). Adding a mount changes the +// app's VolumeRefList count, which zedmanager always treats as +// requiring a purge (a restart of the domain that preserves the app +// instance's identity -- not a full delete+recreate); UpdateApplication +// bumps the purge counter for this automatically. Wait through +// PURGING -> RUNNING, then verify the guest's disk count increases by +// exactly one. +// 3. Cleanup: delete the app, wait for ZSwState_INVALID, then delete the +// extra disk's volume (independent of the app's own lifecycle -- see +// MountConfig). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestExtraDiskAttach(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "disk-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.22.22.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.22.22.2"), + End: evetest.IPAddress("10.22.22.254"), + }, + Gateway: evetest.IPAddress("10.22.22.1"), + MTU: 1500, + }) + + // Step 1: deploy the app with no extra volumes. appConfig is kept around + // so the later UpdateApplication call only changes Mounts, leaving + // every other field as originally deployed. + appConfig := evetest.ApplicationInstanceConfig{ + DisplayName: "disk-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2222, AppPort: 22}, + }, + }, + }, + } + appUUID := devConfig.AddApplication(appConfig) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + defer stopAppWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("app-deployed") + + timeout := 15 * time.Minute + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("app-running") + + sshTimeout := 20 * time.Second + polling := 5 * time.Second + log := evetest.Logger() + + log.Infof("Waiting for disk-app's SSH daemon to become reachable") + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("ssh-ready") + + baselineDisks, _, err := countGuestDisks(device, appUUID, sshTimeout) + t.Expect(err).ToNot(HaveOccurred()) + log.Infof("Baseline guest disk count: %d", baselineDisks) + + // Step 2: create a standalone blank volume and mount it with an empty + // MountDir -- attached as a raw (unmounted) disk. + const extraDiskSize = 16 * evetest.MiB + extraDiskVolUUID := devConfig.AddBlankVolume("disk-app-extra-disk", extraDiskSize) + appConfig.Mounts = []evetest.MountConfig{ + {VolumeUUID: extraDiskVolUUID, MountDir: ""}, + } + devConfig.UpdateApplication(appUUID, appConfig) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("extra-disk-attached") + + log.Infof("Waiting for disk-app to purge (restart) with the extra disk") + t.Eventually(appUpdates, 2*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "disk-app enters a transient purge state", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_PURGING || + info.State == eveinfo.ZSwState_HALTING + }))) + device.WaitUntilAppIsRunning(appUUID, timeout) + evetest.Checkpoint("extra-disk-purged") + + log.Infof("Waiting for the extra disk to show up inside the guest") + t.Eventually(func() (int, error) { + n, _, err := countGuestDisks(device, appUUID, sshTimeout) + return n, err + }, timeout, polling).Should(Equal(baselineDisks + 1)) + + // Step 3: cleanup. + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app is gone", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + + // The extra disk's volume is independent of the app and outlives it; + // remove it explicitly. + devConfig.DeleteVolume(extraDiskVolUUID) + device.ApplyConfig(devConfig, false, false) +} + +// countGuestDisks returns the number of block devices of type "disk" (i.e. +// excluding partitions, loop devices, etc.) reported by lsblk inside the +// application identified by appUUID. +func countGuestDisks(device *evetest.EdgeDevice, appUUID uuid.UUID, + timeout time.Duration) (int, string, error) { + out, stderr, err := device.RunShellScriptInsideApp(appUUID, ubuntuCtrAppAuth, + "lsblk -d -n -o TYPE | grep -c disk", timeout, 0) + if err != nil { + return 0, stderr, err + } + n, convErr := strconv.Atoi(strings.TrimSpace(out)) + if convErr != nil { + return 0, stderr, convErr + } + return n, stderr, nil +} diff --git a/evetest/tests/storage/helpers_test.go b/evetest/tests/storage/helpers_test.go new file mode 100644 index 00000000000..f80180a9644 --- /dev/null +++ b/evetest/tests/storage/helpers_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" +) + +// ubuntuCtrAppAuth is the fixed SSH credential baked into the +// lfedge/evetest-ubuntu-ctr image, reused by every storage test in this +// package that needs to run commands inside a container app. +var ubuntuCtrAppAuth = evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", +} + +// appHasError reports whether info is in the ERROR state, for use as a +// StopIf fast-fail condition on Eventually assertions waiting on app state. +func appHasError(info *eveinfo.ZInfoApp) (string, bool) { + if info.State == eveinfo.ZSwState_ERROR { + return "Application instance is in error state", true + } + return "", false +} + +// volumeHasError reports whether info carries a VolumeErr, for use as a +// StopIf fast-fail condition on Eventually assertions waiting on volume state. +func volumeHasError(info *eveinfo.ZInfoVolume) (string, bool) { + if desc := info.GetVolumeErr().GetDescription(); desc != "" { + return "Volume reports an error: " + desc, true + } + return "", false +} + +// flattenStorageDisks walks a ZInfoDevice.StorageInfo tree (top-level pools +// plus nested mirror/RAID StorageChildren groups) and returns every disk +// found anywhere in it, as a flat list. +func flattenStorageDisks(pools []*eveinfo.StorageInfo) []*eveinfo.StorageDiskState { + var disks []*eveinfo.StorageDiskState + var walkChildren func(children []*eveinfo.StorageChildren) + walkChildren = func(children []*eveinfo.StorageChildren) { + for _, c := range children { + disks = append(disks, c.GetDisks()...) + walkChildren(c.GetChildren()) + } + } + for _, pool := range pools { + disks = append(disks, pool.GetDisks()...) + walkChildren(pool.GetChildren()) + } + return disks +} + +// diskStatus returns the StorageStatus of the disk named diskName anywhere +// in the given (already-flattened) disk list, and whether it was found. +func diskStatus(disks []*eveinfo.StorageDiskState, diskName string) ( + eveinfo.StorageStatus, bool) { + for _, d := range disks { + if d.GetDiskName().GetName() == diskName { + return d.GetStatus(), true + } + } + return eveinfo.StorageStatus_STORAGE_STATUS_UNSPECIFIED, false +} diff --git a/evetest/tests/storage/mount_test.go b/evetest/tests/storage/mount_test.go new file mode 100644 index 00000000000..7cd07ab3a75 --- /dev/null +++ b/evetest/tests/storage/mount_test.go @@ -0,0 +1,276 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// TestMountedVolumes verifies that an application can have additional, +// independently created volumes mounted alongside its root disk, that +// a mount can be dropped from a running application without affecting +// the underlying volume, and that the same volume can then be mounted again +// at a different MountDir without redownloading its content. Adding or +// removing a mount changes the app's VolumeRefList count, which EVE +// always treats as requiring a purge (a restart of the domain that preserves +// the app instance's identity -- not a full delete+recreate); UpdateApplication +// bumps the purge counter for this automatically, so each of these steps is +// followed by a PURGING -> RUNNING wait. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+app port; also provides +// Internet access needed to pull the docker images used as mount content. +// +// Phases +// ------ +// 1. Create two standalone volumes (AddVolume): tstVol from +// docker://hello-world:linux, dirVol from docker://busybox:latest. Deploy +// "mount-app" (lfedge/evetest-ubuntu-ctr:1.0, HVM) with an SSH +// port-forward (2222->22) and Mounts referencing both volumes: tstVol at +// /tst, dirVol at /dir. Wait for RUNNING and for the app's SSH daemon to +// become reachable. +// 2. Verify (via RunShellScriptInsideApp) that /tst contains the +// hello-world image's content and /dir contains busybox's. Then write a +// marker file into /tst: hello-world's own baked-in content would look +// identical whether tstVol is reused or redownloaded from scratch, so +// the marker (written outside the image's own content) is what actually +// proves reuse in step 4. +// 3. Update the app, dropping the /tst mount (UpdateApplication with only +// dirVol in Mounts), and re-apply -- this triggers a purge (see above), +// not a full redeploy. Verify /tst no longer has hello-world's content. +// tstVol itself still exists (untouched by UpdateApplication; see +// MountConfig). +// 4. Update the app again, mounting tstVol at MountDir "/dst" instead of +// /tst. Verify /dst has both hello-world's content and the marker file +// written in step 2 -- proof that this is the same volume from step 1, +// not redownloaded -- while /dir (never touched) still has busybox's. +// 5. Cleanup: delete the app (removes its root volume only), wait for +// ZSwState_INVALID, then delete tstVol and dirVol explicitly (their +// lifecycle is independent of the app -- see MountConfig). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestMountedVolumes(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + + niUUID := devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "mount-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.21.21.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.21.21.2"), + End: evetest.IPAddress("10.21.21.254"), + }, + Gateway: evetest.IPAddress("10.21.21.1"), + MTU: 1500, + }) + + // Step 1: create the two volumes mounts will reference, then deploy the + // app with Mounts pointing at them. appConfig is kept around and mutated + // in later steps so each UpdateApplication call only changes Mounts, + // leaving every other field (Image, CPUs, adapters, ...) exactly as + // originally deployed. + tstVolUUID := devConfig.AddVolume("mount-app-tst", + evetest.DockerContainer{ImageName: "hello-world", Tag: "linux"}, 0) + dirVolUUID := devConfig.AddVolume("mount-app-dir", + evetest.DockerContainer{ImageName: "busybox", Tag: "latest"}, 0) + + appConfig := evetest.ApplicationInstanceConfig{ + DisplayName: "mount-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + Mounts: []evetest.MountConfig{ + {VolumeUUID: tstVolUUID, MountDir: "/tst"}, + {VolumeUUID: dirVolUUID, MountDir: "/dir"}, + }, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + {Protocol: evetest.NetworkProtocolTCP, EdgeNodePort: 2222, AppPort: 22}, + }, + }, + }, + } + appUUID := devConfig.AddApplication(appConfig) + + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + defer stopAppWatch() + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("app-deployed") + + timeout := 15 * time.Minute + timeoutExcludingDownload := 5 * time.Minute + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("app-running") + + sshTimeout := 20 * time.Second + polling := 5 * time.Second + log := evetest.Logger() + + log.Infof("Waiting for mount-app's SSH daemon to become reachable") + t.Eventually(func(t Gomega) { + _, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + }, timeout, polling).Should(Succeed()) + evetest.Checkpoint("ssh-ready") + + // Step 2: verify both mounts have the expected content. + log.Infof("Verifying /tst (hello-world) and /dir (busybox) mount content") + t.Eventually(func() (string, error) { + out, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "ls /tst", sshTimeout, 0) + return out, err + }, timeout, polling).Should(ContainSubstring("hello")) + t.Eventually(func() (string, error) { + out, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "ls /dir/bin", sshTimeout, 0) + return out, err + }, timeout, polling).Should(ContainSubstring("busybox")) + evetest.Checkpoint("mounts-verified") + + // Write a marker file into tstVol before detaching it. hello-world's own + // baked-in content would look identical whether tstVol is reused or + // redownloaded from scratch; a marker written here (outside the image's + // content) only survives into step 4 if reattaching /dst reuses this + // exact volume rather than recreating it. + const markerContent = "mount-test-marker" + _, _, err := device.RunShellScriptInsideApp(appUUID, ubuntuCtrAppAuth, + "echo "+markerContent+" > /tst/marker.txt && sync", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + + waitForPurge := func() { + t.Eventually(appUpdates, 2*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "mount-app enters a transient purge state", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_PURGING || + info.State == eveinfo.ZSwState_HALTING + }))) + device.WaitUntilAppIsRunning(appUUID, timeout) + } + + // Step 3: drop the /tst mount. tstVol itself is untouched -- only the + // app's VolumeRefList changes. + log.Infof("Dropping the /tst mount") + appConfig.Mounts = []evetest.MountConfig{ + {VolumeUUID: dirVolUUID, MountDir: "/dir"}, + } + devConfig.UpdateApplication(appUUID, appConfig) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("tst-detached") + waitForPurge() + + t.Eventually(func() (string, error) { + out, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "ls /tst", sshTimeout, 0) + return out, err + }, timeout, polling).ShouldNot(ContainSubstring("hello")) + + // Step 4: mount tstVol again, now at MountDir "/dst" -- the same volume, + // so its already-downloaded content is reused, not redownloaded. + log.Infof("Mounting tstVol at /dst") + appConfig.Mounts = []evetest.MountConfig{ + {VolumeUUID: tstVolUUID, MountDir: "/dst"}, + {VolumeUUID: dirVolUUID, MountDir: "/dir"}, + } + devConfig.UpdateApplication(appUUID, appConfig) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("tst-reattached") + waitForPurge() + + t.Eventually(func() (string, error) { + out, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "ls /dst", sshTimeout, 0) + return out, err + }, timeout, polling).Should(ContainSubstring("hello")) + + // The marker written into tstVol before detaching it must have survived + // -- proof that reattaching at /dst reused the same volume (and its + // downloaded content), rather than recreating it from scratch. + out, _, err := device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "cat /dst/marker.txt", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.TrimSpace(out)).To(Equal(markerContent)) + + // /dir was never touched by the detach/reattach and must still be intact. + out, _, err = device.RunShellScriptInsideApp( + appUUID, ubuntuCtrAppAuth, "ls /dir/bin", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(out).To(ContainSubstring("busybox")) + + // Step 5: cleanup. + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "app is gone", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + + // tstVol and dirVol outlive the app; remove them explicitly. + devConfig.DeleteVolume(tstVolUUID) + devConfig.DeleteVolume(dirVolUUID) + device.ApplyConfig(devConfig, false, false) +} diff --git a/evetest/tests/storage/testsuite_test.go b/evetest/tests/storage/testsuite_test.go new file mode 100644 index 00000000000..23d97192119 --- /dev/null +++ b/evetest/tests/storage/testsuite_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestStorageSuite drives storage-layer regression tests. +func TestStorageSuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + evetest.RunTestSuite( + // Non-ZFS tests first, grouped together so the device can be reused + // across all of them. + evetest.TestCase{ + Test: TestVolumes, + }, + evetest.TestCase{ + Test: TestMountedVolumes, + }, + evetest.TestCase{ + Test: TestExtraDiskAttach, + }, + // ZFS tests next, grouped together for the same reason. + evetest.TestCase{ + Test: TestZVolProvisionedSizeReported, + }, + evetest.TestCase{ + Test: TestVaultZvolTrimReclaimsBlocks, + }, + // Last: needs its own freshly created device with extra disks, so + // placement relative to the other tests does not matter for reuse. + evetest.TestCase{ + Test: TestZFSDiskLayout, + }, + ) +} diff --git a/evetest/tests/storage/vault_trim_test.go b/evetest/tests/storage/vault_trim_test.go index 60bdd2695d5..4eb25634c19 100644 --- a/evetest/tests/storage/vault_trim_test.go +++ b/evetest/tests/storage/vault_trim_test.go @@ -28,22 +28,37 @@ import ( // // The test writes 256 MiB of incompressible data (/dev/urandom bypasses ZFS // zstd compression), deletes it to create ghost blocks, then verifies that -// fstrim causes logicalused to drop. Skipped on non-kubevirt or non-ZFS nodes. +// fstrim causes logicalused to drop. Requires a ZFS node; the underlying bug +// was found on EVE-K (Longhorn replica churn), but the mechanism is purely a +// ZFS/fstrim concern and applies equally to EVE-KVM, so the hypervisor is a +// parameter rather than hard-coded. +// +// Parameters +// ---------- +// - HYPERVISOR (defaults to KVM). func TestVaultZvolTrimReclaimsBlocks(test *testing.T) { evetestT := evetest.Init(test) t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + devName := "edge-dev" evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKubevirt, + WithHypervisor: hypervisor, WithFilesystem: evetest.FilesystemZFS, DeviceReusePolicy: evetest.UseAsIs, }, ) device := evetest.GetEdgeDevice(devName) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } // evetest.Setup returns once the device is onboarded and has fetched its // config; it does NOT wait for the vault to be unlocked/mounted. Gate the @@ -51,20 +66,14 @@ func TestVaultZvolTrimReclaimsBlocks(test *testing.T) { // write lands on the parent persist dataset's mountpoint directory (the // ext4-on-zvol is not mounted yet) and logicalused on the zvol never moves. - // Wait for vaultmgr to report the default vault ConversionComplete. Read - // the VaultStatus pubsub JSON on-device via a shell (not ReadPublication): - // ReadPublication/ReadFile Fatalf on a not-yet-published file, and the - // pubsub key "Application Data Store" contains spaces that break scp's - // remote path. A failed cat just fails the poll and we retry. - vaultStatusPath := `/run/vaultmgr/VaultStatus/` + pillartypes.DefaultVaultName + `.json` + // Wait for vaultmgr to report the default vault ConversionComplete. t.Eventually(func() bool { - out, _, err := device.RunShellScript( - `eve exec pillar cat "`+vaultStatusPath+`"`, 15*time.Second, 0) - if err != nil { + var status pillartypes.VaultStatus + if !evetest.ReadPublication(device, "vaultmgr", false, + pillartypes.DefaultVaultName, &status) { return false // status not published yet } - return strings.Contains(strings.ReplaceAll(out, " ", ""), - `"ConversionComplete":true`) + return status.ConversionComplete }, 5*time.Minute, 5*time.Second).Should(BeTrue(), "vaultmgr must report the default vault ConversionComplete before writing") diff --git a/evetest/tests/storage/volumes_test.go b/evetest/tests/storage/volumes_test.go new file mode 100644 index 00000000000..dafe6fc056c --- /dev/null +++ b/evetest/tests/storage/volumes_test.go @@ -0,0 +1,344 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + evemetrics "github.com/lf-edge/eve-api/go/metrics" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + uuid "github.com/satori/go.uuid" +) + +// TestVolumes exercises the lifecycle of standalone (app-unreferenced) +// volumes: creation from every supported content source/disk format and +// deletion, plus how volumemgr behaves when /persist runs low on space -- +// both for a standalone volume and for an application's own root volume. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability +// and to pull docker images; volumemgr creates standalone +// (app-unreferenced) volumes on its own, so no network instance is +// needed for the standalone-volume part of this test. +// +// Phases +// ------ +// 1. Set up a device with a single DHCP mgmt port. +// 2. Create 5 standalone volumes, one per supported disk format/source +// (docker, qcow2, qcow, vmdk, vhdx -- the qcow2/qcow/vmdk/vhdx images are +// blank files generated locally via CreateBlankImageFile and served +// by evetest's built-in HTTP image server). Wait for all 5 to reach +// ZSwState_CREATED_VOLUME. +// 3. Disk-space-exhaustion scenario: +// a. Read the device's current /persist disk metric (total/free). +// b. Create "blank-vol-1", sized so that the free space remaining after +// it is allocated drops just below half of /persist's total size -- +// small enough to fit itself, but leaving too little room for a +// second, half-total-sized volume. +// c. Attempt to create "blank-vol-2" sized at half of /persist's total. +// Verify it fails with a VolumeErr mentioning "Remaining" (insufficient +// disk space), instead of reaching CREATED_VOLUME. +// d. Delete "blank-vol-1" (freeing space) and verify "blank-vol-2" then +// succeeds (reaches CREATED_VOLUME). +// e. Deploy an app whose root volume is also sized at half of +// /persist's total (competing with blank-vol-2 for the same space). +// Verify it fails to activate with an AppErr mentioning "Remaining". +// f. Delete "blank-vol-2" (freeing space) and verify the app then +// reaches RUNNING. +// g. Purge the app (PurgeApplication) and verify it goes through +// PURGING and back to RUNNING. +// h. Delete the app. +// 4. Delete all 5 standalone volumes from step 2 and verify they are all +// reported as gone (ZSwState_INVALID). +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +func TestVolumes(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + evetest.RequireInternetConnectivity{}, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + + timeout := 15 * time.Minute + log := evetest.Logger() + + // Step 2: create one standalone volume per supported format/source. + const perVolSize = 200 * evetest.MiB + type namedVolume struct { + name string + uuid uuid.UUID + updates <-chan *eveinfo.ZInfoVolume + stop func() + } + vols := []namedVolume{ + {name: "v-docker"}, + {name: "v-qcow2"}, + {name: "v-qcow"}, + {name: "v-vmdk"}, + {name: "v-vhdx"}, + } + vols[0].uuid = devConfig.AddVolume("v-docker", evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0"}, perVolSize) + + qcow2Path, qcow2SHA256 := evetest.CreateBlankImageFile( + "v-qcow2.qcow2", eveconfig.Format_QCOW2, perVolSize) + vols[1].uuid = devConfig.AddVolume("v-qcow2", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW2, + ImageRelativePath: qcow2Path, + ImageSHA256: qcow2SHA256, + ServerAddress: evetest.GetImageServerIPv4().String(), + }, perVolSize) + + qcowPath, qcowSHA256 := evetest.CreateBlankImageFile( + "v-qcow.qcow", eveconfig.Format_QCOW, perVolSize) + vols[2].uuid = devConfig.AddVolume("v-qcow", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_QCOW, + ImageRelativePath: qcowPath, + ImageSHA256: qcowSHA256, + ServerAddress: evetest.GetImageServerIPv4().String(), + }, perVolSize) + + vmdkPath, vmdkSHA256 := evetest.CreateBlankImageFile( + "v-vmdk.vmdk", eveconfig.Format_VMDK, perVolSize) + vols[3].uuid = devConfig.AddVolume("v-vmdk", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_VMDK, + ImageRelativePath: vmdkPath, + ImageSHA256: vmdkSHA256, + ServerAddress: evetest.GetImageServerIPv4().String(), + }, perVolSize) + + vhdxPath, vhdxSHA256 := evetest.CreateBlankImageFile( + "v-vhdx.vhdx", eveconfig.Format_VHDX, perVolSize) + vols[4].uuid = devConfig.AddVolume("v-vhdx", evetest.HTTPStorage{ + ImageFormat: eveconfig.Format_VHDX, + ImageRelativePath: vhdxPath, + ImageSHA256: vhdxSHA256, + ServerAddress: evetest.GetImageServerIPv4().String(), + }, perVolSize) + + for i := range vols { + vols[i].updates, vols[i].stop = device.WatchVolumeInfo(vols[i].uuid) + } + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("standalone-volumes-config-applied") + + log.Infof("Waiting for all 5 standalone volumes to be created") + for _, v := range vols { + v := v + t.Eventually(v.updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("volume %s is created", v.name), + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + } + evetest.Checkpoint("standalone-volumes-created") + + // Step 3: disk-space-exhaustion scenario. + const mib = evetest.MiB + const safetyMarginMiB = 200 + + metricsUpdates, stopMetricsWatch := device.WatchDeviceMetrics() + var persistDisk *evemetrics.DiskMetric + t.Eventually(metricsUpdates, 5*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "device reports a non-zero /persist disk metric", + func(m *evemetrics.DeviceMetric) bool { + for _, d := range m.GetDisk() { + if d.GetMountPath() == "/persist" && d.GetTotal() > 0 { + persistDisk = d + return true + } + } + return false + }))) + stopMetricsWatch() + + totalMiB := persistDisk.GetTotal() + freeMiB := persistDisk.GetFree() + halfTotalMiB := totalMiB / 2 + log.Infof("/persist: total=%d MiB free=%d MiB half_total=%d MiB", + totalMiB, freeMiB, halfTotalMiB) + t.Expect(freeMiB-halfTotalMiB).To(BeNumerically(">=", safetyMarginMiB), + "insufficient /persist free space for the test "+ + "(free=%d MiB, half_total=%d MiB, need free >= half_total + %d MiB)", + freeMiB, halfTotalMiB, safetyMarginMiB) + // blankVol1Size: large enough that the free space remaining after + // blank-vol-1 is below half_total (so blank-vol-2 cannot fit), but + // bounded by current free space so blank-vol-1 itself still fits. + blankVol1SizeMiB := freeMiB - halfTotalMiB + safetyMarginMiB + halfTotalBytes := halfTotalMiB * mib + blankVol1Bytes := blankVol1SizeMiB * mib + + blankVol1UUID := devConfig.AddBlankVolume("blank-vol-1", blankVol1Bytes) + blankVol1Updates, stopBlankVol1Watch := device.WatchVolumeInfo(blankVol1UUID) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("blank-vol-1-config-applied") + + t.Eventually(blankVol1Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "blank-vol-1 is created", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }).StopIf(volumeHasError))) + stopBlankVol1Watch() + evetest.Checkpoint("blank-vol-1-created") + + // blank-vol-2 is expected to fail: not enough free space remains. + blankVol2UUID := devConfig.AddBlankVolume("blank-vol-2", halfTotalBytes) + blankVol2Updates, stopBlankVol2Watch := device.WatchVolumeInfo(blankVol2UUID) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("blank-vol-2-config-applied") + + log.Infof("Waiting for blank-vol-2 to fail due to insufficient disk space") + t.Eventually(blankVol2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "blank-vol-2 reports insufficient remaining disk space", + func(info *eveinfo.ZInfoVolume) bool { + return containsIgnoreCase(info.GetVolumeErr().GetDescription(), "Remaining") + }))) + evetest.Checkpoint("blank-vol-2-failed-as-expected") + + // Delete blank-vol-1 to free up space for blank-vol-2. + devConfig.DeleteVolume(blankVol1UUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(blankVol2Updates, timeout).Should(Receive(matchers.SatisfyPredicate( + "blank-vol-2 is created after blank-vol-1 is freed", + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_CREATED_VOLUME + }))) + stopBlankVol2Watch() + evetest.Checkpoint("blank-vol-2-created") + + // Deploy an app whose root volume competes with blank-vol-2 for the same + // space; it is expected to fail to activate. + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "vol-space-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", Tag: "1.0"}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + DiskBytes: halfTotalBytes, + }) + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("vol-space-app-config-applied") + + log.Infof("Waiting for vol-space-app to fail due to insufficient disk space") + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "vol-space-app reports insufficient remaining disk space", + func(info *eveinfo.ZInfoApp) bool { + for _, appErr := range info.GetAppErr() { + if containsIgnoreCase(appErr.GetDescription(), "Remaining") { + return true + } + } + return false + }))) + evetest.Checkpoint("vol-space-app-failed-as-expected") + + // Delete blank-vol-2 to free up space; the app should then start. + devConfig.DeleteVolume(blankVol2UUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "vol-space-app reaches RUNNING", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_RUNNING + }))) + evetest.Checkpoint("vol-space-app-running") + + // Purge the app and verify it comes back up. + log.Infof("Purging vol-space-app") + device.PurgeApplication(appUUID, false, 0) + t.Eventually(appUpdates, 2*time.Minute).Should(Receive(matchers.SatisfyPredicate( + "vol-space-app enters a transient purge state", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_PURGING || + info.State == eveinfo.ZSwState_HALTING + }))) + device.WaitUntilAppIsRunning(appUUID, timeout) + evetest.Checkpoint("vol-space-app-purged") + + // Resync devConfig with the config as mutated by PurgeApplication before + // deleting the app through it. + devConfig = device.GetConfig() + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "vol-space-app is gone", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }).StopIf(appHasError))) + stopAppWatch() + + // Step 4: delete the 5 standalone volumes from step 2 and verify they + // are all gone. + for _, v := range vols { + devConfig.DeleteVolume(v.uuid) + } + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("standalone-volumes-delete-applied") + + for _, v := range vols { + v := v + t.Eventually(v.updates, timeout).Should(Receive(matchers.SatisfyPredicate( + fmt.Sprintf("volume %s is gone", v.name), + func(info *eveinfo.ZInfoVolume) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) + v.stop() + } +} + +// containsIgnoreCase reports whether s contains substr, ignoring case. +func containsIgnoreCase(s, substr string) bool { + return strings.Contains(strings.ToLower(s), strings.ToLower(substr)) +} diff --git a/evetest/tests/storage/zfs_layout_test.go b/evetest/tests/storage/zfs_layout_test.go new file mode 100644 index 00000000000..bf930ec3c4b --- /dev/null +++ b/evetest/tests/storage/zfs_layout_test.go @@ -0,0 +1,228 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// TestZFSDiskLayout exercises EVE's ZFS disk-array layout/RAID +// reconfiguration end to end, using a fixed set of 5 extra virtio disks +// provisioned up front (RequireEdgeDevice.ExtraDisks) so every stage always +// runs regardless of the test environment: raid1 (2 disks) -> raid10 (4 +// disks) -> offline one disk (degraded) -> restore -> replace one disk with +// the 5th (spare) disk. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- only needed for controller reachability. +// +// Phases +// ------ +// 1. Set up a device with WithFilesystem: FilesystemZFS and 5 extra 2 GiB +// virtio disks (/dev/vdb.../dev/vdf; /dev/vda is the boot disk). At this +// point the extra disks are not yet part of the zpool (EVE's installer +// only ever creates the pool from the boot disk's own P3 partition). +// Reduce timer.metric.diskscan.interval to its minimum so config-driven +// disk changes are picked up quickly. +// 2. Verify the initial pool (boot disk only) is reported +// STORAGE_STATUS_ONLINE. +// 3. Apply a RAID1 layout (DiskLayoutRAID1, disks vdb+vdc) via +// SetDisksLayout. Verify StorageInfo reports vdb. +// 4. Apply a RAID10 layout (DiskLayoutRAID10, disks vdb-vde). Verify +// StorageInfo reports vdc and vdd. +// 5. Mark the first disk (vdb) offline. Verify it is reported +// STORAGE_STATUS_OFFLINE and the pool becomes STORAGE_STATUS_DEGRADED. +// 6. Bring the disk back online (remove the offline marker). Verify no +// disk is reported DEGRADED/OFFLINE anymore. +// 7. Replace the first disk (vdb) with the spare (vdf, the 5th extra disk). +// Verify StorageInfo reports vdf and no longer reports vdb. +// +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). ZFS disk-layout/RAID configuration +// itself is independent of hypervisor choice; the parameter exists for +// consistency with the rest of TestStorageSuite. Note that DiskName's +// /dev/vdX naming for the extra disks has only been verified under KVM. +func TestZFSDiskLayout(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + + devName := "edge-dev" + const extraDiskSize = 2 * evetest.GiB + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + WithFilesystem: evetest.FilesystemZFS, + ExtraDisks: []uint64{ + extraDiskSize, extraDiskSize, extraDiskSize, extraDiskSize, extraDiskSize}, + DeviceReusePolicy: evetest.CreateFromScratchWithLiveImage, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DiskScanMetricInterval, 15) + devConfig.SetConfigProperties(cfgProps) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("mgmt-network-ready") + + devInfoUpdates, stopDevInfoWatch := device.WatchDeviceInfo() + defer stopDevInfoWatch() + + timeout := 10 * time.Minute + log := evetest.Logger() + + // Step 2: the initial pool (boot disk only) must be online. + log.Infof("Waiting for the initial (boot-disk-only) pool to be online") + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "storage pool is online", + func(info *eveinfo.ZInfoDevice) bool { + for _, pool := range info.GetStorageInfo() { + if pool.GetStorageState() == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE { + return true + } + } + return false + }))) + evetest.Checkpoint("initial-pool-online") + + // Step 3: RAID1 layout (vdb + vdc). + log.Infof("Applying RAID1 layout (vdb+vdc)") + devConfig.SetDisksLayout(evetest.DisksLayout{LayoutType: evetest.DiskLayoutRAID1}) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("raid1-config-applied") + + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "storage info reports /dev/vdb online", + func(info *eveinfo.ZInfoDevice) bool { + status, found := diskStatus( + flattenStorageDisks(info.GetStorageInfo()), evetest.DiskName(0)) + return found && status == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE + }))) + evetest.Checkpoint("raid1-applied") + + // Step 4: grow to RAID10 layout (vdb-vde). + log.Infof("Applying RAID10 layout (vdb-vde)") + devConfig.SetDisksLayout(evetest.DisksLayout{LayoutType: evetest.DiskLayoutRAID10}) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("raid10-config-applied") + + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "storage info reports /dev/vdc and /dev/vdd online", + func(info *eveinfo.ZInfoDevice) bool { + disks := flattenStorageDisks(info.GetStorageInfo()) + statusC, foundC := diskStatus(disks, evetest.DiskName(1)) + statusD, foundD := diskStatus(disks, evetest.DiskName(2)) + return foundC && statusC == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE && + foundD && statusD == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE + }))) + evetest.Checkpoint("raid10-applied") + + // Step 5: take the first disk (vdb) offline; pool becomes degraded. + log.Infof("Marking /dev/vdb offline") + devConfig.SetDisksLayout(evetest.DisksLayout{ + LayoutType: evetest.DiskLayoutRAID10, + OfflineDisks: []uint{0}, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("disk-offline-config-applied") + + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "/dev/vdb is offline and the pool is degraded", + func(info *eveinfo.ZInfoDevice) bool { + disks := flattenStorageDisks(info.GetStorageInfo()) + status, found := diskStatus(disks, evetest.DiskName(0)) + if !found || status != eveinfo.StorageStatus_STORAGE_STATUS_OFFLINE { + return false + } + for _, pool := range info.GetStorageInfo() { + if pool.GetStorageState() == eveinfo.StorageStatus_STORAGE_STATUS_DEGRADED { + return true + } + } + return false + }))) + evetest.Checkpoint("disk-offlined") + + // Step 6: bring the disk back online; the pool should no longer be degraded. + log.Infof("Bringing /dev/vdb back online") + devConfig.SetDisksLayout(evetest.DisksLayout{LayoutType: evetest.DiskLayoutRAID10}) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("disk-restore-config-applied") + + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "no disk is degraded or offline", + func(info *eveinfo.ZInfoDevice) bool { + for _, pool := range info.GetStorageInfo() { + if pool.GetStorageState() == eveinfo.StorageStatus_STORAGE_STATUS_DEGRADED { + return false + } + } + disks := flattenStorageDisks(info.GetStorageInfo()) + status, found := diskStatus(disks, evetest.DiskName(0)) + return found && status == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE + }))) + evetest.Checkpoint("disk-restored") + + // Step 7: replace the first disk (vdb) with the spare (vdf). + log.Infof("Replacing /dev/vdb with the spare disk /dev/vdf") + devConfig.SetDisksLayout(evetest.DisksLayout{ + LayoutType: evetest.DiskLayoutRAID10, + ReplaceDisks: []uint{0}, + }) + device.ApplyConfig(devConfig, false, false) + evetest.Checkpoint("disk-replace-config-applied") + + t.Eventually(devInfoUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "/dev/vdf replaced /dev/vdb and the pool is not degraded", + func(info *eveinfo.ZInfoDevice) bool { + for _, pool := range info.GetStorageInfo() { + if pool.GetStorageState() == eveinfo.StorageStatus_STORAGE_STATUS_DEGRADED { + return false + } + } + disks := flattenStorageDisks(info.GetStorageInfo()) + _, oldFound := diskStatus(disks, evetest.DiskName(0)) + newStatus, newFound := diskStatus(disks, evetest.DiskName(4)) + return newFound && newStatus == eveinfo.StorageStatus_STORAGE_STATUS_ONLINE && !oldFound + }))) + evetest.Checkpoint("disk-replaced") +} diff --git a/evetest/tests/storage/zvol_provisioned_size_test.go b/evetest/tests/storage/zvol_provisioned_size_test.go index 74912b7f407..aaa50abd6b5 100644 --- a/evetest/tests/storage/zvol_provisioned_size_test.go +++ b/evetest/tests/storage/zvol_provisioned_size_test.go @@ -41,13 +41,17 @@ import ( // it. volumemgr creates standalone (app-unreferenced) volumes on its own, so no // application is needed to trigger creation. // -// Requires a KVM device whose /persist is ZFS: a non-container, non-ISO volume +// Requires a device whose /persist is ZFS: a non-container, non-ISO volume // is only backed by a zvol when /persist is ZFS. On EXT4 there is no zvol and // the code path under test does not run, so the test skips. // +// Test params +// ----------- +// - HYPERVISOR (defaults to KVM). +// // Phases // ------ -// 1. Set up a KVM + ZFS device with a single DHCP mgmt port. +// 1. Set up a ZFS device with a single DHCP mgmt port. // 2. Apply a config that declares one standalone empty 1 GiB block-device // volume (no app, no network instance). // 3. Wait for volumemgr to create the volume (state CREATED_VOLUME) and, @@ -60,6 +64,11 @@ func TestZVolProvisionedSizeReported(test *testing.T) { t := NewGomegaWithT(evetestT) defer evetest.Close() + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + // A 1 GiB volume is an exact multiple of the ZFS volblocksize (16 KiB), so // the resulting zvol volsize -- and therefore the reported provisioned size // -- is exactly 1 GiB with no block-size rounding. @@ -71,7 +80,7 @@ func TestZVolProvisionedSizeReported(test *testing.T) { evetest.Setup( evetest.RequireEdgeDevice{ Name: devName, - WithHypervisor: evetest.HypervisorKVM, + WithHypervisor: hypervisor, WithFilesystem: evetest.FilesystemZFS, DeviceReusePolicy: evetest.ResetDeviceConfig, }, @@ -106,6 +115,9 @@ func TestZVolProvisionedSizeReported(test *testing.T) { volInfoUpdates, stopVolInfoWatch := device.WatchVolumeInfo(volUUID) defer stopVolInfoWatch() device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } evetest.Checkpoint("blank-volume-config-applied") // Wait for the volume to be created. CREATED_VOLUME is the volume-equivalent diff --git a/evetest/tests/upgrade/testsuite_test.go b/evetest/tests/upgrade/testsuite_test.go index 0518022f0ba..0ac25b1a6b5 100644 --- a/evetest/tests/upgrade/testsuite_test.go +++ b/evetest/tests/upgrade/testsuite_test.go @@ -26,8 +26,18 @@ func TestUpgradeSuite(test *testing.T) { initialEVEVersionForKVM = "16.0.0-lts" // EVE-K (k3s/kubevirt-based EVE) is officially supported starting from 17.0.0. - // TODO: strip the rc suffix once 17.0.0 is released. - initialEVEVersionForKubevirt = "17.0.0-rc2" + initialEVEVersionForKubevirt = "17.0.0-lts" + + // The OCI-datastore variant points EVE at evetest's own embedded OCI + // registry (see PushDockerImageToLocalRegistry), which is fronted by + // the harness's self-signed CA. Resolving an OCI tag against a + // custom-CA registry only started honoring the datastore's trusted + // certificates in pillar commit 90fb8664 ("downloader: ds custom + // certs when resolving OCI"); before that, tag resolution opened the + // registry without them and failed TLS verification even though the + // certs were (and still are) applied correctly for the subsequent + // download. That fix is included in 17.0.0-lts (but not in 16.0.0-lts). + initialEVEVersionForOCIDatastore = "17.0.0-lts" // Enough for the pre-10GB partition layout, but not enough for EVE 17.0.0+, // which is why *WithSmallDisk variants expect revert. @@ -54,6 +64,19 @@ func TestUpgradeSuite(test *testing.T) { {Key: evetest.HypervisorParameterKey, Value: evetest.HypervisorKVM}, }, }, + { + Name: "TestEVEUpgradeKVMtoKVMWithOCIDatastore", + Parameters: []evetest.TestParameterValue{ + // Initial + {Key: initialEVEVersionParamKey, Value: initialEVEVersionForOCIDatastore}, + {Key: initialHypervisorParamKey, Value: evetest.HypervisorKVM}, + // Target + {Key: evetest.HypervisorParameterKey, Value: evetest.HypervisorKVM}, + // Have EVE pull the target rootfs directly from its OCI registry + // instead of the default evetest-hosted HTTP datastore. + {Key: datastoreTypeParamKey, Value: evetest.BaseOSDatastoreOCI}, + }, + }, { Name: "TestEVEUpgradeKubevirtToKubevirt", Parameters: []evetest.TestParameterValue{ diff --git a/evetest/tests/upgrade/upgrade_test.go b/evetest/tests/upgrade/upgrade_test.go index d550abe7ebf..95755b532a7 100644 --- a/evetest/tests/upgrade/upgrade_test.go +++ b/evetest/tests/upgrade/upgrade_test.go @@ -25,6 +25,7 @@ const ( initialEVEVersionParamKey = "INITIAL_EVE_VERSION" initialHypervisorParamKey = "INITIAL_HYPERVISOR" expectRevertParamKey = "EXPECT_REVERT" + datastoreTypeParamKey = "DATASTORE_TYPE" appSSHUser = "root" appSSHPassword = "testpassword" @@ -51,6 +52,10 @@ const ( // - DISK_SIZE_MB: device disk size in MiB (0 = framework default 65536 MiB) // - EXPECT_REVERT: if true, the upgrade is expected to fail and EVE to revert // to the previous version (default: false) +// - DATASTORE_TYPE: how the target rootfs is delivered to the device -- +// "http" (evetest extracts the rootfs and serves it over its own embedded +// HTTP image server) or "oci" (EVE pulls the target image directly from +// its OCI registry, e.g. Docker Hub) (default: "http") // // A container app (evetest-ubuntu-ctr) is deployed before the upgrade and verified // to be healthy both before and after the upgrade (or revert). @@ -90,6 +95,16 @@ func TestEVEUpgrade(test *testing.T) { Default: "false", }, }, + evetest.TestParameterDefinition{ + Key: datastoreTypeParamKey, + DefaultValue: evetest.BaseOSDatastoreHTTP, + Description: evetest.TestParameterDescription{ + Summary: "How the target rootfs is delivered to the device: evetest-hosted " + + "HTTP, or an OCI registry pull performed directly by EVE", + Default: "http", + AllowedValues: "http|oci", + }, + }, ) // Get parameter values set for this test execution. @@ -104,6 +119,7 @@ func TestEVEUpgrade(test *testing.T) { targetVersion := evetest.GetEVEVersionParameterValue() targetHypervisor := evetest.GetHypervisorParameterValue() expectRevert := evetest.GetTestParameter[bool](expectRevertParamKey) + datastoreType := evetest.GetTestParameter[evetest.BaseOSDatastoreType](datastoreTypeParamKey) // The pre-upgrade device pins INITIAL_EVE_VERSION, so it always boots that // released version from a container image; the live transport applies to the @@ -111,8 +127,9 @@ func TestEVEUpgrade(test *testing.T) { // survive an upgrade from the last release?". Naming a target version and // asking for the live transport is only contradictory when that version is // not one of the local builds, which UpgradeEVE reports. + log := evetest.Logger() if evetest.LocalLiveImageRequested() { - evetestT.Logf("Upgrading to the local EVE build (%s%s is set); the "+ + log.Infof("Upgrading to the local EVE build (%s%s is set); the "+ "pre-upgrade version %q still comes from a container image", constants.EnvPrefix, constants.EVELiveImageEnv, initialVersion) } @@ -219,7 +236,6 @@ func TestEVEUpgrade(test *testing.T) { // Verify the app is reachable before the upgrade. appAuth := evetest.UsernamePasswordAuth{Username: appSSHUser, Password: appSSHPassword} sshTimeout := 20 * time.Second - log := evetest.Logger() log.Infof("Verifying app is reachable before upgrade") t.Eventually(func(t Gomega) { out, _, err := device.RunShellScriptInsideApp( @@ -230,7 +246,7 @@ func TestEVEUpgrade(test *testing.T) { evetest.Checkpoint("pre-upgrade") - device.UpgradeEVE(targetVersion, targetHypervisor, true, expectRevert) + device.UpgradeEVE(targetVersion, targetHypervisor, datastoreType, true, expectRevert) if expectRevert { evetest.Checkpoint("upgrade-reverted") diff --git a/pkg/pillar/cmd/zedagent/handlemetrics.go b/pkg/pillar/cmd/zedagent/handlemetrics.go index 35f3461623a..2259bc44923 100644 --- a/pkg/pillar/cmd/zedagent/handlemetrics.go +++ b/pkg/pillar/cmd/zedagent/handlemetrics.go @@ -1232,7 +1232,7 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string, networkInfo.DevName = *proto.String(name) niStatus := appIfnameToNetworkInstance(ctx, aiStatus, ifname) if niStatus != nil { - networkInfo.NtpServers = utils.ToStrings(niStatus.NtpServers) + networkInfo.NtpServers = utils.ToStrings(niStatus.CombinedNTPServers) networkInfo.DefaultRouters = []string{niStatus.Gateway.String()} networkInfo.Dns = &info.ZInfoDNS{ DNSservers: []string{}, diff --git a/pkg/pillar/cmd/zedrouter/networkinstance.go b/pkg/pillar/cmd/zedrouter/networkinstance.go index fb5bfe61372..c6b3ee058a3 100644 --- a/pkg/pillar/cmd/zedrouter/networkinstance.go +++ b/pkg/pillar/cmd/zedrouter/networkinstance.go @@ -256,9 +256,9 @@ func (z *zedrouter) updateNIPorts(niConfig types.NetworkInstanceConfig, newNTPServers = generics.FilterDuplicatesFn(newNTPServers, netutils.EqualHostnameOrIPs) changed = changed || !generics.EqualSets(niStatus.Ports, validatedPortLLs) niStatus.Ports = validatedPortLLs - changed = changed || !generics.EqualSetsFn(niStatus.NTPServers, newNTPServers, + changed = changed || !generics.EqualSetsFn(niStatus.CombinedNTPServers, newNTPServers, netutils.EqualHostnameOrIPs) - niStatus.NTPServers = newNTPServers + niStatus.CombinedNTPServers = newNTPServers // Update BridgeMac for Switch NI bridge created by NIM. if z.niBridgeIsCreatedByNIM(niConfig) { // Only switch NI with single port may have the bridge created by NIM. diff --git a/pkg/pillar/types/zedroutertypes.go b/pkg/pillar/types/zedroutertypes.go index 7a66105a35b..9c29c6d80b6 100644 --- a/pkg/pillar/types/zedroutertypes.go +++ b/pkg/pillar/types/zedroutertypes.go @@ -1090,10 +1090,14 @@ type NetworkInstanceStatus struct { // Labels of device ports used for external connectivity. // The list is empty for air-gapped network instances. Ports []string - // List of NTP servers published to applications connected to this network instance. - // This includes the NTP server from the NI config (if any) and all NTP servers - // associated with ports used by the network instance for external connectivity. - NTPServers []netutils.HostnameOrIP + // CombinedNTPServers is the list of NTP servers published to applications + // connected to this network instance. It combines the NTP server from the + // NI config (if any) with all NTP servers associated with ports used by + // the network instance for external connectivity. Named distinctly from + // the embedded NetworkInstanceConfig.NtpServers (the NI-only raw config) + // since the two previously differed only by case, which let a caller + // silently read the wrong one through field promotion. + CombinedNTPServers []netutils.HostnameOrIP // The intended state of the routing table. // Includes user-configured static routes and potentially also automatically // generated default route.