From a6796b7562b3c021e67a6f154f9581874d5f4703 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 12 Aug 2026 10:56:15 -0600 Subject: [PATCH] e2e: extract the shared harness into lib.sh This moves the cluster setup out per-version-hpa.sh into e2e/lib.sh, in anticpation of adding more cluster e2e tests for immenient work. --- e2e/lib.sh | 233 +++++++++++++++++++++++++++++++++++++++++ e2e/per-version-hpa.sh | 192 +++------------------------------ 2 files changed, 249 insertions(+), 176 deletions(-) create mode 100644 e2e/lib.sh diff --git a/e2e/lib.sh b/e2e/lib.sh new file mode 100644 index 0000000..fd87ea4 --- /dev/null +++ b/e2e/lib.sh @@ -0,0 +1,233 @@ +# shellcheck shell=bash +# +# Shared harness for the operator's e2e scripts: a disposable kind cluster with the CRDs, +# the operator built from this checkout, a single-node RestateCluster, and the example +# "greeter" service — plus assertion/diagnostic helpers. +# +# Source this, then call only the setup steps your scenario needs (metrics-server, for +# instance, is only interesting to the autoscaling script). Override any of the config +# below by setting it *before* sourcing. +# +# Prerequisites on PATH: kind, kubectl, helm, docker, jq, just, curl. + +# ---- config --------------------------------------------------------------- +CLUSTER_NAME="${CLUSTER_NAME:-restate-operator-e2e}" +OPERATOR_NS="${OPERATOR_NS:-restate-operator}" +RESTATE_NS="${RESTATE_NS:-restate}" # namespace the operator creates for the cluster +APP_NS="${APP_NS:-default}" # namespace for the RestateDeployment +OPERATOR_IMAGE="${OPERATOR_IMAGE:-ghcr.io/restatedev/restate-operator:local}" +GREETER_IMAGE="${GREETER_IMAGE:-dev.local/restatedev/restate-operator/greeter:local}" +# In-cluster Restate ports. +CLUSTER_INGRESS_PORT=8080 +CLUSTER_ADMIN_PORT=9070 +# Local ports for our port-forward. Deliberately uncommon so we don't collide with (and +# silently hit) an unrelated Restate port-forward — e.g. a Restate Cloud control plane +# forwarded to 8080/9070. Override if these are taken. +INGRESS_PORT="${INGRESS_PORT:-18080}" +ADMIN_PORT="${ADMIN_PORT:-19070}" +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +CTX="kind-${CLUSTER_NAME}" + +# ---- output helpers ------------------------------------------------------- +RED=$'\e[31m'; GREEN=$'\e[32m'; YELLOW=$'\e[33m'; BLUE=$'\e[34m'; BOLD=$'\e[1m'; RST=$'\e[0m' +info() { echo "${BLUE}==>${RST} $*"; } +step() { echo; echo "${BOLD}### $*${RST}"; } +pass() { echo "${GREEN}PASS${RST} $*"; PASSES=$((PASSES+1)); } +fail() { echo "${RED}FAIL${RST} $*"; FAILS=$((FAILS+1)); } +warn() { echo "${YELLOW}WARN${RST} $*"; } +die() { echo "${RED}fatal:${RST} $*" >&2; exit 1; } +PASSES=0; FAILS=0; PF_PID="" + +kc() { kubectl --context "$CTX" "$@"; } + +# Poll predicate (a command/function) until it succeeds or `timeout` elapses. +# The predicate runs in the current shell, so it can use helpers and variables. +wait_until() { + local desc="$1" timeout="$2"; shift 2 + local deadline=$(( $(date +%s) + timeout )) + while true; do + if "$@" >/dev/null 2>&1; then return 0; fi + if (( $(date +%s) >= deadline )); then + warn "timed out after ${timeout}s waiting for: ${desc}" + return 1 + fi + sleep 3 + done +} + +# Elapsed seconds around a command, printed on stdout. Used to assert that a forced +# teardown really did skip the drain delay rather than merely finishing eventually. +time_it() { + local start; start=$(date +%s) + "$@" >/dev/null 2>&1 || true + echo $(( $(date +%s) - start )) +} + +e2e_trap_cleanup() { + [[ -n "$PF_PID" ]] && kill "$PF_PID" >/dev/null 2>&1 || true + if [[ "${KEEP:-0}" != "1" ]]; then + info "deleting kind cluster ${CLUSTER_NAME} (set KEEP=1 to keep it)" + kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true + else + info "KEEP=1 — leaving cluster ${CLUSTER_NAME} up" + fi +} + +# ---- setup steps ---------------------------------------------------------- +require_tools() { + step "Preflight" + local bin + for bin in kind kubectl helm docker jq just curl; do + command -v "$bin" >/dev/null 2>&1 || die "missing required tool: $bin" + done + info "all tools present" +} + +create_kind_cluster() { + step "Create kind cluster" + # Reuse an existing cluster only if it is actually reachable. kind can leave a cluster + # listed while its kubeconfig context is gone (e.g. after a partial delete); in that + # case recreate it rather than failing later. + if kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then + kind export kubeconfig --name "$CLUSTER_NAME" >/dev/null 2>&1 || true + if kc cluster-info >/dev/null 2>&1; then + info "cluster ${CLUSTER_NAME} healthy, reusing" + return 0 + fi + warn "cluster ${CLUSTER_NAME} is listed but unreachable; recreating" + kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true + fi + kind create cluster --name "$CLUSTER_NAME" +} + +install_metrics_server() { + step "Install metrics-server (kind-patched)" + kc apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + # kind kubelets serve metrics over a self-signed cert; allow it (idempotently — don't + # append the flag twice when reusing a cluster). + if ! kc -n kube-system get deploy metrics-server \ + -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null | grep -q kubelet-insecure-tls; then + kc -n kube-system patch deployment metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' + fi + kc -n kube-system rollout status deployment/metrics-server --timeout=180s +} + +install_crds() { + step "Install CRDs" + kc apply --server-side -f "${REPO_ROOT}/crd/restateclusters.yaml" + kc apply --server-side -f "${REPO_ROOT}/crd/restatedeployments.yaml" +} + +build_and_load_images() { + if [[ "${SKIP_BUILD:-0}" == "1" ]]; then + info "SKIP_BUILD=1 — reusing already-loaded images" + return 0 + fi + step "Build + load operator image" + ( cd "$REPO_ROOT" && just docker ) + kind load docker-image "$OPERATOR_IMAGE" --name "$CLUSTER_NAME" + + step "Build + load greeter image" + ( cd "$REPO_ROOT/examples/services/greeter" && just docker ) + kind load docker-image "$GREETER_IMAGE" --name "$CLUSTER_NAME" +} + +deploy_operator() { + step "Deploy operator" + helm --kube-context "$CTX" upgrade --install restate-operator \ + "${REPO_ROOT}/charts/restate-operator-helm" \ + --namespace "$OPERATOR_NS" --create-namespace \ + --set version=local --wait --timeout 180s + # force a fresh pod so a rebuilt :local image is picked up + kc -n "$OPERATOR_NS" rollout restart deployment -l app=restate-operator >/dev/null 2>&1 || true + kc -n "$OPERATOR_NS" rollout status deployment -l app=restate-operator --timeout=120s +} + +deploy_restate_cluster() { + step "Deploy + provision RestateCluster" + kc apply --server-side -f "${REPO_ROOT}/examples/cluster/cluster.yaml" + wait_until "restate-0 pod to exist" 180 kc -n "$RESTATE_NS" get pod restate-0 + kc -n "$RESTATE_NS" wait --for=condition=Ready pod/restate-0 --timeout=240s \ + || die "restate-0 did not become ready" + # single-node clusters must be provisioned once; ignore 'already provisioned'. + kc -n "$RESTATE_NS" exec restate-0 -- restatectl provision --yes >/dev/null 2>&1 \ + || kc -n "$RESTATE_NS" exec restate-0 -- restatectl provision >/dev/null 2>&1 \ + || warn "provision returned non-zero (may already be provisioned)" +} + +port_forward_restate() { + step "Port-forward Restate ingress + admin" + kc -n "$RESTATE_NS" port-forward svc/restate \ + "${INGRESS_PORT}:${CLUSTER_INGRESS_PORT}" "${ADMIN_PORT}:${CLUSTER_ADMIN_PORT}" \ + >/tmp/restate-pf.log 2>&1 & + PF_PID=$! + sleep 2 + # Fail loudly if the forward could not bind — otherwise a stale listener on these ports + # (e.g. another Restate) would silently answer our health check and curls. + if ! kill -0 "$PF_PID" 2>/dev/null || grep -q "address already in use" /tmp/restate-pf.log; then + cat /tmp/restate-pf.log >&2 + die "port-forward to Restate failed (ports ${INGRESS_PORT}/${ADMIN_PORT} in use?). Set INGRESS_PORT/ADMIN_PORT to free ports." + fi + wait_until "admin API reachable" 60 admin_healthy || die "admin API never came up" +} + +# ---- kubernetes queries --------------------------------------------------- +rs_replicas() { kc -n "$APP_NS" get rs "$1" -o jsonpath='{.spec.replicas}' 2>/dev/null; } + +# ---- admin API ------------------------------------------------------------ +admin_healthy() { curl -sf "localhost:${ADMIN_PORT}/health" >/dev/null 2>&1; } + +# Run a SQL query against the admin API, echoing the raw JSON response. +admin_query() { + curl -sS "localhost:${ADMIN_PORT}/query" \ + -H 'content-type: application/json' -H 'accept: application/json' \ + -d "{\"query\": $(jq -Rs . <<<"$1")}" 2>/dev/null +} + +# True while any non-completed invocation against Greeter exists. +greeter_has_active_invocations() { + admin_query "SELECT count(*) AS c FROM sys_invocation_status WHERE status != 'completed' AND target_service_name = 'Greeter'" \ + | jq -e '.rows[0].c > 0' >/dev/null 2>&1 +} +greeter_drained() { ! greeter_has_active_invocations; } + +# True if Restate still has a registration for this deployment id. +deployment_registered() { + admin_query "SELECT count(*) AS c FROM sys_deployment WHERE id = '$1'" \ + | jq -e '.rows[0].c > 0' >/dev/null 2>&1 +} +deployment_deregistered() { ! deployment_registered "$1"; } + +# ---- invocation helpers --------------------------------------------------- +# Fire a long async invocation, pinning it to whatever version is latest now. Returns +# immediately (Restate ingress /send) and echoes the invocation id. +slow_greet() { + local resp id + resp="$(curl -sS "localhost:${INGRESS_PORT}/Greeter/slowGreet/send" \ + -H 'content-type: application/json' \ + -d "{\"name\":\"e2e\",\"delaySeconds\":${1}}" 2>&1)" + id="$(jq -r '.invocationId // empty' <<<"$resp" 2>/dev/null)" + if [[ -z "$id" ]]; then warn "slowGreet /send rejected: ${resp}"; return 1; fi + echo "$id" +} + +# ---- diagnostics / results ------------------------------------------------ +operator_logs() { kc -n "$OPERATOR_NS" logs -l app=restate-operator --tail="${1:-200}" 2>/dev/null; } + +# Dump operator logs + resource state; call when an assertion fails, so a failing run is +# debuggable without KEEP=1. +dump_diagnostics() { + echo "${YELLOW}--- diagnostics ---${RST}" + echo " ### RestateDeployment / ReplicaSets / HPAs / pods (${APP_NS})" + kc -n "$APP_NS" get restatedeployment,rs,hpa,pods -o wide 2>&1 | sed 's/^/ /' || true + echo " ### operator logs (tail 80)" + operator_logs 80 2>&1 | sed 's/^/ /' || true +} + +e2e_results() { + step "Results" + echo " ${GREEN}${PASSES} passed${RST}, ${RED}${FAILS} failed${RST}" + (( FAILS == 0 )) || { dump_diagnostics; die "e2e had failures"; } + info "all assertions passed" +} diff --git a/e2e/per-version-hpa.sh b/e2e/per-version-hpa.sh index 118a91e..05e0915 100755 --- a/e2e/per-version-hpa.sh +++ b/e2e/per-version-hpa.sh @@ -31,63 +31,14 @@ # set -euo pipefail -# ---- config --------------------------------------------------------------- CLUSTER_NAME="${CLUSTER_NAME:-restate-hpa-e2e}" -OPERATOR_NS="${OPERATOR_NS:-restate-operator}" -RESTATE_NS="${RESTATE_NS:-restate}" # namespace the operator creates for the cluster -APP_NS="${APP_NS:-default}" # namespace for the RestateDeployment RD_NAME="${RD_NAME:-greeter-rsd}" -OPERATOR_IMAGE="ghcr.io/restatedev/restate-operator:local" -GREETER_IMAGE="dev.local/restatedev/restate-operator/greeter:local" PIN_SECONDS="${PIN_SECONDS:-150}" # how long the pinning slowGreet stays in-flight -# In-cluster Restate ports. -CLUSTER_INGRESS_PORT=8080 -CLUSTER_ADMIN_PORT=9070 -# Local ports for our port-forward. Deliberately uncommon so we don't collide -# with (and silently hit) an unrelated Restate port-forward — e.g. a Restate -# Cloud control plane forwarded to 8080/9070. Override if these are taken. -INGRESS_PORT="${INGRESS_PORT:-18080}" -ADMIN_PORT="${ADMIN_PORT:-19070}" -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CTX="kind-${CLUSTER_NAME}" - -# ---- output helpers ------------------------------------------------------- -RED=$'\e[31m'; GREEN=$'\e[32m'; YELLOW=$'\e[33m'; BLUE=$'\e[34m'; BOLD=$'\e[1m'; RST=$'\e[0m' -info() { echo "${BLUE}==>${RST} $*"; } -step() { echo; echo "${BOLD}### $*${RST}"; } -pass() { echo "${GREEN}PASS${RST} $*"; PASSES=$((PASSES+1)); } -fail() { echo "${RED}FAIL${RST} $*"; FAILS=$((FAILS+1)); } -warn() { echo "${YELLOW}WARN${RST} $*"; } -die() { echo "${RED}fatal:${RST} $*" >&2; exit 1; } -PASSES=0; FAILS=0; PF_PID="" - -kc() { kubectl --context "$CTX" "$@"; } - -# Poll predicate (a command/function) until it succeeds or `timeout` elapses. -# The predicate runs in the current shell, so it can use helpers and variables. -wait_until() { - local desc="$1" timeout="$2"; shift 2 - local deadline=$(( $(date +%s) + timeout )) - while true; do - if "$@" >/dev/null 2>&1; then return 0; fi - if (( $(date +%s) >= deadline )); then - warn "timed out after ${timeout}s waiting for: ${desc}" - return 1 - fi - sleep 3 - done -} -cleanup() { - [[ -n "$PF_PID" ]] && kill "$PF_PID" >/dev/null 2>&1 || true - if [[ "${KEEP:-0}" != "1" ]]; then - info "deleting kind cluster ${CLUSTER_NAME} (set KEEP=1 to keep it)" - kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true - else - info "KEEP=1 — leaving cluster ${CLUSTER_NAME} up" - fi -} -trap cleanup EXIT +# shellcheck source=e2e/lib.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib.sh" + +trap e2e_trap_cleanup EXIT # ---- predicates / queries ------------------------------------------------- operator_hpa_names() { @@ -97,120 +48,22 @@ operator_hpa_names() { } operator_hpa_count() { operator_hpa_names | grep -c . || true; } hpa_count_is() { [[ "$(operator_hpa_count)" == "$1" ]]; } -rs_replicas() { kc -n "$APP_NS" get rs "$1" -o jsonpath='{.spec.replicas}' 2>/dev/null; } rs_replicas_is() { [[ "$(rs_replicas "$1")" == "$2" ]]; } rs_gone_or_zero() { ! kc -n "$APP_NS" get rs "$1" >/dev/null 2>&1 || rs_replicas_is "$1" 0; } rd_ready() { kc -n "$APP_NS" get restatedeployment "$RD_NAME" \ -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -qx True } -admin_healthy() { curl -sf "localhost:${ADMIN_PORT}/health" >/dev/null 2>&1; } - -# Dump operator logs + resource state; called automatically when an assertion -# fails, so a failing run is debuggable without KEEP=1. -dump_diagnostics() { - echo "${YELLOW}--- diagnostics ---${RST}" - echo " ### RestateDeployment / ReplicaSets / HPAs / pods (${APP_NS})" - kc -n "$APP_NS" get restatedeployment,rs,hpa,pods -o wide 2>&1 | sed 's/^/ /' || true - echo " ### operator logs (tail 80)" - kc -n "$OPERATOR_NS" logs -l app=restate-operator --tail=80 2>&1 | sed 's/^/ /' || true -} -# True while any non-completed invocation against Greeter exists (best-effort; -# depends on the admin /query JSON shape — used only as a secondary signal). -greeter_has_active_invocations() { - local q='SELECT count(*) AS c FROM sys_invocation_status WHERE status != '"'"'completed'"'"' AND target_service_name = '"'"'Greeter'"'" - curl -sS "localhost:${ADMIN_PORT}/query" \ - -H 'content-type: application/json' -H 'accept: application/json' \ - -d "{\"query\": $(jq -Rs . <<<"$q")}" 2>/dev/null \ - | jq -e '.rows[0].c > 0' >/dev/null 2>&1 -} -greeter_drained() { ! greeter_has_active_invocations; } - -# ---- preflight ------------------------------------------------------------ -step "Preflight" -for bin in kind kubectl helm docker jq just curl; do - command -v "$bin" >/dev/null 2>&1 || die "missing required tool: $bin" -done -info "all tools present" - -# ---- cluster -------------------------------------------------------------- -step "Create kind cluster" -# Reuse an existing cluster only if it is actually reachable. kind can leave a -# cluster listed while its kubeconfig context is gone (e.g. after a partial -# delete); in that case recreate it rather than failing later. -if kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then - kind export kubeconfig --name "$CLUSTER_NAME" >/dev/null 2>&1 || true - if kc cluster-info >/dev/null 2>&1; then - info "cluster ${CLUSTER_NAME} healthy, reusing" - else - warn "cluster ${CLUSTER_NAME} is listed but unreachable; recreating" - kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true - kind create cluster --name "$CLUSTER_NAME" - fi -else - kind create cluster --name "$CLUSTER_NAME" -fi - -step "Install metrics-server (kind-patched)" -kc apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml -# kind kubelets serve metrics over a self-signed cert; allow it (idempotently — -# don't append the flag twice when reusing a cluster). -if ! kc -n kube-system get deploy metrics-server \ - -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null | grep -q kubelet-insecure-tls; then - kc -n kube-system patch deployment metrics-server --type=json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' -fi -kc -n kube-system rollout status deployment/metrics-server --timeout=180s - -step "Install CRDs" -kc apply --server-side -f "${REPO_ROOT}/crd/restateclusters.yaml" -kc apply --server-side -f "${REPO_ROOT}/crd/restatedeployments.yaml" - -# ---- images + operator ---------------------------------------------------- -if [[ "${SKIP_BUILD:-0}" != "1" ]]; then - step "Build + load operator image" - ( cd "$REPO_ROOT" && just docker ) - kind load docker-image "$OPERATOR_IMAGE" --name "$CLUSTER_NAME" - - step "Build + load greeter image" - ( cd "$REPO_ROOT/examples/services/greeter" && just docker ) - kind load docker-image "$GREETER_IMAGE" --name "$CLUSTER_NAME" -fi - -step "Deploy operator" -helm --kube-context "$CTX" upgrade --install restate-operator \ - "${REPO_ROOT}/charts/restate-operator-helm" \ - --namespace "$OPERATOR_NS" --create-namespace \ - --set version=local --wait --timeout 180s -# force a fresh pod so a rebuilt :local image is picked up -kc -n "$OPERATOR_NS" rollout restart deployment -l app=restate-operator >/dev/null 2>&1 || true -kc -n "$OPERATOR_NS" rollout status deployment -l app=restate-operator --timeout=120s - -# ---- restate cluster ------------------------------------------------------ -step "Deploy + provision RestateCluster" -kc apply --server-side -f "${REPO_ROOT}/examples/cluster/cluster.yaml" -wait_until "restate-0 pod to exist" 180 kc -n "$RESTATE_NS" get pod restate-0 -kc -n "$RESTATE_NS" wait --for=condition=Ready pod/restate-0 --timeout=240s \ - || die "restate-0 did not become ready" -# single-node clusters must be provisioned once; ignore 'already provisioned'. -kc -n "$RESTATE_NS" exec restate-0 -- restatectl provision --yes >/dev/null 2>&1 \ - || kc -n "$RESTATE_NS" exec restate-0 -- restatectl provision >/dev/null 2>&1 \ - || warn "provision returned non-zero (may already be provisioned)" - -step "Port-forward Restate ingress + admin" -kc -n "$RESTATE_NS" port-forward svc/restate \ - "${INGRESS_PORT}:${CLUSTER_INGRESS_PORT}" "${ADMIN_PORT}:${CLUSTER_ADMIN_PORT}" \ - >/tmp/restate-pf.log 2>&1 & -PF_PID=$! -sleep 2 -# Fail loudly if the forward could not bind — otherwise a stale listener on these -# ports (e.g. another Restate) would silently answer our health check and curls. -if ! kill -0 "$PF_PID" 2>/dev/null || grep -q "address already in use" /tmp/restate-pf.log; then - cat /tmp/restate-pf.log >&2 - die "port-forward to Restate failed (ports ${INGRESS_PORT}/${ADMIN_PORT} in use?). Set INGRESS_PORT/ADMIN_PORT to free ports." -fi -wait_until "admin API reachable" 60 admin_healthy || die "admin API never came up" +# ---- setup ---------------------------------------------------------------- +require_tools +create_kind_cluster +install_metrics_server +install_crds +build_and_load_images +deploy_operator +deploy_restate_cluster +port_forward_restate # ---- manifest + invocation helpers ---------------------------------------- # Emit a RestateDeployment for the given version. Short drainDelay + zero history @@ -285,9 +138,7 @@ deploy_version() { rd_manifest "$1" | kc apply -f - ; } PIN_DEADLINE=0 pin_invocation() { local id - id="$(curl -sS "localhost:${INGRESS_PORT}/Greeter/slowGreet/send" \ - -H 'content-type: application/json' \ - -d "{\"name\":\"drain\",\"delaySeconds\":${PIN_SECONDS}}" | jq -r '.invocationId // empty')" + id="$(slow_greet "$PIN_SECONDS" || true)" PIN_DEADLINE=$(( $(date +%s) + PIN_SECONDS )) [[ -n "$id" ]] && info "pinned invocation ${id} (in-flight ${PIN_SECONDS}s)" \ || warn "could not capture invocation id from ingress /send" @@ -389,15 +240,7 @@ burn_on_current() { # Fire a long idle (suspended) keep-alive pinned to the current latest version, # so it stays "active" in Restate — keeping its HPA and avoiding drain — even # after the burst load ends, letting us see it scale back to floor (not vanish). -keepalive_on_current() { - local resp id - resp="$(curl -sS "localhost:${INGRESS_PORT}/Greeter/slowGreet/send" \ - -H 'content-type: application/json' \ - -d "{\"name\":\"keepalive\",\"delaySeconds\":${KEEPALIVE_SECONDS}}" 2>&1)" - id="$(jq -r '.invocationId // empty' <<<"$resp" 2>/dev/null)" - if [[ -z "$id" ]]; then warn "keepalive /send rejected: ${resp}"; return 1; fi - echo "$id" -} +keepalive_on_current() { slow_greet "$KEEPALIVE_SECONDS"; } cpu_util() { kc -n "$APP_NS" get hpa "$1" -o jsonpath='{.status.currentMetrics[0].resource.current.averageUtilization}' 2>/dev/null; } @@ -616,7 +459,4 @@ else fi # =========================================================================== -step "Results" -echo " ${GREEN}${PASSES} passed${RST}, ${RED}${FAILS} failed${RST}" -(( FAILS == 0 )) || { dump_diagnostics; die "e2e had failures"; } -info "all assertions passed" +e2e_results