From d2303d32f4ecbbc53a9a2012d4ef033d0b07be24 Mon Sep 17 00:00:00 2001 From: Michal Pryc Date: Wed, 1 Jul 2026 17:42:07 +0200 Subject: [PATCH 1/3] Add shared dev environment for medik8s operators Shared Kind-based development environment for all medik8s operators. Provides consistent cluster setup, operator deployment, failure simulation, and observability across NHC, SNR, FAR, MDR, NMO, and SBR. Features: - Kind cluster with 1 CP + 3 workers, cert-manager, OLM, softdog - Build/deploy operators with automatic cert-manager webhook configuration - Failure simulations: kubelet stop, network partition, storm - Observability: logs, describe, summary, events - External cluster support (OCP, etc.) with SKIP_KIND=true - ttl.sh ephemeral registry support (DEV_REGISTRY=ttl.sh) - Rootless podman support with cpuset cgroup detection - Shellcheck CI via GitHub Actions Signed-off-by: Michal Pryc Co-Authored-By: Claude Opus 4.6 --- .github/workflows/shellcheck.yml | 21 ++ README.md | 26 +++ dev/README.md | 244 +++++++++++++++++++++++ dev/common.sh | 31 +++ dev/create-nhc.sh | 56 ++++++ dev/describe.sh | 106 ++++++++++ dev/dev.mk | 325 +++++++++++++++++++++++++++++++ dev/enable-certmanager.sh | 112 +++++++++++ dev/kind-config.yaml | 15 ++ dev/setup.sh | 244 +++++++++++++++++++++++ dev/simulate-failure.sh | 174 +++++++++++++++++ dev/summary.sh | 56 ++++++ dev/teardown.sh | 43 ++++ 13 files changed, 1453 insertions(+) create mode 100644 .github/workflows/shellcheck.yml create mode 100644 dev/README.md create mode 100755 dev/common.sh create mode 100755 dev/create-nhc.sh create mode 100755 dev/describe.sh create mode 100644 dev/dev.mk create mode 100755 dev/enable-certmanager.sh create mode 100644 dev/kind-config.yaml create mode 100755 dev/setup.sh create mode 100755 dev/simulate-failure.sh create mode 100755 dev/summary.sh create mode 100755 dev/teardown.sh diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000..917fe63 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,21 @@ +name: shellcheck + +on: + push: + branches: [main] + paths: ["dev/**/*.sh"] + pull_request: + paths: ["dev/**/*.sh"] + +jobs: + shellcheck: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Run shellcheck + uses: ludeeus/action-shellcheck@2.0.0 + with: + scandir: dev + severity: warning diff --git a/README.md b/README.md index acb91fa..d850063 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,28 @@ # tools Tools etc. which are not specific for one of the medik8s operators + +## Dev Environment + +Shared development environment for all medik8s operators. See [dev/README.md](dev/README.md) for full documentation. + +### Quick Start + +```bash +# 1. Add to your operator's Makefile (one-time): +# TOOLS_DIR ?= $(shell cd .. && pwd)/tools +# -include $(TOOLS_DIR)/dev/dev.mk + +# 2. Create the dev cluster: +make dev-setup + +# 3. Build and deploy your operator: +make dev-deploy + +# 4. Simulate failures: +make dev-simulate-failure +``` + +## Other Tools + +- `findIndexImage/` — Find OCP index images for IIB discovery +- `scripts/` — Build and deploy scripts for NHC+SNR diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..30e4083 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,244 @@ +# Medik8s Development Environment + +A shared, consistent development environment for all medik8s operators. + +## Quick Start + +```bash +# One-time: add the dev environment snippet to your operator's Makefile +# (see Setup section below for options) + +make dev-setup # Create Kind cluster +make dev-deploy # Build and deploy operator +make dev-describe # Verify everything is running +make dev-simulate-failure && kubectl get nodes -w # Test remediation +make dev-recover # Restore cluster +make dev-teardown # Clean up +``` + +### Full Walkthrough (NHC + SNR) + +This walks through the complete remediation flow: deploy both operators, +simulate a node failure, and watch NHC trigger SNR to remediate it. + +```bash +# 1. Create the Kind cluster (from any operator directory) +pushd self-node-remediation +make dev-setup +popd + +# 2. Deploy SNR +pushd self-node-remediation +make dev-deploy +popd + +# 3. Deploy NHC (auto-creates NodeHealthCheck CR linking to SNR) +pushd node-healthcheck-operator +make dev-deploy +popd + +# 4. Verify everything is running +make dev-wait +make dev-describe + +# 5. Simulate a node failure +make dev-simulate-failure + +# 6. Watch the remediation flow (run in separate terminals) +kubectl get nodes -w +kubectl get selfnoderemediation -A -w +make dev-events + +# 7. Debug if needed +make dev-logs # tail operator logs +make dev-describe # full resource summary +make dev-summary # remediation flow timeline +make dev-shell # shell into a worker node + +# 8. Recover all workers +make dev-recover + +# 9. Teardown +make dev-teardown +``` + +## Prerequisites + +- [Go](https://go.dev/doc/install) (version matching the operator's go.mod) +- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) (v0.20+) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) or [oc](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/) +- [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/getting-started/installation) +- [operator-sdk](https://sdk.operatorframework.io/docs/installation/) (optional, for OLM bundle testing) + +### System limits + +The setup script checks inotify limits and exits with an error if they are too low (auto-fixes when running as root). Fix manually before running `make dev-setup`: + +```bash +sudo sysctl -w fs.inotify.max_user_instances=8192 +sudo sysctl -w fs.inotify.max_user_watches=524288 +``` + +To make persistent, add to `/etc/sysctl.d/99-kind.conf`: +``` +fs.inotify.max_user_instances=8192 +fs.inotify.max_user_watches=524288 +``` + +### Podman rootless setup + +Rootless podman requires `cpuset` cgroup delegation for Kind worker nodes. Without it, kubelet cannot start inside the containers. The setup script detects this and prints instructions. + +```bash +sudo mkdir -p /etc/systemd/system/user@.service.d +sudo tee /etc/systemd/system/user@.service.d/delegate.conf <> .gitignore +``` + +To update the downloaded copy, delete it and re-run: +```bash +rm -rf .tools && make dev-help +``` + +All `dev-*` targets are now available. + +## What It Creates + +- **1 control-plane + 3 worker nodes** (SNR needs 2+ workers for peer health; multi-CP requires a real cluster) +- **Worker labels** (`node-role.kubernetes.io/worker`) +- **softdog** kernel module on workers (for SNR watchdog testing) +- **Namespaces**: `medik8s-system` (privileged PSA) and `medik8s-leases` for shared resources +- **cert-manager** (required for operator webhook TLS certificates) +- **inotify limits** increased automatically when running as root +- **OLM** (if operator-sdk is available) + +Re-running `make dev-setup` on an existing cluster is safe — it re-applies configuration without recreating. + +**Note:** Each operator deploys into its own namespace (e.g. `self-node-remediation`, `node-healthcheck-operator-system`) as defined in its kustomization files (either `config/default/kustomization.yaml` or a component/patch kustomization). The `dev-deploy` target automatically creates cert-manager certificates, patches the deployment to mount webhook TLS secrets, and waits for the deployment to become ready. If both NHC and SNR are deployed, it also creates a NodeHealthCheck CR linking them. + +## Failure Simulations + +| Command | What it does | +|---------|-------------| +| `make dev-simulate-failure` | Stop kubelet on a worker → node goes NotReady → NHC creates remediation CR | +| `make dev-simulate-network` | Block API server from a worker → tests SNR peer health decisions | +| `make dev-simulate-storm` | Stop kubelet on 2/3 workers → NHC detects storm, pauses remediation | +| `make dev-recover` | Restart kubelet, restore network, clean up CRs | + +## All Targets + +| Target | Description | +|--------|-------------| +| `dev-setup` | Create Kind cluster with all dependencies (or configure external cluster with `SKIP_KIND=true`) | +| `dev-teardown` | Destroy Kind cluster | +| `dev-build` | Build operator image and load into Kind (or push to ttl.sh) | +| `dev-deploy` | Build + install CRDs + deploy + configure cert-manager | +| `dev-redeploy` | Rebuild and restart (deletes pods to pick up new image) | +| `dev-undeploy` | Remove operator from cluster | +| `dev-create-nhc` | Create NodeHealthCheck CR referencing SNR | +| `dev-logs` | Tail operator controller-manager logs | +| `dev-describe` | Full summary (nodes, pods, CRs, leases, events) | +| `dev-summary` | Remediation flow timeline | +| `dev-events` | Show recent remediation-related events | +| `dev-wait` | Wait for all operator pods to be ready | +| `dev-shell` | Open a shell on a Kind node (`NODE=`, default: first worker) | +| `dev-simulate-failure` | Stop kubelet on a worker | +| `dev-simulate-storm` | Stop kubelet on 2 workers | +| `dev-simulate-network` | Block API server from a worker | +| `dev-recover` | Recover all workers and clean up | +| `dev-help` | Show all dev targets | + +## Using an Existing Cluster (OCP, etc.) + +For external clusters (OCP, etc.), set `SKIP_KIND=true` to skip Kind +creation. Images are pushed to [ttl.sh](https://ttl.sh) — an anonymous, +ephemeral registry that requires no auth. + +```bash +# Point kubectl at your cluster, then use the same workflow +export KUBECONFIG=~/.kube/my-ocp-cluster +export SKIP_KIND=true + +make dev-setup # Configures namespaces + cert-manager (no Kind) +make dev-deploy # Builds image, pushes to ttl.sh, deploys +make dev-describe # Verify everything is running +``` + +Exporting `SKIP_KIND=true` ensures all targets know this is an external cluster. +Without it, auto-detection might find a local Kind cluster and use the wrong +image delivery method. + +You can also use ttl.sh with a Kind cluster: + +```bash +DEV_REGISTRY=ttl.sh make dev-deploy +``` + +**Note:** Failure simulations (`dev-simulate-failure`, etc.) require +`docker exec`/`podman exec` access to Kind node containers. On external +clusters, trigger failures through your cluster's own mechanisms. + +## Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `SKIP_KIND` | `false` | Set to `true` to skip Kind creation (external cluster) | +| `DEV_REGISTRY` | `local` (Kind) / `ttl.sh` (external) | Image delivery: `local` or `ttl.sh` | +| `DEV_IMG` | auto-generated | Override to use a custom image name | +| `TTL_SH_TTL` | `2h` | Image expiry when using ttl.sh | +| `MEDIK8S_CLUSTER_NAME` | `medik8s-dev` | Kind cluster name | +| `CONTAINER_TOOL` | auto-detected | `docker` or `podman` | +| `KUBECTL` | auto-detected | `kubectl` or `oc` | + +## Operator Coverage + +| Operator | Coverage | Notes | +|----------|----------|-------| +| NHC | Full | Node conditions, storm recovery, escalation, CP protection | +| NMO | Full | Cordon, drain, PDB-aware eviction | +| SNR | ~85% | Peer health, softdog watchdog, API check. No hardware watchdog. | +| FAR | Controller logic | No real fence agents — controller reconciliation is testable | +| MDR | Controller logic | No Machine API — reconciliation testable via envtest (`make test`) | +| SBR | Unit only | No shared storage | + +## Limitations + +- **FAR fence agent execution** — no IPMI/BMC or cloud APIs +- **MDR Machine API** — Kind has no Machine objects +- **SBR shared storage** — no ODF +- **Hardware watchdog** — only softdog (software) + +For full-fidelity testing, use an OpenShift cluster. diff --git a/dev/common.sh b/dev/common.sh new file mode 100755 index 0000000..2116bdb --- /dev/null +++ b/dev/common.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Shared helpers for medik8s dev scripts. +# Source this from other scripts: source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +# Detect kubectl or oc +detect_kubectl() { + if command -v kubectl &>/dev/null; then + echo kubectl + elif command -v oc &>/dev/null; then + echo oc + else + echo "Error: kubectl or oc is required but neither is installed." >&2 + echo "Install kubectl from: https://kubernetes.io/docs/tasks/tools/" >&2 + exit 1 + fi +} + +# Detect container tool (docker preferred, podman fallback) +detect_container_tool() { + if command -v docker &>/dev/null; then + echo docker + elif command -v podman &>/dev/null; then + echo podman + else + echo "Error: docker or podman is required but neither is installed." >&2 + exit 1 + fi +} + +KUBECTL="${KUBECTL:-$(detect_kubectl)}" +CONTAINER_TOOL="${CONTAINER_TOOL:-$(detect_container_tool)}" diff --git a/dev/create-nhc.sh b/dev/create-nhc.sh new file mode 100755 index 0000000..637b285 --- /dev/null +++ b/dev/create-nhc.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Creates a NodeHealthCheck CR that references SNR for remediation. +# Usage: create-nhc.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +# Check if the NHC CRD exists +if ! ${KUBECTL} get crd nodehealthchecks.remediation.medik8s.io &>/dev/null; then + echo "Error: NodeHealthCheck CRD not found. Deploy NHC first (make dev-deploy from the NHC directory)." + exit 1 +fi + +# Find the SNR template namespace +SNR_NS=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) +SNR_TEMPLATE=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + +if [ -z "${SNR_NS}" ] || [ -z "${SNR_TEMPLATE}" ]; then + echo "Error: No SelfNodeRemediationTemplate found. Deploy SNR first (make dev-deploy from the SNR directory)." + exit 1 +fi + +echo "Creating NodeHealthCheck CR referencing ${SNR_TEMPLATE} in ${SNR_NS}..." + +${KUBECTL} apply -f - </dev/null || true +echo "" + +echo "--- Operator Pods ---" +${KUBECTL} get pods -A -l control-plane=controller-manager \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp \ + 2>/dev/null || echo " (none)" +echo "" + +# NodeHealthCheck +if ${KUBECTL} get crd nodehealthchecks.remediation.medik8s.io >/dev/null 2>&1; then + echo "--- NodeHealthCheck ---" + NHCS=$(${KUBECTL} get nodehealthcheck --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null) + if [ -n "$NHCS" ]; then + for nhc in $NHCS; do + echo "" + echo " ${nhc}:" + PHASE=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.phase}' 2>/dev/null) + HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.healthyNodes}' 2>/dev/null) + OBSERVED=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.observedNodes}' 2>/dev/null) + UNHEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.unhealthyNodes}' 2>/dev/null) + MIN_HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.spec.minHealthy}' 2>/dev/null) + echo " Phase: ${PHASE:-N/A}" + echo " Nodes: ${HEALTHY:-0} healthy / ${OBSERVED:-0} observed (${UNHEALTHY:-0} unhealthy)" + echo " MinHealthy: ${MIN_HEALTHY:-N/A}" + done + else + echo " (none) — create with: make dev-create-nhc" + fi + echo "" +fi + +# SelfNodeRemediation +if ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- SelfNodeRemediation ---" + if ${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | grep -q .; then + ${KUBECTL} get selfnoderemediation -A \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STARTED:.status.startTime,PHASE:.status.phase \ + 2>/dev/null + else + echo " (none)" + fi + echo "" + echo " Templates:" + ${KUBECTL} get selfnoderemediationtemplate -A --no-headers 2>/dev/null \ + | awk '{print " " $2 " (ns: " $1 ")"}' || echo " (none)" + echo "" + echo " Config:" + ${KUBECTL} get selfnoderemediationconfig -A --no-headers 2>/dev/null \ + | awk '{print " " $2 " (ns: " $1 ")"}' || echo " (none)" + echo "" +fi + +# FenceAgentsRemediation +if ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- FenceAgentsRemediation ---" + ${KUBECTL} get fenceagentsremediation -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# MachineDeletionRemediation +if ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- MachineDeletionRemediation ---" + ${KUBECTL} get machinedeletionremediation -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# NodeMaintenance +if ${KUBECTL} get crd nodemaintenances.nodemaintenance.medik8s.io >/dev/null 2>&1; then + echo "--- NodeMaintenance ---" + ${KUBECTL} get nodemaintenance -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# Leases +if ${KUBECTL} get namespace medik8s-leases >/dev/null 2>&1; then + echo "--- Leases (medik8s-leases) ---" + if ${KUBECTL} get leases -n medik8s-leases --no-headers 2>/dev/null | grep -q .; then + ${KUBECTL} get leases -n medik8s-leases 2>/dev/null + else + echo " (none)" + fi + echo "" +fi + +# Recent events +echo "--- Recent Remediation Events ---" +${KUBECTL} get events -A --sort-by=.lastTimestamp 2>/dev/null \ + | grep -iE 'remediat|healthcheck|maintenance|fence|unhealthy|notready' \ + | tail -10 || echo " (none)" diff --git a/dev/dev.mk b/dev/dev.mk new file mode 100644 index 0000000..8751f94 --- /dev/null +++ b/dev/dev.mk @@ -0,0 +1,325 @@ +# dev.mk — Shared Makefile targets for medik8s local development +# +# Include this from any operator's Makefile: +# TOOLS_DIR ?= $(shell cd .. && pwd)/tools +# -include $(TOOLS_DIR)/dev/dev.mk +# +# All targets are prefixed with 'dev-' to avoid collisions. + +# Dev environment configuration +MEDIK8S_CLUSTER_NAME ?= medik8s-dev +MEDIK8S_NAMESPACE ?= medik8s-system +TOOLS_DIR ?= $(shell cd .. && pwd)/tools +DEV_DIR := $(TOOLS_DIR)/dev + +# Detect cluster type: "kind" if a Kind cluster exists, "external" otherwise. +# Uses CONTAINER_TOOL to set KIND_EXPERIMENTAL_PROVIDER (needed for podman). +# When SKIP_KIND=true, force external mode (the user explicitly opted out of Kind). +# Override with DEV_CLUSTER_TYPE=external to force external mode in other cases. +ifeq ($(SKIP_KIND),true) + DEV_CLUSTER_TYPE ?= external +else + DEV_CLUSTER_TYPE ?= $(shell \ + if KIND_EXPERIMENTAL_PROVIDER=$(CONTAINER_TOOL) kind get clusters 2>/dev/null | grep -q '^$(MEDIK8S_CLUSTER_NAME)$$'; then echo kind; \ + elif $(KUBECTL) cluster-info --context 'kind-$(MEDIK8S_CLUSTER_NAME)' >/dev/null 2>&1; then echo kind; \ + else echo external; \ + fi \ + ) +endif + +# Image delivery: +# local: loaded directly into Kind nodes (no registry, no pull) +# ttl.sh: pushed to ttl.sh (anonymous, ephemeral, no auth required) +# +# Defaults to "local" for Kind clusters, "ttl.sh" for external. +# Set DEV_REGISTRY=ttl.sh to force using ttl.sh even with Kind. +DEV_REGISTRY ?= $(if $(filter kind,$(DEV_CLUSTER_TYPE)),local,ttl.sh) +TTL_SH_TTL ?= 2h +ifeq ($(DEV_REGISTRY),local) + DEV_IMG ?= localhost:5000/medik8s/$(OPERATOR_NAME):dev +else + DEV_IMG ?= ttl.sh/medik8s-$(OPERATOR_NAME)-$(shell echo $$USER | head -c 8):$(TTL_SH_TTL) +endif + +# CONTAINER_TOOL defines the container tool to be used for building images. +# By default uses docker if available, falls back to podman. +# You may also set CONTAINER_TOOL directly as an environment variable. +CONTAINER_TOOL ?= $(shell \ + if command -v docker >/dev/null 2>&1; then echo docker; \ + elif command -v podman >/dev/null 2>&1; then echo podman; \ + else echo ""; \ + fi \ +) +ifeq ($(CONTAINER_TOOL),) + $(error No container tool found. Please install docker or podman.) +endif + +# Detect kubectl or oc +KUBECTL ?= $(shell \ + if command -v kubectl >/dev/null 2>&1; then echo kubectl; \ + elif command -v oc >/dev/null 2>&1; then echo oc; \ + else echo ""; \ + fi \ +) +ifeq ($(KUBECTL),) + $(error No kubectl or oc found. Please install kubectl or oc.) +endif + +# Verify Go is available +ifeq ($(shell command -v go 2>/dev/null),) + $(error Go not found. Please install Go from https://go.dev/doc/install or add it to your PATH.) +endif + +# Warn early if OPERATOR_NAME is not set — dev-build and dev-deploy will fail without it. +ifndef OPERATOR_NAME + $(warning OPERATOR_NAME is not set. Targets dev-build, dev-deploy, dev-redeploy will not work.) +endif + +export MEDIK8S_CLUSTER_NAME +export MEDIK8S_NAMESPACE + +# Helper to find the namespace for this operator's deployment. +# Looks for deployments with the controller-manager label, filtering by operator name. +_dev_find_ns = $(shell \ + NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)\|$(subst -,.,$(OPERATOR_NAME))' | head -1); \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null | grep '$(OPERATOR_NAME)' | awk '{print $$1}' | head -1); \ + fi; \ + echo "$$NS" \ +) + +##@ Dev Environment + +.PHONY: dev-setup +dev-setup: ## Create Kind cluster and configure dependencies (use SKIP_KIND=true for existing clusters) +ifeq ($(SKIP_KIND),true) + @$(DEV_DIR)/setup.sh --skip-kind +else + @$(DEV_DIR)/setup.sh +endif + +.PHONY: dev-teardown +dev-teardown: ## Destroy the Kind dev cluster +ifeq ($(SKIP_KIND),true) + @echo "External cluster — nothing to tear down. Use 'make dev-undeploy' to remove operators." +else + @$(DEV_DIR)/teardown.sh +endif + +.PHONY: dev-build +dev-build: ## Build operator image and load into Kind or push to ttl.sh + @# For Kind: patch imagePullPolicy to IfNotPresent (no registry, images loaded directly). + @# For external: keep imagePullPolicy as Always (image pulled from ttl.sh). + @# The SNR controller reconciles DaemonSets from templates baked into the image, + @# so this must be done before the container build, not after. + @# Files are restored after build (even on failure) via trap. +ifeq ($(DEV_REGISTRY),local) + @patched=""; \ + for f in $$(find install/ -name '*.yaml' 2>/dev/null); do \ + if grep -q 'imagePullPolicy: Always' "$$f"; then \ + sed -i 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/' "$$f"; \ + patched="$$patched $$f"; \ + echo " Patched $$f imagePullPolicy for dev build."; \ + fi; \ + done; \ + restore() { for f in $$patched; do sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: Always/' "$$f"; done; }; \ + trap restore EXIT; \ + $(CONTAINER_TOOL) build -t $(DEV_IMG) . + $(CONTAINER_TOOL) save -o /tmp/dev-image-$(OPERATOR_NAME).tar $(DEV_IMG) + KIND_EXPERIMENTAL_PROVIDER=$(if $(filter podman,$(CONTAINER_TOOL)),podman,docker) \ + kind load image-archive /tmp/dev-image-$(OPERATOR_NAME).tar --name $(MEDIK8S_CLUSTER_NAME) + rm -f /tmp/dev-image-$(OPERATOR_NAME).tar +else + $(CONTAINER_TOOL) build -t $(DEV_IMG) . + $(CONTAINER_TOOL) push $(DEV_IMG) + @echo "" + @echo " Image pushed to $(DEV_IMG)" + @echo " Image will expire after $(TTL_SH_TTL)." +endif + +.PHONY: dev-deploy +dev-deploy: dev-build install $(if $(ENVSUBST),envsubst) ## Build, load image, install CRDs, and deploy operator + cd config/manager && $(KUSTOMIZE) edit set image controller=$(DEV_IMG) + @# Use envsubst if the operator provides one (SNR uses ${IMG} in manifests) + @ENVSUBST_BIN="$(ENVSUBST)"; \ + if [ -n "$$ENVSUBST_BIN" ] && [ -x "$$ENVSUBST_BIN" ]; then \ + export IMG=$(DEV_IMG) && $(KUSTOMIZE) build config/default | $$ENVSUBST_BIN | $(KUBECTL) apply -f -; \ + else \ + $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -; \ + fi + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + @# Detect the operator namespace from kustomization files (reliable, no cluster query needed). + @# The namespace may be in config/default/ or in a component/patch kustomization.yaml. + @NS=$$(grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null | head -1 | awk '{print $$2}'); \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)' | head -1); \ + fi; \ + if [ -n "$$NS" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + SVC=$$($(KUBECTL) get svc -n $$NS --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | grep webhook | head -1); \ + if [ -n "$$DEPLOY" ] && [ -n "$$SVC" ]; then \ + $(DEV_DIR)/enable-certmanager.sh $$NS $$DEPLOY $$SVC; \ + else \ + echo " Skipping cert-manager setup (no webhook service found)."; \ + fi; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -n "$$DEPLOY" ]; then \ + echo "=== Waiting for operator deployment to be ready ==="; \ + $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ + echo "Warning: deployment $$DEPLOY is not ready. Check logs with 'make dev-logs'."; \ + fi; \ + else \ + echo " Skipping cert-manager setup (deployment not found)."; \ + fi + @# Create NHC CR after webhooks are ready (cert-manager must be configured first) + @if $(KUBECTL) get crd nodehealthchecks.remediation.medik8s.io &>/dev/null && \ + $(KUBECTL) get selfnoderemediationtemplate -A --no-headers 2>/dev/null | grep -q .; then \ + $(DEV_DIR)/create-nhc.sh; \ + fi + +.PHONY: dev-undeploy +dev-undeploy: ## Remove operator from dev cluster + @if [ -n "$(KUSTOMIZE)" ] && [ -f config/default/kustomization.yaml ]; then \ + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found -f -; \ + else \ + NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + echo "Deleting namespace $$NS..."; \ + $(KUBECTL) delete namespace "$$NS" --ignore-not-found; \ + else \ + echo "No operator deployment found to undeploy."; \ + fi; \ + fi + +.PHONY: dev-redeploy +dev-redeploy: dev-build ## Rebuild image and restart operator pods (deletes pods to pick up new image) + @NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + echo "Deleting operator pods in $$NS to pick up new image..."; \ + $(KUBECTL) delete pods -n $$NS -l control-plane=controller-manager --force --grace-period=0 2>/dev/null || true; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -n "$$DEPLOY" ]; then \ + $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ + echo "Warning: deployment is not ready. Check logs with 'make dev-logs'."; \ + fi; \ + else \ + echo "Could not find deployment to restart. Run 'make dev-deploy' first."; \ + fi + +.PHONY: dev-logs +dev-logs: ## Tail operator controller-manager logs + @NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + POD=$$($(KUBECTL) get pods -n $$NS -l control-plane=controller-manager -o name 2>/dev/null | head -1); \ + if [ -n "$$POD" ]; then \ + $(KUBECTL) logs -f -n $$NS $$POD --all-containers --tail=50; \ + else \ + echo "No controller-manager pod found in $$NS. Is the operator running?"; \ + fi; \ + else \ + echo "No controller-manager pod found. Run 'make dev-deploy' first."; \ + fi + +.PHONY: dev-wait +dev-wait: ## Wait for all medik8s operator pods to be ready + @echo "=== Waiting for operator deployments to be ready ===" + @FOUND=false; \ + for ns in $$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do \ + for deploy in $$($(KUBECTL) get deployment -n $$ns -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do \ + FOUND=true; \ + echo " Waiting for $$ns/$$deploy..."; \ + $(KUBECTL) wait --for=condition=Available deployment/$$deploy -n $$ns --timeout=120s || \ + echo " Warning: $$ns/$$deploy is not ready."; \ + done; \ + done; \ + if [ "$$FOUND" = false ]; then \ + echo " No operator deployments found. Run 'make dev-deploy' first."; \ + fi + +.PHONY: dev-events +dev-events: ## Show recent events related to medik8s resources + @echo "=== Recent Events (last 10 minutes) ===" + @$(KUBECTL) get events -A --sort-by=.lastTimestamp --field-selector reason!=Pulling,reason!=Pulled 2>/dev/null | \ + grep -iE 'remediat|healthcheck|maintenance|fence|unhealthy|notready|taint' || \ + echo " No remediation-related events found." + @echo "" + @echo "=== All Recent Events ===" + @$(KUBECTL) get events -A --sort-by=.lastTimestamp 2>/dev/null | tail -20 + +.PHONY: dev-summary +dev-summary: ## Show remediation flow timeline (what happened during simulate/recover) + @$(DEV_DIR)/summary.sh + +.PHONY: dev-describe +dev-describe: ## Full summary of all medik8s resources (nodes, pods, CRs, leases, events) + @$(DEV_DIR)/describe.sh + +.PHONY: dev-shell +dev-shell: ## Open a shell on a Kind node (use NODE=, default: first worker) + @NODES=$$(kind get nodes --name $(MEDIK8S_CLUSTER_NAME) 2>/dev/null); \ + if [ -z "$$NODES" ]; then \ + NODES=$$($(KUBECTL) get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); \ + fi; \ + if [ -z "$$NODES" ]; then \ + echo "No nodes found. Is the cluster running?"; \ + exit 1; \ + fi; \ + TARGET="$(NODE)"; \ + if [ -z "$$TARGET" ]; then \ + TARGET=$$(echo "$$NODES" | grep worker | head -1); \ + fi; \ + if [ -z "$$TARGET" ]; then \ + TARGET=$$(echo "$$NODES" | head -1); \ + fi; \ + echo "Opening shell on $$TARGET..."; \ + echo " (type 'exit' to return)"; \ + $(CONTAINER_TOOL) exec -it "$$TARGET" bash + +.PHONY: dev-create-nhc +dev-create-nhc: ## Create a NodeHealthCheck CR that triggers SNR remediation + @$(DEV_DIR)/create-nhc.sh + +.PHONY: dev-simulate-failure +dev-simulate-failure: ## Stop kubelet on a worker to trigger remediation (use SCENARIO= for other scenarios) + @$(DEV_DIR)/simulate-failure.sh $(or $(SCENARIO),kubelet-stop) + +.PHONY: dev-simulate-storm +dev-simulate-storm: ## Simulate storm: stop kubelet on 2 workers + @$(DEV_DIR)/simulate-failure.sh storm + +.PHONY: dev-simulate-network +dev-simulate-network: ## Block API server from a worker to test SNR peer health + @$(DEV_DIR)/simulate-failure.sh network-partition + +.PHONY: dev-recover +dev-recover: ## Recover all workers (restart kubelet, restore network, clean CRs) + @$(DEV_DIR)/simulate-failure.sh recover + +.PHONY: dev-help +dev-help: ## Show dev environment help + @echo "Medik8s Development Environment" + @echo "" + @echo "Lifecycle:" + @echo " make dev-setup Create Kind cluster (1 CP + 3 workers)" + @echo " make dev-teardown Destroy cluster" + @echo "" + @echo "Build & Deploy:" + @echo " make dev-build Build image and load into Kind" + @echo " make dev-deploy Build + install CRDs + deploy operator" + @echo " make dev-redeploy Rebuild and restart (fast iteration)" + @echo " make dev-undeploy Remove operator from cluster" + @echo " make dev-create-nhc Create NodeHealthCheck CR (links NHC to SNR)" + @echo "" + @echo "Observe:" + @echo " make dev-logs Tail operator logs" + @echo " make dev-describe Full summary (nodes, pods, CRs, leases, events)" + @echo " make dev-summary Remediation flow timeline" + @echo " make dev-events Show recent remediation-related events" + @echo " make dev-wait Wait for all operator pods to be ready" + @echo " make dev-shell Open shell on a Kind node (NODE=)" + @echo "" + @echo "Simulate Failures:" + @echo " make dev-simulate-failure Stop kubelet on a worker" + @echo " make dev-simulate-storm Stop kubelet on 2 workers (storm test)" + @echo " make dev-simulate-network Block API server from a worker" + @echo " make dev-recover Recover all workers" diff --git a/dev/enable-certmanager.sh b/dev/enable-certmanager.sh new file mode 100755 index 0000000..021aedf --- /dev/null +++ b/dev/enable-certmanager.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Creates cert-manager Certificate for an operator's webhook and patches +# the deployment to mount the TLS secret. +# Usage: enable-certmanager.sh + +set -euo pipefail + +if [ $# -lt 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +NAMESPACE="$1" +DEPLOY_NAME="$2" +SERVICE_NAME="$3" + +# Check if certificate already exists and is ready AND deployment has the volume mount +if ${KUBECTL} get certificate serving-cert -n "${NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q True; then + if ${KUBECTL} get deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.volumes[*].name}' 2>/dev/null | grep -q cert; then + echo " cert-manager already configured in ${NAMESPACE}." + exit 0 + fi + echo " Certificate ready but deployment needs patching..." +fi + +echo " Creating cert-manager Issuer and Certificate in ${NAMESPACE}..." + +# Retry the apply — cert-manager webhook may not be ready immediately after deployment +for i in $(seq 1 30); do + if ${KUBECTL} apply -f - </dev/null +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: ${NAMESPACE} +spec: + selfSigned: {} +INNEREOF + then + break + fi + if [ "$i" -eq 1 ]; then + echo " Waiting for cert-manager webhook to accept requests..." + fi + if [ "$i" -eq 30 ]; then + echo " Error: cert-manager webhook not ready after 30s." >&2 + exit 1 + fi + sleep 1 +done + +${KUBECTL} apply -f - </dev/null); do + # Only annotate webhooks that reference services in our namespace + if ${KUBECTL} get "${wh}" -o yaml 2>/dev/null | grep -q "namespace: ${NAMESPACE}"; then + ${KUBECTL} annotate "${wh}" cert-manager.io/inject-ca-from="${NAMESPACE}/serving-cert" --overwrite 2>/dev/null || true + fi + done +done + +# Patch deployment to mount the TLS secret at the path controller-runtime expects +echo " Patching deployment to mount webhook TLS secret..." +${KUBECTL} patch deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --type=strategic -p='{ + "spec": { + "template": { + "spec": { + "volumes": [{ + "name": "cert", + "secret": { + "secretName": "webhook-server-cert", + "defaultMode": 420 + } + }], + "containers": [{ + "name": "manager", + "volumeMounts": [{ + "name": "cert", + "mountPath": "/tmp/k8s-webhook-server/serving-certs", + "readOnly": true + }] + }] + } + } + } +}' + +echo " Waiting for rollout..." +${KUBECTL} rollout status deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --timeout=120s diff --git a/dev/kind-config.yaml b/dev/kind-config.yaml new file mode 100644 index 0000000..c2a9da7 --- /dev/null +++ b/dev/kind-config.yaml @@ -0,0 +1,15 @@ +# Kind cluster configuration for medik8s development +# Creates 1 control-plane + 3 worker nodes. +# SNR requires 2+ workers for peer health checks. +# Single CP is sufficient for development; multi-CP etcd quorum testing +# requires a real OpenShift cluster. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/dev/setup.sh b/dev/setup.sh new file mode 100755 index 0000000..a462f5e --- /dev/null +++ b/dev/setup.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# Medik8s development environment setup +# Creates a Kind cluster with 1 CP + 3 worker nodes, installs OLM, +# and prepares the namespace for operator deployment. +# +# Usage: ./setup.sh [--skip-olm] [--name ] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" +DEV_NS="${MEDIK8S_NAMESPACE:-medik8s-system}" +INSTALL_OLM=true +SKIP_KIND=false +KIND_CONFIG="${SCRIPT_DIR}/kind-config.yaml" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --skip-kind) + SKIP_KIND=true + shift + ;; + --skip-olm) + INSTALL_OLM=false + shift + ;; + --name) + CLUSTER_NAME="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [--skip-kind] [--skip-olm] [--name ]" + echo "" + echo "Options:" + echo " --skip-kind Skip Kind cluster creation (use existing cluster)" + echo " --skip-olm Skip OLM installation" + echo " --name Kind cluster name (default: medik8s-dev)" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Check prerequisites +check_tool() { + if ! command -v "$1" &>/dev/null; then + echo "Error: $1 is required but not installed." + echo "Install it from: $2" + exit 1 + fi +} + +echo "Using kubectl command: ${KUBECTL}" +echo "Using container tool: ${CONTAINER_TOOL}" + +if [ "${SKIP_KIND}" = true ]; then + echo "Using existing cluster (--skip-kind)." + # Verify cluster connectivity + if ! ${KUBECTL} cluster-info >/dev/null 2>&1; then + echo "Error: cannot connect to cluster. Check your kubeconfig." + exit 1 + fi +else + check_tool kind "https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + check_tool go "https://go.dev/doc/install" + export KIND_EXPERIMENTAL_PROVIDER="${CONTAINER_TOOL}" + + # Check inotify limits — Kind nodes inherit host limits and operators need many watchers. + INOTIFY_INSTANCES=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) + INOTIFY_WATCHES=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0) + if [ "${INOTIFY_INSTANCES}" -lt 512 ] || [ "${INOTIFY_WATCHES}" -lt 524288 ]; then + echo "" + echo "Warning: inotify limits are too low for running multiple operators in Kind." + echo " Current: max_user_instances=${INOTIFY_INSTANCES}, max_user_watches=${INOTIFY_WATCHES}" + echo " Recommended: max_user_instances=8192, max_user_watches=524288" + echo "" + echo "Fix (requires sudo):" + echo " sudo sysctl -w fs.inotify.max_user_instances=8192" + echo " sudo sysctl -w fs.inotify.max_user_watches=524288" + echo "" + echo "To make persistent, add to /etc/sysctl.d/99-kind.conf:" + echo " fs.inotify.max_user_instances=8192" + echo " fs.inotify.max_user_watches=524288" + echo "" + # Try to fix automatically if running as root + if [ "$(id -u)" = "0" ]; then + echo "Running as root — fixing automatically." + sysctl -w fs.inotify.max_user_instances=8192 >/dev/null + sysctl -w fs.inotify.max_user_watches=524288 >/dev/null + else + echo "Error: inotify limits are too low and must be fixed before continuing." + echo "Without sufficient inotify limits, Kind nodes may fail to join and operators will crash." + exit 1 + fi + fi + + # Check if cluster already exists. + # Try 'kind get clusters' first, but also check kubectl connectivity — + # the cluster may have been created with sudo (rootful podman) and won't + # appear in rootless 'kind get clusters'. + CLUSTER_EXISTS=false + if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + CLUSTER_EXISTS=true + elif ${KUBECTL} cluster-info --context "kind-${CLUSTER_NAME}" &>/dev/null; then + CLUSTER_EXISTS=true + echo "Note: cluster '${CLUSTER_NAME}' found via kubectl (created outside current user's Kind)." + fi + + if [ "${CLUSTER_EXISTS}" = false ]; then + # When using rootless podman, verify cgroup delegation includes cpuset. + # Without cpuset, kubelet inside Kind worker nodes cannot start. + if [ "${CONTAINER_TOOL}" = "podman" ] && [ "$(id -u)" != "0" ]; then + CGROUP_SUBTREE="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.subtree_control" + if [ -f "${CGROUP_SUBTREE}" ]; then + if ! grep -q 'cpuset' "${CGROUP_SUBTREE}" 2>/dev/null; then + echo "" + echo "Error: rootless podman detected but 'cpuset' cgroup controller is not delegated." + echo "Kind worker nodes will fail to start without it." + echo "" + echo "Fix: create a systemd override to delegate the required controllers:" + echo "" + echo " sudo mkdir -p /etc/systemd/system/user@.service.d" + echo " sudo tee /etc/systemd/system/user@.service.d/delegate.conf < ~/.kube/config" + echo "Then re-run this command — it will detect the existing cluster and configure it." + exit 1 + fi + fi + fi + + echo "=== Creating Kind cluster '${CLUSTER_NAME}' ===" + kind create cluster --config "${KIND_CONFIG}" --name "${CLUSTER_NAME}" + else + echo "=== Cluster '${CLUSTER_NAME}' already exists — skipping creation, re-applying configuration ===" + fi + + echo "=== Waiting for all nodes to be Ready ===" + ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s + + echo "=== Labeling worker nodes ===" + # Label any non-CP nodes with the worker role (idempotent) + LABELED=0 + for node in $(${KUBECTL} get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + if ! ${KUBECTL} get node "$node" -o jsonpath='{.metadata.labels}' 2>/dev/null | grep -q 'node-role.kubernetes.io/control-plane'; then + if ${KUBECTL} get node "$node" -o jsonpath='{.metadata.labels}' 2>/dev/null | grep -q 'node-role.kubernetes.io/worker'; then + continue + fi + ${KUBECTL} label node "$node" node-role.kubernetes.io/worker="" 2>/dev/null || true + echo " labeled $node" + LABELED=$((LABELED + 1)) + fi + done + if [ "${LABELED}" -eq 0 ]; then + echo " All worker nodes already labeled." + fi + + echo "=== Loading softdog kernel module on worker nodes (for SNR watchdog) ===" + NODES=$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null) + if [ -z "${NODES}" ]; then + NODES=$(${KUBECTL} get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null) + fi + for node in ${NODES}; do + if echo "$node" | grep -q 'worker'; then + ${CONTAINER_TOOL} exec "$node" modprobe softdog 2>/dev/null && \ + echo " softdog loaded on $node" || \ + echo " Warning: could not load softdog on $node (SNR watchdog reboot testing will be limited)" + fi + done +fi + +echo "=== Ensuring namespace '${DEV_NS}' ===" +if ${KUBECTL} get namespace "${DEV_NS}" &>/dev/null; then + echo " Namespace '${DEV_NS}' already exists." +else + ${KUBECTL} create namespace "${DEV_NS}" +fi +${KUBECTL} label --overwrite ns "${DEV_NS}" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/audit=privileged \ + pod-security.kubernetes.io/warn=privileged 2>&1 | grep -v 'not labeled' || true + +# Also create the medik8s-leases namespace (used by common lease manager) +if ! ${KUBECTL} get namespace medik8s-leases &>/dev/null; then + ${KUBECTL} create namespace medik8s-leases +else + echo " Namespace 'medik8s-leases' already exists." +fi + +echo "=== Installing cert-manager ===" +if ${KUBECTL} get crd certificates.cert-manager.io &>/dev/null; then + echo " cert-manager already installed (CRDs found)." +else + ${KUBECTL} apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml + echo " Waiting for cert-manager to be ready..." + ${KUBECTL} wait --for=condition=Available deployment --all -n cert-manager --timeout=120s +fi + +if [ "$INSTALL_OLM" = true ]; then + if command -v operator-sdk &>/dev/null; then + echo "=== Installing OLM ===" + operator-sdk olm install 2>/dev/null || { + echo " OLM may already be installed or operator-sdk olm install failed." + echo " Continuing without OLM. Use 'make deploy' instead of 'make bundle-run'." + } + else + echo "=== Skipping OLM (operator-sdk not found) ===" + echo " Install operator-sdk for OLM bundle testing, or use 'make deploy' for direct deployment." + fi +fi + +echo "" +echo "=== Medik8s dev environment ready ===" +echo "" +echo " Cluster: ${CLUSTER_NAME}" +echo " Namespace: ${DEV_NS}" +echo " Nodes: $(${KUBECTL} get nodes --no-headers 2>/dev/null | wc -l) (1 CP + 3 workers)" +echo " OLM: $(${KUBECTL} get deployment -n olm olm-operator --no-headers >/dev/null 2>&1 && echo 'installed' || echo 'not installed')" +echo "" +echo " Next steps:" +echo " cd " +echo " make dev-deploy # Build and deploy operator" +echo " make dev-simulate-failure # Trigger node failure" +echo " make dev-logs # Watch operator logs" +echo " make dev-describe # Check cluster state" +echo "" diff --git a/dev/simulate-failure.sh b/dev/simulate-failure.sh new file mode 100755 index 0000000..3a4a8dc --- /dev/null +++ b/dev/simulate-failure.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# Medik8s failure simulation for development +# Simulates node failures to trigger remediation flows. +# +# Usage: ./simulate-failure.sh [options] +# +# Scenarios: +# kubelet-stop Stop kubelet on a worker node (triggers NHC → SNR) +# network-partition Block API server access from a worker (tests SNR peer health) +# storm Stop kubelet on 2 workers simultaneously (tests NHC storm recovery) +# recover Restart kubelet and restore network on all workers + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" +SCENARIO="${1:-}" + +if [ -z "$SCENARIO" ]; then + echo "Usage: $0 " + echo "" + echo "Scenarios:" + echo " kubelet-stop Stop kubelet on a worker (triggers remediation)" + echo " network-partition Block API server from a worker (tests peer health)" + echo " storm Stop kubelet on 2 workers (tests storm protection)" + echo " recover Recover all workers" + exit 1 +fi + +get_worker_nodes() { + # Try kind first; if it can't see the cluster (e.g. created with sudo), + # fall back to kubectl node names (which match Kind container names). + local nodes + nodes=$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null | grep worker | sort) + if [ -z "$nodes" ]; then + nodes=$(${KUBECTL} get nodes -l node-role.kubernetes.io/worker --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | sort) + fi + echo "$nodes" +} + +mapfile -t WORKERS < <(get_worker_nodes) +if [ ${#WORKERS[@]} -lt 2 ]; then + echo "Error: need at least 2 worker nodes. Found: ${#WORKERS[@]}" + exit 1 +fi + +# Verify we can access the containers. If the cluster was created with sudo, +# the containers are owned by root and we need to run this script with sudo too. +if ! ${CONTAINER_TOOL} container inspect "${WORKERS[0]}" &>/dev/null; then + echo "" + echo "Error: cannot access container '${WORKERS[0]}'." + echo "The cluster was likely created with 'sudo kind create cluster'." + echo "" + echo "Run this script with sudo:" + echo " sudo $0 $SCENARIO" + echo "" + echo "Or via make:" + echo " sudo make dev-simulate-failure" + exit 1 +fi + +# If running as root (sudo), prefix hints with sudo so users know to do the same +SUDO_HINT="" +if [ "$(id -u)" = "0" ]; then + SUDO_HINT="sudo " +fi + +case "$SCENARIO" in + kubelet-stop) + TARGET="${WORKERS[0]}" + echo "=== Stopping kubelet on ${TARGET} ===" + echo "This will make the node NotReady after ~40s." + echo "NHC will detect it after the configured unhealthyCondition duration." + ${CONTAINER_TOOL} exec "${TARGET}" systemctl stop kubelet + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get selfnoderemediation -A -w" + echo " ${KUBECTL} get nodehealthcheck -o yaml" + echo "" + echo "To recover: ${SUDO_HINT}$0 recover" + ;; + + network-partition) + TARGET="${WORKERS[0]}" + echo "=== Blocking API server access from ${TARGET} ===" + echo "This tests SNR's peer health decision engine." + # Block port 6443 (API server) + ${CONTAINER_TOOL} exec "${TARGET}" iptables -A OUTPUT -p tcp --dport 6443 -j DROP 2>/dev/null || { + echo "Error: iptables failed. The Kind node may not have iptables." + exit 1 + } + echo "" + echo "SNR agent on ${TARGET} will:" + echo " 1. Fail API server health checks" + echo " 2. Query peers via gRPC" + echo " 3. Decide whether to self-remediate" + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} logs -n medik8s-system -l app.kubernetes.io/component=agent --field-selector spec.nodeName=${TARGET} -f" + echo "" + echo "To recover: ${SUDO_HINT}$0 recover" + ;; + + storm) + echo "=== Simulating storm: stopping kubelet on ${WORKERS[0]} and ${WORKERS[1]} ===" + echo "With 3 workers and minHealthy=51%, NHC should NOT remediate (only 33% healthy)." + ${CONTAINER_TOOL} exec "${WORKERS[0]}" systemctl stop kubelet & + ${CONTAINER_TOOL} exec "${WORKERS[1]}" systemctl stop kubelet & + wait + echo "" + echo "Expected behavior:" + echo " - Both nodes go NotReady" + echo " - NHC detects storm (healthy < minHealthy threshold)" + echo " - NO remediation CRs created" + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get nodehealthcheck -o jsonpath='{.items[0].status}' | python3 -m json.tool" + echo "" + echo "To recover one node and trigger remediation of the other:" + echo " ${SUDO_HINT}${CONTAINER_TOOL} exec ${WORKERS[0]} systemctl start kubelet" + echo "" + echo "To recover all: ${SUDO_HINT}$0 recover" + ;; + + recover) + echo "=== Recovering all workers ===" + for node in "${WORKERS[@]}"; do + echo " Recovering ${node}..." + # Restore kubelet + ${CONTAINER_TOOL} exec "${node}" systemctl start kubelet 2>/dev/null || true + # Remove iptables rules + ${CONTAINER_TOOL} exec "${node}" iptables -D OUTPUT -p tcp --dport 6443 -j DROP 2>/dev/null || true + done + echo "" + echo "Waiting for nodes to become Ready..." + ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s 2>/dev/null || { + echo "Some nodes may take longer to recover. Check with: kubectl get nodes" + } + echo "" + echo "Waiting for SNR controller to finish remediation cleanup..." + # The SNR controller needs to observe the node is healthy, remove taints, + # and strip its finalizer before the CR can be deleted. Give it time. + REMAINING=0 + WAITED=0 + while [ $WAITED -lt 120 ]; do + REMAINING=$(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | wc -l) + if [ "$REMAINING" -eq 0 ]; then + break + fi + echo " $REMAINING remediation CR(s) still being processed..." + sleep 5 + WAITED=$((WAITED + 5)) + done + # Clean up any stragglers (e.g. from other remediators) + ${KUBECTL} delete fenceagentsremediation --all -A 2>/dev/null || true + if [ "$REMAINING" -gt 0 ]; then + echo " Warning: $REMAINING SNR CR(s) still have finalizers after 120s." + echo " You may need to wait longer or check SNR controller logs." + fi + echo "Done." + ;; + + *) + echo "Unknown scenario: ${SCENARIO}" + echo "Run '$0' without arguments for usage." + exit 1 + ;; +esac diff --git a/dev/summary.sh b/dev/summary.sh new file mode 100755 index 0000000..819cf82 --- /dev/null +++ b/dev/summary.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Show remediation flow timeline — what happened during simulate/recover. +# Usage: summary.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +echo "========================================" +echo " Remediation Flow Summary" +echo "========================================" +echo "" + +echo "--- Nodes ---" +${KUBECTL} get nodes -o custom-columns=NAME:.metadata.name,STATUS:.status.conditions[-1:].type,READY:.status.conditions[-1:].status 2>/dev/null +echo "" + +echo "--- NHC Status ---" +NHCS=$(${KUBECTL} get nodehealthcheck --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null || true) +if [ -n "$NHCS" ]; then + for nhc in $NHCS; do + PHASE=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.phase}' 2>/dev/null) + HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.healthyNodes}' 2>/dev/null) + OBSERVED=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.observedNodes}' 2>/dev/null) + echo " ${nhc}: Phase=${PHASE:-N/A} Nodes: ${HEALTHY:-0}/${OBSERVED:-0} healthy" + done +else + echo " (no NodeHealthCheck CRs found)" +fi +echo "" + +echo "--- Active Remediation CRs ---" +SNRS_OUT=$(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null || true) +FARS_OUT=$(${KUBECTL} get fenceagentsremediation -A --no-headers 2>/dev/null || true) +[ -n "$SNRS_OUT" ] && echo "$SNRS_OUT" | awk '{print " " $2 " (ns: " $1 ")"}' +[ -n "$FARS_OUT" ] && echo "$FARS_OUT" | awk '{print " " $2 " (ns: " $1 ")"}' +SNRS=$(echo "$SNRS_OUT" | grep -c . || true) +FARS=$(echo "$FARS_OUT" | grep -c . || true) +if [ "$SNRS" -eq 0 ] && [ "$FARS" -eq 0 ]; then + echo " (none — all remediation completed)" +fi +echo "" + +echo "--- Remediation Timeline ---" +${KUBECTL} get events -A --sort-by=.lastTimestamp 2>/dev/null \ + | grep -E 'NodeNotReady|RemediationStarted|AddFinalizer|UpdateTimeAssumedRebooted|TaintManagerEviction|RemediationFinished|RemoveFinalizer' \ + | awk '{printf " %-8s %-10s %-50s %s\n", $1, $2, $5, substr($0, index($0,$6))}' \ + || echo " No remediation events found." +echo "" + +echo "--- Operator Pods ---" +${KUBECTL} get pods -A -l control-plane=controller-manager --no-headers \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount \ + 2>/dev/null || echo " (none)" diff --git a/dev/teardown.sh b/dev/teardown.sh new file mode 100755 index 0000000..357cc2d --- /dev/null +++ b/dev/teardown.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Medik8s development environment teardown +# Destroys the Kind cluster created by setup.sh +# +# Usage: ./teardown.sh [--name ] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" + +while [[ $# -gt 0 ]]; do + case $1 in + --name) + CLUSTER_NAME="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +if ! command -v kind &>/dev/null; then + echo "Error: kind is not installed." + exit 1 +fi + +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "=== Deleting Kind cluster '${CLUSTER_NAME}' ===" + kind delete cluster --name "${CLUSTER_NAME}" + echo "Done." +elif ${KUBECTL} cluster-info --context "kind-${CLUSTER_NAME}" >/dev/null 2>&1; then + echo "Cluster '${CLUSTER_NAME}' exists but is not visible to 'kind get clusters'." + echo "It was likely created with sudo. Delete it with:" + echo " sudo kind delete cluster --name ${CLUSTER_NAME}" + exit 1 +else + echo "Cluster '${CLUSTER_NAME}' does not exist." +fi From d4b39bd353a7101f7a06a32233261b0dbb13d22b Mon Sep 17 00:00:00 2001 From: Michal Pryc Date: Thu, 9 Jul 2026 18:29:30 +0200 Subject: [PATCH 2/3] Address review feedback: multi-remediator support, label fallbacks, robustness fixes Key changes addressing Or Raz's review: - Fix dev-deploy Error 2: backup/restore kustomization.yaml with cp/mv+trap instead of kustomize edit restore; wrap grep in { || true } for pipefail safety - Fix CONTAINER_TOOL mismatch: use `override CONTAINER_TOOL :=` so dev targets always use the same tool as setup.sh regardless of operator Makefile defaults - Fix NHC label mismatch: add `app.kubernetes.io/component=controller-manager` fallback in _dev_find_ns, dev-deploy, dev-wait, dev-logs, dev-redeploy, dev-describe - Fix dev-wait exit code: exit 1 when no deployments found - Fix inotify check: add --skip-inotify-check flag for single-operator use - Fix Kind version: enforce >= 0.22.0 (K8s 1.29+ for cert-manager selectableFields) - Fix cert-manager DNS names: read namePrefix from all kustomization locations - Fix enable-certmanager.sh: skip patch when deployment already has TLS volume - Support multiple remediators: create-nhc.sh auto-detects SNR/FAR/MDR templates - Add NHC_UNHEALTHY_DURATION config (default: 300s, matching downstream docs) - Add KIND_HA=true support with kind-config-ha.yaml (3 CP + 3 workers) - Add dev-bundle-run / dev-bundle-cleanup targets for OLM bundle deployment - Filter kustomize commonLabels deprecation warnings from output - Merge dev-build recipe lines so trap covers container build failures - describe.sh: show pod node placement and leader election info - simulate-failure.sh: show monitor hints for all installed remediator CRDs - Update README: lazy clone snippet, Kind version, OLM v0 note, SNR envsubst limitation, operator coverage known issues, external cluster workflow Signed-off-by: Michal Pryc Co-Authored-By: Claude Opus 4.6 --- dev/README.md | 52 ++++++++++------- dev/create-nhc.sh | 85 ++++++++++++++++++++++----- dev/describe.sh | 40 +++++++++++-- dev/dev.mk | 119 ++++++++++++++++++++++++++------------ dev/enable-certmanager.sh | 51 ++++++++-------- dev/kind-config-ha.yaml | 16 +++++ dev/setup.sh | 97 +++++++++++++++++++++---------- dev/simulate-failure.sh | 29 ++++++---- 8 files changed, 345 insertions(+), 144 deletions(-) create mode 100644 dev/kind-config-ha.yaml diff --git a/dev/README.md b/dev/README.md index 30e4083..715ddb6 100644 --- a/dev/README.md +++ b/dev/README.md @@ -22,50 +22,46 @@ This walks through the complete remediation flow: deploy both operators, simulate a node failure, and watch NHC trigger SNR to remediate it. ```bash -# 1. Create the Kind cluster (from any operator directory) +# 1. Deploy SNR (creates Kind cluster on first run) pushd self-node-remediation make dev-setup -popd - -# 2. Deploy SNR -pushd self-node-remediation make dev-deploy popd -# 3. Deploy NHC (auto-creates NodeHealthCheck CR linking to SNR) +# 2. Deploy NHC (auto-creates NodeHealthCheck CR linking to SNR) pushd node-healthcheck-operator make dev-deploy popd -# 4. Verify everything is running +# 3. Verify everything is running make dev-wait make dev-describe -# 5. Simulate a node failure +# 4. Simulate a node failure make dev-simulate-failure -# 6. Watch the remediation flow (run in separate terminals) +# 5. Watch the remediation flow (run in separate terminals) kubectl get nodes -w kubectl get selfnoderemediation -A -w make dev-events -# 7. Debug if needed +# 6. Debug if needed make dev-logs # tail operator logs make dev-describe # full resource summary make dev-summary # remediation flow timeline make dev-shell # shell into a worker node -# 8. Recover all workers +# 7. Recover all workers make dev-recover -# 9. Teardown +# 8. Teardown make dev-teardown ``` ## Prerequisites - [Go](https://go.dev/doc/install) (version matching the operator's go.mod) -- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) (v0.20+) +- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) (v0.22.0+, for K8s 1.29+ compatibility) - [kubectl](https://kubernetes.io/docs/tasks/tools/) or [oc](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/) - [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/getting-started/installation) - [operator-sdk](https://sdk.operatorframework.io/docs/installation/) (optional, for OLM bundle testing) @@ -108,6 +104,11 @@ Add the following snippet at the end of your operator's Makefile: # Shared dev environment # Uses a local sibling checkout if available (e.g. ../tools), # otherwise downloads the tools repo into .tools/ on first dev-* target use. +# +# IMPORTANT: Do NOT use $(shell git clone ...) here — $(shell) executes at +# Makefile parse time, so any make invocation (make build, make test, make help) +# would trigger a git clone. The dev-% fallback rule below is lazy: the clone +# only runs when a dev-* target is actually invoked. TOOLS_DIR ?= $(shell cd .. && pwd)/tools DEV_MK := $(TOOLS_DIR)/dev/dev.mk ifeq ($(wildcard $(DEV_MK)),) @@ -118,7 +119,9 @@ endif ifeq ($(wildcard $(DEV_MK)),) dev-%: @echo "Downloading medik8s/tools into $(TOOLS_DIR)..." + @if [ -d $(TOOLS_DIR) ]; then echo " Removing stale $(TOOLS_DIR)..."; rm -rf $(TOOLS_DIR); fi @git clone --depth 1 https://github.com/medik8s/tools.git $(TOOLS_DIR) + @test -f $(DEV_MK) || { echo "Error: $(DEV_MK) not found after clone."; exit 1; } @$(MAKE) $@ endif ``` @@ -137,17 +140,17 @@ All `dev-*` targets are now available. ## What It Creates -- **1 control-plane + 3 worker nodes** (SNR needs 2+ workers for peer health; multi-CP requires a real cluster) +- **1 control-plane + 3 worker nodes** (SNR needs 2+ workers for peer health; use `KIND_HA=true` for 3 CP + 3 workers) - **Worker labels** (`node-role.kubernetes.io/worker`) -- **softdog** kernel module on workers (for SNR watchdog testing) +- **softdog** kernel module on workers (for SNR/SBR watchdog testing) - **Namespaces**: `medik8s-system` (privileged PSA) and `medik8s-leases` for shared resources - **cert-manager** (required for operator webhook TLS certificates) - **inotify limits** increased automatically when running as root -- **OLM** (if operator-sdk is available) +- **OLM** (if operator-sdk is available; uses `operator-sdk olm install` which installs OLM v0 — OLM v1 requires separate setup) Re-running `make dev-setup` on an existing cluster is safe — it re-applies configuration without recreating. -**Note:** Each operator deploys into its own namespace (e.g. `self-node-remediation`, `node-healthcheck-operator-system`) as defined in its kustomization files (either `config/default/kustomization.yaml` or a component/patch kustomization). The `dev-deploy` target automatically creates cert-manager certificates, patches the deployment to mount webhook TLS secrets, and waits for the deployment to become ready. If both NHC and SNR are deployed, it also creates a NodeHealthCheck CR linking them. +**Note:** Each operator deploys into its own namespace (e.g. `self-node-remediation`, `node-healthcheck-operator-system`) as defined in its kustomization files (either `config/default/kustomization.yaml` or a component/patch kustomization). The `dev-deploy` target automatically creates cert-manager certificates, patches the deployment to mount webhook TLS secrets, and waits for the deployment to become ready. If NHC and a remediator (SNR, FAR, or MDR) are both deployed, it also creates a NodeHealthCheck CR linking them. ## Failure Simulations @@ -162,13 +165,15 @@ Re-running `make dev-setup` on an existing cluster is safe — it re-applies con | Target | Description | |--------|-------------| -| `dev-setup` | Create Kind cluster with all dependencies (or configure external cluster with `SKIP_KIND=true`) | +| `dev-setup` | Create Kind cluster with all dependencies (`SKIP_KIND=true` for external cluster, `KIND_HA=true` for 3 CP + 3 workers) | | `dev-teardown` | Destroy Kind cluster | | `dev-build` | Build operator image and load into Kind (or push to ttl.sh) | | `dev-deploy` | Build + install CRDs + deploy + configure cert-manager | | `dev-redeploy` | Rebuild and restart (deletes pods to pick up new image) | | `dev-undeploy` | Remove operator from cluster | -| `dev-create-nhc` | Create NodeHealthCheck CR referencing SNR | +| `dev-bundle-run` | Deploy operator via OLM bundle (requires operator-sdk) | +| `dev-bundle-cleanup` | Remove OLM bundle deployment | +| `dev-create-nhc` | Create NodeHealthCheck CR (auto-detects SNR/FAR/MDR remediator) | | `dev-logs` | Tail operator controller-manager logs | | `dev-describe` | Full summary (nodes, pods, CRs, leases, events) | | `dev-summary` | Remediation flow timeline | @@ -215,6 +220,7 @@ clusters, trigger failures through your cluster's own mechanisms. | Variable | Default | Description | |----------|---------|-------------| +| `KIND_HA` | `false` | Set to `true` for HA cluster (3 CP + 3 workers, for SNR CP testing) | | `SKIP_KIND` | `false` | Set to `true` to skip Kind creation (external cluster) | | `DEV_REGISTRY` | `local` (Kind) / `ttl.sh` (external) | Image delivery: `local` or `ttl.sh` | | `DEV_IMG` | auto-generated | Override to use a custom image name | @@ -222,17 +228,18 @@ clusters, trigger failures through your cluster's own mechanisms. | `MEDIK8S_CLUSTER_NAME` | `medik8s-dev` | Kind cluster name | | `CONTAINER_TOOL` | auto-detected | `docker` or `podman` | | `KUBECTL` | auto-detected | `kubectl` or `oc` | +| `NHC_UNHEALTHY_DURATION` | `300s` | Unhealthy condition duration for NHC CR | ## Operator Coverage | Operator | Coverage | Notes | |----------|----------|-------| | NHC | Full | Node conditions, storm recovery, escalation, CP protection | -| NMO | Full | Cordon, drain, PDB-aware eviction | +| NMO | Full | Cordon, drain, PDB-aware eviction. Pod restarts on startup (missing namespace `list` RBAC — NMO bug, stabilizes after ~4 restarts). | | SNR | ~85% | Peer health, softdog watchdog, API check. No hardware watchdog. | | FAR | Controller logic | No real fence agents — controller reconciliation is testable | | MDR | Controller logic | No Machine API — reconciliation testable via envtest (`make test`) | -| SBR | Unit only | No shared storage | +| SBR | Unit only | No shared storage. Leader election RBAC error on startup (SBR bug). | ## Limitations @@ -240,5 +247,6 @@ clusters, trigger failures through your cluster's own mechanisms. - **MDR Machine API** — Kind has no Machine objects - **SBR shared storage** — no ODF - **Hardware watchdog** — only softdog (software) +- **SNR `${IMG}` placeholders** — SNR manifests use `${IMG}` placeholders expanded by `envsubst`. The `dev-deploy` target handles this automatically, but running `kustomize build config/default | kubectl apply -f -` directly will produce `InvalidImageName` errors. The SNR controller reconciles DaemonSets from templates baked into the image, so image patches don't persist — always use `make dev-deploy` or `make dev-redeploy` for SNR. -For full-fidelity testing, use an OpenShift cluster. +For full-fidelity testing of these scenarios, use an OpenShift cluster with real hardware (BMC/IPMI for FAR, Machine API for MDR, ODF for SBR, hardware watchdog for SNR). The `SKIP_KIND=true` workflow supports deploying to external clusters. diff --git a/dev/create-nhc.sh b/dev/create-nhc.sh index 637b285..ff8384c 100755 --- a/dev/create-nhc.sh +++ b/dev/create-nhc.sh @@ -1,6 +1,7 @@ #!/bin/bash -# Creates a NodeHealthCheck CR that references SNR for remediation. -# Usage: create-nhc.sh +# Creates a NodeHealthCheck CR that references an available remediator. +# Auto-detects deployed remediator templates (SNR, FAR, MDR) and uses the first found. +# Usage: create-nhc.sh [--duration ] set -euo pipefail @@ -8,22 +9,73 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=common.sh source "${SCRIPT_DIR}/common.sh" +# Configurable unhealthy condition duration (default: 300s, matching downstream docs) +NHC_UNHEALTHY_DURATION="${NHC_UNHEALTHY_DURATION:-300s}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --duration) + NHC_UNHEALTHY_DURATION="$2" + shift 2 + ;; + *) + echo "Usage: $0 [--duration ]" + echo " --duration Unhealthy condition duration (default: 300s)" + echo " Environment: NHC_UNHEALTHY_DURATION=300s" + exit 1 + ;; + esac +done + # Check if the NHC CRD exists if ! ${KUBECTL} get crd nodehealthchecks.remediation.medik8s.io &>/dev/null; then echo "Error: NodeHealthCheck CRD not found. Deploy NHC first (make dev-deploy from the NHC directory)." exit 1 fi -# Find the SNR template namespace -SNR_NS=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) -SNR_TEMPLATE=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) +# Auto-detect available remediator template +REMEDIATOR_API="" +REMEDIATOR_KIND="" +REMEDIATOR_NS="" +REMEDIATOR_NAME="" + +# Try SNR first (most common) +if ${KUBECTL} get crd selfnoderemediationtemplates.self-node-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="self-node-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="SelfNodeRemediationTemplate" + fi +fi + +# Try FAR +if [ -z "${REMEDIATOR_API}" ] && ${KUBECTL} get crd fenceagentsremediationtemplates.fence-agents-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get fenceagentsremediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get fenceagentsremediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="fence-agents-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="FenceAgentsRemediationTemplate" + fi +fi + +# Try MDR +if [ -z "${REMEDIATOR_API}" ] && ${KUBECTL} get crd machinedeletionremediationtemplates.machine-deletion-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get machinedeletionremediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get machinedeletionremediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="machine-deletion-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="MachineDeletionRemediationTemplate" + fi +fi -if [ -z "${SNR_NS}" ] || [ -z "${SNR_TEMPLATE}" ]; then - echo "Error: No SelfNodeRemediationTemplate found. Deploy SNR first (make dev-deploy from the SNR directory)." +if [ -z "${REMEDIATOR_API}" ]; then + echo "Error: No remediator template found. Deploy a remediator first (SNR, FAR, or MDR)." exit 1 fi -echo "Creating NodeHealthCheck CR referencing ${SNR_TEMPLATE} in ${SNR_NS}..." +echo "Creating NodeHealthCheck CR referencing ${REMEDIATOR_KIND} '${REMEDIATOR_NAME}' in ${REMEDIATOR_NS}..." ${KUBECTL} apply -f - </dev/null || true echo "" echo "--- Operator Pods ---" -${KUBECTL} get pods -A -l control-plane=controller-manager \ - -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp \ - 2>/dev/null || echo " (none)" +PODS_FOUND=false +for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do + RESULT=$(${KUBECTL} get pods -A -l "$label" \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,NODE:.spec.nodeName,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount \ + --no-headers 2>/dev/null) + if [ -n "$RESULT" ]; then + if [ "$PODS_FOUND" = false ]; then + echo " NAMESPACE NAME NODE STATUS RESTARTS" + fi + echo "$RESULT" | while IFS= read -r line; do echo " $line"; done + PODS_FOUND=true + fi +done +if [ "$PODS_FOUND" = false ]; then + echo " (none)" +fi echo "" # NodeHealthCheck @@ -99,7 +112,26 @@ if ${KUBECTL} get namespace medik8s-leases >/dev/null 2>&1; then echo "" fi -# Recent events +# Leader election +echo "--- Leader Election ---" +LEADER_FOUND=false +for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do + for ns in $(${KUBECTL} get deployment -A -l "$label" --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do + for lease in $(${KUBECTL} get leases -n "$ns" --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + HOLDER=$(${KUBECTL} get lease "$lease" -n "$ns" -o jsonpath='{.spec.holderIdentity}' 2>/dev/null) + if [ -n "$HOLDER" ]; then + echo " $ns/$lease → $HOLDER" + LEADER_FOUND=true + fi + done + done +done +if [ "$LEADER_FOUND" = false ]; then + echo " (none)" +fi +echo "" + +# Recent events — includes both CR and node events echo "--- Recent Remediation Events ---" ${KUBECTL} get events -A --sort-by=.lastTimestamp 2>/dev/null \ | grep -iE 'remediat|healthcheck|maintenance|fence|unhealthy|notready' \ diff --git a/dev/dev.mk b/dev/dev.mk index 8751f94..3a32a41 100644 --- a/dev/dev.mk +++ b/dev/dev.mk @@ -41,10 +41,10 @@ else DEV_IMG ?= ttl.sh/medik8s-$(OPERATOR_NAME)-$(shell echo $$USER | head -c 8):$(TTL_SH_TTL) endif -# CONTAINER_TOOL defines the container tool to be used for building images. -# By default uses docker if available, falls back to podman. -# You may also set CONTAINER_TOOL directly as an environment variable. -CONTAINER_TOOL ?= $(shell \ +# CONTAINER_TOOL for dev targets: auto-detect docker/podman. +# Use override to ensure dev targets use the same tool as setup.sh, +# regardless of what the operator's Makefile sets. +override CONTAINER_TOOL := $(shell \ if command -v docker >/dev/null 2>&1; then echo docker; \ elif command -v podman >/dev/null 2>&1; then echo podman; \ else echo ""; \ @@ -79,9 +79,16 @@ export MEDIK8S_CLUSTER_NAME export MEDIK8S_NAMESPACE # Helper to find the namespace for this operator's deployment. -# Looks for deployments with the controller-manager label, filtering by operator name. +# First tries the kustomization namespace (works even when OPERATOR_NAME != namespace prefix, +# e.g. SBR uses "sbr-operator-system" but OPERATOR_NAME is "storage-based-remediation"). +# Falls back to label-based cluster queries filtered by operator name. _dev_find_ns = $(shell \ + NS=$$({ grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ + if [ -n "$$NS" ] && $(KUBECTL) get namespace "$$NS" >/dev/null 2>&1; then echo "$$NS"; exit 0; fi; \ NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)\|$(subst -,.,$(OPERATOR_NAME))' | head -1); \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)\|$(subst -,.,$(OPERATOR_NAME))' | head -1); \ + fi; \ if [ -z "$$NS" ]; then \ NS=$$($(KUBECTL) get deployment -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null | grep '$(OPERATOR_NAME)' | awk '{print $$1}' | head -1); \ fi; \ @@ -91,12 +98,8 @@ _dev_find_ns = $(shell \ ##@ Dev Environment .PHONY: dev-setup -dev-setup: ## Create Kind cluster and configure dependencies (use SKIP_KIND=true for existing clusters) -ifeq ($(SKIP_KIND),true) - @$(DEV_DIR)/setup.sh --skip-kind -else - @$(DEV_DIR)/setup.sh -endif +dev-setup: ## Create Kind cluster and configure dependencies (use SKIP_KIND=true for existing clusters, KIND_HA=true for 3 CP) + @$(DEV_DIR)/setup.sh $(if $(filter true,$(SKIP_KIND)),--skip-kind) $(if $(filter true,$(KIND_HA)),--ha) .PHONY: dev-teardown dev-teardown: ## Destroy the Kind dev cluster @@ -124,10 +127,10 @@ ifeq ($(DEV_REGISTRY),local) done; \ restore() { for f in $$patched; do sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: Always/' "$$f"; done; }; \ trap restore EXIT; \ - $(CONTAINER_TOOL) build -t $(DEV_IMG) . - $(CONTAINER_TOOL) save -o /tmp/dev-image-$(OPERATOR_NAME).tar $(DEV_IMG) + $(CONTAINER_TOOL) build -t $(DEV_IMG) . && \ + $(CONTAINER_TOOL) save -o /tmp/dev-image-$(OPERATOR_NAME).tar $(DEV_IMG) && \ KIND_EXPERIMENTAL_PROVIDER=$(if $(filter podman,$(CONTAINER_TOOL)),podman,docker) \ - kind load image-archive /tmp/dev-image-$(OPERATOR_NAME).tar --name $(MEDIK8S_CLUSTER_NAME) + kind load image-archive /tmp/dev-image-$(OPERATOR_NAME).tar --name $(MEDIK8S_CLUSTER_NAME) && \ rm -f /tmp/dev-image-$(OPERATOR_NAME).tar else $(CONTAINER_TOOL) build -t $(DEV_IMG) . @@ -139,41 +142,57 @@ endif .PHONY: dev-deploy dev-deploy: dev-build install $(if $(ENVSUBST),envsubst) ## Build, load image, install CRDs, and deploy operator - cd config/manager && $(KUSTOMIZE) edit set image controller=$(DEV_IMG) - @# Use envsubst if the operator provides one (SNR uses ${IMG} in manifests) - @ENVSUBST_BIN="$(ENVSUBST)"; \ + @# Backup kustomization.yaml, set dev image, build+apply, then restore (even on failure). + @cp config/manager/kustomization.yaml config/manager/kustomization.yaml.dev-bak; \ + trap 'mv config/manager/kustomization.yaml.dev-bak config/manager/kustomization.yaml' EXIT; \ + cd config/manager && $(KUSTOMIZE) edit set image controller=$(DEV_IMG) && cd ../.. && \ + ENVSUBST_BIN="$(ENVSUBST)"; \ if [ -n "$$ENVSUBST_BIN" ] && [ -x "$$ENVSUBST_BIN" ]; then \ - export IMG=$(DEV_IMG) && $(KUSTOMIZE) build config/default | $$ENVSUBST_BIN | $(KUBECTL) apply -f -; \ + export IMG=$(DEV_IMG) && $(KUSTOMIZE) build config/default 2>&1 | grep -v "Warning: 'commonLabels'" | $$ENVSUBST_BIN | $(KUBECTL) apply -f -; \ else \ - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -; \ + $(KUSTOMIZE) build config/default 2>&1 | grep -v "Warning: 'commonLabels'" | $(KUBECTL) apply -f -; \ fi - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) @# Detect the operator namespace from kustomization files (reliable, no cluster query needed). @# The namespace may be in config/default/ or in a component/patch kustomization.yaml. - @NS=$$(grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null | head -1 | awk '{print $$2}'); \ + @NS=$$({ grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ if [ -z "$$NS" ]; then \ NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)' | head -1); \ fi; \ if [ -n "$$NS" ]; then \ - DEPLOY=$$($(KUBECTL) get deployment -n $$NS --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ - SVC=$$($(KUBECTL) get svc -n $$NS --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | grep webhook | head -1); \ - if [ -n "$$DEPLOY" ] && [ -n "$$SVC" ]; then \ - $(DEV_DIR)/enable-certmanager.sh $$NS $$DEPLOY $$SVC; \ + if [ -d config/webhook ]; then \ + SVC_RAW=$$(grep -h '^ name:' config/webhook/service.yaml 2>/dev/null | head -1 | awk '{print $$2}'); \ + PREFIX=$$({ grep -rh '^namePrefix:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ + SVC="$${PREFIX}$${SVC_RAW}"; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ + if [ -n "$$DEPLOY" ] && [ -n "$$SVC" ]; then \ + $(DEV_DIR)/enable-certmanager.sh $$NS $$DEPLOY $$SVC; \ + else \ + echo " Warning: config/webhook/ exists but could not determine deployment ($$DEPLOY) or service ($$SVC)."; \ + fi; \ else \ - echo " Skipping cert-manager setup (no webhook service found)."; \ + echo " Skipping cert-manager setup (no config/webhook/ directory)."; \ fi; \ DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ if [ -n "$$DEPLOY" ]; then \ echo "=== Waiting for operator deployment to be ready ==="; \ $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ echo "Warning: deployment $$DEPLOY is not ready. Check logs with 'make dev-logs'."; \ fi; \ else \ - echo " Skipping cert-manager setup (deployment not found)."; \ + echo " Warning: could not detect operator namespace. Skipping cert-manager setup."; \ fi - @# Create NHC CR after webhooks are ready (cert-manager must be configured first) + @# Create NHC CR after webhooks are ready (cert-manager must be configured first). + @# Auto-detects any deployed remediator template (SNR, FAR, MDR). @if $(KUBECTL) get crd nodehealthchecks.remediation.medik8s.io &>/dev/null && \ - $(KUBECTL) get selfnoderemediationtemplate -A --no-headers 2>/dev/null | grep -q .; then \ + ($(KUBECTL) get selfnoderemediationtemplate -A --no-headers 2>/dev/null | grep -q . || \ + $(KUBECTL) get fenceagentsremediationtemplate -A --no-headers 2>/dev/null | grep -q . || \ + $(KUBECTL) get machinedeletionremediationtemplate -A --no-headers 2>/dev/null | grep -q .); then \ $(DEV_DIR)/create-nhc.sh; \ fi @@ -191,13 +210,33 @@ dev-undeploy: ## Remove operator from dev cluster fi; \ fi +.PHONY: dev-bundle-run +dev-bundle-run: dev-build ## Deploy operator via OLM bundle (requires OLM + operator-sdk) + @if ! command -v operator-sdk >/dev/null 2>&1; then \ + echo "Error: operator-sdk is required for bundle-run. Install from: https://sdk.operatorframework.io/docs/installation/"; \ + exit 1; \ + fi + $(MAKE) bundle bundle-build bundle-push bundle-run IMG=$(DEV_IMG) BUNDLE_IMG=$(DEV_IMG)-bundle + +.PHONY: dev-bundle-cleanup +dev-bundle-cleanup: ## Remove OLM bundle deployment + @if ! command -v operator-sdk >/dev/null 2>&1; then \ + echo "Error: operator-sdk is required for bundle-cleanup."; \ + exit 1; \ + fi + $(MAKE) bundle-cleanup BUNDLE_IMG=$(DEV_IMG)-bundle + .PHONY: dev-redeploy dev-redeploy: dev-build ## Rebuild image and restart operator pods (deletes pods to pick up new image) @NS="$(_dev_find_ns)"; \ if [ -n "$$NS" ]; then \ echo "Deleting operator pods in $$NS to pick up new image..."; \ $(KUBECTL) delete pods -n $$NS -l control-plane=controller-manager --force --grace-period=0 2>/dev/null || true; \ + $(KUBECTL) delete pods -n $$NS -l app.kubernetes.io/component=controller-manager --force --grace-period=0 2>/dev/null || true; \ DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ if [ -n "$$DEPLOY" ]; then \ $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ echo "Warning: deployment is not ready. Check logs with 'make dev-logs'."; \ @@ -211,6 +250,9 @@ dev-logs: ## Tail operator controller-manager logs @NS="$(_dev_find_ns)"; \ if [ -n "$$NS" ]; then \ POD=$$($(KUBECTL) get pods -n $$NS -l control-plane=controller-manager -o name 2>/dev/null | head -1); \ + if [ -z "$$POD" ]; then \ + POD=$$($(KUBECTL) get pods -n $$NS -l app.kubernetes.io/component=controller-manager -o name 2>/dev/null | head -1); \ + fi; \ if [ -n "$$POD" ]; then \ $(KUBECTL) logs -f -n $$NS $$POD --all-containers --tail=50; \ else \ @@ -224,16 +266,19 @@ dev-logs: ## Tail operator controller-manager logs dev-wait: ## Wait for all medik8s operator pods to be ready @echo "=== Waiting for operator deployments to be ready ===" @FOUND=false; \ - for ns in $$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do \ - for deploy in $$($(KUBECTL) get deployment -n $$ns -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do \ - FOUND=true; \ - echo " Waiting for $$ns/$$deploy..."; \ - $(KUBECTL) wait --for=condition=Available deployment/$$deploy -n $$ns --timeout=120s || \ - echo " Warning: $$ns/$$deploy is not ready."; \ + for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do \ + for ns in $$($(KUBECTL) get deployment -A -l $$label --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do \ + for deploy in $$($(KUBECTL) get deployment -n $$ns -l $$label --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do \ + FOUND=true; \ + echo " Waiting for $$ns/$$deploy..."; \ + $(KUBECTL) wait --for=condition=Available deployment/$$deploy -n $$ns --timeout=120s || \ + echo " Warning: $$ns/$$deploy is not ready."; \ + done; \ done; \ done; \ if [ "$$FOUND" = false ]; then \ echo " No operator deployments found. Run 'make dev-deploy' first."; \ + exit 1; \ fi .PHONY: dev-events @@ -308,7 +353,9 @@ dev-help: ## Show dev environment help @echo " make dev-deploy Build + install CRDs + deploy operator" @echo " make dev-redeploy Rebuild and restart (fast iteration)" @echo " make dev-undeploy Remove operator from cluster" - @echo " make dev-create-nhc Create NodeHealthCheck CR (links NHC to SNR)" + @echo " make dev-bundle-run Deploy via OLM bundle (requires operator-sdk)" + @echo " make dev-bundle-cleanup Remove OLM bundle deployment" + @echo " make dev-create-nhc Create NodeHealthCheck CR (auto-detects remediator)" @echo "" @echo "Observe:" @echo " make dev-logs Tail operator logs" diff --git a/dev/enable-certmanager.sh b/dev/enable-certmanager.sh index 021aedf..7088672 100755 --- a/dev/enable-certmanager.sh +++ b/dev/enable-certmanager.sh @@ -83,30 +83,33 @@ for wh_type in mutatingwebhookconfigurations validatingwebhookconfigurations; do done # Patch deployment to mount the TLS secret at the path controller-runtime expects -echo " Patching deployment to mount webhook TLS secret..." -${KUBECTL} patch deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --type=strategic -p='{ - "spec": { - "template": { +if ${KUBECTL} get deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.volumes[*].name}' 2>/dev/null | grep -q cert; then + echo " Deployment already has TLS volume mount — skipping patch." +else + echo " Patching deployment to mount webhook TLS secret..." + ${KUBECTL} patch deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --type=strategic -p='{ "spec": { - "volumes": [{ - "name": "cert", - "secret": { - "secretName": "webhook-server-cert", - "defaultMode": 420 + "template": { + "spec": { + "volumes": [{ + "name": "cert", + "secret": { + "secretName": "webhook-server-cert", + "defaultMode": 420 + } + }], + "containers": [{ + "name": "manager", + "volumeMounts": [{ + "name": "cert", + "mountPath": "/tmp/k8s-webhook-server/serving-certs", + "readOnly": true + }] + }] } - }], - "containers": [{ - "name": "manager", - "volumeMounts": [{ - "name": "cert", - "mountPath": "/tmp/k8s-webhook-server/serving-certs", - "readOnly": true - }] - }] + } } - } - } -}' - -echo " Waiting for rollout..." -${KUBECTL} rollout status deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --timeout=120s + }' + echo " Waiting for rollout..." + ${KUBECTL} rollout status deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --timeout=120s +fi diff --git a/dev/kind-config-ha.yaml b/dev/kind-config-ha.yaml new file mode 100644 index 0000000..7af1cd0 --- /dev/null +++ b/dev/kind-config-ha.yaml @@ -0,0 +1,16 @@ +# Kind cluster configuration for medik8s HA development +# Creates 3 control-plane + 3 worker nodes. +# Use with: make dev-setup KIND_HA=true +# Useful for testing SNR control-plane peer health checks. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: + - role: control-plane + - role: control-plane + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/dev/setup.sh b/dev/setup.sh index a462f5e..c54a796 100755 --- a/dev/setup.sh +++ b/dev/setup.sh @@ -12,9 +12,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" +# Shared namespace for dev resources (PSA-privileged). Operators deploy into +# their own namespaces (from kustomization.yaml), not this one. DEV_NS="${MEDIK8S_NAMESPACE:-medik8s-system}" INSTALL_OLM=true SKIP_KIND=false +SKIP_INOTIFY_CHECK=false +KIND_HA="${KIND_HA:-false}" KIND_CONFIG="${SCRIPT_DIR}/kind-config.yaml" # Parse arguments @@ -28,17 +32,27 @@ while [[ $# -gt 0 ]]; do INSTALL_OLM=false shift ;; + --skip-inotify-check) + SKIP_INOTIFY_CHECK=true + shift + ;; + --ha) + KIND_HA=true + shift + ;; --name) CLUSTER_NAME="$2" shift 2 ;; -h|--help) - echo "Usage: $0 [--skip-kind] [--skip-olm] [--name ]" + echo "Usage: $0 [--skip-kind] [--skip-olm] [--skip-inotify-check] [--ha] [--name ]" echo "" echo "Options:" - echo " --skip-kind Skip Kind cluster creation (use existing cluster)" - echo " --skip-olm Skip OLM installation" - echo " --name Kind cluster name (default: medik8s-dev)" + echo " --skip-kind Skip Kind cluster creation (use existing cluster)" + echo " --skip-olm Skip OLM installation" + echo " --skip-inotify-check Skip inotify limits check" + echo " --ha Use HA config (3 CP + 3 workers, for SNR CP testing)" + echo " --name Kind cluster name (default: medik8s-dev)" exit 0 ;; *) @@ -48,6 +62,10 @@ while [[ $# -gt 0 ]]; do esac done +if [ "${KIND_HA}" = true ]; then + KIND_CONFIG="${SCRIPT_DIR}/kind-config-ha.yaml" +fi + # Check prerequisites check_tool() { if ! command -v "$1" &>/dev/null; then @@ -70,34 +88,49 @@ if [ "${SKIP_KIND}" = true ]; then else check_tool kind "https://kind.sigs.k8s.io/docs/user/quick-start/#installation" check_tool go "https://go.dev/doc/install" + + # Kind >= 0.22.0 defaults to K8s 1.29+, required for cert-manager CRD features (selectableFields). + MIN_KIND_VERSION="0.22.0" + KIND_VERSION=$(kind version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + if [ -n "${KIND_VERSION}" ] && printf '%s\n%s\n' "${MIN_KIND_VERSION}" "${KIND_VERSION}" | sort -V -C; then + : # version is sufficient + else + echo "Error: Kind >= ${MIN_KIND_VERSION} is required (found: ${KIND_VERSION:-unknown})." + echo "Install from: https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + exit 1 + fi + export KIND_EXPERIMENTAL_PROVIDER="${CONTAINER_TOOL}" # Check inotify limits — Kind nodes inherit host limits and operators need many watchers. - INOTIFY_INSTANCES=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) - INOTIFY_WATCHES=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0) - if [ "${INOTIFY_INSTANCES}" -lt 512 ] || [ "${INOTIFY_WATCHES}" -lt 524288 ]; then - echo "" - echo "Warning: inotify limits are too low for running multiple operators in Kind." - echo " Current: max_user_instances=${INOTIFY_INSTANCES}, max_user_watches=${INOTIFY_WATCHES}" - echo " Recommended: max_user_instances=8192, max_user_watches=524288" - echo "" - echo "Fix (requires sudo):" - echo " sudo sysctl -w fs.inotify.max_user_instances=8192" - echo " sudo sysctl -w fs.inotify.max_user_watches=524288" - echo "" - echo "To make persistent, add to /etc/sysctl.d/99-kind.conf:" - echo " fs.inotify.max_user_instances=8192" - echo " fs.inotify.max_user_watches=524288" - echo "" - # Try to fix automatically if running as root - if [ "$(id -u)" = "0" ]; then - echo "Running as root — fixing automatically." - sysctl -w fs.inotify.max_user_instances=8192 >/dev/null - sysctl -w fs.inotify.max_user_watches=524288 >/dev/null - else - echo "Error: inotify limits are too low and must be fixed before continuing." - echo "Without sufficient inotify limits, Kind nodes may fail to join and operators will crash." - exit 1 + if [ "${SKIP_INOTIFY_CHECK}" = true ]; then + echo "Warning: inotify limits check skipped (--skip-inotify-check). Nodes may fail to start if limits are too low." + else + INOTIFY_INSTANCES=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) + INOTIFY_WATCHES=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0) + if [ "${INOTIFY_INSTANCES}" -lt 512 ] || [ "${INOTIFY_WATCHES}" -lt 524288 ]; then + echo "" + echo "Error: inotify limits are too low for running multiple operators in Kind." + echo " Current: max_user_instances=${INOTIFY_INSTANCES}, max_user_watches=${INOTIFY_WATCHES}" + echo " Recommended: max_user_instances=8192, max_user_watches=524288" + echo "" + echo "Fix (requires sudo):" + echo " sudo sysctl -w fs.inotify.max_user_instances=8192" + echo " sudo sysctl -w fs.inotify.max_user_watches=524288" + echo "" + echo "To make persistent, add to /etc/sysctl.d/99-kind.conf:" + echo " fs.inotify.max_user_instances=8192" + echo " fs.inotify.max_user_watches=524288" + echo "" + # Try to fix automatically if running as root + if [ "$(id -u)" = "0" ]; then + echo "Running as root — fixing automatically." + sysctl -w fs.inotify.max_user_instances=8192 >/dev/null + sysctl -w fs.inotify.max_user_watches=524288 >/dev/null + else + echo "To skip this check: $0 --skip-inotify-check" + exit 1 + fi fi fi @@ -173,7 +206,7 @@ else echo " All worker nodes already labeled." fi - echo "=== Loading softdog kernel module on worker nodes (for SNR watchdog) ===" + echo "=== Loading softdog kernel module on worker nodes (for SNR/SBR watchdog) ===" NODES=$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null) if [ -z "${NODES}" ]; then NODES=$(${KUBECTL} get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null) @@ -182,7 +215,7 @@ else if echo "$node" | grep -q 'worker'; then ${CONTAINER_TOOL} exec "$node" modprobe softdog 2>/dev/null && \ echo " softdog loaded on $node" || \ - echo " Warning: could not load softdog on $node (SNR watchdog reboot testing will be limited)" + echo " Warning: could not load softdog on $node (SNR/SBR watchdog reboot testing will be limited)" fi done fi @@ -232,7 +265,7 @@ echo "=== Medik8s dev environment ready ===" echo "" echo " Cluster: ${CLUSTER_NAME}" echo " Namespace: ${DEV_NS}" -echo " Nodes: $(${KUBECTL} get nodes --no-headers 2>/dev/null | wc -l) (1 CP + 3 workers)" +echo " Nodes: $(${KUBECTL} get nodes --no-headers 2>/dev/null | wc -l) ($(${KUBECTL} get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | wc -l) CP + $(${KUBECTL} get nodes -l node-role.kubernetes.io/worker --no-headers 2>/dev/null | wc -l) workers)" echo " OLM: $(${KUBECTL} get deployment -n olm olm-operator --no-headers >/dev/null 2>&1 && echo 'installed' || echo 'not installed')" echo "" echo " Next steps:" diff --git a/dev/simulate-failure.sh b/dev/simulate-failure.sh index 3a4a8dc..40eb4e1 100755 --- a/dev/simulate-failure.sh +++ b/dev/simulate-failure.sh @@ -78,8 +78,14 @@ case "$SCENARIO" in echo "" echo "Monitor with:" echo " ${KUBECTL} get nodes -w" - echo " ${KUBECTL} get selfnoderemediation -A -w" echo " ${KUBECTL} get nodehealthcheck -o yaml" + # Show monitor hints for whichever remediator CRDs are installed + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get selfnoderemediation -A -w" + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get fenceagentsremediation -A -w" + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get machinedeletionremediation -A -w" echo "" echo "To recover: ${SUDO_HINT}$0 recover" ;; @@ -143,13 +149,18 @@ case "$SCENARIO" in echo "Some nodes may take longer to recover. Check with: kubectl get nodes" } echo "" - echo "Waiting for SNR controller to finish remediation cleanup..." - # The SNR controller needs to observe the node is healthy, remove taints, - # and strip its finalizer before the CR can be deleted. Give it time. - REMAINING=0 + echo "Waiting for remediation controllers to finish cleanup..." + # Controllers need to observe the node is healthy, remove taints, + # and strip finalizers before CRs can be deleted. Give them time. WAITED=0 while [ $WAITED -lt 120 ]; do - REMAINING=$(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | wc -l) + REMAINING=0 + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | wc -l))) + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get fenceagentsremediation -A --no-headers 2>/dev/null | wc -l))) + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get machinedeletionremediation -A --no-headers 2>/dev/null | wc -l))) if [ "$REMAINING" -eq 0 ]; then break fi @@ -157,11 +168,9 @@ case "$SCENARIO" in sleep 5 WAITED=$((WAITED + 5)) done - # Clean up any stragglers (e.g. from other remediators) - ${KUBECTL} delete fenceagentsremediation --all -A 2>/dev/null || true if [ "$REMAINING" -gt 0 ]; then - echo " Warning: $REMAINING SNR CR(s) still have finalizers after 120s." - echo " You may need to wait longer or check SNR controller logs." + echo " Warning: $REMAINING remediation CR(s) still have finalizers after 120s." + echo " You may need to wait longer or check controller logs." fi echo "Done." ;; From bb0890e413d0f3bcc2d51c31deaa2444585c9973 Mon Sep 17 00:00:00 2001 From: Michal Pryc Date: Thu, 9 Jul 2026 22:26:42 +0200 Subject: [PATCH 3/3] Fix CONTAINER_TOOL ordering, inotify platform guard, deploy error propagation - Move CONTAINER_TOOL auto-detection above DEV_CLUSTER_TYPE in dev.mk so KIND_EXPERIMENTAL_PROVIDER has a valid value during Kind cluster probe - Skip inotify limits check on non-Linux hosts (macOS has no /proc/sys/fs/inotify) - Change dev-deploy kubectl wait from warning to error exit on readiness failure - Add persist-credentials: false to shellcheck workflow checkout step - Fix summary.sh node status: use kubectl get nodes -o wide instead of conditions[-1:] which may not be the Ready condition - Add external cluster support to simulate-failure.sh: prints oc debug / SSH commands by default, DEV_FORCE_SIMULATE=true to auto-execute via oc debug - Add Kind vs OpenShift recovery documentation to README with examples - Make cleanup steps consistent: dev-undeploy (both) + dev-teardown (Kind only) Signed-off-by: Michal Pryc Co-Authored-By: Claude Opus 4.6 --- .github/workflows/shellcheck.yml | 2 + dev/README.md | 45 ++++++++++--- dev/dev.mk | 29 ++++---- dev/setup.sh | 5 +- dev/simulate-failure.sh | 110 +++++++++++++++++++++++++++++++ dev/summary.sh | 2 +- 6 files changed, 168 insertions(+), 25 deletions(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 917fe63..8fc37d0 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -13,6 +13,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Run shellcheck uses: ludeeus/action-shellcheck@2.0.0 diff --git a/dev/README.md b/dev/README.md index 715ddb6..bf91dc0 100644 --- a/dev/README.md +++ b/dev/README.md @@ -13,7 +13,8 @@ make dev-deploy # Build and deploy operator make dev-describe # Verify everything is running make dev-simulate-failure && kubectl get nodes -w # Test remediation make dev-recover # Restore cluster -make dev-teardown # Clean up +make dev-undeploy # Remove operator +make dev-teardown # Destroy Kind cluster (Kind only) ``` ### Full Walkthrough (NHC + SNR) @@ -54,8 +55,9 @@ make dev-shell # shell into a worker node # 7. Recover all workers make dev-recover -# 8. Teardown -make dev-teardown +# 8. Clean up +make dev-undeploy # from each operator directory, removes the operator +make dev-teardown # destroys the Kind cluster (Kind only) ``` ## Prerequisites @@ -161,6 +163,28 @@ Re-running `make dev-setup` on an existing cluster is safe — it re-applies con | `make dev-simulate-storm` | Stop kubelet on 2/3 workers → NHC detects storm, pauses remediation | | `make dev-recover` | Restart kubelet, restore network, clean up CRs | +### Kind vs OpenShift recovery + +On Kind, `dev-simulate-failure` stops kubelet via `docker exec`, which also kills the SNR agent pod on that node. Since a Kind container "reboot" doesn't restart kubelet, the automatic recovery can't complete — `dev-recover` simulates what a real reboot would do. The dev environment tests the detection/decision flow (NHC detects unhealthy node → creates SNR CR), not the full reboot cycle. + +On OpenShift, SNR automatically reboots the unhealthy node, kubelet restarts on boot, and the node rejoins — full self-healing, no manual `dev-recover` needed. + +**Kind** (detection flow only): +```bash +make dev-simulate-failure # stop kubelet via docker exec +kubectl get selfnoderemediation -A -w # watch SNR CR get created +make dev-recover # manually restart kubelet +``` + +**OpenShift** (full end-to-end): +```bash +make dev-simulate-failure # prints oc debug command (safe by default) +# Or auto-execute: +DEV_FORCE_SIMULATE=true make dev-simulate-failure +kubectl get selfnoderemediation -A -w # watch SNR CR + automatic reboot +# No dev-recover needed — SNR reboots the node automatically +``` + ## All Targets | Target | Description | @@ -197,9 +221,11 @@ ephemeral registry that requires no auth. export KUBECONFIG=~/.kube/my-ocp-cluster export SKIP_KIND=true -make dev-setup # Configures namespaces + cert-manager (no Kind) -make dev-deploy # Builds image, pushes to ttl.sh, deploys -make dev-describe # Verify everything is running +make dev-setup # Configures namespaces + cert-manager (no Kind) +make dev-deploy # Builds image, pushes to ttl.sh, deploys +make dev-describe # Verify everything is running +make dev-simulate-failure # Prints oc debug command (safe by default) +make dev-undeploy # Remove operator from cluster ``` Exporting `SKIP_KIND=true` ensures all targets know this is an external cluster. @@ -212,9 +238,9 @@ You can also use ttl.sh with a Kind cluster: DEV_REGISTRY=ttl.sh make dev-deploy ``` -**Note:** Failure simulations (`dev-simulate-failure`, etc.) require -`docker exec`/`podman exec` access to Kind node containers. On external -clusters, trigger failures through your cluster's own mechanisms. +**Note:** On external clusters, `dev-simulate-failure` prints the `oc debug` / SSH +commands to run instead of executing them directly (safety first). Set +`DEV_FORCE_SIMULATE=true` to auto-execute via `oc debug`. ## Configuration @@ -229,6 +255,7 @@ clusters, trigger failures through your cluster's own mechanisms. | `CONTAINER_TOOL` | auto-detected | `docker` or `podman` | | `KUBECTL` | auto-detected | `kubectl` or `oc` | | `NHC_UNHEALTHY_DURATION` | `300s` | Unhealthy condition duration for NHC CR | +| `DEV_FORCE_SIMULATE` | `false` | Set to `true` to auto-execute failure simulation on external clusters (via `oc debug`) | ## Operator Coverage diff --git a/dev/dev.mk b/dev/dev.mk index 3a32a41..0481e17 100644 --- a/dev/dev.mk +++ b/dev/dev.mk @@ -12,6 +12,20 @@ MEDIK8S_NAMESPACE ?= medik8s-system TOOLS_DIR ?= $(shell cd .. && pwd)/tools DEV_DIR := $(TOOLS_DIR)/dev +# CONTAINER_TOOL for dev targets: auto-detect docker/podman. +# Use override to ensure dev targets use the same tool as setup.sh, +# regardless of what the operator's Makefile sets. +# Must be defined before DEV_CLUSTER_TYPE which uses it for KIND_EXPERIMENTAL_PROVIDER. +override CONTAINER_TOOL := $(shell \ + if command -v docker >/dev/null 2>&1; then echo docker; \ + elif command -v podman >/dev/null 2>&1; then echo podman; \ + else echo ""; \ + fi \ +) +ifeq ($(CONTAINER_TOOL),) + $(error No container tool found. Please install docker or podman.) +endif + # Detect cluster type: "kind" if a Kind cluster exists, "external" otherwise. # Uses CONTAINER_TOOL to set KIND_EXPERIMENTAL_PROVIDER (needed for podman). # When SKIP_KIND=true, force external mode (the user explicitly opted out of Kind). @@ -41,19 +55,6 @@ else DEV_IMG ?= ttl.sh/medik8s-$(OPERATOR_NAME)-$(shell echo $$USER | head -c 8):$(TTL_SH_TTL) endif -# CONTAINER_TOOL for dev targets: auto-detect docker/podman. -# Use override to ensure dev targets use the same tool as setup.sh, -# regardless of what the operator's Makefile sets. -override CONTAINER_TOOL := $(shell \ - if command -v docker >/dev/null 2>&1; then echo docker; \ - elif command -v podman >/dev/null 2>&1; then echo podman; \ - else echo ""; \ - fi \ -) -ifeq ($(CONTAINER_TOOL),) - $(error No container tool found. Please install docker or podman.) -endif - # Detect kubectl or oc KUBECTL ?= $(shell \ if command -v kubectl >/dev/null 2>&1; then echo kubectl; \ @@ -182,7 +183,7 @@ dev-deploy: dev-build install $(if $(ENVSUBST),envsubst) ## Build, load image, i if [ -n "$$DEPLOY" ]; then \ echo "=== Waiting for operator deployment to be ready ==="; \ $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ - echo "Warning: deployment $$DEPLOY is not ready. Check logs with 'make dev-logs'."; \ + { echo "Error: deployment $$DEPLOY is not ready. Check logs with 'make dev-logs'."; exit 1; }; \ fi; \ else \ echo " Warning: could not detect operator namespace. Skipping cert-manager setup."; \ diff --git a/dev/setup.sh b/dev/setup.sh index c54a796..4df89f2 100755 --- a/dev/setup.sh +++ b/dev/setup.sh @@ -103,7 +103,10 @@ else export KIND_EXPERIMENTAL_PROVIDER="${CONTAINER_TOOL}" # Check inotify limits — Kind nodes inherit host limits and operators need many watchers. - if [ "${SKIP_INOTIFY_CHECK}" = true ]; then + # Skip on non-Linux (e.g. macOS) where /proc/sys/fs/inotify does not exist. + if [ "$(uname -s)" != "Linux" ]; then + echo " Skipping inotify check (non-Linux host)." + elif [ "${SKIP_INOTIFY_CHECK}" = true ]; then echo "Warning: inotify limits check skipped (--skip-inotify-check). Nodes may fail to start if limits are too low." else INOTIFY_INSTANCES=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) diff --git a/dev/simulate-failure.sh b/dev/simulate-failure.sh index 40eb4e1..02722c9 100755 --- a/dev/simulate-failure.sh +++ b/dev/simulate-failure.sh @@ -47,6 +47,116 @@ if [ ${#WORKERS[@]} -lt 2 ]; then exit 1 fi +# Detect external cluster: if we can't access the first worker as a container, +# this is an external cluster (OCP, etc.) — print commands instead of executing. +EXTERNAL_CLUSTER=false +if ! ${CONTAINER_TOOL} container inspect "${WORKERS[0]}" &>/dev/null; then + EXTERNAL_CLUSTER=true +fi + +# For external clusters, print commands by default. Set DEV_FORCE_SIMULATE=true +# to execute them automatically via oc debug. +if [ "${EXTERNAL_CLUSTER}" = true ]; then + # Check for oc (needed for oc debug on external clusters) + OC_CMD="" + if command -v oc &>/dev/null; then + OC_CMD="oc" + fi + + case "$SCENARIO" in + kubelet-stop) + TARGET="${WORKERS[0]}" + if [ "${DEV_FORCE_SIMULATE:-false}" = true ] && [ -n "${OC_CMD}" ]; then + echo "=== Stopping kubelet on ${TARGET} via oc debug ===" + ${OC_CMD} debug "node/${TARGET}" -- chroot /host systemctl stop kubelet + echo "" + echo "Kubelet stopped on ${TARGET}. Node should go NotReady within ~40s." + else + echo "=== External cluster detected ===" + echo "" + echo "To stop kubelet on a worker, run one of:" + echo "" + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${TARGET} -- chroot /host systemctl stop kubelet" + fi + echo " ssh ${TARGET} \"sudo systemctl stop kubelet\"" + echo "" + echo "Or to let this script do it automatically:" + echo " DEV_FORCE_SIMULATE=true make dev-simulate-failure" + fi + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get nodehealthcheck -o yaml" + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get selfnoderemediation -A -w" + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get fenceagentsremediation -A -w" + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get machinedeletionremediation -A -w" + echo "" + echo "On OpenShift, SNR will automatically reboot the node — no manual recovery needed." + ;; + + network-partition) + TARGET="${WORKERS[0]}" + echo "=== External cluster detected ===" + echo "" + echo "To block API server access from a worker, run:" + echo " ssh ${TARGET} \"sudo iptables -A OUTPUT -p tcp --dport 6443 -j DROP\"" + echo "" + echo "To restore:" + echo " ssh ${TARGET} \"sudo iptables -D OUTPUT -p tcp --dport 6443 -j DROP\"" + ;; + + storm) + echo "=== External cluster detected ===" + echo "" + echo "To simulate a storm (stop kubelet on 2 workers), run:" + echo "" + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${WORKERS[0]} -- chroot /host systemctl stop kubelet &" + echo " oc debug node/${WORKERS[1]} -- chroot /host systemctl stop kubelet &" + echo " wait" + else + echo " ssh ${WORKERS[0]} \"sudo systemctl stop kubelet\" &" + echo " ssh ${WORKERS[1]} \"sudo systemctl stop kubelet\" &" + echo " wait" + fi + echo "" + echo "Or to let this script do it automatically:" + echo " DEV_FORCE_SIMULATE=true make dev-simulate-storm" + ;; + + recover) + echo "=== External cluster: recovery instructions ===" + echo "" + echo "On OpenShift with SNR, the node should reboot and recover automatically." + echo "" + echo "To manually restart kubelet on all workers:" + for node in "${WORKERS[@]}"; do + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${node} -- chroot /host systemctl start kubelet" + else + echo " ssh ${node} \"sudo systemctl start kubelet\"" + fi + done + echo "" + echo "Then wait for nodes:" + echo " ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s" + ;; + + *) + echo "Unknown scenario: ${SCENARIO}" + echo "Run '$0' without arguments for usage." + exit 1 + ;; + esac + exit 0 +fi + +# --- Kind cluster path (direct container access) --- + # Verify we can access the containers. If the cluster was created with sudo, # the containers are owned by root and we need to run this script with sudo too. if ! ${CONTAINER_TOOL} container inspect "${WORKERS[0]}" &>/dev/null; then diff --git a/dev/summary.sh b/dev/summary.sh index 819cf82..dd18cd7 100755 --- a/dev/summary.sh +++ b/dev/summary.sh @@ -14,7 +14,7 @@ echo "========================================" echo "" echo "--- Nodes ---" -${KUBECTL} get nodes -o custom-columns=NAME:.metadata.name,STATUS:.status.conditions[-1:].type,READY:.status.conditions[-1:].status 2>/dev/null +${KUBECTL} get nodes -o wide 2>/dev/null echo "" echo "--- NHC Status ---"