diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000..8fc37d0 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,23 @@ +name: shellcheck + +on: + push: + branches: [main] + paths: ["dev/**/*.sh"] + pull_request: + paths: ["dev/**/*.sh"] + +jobs: + shellcheck: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Run shellcheck + uses: ludeeus/action-shellcheck@2.0.0 + with: + scandir: dev + severity: warning diff --git a/README.md b/README.md index acb91fa..d850063 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,28 @@ # tools Tools etc. which are not specific for one of the medik8s operators + +## Dev Environment + +Shared development environment for all medik8s operators. See [dev/README.md](dev/README.md) for full documentation. + +### Quick Start + +```bash +# 1. Add to your operator's Makefile (one-time): +# TOOLS_DIR ?= $(shell cd .. && pwd)/tools +# -include $(TOOLS_DIR)/dev/dev.mk + +# 2. Create the dev cluster: +make dev-setup + +# 3. Build and deploy your operator: +make dev-deploy + +# 4. Simulate failures: +make dev-simulate-failure +``` + +## Other Tools + +- `findIndexImage/` — Find OCP index images for IIB discovery +- `scripts/` — Build and deploy scripts for NHC+SNR diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..bf91dc0 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,279 @@ +# Medik8s Development Environment + +A shared, consistent development environment for all medik8s operators. + +## Quick Start + +```bash +# One-time: add the dev environment snippet to your operator's Makefile +# (see Setup section below for options) + +make dev-setup # Create Kind cluster +make dev-deploy # Build and deploy operator +make dev-describe # Verify everything is running +make dev-simulate-failure && kubectl get nodes -w # Test remediation +make dev-recover # Restore cluster +make dev-undeploy # Remove operator +make dev-teardown # Destroy Kind cluster (Kind only) +``` + +### Full Walkthrough (NHC + SNR) + +This walks through the complete remediation flow: deploy both operators, +simulate a node failure, and watch NHC trigger SNR to remediate it. + +```bash +# 1. Deploy SNR (creates Kind cluster on first run) +pushd self-node-remediation +make dev-setup +make dev-deploy +popd + +# 2. Deploy NHC (auto-creates NodeHealthCheck CR linking to SNR) +pushd node-healthcheck-operator +make dev-deploy +popd + +# 3. Verify everything is running +make dev-wait +make dev-describe + +# 4. Simulate a node failure +make dev-simulate-failure + +# 5. Watch the remediation flow (run in separate terminals) +kubectl get nodes -w +kubectl get selfnoderemediation -A -w +make dev-events + +# 6. Debug if needed +make dev-logs # tail operator logs +make dev-describe # full resource summary +make dev-summary # remediation flow timeline +make dev-shell # shell into a worker node + +# 7. Recover all workers +make dev-recover + +# 8. Clean up +make dev-undeploy # from each operator directory, removes the operator +make dev-teardown # destroys the Kind cluster (Kind only) +``` + +## Prerequisites + +- [Go](https://go.dev/doc/install) (version matching the operator's go.mod) +- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) (v0.22.0+, for K8s 1.29+ compatibility) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) or [oc](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/) +- [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/getting-started/installation) +- [operator-sdk](https://sdk.operatorframework.io/docs/installation/) (optional, for OLM bundle testing) + +### System limits + +The setup script checks inotify limits and exits with an error if they are too low (auto-fixes when running as root). Fix manually before running `make dev-setup`: + +```bash +sudo sysctl -w fs.inotify.max_user_instances=8192 +sudo sysctl -w fs.inotify.max_user_watches=524288 +``` + +To make persistent, add to `/etc/sysctl.d/99-kind.conf`: +``` +fs.inotify.max_user_instances=8192 +fs.inotify.max_user_watches=524288 +``` + +### Podman rootless setup + +Rootless podman requires `cpuset` cgroup delegation for Kind worker nodes. Without it, kubelet cannot start inside the containers. The setup script detects this and prints instructions. + +```bash +sudo mkdir -p /etc/systemd/system/user@.service.d +sudo tee /etc/systemd/system/user@.service.d/delegate.conf <> .gitignore +``` + +To update the downloaded copy, delete it and re-run: +```bash +rm -rf .tools && make dev-help +``` + +All `dev-*` targets are now available. + +## What It Creates + +- **1 control-plane + 3 worker nodes** (SNR needs 2+ workers for peer health; use `KIND_HA=true` for 3 CP + 3 workers) +- **Worker labels** (`node-role.kubernetes.io/worker`) +- **softdog** kernel module on workers (for SNR/SBR watchdog testing) +- **Namespaces**: `medik8s-system` (privileged PSA) and `medik8s-leases` for shared resources +- **cert-manager** (required for operator webhook TLS certificates) +- **inotify limits** increased automatically when running as root +- **OLM** (if operator-sdk is available; uses `operator-sdk olm install` which installs OLM v0 — OLM v1 requires separate setup) + +Re-running `make dev-setup` on an existing cluster is safe — it re-applies configuration without recreating. + +**Note:** Each operator deploys into its own namespace (e.g. `self-node-remediation`, `node-healthcheck-operator-system`) as defined in its kustomization files (either `config/default/kustomization.yaml` or a component/patch kustomization). The `dev-deploy` target automatically creates cert-manager certificates, patches the deployment to mount webhook TLS secrets, and waits for the deployment to become ready. If NHC and a remediator (SNR, FAR, or MDR) are both deployed, it also creates a NodeHealthCheck CR linking them. + +## Failure Simulations + +| Command | What it does | +|---------|-------------| +| `make dev-simulate-failure` | Stop kubelet on a worker → node goes NotReady → NHC creates remediation CR | +| `make dev-simulate-network` | Block API server from a worker → tests SNR peer health decisions | +| `make dev-simulate-storm` | Stop kubelet on 2/3 workers → NHC detects storm, pauses remediation | +| `make dev-recover` | Restart kubelet, restore network, clean up CRs | + +### Kind vs OpenShift recovery + +On Kind, `dev-simulate-failure` stops kubelet via `docker exec`, which also kills the SNR agent pod on that node. Since a Kind container "reboot" doesn't restart kubelet, the automatic recovery can't complete — `dev-recover` simulates what a real reboot would do. The dev environment tests the detection/decision flow (NHC detects unhealthy node → creates SNR CR), not the full reboot cycle. + +On OpenShift, SNR automatically reboots the unhealthy node, kubelet restarts on boot, and the node rejoins — full self-healing, no manual `dev-recover` needed. + +**Kind** (detection flow only): +```bash +make dev-simulate-failure # stop kubelet via docker exec +kubectl get selfnoderemediation -A -w # watch SNR CR get created +make dev-recover # manually restart kubelet +``` + +**OpenShift** (full end-to-end): +```bash +make dev-simulate-failure # prints oc debug command (safe by default) +# Or auto-execute: +DEV_FORCE_SIMULATE=true make dev-simulate-failure +kubectl get selfnoderemediation -A -w # watch SNR CR + automatic reboot +# No dev-recover needed — SNR reboots the node automatically +``` + +## All Targets + +| Target | Description | +|--------|-------------| +| `dev-setup` | Create Kind cluster with all dependencies (`SKIP_KIND=true` for external cluster, `KIND_HA=true` for 3 CP + 3 workers) | +| `dev-teardown` | Destroy Kind cluster | +| `dev-build` | Build operator image and load into Kind (or push to ttl.sh) | +| `dev-deploy` | Build + install CRDs + deploy + configure cert-manager | +| `dev-redeploy` | Rebuild and restart (deletes pods to pick up new image) | +| `dev-undeploy` | Remove operator from cluster | +| `dev-bundle-run` | Deploy operator via OLM bundle (requires operator-sdk) | +| `dev-bundle-cleanup` | Remove OLM bundle deployment | +| `dev-create-nhc` | Create NodeHealthCheck CR (auto-detects SNR/FAR/MDR remediator) | +| `dev-logs` | Tail operator controller-manager logs | +| `dev-describe` | Full summary (nodes, pods, CRs, leases, events) | +| `dev-summary` | Remediation flow timeline | +| `dev-events` | Show recent remediation-related events | +| `dev-wait` | Wait for all operator pods to be ready | +| `dev-shell` | Open a shell on a Kind node (`NODE=`, default: first worker) | +| `dev-simulate-failure` | Stop kubelet on a worker | +| `dev-simulate-storm` | Stop kubelet on 2 workers | +| `dev-simulate-network` | Block API server from a worker | +| `dev-recover` | Recover all workers and clean up | +| `dev-help` | Show all dev targets | + +## Using an Existing Cluster (OCP, etc.) + +For external clusters (OCP, etc.), set `SKIP_KIND=true` to skip Kind +creation. Images are pushed to [ttl.sh](https://ttl.sh) — an anonymous, +ephemeral registry that requires no auth. + +```bash +# Point kubectl at your cluster, then use the same workflow +export KUBECONFIG=~/.kube/my-ocp-cluster +export SKIP_KIND=true + +make dev-setup # Configures namespaces + cert-manager (no Kind) +make dev-deploy # Builds image, pushes to ttl.sh, deploys +make dev-describe # Verify everything is running +make dev-simulate-failure # Prints oc debug command (safe by default) +make dev-undeploy # Remove operator from cluster +``` + +Exporting `SKIP_KIND=true` ensures all targets know this is an external cluster. +Without it, auto-detection might find a local Kind cluster and use the wrong +image delivery method. + +You can also use ttl.sh with a Kind cluster: + +```bash +DEV_REGISTRY=ttl.sh make dev-deploy +``` + +**Note:** On external clusters, `dev-simulate-failure` prints the `oc debug` / SSH +commands to run instead of executing them directly (safety first). Set +`DEV_FORCE_SIMULATE=true` to auto-execute via `oc debug`. + +## Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `KIND_HA` | `false` | Set to `true` for HA cluster (3 CP + 3 workers, for SNR CP testing) | +| `SKIP_KIND` | `false` | Set to `true` to skip Kind creation (external cluster) | +| `DEV_REGISTRY` | `local` (Kind) / `ttl.sh` (external) | Image delivery: `local` or `ttl.sh` | +| `DEV_IMG` | auto-generated | Override to use a custom image name | +| `TTL_SH_TTL` | `2h` | Image expiry when using ttl.sh | +| `MEDIK8S_CLUSTER_NAME` | `medik8s-dev` | Kind cluster name | +| `CONTAINER_TOOL` | auto-detected | `docker` or `podman` | +| `KUBECTL` | auto-detected | `kubectl` or `oc` | +| `NHC_UNHEALTHY_DURATION` | `300s` | Unhealthy condition duration for NHC CR | +| `DEV_FORCE_SIMULATE` | `false` | Set to `true` to auto-execute failure simulation on external clusters (via `oc debug`) | + +## Operator Coverage + +| Operator | Coverage | Notes | +|----------|----------|-------| +| NHC | Full | Node conditions, storm recovery, escalation, CP protection | +| NMO | Full | Cordon, drain, PDB-aware eviction. Pod restarts on startup (missing namespace `list` RBAC — NMO bug, stabilizes after ~4 restarts). | +| SNR | ~85% | Peer health, softdog watchdog, API check. No hardware watchdog. | +| FAR | Controller logic | No real fence agents — controller reconciliation is testable | +| MDR | Controller logic | No Machine API — reconciliation testable via envtest (`make test`) | +| SBR | Unit only | No shared storage. Leader election RBAC error on startup (SBR bug). | + +## Limitations + +- **FAR fence agent execution** — no IPMI/BMC or cloud APIs +- **MDR Machine API** — Kind has no Machine objects +- **SBR shared storage** — no ODF +- **Hardware watchdog** — only softdog (software) +- **SNR `${IMG}` placeholders** — SNR manifests use `${IMG}` placeholders expanded by `envsubst`. The `dev-deploy` target handles this automatically, but running `kustomize build config/default | kubectl apply -f -` directly will produce `InvalidImageName` errors. The SNR controller reconciles DaemonSets from templates baked into the image, so image patches don't persist — always use `make dev-deploy` or `make dev-redeploy` for SNR. + +For full-fidelity testing of these scenarios, use an OpenShift cluster with real hardware (BMC/IPMI for FAR, Machine API for MDR, ODF for SBR, hardware watchdog for SNR). The `SKIP_KIND=true` workflow supports deploying to external clusters. diff --git a/dev/common.sh b/dev/common.sh new file mode 100755 index 0000000..2116bdb --- /dev/null +++ b/dev/common.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Shared helpers for medik8s dev scripts. +# Source this from other scripts: source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +# Detect kubectl or oc +detect_kubectl() { + if command -v kubectl &>/dev/null; then + echo kubectl + elif command -v oc &>/dev/null; then + echo oc + else + echo "Error: kubectl or oc is required but neither is installed." >&2 + echo "Install kubectl from: https://kubernetes.io/docs/tasks/tools/" >&2 + exit 1 + fi +} + +# Detect container tool (docker preferred, podman fallback) +detect_container_tool() { + if command -v docker &>/dev/null; then + echo docker + elif command -v podman &>/dev/null; then + echo podman + else + echo "Error: docker or podman is required but neither is installed." >&2 + exit 1 + fi +} + +KUBECTL="${KUBECTL:-$(detect_kubectl)}" +CONTAINER_TOOL="${CONTAINER_TOOL:-$(detect_container_tool)}" diff --git a/dev/create-nhc.sh b/dev/create-nhc.sh new file mode 100755 index 0000000..ff8384c --- /dev/null +++ b/dev/create-nhc.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Creates a NodeHealthCheck CR that references an available remediator. +# Auto-detects deployed remediator templates (SNR, FAR, MDR) and uses the first found. +# Usage: create-nhc.sh [--duration ] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +# Configurable unhealthy condition duration (default: 300s, matching downstream docs) +NHC_UNHEALTHY_DURATION="${NHC_UNHEALTHY_DURATION:-300s}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --duration) + NHC_UNHEALTHY_DURATION="$2" + shift 2 + ;; + *) + echo "Usage: $0 [--duration ]" + echo " --duration Unhealthy condition duration (default: 300s)" + echo " Environment: NHC_UNHEALTHY_DURATION=300s" + exit 1 + ;; + esac +done + +# Check if the NHC CRD exists +if ! ${KUBECTL} get crd nodehealthchecks.remediation.medik8s.io &>/dev/null; then + echo "Error: NodeHealthCheck CRD not found. Deploy NHC first (make dev-deploy from the NHC directory)." + exit 1 +fi + +# Auto-detect available remediator template +REMEDIATOR_API="" +REMEDIATOR_KIND="" +REMEDIATOR_NS="" +REMEDIATOR_NAME="" + +# Try SNR first (most common) +if ${KUBECTL} get crd selfnoderemediationtemplates.self-node-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get selfnoderemediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="self-node-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="SelfNodeRemediationTemplate" + fi +fi + +# Try FAR +if [ -z "${REMEDIATOR_API}" ] && ${KUBECTL} get crd fenceagentsremediationtemplates.fence-agents-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get fenceagentsremediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get fenceagentsremediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="fence-agents-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="FenceAgentsRemediationTemplate" + fi +fi + +# Try MDR +if [ -z "${REMEDIATOR_API}" ] && ${KUBECTL} get crd machinedeletionremediationtemplates.machine-deletion-remediation.medik8s.io &>/dev/null; then + REMEDIATOR_NS=$(${KUBECTL} get machinedeletionremediationtemplate -A --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | head -1) + REMEDIATOR_NAME=$(${KUBECTL} get machinedeletionremediationtemplate -A --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1) + if [ -n "${REMEDIATOR_NS}" ] && [ -n "${REMEDIATOR_NAME}" ]; then + REMEDIATOR_API="machine-deletion-remediation.medik8s.io/v1alpha1" + REMEDIATOR_KIND="MachineDeletionRemediationTemplate" + fi +fi + +if [ -z "${REMEDIATOR_API}" ]; then + echo "Error: No remediator template found. Deploy a remediator first (SNR, FAR, or MDR)." + exit 1 +fi + +echo "Creating NodeHealthCheck CR referencing ${REMEDIATOR_KIND} '${REMEDIATOR_NAME}' in ${REMEDIATOR_NS}..." + +${KUBECTL} apply -f - </dev/null || true +echo "" + +echo "--- Operator Pods ---" +PODS_FOUND=false +for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do + RESULT=$(${KUBECTL} get pods -A -l "$label" \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,NODE:.spec.nodeName,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount \ + --no-headers 2>/dev/null) + if [ -n "$RESULT" ]; then + if [ "$PODS_FOUND" = false ]; then + echo " NAMESPACE NAME NODE STATUS RESTARTS" + fi + echo "$RESULT" | while IFS= read -r line; do echo " $line"; done + PODS_FOUND=true + fi +done +if [ "$PODS_FOUND" = false ]; then + echo " (none)" +fi +echo "" + +# NodeHealthCheck +if ${KUBECTL} get crd nodehealthchecks.remediation.medik8s.io >/dev/null 2>&1; then + echo "--- NodeHealthCheck ---" + NHCS=$(${KUBECTL} get nodehealthcheck --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null) + if [ -n "$NHCS" ]; then + for nhc in $NHCS; do + echo "" + echo " ${nhc}:" + PHASE=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.phase}' 2>/dev/null) + HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.healthyNodes}' 2>/dev/null) + OBSERVED=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.observedNodes}' 2>/dev/null) + UNHEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.unhealthyNodes}' 2>/dev/null) + MIN_HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.spec.minHealthy}' 2>/dev/null) + echo " Phase: ${PHASE:-N/A}" + echo " Nodes: ${HEALTHY:-0} healthy / ${OBSERVED:-0} observed (${UNHEALTHY:-0} unhealthy)" + echo " MinHealthy: ${MIN_HEALTHY:-N/A}" + done + else + echo " (none) — create with: make dev-create-nhc" + fi + echo "" +fi + +# SelfNodeRemediation +if ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- SelfNodeRemediation ---" + if ${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | grep -q .; then + ${KUBECTL} get selfnoderemediation -A \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STARTED:.status.startTime,PHASE:.status.phase \ + 2>/dev/null + else + echo " (none)" + fi + echo "" + echo " Templates:" + ${KUBECTL} get selfnoderemediationtemplate -A --no-headers 2>/dev/null \ + | awk '{print " " $2 " (ns: " $1 ")"}' || echo " (none)" + echo "" + echo " Config:" + ${KUBECTL} get selfnoderemediationconfig -A --no-headers 2>/dev/null \ + | awk '{print " " $2 " (ns: " $1 ")"}' || echo " (none)" + echo "" +fi + +# FenceAgentsRemediation +if ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- FenceAgentsRemediation ---" + ${KUBECTL} get fenceagentsremediation -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# MachineDeletionRemediation +if ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io >/dev/null 2>&1; then + echo "--- MachineDeletionRemediation ---" + ${KUBECTL} get machinedeletionremediation -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# NodeMaintenance +if ${KUBECTL} get crd nodemaintenances.nodemaintenance.medik8s.io >/dev/null 2>&1; then + echo "--- NodeMaintenance ---" + ${KUBECTL} get nodemaintenance -A 2>/dev/null | head -10 || echo " (none)" + echo "" +fi + +# Leases +if ${KUBECTL} get namespace medik8s-leases >/dev/null 2>&1; then + echo "--- Leases (medik8s-leases) ---" + if ${KUBECTL} get leases -n medik8s-leases --no-headers 2>/dev/null | grep -q .; then + ${KUBECTL} get leases -n medik8s-leases 2>/dev/null + else + echo " (none)" + fi + echo "" +fi + +# Leader election +echo "--- Leader Election ---" +LEADER_FOUND=false +for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do + for ns in $(${KUBECTL} get deployment -A -l "$label" --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do + for lease in $(${KUBECTL} get leases -n "$ns" --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + HOLDER=$(${KUBECTL} get lease "$lease" -n "$ns" -o jsonpath='{.spec.holderIdentity}' 2>/dev/null) + if [ -n "$HOLDER" ]; then + echo " $ns/$lease → $HOLDER" + LEADER_FOUND=true + fi + done + done +done +if [ "$LEADER_FOUND" = false ]; then + echo " (none)" +fi +echo "" + +# Recent events — includes both CR and node events +echo "--- Recent Remediation Events ---" +${KUBECTL} get events -A --sort-by=.lastTimestamp 2>/dev/null \ + | grep -iE 'remediat|healthcheck|maintenance|fence|unhealthy|notready' \ + | tail -10 || echo " (none)" diff --git a/dev/dev.mk b/dev/dev.mk new file mode 100644 index 0000000..0481e17 --- /dev/null +++ b/dev/dev.mk @@ -0,0 +1,373 @@ +# dev.mk — Shared Makefile targets for medik8s local development +# +# Include this from any operator's Makefile: +# TOOLS_DIR ?= $(shell cd .. && pwd)/tools +# -include $(TOOLS_DIR)/dev/dev.mk +# +# All targets are prefixed with 'dev-' to avoid collisions. + +# Dev environment configuration +MEDIK8S_CLUSTER_NAME ?= medik8s-dev +MEDIK8S_NAMESPACE ?= medik8s-system +TOOLS_DIR ?= $(shell cd .. && pwd)/tools +DEV_DIR := $(TOOLS_DIR)/dev + +# CONTAINER_TOOL for dev targets: auto-detect docker/podman. +# Use override to ensure dev targets use the same tool as setup.sh, +# regardless of what the operator's Makefile sets. +# Must be defined before DEV_CLUSTER_TYPE which uses it for KIND_EXPERIMENTAL_PROVIDER. +override CONTAINER_TOOL := $(shell \ + if command -v docker >/dev/null 2>&1; then echo docker; \ + elif command -v podman >/dev/null 2>&1; then echo podman; \ + else echo ""; \ + fi \ +) +ifeq ($(CONTAINER_TOOL),) + $(error No container tool found. Please install docker or podman.) +endif + +# Detect cluster type: "kind" if a Kind cluster exists, "external" otherwise. +# Uses CONTAINER_TOOL to set KIND_EXPERIMENTAL_PROVIDER (needed for podman). +# When SKIP_KIND=true, force external mode (the user explicitly opted out of Kind). +# Override with DEV_CLUSTER_TYPE=external to force external mode in other cases. +ifeq ($(SKIP_KIND),true) + DEV_CLUSTER_TYPE ?= external +else + DEV_CLUSTER_TYPE ?= $(shell \ + if KIND_EXPERIMENTAL_PROVIDER=$(CONTAINER_TOOL) kind get clusters 2>/dev/null | grep -q '^$(MEDIK8S_CLUSTER_NAME)$$'; then echo kind; \ + elif $(KUBECTL) cluster-info --context 'kind-$(MEDIK8S_CLUSTER_NAME)' >/dev/null 2>&1; then echo kind; \ + else echo external; \ + fi \ + ) +endif + +# Image delivery: +# local: loaded directly into Kind nodes (no registry, no pull) +# ttl.sh: pushed to ttl.sh (anonymous, ephemeral, no auth required) +# +# Defaults to "local" for Kind clusters, "ttl.sh" for external. +# Set DEV_REGISTRY=ttl.sh to force using ttl.sh even with Kind. +DEV_REGISTRY ?= $(if $(filter kind,$(DEV_CLUSTER_TYPE)),local,ttl.sh) +TTL_SH_TTL ?= 2h +ifeq ($(DEV_REGISTRY),local) + DEV_IMG ?= localhost:5000/medik8s/$(OPERATOR_NAME):dev +else + DEV_IMG ?= ttl.sh/medik8s-$(OPERATOR_NAME)-$(shell echo $$USER | head -c 8):$(TTL_SH_TTL) +endif + +# Detect kubectl or oc +KUBECTL ?= $(shell \ + if command -v kubectl >/dev/null 2>&1; then echo kubectl; \ + elif command -v oc >/dev/null 2>&1; then echo oc; \ + else echo ""; \ + fi \ +) +ifeq ($(KUBECTL),) + $(error No kubectl or oc found. Please install kubectl or oc.) +endif + +# Verify Go is available +ifeq ($(shell command -v go 2>/dev/null),) + $(error Go not found. Please install Go from https://go.dev/doc/install or add it to your PATH.) +endif + +# Warn early if OPERATOR_NAME is not set — dev-build and dev-deploy will fail without it. +ifndef OPERATOR_NAME + $(warning OPERATOR_NAME is not set. Targets dev-build, dev-deploy, dev-redeploy will not work.) +endif + +export MEDIK8S_CLUSTER_NAME +export MEDIK8S_NAMESPACE + +# Helper to find the namespace for this operator's deployment. +# First tries the kustomization namespace (works even when OPERATOR_NAME != namespace prefix, +# e.g. SBR uses "sbr-operator-system" but OPERATOR_NAME is "storage-based-remediation"). +# Falls back to label-based cluster queries filtered by operator name. +_dev_find_ns = $(shell \ + NS=$$({ grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ + if [ -n "$$NS" ] && $(KUBECTL) get namespace "$$NS" >/dev/null 2>&1; then echo "$$NS"; exit 0; fi; \ + NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)\|$(subst -,.,$(OPERATOR_NAME))' | head -1); \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)\|$(subst -,.,$(OPERATOR_NAME))' | head -1); \ + fi; \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null | grep '$(OPERATOR_NAME)' | awk '{print $$1}' | head -1); \ + fi; \ + echo "$$NS" \ +) + +##@ Dev Environment + +.PHONY: dev-setup +dev-setup: ## Create Kind cluster and configure dependencies (use SKIP_KIND=true for existing clusters, KIND_HA=true for 3 CP) + @$(DEV_DIR)/setup.sh $(if $(filter true,$(SKIP_KIND)),--skip-kind) $(if $(filter true,$(KIND_HA)),--ha) + +.PHONY: dev-teardown +dev-teardown: ## Destroy the Kind dev cluster +ifeq ($(SKIP_KIND),true) + @echo "External cluster — nothing to tear down. Use 'make dev-undeploy' to remove operators." +else + @$(DEV_DIR)/teardown.sh +endif + +.PHONY: dev-build +dev-build: ## Build operator image and load into Kind or push to ttl.sh + @# For Kind: patch imagePullPolicy to IfNotPresent (no registry, images loaded directly). + @# For external: keep imagePullPolicy as Always (image pulled from ttl.sh). + @# The SNR controller reconciles DaemonSets from templates baked into the image, + @# so this must be done before the container build, not after. + @# Files are restored after build (even on failure) via trap. +ifeq ($(DEV_REGISTRY),local) + @patched=""; \ + for f in $$(find install/ -name '*.yaml' 2>/dev/null); do \ + if grep -q 'imagePullPolicy: Always' "$$f"; then \ + sed -i 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/' "$$f"; \ + patched="$$patched $$f"; \ + echo " Patched $$f imagePullPolicy for dev build."; \ + fi; \ + done; \ + restore() { for f in $$patched; do sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: Always/' "$$f"; done; }; \ + trap restore EXIT; \ + $(CONTAINER_TOOL) build -t $(DEV_IMG) . && \ + $(CONTAINER_TOOL) save -o /tmp/dev-image-$(OPERATOR_NAME).tar $(DEV_IMG) && \ + KIND_EXPERIMENTAL_PROVIDER=$(if $(filter podman,$(CONTAINER_TOOL)),podman,docker) \ + kind load image-archive /tmp/dev-image-$(OPERATOR_NAME).tar --name $(MEDIK8S_CLUSTER_NAME) && \ + rm -f /tmp/dev-image-$(OPERATOR_NAME).tar +else + $(CONTAINER_TOOL) build -t $(DEV_IMG) . + $(CONTAINER_TOOL) push $(DEV_IMG) + @echo "" + @echo " Image pushed to $(DEV_IMG)" + @echo " Image will expire after $(TTL_SH_TTL)." +endif + +.PHONY: dev-deploy +dev-deploy: dev-build install $(if $(ENVSUBST),envsubst) ## Build, load image, install CRDs, and deploy operator + @# Backup kustomization.yaml, set dev image, build+apply, then restore (even on failure). + @cp config/manager/kustomization.yaml config/manager/kustomization.yaml.dev-bak; \ + trap 'mv config/manager/kustomization.yaml.dev-bak config/manager/kustomization.yaml' EXIT; \ + cd config/manager && $(KUSTOMIZE) edit set image controller=$(DEV_IMG) && cd ../.. && \ + ENVSUBST_BIN="$(ENVSUBST)"; \ + if [ -n "$$ENVSUBST_BIN" ] && [ -x "$$ENVSUBST_BIN" ]; then \ + export IMG=$(DEV_IMG) && $(KUSTOMIZE) build config/default 2>&1 | grep -v "Warning: 'commonLabels'" | $$ENVSUBST_BIN | $(KUBECTL) apply -f -; \ + else \ + $(KUSTOMIZE) build config/default 2>&1 | grep -v "Warning: 'commonLabels'" | $(KUBECTL) apply -f -; \ + fi + @# Detect the operator namespace from kustomization files (reliable, no cluster query needed). + @# The namespace may be in config/default/ or in a component/patch kustomization.yaml. + @NS=$$({ grep -rh '^namespace:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ + if [ -z "$$NS" ]; then \ + NS=$$($(KUBECTL) get deployment -A -l control-plane=controller-manager --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | grep -i '$(OPERATOR_NAME)' | head -1); \ + fi; \ + if [ -n "$$NS" ]; then \ + if [ -d config/webhook ]; then \ + SVC_RAW=$$(grep -h '^ name:' config/webhook/service.yaml 2>/dev/null | head -1 | awk '{print $$2}'); \ + PREFIX=$$({ grep -rh '^namePrefix:' config/default/kustomization.yaml config/patches/*/kustomization.yaml config/components/*/kustomization.yaml 2>/dev/null || true; } | head -1 | awk '{print $$2}'); \ + SVC="$${PREFIX}$${SVC_RAW}"; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ + if [ -n "$$DEPLOY" ] && [ -n "$$SVC" ]; then \ + $(DEV_DIR)/enable-certmanager.sh $$NS $$DEPLOY $$SVC; \ + else \ + echo " Warning: config/webhook/ exists but could not determine deployment ($$DEPLOY) or service ($$SVC)."; \ + fi; \ + else \ + echo " Skipping cert-manager setup (no config/webhook/ directory)."; \ + fi; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ + if [ -n "$$DEPLOY" ]; then \ + echo "=== Waiting for operator deployment to be ready ==="; \ + $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ + { echo "Error: deployment $$DEPLOY is not ready. Check logs with 'make dev-logs'."; exit 1; }; \ + fi; \ + else \ + echo " Warning: could not detect operator namespace. Skipping cert-manager setup."; \ + fi + @# Create NHC CR after webhooks are ready (cert-manager must be configured first). + @# Auto-detects any deployed remediator template (SNR, FAR, MDR). + @if $(KUBECTL) get crd nodehealthchecks.remediation.medik8s.io &>/dev/null && \ + ($(KUBECTL) get selfnoderemediationtemplate -A --no-headers 2>/dev/null | grep -q . || \ + $(KUBECTL) get fenceagentsremediationtemplate -A --no-headers 2>/dev/null | grep -q . || \ + $(KUBECTL) get machinedeletionremediationtemplate -A --no-headers 2>/dev/null | grep -q .); then \ + $(DEV_DIR)/create-nhc.sh; \ + fi + +.PHONY: dev-undeploy +dev-undeploy: ## Remove operator from dev cluster + @if [ -n "$(KUSTOMIZE)" ] && [ -f config/default/kustomization.yaml ]; then \ + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found -f -; \ + else \ + NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + echo "Deleting namespace $$NS..."; \ + $(KUBECTL) delete namespace "$$NS" --ignore-not-found; \ + else \ + echo "No operator deployment found to undeploy."; \ + fi; \ + fi + +.PHONY: dev-bundle-run +dev-bundle-run: dev-build ## Deploy operator via OLM bundle (requires OLM + operator-sdk) + @if ! command -v operator-sdk >/dev/null 2>&1; then \ + echo "Error: operator-sdk is required for bundle-run. Install from: https://sdk.operatorframework.io/docs/installation/"; \ + exit 1; \ + fi + $(MAKE) bundle bundle-build bundle-push bundle-run IMG=$(DEV_IMG) BUNDLE_IMG=$(DEV_IMG)-bundle + +.PHONY: dev-bundle-cleanup +dev-bundle-cleanup: ## Remove OLM bundle deployment + @if ! command -v operator-sdk >/dev/null 2>&1; then \ + echo "Error: operator-sdk is required for bundle-cleanup."; \ + exit 1; \ + fi + $(MAKE) bundle-cleanup BUNDLE_IMG=$(DEV_IMG)-bundle + +.PHONY: dev-redeploy +dev-redeploy: dev-build ## Rebuild image and restart operator pods (deletes pods to pick up new image) + @NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + echo "Deleting operator pods in $$NS to pick up new image..."; \ + $(KUBECTL) delete pods -n $$NS -l control-plane=controller-manager --force --grace-period=0 2>/dev/null || true; \ + $(KUBECTL) delete pods -n $$NS -l app.kubernetes.io/component=controller-manager --force --grace-period=0 2>/dev/null || true; \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l control-plane=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + if [ -z "$$DEPLOY" ]; then \ + DEPLOY=$$($(KUBECTL) get deployment -n $$NS -l app.kubernetes.io/component=controller-manager --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1); \ + fi; \ + if [ -n "$$DEPLOY" ]; then \ + $(KUBECTL) wait --for=condition=Available deployment/$$DEPLOY -n $$NS --timeout=120s || \ + echo "Warning: deployment is not ready. Check logs with 'make dev-logs'."; \ + fi; \ + else \ + echo "Could not find deployment to restart. Run 'make dev-deploy' first."; \ + fi + +.PHONY: dev-logs +dev-logs: ## Tail operator controller-manager logs + @NS="$(_dev_find_ns)"; \ + if [ -n "$$NS" ]; then \ + POD=$$($(KUBECTL) get pods -n $$NS -l control-plane=controller-manager -o name 2>/dev/null | head -1); \ + if [ -z "$$POD" ]; then \ + POD=$$($(KUBECTL) get pods -n $$NS -l app.kubernetes.io/component=controller-manager -o name 2>/dev/null | head -1); \ + fi; \ + if [ -n "$$POD" ]; then \ + $(KUBECTL) logs -f -n $$NS $$POD --all-containers --tail=50; \ + else \ + echo "No controller-manager pod found in $$NS. Is the operator running?"; \ + fi; \ + else \ + echo "No controller-manager pod found. Run 'make dev-deploy' first."; \ + fi + +.PHONY: dev-wait +dev-wait: ## Wait for all medik8s operator pods to be ready + @echo "=== Waiting for operator deployments to be ready ===" + @FOUND=false; \ + for label in control-plane=controller-manager app.kubernetes.io/component=controller-manager; do \ + for ns in $$($(KUBECTL) get deployment -A -l $$label --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do \ + for deploy in $$($(KUBECTL) get deployment -n $$ns -l $$label --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do \ + FOUND=true; \ + echo " Waiting for $$ns/$$deploy..."; \ + $(KUBECTL) wait --for=condition=Available deployment/$$deploy -n $$ns --timeout=120s || \ + echo " Warning: $$ns/$$deploy is not ready."; \ + done; \ + done; \ + done; \ + if [ "$$FOUND" = false ]; then \ + echo " No operator deployments found. Run 'make dev-deploy' first."; \ + exit 1; \ + fi + +.PHONY: dev-events +dev-events: ## Show recent events related to medik8s resources + @echo "=== Recent Events (last 10 minutes) ===" + @$(KUBECTL) get events -A --sort-by=.lastTimestamp --field-selector reason!=Pulling,reason!=Pulled 2>/dev/null | \ + grep -iE 'remediat|healthcheck|maintenance|fence|unhealthy|notready|taint' || \ + echo " No remediation-related events found." + @echo "" + @echo "=== All Recent Events ===" + @$(KUBECTL) get events -A --sort-by=.lastTimestamp 2>/dev/null | tail -20 + +.PHONY: dev-summary +dev-summary: ## Show remediation flow timeline (what happened during simulate/recover) + @$(DEV_DIR)/summary.sh + +.PHONY: dev-describe +dev-describe: ## Full summary of all medik8s resources (nodes, pods, CRs, leases, events) + @$(DEV_DIR)/describe.sh + +.PHONY: dev-shell +dev-shell: ## Open a shell on a Kind node (use NODE=, default: first worker) + @NODES=$$(kind get nodes --name $(MEDIK8S_CLUSTER_NAME) 2>/dev/null); \ + if [ -z "$$NODES" ]; then \ + NODES=$$($(KUBECTL) get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); \ + fi; \ + if [ -z "$$NODES" ]; then \ + echo "No nodes found. Is the cluster running?"; \ + exit 1; \ + fi; \ + TARGET="$(NODE)"; \ + if [ -z "$$TARGET" ]; then \ + TARGET=$$(echo "$$NODES" | grep worker | head -1); \ + fi; \ + if [ -z "$$TARGET" ]; then \ + TARGET=$$(echo "$$NODES" | head -1); \ + fi; \ + echo "Opening shell on $$TARGET..."; \ + echo " (type 'exit' to return)"; \ + $(CONTAINER_TOOL) exec -it "$$TARGET" bash + +.PHONY: dev-create-nhc +dev-create-nhc: ## Create a NodeHealthCheck CR that triggers SNR remediation + @$(DEV_DIR)/create-nhc.sh + +.PHONY: dev-simulate-failure +dev-simulate-failure: ## Stop kubelet on a worker to trigger remediation (use SCENARIO= for other scenarios) + @$(DEV_DIR)/simulate-failure.sh $(or $(SCENARIO),kubelet-stop) + +.PHONY: dev-simulate-storm +dev-simulate-storm: ## Simulate storm: stop kubelet on 2 workers + @$(DEV_DIR)/simulate-failure.sh storm + +.PHONY: dev-simulate-network +dev-simulate-network: ## Block API server from a worker to test SNR peer health + @$(DEV_DIR)/simulate-failure.sh network-partition + +.PHONY: dev-recover +dev-recover: ## Recover all workers (restart kubelet, restore network, clean CRs) + @$(DEV_DIR)/simulate-failure.sh recover + +.PHONY: dev-help +dev-help: ## Show dev environment help + @echo "Medik8s Development Environment" + @echo "" + @echo "Lifecycle:" + @echo " make dev-setup Create Kind cluster (1 CP + 3 workers)" + @echo " make dev-teardown Destroy cluster" + @echo "" + @echo "Build & Deploy:" + @echo " make dev-build Build image and load into Kind" + @echo " make dev-deploy Build + install CRDs + deploy operator" + @echo " make dev-redeploy Rebuild and restart (fast iteration)" + @echo " make dev-undeploy Remove operator from cluster" + @echo " make dev-bundle-run Deploy via OLM bundle (requires operator-sdk)" + @echo " make dev-bundle-cleanup Remove OLM bundle deployment" + @echo " make dev-create-nhc Create NodeHealthCheck CR (auto-detects remediator)" + @echo "" + @echo "Observe:" + @echo " make dev-logs Tail operator logs" + @echo " make dev-describe Full summary (nodes, pods, CRs, leases, events)" + @echo " make dev-summary Remediation flow timeline" + @echo " make dev-events Show recent remediation-related events" + @echo " make dev-wait Wait for all operator pods to be ready" + @echo " make dev-shell Open shell on a Kind node (NODE=)" + @echo "" + @echo "Simulate Failures:" + @echo " make dev-simulate-failure Stop kubelet on a worker" + @echo " make dev-simulate-storm Stop kubelet on 2 workers (storm test)" + @echo " make dev-simulate-network Block API server from a worker" + @echo " make dev-recover Recover all workers" diff --git a/dev/enable-certmanager.sh b/dev/enable-certmanager.sh new file mode 100755 index 0000000..7088672 --- /dev/null +++ b/dev/enable-certmanager.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Creates cert-manager Certificate for an operator's webhook and patches +# the deployment to mount the TLS secret. +# Usage: enable-certmanager.sh + +set -euo pipefail + +if [ $# -lt 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +NAMESPACE="$1" +DEPLOY_NAME="$2" +SERVICE_NAME="$3" + +# Check if certificate already exists and is ready AND deployment has the volume mount +if ${KUBECTL} get certificate serving-cert -n "${NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q True; then + if ${KUBECTL} get deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.volumes[*].name}' 2>/dev/null | grep -q cert; then + echo " cert-manager already configured in ${NAMESPACE}." + exit 0 + fi + echo " Certificate ready but deployment needs patching..." +fi + +echo " Creating cert-manager Issuer and Certificate in ${NAMESPACE}..." + +# Retry the apply — cert-manager webhook may not be ready immediately after deployment +for i in $(seq 1 30); do + if ${KUBECTL} apply -f - </dev/null +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: ${NAMESPACE} +spec: + selfSigned: {} +INNEREOF + then + break + fi + if [ "$i" -eq 1 ]; then + echo " Waiting for cert-manager webhook to accept requests..." + fi + if [ "$i" -eq 30 ]; then + echo " Error: cert-manager webhook not ready after 30s." >&2 + exit 1 + fi + sleep 1 +done + +${KUBECTL} apply -f - </dev/null); do + # Only annotate webhooks that reference services in our namespace + if ${KUBECTL} get "${wh}" -o yaml 2>/dev/null | grep -q "namespace: ${NAMESPACE}"; then + ${KUBECTL} annotate "${wh}" cert-manager.io/inject-ca-from="${NAMESPACE}/serving-cert" --overwrite 2>/dev/null || true + fi + done +done + +# Patch deployment to mount the TLS secret at the path controller-runtime expects +if ${KUBECTL} get deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.volumes[*].name}' 2>/dev/null | grep -q cert; then + echo " Deployment already has TLS volume mount — skipping patch." +else + echo " Patching deployment to mount webhook TLS secret..." + ${KUBECTL} patch deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --type=strategic -p='{ + "spec": { + "template": { + "spec": { + "volumes": [{ + "name": "cert", + "secret": { + "secretName": "webhook-server-cert", + "defaultMode": 420 + } + }], + "containers": [{ + "name": "manager", + "volumeMounts": [{ + "name": "cert", + "mountPath": "/tmp/k8s-webhook-server/serving-certs", + "readOnly": true + }] + }] + } + } + } + }' + echo " Waiting for rollout..." + ${KUBECTL} rollout status deployment "${DEPLOY_NAME}" -n "${NAMESPACE}" --timeout=120s +fi diff --git a/dev/kind-config-ha.yaml b/dev/kind-config-ha.yaml new file mode 100644 index 0000000..7af1cd0 --- /dev/null +++ b/dev/kind-config-ha.yaml @@ -0,0 +1,16 @@ +# Kind cluster configuration for medik8s HA development +# Creates 3 control-plane + 3 worker nodes. +# Use with: make dev-setup KIND_HA=true +# Useful for testing SNR control-plane peer health checks. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: + - role: control-plane + - role: control-plane + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/dev/kind-config.yaml b/dev/kind-config.yaml new file mode 100644 index 0000000..c2a9da7 --- /dev/null +++ b/dev/kind-config.yaml @@ -0,0 +1,15 @@ +# Kind cluster configuration for medik8s development +# Creates 1 control-plane + 3 worker nodes. +# SNR requires 2+ workers for peer health checks. +# Single CP is sufficient for development; multi-CP etcd quorum testing +# requires a real OpenShift cluster. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/dev/setup.sh b/dev/setup.sh new file mode 100755 index 0000000..4df89f2 --- /dev/null +++ b/dev/setup.sh @@ -0,0 +1,280 @@ +#!/bin/bash +# Medik8s development environment setup +# Creates a Kind cluster with 1 CP + 3 worker nodes, installs OLM, +# and prepares the namespace for operator deployment. +# +# Usage: ./setup.sh [--skip-olm] [--name ] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" +# Shared namespace for dev resources (PSA-privileged). Operators deploy into +# their own namespaces (from kustomization.yaml), not this one. +DEV_NS="${MEDIK8S_NAMESPACE:-medik8s-system}" +INSTALL_OLM=true +SKIP_KIND=false +SKIP_INOTIFY_CHECK=false +KIND_HA="${KIND_HA:-false}" +KIND_CONFIG="${SCRIPT_DIR}/kind-config.yaml" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --skip-kind) + SKIP_KIND=true + shift + ;; + --skip-olm) + INSTALL_OLM=false + shift + ;; + --skip-inotify-check) + SKIP_INOTIFY_CHECK=true + shift + ;; + --ha) + KIND_HA=true + shift + ;; + --name) + CLUSTER_NAME="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [--skip-kind] [--skip-olm] [--skip-inotify-check] [--ha] [--name ]" + echo "" + echo "Options:" + echo " --skip-kind Skip Kind cluster creation (use existing cluster)" + echo " --skip-olm Skip OLM installation" + echo " --skip-inotify-check Skip inotify limits check" + echo " --ha Use HA config (3 CP + 3 workers, for SNR CP testing)" + echo " --name Kind cluster name (default: medik8s-dev)" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +if [ "${KIND_HA}" = true ]; then + KIND_CONFIG="${SCRIPT_DIR}/kind-config-ha.yaml" +fi + +# Check prerequisites +check_tool() { + if ! command -v "$1" &>/dev/null; then + echo "Error: $1 is required but not installed." + echo "Install it from: $2" + exit 1 + fi +} + +echo "Using kubectl command: ${KUBECTL}" +echo "Using container tool: ${CONTAINER_TOOL}" + +if [ "${SKIP_KIND}" = true ]; then + echo "Using existing cluster (--skip-kind)." + # Verify cluster connectivity + if ! ${KUBECTL} cluster-info >/dev/null 2>&1; then + echo "Error: cannot connect to cluster. Check your kubeconfig." + exit 1 + fi +else + check_tool kind "https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + check_tool go "https://go.dev/doc/install" + + # Kind >= 0.22.0 defaults to K8s 1.29+, required for cert-manager CRD features (selectableFields). + MIN_KIND_VERSION="0.22.0" + KIND_VERSION=$(kind version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + if [ -n "${KIND_VERSION}" ] && printf '%s\n%s\n' "${MIN_KIND_VERSION}" "${KIND_VERSION}" | sort -V -C; then + : # version is sufficient + else + echo "Error: Kind >= ${MIN_KIND_VERSION} is required (found: ${KIND_VERSION:-unknown})." + echo "Install from: https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + exit 1 + fi + + export KIND_EXPERIMENTAL_PROVIDER="${CONTAINER_TOOL}" + + # Check inotify limits — Kind nodes inherit host limits and operators need many watchers. + # Skip on non-Linux (e.g. macOS) where /proc/sys/fs/inotify does not exist. + if [ "$(uname -s)" != "Linux" ]; then + echo " Skipping inotify check (non-Linux host)." + elif [ "${SKIP_INOTIFY_CHECK}" = true ]; then + echo "Warning: inotify limits check skipped (--skip-inotify-check). Nodes may fail to start if limits are too low." + else + INOTIFY_INSTANCES=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) + INOTIFY_WATCHES=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0) + if [ "${INOTIFY_INSTANCES}" -lt 512 ] || [ "${INOTIFY_WATCHES}" -lt 524288 ]; then + echo "" + echo "Error: inotify limits are too low for running multiple operators in Kind." + echo " Current: max_user_instances=${INOTIFY_INSTANCES}, max_user_watches=${INOTIFY_WATCHES}" + echo " Recommended: max_user_instances=8192, max_user_watches=524288" + echo "" + echo "Fix (requires sudo):" + echo " sudo sysctl -w fs.inotify.max_user_instances=8192" + echo " sudo sysctl -w fs.inotify.max_user_watches=524288" + echo "" + echo "To make persistent, add to /etc/sysctl.d/99-kind.conf:" + echo " fs.inotify.max_user_instances=8192" + echo " fs.inotify.max_user_watches=524288" + echo "" + # Try to fix automatically if running as root + if [ "$(id -u)" = "0" ]; then + echo "Running as root — fixing automatically." + sysctl -w fs.inotify.max_user_instances=8192 >/dev/null + sysctl -w fs.inotify.max_user_watches=524288 >/dev/null + else + echo "To skip this check: $0 --skip-inotify-check" + exit 1 + fi + fi + fi + + # Check if cluster already exists. + # Try 'kind get clusters' first, but also check kubectl connectivity — + # the cluster may have been created with sudo (rootful podman) and won't + # appear in rootless 'kind get clusters'. + CLUSTER_EXISTS=false + if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + CLUSTER_EXISTS=true + elif ${KUBECTL} cluster-info --context "kind-${CLUSTER_NAME}" &>/dev/null; then + CLUSTER_EXISTS=true + echo "Note: cluster '${CLUSTER_NAME}' found via kubectl (created outside current user's Kind)." + fi + + if [ "${CLUSTER_EXISTS}" = false ]; then + # When using rootless podman, verify cgroup delegation includes cpuset. + # Without cpuset, kubelet inside Kind worker nodes cannot start. + if [ "${CONTAINER_TOOL}" = "podman" ] && [ "$(id -u)" != "0" ]; then + CGROUP_SUBTREE="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.subtree_control" + if [ -f "${CGROUP_SUBTREE}" ]; then + if ! grep -q 'cpuset' "${CGROUP_SUBTREE}" 2>/dev/null; then + echo "" + echo "Error: rootless podman detected but 'cpuset' cgroup controller is not delegated." + echo "Kind worker nodes will fail to start without it." + echo "" + echo "Fix: create a systemd override to delegate the required controllers:" + echo "" + echo " sudo mkdir -p /etc/systemd/system/user@.service.d" + echo " sudo tee /etc/systemd/system/user@.service.d/delegate.conf < ~/.kube/config" + echo "Then re-run this command — it will detect the existing cluster and configure it." + exit 1 + fi + fi + fi + + echo "=== Creating Kind cluster '${CLUSTER_NAME}' ===" + kind create cluster --config "${KIND_CONFIG}" --name "${CLUSTER_NAME}" + else + echo "=== Cluster '${CLUSTER_NAME}' already exists — skipping creation, re-applying configuration ===" + fi + + echo "=== Waiting for all nodes to be Ready ===" + ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s + + echo "=== Labeling worker nodes ===" + # Label any non-CP nodes with the worker role (idempotent) + LABELED=0 + for node in $(${KUBECTL} get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + if ! ${KUBECTL} get node "$node" -o jsonpath='{.metadata.labels}' 2>/dev/null | grep -q 'node-role.kubernetes.io/control-plane'; then + if ${KUBECTL} get node "$node" -o jsonpath='{.metadata.labels}' 2>/dev/null | grep -q 'node-role.kubernetes.io/worker'; then + continue + fi + ${KUBECTL} label node "$node" node-role.kubernetes.io/worker="" 2>/dev/null || true + echo " labeled $node" + LABELED=$((LABELED + 1)) + fi + done + if [ "${LABELED}" -eq 0 ]; then + echo " All worker nodes already labeled." + fi + + echo "=== Loading softdog kernel module on worker nodes (for SNR/SBR watchdog) ===" + NODES=$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null) + if [ -z "${NODES}" ]; then + NODES=$(${KUBECTL} get nodes --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null) + fi + for node in ${NODES}; do + if echo "$node" | grep -q 'worker'; then + ${CONTAINER_TOOL} exec "$node" modprobe softdog 2>/dev/null && \ + echo " softdog loaded on $node" || \ + echo " Warning: could not load softdog on $node (SNR/SBR watchdog reboot testing will be limited)" + fi + done +fi + +echo "=== Ensuring namespace '${DEV_NS}' ===" +if ${KUBECTL} get namespace "${DEV_NS}" &>/dev/null; then + echo " Namespace '${DEV_NS}' already exists." +else + ${KUBECTL} create namespace "${DEV_NS}" +fi +${KUBECTL} label --overwrite ns "${DEV_NS}" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/audit=privileged \ + pod-security.kubernetes.io/warn=privileged 2>&1 | grep -v 'not labeled' || true + +# Also create the medik8s-leases namespace (used by common lease manager) +if ! ${KUBECTL} get namespace medik8s-leases &>/dev/null; then + ${KUBECTL} create namespace medik8s-leases +else + echo " Namespace 'medik8s-leases' already exists." +fi + +echo "=== Installing cert-manager ===" +if ${KUBECTL} get crd certificates.cert-manager.io &>/dev/null; then + echo " cert-manager already installed (CRDs found)." +else + ${KUBECTL} apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml + echo " Waiting for cert-manager to be ready..." + ${KUBECTL} wait --for=condition=Available deployment --all -n cert-manager --timeout=120s +fi + +if [ "$INSTALL_OLM" = true ]; then + if command -v operator-sdk &>/dev/null; then + echo "=== Installing OLM ===" + operator-sdk olm install 2>/dev/null || { + echo " OLM may already be installed or operator-sdk olm install failed." + echo " Continuing without OLM. Use 'make deploy' instead of 'make bundle-run'." + } + else + echo "=== Skipping OLM (operator-sdk not found) ===" + echo " Install operator-sdk for OLM bundle testing, or use 'make deploy' for direct deployment." + fi +fi + +echo "" +echo "=== Medik8s dev environment ready ===" +echo "" +echo " Cluster: ${CLUSTER_NAME}" +echo " Namespace: ${DEV_NS}" +echo " Nodes: $(${KUBECTL} get nodes --no-headers 2>/dev/null | wc -l) ($(${KUBECTL} get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | wc -l) CP + $(${KUBECTL} get nodes -l node-role.kubernetes.io/worker --no-headers 2>/dev/null | wc -l) workers)" +echo " OLM: $(${KUBECTL} get deployment -n olm olm-operator --no-headers >/dev/null 2>&1 && echo 'installed' || echo 'not installed')" +echo "" +echo " Next steps:" +echo " cd " +echo " make dev-deploy # Build and deploy operator" +echo " make dev-simulate-failure # Trigger node failure" +echo " make dev-logs # Watch operator logs" +echo " make dev-describe # Check cluster state" +echo "" diff --git a/dev/simulate-failure.sh b/dev/simulate-failure.sh new file mode 100755 index 0000000..02722c9 --- /dev/null +++ b/dev/simulate-failure.sh @@ -0,0 +1,293 @@ +#!/bin/bash +# Medik8s failure simulation for development +# Simulates node failures to trigger remediation flows. +# +# Usage: ./simulate-failure.sh [options] +# +# Scenarios: +# kubelet-stop Stop kubelet on a worker node (triggers NHC → SNR) +# network-partition Block API server access from a worker (tests SNR peer health) +# storm Stop kubelet on 2 workers simultaneously (tests NHC storm recovery) +# recover Restart kubelet and restore network on all workers + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" +SCENARIO="${1:-}" + +if [ -z "$SCENARIO" ]; then + echo "Usage: $0 " + echo "" + echo "Scenarios:" + echo " kubelet-stop Stop kubelet on a worker (triggers remediation)" + echo " network-partition Block API server from a worker (tests peer health)" + echo " storm Stop kubelet on 2 workers (tests storm protection)" + echo " recover Recover all workers" + exit 1 +fi + +get_worker_nodes() { + # Try kind first; if it can't see the cluster (e.g. created with sudo), + # fall back to kubectl node names (which match Kind container names). + local nodes + nodes=$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null | grep worker | sort) + if [ -z "$nodes" ]; then + nodes=$(${KUBECTL} get nodes -l node-role.kubernetes.io/worker --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | sort) + fi + echo "$nodes" +} + +mapfile -t WORKERS < <(get_worker_nodes) +if [ ${#WORKERS[@]} -lt 2 ]; then + echo "Error: need at least 2 worker nodes. Found: ${#WORKERS[@]}" + exit 1 +fi + +# Detect external cluster: if we can't access the first worker as a container, +# this is an external cluster (OCP, etc.) — print commands instead of executing. +EXTERNAL_CLUSTER=false +if ! ${CONTAINER_TOOL} container inspect "${WORKERS[0]}" &>/dev/null; then + EXTERNAL_CLUSTER=true +fi + +# For external clusters, print commands by default. Set DEV_FORCE_SIMULATE=true +# to execute them automatically via oc debug. +if [ "${EXTERNAL_CLUSTER}" = true ]; then + # Check for oc (needed for oc debug on external clusters) + OC_CMD="" + if command -v oc &>/dev/null; then + OC_CMD="oc" + fi + + case "$SCENARIO" in + kubelet-stop) + TARGET="${WORKERS[0]}" + if [ "${DEV_FORCE_SIMULATE:-false}" = true ] && [ -n "${OC_CMD}" ]; then + echo "=== Stopping kubelet on ${TARGET} via oc debug ===" + ${OC_CMD} debug "node/${TARGET}" -- chroot /host systemctl stop kubelet + echo "" + echo "Kubelet stopped on ${TARGET}. Node should go NotReady within ~40s." + else + echo "=== External cluster detected ===" + echo "" + echo "To stop kubelet on a worker, run one of:" + echo "" + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${TARGET} -- chroot /host systemctl stop kubelet" + fi + echo " ssh ${TARGET} \"sudo systemctl stop kubelet\"" + echo "" + echo "Or to let this script do it automatically:" + echo " DEV_FORCE_SIMULATE=true make dev-simulate-failure" + fi + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get nodehealthcheck -o yaml" + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get selfnoderemediation -A -w" + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get fenceagentsremediation -A -w" + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get machinedeletionremediation -A -w" + echo "" + echo "On OpenShift, SNR will automatically reboot the node — no manual recovery needed." + ;; + + network-partition) + TARGET="${WORKERS[0]}" + echo "=== External cluster detected ===" + echo "" + echo "To block API server access from a worker, run:" + echo " ssh ${TARGET} \"sudo iptables -A OUTPUT -p tcp --dport 6443 -j DROP\"" + echo "" + echo "To restore:" + echo " ssh ${TARGET} \"sudo iptables -D OUTPUT -p tcp --dport 6443 -j DROP\"" + ;; + + storm) + echo "=== External cluster detected ===" + echo "" + echo "To simulate a storm (stop kubelet on 2 workers), run:" + echo "" + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${WORKERS[0]} -- chroot /host systemctl stop kubelet &" + echo " oc debug node/${WORKERS[1]} -- chroot /host systemctl stop kubelet &" + echo " wait" + else + echo " ssh ${WORKERS[0]} \"sudo systemctl stop kubelet\" &" + echo " ssh ${WORKERS[1]} \"sudo systemctl stop kubelet\" &" + echo " wait" + fi + echo "" + echo "Or to let this script do it automatically:" + echo " DEV_FORCE_SIMULATE=true make dev-simulate-storm" + ;; + + recover) + echo "=== External cluster: recovery instructions ===" + echo "" + echo "On OpenShift with SNR, the node should reboot and recover automatically." + echo "" + echo "To manually restart kubelet on all workers:" + for node in "${WORKERS[@]}"; do + if [ -n "${OC_CMD}" ]; then + echo " oc debug node/${node} -- chroot /host systemctl start kubelet" + else + echo " ssh ${node} \"sudo systemctl start kubelet\"" + fi + done + echo "" + echo "Then wait for nodes:" + echo " ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s" + ;; + + *) + echo "Unknown scenario: ${SCENARIO}" + echo "Run '$0' without arguments for usage." + exit 1 + ;; + esac + exit 0 +fi + +# --- Kind cluster path (direct container access) --- + +# Verify we can access the containers. If the cluster was created with sudo, +# the containers are owned by root and we need to run this script with sudo too. +if ! ${CONTAINER_TOOL} container inspect "${WORKERS[0]}" &>/dev/null; then + echo "" + echo "Error: cannot access container '${WORKERS[0]}'." + echo "The cluster was likely created with 'sudo kind create cluster'." + echo "" + echo "Run this script with sudo:" + echo " sudo $0 $SCENARIO" + echo "" + echo "Or via make:" + echo " sudo make dev-simulate-failure" + exit 1 +fi + +# If running as root (sudo), prefix hints with sudo so users know to do the same +SUDO_HINT="" +if [ "$(id -u)" = "0" ]; then + SUDO_HINT="sudo " +fi + +case "$SCENARIO" in + kubelet-stop) + TARGET="${WORKERS[0]}" + echo "=== Stopping kubelet on ${TARGET} ===" + echo "This will make the node NotReady after ~40s." + echo "NHC will detect it after the configured unhealthyCondition duration." + ${CONTAINER_TOOL} exec "${TARGET}" systemctl stop kubelet + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get nodehealthcheck -o yaml" + # Show monitor hints for whichever remediator CRDs are installed + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get selfnoderemediation -A -w" + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get fenceagentsremediation -A -w" + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + echo " ${KUBECTL} get machinedeletionremediation -A -w" + echo "" + echo "To recover: ${SUDO_HINT}$0 recover" + ;; + + network-partition) + TARGET="${WORKERS[0]}" + echo "=== Blocking API server access from ${TARGET} ===" + echo "This tests SNR's peer health decision engine." + # Block port 6443 (API server) + ${CONTAINER_TOOL} exec "${TARGET}" iptables -A OUTPUT -p tcp --dport 6443 -j DROP 2>/dev/null || { + echo "Error: iptables failed. The Kind node may not have iptables." + exit 1 + } + echo "" + echo "SNR agent on ${TARGET} will:" + echo " 1. Fail API server health checks" + echo " 2. Query peers via gRPC" + echo " 3. Decide whether to self-remediate" + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} logs -n medik8s-system -l app.kubernetes.io/component=agent --field-selector spec.nodeName=${TARGET} -f" + echo "" + echo "To recover: ${SUDO_HINT}$0 recover" + ;; + + storm) + echo "=== Simulating storm: stopping kubelet on ${WORKERS[0]} and ${WORKERS[1]} ===" + echo "With 3 workers and minHealthy=51%, NHC should NOT remediate (only 33% healthy)." + ${CONTAINER_TOOL} exec "${WORKERS[0]}" systemctl stop kubelet & + ${CONTAINER_TOOL} exec "${WORKERS[1]}" systemctl stop kubelet & + wait + echo "" + echo "Expected behavior:" + echo " - Both nodes go NotReady" + echo " - NHC detects storm (healthy < minHealthy threshold)" + echo " - NO remediation CRs created" + echo "" + echo "Monitor with:" + echo " ${KUBECTL} get nodes -w" + echo " ${KUBECTL} get nodehealthcheck -o jsonpath='{.items[0].status}' | python3 -m json.tool" + echo "" + echo "To recover one node and trigger remediation of the other:" + echo " ${SUDO_HINT}${CONTAINER_TOOL} exec ${WORKERS[0]} systemctl start kubelet" + echo "" + echo "To recover all: ${SUDO_HINT}$0 recover" + ;; + + recover) + echo "=== Recovering all workers ===" + for node in "${WORKERS[@]}"; do + echo " Recovering ${node}..." + # Restore kubelet + ${CONTAINER_TOOL} exec "${node}" systemctl start kubelet 2>/dev/null || true + # Remove iptables rules + ${CONTAINER_TOOL} exec "${node}" iptables -D OUTPUT -p tcp --dport 6443 -j DROP 2>/dev/null || true + done + echo "" + echo "Waiting for nodes to become Ready..." + ${KUBECTL} wait --for=condition=Ready node --all --timeout=120s 2>/dev/null || { + echo "Some nodes may take longer to recover. Check with: kubectl get nodes" + } + echo "" + echo "Waiting for remediation controllers to finish cleanup..." + # Controllers need to observe the node is healthy, remove taints, + # and strip finalizers before CRs can be deleted. Give them time. + WAITED=0 + while [ $WAITED -lt 120 ]; do + REMAINING=0 + ${KUBECTL} get crd selfnoderemediations.self-node-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null | wc -l))) + ${KUBECTL} get crd fenceagentsremediations.fence-agents-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get fenceagentsremediation -A --no-headers 2>/dev/null | wc -l))) + ${KUBECTL} get crd machinedeletionremediations.machine-deletion-remediation.medik8s.io &>/dev/null && \ + REMAINING=$((REMAINING + $(${KUBECTL} get machinedeletionremediation -A --no-headers 2>/dev/null | wc -l))) + if [ "$REMAINING" -eq 0 ]; then + break + fi + echo " $REMAINING remediation CR(s) still being processed..." + sleep 5 + WAITED=$((WAITED + 5)) + done + if [ "$REMAINING" -gt 0 ]; then + echo " Warning: $REMAINING remediation CR(s) still have finalizers after 120s." + echo " You may need to wait longer or check controller logs." + fi + echo "Done." + ;; + + *) + echo "Unknown scenario: ${SCENARIO}" + echo "Run '$0' without arguments for usage." + exit 1 + ;; +esac diff --git a/dev/summary.sh b/dev/summary.sh new file mode 100755 index 0000000..dd18cd7 --- /dev/null +++ b/dev/summary.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Show remediation flow timeline — what happened during simulate/recover. +# Usage: summary.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +echo "========================================" +echo " Remediation Flow Summary" +echo "========================================" +echo "" + +echo "--- Nodes ---" +${KUBECTL} get nodes -o wide 2>/dev/null +echo "" + +echo "--- NHC Status ---" +NHCS=$(${KUBECTL} get nodehealthcheck --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null || true) +if [ -n "$NHCS" ]; then + for nhc in $NHCS; do + PHASE=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.phase}' 2>/dev/null) + HEALTHY=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.healthyNodes}' 2>/dev/null) + OBSERVED=$(${KUBECTL} get nodehealthcheck "$nhc" -o jsonpath='{.status.observedNodes}' 2>/dev/null) + echo " ${nhc}: Phase=${PHASE:-N/A} Nodes: ${HEALTHY:-0}/${OBSERVED:-0} healthy" + done +else + echo " (no NodeHealthCheck CRs found)" +fi +echo "" + +echo "--- Active Remediation CRs ---" +SNRS_OUT=$(${KUBECTL} get selfnoderemediation -A --no-headers 2>/dev/null || true) +FARS_OUT=$(${KUBECTL} get fenceagentsremediation -A --no-headers 2>/dev/null || true) +[ -n "$SNRS_OUT" ] && echo "$SNRS_OUT" | awk '{print " " $2 " (ns: " $1 ")"}' +[ -n "$FARS_OUT" ] && echo "$FARS_OUT" | awk '{print " " $2 " (ns: " $1 ")"}' +SNRS=$(echo "$SNRS_OUT" | grep -c . || true) +FARS=$(echo "$FARS_OUT" | grep -c . || true) +if [ "$SNRS" -eq 0 ] && [ "$FARS" -eq 0 ]; then + echo " (none — all remediation completed)" +fi +echo "" + +echo "--- Remediation Timeline ---" +${KUBECTL} get events -A --sort-by=.lastTimestamp 2>/dev/null \ + | grep -E 'NodeNotReady|RemediationStarted|AddFinalizer|UpdateTimeAssumedRebooted|TaintManagerEviction|RemediationFinished|RemoveFinalizer' \ + | awk '{printf " %-8s %-10s %-50s %s\n", $1, $2, $5, substr($0, index($0,$6))}' \ + || echo " No remediation events found." +echo "" + +echo "--- Operator Pods ---" +${KUBECTL} get pods -A -l control-plane=controller-manager --no-headers \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount \ + 2>/dev/null || echo " (none)" diff --git a/dev/teardown.sh b/dev/teardown.sh new file mode 100755 index 0000000..357cc2d --- /dev/null +++ b/dev/teardown.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Medik8s development environment teardown +# Destroys the Kind cluster created by setup.sh +# +# Usage: ./teardown.sh [--name ] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CLUSTER_NAME="${MEDIK8S_CLUSTER_NAME:-medik8s-dev}" + +while [[ $# -gt 0 ]]; do + case $1 in + --name) + CLUSTER_NAME="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +if ! command -v kind &>/dev/null; then + echo "Error: kind is not installed." + exit 1 +fi + +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "=== Deleting Kind cluster '${CLUSTER_NAME}' ===" + kind delete cluster --name "${CLUSTER_NAME}" + echo "Done." +elif ${KUBECTL} cluster-info --context "kind-${CLUSTER_NAME}" >/dev/null 2>&1; then + echo "Cluster '${CLUSTER_NAME}' exists but is not visible to 'kind get clusters'." + echo "It was likely created with sudo. Delete it with:" + echo " sudo kind delete cluster --name ${CLUSTER_NAME}" + exit 1 +else + echo "Cluster '${CLUSTER_NAME}' does not exist." +fi