From 9030700ffe13bc9da594bf639a28910c7d51c9b2 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Wed, 27 May 2026 18:16:47 +0100 Subject: [PATCH 1/7] feat: confidential compute --- .goreleaser-docker.yaml | 30 +++ .goreleaser.yaml | 13 + _run/common-commands.mk | 7 +- _run/kube/Makefile | 60 ++++- _run/kube/cc-containerd-setup.sh | 22 ++ _run/kube/cc-setup.yaml | 29 +++ _run/kube/deployment-cc.yaml | 40 +++ _run/kube/provider-cc.yaml | 13 + cluster/client.go | 11 + cluster/inventory.go | 71 +++++- cluster/kube/builder/builder.go | 20 ++ cluster/kube/builder/deployment.go | 4 +- cluster/kube/builder/statefulset.go | 4 +- cluster/kube/builder/workload.go | 97 +++++++- cluster/kube/client_attestation.go | 84 +++++++ .../operators/clients/inventory/inventory.go | 29 ++- cluster/kube/webhook/registration.go | 134 ++++++++++ cluster/kube/webhook/sidecar.go | 234 ++++++++++++++++++ cluster/kube/webhook/sidecar_test.go | 182 ++++++++++++++ cluster/kube/webhook/webhook.go | 152 ++++++++++++ cluster/types/v1beta3/types.go | 18 +- cmd/provider-services/cmd/flags.go | 30 +++ cmd/provider-services/cmd/leaseAttestation.go | 152 ++++++++++++ cmd/provider-services/cmd/root.go | 1 + cmd/provider-services/cmd/run.go | 134 ++++++++++ gateway/rest/attestation.go | 182 ++++++++++++++ gateway/rest/integration_test.go | 2 +- gateway/rest/router.go | 17 +- gateway/rest/server.go | 3 +- go.mod | 43 ++-- go.sum | 82 +++--- mocks/cluster/Client_mock.go | 15 ++ pkg/apis/akash.network/crd.yaml | 2 + pkg/apis/akash.network/v2beta2/manifest.go | 11 +- sidecar/attestation/Dockerfile | 3 + sidecar/attestation/main.go | 66 +++++ sidecar/attestation/server.go | 135 ++++++++++ sidecar/attestation/tee/configfs.go | 64 +++++ sidecar/attestation/tee/mock.go | 103 ++++++++ sidecar/attestation/tee/sevguest.go | 111 +++++++++ sidecar/attestation/tee/tdx.go | 101 ++++++++ sidecar/attestation/tee/tee.go | 90 +++++++ sidecar/attestation/tlsbinding.go | 94 +++++++ 43 files changed, 2602 insertions(+), 93 deletions(-) create mode 100755 _run/kube/cc-containerd-setup.sh create mode 100644 _run/kube/cc-setup.yaml create mode 100644 _run/kube/deployment-cc.yaml create mode 100644 _run/kube/provider-cc.yaml create mode 100644 cluster/kube/client_attestation.go create mode 100644 cluster/kube/webhook/registration.go create mode 100644 cluster/kube/webhook/sidecar.go create mode 100644 cluster/kube/webhook/sidecar_test.go create mode 100644 cluster/kube/webhook/webhook.go create mode 100644 cmd/provider-services/cmd/leaseAttestation.go create mode 100644 gateway/rest/attestation.go create mode 100644 sidecar/attestation/Dockerfile create mode 100644 sidecar/attestation/main.go create mode 100644 sidecar/attestation/server.go create mode 100644 sidecar/attestation/tee/configfs.go create mode 100644 sidecar/attestation/tee/mock.go create mode 100644 sidecar/attestation/tee/sevguest.go create mode 100644 sidecar/attestation/tee/tdx.go create mode 100644 sidecar/attestation/tee/tee.go create mode 100644 sidecar/attestation/tlsbinding.go diff --git a/.goreleaser-docker.yaml b/.goreleaser-docker.yaml index bb978aa3..5bbfd77a 100644 --- a/.goreleaser-docker.yaml +++ b/.goreleaser-docker.yaml @@ -10,6 +10,19 @@ before: - apt update - apt install -y pkg-config libudev-dev:amd64 libudev-dev:arm64 builds: + - id: attestation-sidecar-linux-amd64 + binary: attestation-sidecar + main: ./sidecar/attestation + goarch: + - amd64 + goos: + - linux + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w - id: provider-services-linux-arm64 binary: provider-services main: ./cmd/provider-services @@ -160,3 +173,20 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} image_templates: - "{{ .Env.DOCKER_IMAGE }}:latest-arm64-debug" + - dockerfile: sidecar/attestation/Dockerfile + ids: + - attestation-sidecar-linux-amd64 + use: buildx + goos: linux + goarch: amd64 + build_flag_templates: + - --platform=linux/amd64 + - --label=org.opencontainers.image.title=attestation-sidecar + - --label=org.opencontainers.image.description=Akash attestation sidecar for confidential compute + - --label=org.opencontainers.image.url={{.GitURL}} + - --label=org.opencontainers.image.source={{.GitURL}} + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + image_templates: + - "{{ .Env.DOCKER_IMAGE }}-attestation-sidecar:latest-amd64" diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 1302959c..cef676e4 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -174,6 +174,19 @@ builds: - -extldflags "-L./.cache/lib -lwasmvm_muslc.x86_64 -Wl,-z,muldefs -lm -lrt -lc" gcflags: - "all=-N -l" + - id: attestation-sidecar-linux-amd64 + binary: attestation-sidecar + main: ./sidecar/attestation + goarch: + - amd64 + goos: + - linux + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w universal_binaries: - id: darwin-universal ids: diff --git a/_run/common-commands.mk b/_run/common-commands.mk index 6beb0c4a..5eaf474a 100644 --- a/_run/common-commands.mk +++ b/_run/common-commands.mk @@ -14,7 +14,8 @@ PRICE ?= 10uakt CERT_HOSTNAME ?= localhost LEASE_SERVICES ?= web -RESOURCE_SERVER_HOST ?= localhost:8445 +RESOURCE_SERVER_HOST ?= localhost:8445 +GATEWAY_GRPC_ENDPOINT ?= localhost:8444 GW_AUTH_TYPE ?= jwt @@ -293,6 +294,10 @@ provider-lease-events: --provider "$(PROVIDER_ADDRESS)" \ --auth-type "$(GW_AUTH_TYPE)" +.PHONY: provider-lease-attestation +provider-lease-attestation: + $(PROVIDER_SERVICES) lease-attestation --dseq "$(DSEQ)" --gseq "$(GSEQ)" --oseq "$(OSEQ)" --from "$(KEY_NAME)" --provider "$(PROVIDER_ADDRESS)" --auth-type "$(GW_AUTH_TYPE)" + .PHONY: provider-lease-status provider-lease-status: $(PROVIDER_SERVICES) lease-status \ diff --git a/_run/kube/Makefile b/_run/kube/Makefile index d259bf7b..a06a5fff 100644 --- a/_run/kube/Makefile +++ b/_run/kube/Makefile @@ -1,4 +1,5 @@ -GATEWAY_API ?= false +GATEWAY_API ?= false +CONFIDENTIAL_COMPUTE ?= false KUBE_SETUP_PREREQUISITES ?= \ @@ -23,7 +24,14 @@ kube-setup-ingress-default: kube-setup-ingress-gateway @echo "Gateway API ingress setup complete" endif -SDL_PATH ?= grafana.yaml +# Confidential compute mode overrides +ifeq ($(CONFIDENTIAL_COMPUTE),true) +SDL_PATH := deployment-cc.yaml +PROVIDER_CONFIG_PATH := provider-cc.yaml +ATTESTATION_SIDECAR_IMAGE ?= ghcr.io/akash-network/attestation-sidecar:latest +else +SDL_PATH ?= grafana.yaml +endif GATEWAY_HOSTNAME ?= localhost GATEWAY_HOST ?= $(GATEWAY_HOSTNAME):8443 @@ -42,7 +50,12 @@ provider-run: --bid-price-strategy "randomRange" \ --deployment-runtime-class "none" \ --ip-operator=true \ - $(if $(filter true,$(GATEWAY_API)),--ingress-mode=gateway-api) + $(if $(filter true,$(GATEWAY_API)),--ingress-mode=gateway-api) \ + $(if $(filter true,$(CONFIDENTIAL_COMPUTE)),\ + --attestation-webhook-enabled=true \ + --attestation-sidecar-image="$(ATTESTATION_SIDECAR_IMAGE)" \ + --attestation-mock=true \ + ) .PHONY: provider-lease-ping provider-lease-ping: @@ -52,6 +65,45 @@ provider-lease-ping: hostname-operator: $(PROVIDER_SERVICES) hostname-operator +# Confidential compute cluster setup: creates RuntimeClass objects and labels +# the Kind node with TEE capability labels so the provider can schedule CC +# workloads. The pods won't actually run with Kata (Kind doesn't have it), +# but the webhook, annotations, and sidecar injection can be verified. +.PHONY: kube-setup-cc +kube-setup-cc: + kubectl apply -f cc-setup.yaml + @echo "Labeling kind node for CC..." + @for node in $$(kubectl get nodes -o name); do \ + kubectl label $$node \ + katacontainers.io/kata-runtime=true \ + amd.feature.node.kubernetes.io/snp=true \ + intel.feature.node.kubernetes.io/tdx=true \ + nvidia.com/cc.ready.state=true \ + --overwrite; \ + done + @echo "Registering CC runtime handlers in containerd (aliased to runc)..." + @for node in $$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do \ + docker cp cc-containerd-setup.sh $$node:/cc-containerd-setup.sh; \ + docker exec $$node bash /cc-containerd-setup.sh; \ + done + @echo "Loading attestation sidecar image into Kind..." + $(KIND) load docker-image "$(ATTESTATION_SIDECAR_IMAGE)" --name "$(KIND_NAME)" 2>/dev/null || \ + echo "Sidecar image not found locally. Build it first or set ATTESTATION_SIDECAR_IMAGE." + @echo "CC setup complete. RuntimeClasses, node labels, and containerd handlers applied." + +.PHONY: kube-teardown-cc +kube-teardown-cc: + -kubectl delete -f cc-setup.yaml 2>/dev/null + @for node in $$(kubectl get nodes -o name); do \ + kubectl label $$node \ + katacontainers.io/kata-runtime- \ + amd.feature.node.kubernetes.io/snp- \ + intel.feature.node.kubernetes.io/tdx- \ + nvidia.com/cc.ready.state- \ + 2>/dev/null; \ + done + @echo "CC teardown complete." + .PHONY: clean-kube clean-kube: @@ -61,6 +113,8 @@ kube-deployments-rollout: #$(patsubst %, kube-deployment-rollout-%,$(KUSTOMIZE_I .PHONY: kube-setup-kube ifeq ($(GATEWAY_API),true) kube-setup-kube: kube-setup-hostname-operator-gateway +else ifeq ($(CONFIDENTIAL_COMPUTE),true) +kube-setup-kube: kube-setup-cc else kube-setup-kube: endif diff --git a/_run/kube/cc-containerd-setup.sh b/_run/kube/cc-containerd-setup.sh new file mode 100755 index 00000000..75e56abd --- /dev/null +++ b/_run/kube/cc-containerd-setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Registers mock CC runtime handlers in containerd, aliased to runc. +# Run inside the Kind node via: docker exec bash /path/to/this/script +set -e + +if grep -q kata-qemu-snp /etc/containerd/config.toml; then + echo "CC runtime handlers already registered" + exit 0 +fi + +for rt in kata-qemu-snp kata-qemu-nvidia-gpu-snp kata-qemu-tdx kata-qemu-nvidia-gpu-tdx; do + cat >> /etc/containerd/config.toml < 0; resources[i].Count-- { - sparams, nStatus, cStatus := currInventory.tryAdjust(nodeIdx, adjusted) + sparams, nStatus, cStatus := currInventory.tryAdjust(nodeIdx, adjusted, cfg.ConfidentialCompute, cfg.TEEType) if !cStatus { // cannot satisfy cluster-wide resources, stop lookup break nodes diff --git a/cluster/kube/webhook/registration.go b/cluster/kube/webhook/registration.go new file mode 100644 index 00000000..d01d5861 --- /dev/null +++ b/cluster/kube/webhook/registration.go @@ -0,0 +1,134 @@ +package webhook + +import ( + "context" + "fmt" + + "cosmossdk.io/log" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +const ( + webhookName = "attestation-sidecar-injector.akash.network" + webhookConfigName = "akash-attestation-sidecar-injector" +) + +// RegisterWebhookConfiguration creates or updates the MutatingWebhookConfiguration +// in the cluster. This tells the K8s API server to send pod CREATE requests +// (in Akash-managed namespaces) to our webhook for sidecar injection. +// +// The caBundle must be PEM-encoded. When webhookURL is non-empty, it registers +// with a URL endpoint (for local dev). Otherwise it uses a K8s Service reference. +func RegisterWebhookConfiguration(ctx context.Context, kc kubernetes.Interface, log log.Logger, serviceName, serviceNamespace string, caBundle []byte, webhookPort int32, webhookURL string) error { + failPolicy := admissionregistrationv1.Fail // fail-closed + sideEffects := admissionregistrationv1.SideEffectClassNone + matchPolicy := admissionregistrationv1.Equivalent + timeoutSec := int32(5) + + path := "/mutate" + + var clientConfig admissionregistrationv1.WebhookClientConfig + if webhookURL != "" { + // Local dev: provider runs outside the cluster, use URL endpoint + url := webhookURL + path + clientConfig = admissionregistrationv1.WebhookClientConfig{ + URL: &url, + CABundle: caBundle, + } + } else { + // Production: provider runs as a K8s service + clientConfig = admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: serviceName, + Namespace: serviceNamespace, + Path: &path, + Port: &webhookPort, + }, + CABundle: caBundle, + } + } + + cfg := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookConfigName, + Labels: map[string]string{ + "app.kubernetes.io/name": "attestation-webhook", + "app.kubernetes.io/part-of": "provider", + "app.kubernetes.io/component": "admission-webhook", + }, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: webhookName, + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failPolicy, + SideEffects: &sideEffects, + MatchPolicy: &matchPolicy, + TimeoutSeconds: &timeoutSec, + ClientConfig: clientConfig, + // Only intercept pod creation in Akash-managed namespaces. + // System pods and non-Akash workloads are never affected. + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + builder.AkashManagedLabelName: "true", + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + }, + }, + } + + client := kc.AdmissionregistrationV1().MutatingWebhookConfigurations() + + existing, err := client.Get(ctx, webhookConfigName, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + _, err = client.Create(ctx, cfg, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create MutatingWebhookConfiguration: %w", err) + } + log.Info("created MutatingWebhookConfiguration", "name", webhookConfigName) + return nil + } + if err != nil { + return fmt.Errorf("get MutatingWebhookConfiguration: %w", err) + } + + cfg.ResourceVersion = existing.ResourceVersion + _, err = client.Update(ctx, cfg, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("update MutatingWebhookConfiguration: %w", err) + } + log.Info("updated MutatingWebhookConfiguration", "name", webhookConfigName) + return nil +} + +// DeregisterWebhookConfiguration removes the MutatingWebhookConfiguration +// from the cluster. Call this on shutdown to avoid dangling webhooks that +// block pod creation when the provider is not running. +func DeregisterWebhookConfiguration(ctx context.Context, kc kubernetes.Interface, log log.Logger) { + err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete( + ctx, webhookConfigName, metav1.DeleteOptions{}, + ) + if err != nil && !kerrors.IsNotFound(err) { + log.Error("failed to deregister MutatingWebhookConfiguration", "err", err) + return + } + log.Info("deregistered MutatingWebhookConfiguration", "name", webhookConfigName) +} diff --git a/cluster/kube/webhook/sidecar.go b/cluster/kube/webhook/sidecar.go new file mode 100644 index 00000000..cea0f3ea --- /dev/null +++ b/cluster/kube/webhook/sidecar.go @@ -0,0 +1,234 @@ +package webhook + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +const ( + sidecarContainerName = "akash-attestation-sidecar" + sidecarPort = int32(8790) + tsmVolumeName = "tsm-reports" + sevGuestVolumeName = "sev-guest" +) + +// ShouldInject returns true if the pod should have the attestation sidecar injected. +// Trigger conditions: +// - Pod has the akash.network managed label +// - Pod has a Kata CC RuntimeClassName +// - Tenant has not opted out via attestation-disabled annotation +// - Sidecar is not already present (idempotency) +func ShouldInject(pod *corev1.Pod) bool { + if pod.Labels[builder.AkashManagedLabelName] != builder.ValTrue { + return false + } + + if pod.Spec.RuntimeClassName == nil { + return false + } + + if !builder.IsConfidentialComputeRuntimeClass(*pod.Spec.RuntimeClassName) { + return false + } + + // Tenant opted out of attestation sidecar injection + if pod.Annotations[builder.AkashAttestationDisabledAnnotation] == "true" { + return false + } + + for _, c := range pod.Spec.Containers { + if c.Name == sidecarContainerName { + return false // already injected + } + } + + return true +} + +type jsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +// BuildSidecarPatch generates a JSON Patch (RFC 6902) that injects the +// attestation sidecar container, required volumes, and volume mounts into +// the pod spec. +// +// The sidecar runs INSIDE the Kata VM (same pod, same VM as the tenant workload). +// It requires CAP_SYS_ADMIN for configfs mount operations within the guest kernel. +func BuildSidecarPatch(pod *corev1.Pod, sidecarImage string, extraEnv []corev1.EnvVar) ([]byte, error) { + if sidecarImage == "" { + return nil, fmt.Errorf("attestation sidecar image not configured") + } + + runtimeClass := "" + if pod.Spec.RuntimeClassName != nil { + runtimeClass = *pod.Spec.RuntimeClassName + } + + isGPU := builder.IsGPURuntimeClass(runtimeClass) + + mockMode := isMockMode(extraEnv) + container := buildSidecarContainer(sidecarImage, isGPU, extraEnv) + volumes := buildSidecarVolumes(isGPU, mockMode) + + var patches []jsonPatch + + // Add sidecar container + if len(pod.Spec.Containers) == 0 { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/containers", + Value: []corev1.Container{container}, + }) + } else { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/containers/-", + Value: container, + }) + } + + // Add volumes if the pod has no volumes, create the array with all + // volumes at once; otherwise append individually. + if len(pod.Spec.Volumes) == 0 { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/volumes", + Value: volumes, + }) + } else { + for _, vol := range volumes { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/volumes/-", + Value: vol, + }) + } + } + + return json.Marshal(patches) +} + +func buildSidecarContainer(image string, isGPU bool, extraEnv []corev1.EnvVar) corev1.Container { + c := corev1.Container{ + Name: sidecarContainerName, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/attestation-sidecar"}, + Ports: []corev1.ContainerPort{{ + Name: "attestation", + ContainerPort: sidecarPort, + Protocol: corev1.ProtocolTCP, + }}, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("32Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + }, + Env: append([]corev1.EnvVar{ + {Name: "ATTESTATION_LISTEN_ADDR", Value: fmt.Sprintf(":%d", sidecarPort)}, + }, extraEnv...), + } + + // Volume mounts: in mock mode use writable paths (distroless /sys is read-only). + // In production, Kata VMs have a writable guest /sys/kernel/config. + mockMode := isMockMode(extraEnv) + if mockMode { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: tsmVolumeName, + MountPath: "/var/run/mock-tee/tsm-report", + }) + if !isGPU { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: sevGuestVolumeName, + MountPath: "/var/run/mock-tee/sev-guest", + }) + } + } else { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: tsmVolumeName, + MountPath: "/sys/kernel/config/tsm/report", + }) + if !isGPU { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: sevGuestVolumeName, + MountPath: "/dev/sev-guest", + }) + } + } + + return c +} + +func isMockMode(extraEnv []corev1.EnvVar) bool { + for _, env := range extraEnv { + if env.Name == "ATTESTATION_MOCK" && env.Value == "true" { + return true + } + } + return false +} + +func buildSidecarVolumes(isGPU bool, mockMode bool) []corev1.Volume { + // Mock mode: use emptyDir so pods can start on non-TEE hosts (Kind). + // The mock TEE provider generates synthetic reports in-memory and + // doesn't access these mount paths. + if mockMode { + volumes := []corev1.Volume{ + {Name: tsmVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } + if !isGPU { + volumes = append(volumes, corev1.Volume{ + Name: sevGuestVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + } + return volumes + } + + // Production: hostPath volumes inside the Kata VM guest filesystem. + hostPathDirOrCreate := corev1.HostPathDirectoryOrCreate + hostPathCharDev := corev1.HostPathCharDev + + volumes := []corev1.Volume{ + { + Name: tsmVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/kernel/config/tsm/report", + Type: &hostPathDirOrCreate, + }, + }, + }, + } + + if !isGPU { + volumes = append(volumes, corev1.Volume{ + Name: sevGuestVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/sev-guest", + Type: &hostPathCharDev, + }, + }, + }) + } + + return volumes +} diff --git a/cluster/kube/webhook/sidecar_test.go b/cluster/kube/webhook/sidecar_test.go new file mode 100644 index 00000000..ddd0c7a3 --- /dev/null +++ b/cluster/kube/webhook/sidecar_test.go @@ -0,0 +1,182 @@ +package webhook + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +func strPtr(s string) *string { return &s } + +func TestShouldInject_CCPodWithGPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUSNP), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_CCPodCPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_NonCCPod(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr("nvidia"), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_NoRuntimeClass(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{}, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_AttestationDisabled(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + Annotations: map[string]string{builder.AkashAttestationDisabledAnnotation: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_MissingLabel(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{}}, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_AlreadyInjected(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + Containers: []corev1.Container{ + {Name: sidecarContainerName}, + }, + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_TDXCPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuTDX), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_TDXWithGPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUTDX), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestBuildSidecarPatch_GPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUSNP), + Containers: []corev1.Container{ + {Name: "workload", Image: "myimage"}, + }, + }, + } + + patchBytes, err := BuildSidecarPatch(pod, "ghcr.io/akash-network/attestation-sidecar:latest", nil) + require.NoError(t, err) + require.NotEmpty(t, patchBytes) + + var patches []jsonPatch + err = json.Unmarshal(patchBytes, &patches) + require.NoError(t, err) + + // Should have container patch + volumes array patch + require.Len(t, patches, 2) // container + volumes +} + +func TestBuildSidecarPatch_CPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + Containers: []corev1.Container{ + {Name: "workload", Image: "myimage"}, + }, + }, + } + + patchBytes, err := BuildSidecarPatch(pod, "ghcr.io/akash-network/attestation-sidecar:latest", nil) + require.NoError(t, err) + + var patches []jsonPatch + err = json.Unmarshal(patchBytes, &patches) + require.NoError(t, err) + + // Should have container patch + volumes array patch (TSM + sev-guest in one array) + require.Len(t, patches, 2) +} + +func TestBuildSidecarPatch_EmptyImage(t *testing.T) { + pod := &corev1.Pod{ + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + + _, err := BuildSidecarPatch(pod, "", nil) + require.Error(t, err) +} diff --git a/cluster/kube/webhook/webhook.go b/cluster/kube/webhook/webhook.go new file mode 100644 index 00000000..53244487 --- /dev/null +++ b/cluster/kube/webhook/webhook.go @@ -0,0 +1,152 @@ +package webhook + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "cosmossdk.io/log" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" +) + +var ( + runtimeScheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(runtimeScheme) + deserializer = codecs.UniversalDeserializer() +) + +// Config holds configuration for the attestation webhook server. +type Config struct { + SidecarImage string + SidecarEnv []corev1.EnvVar // additional env vars injected into the sidecar container + ListenAddr string + TLSCert tls.Certificate + Log log.Logger +} + +// Server handles mutating admission webhook requests for attestation +// sidecar injection into confidential compute pods. +type Server struct { + server *http.Server + config Config +} + +// NewServer creates a new attestation webhook server. +// Failure mode: fail-closed. A CC workload without an attestation sidecar +// is silently unverifiable — the tenant cannot confirm TEE execution. +func NewServer(cfg Config) *Server { + s := &Server{config: cfg} + + mux := http.NewServeMux() + mux.HandleFunc("/mutate", s.handleMutate) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + s.server = &http.Server{ + Addr: cfg.ListenAddr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cfg.TLSCert}, + MinVersion: tls.VersionTLS12, + }, + } + + return s +} + +// ListenAndServeTLS starts the webhook HTTPS server. +func (s *Server) ListenAndServeTLS() error { + return s.server.ListenAndServeTLS("", "") +} + +// Shutdown gracefully shuts down the server. +func (s *Server) Shutdown(ctx context.Context) error { + return s.server.Shutdown(ctx) +} + +func (s *Server) handleMutate(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + s.config.Log.Error("failed to read request body", "err", err) + http.Error(w, "failed to read body", http.StatusBadRequest) + return + } + + if len(body) == 0 { + http.Error(w, "empty body", http.StatusBadRequest) + return + } + + var admissionReview admissionv1.AdmissionReview + if _, _, err := deserializer.Decode(body, nil, &admissionReview); err != nil { + s.config.Log.Error("failed to decode admission review", "err", err) + http.Error(w, fmt.Sprintf("decode error: %v", err), http.StatusBadRequest) + return + } + + response := s.mutate(admissionReview.Request) + + admissionReview.Response = response + admissionReview.Response.UID = admissionReview.Request.UID + + resp, err := json.Marshal(admissionReview) + if err != nil { + s.config.Log.Error("failed to marshal response", "err", err) + http.Error(w, fmt.Sprintf("marshal error: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(resp) //nolint:errcheck +} + +func (s *Server) mutate(req *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + if req == nil { + return &admissionv1.AdmissionResponse{Allowed: true} + } + + var pod corev1.Pod + if err := json.Unmarshal(req.Object.Raw, &pod); err != nil { + s.config.Log.Error("failed to unmarshal pod", "err", err) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: fmt.Sprintf("failed to unmarshal pod: %v", err), + }, + } + } + + if !ShouldInject(&pod) { + return &admissionv1.AdmissionResponse{Allowed: true} + } + + patch, err := BuildSidecarPatch(&pod, s.config.SidecarImage, s.config.SidecarEnv) + if err != nil { + s.config.Log.Error("failed to build sidecar patch", "err", err) + // Fail-closed: reject the pod if we can't build the patch + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: fmt.Sprintf("failed to build attestation sidecar patch: %v", err), + }, + } + } + + patchType := admissionv1.PatchTypeJSONPatch + return &admissionv1.AdmissionResponse{ + Allowed: true, + Patch: patch, + PatchType: &patchType, + } +} diff --git a/cluster/types/v1beta3/types.go b/cluster/types/v1beta3/types.go index 392a44af..3aba0b7c 100644 --- a/cluster/types/v1beta3/types.go +++ b/cluster/types/v1beta3/types.go @@ -52,7 +52,9 @@ type LeaseEvent struct { } type InventoryOptions struct { - DryRun bool + DryRun bool + ConfidentialCompute bool + TEEType string // "amd-sev-snp" or "intel-tdx" } type InventoryOption func(*InventoryOptions) *InventoryOptions @@ -64,6 +66,20 @@ func WithDryRun() InventoryOption { } } +func WithConfidentialCompute() InventoryOption { + return func(opts *InventoryOptions) *InventoryOptions { + opts.ConfidentialCompute = true + return opts + } +} + +func WithTEEType(teeType string) InventoryOption { + return func(opts *InventoryOptions) *InventoryOptions { + opts.TEEType = teeType + return opts + } +} + type Inventory interface { Adjust(ReservationGroup, ...InventoryOption) error Metrics() inventoryV1.Metrics diff --git a/cmd/provider-services/cmd/flags.go b/cmd/provider-services/cmd/flags.go index 8fff9a03..e434e684 100644 --- a/cmd/provider-services/cmd/flags.go +++ b/cmd/provider-services/cmd/flags.go @@ -354,5 +354,35 @@ func addRunFlags(cmd *cobra.Command) error { return err } + cmd.Flags().Bool(FlagAttestationWebhookEnabled, false, "Enable attestation sidecar mutating admission webhook for confidential compute") + if err := viper.BindPFlag(FlagAttestationWebhookEnabled, cmd.Flags().Lookup(FlagAttestationWebhookEnabled)); err != nil { + return err + } + + cmd.Flags().Int(FlagAttestationWebhookPort, 9443, "Port for the attestation sidecar webhook server") + if err := viper.BindPFlag(FlagAttestationWebhookPort, cmd.Flags().Lookup(FlagAttestationWebhookPort)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationSidecarImage, "", "Docker image for the attestation sidecar (required when webhook is enabled)") + if err := viper.BindPFlag(FlagAttestationSidecarImage, cmd.Flags().Lookup(FlagAttestationSidecarImage)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationExpectedMeasurement, "", "Expected SEV-SNP launch measurement (hex) for the attestation directory. Advisory hint only (invariant #4)") + if err := viper.BindPFlag(FlagAttestationExpectedMeasurement, cmd.Flags().Lookup(FlagAttestationExpectedMeasurement)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationExpectedImageDigest, "", "Expected Kata VM image digest for the attestation directory. Advisory hint only (invariant #4)") + if err := viper.BindPFlag(FlagAttestationExpectedImageDigest, cmd.Flags().Lookup(FlagAttestationExpectedImageDigest)); err != nil { + return err + } + + cmd.Flags().Bool(FlagAttestationMockMode, false, "Run attestation sidecar in mock mode (synthetic reports, no TEE hardware). For local development only") + if err := viper.BindPFlag(FlagAttestationMockMode, cmd.Flags().Lookup(FlagAttestationMockMode)); err != nil { + return err + } + return nil } diff --git a/cmd/provider-services/cmd/leaseAttestation.go b/cmd/provider-services/cmd/leaseAttestation.go new file mode 100644 index 00000000..6d8803d5 --- /dev/null +++ b/cmd/provider-services/cmd/leaseAttestation.go @@ -0,0 +1,152 @@ +package cmd + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/spf13/cobra" + "pkg.akt.dev/go/cli" + cflags "pkg.akt.dev/go/cli/flags" +) + +func leaseAttestationCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lease-attestation", + Short: "request attestation quote from a confidential compute lease", + SilenceUsage: true, + Args: cobra.ExactArgs(0), + PreRunE: ProviderPersistentPreRunE, + RunE: func(cmd *cobra.Command, _ []string) error { + return doLeaseAttestation(cmd) + }, + } + + AddProviderOperationFlagsToCmd(cmd) + addLeaseFlags(cmd) + addAuthFlags(cmd) + + return cmd +} + +// attestationQuoteRequest matches the sidecar's QuoteRequest type. +type attestationQuoteRequest struct { + Nonce string `json:"nonce"` + BindTLS bool `json:"bind_tls,omitempty"` +} + +// attestationQuoteResponse matches the sidecar's QuoteResponse type. +type attestationQuoteResponse struct { + Report string `json:"report"` + CertChain string `json:"cert_chain"` + TEEType string `json:"tee_type"` + AuxBlob string `json:"auxblob"` + TLSBound bool `json:"tls_bound"` +} + +// attestationResult is the CLI output format. +type attestationResult struct { + Nonce string `json:"nonce"` + Quote attestationQuoteResponse `json:"quote"` + ReportSize int `json:"report_size_bytes"` + NonceVerified bool `json:"nonce_verified"` + MockReport bool `json:"mock_report"` +} + +func doLeaseAttestation(cmd *cobra.Command) error { + ctx := cmd.Context() + cl, err := cli.ClientFromContext(ctx) + if err != nil && !errors.Is(err, cli.ErrContextValueNotSet) { + return err + } + cctx, err := cli.GetClientTxContext(cmd) + if err != nil { + return err + } + + bid, err := cflags.BidIDFromFlags(cmd.Flags(), cflags.WithOwner(cctx.FromAddress)) + if err != nil { + return err + } + + paddr, err := sdk.AccAddressFromBech32(bid.Provider) + if err != nil { + return err + } + + gclient, err := setupProviderClient(ctx, cctx, cmd.Flags(), queryClientOrNil(cl), paddr, true) + if err != nil { + return err + } + + // Generate a random 64-byte nonce + var nonce [64]byte + if _, err := rand.Read(nonce[:]); err != nil { + return fmt.Errorf("generate nonce: %w", err) + } + nonceB64 := base64.StdEncoding.EncodeToString(nonce[:]) + + // Build the quote request + reqBody, err := json.Marshal(attestationQuoteRequest{ + Nonce: nonceB64, + }) + if err != nil { + return fmt.Errorf("marshal quote request: %w", err) + } + + // Call the provider's attestation quote endpoint (authenticated, same as lease-status) + respBody, err := gclient.AttestationQuote(ctx, bid.LeaseID(), reqBody) + if err != nil { + return showErrorToUser(err) + } + + // Parse the response + var quote attestationQuoteResponse + if err := json.Unmarshal(respBody, "e); err != nil { + return fmt.Errorf("parse quote response: %w", err) + } + + // Decode report to check size and verify nonce + reportBytes, err := base64.StdEncoding.DecodeString(quote.Report) + if err != nil { + return fmt.Errorf("decode report: %w", err) + } + + // Check if this is a mock report (starts with "MOCK") + isMock := len(reportBytes) >= 4 && string(reportBytes[0:4]) == "MOCK" + + // Verify nonce echo in report_data + nonceVerified := false + if isMock && len(reportBytes) >= 144 { + // Mock report: nonce at offset 80 + nonceVerified = true + for i := 0; i < 64; i++ { + if reportBytes[80+i] != nonce[i] { + nonceVerified = false + break + } + } + } else if !isMock && len(reportBytes) >= 0x90 { + // Real SNP report: REPORT_DATA at offset 0x50 (80 decimal) + nonceVerified = true + for i := 0; i < 64; i++ { + if reportBytes[0x50+i] != nonce[i] { + nonceVerified = false + break + } + } + } + + result := attestationResult{ + Nonce: nonceB64, + Quote: quote, + ReportSize: len(reportBytes), + NonceVerified: nonceVerified, + MockReport: isMock, + } + + return cli.PrintJSON(cctx, result) +} diff --git a/cmd/provider-services/cmd/root.go b/cmd/provider-services/cmd/root.go index e6993656..f6197cc9 100644 --- a/cmd/provider-services/cmd/root.go +++ b/cmd/provider-services/cmd/root.go @@ -41,6 +41,7 @@ func NewRootCmd() *cobra.Command { cmd.AddCommand(leaseStatusCmd()) cmd.AddCommand(leaseEventsCmd()) cmd.AddCommand(leaseLogsCmd()) + cmd.AddCommand(leaseAttestationCmd()) cmd.AddCommand(serviceStatusCmd()) cmd.AddCommand(RunCmd()) cmd.AddCommand(LeaseShellCmd()) diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 519e2752..91eeed60 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -2,9 +2,17 @@ package cmd import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + crypto_rand "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "io" + "math/big" "net" "net/http" "os" @@ -45,6 +53,7 @@ import ( kubehostname "github.com/akash-network/provider/cluster/kube/operators/clients/hostname" kubeinventory "github.com/akash-network/provider/cluster/kube/operators/clients/inventory" kubeip "github.com/akash-network/provider/cluster/kube/operators/clients/ip" + attestwebhook "github.com/akash-network/provider/cluster/kube/webhook" cip "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip" clfromctx "github.com/akash-network/provider/cluster/types/v1beta3/fromctx" providerflags "github.com/akash-network/provider/cmd/provider-services/cmd/flags" @@ -129,6 +138,12 @@ const ( FlagGatewayName = "gateway-name" FlagGatewayNamespace = "gateway-namespace" FlagGatewayProvider = "gateway-provider" + FlagAttestationWebhookEnabled = "attestation-webhook-enabled" + FlagAttestationWebhookPort = "attestation-webhook-port" + FlagAttestationSidecarImage = "attestation-sidecar-image" + FlagAttestationExpectedMeasurement = "attestation-expected-measurement" + FlagAttestationExpectedImageDigest = "attestation-expected-image-digest" + FlagAttestationMockMode = "attestation-mock" ) const ( @@ -771,6 +786,10 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { gwaddr, cctx.FromAddress, clusterSettings, + gwrest.AttestationConfig{ + ExpectedLaunchMeasurement: viper.GetString(FlagAttestationExpectedMeasurement), + ExpectedImageDigest: viper.GetString(FlagAttestationExpectedImageDigest), + }, ) if err != nil { return err @@ -802,6 +821,121 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { return gwRest.Close() }) + // Start attestation webhook server if enabled + if viper.GetBool(FlagAttestationWebhookEnabled) { + sidecarImage := viper.GetString(FlagAttestationSidecarImage) + if sidecarImage == "" { + return fmt.Errorf("%w: %s is required when %s is enabled", + errInvalidConfig, FlagAttestationSidecarImage, FlagAttestationWebhookEnabled) + } + webhookPort := viper.GetInt(FlagAttestationWebhookPort) + webhookAddr := fmt.Sprintf(":%d", webhookPort) + + // Load TLS cert: use gateway cert files if provided, otherwise + // generate a self-signed cert (sufficient for local dev / mock mode). + var tlsCert tls.Certificate + certFile := viper.GetString(FlagGatewayTLSCert) + keyFile := viper.GetString(FlagGatewayTLSKey) + if certFile != "" && keyFile != "" { + tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("attestation webhook: load TLS cert: %w", err) + } + } else { + // Generate a self-signed cert for the webhook. + // In production, use --gateway-tls-cert/--gateway-tls-key with a cert + // matching the webhook's K8s service DNS name. + key, genErr := ecdsa.GenerateKey(elliptic.P256(), crypto_rand.Reader) + if genErr != nil { + return fmt.Errorf("attestation webhook: generate key: %w", genErr) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "akash-attestation-webhook"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost", "host.docker.internal"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + certDER, genErr := x509.CreateCertificate(crypto_rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if genErr != nil { + return fmt.Errorf("attestation webhook: create cert: %w", genErr) + } + tlsCert = tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: key, + } + logger.Info("generated self-signed TLS cert for attestation webhook") + } + + webhookLog := logger.With("module", "attestation-webhook") + + var sidecarEnv []corev1.EnvVar + if viper.GetBool(FlagAttestationMockMode) { + webhookLog.Info("attestation mock mode enabled — sidecar will produce synthetic reports") + sidecarEnv = append(sidecarEnv, corev1.EnvVar{ + Name: "ATTESTATION_MOCK", Value: "true", + }) + } + + webhookSrv := attestwebhook.NewServer(attestwebhook.Config{ + SidecarImage: sidecarImage, + SidecarEnv: sidecarEnv, + ListenAddr: webhookAddr, + TLSCert: tlsCert, + Log: webhookLog, + }) + + // Register the MutatingWebhookConfiguration so the K8s API server + // sends pod CREATE requests to our webhook for sidecar injection. + // The CA bundle must be PEM-encoded. For self-signed certs, it's the leaf cert itself. + caBundle := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: tlsCert.Certificate[0], + }) + if webhookKC, kcErr := fromctx.KubeClientFromCtx(ctx); kcErr == nil { + ns := viper.GetString(providerflags.FlagK8sManifestNS) + + // When running locally (mock mode), use a URL endpoint so the Kind + // cluster can reach the webhook on the host machine. + var webhookURL string + if viper.GetBool(FlagAttestationMockMode) { + webhookURL = fmt.Sprintf("https://host.docker.internal:%d", webhookPort) + } + + regErr := attestwebhook.RegisterWebhookConfiguration( + ctx, webhookKC, webhookLog, + "akash-provider", ns, caBundle, int32(webhookPort), webhookURL, //nolint:gosec // port is bounded by flag default + ) + if regErr != nil { + return fmt.Errorf("register attestation webhook: %w", regErr) + } + } else { + webhookLog.Error("kube client unavailable, skipping webhook registration", "err", kcErr) + } + + group.Go(func() error { + logger.Info("starting attestation webhook", "addr", webhookAddr) + return webhookSrv.ListenAndServeTLS() + }) + + group.Go(func() error { + <-ctx.Done() + // Deregister webhook on shutdown to avoid dangling fail-closed + // webhooks blocking pod creation when the provider is down. + if webhookKC, kcErr := fromctx.KubeClientFromCtx(ctx); kcErr == nil { + attestwebhook.DeregisterWebhookConfiguration( + context.Background(), webhookKC, webhookLog, + ) + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return webhookSrv.Shutdown(shutdownCtx) + }) + } + if metricsRouter != nil { group.Go(func() error { // nolint: gosec diff --git a/gateway/rest/attestation.go b/gateway/rest/attestation.go new file mode 100644 index 00000000..114ff7f7 --- /dev/null +++ b/gateway/rest/attestation.go @@ -0,0 +1,182 @@ +package rest + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + + "cosmossdk.io/log" + sdk "github.com/cosmos/cosmos-sdk/types" + gcontext "github.com/gorilla/context" + "github.com/gorilla/mux" + mtypes "pkg.akt.dev/go/node/market/v1" + + "github.com/akash-network/provider/cluster" + "github.com/akash-network/provider/cluster/kube/builder" +) + +const attestationProtocolVersion = "2" + +// AttestationConfig holds provider-configured attestation parameters +// that are returned as advisory hints in the directory response. +// All values are explicitly untrusted (invariant #4). +type AttestationConfig struct { + ExpectedLaunchMeasurement string + ExpectedImageDigest string +} + +// AttestationDirectoryResponse is the response from the directory endpoint. +// This endpoint is explicitly UNTRUSTED (architectural invariant #3). +// All fields are advisory routing hints only. The tenant must verify +// all claims against hardware-signed attestation evidence. +type AttestationDirectoryResponse struct { + // Endpoint is the lease-scoped URL to reach the attestation sidecar. + // Tenant calls POST on this path with their nonce. + Endpoint string `json:"endpoint"` + + // ExpectedLaunchMeasurement is an advisory hint the tenant can compare + // against the MEASUREMENT field in the hardware-signed SNP report. + // NOT a standalone trust signal (invariant #4). + ExpectedLaunchMeasurement string `json:"expected_launch_measurement,omitempty"` + + // ExpectedImageDigest is an advisory hint for the Kata VM image digest. + // NOT a standalone trust signal (invariant #4). + ExpectedImageDigest string `json:"expected_image_digest,omitempty"` + + ProtocolVersion string `json:"protocol_version"` + RuntimeClass string `json:"runtime_class"` + TEEType string `json:"tee_type"` // "amd-sev-snp" +} + +// createAttestationDirectoryHandler returns the directory endpoint handler. +// The directory API is unauthenticated — the tenant needs to discover the +// sidecar before establishing a trusted channel. Responses contain no secrets. +func createAttestationDirectoryHandler(log log.Logger, cclient cluster.ReadClient, attestCfg AttestationConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + dseq, err := strconv.ParseUint(vars["dseq"], 10, 64) + if err != nil { + http.Error(w, "invalid dseq", http.StatusBadRequest) + return + } + gseq, err := strconv.ParseUint(vars["gseq"], 10, 32) + if err != nil { + http.Error(w, "invalid gseq", http.StatusBadRequest) + return + } + oseq, err := strconv.ParseUint(vars["oseq"], 10, 32) + if err != nil { + http.Error(w, "invalid oseq", http.StatusBadRequest) + return + } + + log.Debug("attestation directory request", + "dseq", dseq, "gseq", gseq, "oseq", oseq) + + // The provider address is stored in the request context by the router middleware. + providerAddr, ok := gcontext.Get(r, providerContextKey).(sdk.Address) + if !ok { + http.Error(w, "provider address unavailable", http.StatusInternalServerError) + return + } + + // Construct a LeaseID to look up the manifest group. + // The directory endpoint is unauthenticated so we don't have the owner address. + // GetManifestGroup resolves the lease namespace from the full LeaseID via + // a SHA-224 hash. Without the owner, we pass an empty owner — the namespace + // hash won't match any real lease. Instead, we need to search differently. + // + // For the directory endpoint, we construct a partial lease ID. The provider + // can look up by dseq since it has the CRD manifest labeled with dseq. + // The GetManifestGroup uses LidNS which requires a full LeaseID. + // Since this endpoint is unauthenticated and untrusted (invariant #3), + // we accept that the owner field is empty — the lookup will work only + // if the caller also provides it as a query parameter. + owner := r.URL.Query().Get("owner") + + lid := mtypes.LeaseID{ + Owner: owner, + DSeq: dseq, + GSeq: uint32(gseq), //nolint:gosec + OSeq: uint32(oseq), //nolint:gosec + Provider: providerAddr.String(), + } + + // Look up the manifest to find the runtime class. + found, mgroup, err := cclient.GetManifestGroup(r.Context(), lid) + if err != nil { + log.Error("attestation directory: manifest lookup failed", "err", err) + writeClusterError(w, err) + return + } + if !found { + http.Error(w, "lease not found", http.StatusNotFound) + return + } + + // Find the first service with a CC runtime class. + var runtimeClass string + for _, svc := range mgroup.Services { + if svc.SchedulerParams != nil && builder.IsConfidentialComputeRuntimeClass(svc.SchedulerParams.RuntimeClass) { + runtimeClass = svc.SchedulerParams.RuntimeClass + break + } + } + + if runtimeClass == "" { + http.Error(w, "lease is not a confidential compute deployment", http.StatusNotFound) + return + } + + teeType := builder.TEETypeForRuntimeClass(runtimeClass) + + resp := AttestationDirectoryResponse{ + Endpoint: fmt.Sprintf("/lease/%d/%d/%d/attestation/quote", dseq, gseq, oseq), + ExpectedLaunchMeasurement: attestCfg.ExpectedLaunchMeasurement, + ExpectedImageDigest: attestCfg.ExpectedImageDigest, + ProtocolVersion: attestationProtocolVersion, + RuntimeClass: runtimeClass, + TEEType: teeType, + } + + // Explicitly untrusted — staleness is HTTP-native (invariant #3) + w.Header().Set("Cache-Control", "max-age=60, must-revalidate") + w.Header().Set("Content-Type", contentTypeJSON) + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +// createAttestationQuoteHandler handles attestation quote requests by calling +// the sidecar via the cluster.Client interface, which resolves the pod IP from +// K8s and calls the sidecar directly — the same pattern used by Exec and LeaseLogs. +// +// The provider forwards the tenant's nonce verbatim to the sidecar and returns +// the hardware-signed evidence verbatim. The provider never inspects or modifies +// either payload (invariants #1 and #5). +func createAttestationQuoteHandler(log log.Logger, cclient cluster.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + leaseID := requestLeaseID(r) + + log.Debug("attestation quote request", "lease", leaseID.String()) + + body, err := io.ReadAll(io.LimitReader(r.Body, 4096)) + if err != nil { + http.Error(w, "failed to read request body", http.StatusBadRequest) + return + } + + respBody, statusCode, err := cclient.AttestationQuote(r.Context(), leaseID, body) + if err != nil { + log.Error("attestation quote failed", "err", err, "lease", leaseID.String()) + http.Error(w, fmt.Sprintf("attestation quote failed: %v", err), statusCode) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + w.Write(respBody) //nolint:errcheck + } +} diff --git a/gateway/rest/integration_test.go b/gateway/rest/integration_test.go index 79269a80..3f3beac8 100644 --- a/gateway/rest/integration_test.go +++ b/gateway/rest/integration_test.go @@ -521,7 +521,7 @@ func withServer( ) { addr := sdk.AccAddress(keys[0].PubKey().Address()) - router := newRouter(testutil.Logger(t), addr, pclient, map[interface{}]interface{}{}) + router := newRouter(testutil.Logger(t), addr, pclient, map[interface{}]interface{}{}, AttestationConfig{}) pstorage, err := memory.NewMemory() require.NoError(t, err) diff --git a/gateway/rest/router.go b/gateway/rest/router.go index 3075206c..a97ed49b 100644 --- a/gateway/rest/router.go +++ b/gateway/rest/router.go @@ -89,7 +89,7 @@ type wsStreamConfig struct { client cluster.ReadClient } -func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxConfig map[interface{}]interface{}, middlewares ...mux.MiddlewareFunc) *mux.Router { +func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxConfig map[interface{}]interface{}, attestCfg AttestationConfig, middlewares ...mux.MiddlewareFunc) *mux.Router { router := mux.NewRouter() // store provider address in context as a lease's endpoints below need it @@ -123,6 +123,14 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxCon createStatusHandler(log, pclient, addr)). Methods("GET") + // GET /attestation/directory/{dseq}/{gseq}/{oseq} + // Attestation directory — explicitly UNTRUSTED (invariant #3). + // Returns advisory routing hints for the attestation sidecar. + // Unauthenticated: tenant needs this before establishing a trusted channel. + router.HandleFunc("/attestation/directory/{dseq}/{gseq}/{oseq}", + createAttestationDirectoryHandler(log, pclient.Cluster(), attestCfg)). + Methods(http.MethodGet) + authedRouter := router.NewRoute().Subrouter() authedRouter.Use( authorizeProviderMiddleware, @@ -155,6 +163,13 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxCon requireLeaseID, ) + // POST /lease//attestation/quote + // Calls the attestation sidecar inside the CC pod via direct K8s pod IP access. + // Provider forwards nonce/response verbatim (invariants #1, #5). + lrouter.HandleFunc("/attestation/quote", + createAttestationQuoteHandler(log, pclient.Cluster())). + Methods(http.MethodPost) + mrouter = lrouter.NewRoute().Subrouter() mrouter.Use(requireEndpointScopeForLeaseID(ajwt.PermissionScopeGetManifest)) diff --git a/gateway/rest/server.go b/gateway/rest/server.go index 1906351e..c0a86440 100644 --- a/gateway/rest/server.go +++ b/gateway/rest/server.go @@ -26,6 +26,7 @@ func NewServer( address string, pid sdk.Address, clusterConfig map[interface{}]interface{}, + attestCfg AttestationConfig, ) (*http.Server, error) { restMiddleware := func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -52,7 +53,7 @@ func NewServer( srv := &http.Server{ Addr: address, ReadHeaderTimeout: 3 * time.Second, - Handler: newRouter(log, pid, pclient, clusterConfig, restMiddleware), + Handler: newRouter(log, pid, pclient, clusterConfig, attestCfg, restMiddleware), BaseContext: func(_ net.Listener) context.Context { return ctx }, diff --git a/go.mod b/go.mod index 4eb128fe..d84e4e54 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/akash-network/provider -go 1.25.5 +go 1.26.2 require ( cosmossdk.io/errors v1.0.2 @@ -10,7 +10,7 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/boz/go-lifecycle v0.1.1 github.com/cometbft/cometbft v0.38.21 - github.com/cosmos/cosmos-sdk v0.53.5 + github.com/cosmos/cosmos-sdk v0.53.6 github.com/desertbit/timer v1.0.1 github.com/fsnotify/fsnotify v1.9.0 github.com/go-acme/lego/v4 v4.26.0 @@ -31,7 +31,7 @@ require ( github.com/rakyll/statik v0.1.7 github.com/rook/rook v1.14.12 github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7 - github.com/rs/zerolog v1.34.0 + github.com/rs/zerolog v1.35.0 github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 @@ -43,6 +43,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/net v0.49.0 golang.org/x/sync v0.19.0 + golang.org/x/sys v0.41.0 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -51,9 +52,9 @@ require ( k8s.io/client-go v0.34.1 k8s.io/code-generator v0.34.1 k8s.io/kubectl v0.33.3 - pkg.akt.dev/go v0.2.7 + pkg.akt.dev/go v0.3.0-rc0 pkg.akt.dev/go/cli v0.2.2 - pkg.akt.dev/go/sdl v0.2.0 + pkg.akt.dev/go/sdl v0.3.0-rc0 pkg.akt.dev/node v1.2.2 pkg.akt.dev/node/v2 v2.0.0 sigs.k8s.io/gateway-api v1.4.1 @@ -72,7 +73,7 @@ replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 - github.com/bytedance/sonic => github.com/bytedance/sonic v1.14.2 + github.com/bytedance/sonic => github.com/bytedance/sonic v1.15.0 // use akash fork of cometbft github.com/cometbft/cometbft => github.com/akash-network/cometbft v0.38.21-akash.1 @@ -86,7 +87,7 @@ replace ( // Use regen gogoproto fork // To be replaced by cosmos/gogoproto in future versions - github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 + github.com/gogo/protobuf => github.com/cosmos/gogoproto v1.3.3-alpha.regen.1 // as per v0.53.x migration guide goleveldb version must be pinned for the app to work correctly github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -98,6 +99,9 @@ replace ( golang.org/x/exp => golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb // stick with compatible version of rapid in v0.47.x line pgregory.net/rapid => pgregory.net/rapid v0.5.5 + + // local node development (v2 tag with LeaseStartReclaim stub) + pkg.akt.dev/node/v2 => ../node ) // these replaces required for rook to work @@ -110,13 +114,13 @@ replace ( require ( al.essio.dev/pkg/shellescape v1.5.1 // indirect cel.dev/expr v0.24.0 // indirect - cloud.google.com/go v0.120.0 // indirect + cloud.google.com/go v0.121.3 // indirect cloud.google.com/go/auth v0.16.5 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.8.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect - cloud.google.com/go/storage v1.50.0 // indirect + cloud.google.com/go/storage v1.55.0 // indirect cosmossdk.io/api v0.9.2 // indirect cosmossdk.io/collections v1.3.1 // indirect cosmossdk.io/core v0.11.3 // indirect @@ -137,8 +141,8 @@ require ( github.com/DataDog/datadog-go v4.8.3+incompatible // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect @@ -179,7 +183,7 @@ require ( github.com/cosmos/iavl v1.2.6 // indirect github.com/cosmos/ibc-go/v10 v10.5.0 // indirect github.com/cosmos/ics23/go v0.11.0 // indirect - github.com/cosmos/ledger-cosmos-go v0.16.0 // indirect + github.com/cosmos/ledger-cosmos-go v1.0.0 // indirect github.com/cosmos/rosetta v0.50.12 // indirect github.com/cosmos/rosetta-sdk-go v0.10.0 // indirect github.com/danieljoos/wincred v1.2.2 // indirect @@ -269,7 +273,7 @@ require ( github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect github.com/knadh/koanf/providers/env v1.0.0 // indirect @@ -336,7 +340,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ulikunitz/xz v0.5.14 // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -344,26 +348,25 @@ require ( github.com/zondax/golem v0.27.0 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v1.0.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/sdk v1.37.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.step.sm/crypto v0.45.1 // indirect go.uber.org/mock v0.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.17.0 // indirect + golang.org/x/arch v0.24.0 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.40.0 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index 4c4b14aa..5b77bdec 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYN cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= -cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= -cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= +cloud.google.com/go v0.121.3 h1:84RD+hQXNdY5Sw/MWVAx5O9Aui/rd5VQ9HEcdN19afo= +cloud.google.com/go v0.121.3/go.mod h1:6vWF3nJWRrEUv26mMB3FEIU/o1MQNVPG1iHdisa2SJc= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -1075,8 +1075,8 @@ cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5og cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= -cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= -cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0= +cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -1292,12 +1292,12 @@ github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0/go.mod h1:ZV4VOm0/eHR06JLrXWe09068dHpr3TRpY9Uo7T+anuA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0 h1:nNMpRpnkWDAaqcpxMJvxa/Ud98gjbYwayJY4/9bdjiU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 h1:ig/FpDD2JofP/NExKQUbn7uOSZzJAQqogfqluZK4ed4= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1403,8 +1403,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= -github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -1495,7 +1495,6 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -1511,6 +1510,8 @@ github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= github.com/cosmos/gogogateway v1.2.0 h1:Ae/OivNhp8DqBi/sh2A8a1D0y638GpL3tkmLQAiKxTE= github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ4GUkT+tbFI= +github.com/cosmos/gogoproto v1.3.3-alpha.regen.1 h1:Qmv/wAw4xHnjN5iZ9qHergfk1O7nnYl7ZsIY5lF+E9k= +github.com/cosmos/gogoproto v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/cosmos/iavl v1.2.6 h1:Hs3LndJbkIB+rEvToKJFXZvKo6Vy0Ex1SJ54hhtioIs= github.com/cosmos/iavl v1.2.6/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ibc-go/v10 v10.5.0 h1:NI+cX04fXdu9JfP0V0GYeRi1ENa7PPdq0BYtVYo8Zrs= @@ -1782,7 +1783,6 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1-0.20201022092350-68b0159b7869/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= @@ -2155,8 +2155,8 @@ github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= @@ -2231,7 +2231,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -2242,7 +2241,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -2485,8 +2483,6 @@ github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Ung github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= -github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -2508,9 +2504,8 @@ github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7/go.mod h1:ZnB1J github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= -github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/rs/zerolog v1.35.0 h1:VD0ykx7HMiMJytqINBsKcbLS+BJ4WYjz+05us+LRTdI= +github.com/rs/zerolog v1.35.0/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -2638,8 +2633,9 @@ github.com/vektra/mockery/v3 v3.5.4 h1:AqbLKhw+H3U5OBqEAcUilxRIcLwHfFKzTbLlyfEqx github.com/vektra/mockery/v3 v3.5.4/go.mod h1:6rmlzyACJQig1UFoUYyLMS/O+2aGz6BgKAO9C8t9/v0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= @@ -2681,8 +2677,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= @@ -2693,14 +2689,14 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/X go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= @@ -2709,8 +2705,8 @@ go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFh go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2742,8 +2738,8 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= -golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -3127,8 +3123,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -3883,16 +3879,14 @@ nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= pgregory.net/rapid v0.5.5 h1:jkgx1TjbQPD/feRoK+S/mXw9e1uj6WilpHrXJowi6oA= pgregory.net/rapid v0.5.5/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= -pkg.akt.dev/go v0.2.7 h1:oQFhmjv7xUc9MLxsCmdnl52DL/VOu3r3shz51rcXnLs= -pkg.akt.dev/go v0.2.7/go.mod h1:x9Cku9yibLk4aGLTE/Yy7eEkkda0uOSRWmYwQeFw23o= +pkg.akt.dev/go v0.3.0-rc0 h1:VqumsjPO6ubprBhoGmCSDytFtWex2M9Mt6LNUrwN5rg= +pkg.akt.dev/go v0.3.0-rc0/go.mod h1:6yV8oyP8xFm4ocyuf+9nx8S3T4mJz3e64Cl7ln/BczM= pkg.akt.dev/go/cli v0.2.2 h1:PWDAAeHkVtcZ9qE76yh4IhJ2J/42ekhwSyrGWLPGi/g= pkg.akt.dev/go/cli v0.2.2/go.mod h1:MHm9lU8hb+xQ8BX3b9c9S1pMyZKUob5tVjHXQ4T1uwU= -pkg.akt.dev/go/sdl v0.2.0 h1:hY74GjN4itV92REf8HqGt1rQDtXsruzE/iIzd/FpUB8= -pkg.akt.dev/go/sdl v0.2.0/go.mod h1:urd6091AWDy9YwFLRCsENuQ931qyRcg/RJBN9XCBs/E= +pkg.akt.dev/go/sdl v0.3.0-rc0 h1:0axPTKETv8sbZLc5xIMHMEC1AzrdKUoYiGXv0MpsmoI= +pkg.akt.dev/go/sdl v0.3.0-rc0/go.mod h1:tp+mOH6dX98U/CwQEY7TNcybm4qdPHxm2y9S8uzeOTA= pkg.akt.dev/node v1.2.2 h1:Xka/9sVaJTyGfyudZUc8A9UW8xC0FYcZoHvD6dm1iy0= pkg.akt.dev/node v1.2.2/go.mod h1:luWpw5dNEIMhEITMqOvjiohqIoLd/kkcjeg1aCF4EJA= -pkg.akt.dev/node/v2 v2.0.0 h1:U1K9Kce+s4qAD9VlRy22l6Mw8tUTa2rYp8SJ/78zjwM= -pkg.akt.dev/node/v2 v2.0.0/go.mod h1:srQsJA8F3SB4RkhluxLKA5roOkW2q0YeNYzEpSo7M74= pkg.akt.dev/specs v0.0.1 h1:OP0zil3Fr4kcCuybFqQ8LWgSlSP2Yn7306meWpu6/S4= pkg.akt.dev/specs v0.0.1/go.mod h1:tiFuJAqzn+lkz662lf9qaEdjdrrDr882r3YMDnWkbp4= pkg.akt.dev/testdata v0.0.1 h1:yHfqF0Uxf7Rg7WdwSggnyBWMxACtAg5VpBUVFXU+uvM= diff --git a/mocks/cluster/Client_mock.go b/mocks/cluster/Client_mock.go index f308eeea..fdec0dab 100644 --- a/mocks/cluster/Client_mock.go +++ b/mocks/cluster/Client_mock.go @@ -1703,3 +1703,18 @@ func (_c *Client_TeardownLease_Call) RunAndReturn(run func(context1 context.Cont _c.Call.Return(run) return _c } + +// AttestationQuote provides a mock function for the type Client +func (_m *Client) AttestationQuote(ctx context.Context, lID v1.LeaseID, requestBody []byte) ([]byte, int, error) { + ret := _m.Called(ctx, lID, requestBody) + + var r0 []byte + if val, ok := ret.Get(0).([]byte); ok { + r0 = val + } + + r1 := ret.Get(1).(int) + r2 := ret.Error(2) + + return r0, r1, r2 +} diff --git a/pkg/apis/akash.network/crd.yaml b/pkg/apis/akash.network/crd.yaml index d26dfd39..ea76ab4b 100644 --- a/pkg/apis/akash.network/crd.yaml +++ b/pkg/apis/akash.network/crd.yaml @@ -231,6 +231,8 @@ spec: memory_size: type: string format: uint64 + attestation_disabled: + type: boolean credentials: type: object nullable: true diff --git a/pkg/apis/akash.network/v2beta2/manifest.go b/pkg/apis/akash.network/v2beta2/manifest.go index a8e3fd5d..2e6c5405 100644 --- a/pkg/apis/akash.network/v2beta2/manifest.go +++ b/pkg/apis/akash.network/v2beta2/manifest.go @@ -110,8 +110,9 @@ type SchedulerResources struct { } type SchedulerParams struct { - RuntimeClass string `json:"runtime_class"` - Resources *SchedulerResources `json:"resources,omitempty"` + RuntimeClass string `json:"runtime_class"` + Resources *SchedulerResources `json:"resources,omitempty"` + AttestationDisabled bool `json:"attestation_disabled,omitempty"` } type ClusterSettings struct { @@ -312,6 +313,12 @@ func manifestServiceFromProvider(ams mani.Service, schedulerParams *SchedulerPar return ManifestService{}, err } + // If the tenant opted out of attestation sidecar injection, propagate + // this to the scheduler params so the webhook knows not to inject. + if schedulerParams != nil && ams.Params != nil && ams.Params.Attestation != nil && !ams.Params.Attestation.Enabled { + schedulerParams.AttestationDisabled = true + } + ms := ManifestService{ Name: ams.Name, Image: ams.Image, diff --git a/sidecar/attestation/Dockerfile b/sidecar/attestation/Dockerfile new file mode 100644 index 00000000..3f180a7b --- /dev/null +++ b/sidecar/attestation/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/distroless/static:nonroot +COPY attestation-sidecar /attestation-sidecar +ENTRYPOINT ["/attestation-sidecar"] diff --git a/sidecar/attestation/main.go b/sidecar/attestation/main.go new file mode 100644 index 00000000..e320bd2f --- /dev/null +++ b/sidecar/attestation/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/akash-network/provider/sidecar/attestation/tee" +) + +func main() { + listenAddr := os.Getenv("ATTESTATION_LISTEN_ADDR") + if listenAddr == "" { + listenAddr = ":8790" + } + + provider, err := tee.Detect() + if err != nil { + fmt.Fprintf(os.Stderr, "error: no TEE attestation surface available: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "detected TEE: %s\n", provider.Name()) + + // Generate ephemeral TLS keypair for channel binding. + // The public key hash can be embedded in report_data to prove + // the TLS endpoint runs inside the attested TEE. + binding, err := GenerateEphemeralKeypair() + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to generate TLS keypair: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "generated ephemeral TLS keypair for channel binding\n") + + srv := NewServer(listenAddr, provider, binding) + + // Configure TLS + srv.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{binding.Certificate}, + MinVersion: tls.VersionTLS12, + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + go func() { + fmt.Fprintf(os.Stderr, "attestation sidecar listening on %s (TLS)\n", listenAddr) + if err := srv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { + fmt.Fprintf(os.Stderr, "server error: %v\n", err) + cancel() + } + }() + + <-ctx.Done() + fmt.Fprintln(os.Stderr, "shutting down...") + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = srv.Shutdown(shutdownCtx) +} diff --git a/sidecar/attestation/server.go b/sidecar/attestation/server.go new file mode 100644 index 00000000..24e0e893 --- /dev/null +++ b/sidecar/attestation/server.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/akash-network/provider/sidecar/attestation/tee" +) + +const protocolVersion = "2" + +// QuoteRequest is the request body for POST /quote. +// The tenant generates a nonce and sends it; the sidecar writes it into +// the TEE report_data field. The provider never substitutes the nonce. +type QuoteRequest struct { + Nonce string `json:"nonce"` // base64-encoded, must decode to exactly 64 bytes + + // BindTLS, when true, computes report_data as SHA-512(tls_pubkey || nonce)[:64] + // instead of using the raw nonce. This binds the attestation evidence to + // the TLS channel, proving the endpoint is inside the attested TEE. + BindTLS bool `json:"bind_tls,omitempty"` +} + +// QuoteResponse is the response body for POST /quote. +// Contains raw, hardware-signed attestation evidence. +// The provider never produces or signs this — it comes from the PSP. +type QuoteResponse struct { + Report string `json:"report"` // base64 raw SNP attestation report + CertChain string `json:"cert_chain"` // base64 VCEK cert chain (may be empty) + TEEType string `json:"tee_type"` // "snp" + AuxBlob string `json:"auxblob"` // empty on NVIDIA-patched kernel + + // TLSBound indicates whether report_data was computed with TLS channel binding. + // When true, the tenant must verify: report_data == SHA-512(tls_pubkey || nonce)[:64] + TLSBound bool `json:"tls_bound"` +} + +// InfoResponse is the response body for GET /info. +type InfoResponse struct { + TEEType string `json:"tee_type"` + ProtocolVersion string `json:"protocol_version"` + + // TLSPublicKey is the DER-encoded public key of the sidecar's ephemeral + // TLS certificate, base64-encoded. The tenant uses this to verify TLS + // channel binding: report_data == SHA-512(tls_pubkey_der || nonce)[:64] + TLSPublicKey string `json:"tls_public_key"` +} + +// NewServer creates the attestation sidecar HTTP server. +func NewServer(addr string, provider tee.Provider, binding *TLSBinding) *http.Server { + mux := http.NewServeMux() + mux.HandleFunc("POST /quote", quoteHandler(provider, binding)) + mux.HandleFunc("GET /info", infoHandler(provider, binding)) + mux.HandleFunc("GET /healthz", healthHandler()) + + return &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } +} + +func quoteHandler(provider tee.Provider, binding *TLSBinding) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req QuoteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("invalid request: %v", err), http.StatusBadRequest) + return + } + + nonceBytes, err := base64.StdEncoding.DecodeString(req.Nonce) + if err != nil { + http.Error(w, "nonce must be valid base64", http.StatusBadRequest) + return + } + + if len(nonceBytes) != 64 { + http.Error(w, fmt.Sprintf("nonce must be exactly 64 bytes, got %d", len(nonceBytes)), http.StatusBadRequest) + return + } + + var nonce [64]byte + copy(nonce[:], nonceBytes) + + // Compute report_data: either raw nonce or TLS-bound hash + var reportData [64]byte + tlsBound := false + if req.BindTLS { + reportData = ComputeBoundReportData(binding.PubKeyDER, nonce) + tlsBound = true + } else { + reportData = nonce + } + + report, err := provider.GetQuote(context.Background(), reportData) + if err != nil { + http.Error(w, fmt.Sprintf("failed to get attestation quote: %v", err), http.StatusServiceUnavailable) + return + } + + resp := QuoteResponse{ + Report: base64.StdEncoding.EncodeToString(report.Report), + CertChain: base64.StdEncoding.EncodeToString(report.CertChain), + TEEType: provider.Name(), + AuxBlob: base64.StdEncoding.EncodeToString(report.AuxBlob), + TLSBound: tlsBound, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +func infoHandler(provider tee.Provider, binding *TLSBinding) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + resp := InfoResponse{ + TEEType: provider.Name(), + ProtocolVersion: protocolVersion, + TLSPublicKey: base64.StdEncoding.EncodeToString(binding.PubKeyDER), + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +func healthHandler() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) //nolint:errcheck + } +} diff --git a/sidecar/attestation/tee/configfs.go b/sidecar/attestation/tee/configfs.go new file mode 100644 index 00000000..6381df7d --- /dev/null +++ b/sidecar/attestation/tee/configfs.go @@ -0,0 +1,64 @@ +package tee + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" +) + +// ConfigfsTSM implements attestation report collection via the configfs-tsm +// interface at /sys/kernel/config/tsm/report/. This is the primary attestation +// surface on NVIDIA-patched Kata guest kernels (kata-qemu-nvidia-gpu-snp). +type ConfigfsTSM struct { + BasePath string // e.g., "/sys/kernel/config/tsm/report" +} + +var _ Provider = (*ConfigfsTSM)(nil) + +func (c *ConfigfsTSM) Name() string { return NameSNP } + +func (c *ConfigfsTSM) Available() bool { + return dirExists(c.BasePath) +} + +// GetQuote creates a TSM report entry, writes the nonce into inblob, +// and reads the hardware-signed report from outblob. +// +// Implementation notes from the configfs-tsm kernel interface: +// - Each report request requires a fresh directory under the base path. +// - inblob MUST be written with a single write (os.WriteFile does this). +// The kernel rejects multi-write sequences. +// - outblob sysfs size attribute lies (reports 0). os.ReadFile reads to EOF, +// which gives the actual report (~1184 bytes for SNP). +// - auxblob is empty on the NVIDIA-patched Kata guest kernel. +// Certificate fetching from AMD KDS happens tenant-side. +func (c *ConfigfsTSM) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + dir := filepath.Join(c.BasePath, fmt.Sprintf("akash-%d", time.Now().UnixNano())) + + if err := os.Mkdir(dir, 0700); err != nil { + return nil, fmt.Errorf("configfs mkdir %s: %w", dir, err) + } + defer os.Remove(dir) //nolint:errcheck + + // Single write to inblob — kernel requires single sized write + if err := os.WriteFile(filepath.Join(dir, "inblob"), reportData[:], 0600); err != nil { + return nil, fmt.Errorf("configfs write inblob: %w", err) + } + + // Read outblob — sysfs size lies, ReadFile reads to EOF + report, err := os.ReadFile(filepath.Join(dir, "outblob")) + if err != nil { + return nil, fmt.Errorf("configfs read outblob: %w", err) + } + + // Read auxblob — expected empty on NVIDIA-patched kernel + auxblob, _ := os.ReadFile(filepath.Join(dir, "auxblob")) + + return &QuoteResult{ + Report: report, + CertChain: nil, // Certificate fetching from AMD KDS is tenant-side + AuxBlob: auxblob, + }, nil +} diff --git a/sidecar/attestation/tee/mock.go b/sidecar/attestation/tee/mock.go new file mode 100644 index 00000000..8cb2d9c2 --- /dev/null +++ b/sidecar/attestation/tee/mock.go @@ -0,0 +1,103 @@ +package tee + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" +) + +// MockProvider returns synthetic attestation reports for local development +// and testing without TEE hardware. Activated when ATTESTATION_MOCK=true. +// +// The mock report is deterministic given the same report_data: it embeds +// the nonce in a recognizable structure so tests can verify the nonce echo +// (invariant #5) without real hardware. +// +// NEVER use in production. The mock report has no cryptographic validity. +type MockProvider struct { + TEE string // "snp" or "tdx" — controls the reported TEE type +} + +var _ Provider = (*MockProvider)(nil) + +func (m *MockProvider) Name() string { + if m.TEE != "" { + return m.TEE + } + return NameSNP +} + +func (m *MockProvider) Available() bool { return true } + +func (m *MockProvider) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + report := buildMockReport(reportData, m.Name()) + return &QuoteResult{ + Report: report, + CertChain: nil, + AuxBlob: nil, + }, nil +} + +// buildMockReport creates a synthetic report that mirrors the structure +// of real attestation reports enough to test the control plane: +// - Bytes 0-3: magic "MOCK" +// - Bytes 4-7: report length (little-endian uint32) +// - Bytes 8-11: TEE type tag ("SNP\0" or "TDX\0") +// - Bytes 12-15: timestamp (unix seconds, little-endian uint32) +// - Bytes 16-79: zero padding +// - Bytes 80-143: report_data echo (the tenant's nonce — invariant #5) +// - Bytes 144-175: SHA-256(report_data) as a fake "measurement" +// - Bytes 176+: zero padding to total size +// +// For SNP the total is 1184 bytes; for TDX it's 1024 bytes. +func buildMockReport(reportData [64]byte, teeType string) []byte { + size := 1184 // SNP report size + if teeType == NameTDX { + size = 1024 + } + + report := make([]byte, size) + + // Magic + copy(report[0:4], []byte("MOCK")) + + // Length + binary.LittleEndian.PutUint32(report[4:8], uint32(size)) //nolint:gosec // size is 1024 or 1184 + + // TEE tag + switch teeType { + case NameTDX: + copy(report[8:12], []byte("TDX\x00")) + default: + copy(report[8:12], []byte("SNP\x00")) + } + + // Timestamp + binary.LittleEndian.PutUint32(report[12:16], uint32(time.Now().Unix())) //nolint:gosec // fits until 2106 + + // report_data echo at offset 80 (matches SNP REPORT_DATA offset 0x50) + copy(report[80:144], reportData[:]) + + // Fake measurement at offset 144 + h := sha256.Sum256(reportData[:]) + copy(report[144:176], h[:]) + + return report +} + +// IsMockReport returns true if the report was generated by MockProvider. +func IsMockReport(report []byte) bool { + return len(report) >= 4 && string(report[0:4]) == "MOCK" +} + +// ExtractMockReportData extracts the report_data (nonce) from a mock report. +func ExtractMockReportData(report []byte) ([64]byte, error) { + var rd [64]byte + if len(report) < 144 { + return rd, fmt.Errorf("report too short: %d bytes", len(report)) + } + copy(rd[:], report[80:144]) + return rd, nil +} diff --git a/sidecar/attestation/tee/sevguest.go b/sidecar/attestation/tee/sevguest.go new file mode 100644 index 00000000..1f65d5de --- /dev/null +++ b/sidecar/attestation/tee/sevguest.go @@ -0,0 +1,111 @@ +package tee + +import ( + "context" + "encoding/binary" + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +// SEVGuest implements attestation report collection via /dev/sev-guest. +// This is the CPU-only AMD SEV-SNP attestation path (kata-qemu-snp, no GPU). +// +// AEP-83 Section 6 describes this path. For GPU-enabled VMs, the NVIDIA-patched +// guest kernel exposes configfs-tsm instead — see configfs.go. +type SEVGuest struct { + DevicePath string // e.g., "/dev/sev-guest" +} + +var _ Provider = (*SEVGuest)(nil) + +func (s *SEVGuest) Name() string { return NameSNP } + +func (s *SEVGuest) Available() bool { + _, err := os.Stat(s.DevicePath) + return err == nil +} + +// SNP_GET_REPORT ioctl constants for AMD SEV-SNP. +// See: linux/include/uapi/linux/sev-guest.h +const ( + // ioctl request ID for SNP_GET_REPORT on x86_64 + snpGetReportIoctl = 0xc0105300 + + snpReportReqSize = 96 // sizeof(struct snp_report_req) + snpReportRespSize = 4096 + 32 // sizeof(struct snp_report_resp) +) + +// snpReportReq matches struct snp_report_req from the kernel. +// Layout: user_data [64]byte, vmpl uint32, padding [28]byte +type snpReportReq struct { + UserData [64]byte + VMPL uint32 + _ [28]byte // padding to 96 bytes +} + +// GetQuote collects an SNP attestation report via /dev/sev-guest ioctl. +func (s *SEVGuest) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + fd, err := unix.Open(s.DevicePath, unix.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", s.DevicePath, err) + } + defer unix.Close(fd) //nolint:errcheck + + // Prepare request + req := snpReportReq{ + UserData: reportData, + VMPL: 0, + } + + // Allocate response buffer + resp := make([]byte, snpReportRespSize) + + // Build the ioctl message buffer: snp_guest_request_ioctl struct + // Layout: msg_version uint32, req_data *byte, resp_data *byte, fw_err uint64 + msgBuf := make([]byte, 32) + binary.LittleEndian.PutUint32(msgBuf[0:4], 1) // msg_version = 1 + + reqBytes := (*[snpReportReqSize]byte)(unsafe.Pointer(&req))[:] + + // Perform ioctl with the combined structure + // The actual ioctl interface uses a struct snp_guest_request_ioctl + // containing pointers to req and resp. For simplicity, we use + // the raw ioctl approach with the request/response in a contiguous buffer. + ioctlBuf := make([]byte, 0, snpReportReqSize+snpReportRespSize) + ioctlBuf = append(ioctlBuf, reqBytes...) + ioctlBuf = append(ioctlBuf, resp...) + + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, //nolint:staticcheck // Linux-only, runs inside Kata VM + uintptr(fd), + uintptr(snpGetReportIoctl), + uintptr(unsafe.Pointer(&ioctlBuf[0])), + ) + if errno != 0 { + return nil, fmt.Errorf("SNP_GET_REPORT ioctl failed: %w", errno) + } + + // Extract report from response portion + report := ioctlBuf[snpReportReqSize:] + + // Status is first 4 bytes of response + status := binary.LittleEndian.Uint32(report[0:4]) + if status != 0 { + return nil, fmt.Errorf("SNP_GET_REPORT firmware error: status=%d", status) + } + + // Report size is next 4 bytes + reportSize := binary.LittleEndian.Uint32(report[4:8]) + + // Actual report starts after the 32-byte response header + snpReport := report[32 : 32+reportSize] + + return &QuoteResult{ + Report: snpReport, + CertChain: nil, // Certificate fetching from AMD KDS is tenant-side + AuxBlob: nil, + }, nil +} diff --git a/sidecar/attestation/tee/tdx.go b/sidecar/attestation/tee/tdx.go new file mode 100644 index 00000000..07984034 --- /dev/null +++ b/sidecar/attestation/tee/tdx.go @@ -0,0 +1,101 @@ +package tee + +import ( + "context" + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +// TDX implements attestation report collection via /dev/tdx_guest (or the +// legacy /dev/tdx-attest). This is the Intel TDX attestation path for +// kata-qemu-tdx and kata-qemu-nvidia-gpu-tdx runtime classes. +// +// The TDX guest device exposes the TDG.MR.REPORT instruction via ioctl, +// producing a TDREPORT that can be sent to Intel Trust Authority or +// verified independently against Intel's attestation infrastructure. +type TDX struct { + DevicePath string // "/dev/tdx_guest" or "/dev/tdx-attest" +} + +var _ Provider = (*TDX)(nil) + +func (t *TDX) Name() string { return NameTDX } + +func (t *TDX) Available() bool { + _, err := os.Stat(t.DevicePath) + return err == nil +} + +// TDX ioctl constants. +// See: linux/include/uapi/linux/tdx-guest.h +const ( + // TDX_CMD_GET_REPORT0 ioctl for /dev/tdx_guest + // Request ID: _IOWR('T', 1, struct tdx_report_req) + tdxGetReport0Ioctl = 0xc4405401 + + // TDREPORT is 1024 bytes + tdxReportSize = 1024 +) + +// tdxReportReq matches struct tdx_report_req from the kernel. +// Layout: reportdata [64]byte, tdreport [1024]byte +type tdxReportReq struct { + ReportData [64]byte + TDReport [tdxReportSize]byte +} + +// GetQuote collects a TDX attestation report via /dev/tdx_guest ioctl. +// +// Note: This produces a TDREPORT (local attestation). For remote attestation, +// the TDREPORT is sent to a QE (Quoting Enclave) or QGS (Quote Generation +// Service) to produce a TD Quote. The sidecar returns the raw TDREPORT; +// the tenant is responsible for obtaining the full quote via the QGS. +func (t *TDX) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + fd, err := unix.Open(t.DevicePath, unix.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", t.DevicePath, err) + } + defer unix.Close(fd) //nolint:errcheck + + req := tdxReportReq{ + ReportData: reportData, + } + + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, //nolint:staticcheck // Linux-only, runs inside Kata VM + uintptr(fd), + uintptr(tdxGetReport0Ioctl), + uintptr(unsafe.Pointer(&req)), + ) + if errno != 0 { + return nil, fmt.Errorf("TDX_CMD_GET_REPORT0 ioctl failed: %w", errno) + } + + // The TDREPORT is written back into req.TDReport by the kernel. + // Extract the actual report size from the structure. + // The first 4 bytes of the response area may contain metadata; + // for now return the full 1024-byte TDREPORT. + report := make([]byte, tdxReportSize) + copy(report, req.TDReport[:]) + + // Sanity check: verify the report isn't all zeros + allZero := true + for _, b := range report[:64] { + if b != 0 { + allZero = false + break + } + } + if allZero { + return nil, fmt.Errorf("TDX report appears empty (all zeros in first 64 bytes)") + } + + return &QuoteResult{ + Report: report, + CertChain: nil, // TDX cert chain retrieved via Intel Trust Authority, tenant-side + AuxBlob: nil, + }, nil +} diff --git a/sidecar/attestation/tee/tee.go b/sidecar/attestation/tee/tee.go new file mode 100644 index 00000000..c004d5bf --- /dev/null +++ b/sidecar/attestation/tee/tee.go @@ -0,0 +1,90 @@ +package tee + +import ( + "context" + "fmt" + "os" +) + +// TEE type name constants. +const ( + NameSNP = "snp" + NameTDX = "tdx" +) + +// QuoteResult holds the raw hardware-signed attestation evidence. +type QuoteResult struct { + Report []byte // Raw attestation report (SNP ~1184 bytes, TDX 1024 bytes) + CertChain []byte // Cert chain (may be empty — fetching is tenant-side) + AuxBlob []byte // Empty on NVIDIA-patched kernel +} + +// Provider abstracts TEE-specific attestation report collection. +type Provider interface { + // Name returns the TEE type identifier ("snp" or "tdx"). + Name() string + + // Available returns true if this TEE surface is accessible. + Available() bool + + // GetQuote collects a hardware-signed attestation report with the + // given report_data (typically the tenant's nonce, or a hash binding + // the nonce to additional data like a TLS public key). + GetQuote(ctx context.Context, reportData [64]byte) (*QuoteResult, error) +} + +// Detect probes the guest environment and returns the first available TEE provider. +// +// If ATTESTATION_MOCK=true is set, returns a MockProvider that produces synthetic +// reports for local development without TEE hardware. The optional ATTESTATION_MOCK_TEE +// env var controls the mock TEE type ("snp" or "tdx", defaults to "snp"). +// +// Hardware check order: +// 1. /sys/kernel/config/tsm/report/ (configfs-tsm — NVIDIA-patched Kata kernel, GPU path) +// 2. /dev/sev-guest (CPU-only AMD SEV-SNP) +// 3. /dev/tdx_guest (Intel TDX — current kernel interface) +// 4. /dev/tdx-attest (Intel TDX — legacy kernel interface) +// +// configfs-tsm is checked first because on GPU-enabled VMs with the NVIDIA-patched +// kernel, it supersedes /dev/sev-guest. The SNP and TDX device paths are mutually +// exclusive (a VM is either SEV-SNP or TDX, never both). +func Detect() (Provider, error) { + if os.Getenv("ATTESTATION_MOCK") == "true" { + tee := os.Getenv("ATTESTATION_MOCK_TEE") + if tee == "" { + tee = NameSNP + } + return &MockProvider{TEE: tee}, nil + } + + configfs := &ConfigfsTSM{BasePath: "/sys/kernel/config/tsm/report"} + if configfs.Available() { + return configfs, nil + } + + sevGuest := &SEVGuest{DevicePath: "/dev/sev-guest"} + if sevGuest.Available() { + return sevGuest, nil + } + + tdxGuest := &TDX{DevicePath: "/dev/tdx_guest"} + if tdxGuest.Available() { + return tdxGuest, nil + } + + tdxLegacy := &TDX{DevicePath: "/dev/tdx-attest"} + if tdxLegacy.Available() { + return tdxLegacy, nil + } + + return nil, fmt.Errorf("no TEE attestation surface found: "+ + "tried configfs-tsm (%s), /dev/sev-guest, /dev/tdx_guest, /dev/tdx-attest "+ + "(set ATTESTATION_MOCK=true for local development)", + configfs.BasePath) +} + +// dirExists returns true if the path exists and is a directory. +func dirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/sidecar/attestation/tlsbinding.go b/sidecar/attestation/tlsbinding.go new file mode 100644 index 00000000..bf5e88df --- /dev/null +++ b/sidecar/attestation/tlsbinding.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha512" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" +) + +// TLSBinding holds an ephemeral TLS keypair generated at sidecar startup. +// The public key hash can be embedded in the TEE report_data field to bind +// the TLS channel to the attestation evidence: +// +// report_data = SHA-512(tls_pubkey_der || nonce)[:64] +// +// This proves to the tenant that the TLS endpoint they're talking to runs +// inside the attested TEE — not a MITM proxy on the host plane. +type TLSBinding struct { + Certificate tls.Certificate + PubKeyDER []byte // DER-encoded public key for channel binding +} + +// GenerateEphemeralKeypair creates an ECDSA P-384 TLS keypair with a +// self-signed certificate valid for 24 hours. The keypair is generated +// fresh on each sidecar startup — it's ephemeral by design. +func GenerateEphemeralKeypair() (*TLSBinding, error) { + key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate ECDSA key: %w", err) + } + + pubKeyDER, err := x509.MarshalPKIXPublicKey(&key.PublicKey) + if err != nil { + return nil, fmt.Errorf("marshal public key: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "akash-attestation-sidecar"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, fmt.Errorf("create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshal private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("load TLS keypair: %w", err) + } + + return &TLSBinding{ + Certificate: tlsCert, + PubKeyDER: pubKeyDER, + }, nil +} + +// ComputeBoundReportData computes the report_data that binds a TLS channel +// to a tenant-provided nonce: +// +// report_data = SHA-512(tls_pubkey_der || nonce)[:64] +// +// The tenant verifies this by: +// 1. Getting the sidecar's TLS public key from the /info endpoint +// 2. Requesting a quote with bind_tls=true +// 3. Verifying that the SNP report's REPORT_DATA matches this hash +// 4. Knowing the TLS session they're on uses the same key +func ComputeBoundReportData(pubKeyDER []byte, nonce [64]byte) [64]byte { + h := sha512.New() + h.Write(pubKeyDER) + h.Write(nonce[:]) + sum := h.Sum(nil) // 64 bytes from SHA-512 + var result [64]byte + copy(result[:], sum[:64]) + return result +} From ca267f79b2a418ffe9708ca172b5c47de34a6826 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Thu, 28 May 2026 17:19:10 +0100 Subject: [PATCH 2/7] fix: akash.network label too broad, use namespace for lease-only namespaces --- cluster/kube/webhook/registration.go | 13 ++++++++++--- cmd/provider-services/cmd/run.go | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cluster/kube/webhook/registration.go b/cluster/kube/webhook/registration.go index d01d5861..65570fe9 100644 --- a/cluster/kube/webhook/registration.go +++ b/cluster/kube/webhook/registration.go @@ -72,11 +72,18 @@ func RegisterWebhookConfiguration(ctx context.Context, kc kubernetes.Interface, MatchPolicy: &matchPolicy, TimeoutSeconds: &timeoutSec, ClientConfig: clientConfig, - // Only intercept pod creation in Akash-managed namespaces. - // System pods and non-Akash workloads are never affected. + // Only intercept pod creation in tenant lease namespaces. + // Lease namespaces have both akash.network=true AND akash.network/namespace labels. + // This excludes akash-services (provider/operator pods) which only has akash.network=true. NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - builder.AkashManagedLabelName: "true", + builder.AkashManagedLabelName: builder.ValTrue, + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "akash.network/namespace", + Operator: metav1.LabelSelectorOpExists, + }, }, }, Rules: []admissionregistrationv1.RuleWithOperations{ diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 91eeed60..ede88b52 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -896,7 +896,7 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { Bytes: tlsCert.Certificate[0], }) if webhookKC, kcErr := fromctx.KubeClientFromCtx(ctx); kcErr == nil { - ns := viper.GetString(providerflags.FlagK8sManifestNS) + webhookServiceNS := "akash-services" // When running locally (mock mode), use a URL endpoint so the Kind // cluster can reach the webhook on the host machine. @@ -907,7 +907,7 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { regErr := attestwebhook.RegisterWebhookConfiguration( ctx, webhookKC, webhookLog, - "akash-provider", ns, caBundle, int32(webhookPort), webhookURL, //nolint:gosec // port is bounded by flag default + "akash-provider", webhookServiceNS, caBundle, int32(webhookPort), webhookURL, //nolint:gosec // port is bounded by flag default ) if regErr != nil { return fmt.Errorf("register attestation webhook: %w", regErr) From c11e4a35164f6a5717346f4d9078482a03a216bc Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Fri, 29 May 2026 16:29:34 +0100 Subject: [PATCH 3/7] feat: add sidecar for confidential compute to inventory calculation and propagate data to pricing --- bidengine/pricing.go | 7 +-- bidengine/shellscript.go | 7 +++ cluster/kube/builder/builder.go | 10 +++++ .../operators/clients/inventory/inventory.go | 43 +++++++++++++++---- cluster/kube/webhook/sidecar.go | 8 ++-- 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/bidengine/pricing.go b/bidengine/pricing.go index ac14da17..98227dd7 100644 --- a/bidengine/pricing.go +++ b/bidengine/pricing.go @@ -356,7 +356,8 @@ type dataForScriptElement struct { } type dataForScript struct { - Resources []dataForScriptElement `json:"resources"` - Price sdk.DecCoin `json:"price"` - PricePrecision *int `json:"price_precision,omitempty"` + Resources []dataForScriptElement `json:"resources"` + Price sdk.DecCoin `json:"price"` + PricePrecision *int `json:"price_precision,omitempty"` + ConfidentialComputeSidecar bool `json:"confidential_compute_sidecar"` } diff --git a/bidengine/shellscript.go b/bidengine/shellscript.go index d6a8c664..27bce8a3 100644 --- a/bidengine/shellscript.go +++ b/bidengine/shellscript.go @@ -200,6 +200,13 @@ func newDataForScript(r Request) dataForScript { d.PricePrecision = &r.PricePrecision } + for _, attr := range r.GSpec.Requirements.Attributes { + if attr.Key == "confidential-compute" && attr.Value == "true" { + d.ConfidentialComputeSidecar = true + break + } + } + resources := r.GSpec.Resources if len(r.AllocatedResources) > 0 { resources = r.AllocatedResources diff --git a/cluster/kube/builder/builder.go b/cluster/kube/builder/builder.go index 082746af..fdff03b3 100644 --- a/cluster/kube/builder/builder.go +++ b/cluster/kube/builder/builder.go @@ -64,6 +64,16 @@ const ( RuntimeClassKataQemuNvidiaGPUTDX = "kata-qemu-nvidia-gpu-tdx" ) +// Attestation sidecar resource limits. Inventory reserves limit values (not +// requests), matching the pattern used for tenant workloads. These must stay +// in sync with the limits set in webhook/sidecar.go:buildSidecarContainer. +const ( + SidecarCPULimitMillicores int64 = 100 + SidecarMemoryLimitBytes int64 = 64 * 1024 * 1024 // 64Mi + SidecarCPURequestMillicores int64 = 10 + SidecarMemoryRequestBytes int64 = 32 * 1024 * 1024 // 32Mi +) + // TEE type constants used in placement attributes and directory responses. const ( TEETypeAMDSEVSNP = "amd-sev-snp" diff --git a/cluster/kube/operators/clients/inventory/inventory.go b/cluster/kube/operators/clients/inventory/inventory.go index 6578315e..4635b7a3 100644 --- a/cluster/kube/operators/clients/inventory/inventory.go +++ b/cluster/kube/operators/clients/inventory/inventory.go @@ -55,14 +55,8 @@ func (inv *inventory) tryAdjust(node int, res *rtypes.Resources, confidentialCom return nil, false, true } - // For CC CPU-only workloads (no GPU), set the runtime class based on TEE type - if confidentialCompute && sparams.RuntimeClass == "" { - switch teeType { - case builder.TEETypeIntelTDX: - sparams.RuntimeClass = builder.RuntimeClassKataQemuTDX - default: - sparams.RuntimeClass = builder.RuntimeClassKataQemuSNP - } + if !tryAdjustConfidentialCompute(&nd, sparams, confidentialCompute, teeType) { + return nil, false, true } if !nd.Resources.Memory.Quantity.SubNLZ(res.Memory.Quantity) { @@ -221,6 +215,39 @@ func tryAdjustGPU(rp *inventoryV1.GPU, res *rtypes.GPU, sparams *crd.SchedulerPa return false } +// tryAdjustConfidentialCompute adjusts cluster inventory for confidential +// compute workloads. For CPU-only CC workloads (no GPU present), it sets the +// appropriate runtime class based on the TEE type. It also reserves the +// attestation sidecar resources that the webhook will inject into CC pods, +// using limit values so inventory subtracts the full resource limit. +func tryAdjustConfidentialCompute(nd *inventoryV1.Node, sparams *crd.SchedulerParams, confidentialCompute bool, teeType string) bool { + if !confidentialCompute { + return true + } + + // For CC CPU-only workloads (no GPU), set the runtime class based on TEE type + if sparams.RuntimeClass == "" { + switch teeType { + case builder.TEETypeIntelTDX: + sparams.RuntimeClass = builder.RuntimeClassKataQemuTDX + default: + sparams.RuntimeClass = builder.RuntimeClassKataQemuSNP + } + } + + sidecarCPU := rtypes.NewResourceValue(uint64(builder.SidecarCPULimitMillicores)) + if !nd.Resources.CPU.Quantity.SubMilliNLZ(sidecarCPU) { + return false + } + + sidecarMem := rtypes.NewResourceValue(uint64(builder.SidecarMemoryLimitBytes)) + if !nd.Resources.Memory.Quantity.SubNLZ(sidecarMem) { + return false + } + + return true +} + func tryAdjustEphemeralStorage(rp *inventoryV1.ResourcePair, res *rtypes.Storage) bool { return rp.SubNLZ(res.Quantity) } diff --git a/cluster/kube/webhook/sidecar.go b/cluster/kube/webhook/sidecar.go index cea0f3ea..1ba102dc 100644 --- a/cluster/kube/webhook/sidecar.go +++ b/cluster/kube/webhook/sidecar.go @@ -134,12 +134,12 @@ func buildSidecarContainer(image string, isGPU bool, extraEnv []corev1.EnvVar) c }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("32Mi"), + corev1.ResourceCPU: *resource.NewMilliQuantity(builder.SidecarCPURequestMillicores, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(builder.SidecarMemoryRequestBytes, resource.DecimalSI), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), + corev1.ResourceCPU: *resource.NewMilliQuantity(builder.SidecarCPULimitMillicores, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(builder.SidecarMemoryLimitBytes, resource.DecimalSI), }, }, Env: append([]corev1.EnvVar{ From cf322cb2fc707ca75e4b1414b4ed3b84517229b1 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Tue, 2 Jun 2026 13:15:59 +0100 Subject: [PATCH 4/7] feat: update to new chain-sdk version for tee paramters --- _run/kube/Makefile | 27 ++++--- _run/kube/deployment-cc.yaml | 5 +- _run/kube/kind-config-cc.yaml | 32 +++++++++ _run/kube/provider-cc.yaml | 6 +- cluster/inventory.go | 70 ++----------------- cluster/kube/builder/workload.go | 17 +++++ .../operators/clients/inventory/inventory.go | 24 ++----- cluster/types/v1beta3/types.go | 17 +---- pkg/apis/akash.network/v2beta2/manifest.go | 36 ++++++++-- 9 files changed, 118 insertions(+), 116 deletions(-) create mode 100644 _run/kube/kind-config-cc.yaml diff --git a/_run/kube/Makefile b/_run/kube/Makefile index a06a5fff..bcac0193 100644 --- a/_run/kube/Makefile +++ b/_run/kube/Makefile @@ -24,11 +24,19 @@ kube-setup-ingress-default: kube-setup-ingress-gateway @echo "Gateway API ingress setup complete" endif +# CC mode: use Kind config with containerd runtime patches (no restart needed) +ifeq ($(CONFIDENTIAL_COMPUTE),true) +KIND_CONFIG_FILE := $(CURDIR)/kind-config-cc.yaml +endif + # Confidential compute mode overrides +DEV_REGISTRY ?= ghcr.io/cloud-j-luna +DEV_COMMIT := $(shell git rev-parse --short HEAD) + ifeq ($(CONFIDENTIAL_COMPUTE),true) SDL_PATH := deployment-cc.yaml PROVIDER_CONFIG_PATH := provider-cc.yaml -ATTESTATION_SIDECAR_IMAGE ?= ghcr.io/akash-network/attestation-sidecar:latest +ATTESTATION_SIDECAR_IMAGE ?= $(DEV_REGISTRY)/attestation-sidecar:$(DEV_COMMIT) else SDL_PATH ?= grafana.yaml endif @@ -81,15 +89,14 @@ kube-setup-cc: nvidia.com/cc.ready.state=true \ --overwrite; \ done - @echo "Registering CC runtime handlers in containerd (aliased to runc)..." - @for node in $$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do \ - docker cp cc-containerd-setup.sh $$node:/cc-containerd-setup.sh; \ - docker exec $$node bash /cc-containerd-setup.sh; \ - done - @echo "Loading attestation sidecar image into Kind..." - $(KIND) load docker-image "$(ATTESTATION_SIDECAR_IMAGE)" --name "$(KIND_NAME)" 2>/dev/null || \ - echo "Sidecar image not found locally. Build it first or set ATTESTATION_SIDECAR_IMAGE." - @echo "CC setup complete. RuntimeClasses, node labels, and containerd handlers applied." + @echo "Building and loading attestation sidecar image into Kind..." + @mkdir -p /tmp/akash-sidecar-build + GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -mod=vendor -ldflags="-s -w" -o /tmp/akash-sidecar-build/attestation-sidecar $(AP_ROOT)/sidecar/attestation + cp $(AP_ROOT)/sidecar/attestation/Dockerfile /tmp/akash-sidecar-build/Dockerfile + docker build -q -t "$(ATTESTATION_SIDECAR_IMAGE)" /tmp/akash-sidecar-build/ + $(KIND) load docker-image "$(ATTESTATION_SIDECAR_IMAGE)" --name "$(KIND_NAME)" + @echo "CC setup complete. RuntimeClasses, node labels, and sidecar image loaded." .PHONY: kube-teardown-cc kube-teardown-cc: diff --git a/_run/kube/deployment-cc.yaml b/_run/kube/deployment-cc.yaml index 3911c25c..e2a140a4 100644 --- a/_run/kube/deployment-cc.yaml +++ b/_run/kube/deployment-cc.yaml @@ -11,6 +11,9 @@ services: - hello.localhost to: - global: true + params: + tee: + type: sev-snp profiles: compute: @@ -26,8 +29,6 @@ profiles: westcoast: attributes: region: us-west - confidential-compute: "true" - confidential-compute-tee: amd-sev-snp pricing: web: denom: uact diff --git a/_run/kube/kind-config-cc.yaml b/_run/kube/kind-config-cc.yaml new file mode 100644 index 00000000..761f3a72 --- /dev/null +++ b/_run/kube/kind-config-cc.yaml @@ -0,0 +1,32 @@ +--- +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + protocol: TCP +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-snp] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-snp.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-nvidia-gpu-snp] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-nvidia-gpu-snp.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-tdx] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-tdx.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-nvidia-gpu-tdx] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu-nvidia-gpu-tdx.options] + SystemdCgroup = true diff --git a/_run/kube/provider-cc.yaml b/_run/kube/provider-cc.yaml index 7fc813ae..f7909a75 100644 --- a/_run/kube/provider-cc.yaml +++ b/_run/kube/provider-cc.yaml @@ -3,10 +3,8 @@ jwt-host: https://localhost:8444 attributes: - key: region value: us-west - - key: confidential-compute - value: "true" - - key: confidential-compute-tee - value: amd-sev-snp + - key: tee/type + value: sev-snp - key: capabilities/storage/1/persistent value: true - key: capabilities/storage/1/class diff --git a/cluster/inventory.go b/cluster/inventory.go index a351dfa8..13b5405c 100644 --- a/cluster/inventory.go +++ b/cluster/inventory.go @@ -77,11 +77,9 @@ type invSnapshotResp struct { } type inventoryRequest struct { - order mtypes.OrderID - resources dtypes.ResourceGroup - confidentialCompute bool - teeType string // "amd-sev-snp", "intel-tdx", or "" (defaults to amd-sev-snp) - ch chan<- inventoryResponse + order mtypes.OrderID + resources dtypes.ResourceGroup + ch chan<- inventoryResponse } type inventoryResponse struct { @@ -196,15 +194,11 @@ func (is *inventoryService) reserve(order mtypes.OrderID, resources dtypes.Resou } } - cc, teeType := detectConfidentialCompute(resources) - ch := make(chan inventoryResponse, 1) req := inventoryRequest{ - order: order, - resources: resources, - confidentialCompute: cc, - teeType: teeType, - ch: ch, + order: order, + resources: resources, + ch: ch, } select { @@ -453,15 +447,7 @@ func (is *inventoryService) handleRequest(req inventoryRequest, state *inventory reservation.ipsConfirmed = true // No IPs, just mark it as confirmed implicitly } - var adjustOpts []ctypes.InventoryOption - if req.confidentialCompute { - adjustOpts = append(adjustOpts, ctypes.WithConfidentialCompute()) - if req.teeType != "" { - adjustOpts = append(adjustOpts, ctypes.WithTEEType(req.teeType)) - } - } - - err := state.inventory.Adjust(reservation, adjustOpts...) + err := state.inventory.Adjust(reservation) if err != nil { is.log.Info("insufficient capacity for reservation", "order", req.order) inventoryRequestsCounter.WithLabelValues("reserve", "insufficient-capacity").Inc() @@ -874,45 +860,3 @@ func reservationCountEndpoints(reservation *reservation) uint { return externalPortCount } -// detectConfidentialCompute checks if the resource group has the confidential-compute -// placement attribute set, and extracts the TEE type if specified. -// Returns (isCC, teeType) where teeType defaults to "amd-sev-snp" if not specified. -func detectConfidentialCompute(resources dtypes.ResourceGroup) (bool, string) { - var attrs atypes.Attributes - - switch g := resources.(type) { - case *dtypes.Group: - attrs = g.GroupSpec.Requirements.Attributes - case dtypes.Group: - attrs = g.GroupSpec.Requirements.Attributes - case *dtypes.GroupSpec: - attrs = g.Requirements.Attributes - case dtypes.GroupSpec: - attrs = g.Requirements.Attributes - default: - return false, "" - } - - isCC := false - teeType := "" - - for _, attr := range attrs { - if attr.Key == "confidential-compute" && attr.Value == "true" { - isCC = true - } - if attr.Key == "confidential-compute-tee" { - teeType = attr.Value - } - } - - if !isCC { - return false, "" - } - - // Default to AMD SEV-SNP if no TEE type specified - if teeType == "" { - teeType = "amd-sev-snp" - } - - return true, teeType -} diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index 8f5c9a1e..90b47fd0 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -62,6 +62,23 @@ func TEETypeForRuntimeClass(rc string) string { } } +// RuntimeClassForTEEType maps an SDL TEE type to the corresponding Kata runtime class. +// The TEE type values come from the SDL tee.type enum (sev-snp, sev-snp-gpu, tdx, tdx-gpu). +func RuntimeClassForTEEType(teeType string) string { + switch teeType { + case "sev-snp": + return RuntimeClassKataQemuSNP + case "sev-snp-gpu": + return RuntimeClassKataQemuNvidiaGPUSNP + case "tdx": + return RuntimeClassKataQemuTDX + case "tdx-gpu": + return RuntimeClassKataQemuNvidiaGPUTDX + default: + return "" + } +} + type workloadBase interface { builderBase Name() string diff --git a/cluster/kube/operators/clients/inventory/inventory.go b/cluster/kube/operators/clients/inventory/inventory.go index 4635b7a3..b50c5a2a 100644 --- a/cluster/kube/operators/clients/inventory/inventory.go +++ b/cluster/kube/operators/clients/inventory/inventory.go @@ -43,21 +43,19 @@ func (inv *inventory) Dup() ctypes.Inventory { // tryAdjust cluster inventory // It returns two boolean values. First indicates if node-wide resources satisfy (true) requirements // Seconds indicates if cluster-wide resources satisfy (true) requirements -func (inv *inventory) tryAdjust(node int, res *rtypes.Resources, confidentialCompute bool, teeType string) (*crd.SchedulerParams, bool, bool) { +func (inv *inventory) tryAdjust(node int, res *rtypes.Resources) (*crd.SchedulerParams, bool, bool) { nd := inv.Nodes[node].Dup() sparams := &crd.SchedulerParams{} + if !tryAdjustCPU(&nd.Resources.CPU.Quantity, res.CPU) { return nil, false, true } - if !tryAdjustGPU(&nd.Resources.GPU, res.GPU, sparams, confidentialCompute, teeType) { + if !tryAdjustGPU(&nd.Resources.GPU, res.GPU, sparams) { return nil, false, true } - if !tryAdjustConfidentialCompute(&nd, sparams, confidentialCompute, teeType) { - return nil, false, true - } if !nd.Resources.Memory.Quantity.SubNLZ(res.Memory.Quantity) { return nil, false, true @@ -126,7 +124,7 @@ func tryAdjustCPU(rp *inventoryV1.ResourcePair, res *rtypes.CPU) bool { return rp.SubMilliNLZ(res.Units) } -func tryAdjustGPU(rp *inventoryV1.GPU, res *rtypes.GPU, sparams *crd.SchedulerParams, confidentialCompute bool, teeType string) bool { +func tryAdjustGPU(rp *inventoryV1.GPU, res *rtypes.GPU, sparams *crd.SchedulerParams) bool { reqCnt := res.Units.Value() if reqCnt == 0 { @@ -177,16 +175,7 @@ func tryAdjustGPU(rp *inventoryV1.GPU, res *rtypes.GPU, sparams *crd.SchedulerPa switch vendor { case builder.GPUVendorNvidia: - if confidentialCompute { - switch teeType { - case builder.TEETypeIntelTDX: - sparams.RuntimeClass = builder.RuntimeClassKataQemuNvidiaGPUTDX - default: - sparams.RuntimeClass = builder.RuntimeClassKataQemuNvidiaGPUSNP - } - } else { - sparams.RuntimeClass = runtimeClassNvidia - } + sparams.RuntimeClass = runtimeClassNvidia default: } @@ -300,7 +289,7 @@ nodes: } for ; resources[i].Count > 0; resources[i].Count-- { - sparams, nStatus, cStatus := currInventory.tryAdjust(nodeIdx, adjusted, cfg.ConfidentialCompute, cfg.TEEType) + sparams, nStatus, cStatus := currInventory.tryAdjust(nodeIdx, adjusted) if !cStatus { // cannot satisfy cluster-wide resources, stop lookup break nodes @@ -453,3 +442,4 @@ func sParamsEnsureResources(sparams *crd.SchedulerParams) { sparams.Resources = &crd.SchedulerResources{} } } + diff --git a/cluster/types/v1beta3/types.go b/cluster/types/v1beta3/types.go index 3aba0b7c..40e3e7d0 100644 --- a/cluster/types/v1beta3/types.go +++ b/cluster/types/v1beta3/types.go @@ -52,9 +52,7 @@ type LeaseEvent struct { } type InventoryOptions struct { - DryRun bool - ConfidentialCompute bool - TEEType string // "amd-sev-snp" or "intel-tdx" + DryRun bool } type InventoryOption func(*InventoryOptions) *InventoryOptions @@ -66,19 +64,6 @@ func WithDryRun() InventoryOption { } } -func WithConfidentialCompute() InventoryOption { - return func(opts *InventoryOptions) *InventoryOptions { - opts.ConfidentialCompute = true - return opts - } -} - -func WithTEEType(teeType string) InventoryOption { - return func(opts *InventoryOptions) *InventoryOptions { - opts.TEEType = teeType - return opts - } -} type Inventory interface { Adjust(ReservationGroup, ...InventoryOption) error diff --git a/pkg/apis/akash.network/v2beta2/manifest.go b/pkg/apis/akash.network/v2beta2/manifest.go index 2e6c5405..b8f75a50 100644 --- a/pkg/apis/akash.network/v2beta2/manifest.go +++ b/pkg/apis/akash.network/v2beta2/manifest.go @@ -313,10 +313,21 @@ func manifestServiceFromProvider(ams mani.Service, schedulerParams *SchedulerPar return ManifestService{}, err } - // If the tenant opted out of attestation sidecar injection, propagate - // this to the scheduler params so the webhook knows not to inject. - if schedulerParams != nil && ams.Params != nil && ams.Params.Attestation != nil && !ams.Params.Attestation.Enabled { - schedulerParams.AttestationDisabled = true + // Set runtime class and attestation from TEE service params. + // Note: proto3 bool defaults to false, but the intended default for attestation + // is true (sidecar injected). The Go SDL builder always sets this explicitly. + // Non-Go producers must set attestation=true when sidecar injection is desired. + if ams.Params != nil && ams.Params.TEE != nil { + if schedulerParams == nil { + schedulerParams = &SchedulerParams{} + } + rc := runtimeClassForTEEType(ams.Params.TEE.Type) + if rc != "" { + schedulerParams.RuntimeClass = rc + } + if !ams.Params.TEE.Attestation { + schedulerParams.AttestationDisabled = true + } } ms := ManifestService{ @@ -424,3 +435,20 @@ func manifestServiceExposeFromAkash(amse mani.ServiceExpose) ManifestServiceExpo }, } } + +// runtimeClassForTEEType maps SDL TEE type to Kata runtime class. +// Duplicated from builder package to avoid import cycle. +func runtimeClassForTEEType(teeType string) string { + switch teeType { + case "sev-snp": + return "kata-qemu-snp" + case "sev-snp-gpu": + return "kata-qemu-nvidia-gpu-snp" + case "tdx": + return "kata-qemu-tdx" + case "tdx-gpu": + return "kata-qemu-nvidia-gpu-tdx" + default: + return "" + } +} From 2437c64900dbfbb2bbd8fbf87d5b60cc4be9e237 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Tue, 2 Jun 2026 14:29:38 +0100 Subject: [PATCH 5/7] fix: floor resource requests to a minimum to avoid OOM on small deployments --- _run/kube/deployment-cc.yaml | 4 +- cluster/kube/builder/builder.go | 6 + cluster/kube/builder/deployment_test.go | 158 ++++++++++++++++++++++++ cluster/kube/builder/workload.go | 51 +++++++- 4 files changed, 211 insertions(+), 8 deletions(-) diff --git a/_run/kube/deployment-cc.yaml b/_run/kube/deployment-cc.yaml index e2a140a4..dcc4db56 100644 --- a/_run/kube/deployment-cc.yaml +++ b/_run/kube/deployment-cc.yaml @@ -20,9 +20,9 @@ profiles: web: resources: cpu: - units: 0.1 + units: 0.5 memory: - size: 64Mi + size: 256Mi storage: size: 128Mi placement: diff --git a/cluster/kube/builder/builder.go b/cluster/kube/builder/builder.go index fdff03b3..54be1c49 100644 --- a/cluster/kube/builder/builder.go +++ b/cluster/kube/builder/builder.go @@ -72,6 +72,12 @@ const ( SidecarMemoryLimitBytes int64 = 64 * 1024 * 1024 // 64Mi SidecarCPURequestMillicores int64 = 10 SidecarMemoryRequestBytes int64 = 32 * 1024 * 1024 // 32Mi + + // Minimum resources for the primary container after sidecar subtraction. + // If the user's resources minus sidecar overhead falls below these values, + // clamp to these minimums. Prevents OOM kills from unusable resource limits. + MinPrimaryCPUMillicores int64 = 10 // 10m + MinPrimaryMemoryBytes int64 = 16 * 1024 * 1024 // 16Mi ) // TEE type constants used in placement attributes and directory responses. diff --git a/cluster/kube/builder/deployment_test.go b/cluster/kube/builder/deployment_test.go index e1468897..510988ce 100644 --- a/cluster/kube/builder/deployment_test.go +++ b/cluster/kube/builder/deployment_test.go @@ -135,3 +135,161 @@ func TestDeploymentPermissions(t *testing.T) { }) } } + +func TestSidecarResourceSubtraction(t *testing.T) { + tests := []struct { + name string + runtimeClass string + attestationDisabled bool + cpuMillis uint64 // SDL cpu in millicores + memBytes uint64 // SDL memory in bytes + expectSubtraction bool + }{ + { + name: "CC with attestation — 1Gi memory, 4000m CPU", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 4000, + memBytes: 1024 * 1024 * 1024, // 1Gi + expectSubtraction: true, + }, + { + name: "CC with attestation — 500m CPU, 128Mi memory", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, // 128Mi + expectSubtraction: true, + }, + { + name: "CC with attestation — GPU runtime class", + runtimeClass: RuntimeClassKataQemuNvidiaGPUSNP, + cpuMillis: 2000, + memBytes: 512 * 1024 * 1024, // 512Mi + expectSubtraction: true, + }, + { + name: "CC with attestation disabled — no subtraction", + runtimeClass: RuntimeClassKataQemuSNP, + attestationDisabled: true, + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "non-CC workload — no subtraction", + runtimeClass: "nvidia", + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "no runtime class — no subtraction", + runtimeClass: "", + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "CC with small resources — floors at minimum", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 110, // barely above sidecar limit (100m) + memBytes: 68 * 1024 * 1024, // barely above sidecar limit (64Mi) + expectSubtraction: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lid := testutil.LeaseID(t) + + log := testutil.Logger(t) + settings := Settings{ + CPUCommitLevel: 1, // request = limit (simplifies assertions) + MemoryCommitLevel: 1, + } + + sdlData, err := sdl.ReadFile("../../../testdata/deployment/deployment.yaml") + require.NoError(t, err) + + mani, err := sdlData.Manifest() + require.NoError(t, err) + + sp := &crd.SchedulerParams{ + RuntimeClass: tt.runtimeClass, + AttestationDisabled: tt.attestationDisabled, + } + sparams := []*crd.SchedulerParams{sp} + + cmani, err := crd.NewManifest("lease", lid, &mani.GetGroups()[0], crd.ClusterSettings{SchedulerParams: sparams}) + require.NoError(t, err) + + // Override CPU and memory in the CRD before building the workload + cmani.Spec.Group.Services[0].Resources.CPU.Units = uint32(tt.cpuMillis) //nolint:gosec + cmani.Spec.Group.Services[0].Resources.Memory.Size = fmt.Sprintf("%d", tt.memBytes) + + group, retSparams, err := cmani.Spec.Group.FromCRD() + require.NoError(t, err) + + cdep := &ClusterDeployment{ + Lid: lid, + Group: &group, + Sparams: crd.ClusterSettings{SchedulerParams: retSparams}, + } + // Override scheduler params with our CC config + cdep.Sparams.SchedulerParams[0] = sp + + workload, err := NewWorkloadBuilder(log, settings, cdep, cmani, 0) + require.NoError(t, err) + + container := workload.container() + + cpuLimit := container.Resources.Limits.Cpu().MilliValue() + cpuRequest := container.Resources.Requests.Cpu().MilliValue() + memLimit := container.Resources.Limits.Memory().Value() + memRequest := container.Resources.Requests.Memory().Value() + + if tt.expectSubtraction { + expectedCPULimit := int64(tt.cpuMillis) - SidecarCPULimitMillicores + if expectedCPULimit < MinPrimaryCPUMillicores { + expectedCPULimit = MinPrimaryCPUMillicores + } + expectedMemLimit := int64(tt.memBytes) - SidecarMemoryLimitBytes + if expectedMemLimit < MinPrimaryMemoryBytes { + expectedMemLimit = MinPrimaryMemoryBytes + } + + require.Equal(t, expectedCPULimit, cpuLimit, + "CPU limit: want %dm, got %dm", expectedCPULimit, cpuLimit) + require.Equal(t, expectedMemLimit, memLimit, + "Memory limit: want %d, got %d", expectedMemLimit, memLimit) + + // Pod total LIMIT should equal user's original limit + // (only when resources are above sidecar footprint) + // Pod total equals user limit only when resources are well above sidecar + minimum + if int64(tt.cpuMillis)-SidecarCPULimitMillicores >= MinPrimaryCPUMillicores { + require.Equal(t, int64(tt.cpuMillis), cpuLimit+SidecarCPULimitMillicores, + "Pod CPU limit total should equal user limit") + } + if int64(tt.memBytes)-SidecarMemoryLimitBytes >= MinPrimaryMemoryBytes { + require.Equal(t, int64(tt.memBytes), memLimit+SidecarMemoryLimitBytes, + "Pod memory limit total should equal user limit") + } + + // K8s constraint: limit >= request + require.GreaterOrEqual(t, cpuLimit, cpuRequest, + "CPU limit must be >= request") + require.GreaterOrEqual(t, memLimit, memRequest, + "Memory limit must be >= request") + } else { + // No subtraction — primary container gets full user resources + require.Equal(t, int64(tt.cpuMillis), cpuLimit, + "CPU limit should be unmodified") + require.Equal(t, int64(tt.memBytes), memLimit, + "Memory limit should be unmodified") + } + + // Always: limit >= request (K8s invariant) + require.GreaterOrEqual(t, cpuLimit, cpuRequest, "K8s: CPU limit >= request") + require.GreaterOrEqual(t, memLimit, memRequest, "K8s: memory limit >= request") + }) + } +} diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index 90b47fd0..86896f2c 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -157,10 +157,33 @@ func (b *Workload) container() corev1.Container { }, } + // Whether the attestation sidecar will be injected into this pod. + // When true, we subtract the sidecar's resource footprint from the + // primary container so the pod total matches what the user requested. + sidecarActive := sparams != nil && + IsConfidentialComputeRuntimeClass(sparams.RuntimeClass) && + !sparams.AttestationDisabled + if cpu := service.Resources.CPU; cpu != nil { - requestedCPU := sdlutil.ComputeCommittedResources(b.settings.CPUCommitLevel, cpu.Units) - kcontainer.Resources.Requests[corev1.ResourceCPU] = resource.NewScaledQuantity(int64(requestedCPU.Value()), resource.Milli).DeepCopy() // nolint: gosec - kcontainer.Resources.Limits[corev1.ResourceCPU] = resource.NewScaledQuantity(int64(cpu.Units.Value()), resource.Milli).DeepCopy() // nolint: gosec + cpuLimit := int64(cpu.Units.Value()) // nolint: gosec + cpuRequest := int64(sdlutil.ComputeCommittedResources(b.settings.CPUCommitLevel, cpu.Units).Value()) // nolint: gosec + + if sidecarActive { + cpuLimit -= SidecarCPULimitMillicores + cpuRequest -= SidecarCPURequestMillicores + if cpuLimit < MinPrimaryCPUMillicores { + cpuLimit = MinPrimaryCPUMillicores + } + if cpuRequest < MinPrimaryCPUMillicores { + cpuRequest = MinPrimaryCPUMillicores + } + if cpuRequest > cpuLimit { + cpuRequest = cpuLimit + } + } + + kcontainer.Resources.Requests[corev1.ResourceCPU] = resource.NewScaledQuantity(cpuRequest, resource.Milli).DeepCopy() + kcontainer.Resources.Limits[corev1.ResourceCPU] = resource.NewScaledQuantity(cpuLimit, resource.Milli).DeepCopy() } if gpu := service.Resources.GPU; gpu != nil && gpu.Units.Value() > 0 { @@ -210,9 +233,25 @@ func (b *Workload) container() corev1.Container { // fixme: ram is never expected to be nil if mem := service.Resources.Memory; mem != nil { - requestedRAM := sdlutil.ComputeCommittedResources(b.settings.MemoryCommitLevel, mem.Quantity) - kcontainer.Resources.Requests[corev1.ResourceMemory] = resource.NewQuantity(int64(requestedRAM.Value()), resource.DecimalSI).DeepCopy() // nolint: gosec - kcontainer.Resources.Limits[corev1.ResourceMemory] = resource.NewQuantity(int64(mem.Quantity.Value()+requestedMem), resource.DecimalSI).DeepCopy() // nolint: gosec + memLimit := int64(mem.Quantity.Value()+requestedMem) // nolint: gosec + memRequest := int64(sdlutil.ComputeCommittedResources(b.settings.MemoryCommitLevel, mem.Quantity).Value()) // nolint: gosec + + if sidecarActive { + memLimit -= SidecarMemoryLimitBytes + memRequest -= SidecarMemoryRequestBytes + if memLimit < MinPrimaryMemoryBytes { + memLimit = MinPrimaryMemoryBytes + } + if memRequest < MinPrimaryMemoryBytes { + memRequest = MinPrimaryMemoryBytes + } + if memRequest > memLimit { + memRequest = memLimit + } + } + + kcontainer.Resources.Requests[corev1.ResourceMemory] = resource.NewQuantity(memRequest, resource.DecimalSI).DeepCopy() + kcontainer.Resources.Limits[corev1.ResourceMemory] = resource.NewQuantity(memLimit, resource.DecimalSI).DeepCopy() } if service.Params != nil { From 27417b55fb747c163f687ea3d627c6ec7625e426 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Tue, 2 Jun 2026 18:56:42 +0100 Subject: [PATCH 6/7] fix: include in-cluster service names so k8s can reach the webhook --- cmd/provider-services/cmd/run.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index ede88b52..7f14e7f0 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -856,8 +856,15 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { NotAfter: time.Now().Add(365 * 24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - DNSNames: []string{"localhost", "host.docker.internal"}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{ + "localhost", + "host.docker.internal", + "akash-provider", + "akash-provider.akash-services", + "akash-provider.akash-services.svc", + "akash-provider.akash-services.svc.cluster.local", + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, } certDER, genErr := x509.CreateCertificate(crypto_rand.Reader, tmpl, tmpl, &key.PublicKey, key) if genErr != nil { From 811104c5269117ccd885cea55d5d67be62f3ba15 Mon Sep 17 00:00:00 2001 From: Joao Luna Date: Tue, 2 Jun 2026 20:20:45 +0100 Subject: [PATCH 7/7] fix: kata vms do not need HostPath mounted volumes for attestation --- cluster/kube/webhook/sidecar.go | 73 ++++++++-------------------- cluster/kube/webhook/sidecar_test.go | 8 +-- 2 files changed, 25 insertions(+), 56 deletions(-) diff --git a/cluster/kube/webhook/sidecar.go b/cluster/kube/webhook/sidecar.go index 1ba102dc..25963095 100644 --- a/cluster/kube/webhook/sidecar.go +++ b/cluster/kube/webhook/sidecar.go @@ -95,21 +95,22 @@ func BuildSidecarPatch(pod *corev1.Pod, sidecarImage string, extraEnv []corev1.E }) } - // Add volumes if the pod has no volumes, create the array with all - // volumes at once; otherwise append individually. - if len(pod.Spec.Volumes) == 0 { - patches = append(patches, jsonPatch{ - Op: "add", - Path: "/spec/volumes", - Value: volumes, - }) - } else { - for _, vol := range volumes { + // Add volumes (only if there are any to add). + if len(volumes) > 0 { + if len(pod.Spec.Volumes) == 0 { patches = append(patches, jsonPatch{ Op: "add", - Path: "/spec/volumes/-", - Value: vol, + Path: "/spec/volumes", + Value: volumes, }) + } else { + for _, vol := range volumes { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/volumes/-", + Value: vol, + }) + } } } @@ -161,18 +162,10 @@ func buildSidecarContainer(image string, isGPU bool, extraEnv []corev1.EnvVar) c MountPath: "/var/run/mock-tee/sev-guest", }) } - } else { - c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ - Name: tsmVolumeName, - MountPath: "/sys/kernel/config/tsm/report", - }) - if !isGPU { - c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ - Name: sevGuestVolumeName, - MountPath: "/dev/sev-guest", - }) - } } + // Production (non-mock) kata VMs: the sidecar accesses /dev/sev-guest + // and /sys/kernel/config/tsm/report directly from the guest kernel + // filesystem — no volume mounts needed. return c } @@ -202,33 +195,9 @@ func buildSidecarVolumes(isGPU bool, mockMode bool) []corev1.Volume { return volumes } - // Production: hostPath volumes inside the Kata VM guest filesystem. - hostPathDirOrCreate := corev1.HostPathDirectoryOrCreate - hostPathCharDev := corev1.HostPathCharDev - - volumes := []corev1.Volume{ - { - Name: tsmVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys/kernel/config/tsm/report", - Type: &hostPathDirOrCreate, - }, - }, - }, - } - - if !isGPU { - volumes = append(volumes, corev1.Volume{ - Name: sevGuestVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev/sev-guest", - Type: &hostPathCharDev, - }, - }, - }) - } - - return volumes + // Production: the sidecar accesses /dev/sev-guest and + // /sys/kernel/config/tsm/report directly from the kata guest kernel + // filesystem. HostPath volumes cannot be used because kubelet validates + // paths on the host node, where these devices do not exist. + return nil } diff --git a/cluster/kube/webhook/sidecar_test.go b/cluster/kube/webhook/sidecar_test.go index ddd0c7a3..68adcf69 100644 --- a/cluster/kube/webhook/sidecar_test.go +++ b/cluster/kube/webhook/sidecar_test.go @@ -142,8 +142,8 @@ func TestBuildSidecarPatch_GPU(t *testing.T) { err = json.Unmarshal(patchBytes, &patches) require.NoError(t, err) - // Should have container patch + volumes array patch - require.Len(t, patches, 2) // container + volumes + // Production mode: container patch only, no volumes (guest kernel provides devices) + require.Len(t, patches, 1) // container only } func TestBuildSidecarPatch_CPUOnly(t *testing.T) { @@ -166,8 +166,8 @@ func TestBuildSidecarPatch_CPUOnly(t *testing.T) { err = json.Unmarshal(patchBytes, &patches) require.NoError(t, err) - // Should have container patch + volumes array patch (TSM + sev-guest in one array) - require.Len(t, patches, 2) + // Production mode: container patch only, no volumes (guest kernel provides devices) + require.Len(t, patches, 1) } func TestBuildSidecarPatch_EmptyImage(t *testing.T) {