diff --git a/.goreleaser-docker.yaml b/.goreleaser-docker.yaml index bb978aa3..5bbfd77a 100644 --- a/.goreleaser-docker.yaml +++ b/.goreleaser-docker.yaml @@ -10,6 +10,19 @@ before: - apt update - apt install -y pkg-config libudev-dev:amd64 libudev-dev:arm64 builds: + - id: attestation-sidecar-linux-amd64 + binary: attestation-sidecar + main: ./sidecar/attestation + goarch: + - amd64 + goos: + - linux + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w - id: provider-services-linux-arm64 binary: provider-services main: ./cmd/provider-services @@ -160,3 +173,20 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} image_templates: - "{{ .Env.DOCKER_IMAGE }}:latest-arm64-debug" + - dockerfile: sidecar/attestation/Dockerfile + ids: + - attestation-sidecar-linux-amd64 + use: buildx + goos: linux + goarch: amd64 + build_flag_templates: + - --platform=linux/amd64 + - --label=org.opencontainers.image.title=attestation-sidecar + - --label=org.opencontainers.image.description=Akash attestation sidecar for confidential compute + - --label=org.opencontainers.image.url={{.GitURL}} + - --label=org.opencontainers.image.source={{.GitURL}} + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + image_templates: + - "{{ .Env.DOCKER_IMAGE }}-attestation-sidecar:latest-amd64" diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 1302959c..cef676e4 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -174,6 +174,19 @@ builds: - -extldflags "-L./.cache/lib -lwasmvm_muslc.x86_64 -Wl,-z,muldefs -lm -lrt -lc" gcflags: - "all=-N -l" + - id: attestation-sidecar-linux-amd64 + binary: attestation-sidecar + main: ./sidecar/attestation + goarch: + - amd64 + goos: + - linux + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w universal_binaries: - id: darwin-universal ids: diff --git a/_run/common-commands.mk b/_run/common-commands.mk index 6beb0c4a..5eaf474a 100644 --- a/_run/common-commands.mk +++ b/_run/common-commands.mk @@ -14,7 +14,8 @@ PRICE ?= 10uakt CERT_HOSTNAME ?= localhost LEASE_SERVICES ?= web -RESOURCE_SERVER_HOST ?= localhost:8445 +RESOURCE_SERVER_HOST ?= localhost:8445 +GATEWAY_GRPC_ENDPOINT ?= localhost:8444 GW_AUTH_TYPE ?= jwt @@ -293,6 +294,10 @@ provider-lease-events: --provider "$(PROVIDER_ADDRESS)" \ --auth-type "$(GW_AUTH_TYPE)" +.PHONY: provider-lease-attestation +provider-lease-attestation: + $(PROVIDER_SERVICES) lease-attestation --dseq "$(DSEQ)" --gseq "$(GSEQ)" --oseq "$(OSEQ)" --from "$(KEY_NAME)" --provider "$(PROVIDER_ADDRESS)" --auth-type "$(GW_AUTH_TYPE)" + .PHONY: provider-lease-status provider-lease-status: $(PROVIDER_SERVICES) lease-status \ diff --git a/_run/kube/Makefile b/_run/kube/Makefile index d259bf7b..bcac0193 100644 --- a/_run/kube/Makefile +++ b/_run/kube/Makefile @@ -1,4 +1,5 @@ -GATEWAY_API ?= false +GATEWAY_API ?= false +CONFIDENTIAL_COMPUTE ?= false KUBE_SETUP_PREREQUISITES ?= \ @@ -23,7 +24,22 @@ kube-setup-ingress-default: kube-setup-ingress-gateway @echo "Gateway API ingress setup complete" endif -SDL_PATH ?= grafana.yaml +# CC mode: use Kind config with containerd runtime patches (no restart needed) +ifeq ($(CONFIDENTIAL_COMPUTE),true) +KIND_CONFIG_FILE := $(CURDIR)/kind-config-cc.yaml +endif + +# Confidential compute mode overrides +DEV_REGISTRY ?= ghcr.io/cloud-j-luna +DEV_COMMIT := $(shell git rev-parse --short HEAD) + +ifeq ($(CONFIDENTIAL_COMPUTE),true) +SDL_PATH := deployment-cc.yaml +PROVIDER_CONFIG_PATH := provider-cc.yaml +ATTESTATION_SIDECAR_IMAGE ?= $(DEV_REGISTRY)/attestation-sidecar:$(DEV_COMMIT) +else +SDL_PATH ?= grafana.yaml +endif GATEWAY_HOSTNAME ?= localhost GATEWAY_HOST ?= $(GATEWAY_HOSTNAME):8443 @@ -42,7 +58,12 @@ provider-run: --bid-price-strategy "randomRange" \ --deployment-runtime-class "none" \ --ip-operator=true \ - $(if $(filter true,$(GATEWAY_API)),--ingress-mode=gateway-api) + $(if $(filter true,$(GATEWAY_API)),--ingress-mode=gateway-api) \ + $(if $(filter true,$(CONFIDENTIAL_COMPUTE)),\ + --attestation-webhook-enabled=true \ + --attestation-sidecar-image="$(ATTESTATION_SIDECAR_IMAGE)" \ + --attestation-mock=true \ + ) .PHONY: provider-lease-ping provider-lease-ping: @@ -52,6 +73,44 @@ provider-lease-ping: hostname-operator: $(PROVIDER_SERVICES) hostname-operator +# Confidential compute cluster setup: creates RuntimeClass objects and labels +# the Kind node with TEE capability labels so the provider can schedule CC +# workloads. The pods won't actually run with Kata (Kind doesn't have it), +# but the webhook, annotations, and sidecar injection can be verified. +.PHONY: kube-setup-cc +kube-setup-cc: + kubectl apply -f cc-setup.yaml + @echo "Labeling kind node for CC..." + @for node in $$(kubectl get nodes -o name); do \ + kubectl label $$node \ + katacontainers.io/kata-runtime=true \ + amd.feature.node.kubernetes.io/snp=true \ + intel.feature.node.kubernetes.io/tdx=true \ + nvidia.com/cc.ready.state=true \ + --overwrite; \ + done + @echo "Building and loading attestation sidecar image into Kind..." + @mkdir -p /tmp/akash-sidecar-build + GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -mod=vendor -ldflags="-s -w" -o /tmp/akash-sidecar-build/attestation-sidecar $(AP_ROOT)/sidecar/attestation + cp $(AP_ROOT)/sidecar/attestation/Dockerfile /tmp/akash-sidecar-build/Dockerfile + docker build -q -t "$(ATTESTATION_SIDECAR_IMAGE)" /tmp/akash-sidecar-build/ + $(KIND) load docker-image "$(ATTESTATION_SIDECAR_IMAGE)" --name "$(KIND_NAME)" + @echo "CC setup complete. RuntimeClasses, node labels, and sidecar image loaded." + +.PHONY: kube-teardown-cc +kube-teardown-cc: + -kubectl delete -f cc-setup.yaml 2>/dev/null + @for node in $$(kubectl get nodes -o name); do \ + kubectl label $$node \ + katacontainers.io/kata-runtime- \ + amd.feature.node.kubernetes.io/snp- \ + intel.feature.node.kubernetes.io/tdx- \ + nvidia.com/cc.ready.state- \ + 2>/dev/null; \ + done + @echo "CC teardown complete." + .PHONY: clean-kube clean-kube: @@ -61,6 +120,8 @@ kube-deployments-rollout: #$(patsubst %, kube-deployment-rollout-%,$(KUSTOMIZE_I .PHONY: kube-setup-kube ifeq ($(GATEWAY_API),true) kube-setup-kube: kube-setup-hostname-operator-gateway +else ifeq ($(CONFIDENTIAL_COMPUTE),true) +kube-setup-kube: kube-setup-cc else kube-setup-kube: endif diff --git a/_run/kube/cc-containerd-setup.sh b/_run/kube/cc-containerd-setup.sh new file mode 100755 index 00000000..75e56abd --- /dev/null +++ b/_run/kube/cc-containerd-setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Registers mock CC runtime handlers in containerd, aliased to runc. +# Run inside the Kind node via: docker exec bash /path/to/this/script +set -e + +if grep -q kata-qemu-snp /etc/containerd/config.toml; then + echo "CC runtime handlers already registered" + exit 0 +fi + +for rt in kata-qemu-snp kata-qemu-nvidia-gpu-snp kata-qemu-tdx kata-qemu-nvidia-gpu-tdx; do + cat >> /etc/containerd/config.toml < 0 { resources = r.AllocatedResources diff --git a/cluster/client.go b/cluster/client.go index 51544fe9..8b240a7a 100644 --- a/cluster/client.go +++ b/cluster/client.go @@ -94,6 +94,13 @@ type Client interface { DeclareIP(ctx context.Context, lID mtypes.LeaseID, serviceName string, port uint32, externalPort uint32, proto mani.ServiceProtocol, sharingKey string, overwrite bool) error PurgeDeclaredIP(ctx context.Context, lID mtypes.LeaseID, serviceName string, externalPort uint32, proto mani.ServiceProtocol) error PurgeDeclaredIPs(ctx context.Context, lID mtypes.LeaseID) error + + // AttestationQuote forwards an attestation quote request to the sidecar + // running inside a confidential compute pod. The requestBody (containing the + // tenant's nonce) is sent verbatim to the sidecar; the response (hardware-signed + // evidence) is returned verbatim. The provider never inspects or modifies + // either payload (invariants #1 and #5). + AttestationQuote(ctx context.Context, lID mtypes.LeaseID, requestBody []byte) ([]byte, int, error) } func ErrorIsOkToSendToClient(err error) bool { @@ -310,6 +317,10 @@ func (c *nullClient) PurgeDeclaredIPs(_ context.Context, _ mtypes.LeaseID) error return errNotImplemented } +func (c *nullClient) AttestationQuote(_ context.Context, _ mtypes.LeaseID, _ []byte) ([]byte, int, error) { + return nil, 0, errNotImplemented +} + func (c *nullClient) ObserveIPState(_ context.Context) (<-chan cip.ResourceEvent, error) { return nil, errNotImplemented } diff --git a/cluster/inventory.go b/cluster/inventory.go index 71df2d30..13b5405c 100644 --- a/cluster/inventory.go +++ b/cluster/inventory.go @@ -859,3 +859,4 @@ func reservationCountEndpoints(reservation *reservation) uint { return externalPortCount } + diff --git a/cluster/kube/builder/builder.go b/cluster/kube/builder/builder.go index a7041ca1..54be1c49 100644 --- a/cluster/kube/builder/builder.go +++ b/cluster/kube/builder/builder.go @@ -50,6 +50,42 @@ const ( runtimeClassNvidia = "nvidia" ) +const ( + // AkashAttestationDisabledAnnotation is set on pods where the tenant opted out + // of attestation sidecar injection via the SDL attestation.enabled=false parameter. + AkashAttestationDisabledAnnotation = "akash.network/attestation-disabled" +) + +// Confidential compute runtime classes (Kata Containers + TEE) +const ( + RuntimeClassKataQemuSNP = "kata-qemu-snp" + RuntimeClassKataQemuNvidiaGPUSNP = "kata-qemu-nvidia-gpu-snp" + RuntimeClassKataQemuTDX = "kata-qemu-tdx" + RuntimeClassKataQemuNvidiaGPUTDX = "kata-qemu-nvidia-gpu-tdx" +) + +// Attestation sidecar resource limits. Inventory reserves limit values (not +// requests), matching the pattern used for tenant workloads. These must stay +// in sync with the limits set in webhook/sidecar.go:buildSidecarContainer. +const ( + SidecarCPULimitMillicores int64 = 100 + SidecarMemoryLimitBytes int64 = 64 * 1024 * 1024 // 64Mi + SidecarCPURequestMillicores int64 = 10 + SidecarMemoryRequestBytes int64 = 32 * 1024 * 1024 // 32Mi + + // Minimum resources for the primary container after sidecar subtraction. + // If the user's resources minus sidecar overhead falls below these values, + // clamp to these minimums. Prevents OOM kills from unusable resource limits. + MinPrimaryCPUMillicores int64 = 10 // 10m + MinPrimaryMemoryBytes int64 = 16 * 1024 * 1024 // 16Mi +) + +// TEE type constants used in placement attributes and directory responses. +const ( + TEETypeAMDSEVSNP = "amd-sev-snp" + TEETypeIntelTDX = "intel-tdx" +) + const ( envVarAkashGroupSequence = "AKASH_GROUP_SEQUENCE" envVarAkashDeploymentSequence = "AKASH_DEPLOYMENT_SEQUENCE" diff --git a/cluster/kube/builder/deployment.go b/cluster/kube/builder/deployment.go index f20fd4c8..ee84c16c 100644 --- a/cluster/kube/builder/deployment.go +++ b/cluster/kube/builder/deployment.go @@ -57,7 +57,8 @@ func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:unparam Replicas: b.replicas(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: b.labels(), + Labels: b.labels(), + Annotations: b.podAnnotations(), }, Spec: corev1.PodSpec{ Affinity: b.affinity(), @@ -85,6 +86,7 @@ func (b *deployment) Update(obj *appsv1.Deployment) (*appsv1.Deployment, error) uobj.Spec.Selector.MatchLabels = b.selectorLabels() uobj.Spec.Replicas = b.replicas() uobj.Spec.Template.Labels = b.labels() + uobj.Spec.Template.Annotations = b.podAnnotations() uobj.Spec.Template.Spec.Affinity = b.affinity() uobj.Spec.Template.Spec.RuntimeClassName = b.runtimeClass() uobj.Spec.Template.Spec.AutomountServiceAccountToken = b.automountServiceAccountToken() diff --git a/cluster/kube/builder/deployment_test.go b/cluster/kube/builder/deployment_test.go index e1468897..510988ce 100644 --- a/cluster/kube/builder/deployment_test.go +++ b/cluster/kube/builder/deployment_test.go @@ -135,3 +135,161 @@ func TestDeploymentPermissions(t *testing.T) { }) } } + +func TestSidecarResourceSubtraction(t *testing.T) { + tests := []struct { + name string + runtimeClass string + attestationDisabled bool + cpuMillis uint64 // SDL cpu in millicores + memBytes uint64 // SDL memory in bytes + expectSubtraction bool + }{ + { + name: "CC with attestation — 1Gi memory, 4000m CPU", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 4000, + memBytes: 1024 * 1024 * 1024, // 1Gi + expectSubtraction: true, + }, + { + name: "CC with attestation — 500m CPU, 128Mi memory", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, // 128Mi + expectSubtraction: true, + }, + { + name: "CC with attestation — GPU runtime class", + runtimeClass: RuntimeClassKataQemuNvidiaGPUSNP, + cpuMillis: 2000, + memBytes: 512 * 1024 * 1024, // 512Mi + expectSubtraction: true, + }, + { + name: "CC with attestation disabled — no subtraction", + runtimeClass: RuntimeClassKataQemuSNP, + attestationDisabled: true, + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "non-CC workload — no subtraction", + runtimeClass: "nvidia", + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "no runtime class — no subtraction", + runtimeClass: "", + cpuMillis: 500, + memBytes: 128 * 1024 * 1024, + expectSubtraction: false, + }, + { + name: "CC with small resources — floors at minimum", + runtimeClass: RuntimeClassKataQemuSNP, + cpuMillis: 110, // barely above sidecar limit (100m) + memBytes: 68 * 1024 * 1024, // barely above sidecar limit (64Mi) + expectSubtraction: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lid := testutil.LeaseID(t) + + log := testutil.Logger(t) + settings := Settings{ + CPUCommitLevel: 1, // request = limit (simplifies assertions) + MemoryCommitLevel: 1, + } + + sdlData, err := sdl.ReadFile("../../../testdata/deployment/deployment.yaml") + require.NoError(t, err) + + mani, err := sdlData.Manifest() + require.NoError(t, err) + + sp := &crd.SchedulerParams{ + RuntimeClass: tt.runtimeClass, + AttestationDisabled: tt.attestationDisabled, + } + sparams := []*crd.SchedulerParams{sp} + + cmani, err := crd.NewManifest("lease", lid, &mani.GetGroups()[0], crd.ClusterSettings{SchedulerParams: sparams}) + require.NoError(t, err) + + // Override CPU and memory in the CRD before building the workload + cmani.Spec.Group.Services[0].Resources.CPU.Units = uint32(tt.cpuMillis) //nolint:gosec + cmani.Spec.Group.Services[0].Resources.Memory.Size = fmt.Sprintf("%d", tt.memBytes) + + group, retSparams, err := cmani.Spec.Group.FromCRD() + require.NoError(t, err) + + cdep := &ClusterDeployment{ + Lid: lid, + Group: &group, + Sparams: crd.ClusterSettings{SchedulerParams: retSparams}, + } + // Override scheduler params with our CC config + cdep.Sparams.SchedulerParams[0] = sp + + workload, err := NewWorkloadBuilder(log, settings, cdep, cmani, 0) + require.NoError(t, err) + + container := workload.container() + + cpuLimit := container.Resources.Limits.Cpu().MilliValue() + cpuRequest := container.Resources.Requests.Cpu().MilliValue() + memLimit := container.Resources.Limits.Memory().Value() + memRequest := container.Resources.Requests.Memory().Value() + + if tt.expectSubtraction { + expectedCPULimit := int64(tt.cpuMillis) - SidecarCPULimitMillicores + if expectedCPULimit < MinPrimaryCPUMillicores { + expectedCPULimit = MinPrimaryCPUMillicores + } + expectedMemLimit := int64(tt.memBytes) - SidecarMemoryLimitBytes + if expectedMemLimit < MinPrimaryMemoryBytes { + expectedMemLimit = MinPrimaryMemoryBytes + } + + require.Equal(t, expectedCPULimit, cpuLimit, + "CPU limit: want %dm, got %dm", expectedCPULimit, cpuLimit) + require.Equal(t, expectedMemLimit, memLimit, + "Memory limit: want %d, got %d", expectedMemLimit, memLimit) + + // Pod total LIMIT should equal user's original limit + // (only when resources are above sidecar footprint) + // Pod total equals user limit only when resources are well above sidecar + minimum + if int64(tt.cpuMillis)-SidecarCPULimitMillicores >= MinPrimaryCPUMillicores { + require.Equal(t, int64(tt.cpuMillis), cpuLimit+SidecarCPULimitMillicores, + "Pod CPU limit total should equal user limit") + } + if int64(tt.memBytes)-SidecarMemoryLimitBytes >= MinPrimaryMemoryBytes { + require.Equal(t, int64(tt.memBytes), memLimit+SidecarMemoryLimitBytes, + "Pod memory limit total should equal user limit") + } + + // K8s constraint: limit >= request + require.GreaterOrEqual(t, cpuLimit, cpuRequest, + "CPU limit must be >= request") + require.GreaterOrEqual(t, memLimit, memRequest, + "Memory limit must be >= request") + } else { + // No subtraction — primary container gets full user resources + require.Equal(t, int64(tt.cpuMillis), cpuLimit, + "CPU limit should be unmodified") + require.Equal(t, int64(tt.memBytes), memLimit, + "Memory limit should be unmodified") + } + + // Always: limit >= request (K8s invariant) + require.GreaterOrEqual(t, cpuLimit, cpuRequest, "K8s: CPU limit >= request") + require.GreaterOrEqual(t, memLimit, memRequest, "K8s: memory limit >= request") + }) + } +} diff --git a/cluster/kube/builder/statefulset.go b/cluster/kube/builder/statefulset.go index 7e9365af..e282dbe4 100644 --- a/cluster/kube/builder/statefulset.go +++ b/cluster/kube/builder/statefulset.go @@ -57,7 +57,8 @@ func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:unparam Replicas: b.replicas(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: b.labels(), + Labels: b.labels(), + Annotations: b.podAnnotations(), }, Spec: corev1.PodSpec{ Affinity: b.affinity(), @@ -86,6 +87,7 @@ func (b *statefulSet) Update(obj *appsv1.StatefulSet) (*appsv1.StatefulSet, erro uobj.Spec.Replicas = b.replicas() uobj.Spec.Selector.MatchLabels = b.selectorLabels() uobj.Spec.Template.Labels = b.labels() + uobj.Spec.Template.Annotations = b.podAnnotations() uobj.Spec.Template.Spec.Affinity = b.affinity() uobj.Spec.Template.Spec.RuntimeClassName = b.runtimeClass() uobj.Spec.Template.Spec.AutomountServiceAccountToken = b.automountServiceAccountToken() diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index 5036b012..86896f2c 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -17,12 +17,68 @@ import ( ) const ( - ResourceGPUNvidia = corev1.ResourceName("nvidia.com/gpu") - ResourceGPUAMD = corev1.ResourceName("amd.com/gpu") - GPUVendorNvidia = "nvidia" - GPUVendorAMD = "amd" + ResourceGPUNvidia = corev1.ResourceName("nvidia.com/gpu") + ResourceGPUNvidiaPGPU = corev1.ResourceName("nvidia.com/pgpu") + ResourceGPUAMD = corev1.ResourceName("amd.com/gpu") + GPUVendorNvidia = "nvidia" + GPUVendorAMD = "amd" ) +// IsConfidentialComputeRuntimeClass returns true if the given runtime class +// is a Kata Containers confidential compute variant. +func IsConfidentialComputeRuntimeClass(rc string) bool { + switch rc { + case RuntimeClassKataQemuSNP, RuntimeClassKataQemuNvidiaGPUSNP, + RuntimeClassKataQemuTDX, RuntimeClassKataQemuNvidiaGPUTDX: + return true + } + return false +} + +// IsGPURuntimeClass returns true if the runtime class is a GPU-enabled CC variant. +func IsGPURuntimeClass(rc string) bool { + return rc == RuntimeClassKataQemuNvidiaGPUSNP || rc == RuntimeClassKataQemuNvidiaGPUTDX +} + +// IsSNPRuntimeClass returns true if the runtime class uses AMD SEV-SNP. +func IsSNPRuntimeClass(rc string) bool { + return rc == RuntimeClassKataQemuSNP || rc == RuntimeClassKataQemuNvidiaGPUSNP +} + +// IsTDXRuntimeClass returns true if the runtime class uses Intel TDX. +func IsTDXRuntimeClass(rc string) bool { + return rc == RuntimeClassKataQemuTDX || rc == RuntimeClassKataQemuNvidiaGPUTDX +} + +// TEETypeForRuntimeClass returns the TEE type string for a given CC runtime class. +func TEETypeForRuntimeClass(rc string) string { + switch { + case IsSNPRuntimeClass(rc): + return TEETypeAMDSEVSNP + case IsTDXRuntimeClass(rc): + return TEETypeIntelTDX + default: + return "" + } +} + +// RuntimeClassForTEEType maps an SDL TEE type to the corresponding Kata runtime class. +// The TEE type values come from the SDL tee.type enum (sev-snp, sev-snp-gpu, tdx, tdx-gpu). +func RuntimeClassForTEEType(teeType string) string { + switch teeType { + case "sev-snp": + return RuntimeClassKataQemuSNP + case "sev-snp-gpu": + return RuntimeClassKataQemuNvidiaGPUSNP + case "tdx": + return RuntimeClassKataQemuTDX + case "tdx-gpu": + return RuntimeClassKataQemuNvidiaGPUTDX + default: + return "" + } +} + type workloadBase interface { builderBase Name() string @@ -101,10 +157,33 @@ func (b *Workload) container() corev1.Container { }, } + // Whether the attestation sidecar will be injected into this pod. + // When true, we subtract the sidecar's resource footprint from the + // primary container so the pod total matches what the user requested. + sidecarActive := sparams != nil && + IsConfidentialComputeRuntimeClass(sparams.RuntimeClass) && + !sparams.AttestationDisabled + if cpu := service.Resources.CPU; cpu != nil { - requestedCPU := sdlutil.ComputeCommittedResources(b.settings.CPUCommitLevel, cpu.Units) - kcontainer.Resources.Requests[corev1.ResourceCPU] = resource.NewScaledQuantity(int64(requestedCPU.Value()), resource.Milli).DeepCopy() // nolint: gosec - kcontainer.Resources.Limits[corev1.ResourceCPU] = resource.NewScaledQuantity(int64(cpu.Units.Value()), resource.Milli).DeepCopy() // nolint: gosec + cpuLimit := int64(cpu.Units.Value()) // nolint: gosec + cpuRequest := int64(sdlutil.ComputeCommittedResources(b.settings.CPUCommitLevel, cpu.Units).Value()) // nolint: gosec + + if sidecarActive { + cpuLimit -= SidecarCPULimitMillicores + cpuRequest -= SidecarCPURequestMillicores + if cpuLimit < MinPrimaryCPUMillicores { + cpuLimit = MinPrimaryCPUMillicores + } + if cpuRequest < MinPrimaryCPUMillicores { + cpuRequest = MinPrimaryCPUMillicores + } + if cpuRequest > cpuLimit { + cpuRequest = cpuLimit + } + } + + kcontainer.Resources.Requests[corev1.ResourceCPU] = resource.NewScaledQuantity(cpuRequest, resource.Milli).DeepCopy() + kcontainer.Resources.Limits[corev1.ResourceCPU] = resource.NewScaledQuantity(cpuLimit, resource.Milli).DeepCopy() } if gpu := service.Resources.GPU; gpu != nil && gpu.Units.Value() > 0 { @@ -112,7 +191,11 @@ func (b *Workload) container() corev1.Container { switch sparams.Resources.GPU.Vendor { case GPUVendorNvidia: - resourceName = ResourceGPUNvidia + if IsConfidentialComputeRuntimeClass(sparams.RuntimeClass) { + resourceName = ResourceGPUNvidiaPGPU // VFIO passthrough for CC + } else { + resourceName = ResourceGPUNvidia + } case GPUVendorAMD: resourceName = ResourceGPUAMD default: @@ -150,9 +233,25 @@ func (b *Workload) container() corev1.Container { // fixme: ram is never expected to be nil if mem := service.Resources.Memory; mem != nil { - requestedRAM := sdlutil.ComputeCommittedResources(b.settings.MemoryCommitLevel, mem.Quantity) - kcontainer.Resources.Requests[corev1.ResourceMemory] = resource.NewQuantity(int64(requestedRAM.Value()), resource.DecimalSI).DeepCopy() // nolint: gosec - kcontainer.Resources.Limits[corev1.ResourceMemory] = resource.NewQuantity(int64(mem.Quantity.Value()+requestedMem), resource.DecimalSI).DeepCopy() // nolint: gosec + memLimit := int64(mem.Quantity.Value()+requestedMem) // nolint: gosec + memRequest := int64(sdlutil.ComputeCommittedResources(b.settings.MemoryCommitLevel, mem.Quantity).Value()) // nolint: gosec + + if sidecarActive { + memLimit -= SidecarMemoryLimitBytes + memRequest -= SidecarMemoryRequestBytes + if memLimit < MinPrimaryMemoryBytes { + memLimit = MinPrimaryMemoryBytes + } + if memRequest < MinPrimaryMemoryBytes { + memRequest = MinPrimaryMemoryBytes + } + if memRequest > memLimit { + memRequest = memLimit + } + } + + kcontainer.Resources.Requests[corev1.ResourceMemory] = resource.NewQuantity(memRequest, resource.DecimalSI).DeepCopy() + kcontainer.Resources.Limits[corev1.ResourceMemory] = resource.NewQuantity(memLimit, resource.DecimalSI).DeepCopy() } if service.Params != nil { @@ -264,6 +363,18 @@ func (b *Workload) persistentVolumeClaims() []corev1.PersistentVolumeClaim { return pvcs } +// podAnnotations returns annotations for the pod template, including +// the attestation-disabled annotation when the tenant has opted out. +func (b *Workload) podAnnotations() map[string]string { + params := b.sparams[b.serviceIdx] + if params != nil && params.AttestationDisabled { + return map[string]string{ + AkashAttestationDisabledAnnotation: "true", + } + } + return nil +} + func (b *Workload) runtimeClass() *string { params := b.sparams[b.serviceIdx] @@ -305,6 +416,38 @@ func (b *Workload) affinity() *corev1.Affinity { selectors = append(selectors, nodeSelectorsFromResources(params.Resources)...) } + if params != nil && IsConfidentialComputeRuntimeClass(params.RuntimeClass) { + selectors = append(selectors, corev1.NodeSelectorRequirement{ + Key: "katacontainers.io/kata-runtime", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }) + + // TEE-specific node labels + if IsSNPRuntimeClass(params.RuntimeClass) { + selectors = append(selectors, corev1.NodeSelectorRequirement{ + Key: "amd.feature.node.kubernetes.io/snp", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }) + } + if IsTDXRuntimeClass(params.RuntimeClass) { + selectors = append(selectors, corev1.NodeSelectorRequirement{ + Key: "intel.feature.node.kubernetes.io/tdx", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }) + } + + if IsGPURuntimeClass(params.RuntimeClass) { + selectors = append(selectors, corev1.NodeSelectorRequirement{ + Key: "nvidia.com/cc.ready.state", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }) + } + } + for _, storage := range service.Resources.Storage { attr := storage.Attributes.Find(sdl.StorageAttributePersistent) if persistent, valid := attr.AsBool(); !valid || !persistent { diff --git a/cluster/kube/client_attestation.go b/cluster/kube/client_attestation.go new file mode 100644 index 00000000..f394fb32 --- /dev/null +++ b/cluster/kube/client_attestation.go @@ -0,0 +1,84 @@ +package kube + +import ( + "context" + "fmt" + "net/http" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/net" + + mtypes "pkg.akt.dev/go/node/market/v1" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +const ( + attestationSidecarContainerName = "akash-attestation-sidecar" + attestationSidecarPort = "8790" +) + +// AttestationQuote forwards an attestation quote request to the sidecar running +// inside a confidential compute pod. It uses the K8s API server's pod proxy +// subresource to reach the sidecar, which works both in-cluster and out-of-cluster +// (e.g., macOS local dev via Kind where pod IPs aren't routable from the host). +// +// The request path is: provider → K8s API server → kubelet → pod:8790/quote +// This follows the same pattern as Exec (which also goes through the API server). +func (c *client) AttestationQuote(ctx context.Context, leaseID mtypes.LeaseID, requestBody []byte) ([]byte, int, error) { + namespace := builder.LidNS(leaseID) + + if err := c.leaseExists(ctx, leaseID); err != nil { + return nil, http.StatusNotFound, err + } + + podName, err := c.findAttestationSidecarPod(ctx, namespace) + if err != nil { + return nil, http.StatusNotFound, err + } + + // Use the K8s API server's pod proxy to POST to the sidecar's /quote endpoint. + // Path: /api/v1/namespaces/{ns}/pods/{scheme}:{pod}:{port}/proxy/quote + body, err := c.kc.CoreV1().RESTClient().Post(). + Namespace(namespace). + Resource("pods"). + SubResource("proxy"). + Name(net.JoinSchemeNamePort("https", podName, attestationSidecarPort)). + Suffix("quote"). + SetHeader("Content-Type", "application/json"). + Body(requestBody). + DoRaw(ctx) + + if err != nil { + return nil, http.StatusBadGateway, fmt.Errorf("proxy to attestation sidecar: %w", err) + } + + return body, http.StatusOK, nil +} + +// findAttestationSidecarPod finds a running pod in the lease namespace that +// contains the attestation sidecar container. Returns the pod name. +func (c *client) findAttestationSidecarPod(ctx context.Context, namespace string) (string, error) { + pods, err := wrapKubeCall("pods-list", func() (*corev1.PodList, error) { + return c.kc.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=true", builder.AkashManagedLabelName), + }) + }) + if err != nil { + return "", fmt.Errorf("list pods in %s: %w", namespace, err) + } + + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning { + continue + } + for _, container := range pod.Spec.Containers { + if container.Name == attestationSidecarContainerName { + return pod.Name, nil + } + } + } + + return "", fmt.Errorf("no running pod with attestation sidecar in namespace %s", namespace) +} diff --git a/cluster/kube/operators/clients/inventory/inventory.go b/cluster/kube/operators/clients/inventory/inventory.go index 2b415bc2..b50c5a2a 100644 --- a/cluster/kube/operators/clients/inventory/inventory.go +++ b/cluster/kube/operators/clients/inventory/inventory.go @@ -47,6 +47,7 @@ func (inv *inventory) tryAdjust(node int, res *rtypes.Resources) (*crd.Scheduler nd := inv.Nodes[node].Dup() sparams := &crd.SchedulerParams{} + if !tryAdjustCPU(&nd.Resources.CPU.Quantity, res.CPU) { return nil, false, true } @@ -55,6 +56,7 @@ func (inv *inventory) tryAdjust(node int, res *rtypes.Resources) (*crd.Scheduler return nil, false, true } + if !nd.Resources.Memory.Quantity.SubNLZ(res.Memory.Quantity) { return nil, false, true } @@ -202,6 +204,39 @@ func tryAdjustGPU(rp *inventoryV1.GPU, res *rtypes.GPU, sparams *crd.SchedulerPa return false } +// tryAdjustConfidentialCompute adjusts cluster inventory for confidential +// compute workloads. For CPU-only CC workloads (no GPU present), it sets the +// appropriate runtime class based on the TEE type. It also reserves the +// attestation sidecar resources that the webhook will inject into CC pods, +// using limit values so inventory subtracts the full resource limit. +func tryAdjustConfidentialCompute(nd *inventoryV1.Node, sparams *crd.SchedulerParams, confidentialCompute bool, teeType string) bool { + if !confidentialCompute { + return true + } + + // For CC CPU-only workloads (no GPU), set the runtime class based on TEE type + if sparams.RuntimeClass == "" { + switch teeType { + case builder.TEETypeIntelTDX: + sparams.RuntimeClass = builder.RuntimeClassKataQemuTDX + default: + sparams.RuntimeClass = builder.RuntimeClassKataQemuSNP + } + } + + sidecarCPU := rtypes.NewResourceValue(uint64(builder.SidecarCPULimitMillicores)) + if !nd.Resources.CPU.Quantity.SubMilliNLZ(sidecarCPU) { + return false + } + + sidecarMem := rtypes.NewResourceValue(uint64(builder.SidecarMemoryLimitBytes)) + if !nd.Resources.Memory.Quantity.SubNLZ(sidecarMem) { + return false + } + + return true +} + func tryAdjustEphemeralStorage(rp *inventoryV1.ResourcePair, res *rtypes.Storage) bool { return rp.SubNLZ(res.Quantity) } @@ -407,3 +442,4 @@ func sParamsEnsureResources(sparams *crd.SchedulerParams) { sparams.Resources = &crd.SchedulerResources{} } } + diff --git a/cluster/kube/webhook/registration.go b/cluster/kube/webhook/registration.go new file mode 100644 index 00000000..65570fe9 --- /dev/null +++ b/cluster/kube/webhook/registration.go @@ -0,0 +1,141 @@ +package webhook + +import ( + "context" + "fmt" + + "cosmossdk.io/log" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +const ( + webhookName = "attestation-sidecar-injector.akash.network" + webhookConfigName = "akash-attestation-sidecar-injector" +) + +// RegisterWebhookConfiguration creates or updates the MutatingWebhookConfiguration +// in the cluster. This tells the K8s API server to send pod CREATE requests +// (in Akash-managed namespaces) to our webhook for sidecar injection. +// +// The caBundle must be PEM-encoded. When webhookURL is non-empty, it registers +// with a URL endpoint (for local dev). Otherwise it uses a K8s Service reference. +func RegisterWebhookConfiguration(ctx context.Context, kc kubernetes.Interface, log log.Logger, serviceName, serviceNamespace string, caBundle []byte, webhookPort int32, webhookURL string) error { + failPolicy := admissionregistrationv1.Fail // fail-closed + sideEffects := admissionregistrationv1.SideEffectClassNone + matchPolicy := admissionregistrationv1.Equivalent + timeoutSec := int32(5) + + path := "/mutate" + + var clientConfig admissionregistrationv1.WebhookClientConfig + if webhookURL != "" { + // Local dev: provider runs outside the cluster, use URL endpoint + url := webhookURL + path + clientConfig = admissionregistrationv1.WebhookClientConfig{ + URL: &url, + CABundle: caBundle, + } + } else { + // Production: provider runs as a K8s service + clientConfig = admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: serviceName, + Namespace: serviceNamespace, + Path: &path, + Port: &webhookPort, + }, + CABundle: caBundle, + } + } + + cfg := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookConfigName, + Labels: map[string]string{ + "app.kubernetes.io/name": "attestation-webhook", + "app.kubernetes.io/part-of": "provider", + "app.kubernetes.io/component": "admission-webhook", + }, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: webhookName, + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failPolicy, + SideEffects: &sideEffects, + MatchPolicy: &matchPolicy, + TimeoutSeconds: &timeoutSec, + ClientConfig: clientConfig, + // Only intercept pod creation in tenant lease namespaces. + // Lease namespaces have both akash.network=true AND akash.network/namespace labels. + // This excludes akash-services (provider/operator pods) which only has akash.network=true. + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + builder.AkashManagedLabelName: builder.ValTrue, + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "akash.network/namespace", + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + }, + }, + } + + client := kc.AdmissionregistrationV1().MutatingWebhookConfigurations() + + existing, err := client.Get(ctx, webhookConfigName, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + _, err = client.Create(ctx, cfg, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create MutatingWebhookConfiguration: %w", err) + } + log.Info("created MutatingWebhookConfiguration", "name", webhookConfigName) + return nil + } + if err != nil { + return fmt.Errorf("get MutatingWebhookConfiguration: %w", err) + } + + cfg.ResourceVersion = existing.ResourceVersion + _, err = client.Update(ctx, cfg, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("update MutatingWebhookConfiguration: %w", err) + } + log.Info("updated MutatingWebhookConfiguration", "name", webhookConfigName) + return nil +} + +// DeregisterWebhookConfiguration removes the MutatingWebhookConfiguration +// from the cluster. Call this on shutdown to avoid dangling webhooks that +// block pod creation when the provider is not running. +func DeregisterWebhookConfiguration(ctx context.Context, kc kubernetes.Interface, log log.Logger) { + err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete( + ctx, webhookConfigName, metav1.DeleteOptions{}, + ) + if err != nil && !kerrors.IsNotFound(err) { + log.Error("failed to deregister MutatingWebhookConfiguration", "err", err) + return + } + log.Info("deregistered MutatingWebhookConfiguration", "name", webhookConfigName) +} diff --git a/cluster/kube/webhook/sidecar.go b/cluster/kube/webhook/sidecar.go new file mode 100644 index 00000000..25963095 --- /dev/null +++ b/cluster/kube/webhook/sidecar.go @@ -0,0 +1,203 @@ +package webhook + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +const ( + sidecarContainerName = "akash-attestation-sidecar" + sidecarPort = int32(8790) + tsmVolumeName = "tsm-reports" + sevGuestVolumeName = "sev-guest" +) + +// ShouldInject returns true if the pod should have the attestation sidecar injected. +// Trigger conditions: +// - Pod has the akash.network managed label +// - Pod has a Kata CC RuntimeClassName +// - Tenant has not opted out via attestation-disabled annotation +// - Sidecar is not already present (idempotency) +func ShouldInject(pod *corev1.Pod) bool { + if pod.Labels[builder.AkashManagedLabelName] != builder.ValTrue { + return false + } + + if pod.Spec.RuntimeClassName == nil { + return false + } + + if !builder.IsConfidentialComputeRuntimeClass(*pod.Spec.RuntimeClassName) { + return false + } + + // Tenant opted out of attestation sidecar injection + if pod.Annotations[builder.AkashAttestationDisabledAnnotation] == "true" { + return false + } + + for _, c := range pod.Spec.Containers { + if c.Name == sidecarContainerName { + return false // already injected + } + } + + return true +} + +type jsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +// BuildSidecarPatch generates a JSON Patch (RFC 6902) that injects the +// attestation sidecar container, required volumes, and volume mounts into +// the pod spec. +// +// The sidecar runs INSIDE the Kata VM (same pod, same VM as the tenant workload). +// It requires CAP_SYS_ADMIN for configfs mount operations within the guest kernel. +func BuildSidecarPatch(pod *corev1.Pod, sidecarImage string, extraEnv []corev1.EnvVar) ([]byte, error) { + if sidecarImage == "" { + return nil, fmt.Errorf("attestation sidecar image not configured") + } + + runtimeClass := "" + if pod.Spec.RuntimeClassName != nil { + runtimeClass = *pod.Spec.RuntimeClassName + } + + isGPU := builder.IsGPURuntimeClass(runtimeClass) + + mockMode := isMockMode(extraEnv) + container := buildSidecarContainer(sidecarImage, isGPU, extraEnv) + volumes := buildSidecarVolumes(isGPU, mockMode) + + var patches []jsonPatch + + // Add sidecar container + if len(pod.Spec.Containers) == 0 { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/containers", + Value: []corev1.Container{container}, + }) + } else { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/containers/-", + Value: container, + }) + } + + // Add volumes (only if there are any to add). + if len(volumes) > 0 { + if len(pod.Spec.Volumes) == 0 { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/volumes", + Value: volumes, + }) + } else { + for _, vol := range volumes { + patches = append(patches, jsonPatch{ + Op: "add", + Path: "/spec/volumes/-", + Value: vol, + }) + } + } + } + + return json.Marshal(patches) +} + +func buildSidecarContainer(image string, isGPU bool, extraEnv []corev1.EnvVar) corev1.Container { + c := corev1.Container{ + Name: sidecarContainerName, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/attestation-sidecar"}, + Ports: []corev1.ContainerPort{{ + Name: "attestation", + ContainerPort: sidecarPort, + Protocol: corev1.ProtocolTCP, + }}, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(builder.SidecarCPURequestMillicores, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(builder.SidecarMemoryRequestBytes, resource.DecimalSI), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(builder.SidecarCPULimitMillicores, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(builder.SidecarMemoryLimitBytes, resource.DecimalSI), + }, + }, + Env: append([]corev1.EnvVar{ + {Name: "ATTESTATION_LISTEN_ADDR", Value: fmt.Sprintf(":%d", sidecarPort)}, + }, extraEnv...), + } + + // Volume mounts: in mock mode use writable paths (distroless /sys is read-only). + // In production, Kata VMs have a writable guest /sys/kernel/config. + mockMode := isMockMode(extraEnv) + if mockMode { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: tsmVolumeName, + MountPath: "/var/run/mock-tee/tsm-report", + }) + if !isGPU { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: sevGuestVolumeName, + MountPath: "/var/run/mock-tee/sev-guest", + }) + } + } + // Production (non-mock) kata VMs: the sidecar accesses /dev/sev-guest + // and /sys/kernel/config/tsm/report directly from the guest kernel + // filesystem — no volume mounts needed. + + return c +} + +func isMockMode(extraEnv []corev1.EnvVar) bool { + for _, env := range extraEnv { + if env.Name == "ATTESTATION_MOCK" && env.Value == "true" { + return true + } + } + return false +} + +func buildSidecarVolumes(isGPU bool, mockMode bool) []corev1.Volume { + // Mock mode: use emptyDir so pods can start on non-TEE hosts (Kind). + // The mock TEE provider generates synthetic reports in-memory and + // doesn't access these mount paths. + if mockMode { + volumes := []corev1.Volume{ + {Name: tsmVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } + if !isGPU { + volumes = append(volumes, corev1.Volume{ + Name: sevGuestVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + } + return volumes + } + + // Production: the sidecar accesses /dev/sev-guest and + // /sys/kernel/config/tsm/report directly from the kata guest kernel + // filesystem. HostPath volumes cannot be used because kubelet validates + // paths on the host node, where these devices do not exist. + return nil +} diff --git a/cluster/kube/webhook/sidecar_test.go b/cluster/kube/webhook/sidecar_test.go new file mode 100644 index 00000000..68adcf69 --- /dev/null +++ b/cluster/kube/webhook/sidecar_test.go @@ -0,0 +1,182 @@ +package webhook + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/akash-network/provider/cluster/kube/builder" +) + +func strPtr(s string) *string { return &s } + +func TestShouldInject_CCPodWithGPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUSNP), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_CCPodCPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_NonCCPod(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr("nvidia"), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_NoRuntimeClass(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{}, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_AttestationDisabled(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + Annotations: map[string]string{builder.AkashAttestationDisabledAnnotation: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_MissingLabel(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{}}, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_AlreadyInjected(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + Containers: []corev1.Container{ + {Name: sidecarContainerName}, + }, + }, + } + require.False(t, ShouldInject(pod)) +} + +func TestShouldInject_TDXCPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuTDX), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestShouldInject_TDXWithGPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUTDX), + }, + } + require.True(t, ShouldInject(pod)) +} + +func TestBuildSidecarPatch_GPU(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuNvidiaGPUSNP), + Containers: []corev1.Container{ + {Name: "workload", Image: "myimage"}, + }, + }, + } + + patchBytes, err := BuildSidecarPatch(pod, "ghcr.io/akash-network/attestation-sidecar:latest", nil) + require.NoError(t, err) + require.NotEmpty(t, patchBytes) + + var patches []jsonPatch + err = json.Unmarshal(patchBytes, &patches) + require.NoError(t, err) + + // Production mode: container patch only, no volumes (guest kernel provides devices) + require.Len(t, patches, 1) // container only +} + +func TestBuildSidecarPatch_CPUOnly(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{builder.AkashManagedLabelName: "true"}, + }, + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + Containers: []corev1.Container{ + {Name: "workload", Image: "myimage"}, + }, + }, + } + + patchBytes, err := BuildSidecarPatch(pod, "ghcr.io/akash-network/attestation-sidecar:latest", nil) + require.NoError(t, err) + + var patches []jsonPatch + err = json.Unmarshal(patchBytes, &patches) + require.NoError(t, err) + + // Production mode: container patch only, no volumes (guest kernel provides devices) + require.Len(t, patches, 1) +} + +func TestBuildSidecarPatch_EmptyImage(t *testing.T) { + pod := &corev1.Pod{ + Spec: corev1.PodSpec{ + RuntimeClassName: strPtr(builder.RuntimeClassKataQemuSNP), + }, + } + + _, err := BuildSidecarPatch(pod, "", nil) + require.Error(t, err) +} diff --git a/cluster/kube/webhook/webhook.go b/cluster/kube/webhook/webhook.go new file mode 100644 index 00000000..53244487 --- /dev/null +++ b/cluster/kube/webhook/webhook.go @@ -0,0 +1,152 @@ +package webhook + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "cosmossdk.io/log" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" +) + +var ( + runtimeScheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(runtimeScheme) + deserializer = codecs.UniversalDeserializer() +) + +// Config holds configuration for the attestation webhook server. +type Config struct { + SidecarImage string + SidecarEnv []corev1.EnvVar // additional env vars injected into the sidecar container + ListenAddr string + TLSCert tls.Certificate + Log log.Logger +} + +// Server handles mutating admission webhook requests for attestation +// sidecar injection into confidential compute pods. +type Server struct { + server *http.Server + config Config +} + +// NewServer creates a new attestation webhook server. +// Failure mode: fail-closed. A CC workload without an attestation sidecar +// is silently unverifiable — the tenant cannot confirm TEE execution. +func NewServer(cfg Config) *Server { + s := &Server{config: cfg} + + mux := http.NewServeMux() + mux.HandleFunc("/mutate", s.handleMutate) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + s.server = &http.Server{ + Addr: cfg.ListenAddr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cfg.TLSCert}, + MinVersion: tls.VersionTLS12, + }, + } + + return s +} + +// ListenAndServeTLS starts the webhook HTTPS server. +func (s *Server) ListenAndServeTLS() error { + return s.server.ListenAndServeTLS("", "") +} + +// Shutdown gracefully shuts down the server. +func (s *Server) Shutdown(ctx context.Context) error { + return s.server.Shutdown(ctx) +} + +func (s *Server) handleMutate(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + s.config.Log.Error("failed to read request body", "err", err) + http.Error(w, "failed to read body", http.StatusBadRequest) + return + } + + if len(body) == 0 { + http.Error(w, "empty body", http.StatusBadRequest) + return + } + + var admissionReview admissionv1.AdmissionReview + if _, _, err := deserializer.Decode(body, nil, &admissionReview); err != nil { + s.config.Log.Error("failed to decode admission review", "err", err) + http.Error(w, fmt.Sprintf("decode error: %v", err), http.StatusBadRequest) + return + } + + response := s.mutate(admissionReview.Request) + + admissionReview.Response = response + admissionReview.Response.UID = admissionReview.Request.UID + + resp, err := json.Marshal(admissionReview) + if err != nil { + s.config.Log.Error("failed to marshal response", "err", err) + http.Error(w, fmt.Sprintf("marshal error: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(resp) //nolint:errcheck +} + +func (s *Server) mutate(req *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + if req == nil { + return &admissionv1.AdmissionResponse{Allowed: true} + } + + var pod corev1.Pod + if err := json.Unmarshal(req.Object.Raw, &pod); err != nil { + s.config.Log.Error("failed to unmarshal pod", "err", err) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: fmt.Sprintf("failed to unmarshal pod: %v", err), + }, + } + } + + if !ShouldInject(&pod) { + return &admissionv1.AdmissionResponse{Allowed: true} + } + + patch, err := BuildSidecarPatch(&pod, s.config.SidecarImage, s.config.SidecarEnv) + if err != nil { + s.config.Log.Error("failed to build sidecar patch", "err", err) + // Fail-closed: reject the pod if we can't build the patch + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: fmt.Sprintf("failed to build attestation sidecar patch: %v", err), + }, + } + } + + patchType := admissionv1.PatchTypeJSONPatch + return &admissionv1.AdmissionResponse{ + Allowed: true, + Patch: patch, + PatchType: &patchType, + } +} diff --git a/cluster/types/v1beta3/types.go b/cluster/types/v1beta3/types.go index 392a44af..40e3e7d0 100644 --- a/cluster/types/v1beta3/types.go +++ b/cluster/types/v1beta3/types.go @@ -64,6 +64,7 @@ func WithDryRun() InventoryOption { } } + type Inventory interface { Adjust(ReservationGroup, ...InventoryOption) error Metrics() inventoryV1.Metrics diff --git a/cmd/provider-services/cmd/flags.go b/cmd/provider-services/cmd/flags.go index 8fff9a03..e434e684 100644 --- a/cmd/provider-services/cmd/flags.go +++ b/cmd/provider-services/cmd/flags.go @@ -354,5 +354,35 @@ func addRunFlags(cmd *cobra.Command) error { return err } + cmd.Flags().Bool(FlagAttestationWebhookEnabled, false, "Enable attestation sidecar mutating admission webhook for confidential compute") + if err := viper.BindPFlag(FlagAttestationWebhookEnabled, cmd.Flags().Lookup(FlagAttestationWebhookEnabled)); err != nil { + return err + } + + cmd.Flags().Int(FlagAttestationWebhookPort, 9443, "Port for the attestation sidecar webhook server") + if err := viper.BindPFlag(FlagAttestationWebhookPort, cmd.Flags().Lookup(FlagAttestationWebhookPort)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationSidecarImage, "", "Docker image for the attestation sidecar (required when webhook is enabled)") + if err := viper.BindPFlag(FlagAttestationSidecarImage, cmd.Flags().Lookup(FlagAttestationSidecarImage)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationExpectedMeasurement, "", "Expected SEV-SNP launch measurement (hex) for the attestation directory. Advisory hint only (invariant #4)") + if err := viper.BindPFlag(FlagAttestationExpectedMeasurement, cmd.Flags().Lookup(FlagAttestationExpectedMeasurement)); err != nil { + return err + } + + cmd.Flags().String(FlagAttestationExpectedImageDigest, "", "Expected Kata VM image digest for the attestation directory. Advisory hint only (invariant #4)") + if err := viper.BindPFlag(FlagAttestationExpectedImageDigest, cmd.Flags().Lookup(FlagAttestationExpectedImageDigest)); err != nil { + return err + } + + cmd.Flags().Bool(FlagAttestationMockMode, false, "Run attestation sidecar in mock mode (synthetic reports, no TEE hardware). For local development only") + if err := viper.BindPFlag(FlagAttestationMockMode, cmd.Flags().Lookup(FlagAttestationMockMode)); err != nil { + return err + } + return nil } diff --git a/cmd/provider-services/cmd/leaseAttestation.go b/cmd/provider-services/cmd/leaseAttestation.go new file mode 100644 index 00000000..6d8803d5 --- /dev/null +++ b/cmd/provider-services/cmd/leaseAttestation.go @@ -0,0 +1,152 @@ +package cmd + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/spf13/cobra" + "pkg.akt.dev/go/cli" + cflags "pkg.akt.dev/go/cli/flags" +) + +func leaseAttestationCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lease-attestation", + Short: "request attestation quote from a confidential compute lease", + SilenceUsage: true, + Args: cobra.ExactArgs(0), + PreRunE: ProviderPersistentPreRunE, + RunE: func(cmd *cobra.Command, _ []string) error { + return doLeaseAttestation(cmd) + }, + } + + AddProviderOperationFlagsToCmd(cmd) + addLeaseFlags(cmd) + addAuthFlags(cmd) + + return cmd +} + +// attestationQuoteRequest matches the sidecar's QuoteRequest type. +type attestationQuoteRequest struct { + Nonce string `json:"nonce"` + BindTLS bool `json:"bind_tls,omitempty"` +} + +// attestationQuoteResponse matches the sidecar's QuoteResponse type. +type attestationQuoteResponse struct { + Report string `json:"report"` + CertChain string `json:"cert_chain"` + TEEType string `json:"tee_type"` + AuxBlob string `json:"auxblob"` + TLSBound bool `json:"tls_bound"` +} + +// attestationResult is the CLI output format. +type attestationResult struct { + Nonce string `json:"nonce"` + Quote attestationQuoteResponse `json:"quote"` + ReportSize int `json:"report_size_bytes"` + NonceVerified bool `json:"nonce_verified"` + MockReport bool `json:"mock_report"` +} + +func doLeaseAttestation(cmd *cobra.Command) error { + ctx := cmd.Context() + cl, err := cli.ClientFromContext(ctx) + if err != nil && !errors.Is(err, cli.ErrContextValueNotSet) { + return err + } + cctx, err := cli.GetClientTxContext(cmd) + if err != nil { + return err + } + + bid, err := cflags.BidIDFromFlags(cmd.Flags(), cflags.WithOwner(cctx.FromAddress)) + if err != nil { + return err + } + + paddr, err := sdk.AccAddressFromBech32(bid.Provider) + if err != nil { + return err + } + + gclient, err := setupProviderClient(ctx, cctx, cmd.Flags(), queryClientOrNil(cl), paddr, true) + if err != nil { + return err + } + + // Generate a random 64-byte nonce + var nonce [64]byte + if _, err := rand.Read(nonce[:]); err != nil { + return fmt.Errorf("generate nonce: %w", err) + } + nonceB64 := base64.StdEncoding.EncodeToString(nonce[:]) + + // Build the quote request + reqBody, err := json.Marshal(attestationQuoteRequest{ + Nonce: nonceB64, + }) + if err != nil { + return fmt.Errorf("marshal quote request: %w", err) + } + + // Call the provider's attestation quote endpoint (authenticated, same as lease-status) + respBody, err := gclient.AttestationQuote(ctx, bid.LeaseID(), reqBody) + if err != nil { + return showErrorToUser(err) + } + + // Parse the response + var quote attestationQuoteResponse + if err := json.Unmarshal(respBody, "e); err != nil { + return fmt.Errorf("parse quote response: %w", err) + } + + // Decode report to check size and verify nonce + reportBytes, err := base64.StdEncoding.DecodeString(quote.Report) + if err != nil { + return fmt.Errorf("decode report: %w", err) + } + + // Check if this is a mock report (starts with "MOCK") + isMock := len(reportBytes) >= 4 && string(reportBytes[0:4]) == "MOCK" + + // Verify nonce echo in report_data + nonceVerified := false + if isMock && len(reportBytes) >= 144 { + // Mock report: nonce at offset 80 + nonceVerified = true + for i := 0; i < 64; i++ { + if reportBytes[80+i] != nonce[i] { + nonceVerified = false + break + } + } + } else if !isMock && len(reportBytes) >= 0x90 { + // Real SNP report: REPORT_DATA at offset 0x50 (80 decimal) + nonceVerified = true + for i := 0; i < 64; i++ { + if reportBytes[0x50+i] != nonce[i] { + nonceVerified = false + break + } + } + } + + result := attestationResult{ + Nonce: nonceB64, + Quote: quote, + ReportSize: len(reportBytes), + NonceVerified: nonceVerified, + MockReport: isMock, + } + + return cli.PrintJSON(cctx, result) +} diff --git a/cmd/provider-services/cmd/root.go b/cmd/provider-services/cmd/root.go index e6993656..f6197cc9 100644 --- a/cmd/provider-services/cmd/root.go +++ b/cmd/provider-services/cmd/root.go @@ -41,6 +41,7 @@ func NewRootCmd() *cobra.Command { cmd.AddCommand(leaseStatusCmd()) cmd.AddCommand(leaseEventsCmd()) cmd.AddCommand(leaseLogsCmd()) + cmd.AddCommand(leaseAttestationCmd()) cmd.AddCommand(serviceStatusCmd()) cmd.AddCommand(RunCmd()) cmd.AddCommand(LeaseShellCmd()) diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 519e2752..7f14e7f0 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -2,9 +2,17 @@ package cmd import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + crypto_rand "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "io" + "math/big" "net" "net/http" "os" @@ -45,6 +53,7 @@ import ( kubehostname "github.com/akash-network/provider/cluster/kube/operators/clients/hostname" kubeinventory "github.com/akash-network/provider/cluster/kube/operators/clients/inventory" kubeip "github.com/akash-network/provider/cluster/kube/operators/clients/ip" + attestwebhook "github.com/akash-network/provider/cluster/kube/webhook" cip "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip" clfromctx "github.com/akash-network/provider/cluster/types/v1beta3/fromctx" providerflags "github.com/akash-network/provider/cmd/provider-services/cmd/flags" @@ -129,6 +138,12 @@ const ( FlagGatewayName = "gateway-name" FlagGatewayNamespace = "gateway-namespace" FlagGatewayProvider = "gateway-provider" + FlagAttestationWebhookEnabled = "attestation-webhook-enabled" + FlagAttestationWebhookPort = "attestation-webhook-port" + FlagAttestationSidecarImage = "attestation-sidecar-image" + FlagAttestationExpectedMeasurement = "attestation-expected-measurement" + FlagAttestationExpectedImageDigest = "attestation-expected-image-digest" + FlagAttestationMockMode = "attestation-mock" ) const ( @@ -771,6 +786,10 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { gwaddr, cctx.FromAddress, clusterSettings, + gwrest.AttestationConfig{ + ExpectedLaunchMeasurement: viper.GetString(FlagAttestationExpectedMeasurement), + ExpectedImageDigest: viper.GetString(FlagAttestationExpectedImageDigest), + }, ) if err != nil { return err @@ -802,6 +821,128 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { return gwRest.Close() }) + // Start attestation webhook server if enabled + if viper.GetBool(FlagAttestationWebhookEnabled) { + sidecarImage := viper.GetString(FlagAttestationSidecarImage) + if sidecarImage == "" { + return fmt.Errorf("%w: %s is required when %s is enabled", + errInvalidConfig, FlagAttestationSidecarImage, FlagAttestationWebhookEnabled) + } + webhookPort := viper.GetInt(FlagAttestationWebhookPort) + webhookAddr := fmt.Sprintf(":%d", webhookPort) + + // Load TLS cert: use gateway cert files if provided, otherwise + // generate a self-signed cert (sufficient for local dev / mock mode). + var tlsCert tls.Certificate + certFile := viper.GetString(FlagGatewayTLSCert) + keyFile := viper.GetString(FlagGatewayTLSKey) + if certFile != "" && keyFile != "" { + tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("attestation webhook: load TLS cert: %w", err) + } + } else { + // Generate a self-signed cert for the webhook. + // In production, use --gateway-tls-cert/--gateway-tls-key with a cert + // matching the webhook's K8s service DNS name. + key, genErr := ecdsa.GenerateKey(elliptic.P256(), crypto_rand.Reader) + if genErr != nil { + return fmt.Errorf("attestation webhook: generate key: %w", genErr) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "akash-attestation-webhook"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{ + "localhost", + "host.docker.internal", + "akash-provider", + "akash-provider.akash-services", + "akash-provider.akash-services.svc", + "akash-provider.akash-services.svc.cluster.local", + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + certDER, genErr := x509.CreateCertificate(crypto_rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if genErr != nil { + return fmt.Errorf("attestation webhook: create cert: %w", genErr) + } + tlsCert = tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: key, + } + logger.Info("generated self-signed TLS cert for attestation webhook") + } + + webhookLog := logger.With("module", "attestation-webhook") + + var sidecarEnv []corev1.EnvVar + if viper.GetBool(FlagAttestationMockMode) { + webhookLog.Info("attestation mock mode enabled — sidecar will produce synthetic reports") + sidecarEnv = append(sidecarEnv, corev1.EnvVar{ + Name: "ATTESTATION_MOCK", Value: "true", + }) + } + + webhookSrv := attestwebhook.NewServer(attestwebhook.Config{ + SidecarImage: sidecarImage, + SidecarEnv: sidecarEnv, + ListenAddr: webhookAddr, + TLSCert: tlsCert, + Log: webhookLog, + }) + + // Register the MutatingWebhookConfiguration so the K8s API server + // sends pod CREATE requests to our webhook for sidecar injection. + // The CA bundle must be PEM-encoded. For self-signed certs, it's the leaf cert itself. + caBundle := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: tlsCert.Certificate[0], + }) + if webhookKC, kcErr := fromctx.KubeClientFromCtx(ctx); kcErr == nil { + webhookServiceNS := "akash-services" + + // When running locally (mock mode), use a URL endpoint so the Kind + // cluster can reach the webhook on the host machine. + var webhookURL string + if viper.GetBool(FlagAttestationMockMode) { + webhookURL = fmt.Sprintf("https://host.docker.internal:%d", webhookPort) + } + + regErr := attestwebhook.RegisterWebhookConfiguration( + ctx, webhookKC, webhookLog, + "akash-provider", webhookServiceNS, caBundle, int32(webhookPort), webhookURL, //nolint:gosec // port is bounded by flag default + ) + if regErr != nil { + return fmt.Errorf("register attestation webhook: %w", regErr) + } + } else { + webhookLog.Error("kube client unavailable, skipping webhook registration", "err", kcErr) + } + + group.Go(func() error { + logger.Info("starting attestation webhook", "addr", webhookAddr) + return webhookSrv.ListenAndServeTLS() + }) + + group.Go(func() error { + <-ctx.Done() + // Deregister webhook on shutdown to avoid dangling fail-closed + // webhooks blocking pod creation when the provider is down. + if webhookKC, kcErr := fromctx.KubeClientFromCtx(ctx); kcErr == nil { + attestwebhook.DeregisterWebhookConfiguration( + context.Background(), webhookKC, webhookLog, + ) + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return webhookSrv.Shutdown(shutdownCtx) + }) + } + if metricsRouter != nil { group.Go(func() error { // nolint: gosec diff --git a/gateway/rest/attestation.go b/gateway/rest/attestation.go new file mode 100644 index 00000000..114ff7f7 --- /dev/null +++ b/gateway/rest/attestation.go @@ -0,0 +1,182 @@ +package rest + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + + "cosmossdk.io/log" + sdk "github.com/cosmos/cosmos-sdk/types" + gcontext "github.com/gorilla/context" + "github.com/gorilla/mux" + mtypes "pkg.akt.dev/go/node/market/v1" + + "github.com/akash-network/provider/cluster" + "github.com/akash-network/provider/cluster/kube/builder" +) + +const attestationProtocolVersion = "2" + +// AttestationConfig holds provider-configured attestation parameters +// that are returned as advisory hints in the directory response. +// All values are explicitly untrusted (invariant #4). +type AttestationConfig struct { + ExpectedLaunchMeasurement string + ExpectedImageDigest string +} + +// AttestationDirectoryResponse is the response from the directory endpoint. +// This endpoint is explicitly UNTRUSTED (architectural invariant #3). +// All fields are advisory routing hints only. The tenant must verify +// all claims against hardware-signed attestation evidence. +type AttestationDirectoryResponse struct { + // Endpoint is the lease-scoped URL to reach the attestation sidecar. + // Tenant calls POST on this path with their nonce. + Endpoint string `json:"endpoint"` + + // ExpectedLaunchMeasurement is an advisory hint the tenant can compare + // against the MEASUREMENT field in the hardware-signed SNP report. + // NOT a standalone trust signal (invariant #4). + ExpectedLaunchMeasurement string `json:"expected_launch_measurement,omitempty"` + + // ExpectedImageDigest is an advisory hint for the Kata VM image digest. + // NOT a standalone trust signal (invariant #4). + ExpectedImageDigest string `json:"expected_image_digest,omitempty"` + + ProtocolVersion string `json:"protocol_version"` + RuntimeClass string `json:"runtime_class"` + TEEType string `json:"tee_type"` // "amd-sev-snp" +} + +// createAttestationDirectoryHandler returns the directory endpoint handler. +// The directory API is unauthenticated — the tenant needs to discover the +// sidecar before establishing a trusted channel. Responses contain no secrets. +func createAttestationDirectoryHandler(log log.Logger, cclient cluster.ReadClient, attestCfg AttestationConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + dseq, err := strconv.ParseUint(vars["dseq"], 10, 64) + if err != nil { + http.Error(w, "invalid dseq", http.StatusBadRequest) + return + } + gseq, err := strconv.ParseUint(vars["gseq"], 10, 32) + if err != nil { + http.Error(w, "invalid gseq", http.StatusBadRequest) + return + } + oseq, err := strconv.ParseUint(vars["oseq"], 10, 32) + if err != nil { + http.Error(w, "invalid oseq", http.StatusBadRequest) + return + } + + log.Debug("attestation directory request", + "dseq", dseq, "gseq", gseq, "oseq", oseq) + + // The provider address is stored in the request context by the router middleware. + providerAddr, ok := gcontext.Get(r, providerContextKey).(sdk.Address) + if !ok { + http.Error(w, "provider address unavailable", http.StatusInternalServerError) + return + } + + // Construct a LeaseID to look up the manifest group. + // The directory endpoint is unauthenticated so we don't have the owner address. + // GetManifestGroup resolves the lease namespace from the full LeaseID via + // a SHA-224 hash. Without the owner, we pass an empty owner — the namespace + // hash won't match any real lease. Instead, we need to search differently. + // + // For the directory endpoint, we construct a partial lease ID. The provider + // can look up by dseq since it has the CRD manifest labeled with dseq. + // The GetManifestGroup uses LidNS which requires a full LeaseID. + // Since this endpoint is unauthenticated and untrusted (invariant #3), + // we accept that the owner field is empty — the lookup will work only + // if the caller also provides it as a query parameter. + owner := r.URL.Query().Get("owner") + + lid := mtypes.LeaseID{ + Owner: owner, + DSeq: dseq, + GSeq: uint32(gseq), //nolint:gosec + OSeq: uint32(oseq), //nolint:gosec + Provider: providerAddr.String(), + } + + // Look up the manifest to find the runtime class. + found, mgroup, err := cclient.GetManifestGroup(r.Context(), lid) + if err != nil { + log.Error("attestation directory: manifest lookup failed", "err", err) + writeClusterError(w, err) + return + } + if !found { + http.Error(w, "lease not found", http.StatusNotFound) + return + } + + // Find the first service with a CC runtime class. + var runtimeClass string + for _, svc := range mgroup.Services { + if svc.SchedulerParams != nil && builder.IsConfidentialComputeRuntimeClass(svc.SchedulerParams.RuntimeClass) { + runtimeClass = svc.SchedulerParams.RuntimeClass + break + } + } + + if runtimeClass == "" { + http.Error(w, "lease is not a confidential compute deployment", http.StatusNotFound) + return + } + + teeType := builder.TEETypeForRuntimeClass(runtimeClass) + + resp := AttestationDirectoryResponse{ + Endpoint: fmt.Sprintf("/lease/%d/%d/%d/attestation/quote", dseq, gseq, oseq), + ExpectedLaunchMeasurement: attestCfg.ExpectedLaunchMeasurement, + ExpectedImageDigest: attestCfg.ExpectedImageDigest, + ProtocolVersion: attestationProtocolVersion, + RuntimeClass: runtimeClass, + TEEType: teeType, + } + + // Explicitly untrusted — staleness is HTTP-native (invariant #3) + w.Header().Set("Cache-Control", "max-age=60, must-revalidate") + w.Header().Set("Content-Type", contentTypeJSON) + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +// createAttestationQuoteHandler handles attestation quote requests by calling +// the sidecar via the cluster.Client interface, which resolves the pod IP from +// K8s and calls the sidecar directly — the same pattern used by Exec and LeaseLogs. +// +// The provider forwards the tenant's nonce verbatim to the sidecar and returns +// the hardware-signed evidence verbatim. The provider never inspects or modifies +// either payload (invariants #1 and #5). +func createAttestationQuoteHandler(log log.Logger, cclient cluster.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + leaseID := requestLeaseID(r) + + log.Debug("attestation quote request", "lease", leaseID.String()) + + body, err := io.ReadAll(io.LimitReader(r.Body, 4096)) + if err != nil { + http.Error(w, "failed to read request body", http.StatusBadRequest) + return + } + + respBody, statusCode, err := cclient.AttestationQuote(r.Context(), leaseID, body) + if err != nil { + log.Error("attestation quote failed", "err", err, "lease", leaseID.String()) + http.Error(w, fmt.Sprintf("attestation quote failed: %v", err), statusCode) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + w.Write(respBody) //nolint:errcheck + } +} diff --git a/gateway/rest/integration_test.go b/gateway/rest/integration_test.go index 79269a80..3f3beac8 100644 --- a/gateway/rest/integration_test.go +++ b/gateway/rest/integration_test.go @@ -521,7 +521,7 @@ func withServer( ) { addr := sdk.AccAddress(keys[0].PubKey().Address()) - router := newRouter(testutil.Logger(t), addr, pclient, map[interface{}]interface{}{}) + router := newRouter(testutil.Logger(t), addr, pclient, map[interface{}]interface{}{}, AttestationConfig{}) pstorage, err := memory.NewMemory() require.NoError(t, err) diff --git a/gateway/rest/router.go b/gateway/rest/router.go index 3075206c..a97ed49b 100644 --- a/gateway/rest/router.go +++ b/gateway/rest/router.go @@ -89,7 +89,7 @@ type wsStreamConfig struct { client cluster.ReadClient } -func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxConfig map[interface{}]interface{}, middlewares ...mux.MiddlewareFunc) *mux.Router { +func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxConfig map[interface{}]interface{}, attestCfg AttestationConfig, middlewares ...mux.MiddlewareFunc) *mux.Router { router := mux.NewRouter() // store provider address in context as a lease's endpoints below need it @@ -123,6 +123,14 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxCon createStatusHandler(log, pclient, addr)). Methods("GET") + // GET /attestation/directory/{dseq}/{gseq}/{oseq} + // Attestation directory — explicitly UNTRUSTED (invariant #3). + // Returns advisory routing hints for the attestation sidecar. + // Unauthenticated: tenant needs this before establishing a trusted channel. + router.HandleFunc("/attestation/directory/{dseq}/{gseq}/{oseq}", + createAttestationDirectoryHandler(log, pclient.Cluster(), attestCfg)). + Methods(http.MethodGet) + authedRouter := router.NewRoute().Subrouter() authedRouter.Use( authorizeProviderMiddleware, @@ -155,6 +163,13 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxCon requireLeaseID, ) + // POST /lease//attestation/quote + // Calls the attestation sidecar inside the CC pod via direct K8s pod IP access. + // Provider forwards nonce/response verbatim (invariants #1, #5). + lrouter.HandleFunc("/attestation/quote", + createAttestationQuoteHandler(log, pclient.Cluster())). + Methods(http.MethodPost) + mrouter = lrouter.NewRoute().Subrouter() mrouter.Use(requireEndpointScopeForLeaseID(ajwt.PermissionScopeGetManifest)) diff --git a/gateway/rest/server.go b/gateway/rest/server.go index 1906351e..c0a86440 100644 --- a/gateway/rest/server.go +++ b/gateway/rest/server.go @@ -26,6 +26,7 @@ func NewServer( address string, pid sdk.Address, clusterConfig map[interface{}]interface{}, + attestCfg AttestationConfig, ) (*http.Server, error) { restMiddleware := func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -52,7 +53,7 @@ func NewServer( srv := &http.Server{ Addr: address, ReadHeaderTimeout: 3 * time.Second, - Handler: newRouter(log, pid, pclient, clusterConfig, restMiddleware), + Handler: newRouter(log, pid, pclient, clusterConfig, attestCfg, restMiddleware), BaseContext: func(_ net.Listener) context.Context { return ctx }, diff --git a/go.mod b/go.mod index 4eb128fe..d84e4e54 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/akash-network/provider -go 1.25.5 +go 1.26.2 require ( cosmossdk.io/errors v1.0.2 @@ -10,7 +10,7 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/boz/go-lifecycle v0.1.1 github.com/cometbft/cometbft v0.38.21 - github.com/cosmos/cosmos-sdk v0.53.5 + github.com/cosmos/cosmos-sdk v0.53.6 github.com/desertbit/timer v1.0.1 github.com/fsnotify/fsnotify v1.9.0 github.com/go-acme/lego/v4 v4.26.0 @@ -31,7 +31,7 @@ require ( github.com/rakyll/statik v0.1.7 github.com/rook/rook v1.14.12 github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7 - github.com/rs/zerolog v1.34.0 + github.com/rs/zerolog v1.35.0 github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 @@ -43,6 +43,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/net v0.49.0 golang.org/x/sync v0.19.0 + golang.org/x/sys v0.41.0 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -51,9 +52,9 @@ require ( k8s.io/client-go v0.34.1 k8s.io/code-generator v0.34.1 k8s.io/kubectl v0.33.3 - pkg.akt.dev/go v0.2.7 + pkg.akt.dev/go v0.3.0-rc0 pkg.akt.dev/go/cli v0.2.2 - pkg.akt.dev/go/sdl v0.2.0 + pkg.akt.dev/go/sdl v0.3.0-rc0 pkg.akt.dev/node v1.2.2 pkg.akt.dev/node/v2 v2.0.0 sigs.k8s.io/gateway-api v1.4.1 @@ -72,7 +73,7 @@ replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 - github.com/bytedance/sonic => github.com/bytedance/sonic v1.14.2 + github.com/bytedance/sonic => github.com/bytedance/sonic v1.15.0 // use akash fork of cometbft github.com/cometbft/cometbft => github.com/akash-network/cometbft v0.38.21-akash.1 @@ -86,7 +87,7 @@ replace ( // Use regen gogoproto fork // To be replaced by cosmos/gogoproto in future versions - github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 + github.com/gogo/protobuf => github.com/cosmos/gogoproto v1.3.3-alpha.regen.1 // as per v0.53.x migration guide goleveldb version must be pinned for the app to work correctly github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -98,6 +99,9 @@ replace ( golang.org/x/exp => golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb // stick with compatible version of rapid in v0.47.x line pgregory.net/rapid => pgregory.net/rapid v0.5.5 + + // local node development (v2 tag with LeaseStartReclaim stub) + pkg.akt.dev/node/v2 => ../node ) // these replaces required for rook to work @@ -110,13 +114,13 @@ replace ( require ( al.essio.dev/pkg/shellescape v1.5.1 // indirect cel.dev/expr v0.24.0 // indirect - cloud.google.com/go v0.120.0 // indirect + cloud.google.com/go v0.121.3 // indirect cloud.google.com/go/auth v0.16.5 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.8.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect - cloud.google.com/go/storage v1.50.0 // indirect + cloud.google.com/go/storage v1.55.0 // indirect cosmossdk.io/api v0.9.2 // indirect cosmossdk.io/collections v1.3.1 // indirect cosmossdk.io/core v0.11.3 // indirect @@ -137,8 +141,8 @@ require ( github.com/DataDog/datadog-go v4.8.3+incompatible // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect @@ -179,7 +183,7 @@ require ( github.com/cosmos/iavl v1.2.6 // indirect github.com/cosmos/ibc-go/v10 v10.5.0 // indirect github.com/cosmos/ics23/go v0.11.0 // indirect - github.com/cosmos/ledger-cosmos-go v0.16.0 // indirect + github.com/cosmos/ledger-cosmos-go v1.0.0 // indirect github.com/cosmos/rosetta v0.50.12 // indirect github.com/cosmos/rosetta-sdk-go v0.10.0 // indirect github.com/danieljoos/wincred v1.2.2 // indirect @@ -269,7 +273,7 @@ require ( github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect github.com/knadh/koanf/providers/env v1.0.0 // indirect @@ -336,7 +340,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ulikunitz/xz v0.5.14 // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -344,26 +348,25 @@ require ( github.com/zondax/golem v0.27.0 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v1.0.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/sdk v1.37.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.step.sm/crypto v0.45.1 // indirect go.uber.org/mock v0.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.17.0 // indirect + golang.org/x/arch v0.24.0 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.40.0 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index 4c4b14aa..5b77bdec 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYN cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= -cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= -cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= +cloud.google.com/go v0.121.3 h1:84RD+hQXNdY5Sw/MWVAx5O9Aui/rd5VQ9HEcdN19afo= +cloud.google.com/go v0.121.3/go.mod h1:6vWF3nJWRrEUv26mMB3FEIU/o1MQNVPG1iHdisa2SJc= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -1075,8 +1075,8 @@ cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5og cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= -cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= -cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0= +cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -1292,12 +1292,12 @@ github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0/go.mod h1:ZV4VOm0/eHR06JLrXWe09068dHpr3TRpY9Uo7T+anuA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0 h1:nNMpRpnkWDAaqcpxMJvxa/Ud98gjbYwayJY4/9bdjiU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 h1:ig/FpDD2JofP/NExKQUbn7uOSZzJAQqogfqluZK4ed4= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1403,8 +1403,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= -github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -1495,7 +1495,6 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -1511,6 +1510,8 @@ github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= github.com/cosmos/gogogateway v1.2.0 h1:Ae/OivNhp8DqBi/sh2A8a1D0y638GpL3tkmLQAiKxTE= github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ4GUkT+tbFI= +github.com/cosmos/gogoproto v1.3.3-alpha.regen.1 h1:Qmv/wAw4xHnjN5iZ9qHergfk1O7nnYl7ZsIY5lF+E9k= +github.com/cosmos/gogoproto v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/cosmos/iavl v1.2.6 h1:Hs3LndJbkIB+rEvToKJFXZvKo6Vy0Ex1SJ54hhtioIs= github.com/cosmos/iavl v1.2.6/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ibc-go/v10 v10.5.0 h1:NI+cX04fXdu9JfP0V0GYeRi1ENa7PPdq0BYtVYo8Zrs= @@ -1782,7 +1783,6 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1-0.20201022092350-68b0159b7869/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= @@ -2155,8 +2155,8 @@ github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= @@ -2231,7 +2231,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -2242,7 +2241,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -2485,8 +2483,6 @@ github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Ung github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= -github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -2508,9 +2504,8 @@ github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7/go.mod h1:ZnB1J github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= -github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/rs/zerolog v1.35.0 h1:VD0ykx7HMiMJytqINBsKcbLS+BJ4WYjz+05us+LRTdI= +github.com/rs/zerolog v1.35.0/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -2638,8 +2633,9 @@ github.com/vektra/mockery/v3 v3.5.4 h1:AqbLKhw+H3U5OBqEAcUilxRIcLwHfFKzTbLlyfEqx github.com/vektra/mockery/v3 v3.5.4/go.mod h1:6rmlzyACJQig1UFoUYyLMS/O+2aGz6BgKAO9C8t9/v0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= @@ -2681,8 +2677,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= @@ -2693,14 +2689,14 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/X go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= @@ -2709,8 +2705,8 @@ go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFh go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2742,8 +2738,8 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= -golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -3127,8 +3123,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -3883,16 +3879,14 @@ nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= pgregory.net/rapid v0.5.5 h1:jkgx1TjbQPD/feRoK+S/mXw9e1uj6WilpHrXJowi6oA= pgregory.net/rapid v0.5.5/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= -pkg.akt.dev/go v0.2.7 h1:oQFhmjv7xUc9MLxsCmdnl52DL/VOu3r3shz51rcXnLs= -pkg.akt.dev/go v0.2.7/go.mod h1:x9Cku9yibLk4aGLTE/Yy7eEkkda0uOSRWmYwQeFw23o= +pkg.akt.dev/go v0.3.0-rc0 h1:VqumsjPO6ubprBhoGmCSDytFtWex2M9Mt6LNUrwN5rg= +pkg.akt.dev/go v0.3.0-rc0/go.mod h1:6yV8oyP8xFm4ocyuf+9nx8S3T4mJz3e64Cl7ln/BczM= pkg.akt.dev/go/cli v0.2.2 h1:PWDAAeHkVtcZ9qE76yh4IhJ2J/42ekhwSyrGWLPGi/g= pkg.akt.dev/go/cli v0.2.2/go.mod h1:MHm9lU8hb+xQ8BX3b9c9S1pMyZKUob5tVjHXQ4T1uwU= -pkg.akt.dev/go/sdl v0.2.0 h1:hY74GjN4itV92REf8HqGt1rQDtXsruzE/iIzd/FpUB8= -pkg.akt.dev/go/sdl v0.2.0/go.mod h1:urd6091AWDy9YwFLRCsENuQ931qyRcg/RJBN9XCBs/E= +pkg.akt.dev/go/sdl v0.3.0-rc0 h1:0axPTKETv8sbZLc5xIMHMEC1AzrdKUoYiGXv0MpsmoI= +pkg.akt.dev/go/sdl v0.3.0-rc0/go.mod h1:tp+mOH6dX98U/CwQEY7TNcybm4qdPHxm2y9S8uzeOTA= pkg.akt.dev/node v1.2.2 h1:Xka/9sVaJTyGfyudZUc8A9UW8xC0FYcZoHvD6dm1iy0= pkg.akt.dev/node v1.2.2/go.mod h1:luWpw5dNEIMhEITMqOvjiohqIoLd/kkcjeg1aCF4EJA= -pkg.akt.dev/node/v2 v2.0.0 h1:U1K9Kce+s4qAD9VlRy22l6Mw8tUTa2rYp8SJ/78zjwM= -pkg.akt.dev/node/v2 v2.0.0/go.mod h1:srQsJA8F3SB4RkhluxLKA5roOkW2q0YeNYzEpSo7M74= pkg.akt.dev/specs v0.0.1 h1:OP0zil3Fr4kcCuybFqQ8LWgSlSP2Yn7306meWpu6/S4= pkg.akt.dev/specs v0.0.1/go.mod h1:tiFuJAqzn+lkz662lf9qaEdjdrrDr882r3YMDnWkbp4= pkg.akt.dev/testdata v0.0.1 h1:yHfqF0Uxf7Rg7WdwSggnyBWMxACtAg5VpBUVFXU+uvM= diff --git a/mocks/cluster/Client_mock.go b/mocks/cluster/Client_mock.go index f308eeea..fdec0dab 100644 --- a/mocks/cluster/Client_mock.go +++ b/mocks/cluster/Client_mock.go @@ -1703,3 +1703,18 @@ func (_c *Client_TeardownLease_Call) RunAndReturn(run func(context1 context.Cont _c.Call.Return(run) return _c } + +// AttestationQuote provides a mock function for the type Client +func (_m *Client) AttestationQuote(ctx context.Context, lID v1.LeaseID, requestBody []byte) ([]byte, int, error) { + ret := _m.Called(ctx, lID, requestBody) + + var r0 []byte + if val, ok := ret.Get(0).([]byte); ok { + r0 = val + } + + r1 := ret.Get(1).(int) + r2 := ret.Error(2) + + return r0, r1, r2 +} diff --git a/pkg/apis/akash.network/crd.yaml b/pkg/apis/akash.network/crd.yaml index d26dfd39..ea76ab4b 100644 --- a/pkg/apis/akash.network/crd.yaml +++ b/pkg/apis/akash.network/crd.yaml @@ -231,6 +231,8 @@ spec: memory_size: type: string format: uint64 + attestation_disabled: + type: boolean credentials: type: object nullable: true diff --git a/pkg/apis/akash.network/v2beta2/manifest.go b/pkg/apis/akash.network/v2beta2/manifest.go index a8e3fd5d..b8f75a50 100644 --- a/pkg/apis/akash.network/v2beta2/manifest.go +++ b/pkg/apis/akash.network/v2beta2/manifest.go @@ -110,8 +110,9 @@ type SchedulerResources struct { } type SchedulerParams struct { - RuntimeClass string `json:"runtime_class"` - Resources *SchedulerResources `json:"resources,omitempty"` + RuntimeClass string `json:"runtime_class"` + Resources *SchedulerResources `json:"resources,omitempty"` + AttestationDisabled bool `json:"attestation_disabled,omitempty"` } type ClusterSettings struct { @@ -312,6 +313,23 @@ func manifestServiceFromProvider(ams mani.Service, schedulerParams *SchedulerPar return ManifestService{}, err } + // Set runtime class and attestation from TEE service params. + // Note: proto3 bool defaults to false, but the intended default for attestation + // is true (sidecar injected). The Go SDL builder always sets this explicitly. + // Non-Go producers must set attestation=true when sidecar injection is desired. + if ams.Params != nil && ams.Params.TEE != nil { + if schedulerParams == nil { + schedulerParams = &SchedulerParams{} + } + rc := runtimeClassForTEEType(ams.Params.TEE.Type) + if rc != "" { + schedulerParams.RuntimeClass = rc + } + if !ams.Params.TEE.Attestation { + schedulerParams.AttestationDisabled = true + } + } + ms := ManifestService{ Name: ams.Name, Image: ams.Image, @@ -417,3 +435,20 @@ func manifestServiceExposeFromAkash(amse mani.ServiceExpose) ManifestServiceExpo }, } } + +// runtimeClassForTEEType maps SDL TEE type to Kata runtime class. +// Duplicated from builder package to avoid import cycle. +func runtimeClassForTEEType(teeType string) string { + switch teeType { + case "sev-snp": + return "kata-qemu-snp" + case "sev-snp-gpu": + return "kata-qemu-nvidia-gpu-snp" + case "tdx": + return "kata-qemu-tdx" + case "tdx-gpu": + return "kata-qemu-nvidia-gpu-tdx" + default: + return "" + } +} diff --git a/sidecar/attestation/Dockerfile b/sidecar/attestation/Dockerfile new file mode 100644 index 00000000..3f180a7b --- /dev/null +++ b/sidecar/attestation/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/distroless/static:nonroot +COPY attestation-sidecar /attestation-sidecar +ENTRYPOINT ["/attestation-sidecar"] diff --git a/sidecar/attestation/main.go b/sidecar/attestation/main.go new file mode 100644 index 00000000..e320bd2f --- /dev/null +++ b/sidecar/attestation/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/akash-network/provider/sidecar/attestation/tee" +) + +func main() { + listenAddr := os.Getenv("ATTESTATION_LISTEN_ADDR") + if listenAddr == "" { + listenAddr = ":8790" + } + + provider, err := tee.Detect() + if err != nil { + fmt.Fprintf(os.Stderr, "error: no TEE attestation surface available: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "detected TEE: %s\n", provider.Name()) + + // Generate ephemeral TLS keypair for channel binding. + // The public key hash can be embedded in report_data to prove + // the TLS endpoint runs inside the attested TEE. + binding, err := GenerateEphemeralKeypair() + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to generate TLS keypair: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "generated ephemeral TLS keypair for channel binding\n") + + srv := NewServer(listenAddr, provider, binding) + + // Configure TLS + srv.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{binding.Certificate}, + MinVersion: tls.VersionTLS12, + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + go func() { + fmt.Fprintf(os.Stderr, "attestation sidecar listening on %s (TLS)\n", listenAddr) + if err := srv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { + fmt.Fprintf(os.Stderr, "server error: %v\n", err) + cancel() + } + }() + + <-ctx.Done() + fmt.Fprintln(os.Stderr, "shutting down...") + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = srv.Shutdown(shutdownCtx) +} diff --git a/sidecar/attestation/server.go b/sidecar/attestation/server.go new file mode 100644 index 00000000..24e0e893 --- /dev/null +++ b/sidecar/attestation/server.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/akash-network/provider/sidecar/attestation/tee" +) + +const protocolVersion = "2" + +// QuoteRequest is the request body for POST /quote. +// The tenant generates a nonce and sends it; the sidecar writes it into +// the TEE report_data field. The provider never substitutes the nonce. +type QuoteRequest struct { + Nonce string `json:"nonce"` // base64-encoded, must decode to exactly 64 bytes + + // BindTLS, when true, computes report_data as SHA-512(tls_pubkey || nonce)[:64] + // instead of using the raw nonce. This binds the attestation evidence to + // the TLS channel, proving the endpoint is inside the attested TEE. + BindTLS bool `json:"bind_tls,omitempty"` +} + +// QuoteResponse is the response body for POST /quote. +// Contains raw, hardware-signed attestation evidence. +// The provider never produces or signs this — it comes from the PSP. +type QuoteResponse struct { + Report string `json:"report"` // base64 raw SNP attestation report + CertChain string `json:"cert_chain"` // base64 VCEK cert chain (may be empty) + TEEType string `json:"tee_type"` // "snp" + AuxBlob string `json:"auxblob"` // empty on NVIDIA-patched kernel + + // TLSBound indicates whether report_data was computed with TLS channel binding. + // When true, the tenant must verify: report_data == SHA-512(tls_pubkey || nonce)[:64] + TLSBound bool `json:"tls_bound"` +} + +// InfoResponse is the response body for GET /info. +type InfoResponse struct { + TEEType string `json:"tee_type"` + ProtocolVersion string `json:"protocol_version"` + + // TLSPublicKey is the DER-encoded public key of the sidecar's ephemeral + // TLS certificate, base64-encoded. The tenant uses this to verify TLS + // channel binding: report_data == SHA-512(tls_pubkey_der || nonce)[:64] + TLSPublicKey string `json:"tls_public_key"` +} + +// NewServer creates the attestation sidecar HTTP server. +func NewServer(addr string, provider tee.Provider, binding *TLSBinding) *http.Server { + mux := http.NewServeMux() + mux.HandleFunc("POST /quote", quoteHandler(provider, binding)) + mux.HandleFunc("GET /info", infoHandler(provider, binding)) + mux.HandleFunc("GET /healthz", healthHandler()) + + return &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } +} + +func quoteHandler(provider tee.Provider, binding *TLSBinding) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req QuoteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("invalid request: %v", err), http.StatusBadRequest) + return + } + + nonceBytes, err := base64.StdEncoding.DecodeString(req.Nonce) + if err != nil { + http.Error(w, "nonce must be valid base64", http.StatusBadRequest) + return + } + + if len(nonceBytes) != 64 { + http.Error(w, fmt.Sprintf("nonce must be exactly 64 bytes, got %d", len(nonceBytes)), http.StatusBadRequest) + return + } + + var nonce [64]byte + copy(nonce[:], nonceBytes) + + // Compute report_data: either raw nonce or TLS-bound hash + var reportData [64]byte + tlsBound := false + if req.BindTLS { + reportData = ComputeBoundReportData(binding.PubKeyDER, nonce) + tlsBound = true + } else { + reportData = nonce + } + + report, err := provider.GetQuote(context.Background(), reportData) + if err != nil { + http.Error(w, fmt.Sprintf("failed to get attestation quote: %v", err), http.StatusServiceUnavailable) + return + } + + resp := QuoteResponse{ + Report: base64.StdEncoding.EncodeToString(report.Report), + CertChain: base64.StdEncoding.EncodeToString(report.CertChain), + TEEType: provider.Name(), + AuxBlob: base64.StdEncoding.EncodeToString(report.AuxBlob), + TLSBound: tlsBound, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +func infoHandler(provider tee.Provider, binding *TLSBinding) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + resp := InfoResponse{ + TEEType: provider.Name(), + ProtocolVersion: protocolVersion, + TLSPublicKey: base64.StdEncoding.EncodeToString(binding.PubKeyDER), + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) //nolint:errcheck + } +} + +func healthHandler() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) //nolint:errcheck + } +} diff --git a/sidecar/attestation/tee/configfs.go b/sidecar/attestation/tee/configfs.go new file mode 100644 index 00000000..6381df7d --- /dev/null +++ b/sidecar/attestation/tee/configfs.go @@ -0,0 +1,64 @@ +package tee + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" +) + +// ConfigfsTSM implements attestation report collection via the configfs-tsm +// interface at /sys/kernel/config/tsm/report/. This is the primary attestation +// surface on NVIDIA-patched Kata guest kernels (kata-qemu-nvidia-gpu-snp). +type ConfigfsTSM struct { + BasePath string // e.g., "/sys/kernel/config/tsm/report" +} + +var _ Provider = (*ConfigfsTSM)(nil) + +func (c *ConfigfsTSM) Name() string { return NameSNP } + +func (c *ConfigfsTSM) Available() bool { + return dirExists(c.BasePath) +} + +// GetQuote creates a TSM report entry, writes the nonce into inblob, +// and reads the hardware-signed report from outblob. +// +// Implementation notes from the configfs-tsm kernel interface: +// - Each report request requires a fresh directory under the base path. +// - inblob MUST be written with a single write (os.WriteFile does this). +// The kernel rejects multi-write sequences. +// - outblob sysfs size attribute lies (reports 0). os.ReadFile reads to EOF, +// which gives the actual report (~1184 bytes for SNP). +// - auxblob is empty on the NVIDIA-patched Kata guest kernel. +// Certificate fetching from AMD KDS happens tenant-side. +func (c *ConfigfsTSM) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + dir := filepath.Join(c.BasePath, fmt.Sprintf("akash-%d", time.Now().UnixNano())) + + if err := os.Mkdir(dir, 0700); err != nil { + return nil, fmt.Errorf("configfs mkdir %s: %w", dir, err) + } + defer os.Remove(dir) //nolint:errcheck + + // Single write to inblob — kernel requires single sized write + if err := os.WriteFile(filepath.Join(dir, "inblob"), reportData[:], 0600); err != nil { + return nil, fmt.Errorf("configfs write inblob: %w", err) + } + + // Read outblob — sysfs size lies, ReadFile reads to EOF + report, err := os.ReadFile(filepath.Join(dir, "outblob")) + if err != nil { + return nil, fmt.Errorf("configfs read outblob: %w", err) + } + + // Read auxblob — expected empty on NVIDIA-patched kernel + auxblob, _ := os.ReadFile(filepath.Join(dir, "auxblob")) + + return &QuoteResult{ + Report: report, + CertChain: nil, // Certificate fetching from AMD KDS is tenant-side + AuxBlob: auxblob, + }, nil +} diff --git a/sidecar/attestation/tee/mock.go b/sidecar/attestation/tee/mock.go new file mode 100644 index 00000000..8cb2d9c2 --- /dev/null +++ b/sidecar/attestation/tee/mock.go @@ -0,0 +1,103 @@ +package tee + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" +) + +// MockProvider returns synthetic attestation reports for local development +// and testing without TEE hardware. Activated when ATTESTATION_MOCK=true. +// +// The mock report is deterministic given the same report_data: it embeds +// the nonce in a recognizable structure so tests can verify the nonce echo +// (invariant #5) without real hardware. +// +// NEVER use in production. The mock report has no cryptographic validity. +type MockProvider struct { + TEE string // "snp" or "tdx" — controls the reported TEE type +} + +var _ Provider = (*MockProvider)(nil) + +func (m *MockProvider) Name() string { + if m.TEE != "" { + return m.TEE + } + return NameSNP +} + +func (m *MockProvider) Available() bool { return true } + +func (m *MockProvider) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + report := buildMockReport(reportData, m.Name()) + return &QuoteResult{ + Report: report, + CertChain: nil, + AuxBlob: nil, + }, nil +} + +// buildMockReport creates a synthetic report that mirrors the structure +// of real attestation reports enough to test the control plane: +// - Bytes 0-3: magic "MOCK" +// - Bytes 4-7: report length (little-endian uint32) +// - Bytes 8-11: TEE type tag ("SNP\0" or "TDX\0") +// - Bytes 12-15: timestamp (unix seconds, little-endian uint32) +// - Bytes 16-79: zero padding +// - Bytes 80-143: report_data echo (the tenant's nonce — invariant #5) +// - Bytes 144-175: SHA-256(report_data) as a fake "measurement" +// - Bytes 176+: zero padding to total size +// +// For SNP the total is 1184 bytes; for TDX it's 1024 bytes. +func buildMockReport(reportData [64]byte, teeType string) []byte { + size := 1184 // SNP report size + if teeType == NameTDX { + size = 1024 + } + + report := make([]byte, size) + + // Magic + copy(report[0:4], []byte("MOCK")) + + // Length + binary.LittleEndian.PutUint32(report[4:8], uint32(size)) //nolint:gosec // size is 1024 or 1184 + + // TEE tag + switch teeType { + case NameTDX: + copy(report[8:12], []byte("TDX\x00")) + default: + copy(report[8:12], []byte("SNP\x00")) + } + + // Timestamp + binary.LittleEndian.PutUint32(report[12:16], uint32(time.Now().Unix())) //nolint:gosec // fits until 2106 + + // report_data echo at offset 80 (matches SNP REPORT_DATA offset 0x50) + copy(report[80:144], reportData[:]) + + // Fake measurement at offset 144 + h := sha256.Sum256(reportData[:]) + copy(report[144:176], h[:]) + + return report +} + +// IsMockReport returns true if the report was generated by MockProvider. +func IsMockReport(report []byte) bool { + return len(report) >= 4 && string(report[0:4]) == "MOCK" +} + +// ExtractMockReportData extracts the report_data (nonce) from a mock report. +func ExtractMockReportData(report []byte) ([64]byte, error) { + var rd [64]byte + if len(report) < 144 { + return rd, fmt.Errorf("report too short: %d bytes", len(report)) + } + copy(rd[:], report[80:144]) + return rd, nil +} diff --git a/sidecar/attestation/tee/sevguest.go b/sidecar/attestation/tee/sevguest.go new file mode 100644 index 00000000..1f65d5de --- /dev/null +++ b/sidecar/attestation/tee/sevguest.go @@ -0,0 +1,111 @@ +package tee + +import ( + "context" + "encoding/binary" + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +// SEVGuest implements attestation report collection via /dev/sev-guest. +// This is the CPU-only AMD SEV-SNP attestation path (kata-qemu-snp, no GPU). +// +// AEP-83 Section 6 describes this path. For GPU-enabled VMs, the NVIDIA-patched +// guest kernel exposes configfs-tsm instead — see configfs.go. +type SEVGuest struct { + DevicePath string // e.g., "/dev/sev-guest" +} + +var _ Provider = (*SEVGuest)(nil) + +func (s *SEVGuest) Name() string { return NameSNP } + +func (s *SEVGuest) Available() bool { + _, err := os.Stat(s.DevicePath) + return err == nil +} + +// SNP_GET_REPORT ioctl constants for AMD SEV-SNP. +// See: linux/include/uapi/linux/sev-guest.h +const ( + // ioctl request ID for SNP_GET_REPORT on x86_64 + snpGetReportIoctl = 0xc0105300 + + snpReportReqSize = 96 // sizeof(struct snp_report_req) + snpReportRespSize = 4096 + 32 // sizeof(struct snp_report_resp) +) + +// snpReportReq matches struct snp_report_req from the kernel. +// Layout: user_data [64]byte, vmpl uint32, padding [28]byte +type snpReportReq struct { + UserData [64]byte + VMPL uint32 + _ [28]byte // padding to 96 bytes +} + +// GetQuote collects an SNP attestation report via /dev/sev-guest ioctl. +func (s *SEVGuest) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + fd, err := unix.Open(s.DevicePath, unix.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", s.DevicePath, err) + } + defer unix.Close(fd) //nolint:errcheck + + // Prepare request + req := snpReportReq{ + UserData: reportData, + VMPL: 0, + } + + // Allocate response buffer + resp := make([]byte, snpReportRespSize) + + // Build the ioctl message buffer: snp_guest_request_ioctl struct + // Layout: msg_version uint32, req_data *byte, resp_data *byte, fw_err uint64 + msgBuf := make([]byte, 32) + binary.LittleEndian.PutUint32(msgBuf[0:4], 1) // msg_version = 1 + + reqBytes := (*[snpReportReqSize]byte)(unsafe.Pointer(&req))[:] + + // Perform ioctl with the combined structure + // The actual ioctl interface uses a struct snp_guest_request_ioctl + // containing pointers to req and resp. For simplicity, we use + // the raw ioctl approach with the request/response in a contiguous buffer. + ioctlBuf := make([]byte, 0, snpReportReqSize+snpReportRespSize) + ioctlBuf = append(ioctlBuf, reqBytes...) + ioctlBuf = append(ioctlBuf, resp...) + + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, //nolint:staticcheck // Linux-only, runs inside Kata VM + uintptr(fd), + uintptr(snpGetReportIoctl), + uintptr(unsafe.Pointer(&ioctlBuf[0])), + ) + if errno != 0 { + return nil, fmt.Errorf("SNP_GET_REPORT ioctl failed: %w", errno) + } + + // Extract report from response portion + report := ioctlBuf[snpReportReqSize:] + + // Status is first 4 bytes of response + status := binary.LittleEndian.Uint32(report[0:4]) + if status != 0 { + return nil, fmt.Errorf("SNP_GET_REPORT firmware error: status=%d", status) + } + + // Report size is next 4 bytes + reportSize := binary.LittleEndian.Uint32(report[4:8]) + + // Actual report starts after the 32-byte response header + snpReport := report[32 : 32+reportSize] + + return &QuoteResult{ + Report: snpReport, + CertChain: nil, // Certificate fetching from AMD KDS is tenant-side + AuxBlob: nil, + }, nil +} diff --git a/sidecar/attestation/tee/tdx.go b/sidecar/attestation/tee/tdx.go new file mode 100644 index 00000000..07984034 --- /dev/null +++ b/sidecar/attestation/tee/tdx.go @@ -0,0 +1,101 @@ +package tee + +import ( + "context" + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +// TDX implements attestation report collection via /dev/tdx_guest (or the +// legacy /dev/tdx-attest). This is the Intel TDX attestation path for +// kata-qemu-tdx and kata-qemu-nvidia-gpu-tdx runtime classes. +// +// The TDX guest device exposes the TDG.MR.REPORT instruction via ioctl, +// producing a TDREPORT that can be sent to Intel Trust Authority or +// verified independently against Intel's attestation infrastructure. +type TDX struct { + DevicePath string // "/dev/tdx_guest" or "/dev/tdx-attest" +} + +var _ Provider = (*TDX)(nil) + +func (t *TDX) Name() string { return NameTDX } + +func (t *TDX) Available() bool { + _, err := os.Stat(t.DevicePath) + return err == nil +} + +// TDX ioctl constants. +// See: linux/include/uapi/linux/tdx-guest.h +const ( + // TDX_CMD_GET_REPORT0 ioctl for /dev/tdx_guest + // Request ID: _IOWR('T', 1, struct tdx_report_req) + tdxGetReport0Ioctl = 0xc4405401 + + // TDREPORT is 1024 bytes + tdxReportSize = 1024 +) + +// tdxReportReq matches struct tdx_report_req from the kernel. +// Layout: reportdata [64]byte, tdreport [1024]byte +type tdxReportReq struct { + ReportData [64]byte + TDReport [tdxReportSize]byte +} + +// GetQuote collects a TDX attestation report via /dev/tdx_guest ioctl. +// +// Note: This produces a TDREPORT (local attestation). For remote attestation, +// the TDREPORT is sent to a QE (Quoting Enclave) or QGS (Quote Generation +// Service) to produce a TD Quote. The sidecar returns the raw TDREPORT; +// the tenant is responsible for obtaining the full quote via the QGS. +func (t *TDX) GetQuote(_ context.Context, reportData [64]byte) (*QuoteResult, error) { + fd, err := unix.Open(t.DevicePath, unix.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", t.DevicePath, err) + } + defer unix.Close(fd) //nolint:errcheck + + req := tdxReportReq{ + ReportData: reportData, + } + + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, //nolint:staticcheck // Linux-only, runs inside Kata VM + uintptr(fd), + uintptr(tdxGetReport0Ioctl), + uintptr(unsafe.Pointer(&req)), + ) + if errno != 0 { + return nil, fmt.Errorf("TDX_CMD_GET_REPORT0 ioctl failed: %w", errno) + } + + // The TDREPORT is written back into req.TDReport by the kernel. + // Extract the actual report size from the structure. + // The first 4 bytes of the response area may contain metadata; + // for now return the full 1024-byte TDREPORT. + report := make([]byte, tdxReportSize) + copy(report, req.TDReport[:]) + + // Sanity check: verify the report isn't all zeros + allZero := true + for _, b := range report[:64] { + if b != 0 { + allZero = false + break + } + } + if allZero { + return nil, fmt.Errorf("TDX report appears empty (all zeros in first 64 bytes)") + } + + return &QuoteResult{ + Report: report, + CertChain: nil, // TDX cert chain retrieved via Intel Trust Authority, tenant-side + AuxBlob: nil, + }, nil +} diff --git a/sidecar/attestation/tee/tee.go b/sidecar/attestation/tee/tee.go new file mode 100644 index 00000000..c004d5bf --- /dev/null +++ b/sidecar/attestation/tee/tee.go @@ -0,0 +1,90 @@ +package tee + +import ( + "context" + "fmt" + "os" +) + +// TEE type name constants. +const ( + NameSNP = "snp" + NameTDX = "tdx" +) + +// QuoteResult holds the raw hardware-signed attestation evidence. +type QuoteResult struct { + Report []byte // Raw attestation report (SNP ~1184 bytes, TDX 1024 bytes) + CertChain []byte // Cert chain (may be empty — fetching is tenant-side) + AuxBlob []byte // Empty on NVIDIA-patched kernel +} + +// Provider abstracts TEE-specific attestation report collection. +type Provider interface { + // Name returns the TEE type identifier ("snp" or "tdx"). + Name() string + + // Available returns true if this TEE surface is accessible. + Available() bool + + // GetQuote collects a hardware-signed attestation report with the + // given report_data (typically the tenant's nonce, or a hash binding + // the nonce to additional data like a TLS public key). + GetQuote(ctx context.Context, reportData [64]byte) (*QuoteResult, error) +} + +// Detect probes the guest environment and returns the first available TEE provider. +// +// If ATTESTATION_MOCK=true is set, returns a MockProvider that produces synthetic +// reports for local development without TEE hardware. The optional ATTESTATION_MOCK_TEE +// env var controls the mock TEE type ("snp" or "tdx", defaults to "snp"). +// +// Hardware check order: +// 1. /sys/kernel/config/tsm/report/ (configfs-tsm — NVIDIA-patched Kata kernel, GPU path) +// 2. /dev/sev-guest (CPU-only AMD SEV-SNP) +// 3. /dev/tdx_guest (Intel TDX — current kernel interface) +// 4. /dev/tdx-attest (Intel TDX — legacy kernel interface) +// +// configfs-tsm is checked first because on GPU-enabled VMs with the NVIDIA-patched +// kernel, it supersedes /dev/sev-guest. The SNP and TDX device paths are mutually +// exclusive (a VM is either SEV-SNP or TDX, never both). +func Detect() (Provider, error) { + if os.Getenv("ATTESTATION_MOCK") == "true" { + tee := os.Getenv("ATTESTATION_MOCK_TEE") + if tee == "" { + tee = NameSNP + } + return &MockProvider{TEE: tee}, nil + } + + configfs := &ConfigfsTSM{BasePath: "/sys/kernel/config/tsm/report"} + if configfs.Available() { + return configfs, nil + } + + sevGuest := &SEVGuest{DevicePath: "/dev/sev-guest"} + if sevGuest.Available() { + return sevGuest, nil + } + + tdxGuest := &TDX{DevicePath: "/dev/tdx_guest"} + if tdxGuest.Available() { + return tdxGuest, nil + } + + tdxLegacy := &TDX{DevicePath: "/dev/tdx-attest"} + if tdxLegacy.Available() { + return tdxLegacy, nil + } + + return nil, fmt.Errorf("no TEE attestation surface found: "+ + "tried configfs-tsm (%s), /dev/sev-guest, /dev/tdx_guest, /dev/tdx-attest "+ + "(set ATTESTATION_MOCK=true for local development)", + configfs.BasePath) +} + +// dirExists returns true if the path exists and is a directory. +func dirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/sidecar/attestation/tlsbinding.go b/sidecar/attestation/tlsbinding.go new file mode 100644 index 00000000..bf5e88df --- /dev/null +++ b/sidecar/attestation/tlsbinding.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha512" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" +) + +// TLSBinding holds an ephemeral TLS keypair generated at sidecar startup. +// The public key hash can be embedded in the TEE report_data field to bind +// the TLS channel to the attestation evidence: +// +// report_data = SHA-512(tls_pubkey_der || nonce)[:64] +// +// This proves to the tenant that the TLS endpoint they're talking to runs +// inside the attested TEE — not a MITM proxy on the host plane. +type TLSBinding struct { + Certificate tls.Certificate + PubKeyDER []byte // DER-encoded public key for channel binding +} + +// GenerateEphemeralKeypair creates an ECDSA P-384 TLS keypair with a +// self-signed certificate valid for 24 hours. The keypair is generated +// fresh on each sidecar startup — it's ephemeral by design. +func GenerateEphemeralKeypair() (*TLSBinding, error) { + key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate ECDSA key: %w", err) + } + + pubKeyDER, err := x509.MarshalPKIXPublicKey(&key.PublicKey) + if err != nil { + return nil, fmt.Errorf("marshal public key: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "akash-attestation-sidecar"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, fmt.Errorf("create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshal private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("load TLS keypair: %w", err) + } + + return &TLSBinding{ + Certificate: tlsCert, + PubKeyDER: pubKeyDER, + }, nil +} + +// ComputeBoundReportData computes the report_data that binds a TLS channel +// to a tenant-provided nonce: +// +// report_data = SHA-512(tls_pubkey_der || nonce)[:64] +// +// The tenant verifies this by: +// 1. Getting the sidecar's TLS public key from the /info endpoint +// 2. Requesting a quote with bind_tls=true +// 3. Verifying that the SNP report's REPORT_DATA matches this hash +// 4. Knowing the TLS session they're on uses the same key +func ComputeBoundReportData(pubKeyDER []byte, nonce [64]byte) [64]byte { + h := sha512.New() + h.Write(pubKeyDER) + h.Write(nonce[:]) + sum := h.Sum(nil) // 64 bytes from SHA-512 + var result [64]byte + copy(result[:], sum[:64]) + return result +}