feat(controller): export capacity metrics over OTLP

docs/kubernetes/deployment.md

@@ -159,6 +159,26 @@ controller:
 
 Point your Prometheus stack at the `metrics` container port (for example via a `ServiceMonitor` or `PodMonitoring`).
 
+### Business capacity metrics
+
+The elected controller also exports low-cardinality business capacity metrics over OTLP/HTTP when `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` or `OTEL_EXPORTER_OTLP_ENDPOINT` is set. This is independent of the controller-runtime Prometheus endpoint and remains disabled when neither variable is configured.
+
+```yaml
+extraEnv:
+  - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+    value: http://otel-collector.observability:4318/v1/metrics
+```
+
+| Metric | Unit | Attributes | Description |
+|--------|------|------------|-------------|
+| `controller.pool.pods` | `{pod}` | `namespace`, `pool_name`, `state` | Current Pool Pods, where `state` is `total`, `allocated`, `available`, or `updated` |
+| `controller.pool.cpu.requested` | `{cpu}` | `namespace`, `pool_name`, `state` | Scheduler-equivalent CPU requests represented by total, allocated, or available Pool Pods |
+| `controller.pool.memory.requested` | `By` | `namespace`, `pool_name`, `state` | Scheduler-equivalent memory requests represented by total, allocated, or available Pool Pods |
+| `controller.batchsandbox.count` | `{batchsandbox}` | `namespace`, `phase`, `allocation_mode` | Current BatchSandbox objects by lifecycle phase and pool/direct mode |
+| `controller.batchsandbox.pods` | `{pod}` | `namespace`, `state`, `allocation_mode` | Desired, current, allocated, and ready BatchSandbox Pod counts |
+
+The metrics deliberately omit sandbox, BatchSandbox, and Pod identifiers. Only the leader exports them, so multiple controller replicas do not duplicate cluster totals. An unset initial BatchSandbox phase is exported as `Unknown`. Derive Pool utilization from `allocated / total` and calculate peak, valley, or percentile capacity in the telemetry backend. Actual CPU and memory usage remains available from kubelet/cAdvisor rather than being duplicated here.
+
 ## Configure the Server for Kubernetes
 
 Generate a Kubernetes-oriented server config:

kubernetes/cmd/controller/main.go

@@ -444,6 +444,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	poolAllocator := controller.NewDefaultAllocator(mgr.GetClient())
+	if err := controller.SetupCapacityMetricsWithManager(mgr, poolAllocator); err != nil {
+		setupLog.Error(err, "unable to register capacity metrics")
+		os.Exit(1)
+	}
+
 	if err := (&controller.BatchSandboxReconciler{
 		Client:              mgr.GetClient(),
 		Scheme:              mgr.GetScheme(),
@@ -459,7 +465,7 @@ func main() {
 		Client:     mgr.GetClient(),
 		Scheme:     mgr.GetScheme(),
 		Recorder:   mgr.GetEventRecorderFor("pool-controller"),
-		Allocator:  controller.NewDefaultAllocator(mgr.GetClient()),
+		Allocator:  poolAllocator,
 		RestConfig: mgr.GetConfig(),
 	}).SetupWithManager(mgr, poolConcurrency); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Pool")

kubernetes/go.mod

@@ -13,9 +13,12 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0
+	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	k8s.io/api v0.33.0
 	k8s.io/apimachinery v0.33.0
 	k8s.io/client-go v0.33.0
+	k8s.io/component-helpers v0.33.0
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 	sigs.k8s.io/controller-runtime v0.21.0
@@ -83,7 +86,7 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -102,13 +105,13 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel v1.43.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.43.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0
+	go.opentelemetry.io/otel/sdk v1.43.0
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect

kubernetes/go.sum

@@ -136,8 +136,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -241,6 +241,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
 go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 h1:ao6Oe+wSebTlQ1OEht7jlYTzQKE+pnx/iNywFvTbuuI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0/go.mod h1:u3T6vz0gh/NVzgDgiwkgLxpsSF6PaPmo2il0apGJbls=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
@@ -253,8 +255,8 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
 go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -395,6 +397,8 @@ k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98=
 k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg=
 k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk=
 k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU=
+k8s.io/component-helpers v0.33.0 h1:0AdW0A0mIgljLgtG0hJDdJl52PPqTrtMgOgtm/9i/Ys=
+k8s.io/component-helpers v0.33.0/go.mod h1:9SRiXfLldPw9lEEuSsapMtvT8j/h1JyFFapbtybwKvU=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=

kubernetes/internal/controller/capacity_metrics.go

@@ -0,0 +1,263 @@
+// Copyright 2026 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controller
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	resourcehelper "k8s.io/component-helpers/resource"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1"
+	"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils"
+	"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex"
+)
+
+const (
+	capacityMeterName            = "opensandbox/controller"
+	poolPodsMetricName           = "controller.pool.pods"
+	poolCPURequestsMetricName    = "controller.pool.cpu.requested"
+	poolMemoryRequestsMetricName = "controller.pool.memory.requested"
+	batchSandboxCountMetricName  = "controller.batchsandbox.count"
+	batchSandboxPodsMetricName   = "controller.batchsandbox.pods"
+	unknownBatchSandboxPhase     = "Unknown"
+	allocationModePool           = "pool"
+	allocationModeDirect         = "direct"
+)
+
+type poolAllocationReader interface {
+	GetPoolAllocation(context.Context, *sandboxv1alpha1.Pool) (map[string]string, error)
+}
+
+type poolCapacity struct {
+	namespace string
+	name      string
+	pods      map[string]int64
+	cpu       map[string]float64
+	memory    map[string]int64
+}
+
+type batchSandboxKey struct {
+	namespace      string
+	phase          string
+	allocationMode string
+}
+
+type batchSandboxPodsKey struct {
+	namespace      string
+	state          string
+	allocationMode string
+}
+
+type capacitySnapshot struct {
+	pools              []poolCapacity
+	batchSandboxCounts map[batchSandboxKey]int64
+	batchSandboxPods   map[batchSandboxPodsKey]int64
+}
+
+func registerCapacityMetrics(meter metric.Meter, reader client.Reader, allocations poolAllocationReader) (metric.Registration, error) {
+	poolPods, err := meter.Int64ObservableGauge(
+		poolPodsMetricName,
+		metric.WithDescription("Current Pool Pod count by allocation state"),
+		metric.WithUnit("{pod}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	poolCPU, err := meter.Float64ObservableGauge(
+		poolCPURequestsMetricName,
+		metric.WithDescription("Current CPU requests represented by Pool Pods"),
+		metric.WithUnit("{cpu}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	poolMemory, err := meter.Int64ObservableGauge(
+		poolMemoryRequestsMetricName,
+		metric.WithDescription("Current memory requests represented by Pool Pods"),
+		metric.WithUnit("By"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	batchSandboxCount, err := meter.Int64ObservableGauge(
+		batchSandboxCountMetricName,
+		metric.WithDescription("Current BatchSandbox count by phase and allocation mode"),
+		metric.WithUnit("{batchsandbox}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	batchSandboxPods, err := meter.Int64ObservableGauge(
+		batchSandboxPodsMetricName,
+		metric.WithDescription("Current BatchSandbox Pod count by state and allocation mode"),
+		metric.WithUnit("{pod}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error {
+		snapshot, err := collectCapacitySnapshot(ctx, reader, allocations)
+		if err != nil {
+			return err
+		}
+		for _, pool := range snapshot.pools {
+			for state, value := range pool.pods {
+				attrs := metric.WithAttributes(poolAttributes(pool.namespace, pool.name, state)...)
+				observer.ObserveInt64(poolPods, value, attrs)
+			}
+			for state, value := range pool.cpu {
+				observer.ObserveFloat64(poolCPU, value, metric.WithAttributes(poolAttributes(pool.namespace, pool.name, state)...))
+			}
+			for state, value := range pool.memory {
+				observer.ObserveInt64(poolMemory, value, metric.WithAttributes(poolAttributes(pool.namespace, pool.name, state)...))
+			}
+		}
+		for key, value := range snapshot.batchSandboxCounts {
+			observer.ObserveInt64(batchSandboxCount, value, metric.WithAttributes(
+				attribute.String("namespace", key.namespace),
+				attribute.String("phase", key.phase),
+				attribute.String("allocation_mode", key.allocationMode),
+			))
+		}
+		for key, value := range snapshot.batchSandboxPods {
+			observer.ObserveInt64(batchSandboxPods, value, metric.WithAttributes(
+				attribute.String("namespace", key.namespace),
+				attribute.String("state", key.state),
+				attribute.String("allocation_mode", key.allocationMode),
+			))
+		}
+		return nil
+	}, poolPods, poolCPU, poolMemory, batchSandboxCount, batchSandboxPods)
+}
+
+func collectCapacitySnapshot(ctx context.Context, reader client.Reader, allocations poolAllocationReader) (capacitySnapshot, error) {
+	pools := &sandboxv1alpha1.PoolList{}
+	if err := reader.List(ctx, pools); err != nil {
+		return capacitySnapshot{}, err
+	}
+	batchSandboxes := &sandboxv1alpha1.BatchSandboxList{}
+	if err := reader.List(ctx, batchSandboxes); err != nil {
+		return capacitySnapshot{}, err
+	}
+
+	snapshot := capacitySnapshot{
+		pools:              make([]poolCapacity, 0, len(pools.Items)),
+		batchSandboxCounts: make(map[batchSandboxKey]int64),
+		batchSandboxPods:   make(map[batchSandboxPodsKey]int64),
+	}
+	for i := range pools.Items {
+		pool := &pools.Items[i]
+		if !pool.DeletionTimestamp.IsZero() {
+			continue
+		}
+		capacity, err := collectPoolCapacity(ctx, reader, allocations, pool)
+		if err != nil {
+			return capacitySnapshot{}, err
+		}
+		snapshot.pools = append(snapshot.pools, capacity)
+	}
+	for i := range batchSandboxes.Items {
+		addBatchSandboxCapacity(&snapshot, &batchSandboxes.Items[i])
+	}
+	return snapshot, nil
+}
+
+func collectPoolCapacity(ctx context.Context, reader client.Reader, allocations poolAllocationReader, pool *sandboxv1alpha1.Pool) (poolCapacity, error) {
+	pods := &corev1.PodList{}
+	if err := reader.List(ctx, pods, &client.ListOptions{
+		Namespace:     pool.Namespace,
+		FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}),
+	}); err != nil {
+		return poolCapacity{}, err
+	}
+	allocatedPods, err := allocations.GetPoolAllocation(ctx, pool)
+	if err != nil {
+		return poolCapacity{}, err
+	}
+
+	capacity := poolCapacity{
+		namespace: pool.Namespace,
+		name:      pool.Name,
+		pods: map[string]int64{
+			"total": int64(pool.Status.Total), "allocated": int64(pool.Status.Allocated),
+			"available": int64(pool.Status.Available), "updated": int64(pool.Status.Updated),
+		},
+		cpu:    map[string]float64{"total": 0, "allocated": 0, "available": 0},
+		memory: map[string]int64{"total": 0, "allocated": 0, "available": 0},
+	}
+	for i := range pods.Items {
+		pod := &pods.Items[i]
+		if !pod.DeletionTimestamp.IsZero() {
+			continue
+		}
+		requests := resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{})
+		cpu := float64(requests.Cpu().MilliValue()) / 1000
+		memory := requests.Memory().Value()
+		capacity.cpu["total"] += cpu
+		capacity.memory["total"] += memory
+		if _, allocated := allocatedPods[pod.Name]; allocated {
+			capacity.cpu["allocated"] += cpu
+			capacity.memory["allocated"] += memory
+			continue
+		}
+		if utils.IsPodReady(pod) {
+			capacity.cpu["available"] += cpu
+			capacity.memory["available"] += memory
+		}
+	}
+	return capacity, nil
+}
+
+func addBatchSandboxCapacity(snapshot *capacitySnapshot, sandbox *sandboxv1alpha1.BatchSandbox) {
+	phase := string(sandbox.Status.Phase)
+	if phase == "" {
+		phase = unknownBatchSandboxPhase
+	}
+	allocationMode := allocationModeDirect
+	if sandbox.Spec.PoolRef != "" {
+		allocationMode = allocationModePool
+	}
+	snapshot.batchSandboxCounts[batchSandboxKey{
+		namespace: sandbox.Namespace, phase: phase, allocationMode: allocationMode,
+	}]++
+
+	desired := int64(1)
+	if sandbox.Spec.Replicas != nil {
+		desired = int64(*sandbox.Spec.Replicas)
+	}
+	values := map[string]int64{
+		"desired": desired, "current": int64(sandbox.Status.Replicas),
+		"allocated": int64(sandbox.Status.Allocated), "ready": int64(sandbox.Status.Ready),
+	}
+	for state, value := range values {
+		snapshot.batchSandboxPods[batchSandboxPodsKey{
+			namespace: sandbox.Namespace, state: state, allocationMode: allocationMode,
+		}] += value
+	}
+}
+
+func poolAttributes(namespace, poolName, state string) []attribute.KeyValue {
+	return []attribute.KeyValue{
+		attribute.String("namespace", namespace),
+		attribute.String("pool_name", poolName),
+		attribute.String("state", state),
+	}
+}