remote write to mimir

charts/prometheus-agent/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: prometheus-agent
+description: Prometheus Agent for scraping pods and pushing metrics to Mimir
+type: application
+version: 0.1.0
+appVersion: "2.51.0"

charts/prometheus-agent/templates/_helpers.tpl

@@ -0,0 +1,58 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "prometheus-agent.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "prometheus-agent.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "prometheus-agent.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "prometheus-agent.labels" -}}
+helm.sh/chart: {{ include "prometheus-agent.chart" . }}
+{{ include "prometheus-agent.selectorLabels" . }}
+app.kubernetes.io/version: {{ .Values.image.tag | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "prometheus-agent.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "prometheus-agent.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Service account name
+*/}}
+{{- define "prometheus-agent.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "prometheus-agent.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/prometheus-agent/templates/clusterrole.yaml

@@ -0,0 +1,22 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "prometheus-agent.fullname" . }}
+  labels:
+    {{- include "prometheus-agent.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - nodes/metrics
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs: ["get"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]

charts/prometheus-agent/templates/clusterrolebinding.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "prometheus-agent.fullname" . }}
+  labels:
+    {{- include "prometheus-agent.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "prometheus-agent.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "prometheus-agent.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}

charts/prometheus-agent/templates/configmap.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "prometheus-agent.fullname" . }}
+  labels:
+    {{- include "prometheus-agent.labels" . | nindent 4 }}
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: {{ .Values.scrape.interval }}
+      scrape_timeout: {{ .Values.scrape.timeout }}
+      external_labels:
+        cluster: {{ .Values.externalLabels.cluster | quote }}
+        environment: {{ .Values.externalLabels.environment | quote }}
+        region: {{ .Values.externalLabels.region | quote }}
+
+    remote_write:
+      - url: {{ .Values.remoteWrite.url | quote }}
+        basic_auth:
+          username: {{ .Values.remoteWrite.basicAuth.username | quote }}
+          password: {{ .Values.remoteWrite.basicAuth.password | quote }}
+
+    scrape_configs:
+      # Scrape movement node pods with prometheus.io annotations
+      - job_name: 'movement-nodes'
+        kubernetes_sd_configs:
+          - role: pod
+            {{- if .Values.scrape.namespaces }}
+            namespaces:
+              names:
+                {{- range (splitList "," .Values.scrape.namespaces) }}
+                - {{ . | trim | quote }}
+                {{- end }}
+            {{- end }}
+        relabel_configs:
+          # Only scrape pods with prometheus.io/scrape=true
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: true
+          # Use prometheus.io/port annotation for port
+          - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_prometheus_io_port]
+            action: replace
+            target_label: __address__
+            regex: (.+);(.+)
+            replacement: $1:$2
+          # Use prometheus.io/path annotation for path
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          # Add pod metadata as labels
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            target_label: app
+          # Add node_type label from pod label (movementnetwork.xyz/node_type)
+          - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_node_type]
+            target_label: node_type
+          # Add node_name label from pod label (movementnetwork.xyz/node)
+          - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_node]
+            target_label: node_name
+          # Add network label
+          - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_network]
+            target_label: network

charts/prometheus-agent/templates/deployment.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "prometheus-agent.fullname" . }}
+  labels:
+    {{- include "prometheus-agent.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "prometheus-agent.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "prometheus-agent.selectorLabels" . | nindent 8 }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+    spec:
+      serviceAccountName: {{ include "prometheus-agent.serviceAccountName" . }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        fsGroup: 65534
+      containers:
+        - name: prometheus
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--enable-feature=agent"
+            - "--storage.agent.path=/prometheus"
+            - "--web.enable-lifecycle"
+          ports:
+            - name: http
+              containerPort: 9090
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 15
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/prometheus
+            - name: storage
+              mountPath: /prometheus
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "prometheus-agent.fullname" . }}
+        - name: storage
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/prometheus-agent/templates/serviceaccount.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "prometheus-agent.serviceAccountName" . }}
+  labels:
+    {{- include "prometheus-agent.labels" . | nindent 4 }}
+{{- end }}

charts/prometheus-agent/values.yaml

@@ -0,0 +1,46 @@
+# Prometheus Agent - scrapes pods, pushes to Mimir
+# Deployed per-cluster to collect metrics from all movement nodes
+
+image:
+  repository: prom/prometheus
+  tag: v2.51.0
+  pullPolicy: IfNotPresent
+
+# External labels added to all metrics (REQUIRED)
+# These identify which cluster/environment metrics come from
+externalLabels:
+  cluster: ""           # e.g., "validator-08"
+  environment: "testnet"
+  region: ""            # e.g., "us-west-2"
+
+# Remote write to Mimir (REQUIRED)
+remoteWrite:
+  url: ""               # e.g., "https://mimir.us-west-2.testnet.movementinfra.xyz/api/v1/push"
+  basicAuth:
+    username: ""
+    password: ""
+
+# Scrape configuration
+scrape:
+  interval: 15s
+  timeout: 10s
+  # Scrape pods in these namespaces (comma-separated or empty for all)
+  namespaces: "movement-l1"
+
+# Resources
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+# Service account for pod discovery
+serviceAccount:
+  create: true
+  name: ""
+
+nodeSelector: {}
+tolerations: []
+affinity: {}

examples/validator-vfn/deploy.py

@@ -24,6 +24,7 @@
 SCRIPT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = SCRIPT_DIR.parents[1]
 CHART_DIR = ROOT_DIR / "charts" / "movement-node"
+PROMETHEUS_AGENT_CHART_DIR = ROOT_DIR / "charts" / "prometheus-agent"
 
 # Default VFN connection keys (validator's fullnode-network identity)
 # These are used for the internal V ↔ VFN private connection
@@ -393,6 +394,54 @@ def deploy_node(
     success(f"{node_type.upper()} '{node_name}' deployed successfully")
 
 
+def deploy_prometheus_agent(
+    env_vars: dict,
+    namespace: str,
+    cluster_name: str,
+) -> None:
+    """Deploy Prometheus Agent for metrics collection and push to Mimir."""
+    from tools.helm import HelmManager
+
+    info("Deploying Prometheus Agent for metrics push")
+
+    # Get Mimir configuration
+    mimir_url = env_vars.get("MIMIR_URL", "").strip()
+    mimir_username = env_vars.get("MIMIR_USERNAME", "").strip()
+    mimir_password = env_vars.get("MIMIR_PASSWORD", "").strip()
+
+    if not mimir_url:
+        error("MIMIR_URL not set, skipping Prometheus Agent deployment")
+        return
+
+    if not mimir_username or not mimir_password:
+        error("MIMIR_USERNAME or MIMIR_PASSWORD not set, skipping Prometheus Agent deployment")
+        return
+
+    region = env_vars.get("AWS_REGION", "us-east-1")
+    environment = env_vars.get("NETWORK_NAME", "testnet")
+
+    set_values = {
+        "externalLabels.cluster": cluster_name,
+        "externalLabels.environment": environment,
+        "externalLabels.region": region,
+        "remoteWrite.url": mimir_url,
+        "remoteWrite.basicAuth.username": mimir_username,
+        "remoteWrite.basicAuth.password": mimir_password,
+        "scrape.namespaces": namespace,
+    }
+
+    helm = HelmManager(PROMETHEUS_AGENT_CHART_DIR)
+    helm.upgrade_install(
+        release_name="prometheus-agent",
+        namespace=namespace,
+        create_namespace=False,
+        reset_values=True,
+        set_values=set_values,
+    )
+
+    success("Prometheus Agent deployed successfully")
+
+
 def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Path | None = None) -> None:
     """Deploy validator cluster with intelligent topology handling."""
     tf_dir = terraform_dir or SCRIPT_DIR
@@ -548,6 +597,15 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa
             vfn_service=vfn_name if deploy_vfn else None,
         )
 
+    # Deploy Prometheus Agent if monitoring is enabled
+    deploy_monitoring = env_vars.get("DEPLOY_MONITORING", "false").lower() in ("true", "1", "yes")
+    if deploy_monitoring:
+        deploy_prometheus_agent(
+            env_vars=env_vars,
+            namespace=namespace,
+            cluster_name=validator_name,
+        )
+
     # Step 3: Validation (if requested)
     if validate:
         from tools.validation import validate_deployment, wait_for_pods_ready
@@ -627,6 +685,9 @@ def destroy(env_vars: dict, terraform_dir: Path | None = None) -> None:
     deploy_fullnode = env_vars.get("DEPLOY_FULLNODE", "false").lower() in ("true", "1", "yes")
 
     # Uninstall Helm releases in reverse order
+    deploy_monitoring = env_vars.get("DEPLOY_MONITORING", "false").lower() in ("true", "1", "yes")
+    if deploy_monitoring:
+        helm.uninstall("prometheus-agent", namespace)
     if deploy_fullnode:
         helm.uninstall(fullnode_name, namespace)
     if deploy_vfn: