Add local metrics

fastpath/docker-compose.yml

@@ -4,19 +4,24 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: ooni-fastpath 
+    environment:
+      STATSD_HOST: statsd_exporter
+      STATSD_PORT: "9125"
     ports:
       - "5000:5000"
       - "8472:8472"
     volumes:
-      - .:/app
+      - .:/app:Z
     working_dir: /app
-    profiles:
-      - default
-      - all
+    depends_on:
+      statsd_exporter:
+        condition: service_started
+      clickhouse-server:
+        condition: service_healthy
 
   # This service is used only for testing, in prod we use the actual clickhouse db
   clickhouse-server:
-    image: clickhouse/clickhouse-server:latest
+    image: docker.io/clickhouse/clickhouse-server:latest
     container_name: clickhouse-server
     environment:
       - CLICKHOUSE_DB=default
@@ -27,16 +32,64 @@ services:
       - "8123:8123"
       - "9009:9009"
     volumes:
-       - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql
+      - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql:Z
     healthcheck:
       test: ["CMD", "clickhouse-client", "--query", "select 1;"]
       interval: 30s
       retries: 3
       start_period: 60s
       timeout: 10s
-    profiles:
-      - all
-      - clickhouse 
+
+  statsd_exporter:
+    image: docker.io/prom/statsd-exporter:latest
+    container_name: statsd-exporter
+    # statsd_exporter listens UDP 9125 by default for statsd; exposes HTTP /metrics on 9102
+    environment:
+      STATS_D_EXPORTER_METRICS_PORT: "9102"
+    ports:
+      - "9125:9125/udp"  # UDP ingest (alternative)
+      - "9102:9102"      # HTTP /metrics for Prometheus
+
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9102/metrics || exit 1"]
+      interval: 10s
+      retries: 5
+      start_period: 10s
+      timeout: 2s
+
+  prometheus:
+    image: docker.io/prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:Z
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9090/-/ready || exit 1"]
+      interval: 10s
+      retries: 5
+      start_period: 10s
+      timeout: 2s
+
+  grafana:
+    image: docker.io/grafana/grafana:latest
+    container_name: grafana
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: "admin"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:Z
+      - ./grafana/data:/var/lib/grafana:Z
+
+    ports:
+      - "3000:3000"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:3000/api/health || exit 1"]
+      interval: 10s
+      retries: 5
+      start_period: 10s
+      timeout: 2s
 
 volumes:
   clickhouse-data:

fastpath/fastpath/core.py

@@ -1617,6 +1617,7 @@ def msm_processor(queue):
         metrics.gauge("queue_size", queue.qsize())
 
 
+
 def flag_measurements_with_wrong_date(msm: dict, msmt_uid: str, scores: dict) -> None:
     if not msmt_uid.startswith("20") or len(msmt_uid) < 20:
         return

fastpath/fastpath/metrics.py

@@ -6,13 +6,21 @@
 
 from os.path import basename, splitext
 
+import os
 import statsd  # debdeps: python3-statsd
+import logging
 
 
+log = logging.getLogger("fastpath")
+
 def setup_metrics(host="localhost", name=None):
     """Setup metric generation. Use dotted namespaces e.g.
     "pipeline.centrifugation"
     """
+    statsd_host = os.getenv("STATSD_HOST", host)
+    statsd_port = os.getenv("STATSD_PORT", 8125)
+
+    log.debug("got {statsd_host}:{statsd_port} for statsd")
     if name is None:
         import __main__
 
@@ -21,4 +29,4 @@ def setup_metrics(host="localhost", name=None):
         prefix = name
 
     prefix = prefix.strip(".")
-    return statsd.StatsClient(host, 8125, prefix=prefix)
+    return statsd.StatsClient(statsd_host, statsd_port, prefix=prefix)

fastpath/grafana/data/dashboards/simple-stats.json

@@ -0,0 +1,37 @@
+{
+  "dashboard": {
+    "id": null,
+    "uid": "simple-stats",
+    "title": "Simple Metrics",
+    "schemaVersion": 36,
+    "version": 1,
+    "panels": [
+      {
+        "id": 1,
+        "type": "stat",
+        "title": "Example: statsd_exporter metric",
+        "datasource": "Prometheus",
+        "targets": [
+          {
+            "expr": "statsd_metric_total",
+            "refId": "A"
+          }
+        ],
+        "gridPos": { "h": 4, "w": 12, "x": 0, "y": 0 }
+      },
+      {
+        "id": 2,
+        "type": "graph",
+        "title": "statsd_exporter: requests over time",
+        "datasource": "Prometheus",
+        "targets": [
+          {
+            "expr": "rate(statsd_metric_total[1m])",
+            "refId": "A"
+          }
+        ],
+        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }
+      }
+    ]
+  }
+}