From 274fd2ad42527b8f4ad9d50094f2990bac8cd292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 9 Mar 2026 15:57:21 +0100 Subject: [PATCH 1/3] Add flag to deactivate metadata parsing --- ooniapi/common/src/common/config.py | 5 +++++ .../ooniprobe/src/ooniprobe/routers/reports.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 7b5b18a51..fe2c70455 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -54,6 +54,11 @@ class Settings(BaseSettings): "" # filename of json containing Tor bridges and DirAuth endpoints ) + geoip_metadata_active: bool = Field( + default=False, + description="Activates extra metadata collection for geoip anomalies" + ) + # Used to store history of all manifests and retrieve next manifest to use anonc_manifest_bucket: str = Field( default="CHANGEME", diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py index 6ce1b8510..e7ae297d4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py @@ -15,7 +15,7 @@ from ..common.metrics import timer from ..common.routers import BaseModel from ..common.utils import setnocacheresponse -from ..common.dependencies import ClickhouseDep +from ..common.dependencies import ClickhouseDep, Settings from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep from ..utils import ( error, @@ -198,6 +198,7 @@ async def receive_measurement( asn=asn, msmt_uid=msmt_uid, data=data, + settings=settings ) except Exception as e: log.error(f"Error checking for geoip anomalies: {e}") @@ -228,12 +229,18 @@ def _check_and_register_geoip_anomaly( asn: str, msmt_uid: str, data: bytes, + settings: Settings ) -> None: - # check for geoip anomalies + actual_cc, actual_asn = get_cc_asn(request, cc_reader, asn_reader) if actual_cc != cc or normalize_asn(actual_asn) != normalize_asn(asn): + # expensive: parses measurement body and sends anomaly to clickhouse - platform, software_name, software_version = _parse_metadata(data) + if settings.geoip_metadata_active: + platform, software_name, software_version = _parse_metadata(data) + else: + platform, software_name, software_version = "", "", "" + register_geoip_anomaly( cc, actual_cc, From 17f1c22c7881f403f7633ffb0350bd0accc502ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 9 Mar 2026 16:12:50 +0100 Subject: [PATCH 2/3] Skip failing test due to bad anomaly collection --- ooniapi/services/ooniprobe/tests/integ/test_geolookup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py b/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py index a2474531a..f7628288b 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py @@ -1,6 +1,7 @@ from typing import Tuple import json import time +import pytest import ooniprobe.routers.v1.probe_services as ps from ooniprobe import utils from ooniprobe.dependencies import CCReaderDep, ASNReaderDep @@ -76,6 +77,7 @@ def patched_lookup_probe_network(ipaddr: str, asn_reader) -> Tuple[str, str]: return d.get(ipaddr, ("AS0", "")) +@pytest.mark.skip def test_geoip_mismatch(client, clickhouse_db, monkeypatch): monkeypatch.setattr(utils, "lookup_probe_cc", patched_lookup_probe_cc) From 035545705c2e71aae8c176f8a1e9237f0fe04c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 9 Mar 2026 16:14:49 +0100 Subject: [PATCH 3/3] Set metadata collection to true in conftest for ooniprobe tests --- ooniapi/services/ooniprobe/tests/conftest.py | 1 + ooniapi/services/ooniprobe/tests/integ/test_geolookup.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index d77e95fbb..f8b96a1da 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -177,6 +177,7 @@ def test_settings( anonc_manifest_file="manifest.json", anonc_secret_key=secret_key, tor_targets="./tests/fixtures/data/tor-targets.json", + geoip_metadata_active=True ) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py b/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py index f7628288b..a2474531a 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_geolookup.py @@ -1,7 +1,6 @@ from typing import Tuple import json import time -import pytest import ooniprobe.routers.v1.probe_services as ps from ooniprobe import utils from ooniprobe.dependencies import CCReaderDep, ASNReaderDep @@ -77,7 +76,6 @@ def patched_lookup_probe_network(ipaddr: str, asn_reader) -> Tuple[str, str]: return d.get(ipaddr, ("AS0", "")) -@pytest.mark.skip def test_geoip_mismatch(client, clickhouse_db, monkeypatch): monkeypatch.setattr(utils, "lookup_probe_cc", patched_lookup_probe_cc)