Add ooniprobe private api

ooniapi/services/ooniprobe/pyproject.toml

@@ -14,7 +14,9 @@ dependencies = [
   "ujson ~= 5.9.0",
   "urllib3 ~= 2.1.0",
   "python-dateutil ~= 2.8.2",
+  "pycountry ~= 26.2.16",
   "pydantic-settings ~= 2.1.0",
+  "pydantic_extra_types ~= 2.11.1",
   "statsd ~= 4.0.1",
   "uvicorn ~= 0.25.0",
   "psycopg2 ~= 2.9.9",

ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py

@@ -0,0 +1,273 @@
+"""
+CC to country name lookup table
+
+Regenerate the dict with:
+
+    python3 ooniapi/countries/__init__.py
+
+"""
+
+_countries = {
+    "AD": "Andorra",
+    "AE": "United Arab Emirates",
+    "AF": "Afghanistan",
+    "AG": "Antigua & Barbuda",
+    "AI": "Anguilla",
+    "AL": "Albania",
+    "AM": "Armenia",
+    "AO": "Angola",
+    "AQ": "Antarctica",
+    "AR": "Argentina",
+    "AS": "American Samoa",
+    "AT": "Austria",
+    "AU": "Australia",
+    "AW": "Aruba",
+    "AX": "Åland Islands",
+    "AZ": "Azerbaijan",
+    "BA": "Bosnia",
+    "BB": "Barbados",
+    "BD": "Bangladesh",
+    "BE": "Belgium",
+    "BF": "Burkina Faso",
+    "BG": "Bulgaria",
+    "BH": "Bahrain",
+    "BI": "Burundi",
+    "BJ": "Benin",
+    "BL": "St. Barthélemy",
+    "BM": "Bermuda",
+    "BN": "Brunei",
+    "BO": "Bolivia",
+    "BQ": "Caribbean Netherlands",
+    "BR": "Brazil",
+    "BS": "Bahamas",
+    "BT": "Bhutan",
+    "BV": "Bouvet Island",
+    "BW": "Botswana",
+    "BY": "Belarus",
+    "BZ": "Belize",
+    "CA": "Canada",
+    "CC": "Cocos (Keeling) Islands",
+    "CD": "Congo - Kinshasa",
+    "CF": "Central African Republic",
+    "CG": "Congo - Brazzaville",
+    "CH": "Switzerland",
+    "CI": "Côte d’Ivoire",
+    "CK": "Cook Islands",
+    "CL": "Chile",
+    "CM": "Cameroon",
+    "CN": "China",
+    "CO": "Colombia",
+    "CR": "Costa Rica",
+    "CU": "Cuba",
+    "CV": "Cape Verde",
+    "CW": "Curaçao",
+    "CX": "Christmas Island",
+    "CY": "Cyprus",
+    "CZ": "Czechia",
+    "DE": "Germany",
+    "DJ": "Djibouti",
+    "DK": "Denmark",
+    "DM": "Dominica",
+    "DO": "Dominican Republic",
+    "DZ": "Algeria",
+    "EC": "Ecuador",
+    "EE": "Estonia",
+    "EG": "Egypt",
+    "EH": "Western Sahara",
+    "ER": "Eritrea",
+    "ES": "Spain",
+    "ET": "Ethiopia",
+    "FI": "Finland",
+    "FJ": "Fiji",
+    "FK": "Falkland Islands",
+    "FM": "Micronesia",
+    "FO": "Faroe Islands",
+    "FR": "France",
+    "GA": "Gabon",
+    "GB": "United Kingdom",
+    "GD": "Grenada",
+    "GE": "Georgia",
+    "GF": "French Guiana",
+    "GG": "Guernsey",
+    "GH": "Ghana",
+    "GI": "Gibraltar",
+    "GL": "Greenland",
+    "GM": "Gambia",
+    "GN": "Guinea",
+    "GP": "Guadeloupe",
+    "GQ": "Equatorial Guinea",
+    "GR": "Greece",
+    "GS": "South Georgia & South Sandwich Islands",
+    "GT": "Guatemala",
+    "GU": "Guam",
+    "GW": "Guinea-Bissau",
+    "GY": "Guyana",
+    "HK": "Hong Kong",
+    "HM": "Heard & McDonald Islands",
+    "HN": "Honduras",
+    "HR": "Croatia",
+    "HT": "Haiti",
+    "HU": "Hungary",
+    "ID": "Indonesia",
+    "IE": "Ireland",
+    "IL": "Israel",
+    "IM": "Isle of Man",
+    "IN": "India",
+    "IO": "British Indian Ocean Territory",
+    "IQ": "Iraq",
+    "IR": "Iran",
+    "IS": "Iceland",
+    "IT": "Italy",
+    "JE": "Jersey",
+    "JM": "Jamaica",
+    "JO": "Jordan",
+    "JP": "Japan",
+    "KE": "Kenya",
+    "KG": "Kyrgyzstan",
+    "KH": "Cambodia",
+    "KI": "Kiribati",
+    "KM": "Comoros",
+    "KN": "St. Kitts & Nevis",
+    "KP": "North Korea",
+    "KR": "South Korea",
+    "KW": "Kuwait",
+    "KY": "Cayman Islands",
+    "KZ": "Kazakhstan",
+    "LA": "Laos",
+    "LB": "Lebanon",
+    "LC": "St. Lucia",
+    "LI": "Liechtenstein",
+    "LK": "Sri Lanka",
+    "LR": "Liberia",
+    "LS": "Lesotho",
+    "LT": "Lithuania",
+    "LU": "Luxembourg",
+    "LV": "Latvia",
+    "LY": "Libya",
+    "MA": "Morocco",
+    "MC": "Monaco",
+    "MD": "Moldova",
+    "ME": "Montenegro",
+    "MF": "St. Martin",
+    "MG": "Madagascar",
+    "MH": "Marshall Islands",
+    "MK": "North Macedonia",
+    "ML": "Mali",
+    "MM": "Myanmar",
+    "MN": "Mongolia",
+    "MO": "Macao",
+    "MP": "Northern Mariana Islands",
+    "MQ": "Martinique",
+    "MR": "Mauritania",
+    "MS": "Montserrat",
+    "MT": "Malta",
+    "MU": "Mauritius",
+    "MV": "Maldives",
+    "MW": "Malawi",
+    "MX": "Mexico",
+    "MY": "Malaysia",
+    "MZ": "Mozambique",
+    "NA": "Namibia",
+    "NC": "New Caledonia",
+    "NE": "Niger",
+    "NF": "Norfolk Island",
+    "NG": "Nigeria",
+    "NI": "Nicaragua",
+    "NL": "Netherlands",
+    "NO": "Norway",
+    "NP": "Nepal",
+    "NR": "Nauru",
+    "NU": "Niue",
+    "NZ": "New Zealand",
+    "OM": "Oman",
+    "PA": "Panama",
+    "PE": "Peru",
+    "PF": "French Polynesia",
+    "PG": "Papua New Guinea",
+    "PH": "Philippines",
+    "PK": "Pakistan",
+    "PL": "Poland",
+    "PM": "St. Pierre & Miquelon",
+    "PN": "Pitcairn Islands",
+    "PR": "Puerto Rico",
+    "PS": "Palestine",
+    "PT": "Portugal",
+    "PW": "Palau",
+    "PY": "Paraguay",
+    "QA": "Qatar",
+    "RE": "Réunion",
+    "RO": "Romania",
+    "RS": "Serbia",
+    "RU": "Russia",
+    "RW": "Rwanda",
+    "SA": "Saudi Arabia",
+    "SB": "Solomon Islands",
+    "SC": "Seychelles",
+    "SD": "Sudan",
+    "SE": "Sweden",
+    "SG": "Singapore",
+    "SH": "St. Helena",
+    "SI": "Slovenia",
+    "SJ": "Svalbard & Jan Mayen",
+    "SK": "Slovakia",
+    "SL": "Sierra Leone",
+    "SM": "San Marino",
+    "SN": "Senegal",
+    "SO": "Somalia",
+    "SR": "Suriname",
+    "SS": "South Sudan",
+    "ST": "São Tomé & Príncipe",
+    "SV": "El Salvador",
+    "SX": "Sint Maarten",
+    "SY": "Syria",
+    "SZ": "Eswatini",
+    "TC": "Turks & Caicos Islands",
+    "TD": "Chad",
+    "TF": "French Southern Territories",
+    "TG": "Togo",
+    "TH": "Thailand",
+    "TJ": "Tajikistan",
+    "TK": "Tokelau",
+    "TL": "Timor-Leste",
+    "TM": "Turkmenistan",
+    "TN": "Tunisia",
+    "TO": "Tonga",
+    "TR": "Turkey",
+    "TT": "Trinidad & Tobago",
+    "TV": "Tuvalu",
+    "TW": "Taiwan",
+    "TZ": "Tanzania",
+    "UA": "Ukraine",
+    "UG": "Uganda",
+    "UM": "U.S. Outlying Islands",
+    "US": "United States",
+    "UY": "Uruguay",
+    "UZ": "Uzbekistan",
+    "VA": "Vatican City",
+    "VC": "St. Vincent & Grenadines",
+    "VE": "Venezuela",
+    "VG": "British Virgin Islands",
+    "VI": "U.S. Virgin Islands",
+    "VN": "Vietnam",
+    "VU": "Vanuatu",
+    "WF": "Wallis & Futuna",
+    "WS": "Samoa",
+    "YE": "Yemen",
+    "YT": "Mayotte",
+    "ZA": "South Africa",
+    "ZM": "Zambia",
+    "ZW": "Zimbabwe",
+}
+
+
+def lookup_country(probe_cc: str) -> str:
+    """Translate 2-char country code into country name"""
+    return _countries[probe_cc.upper()]
+
+
+if __name__ == "__main__":
+    import json
+
+    with open("ooniapi/countries/country-list.json") as f:
+        d = {e["iso3166_alpha2"].upper(): e["name"] for e in json.load(f)}
+    print(dict(sorted(d.items())))

ooniapi/services/ooniprobe/src/ooniprobe/data.py

@@ -0,0 +1,56 @@
+dnscheck_inputs = [
+    "https://dns.google/dns-query",
+    "https://8.8.8.8/dns-query",
+    "dot://8.8.8.8:853/",
+    "dot://8.8.4.4:853/",
+    "https://8.8.4.4/dns-query",
+    "https://cloudflare-dns.com/dns-query",
+    "https://1.1.1.1/dns-query",
+    "https://1.0.0.1/dns-query",
+    "dot://1.1.1.1:853/",
+    "dot://1.0.0.1:853/",
+    "https://dns.quad9.net/dns-query",
+    "https://9.9.9.9/dns-query",
+    "dot://9.9.9.9:853/",
+    "dot://dns.quad9.net/",
+    "https://family.cloudflare-dns.com/dns-query",
+    "dot://family.cloudflare-dns.com/dns-query",
+    "https://dns11.quad9.net/dns-query",
+    "dot://dns11.quad9.net/dns-query",
+    "https://dns9.quad9.net/dns-query",
+    "dot://dns9.quad9.net/dns-query",
+    "https://dns12.quad9.net/dns-query",
+    "dot://dns12.quad9.net/dns-query",
+    "https://1dot1dot1dot1.cloudflare-dns.com/dns-query",
+    "dot://1dot1dot1dot1.cloudflare-dns.com/dns-query",
+    "https://dns.adguard.com/dns-query",
+    "dot://dns.adguard.com/dns-query",
+    "https://dns-family.adguard.com/dns-query",
+    "dot://dns-family.adguard.com/dns-query",
+    "https://dns.cloudflare.com/dns-query",
+    "https://adblock.doh.mullvad.net/dns-query",
+    "dot://adblock.doh.mullvad.net/dns-query",
+    "https://dns.alidns.com/dns-query",
+    "dot://dns.alidns.com/dns-query",
+    "https://doh.opendns.com/dns-query",
+    "https://dns.nextdns.io/dns-query",
+    "dot://dns.nextdns.io/dns-query",
+    "https://dns10.quad9.net/dns-query",
+    "dot://dns10.quad9.net/dns-query",
+    "https://security.cloudflare-dns.com/dns-query",
+    "dot://security.cloudflare-dns.com/dns-query",
+    "https://dns.switch.ch/dns-query",
+    "dot://dns.switch.ch/dns-query",
+]
+stunreachability_inputs = [
+    "stun://stun.voip.blackberry.com:3478",
+    "stun://stun.antisip.com:3478",
+    "stun://stun.bluesip.net:3478",
+    "stun://stun.dus.net:3478",
+    "stun://stun.epygi.com:3478",
+    "stun://stun.sonetel.com:3478",
+    "stun://stun.sonetel.net:3478",
+    "stun://stun.uls.co.za:3478",
+    "stun://stun.voipgate.com:3478",
+    "stun://stun.voys.nl:3478",
+]

ooniapi/services/ooniprobe/src/ooniprobe/main.py

@@ -22,7 +22,7 @@
 from .common.version import get_build_label
 from .dependencies import PostgresSessionDep, S3ClientDep, get_manifest
 from .download_geoip import try_update
-from .routers import bouncer, prio_crud, reports
+from .routers import bouncer, prio_crud, reports, private
 from .routers.v1 import probe_services
 from .routers.v2 import vpn
 
@@ -92,6 +92,7 @@ def update_geoip_task():
 app.include_router(reports.router)
 app.include_router(bouncer.router)
 app.include_router(prio_crud.router, prefix="/api")
+app.include_router(private.router, prefix="/api")
 
 
 @app.get("/version")