diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml index 1d87bd2ae..b66269ce6 100644 --- a/ooniapi/services/ooniprobe/pyproject.toml +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -14,7 +14,9 @@ dependencies = [ "ujson ~= 5.9.0", "urllib3 ~= 2.1.0", "python-dateutil ~= 2.8.2", + "pycountry ~= 26.2.16", "pydantic-settings ~= 2.1.0", + "pydantic_extra_types ~= 2.11.1", "statsd ~= 4.0.1", "uvicorn ~= 0.25.0", "psycopg2 ~= 2.9.9", diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py b/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py new file mode 100644 index 000000000..29bd0680c --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py @@ -0,0 +1,273 @@ +""" +CC to country name lookup table + +Regenerate the dict with: + + python3 ooniapi/countries/__init__.py + +""" + +_countries = { + "AD": "Andorra", + "AE": "United Arab Emirates", + "AF": "Afghanistan", + "AG": "Antigua & Barbuda", + "AI": "Anguilla", + "AL": "Albania", + "AM": "Armenia", + "AO": "Angola", + "AQ": "Antarctica", + "AR": "Argentina", + "AS": "American Samoa", + "AT": "Austria", + "AU": "Australia", + "AW": "Aruba", + "AX": "Åland Islands", + "AZ": "Azerbaijan", + "BA": "Bosnia", + "BB": "Barbados", + "BD": "Bangladesh", + "BE": "Belgium", + "BF": "Burkina Faso", + "BG": "Bulgaria", + "BH": "Bahrain", + "BI": "Burundi", + "BJ": "Benin", + "BL": "St. Barthélemy", + "BM": "Bermuda", + "BN": "Brunei", + "BO": "Bolivia", + "BQ": "Caribbean Netherlands", + "BR": "Brazil", + "BS": "Bahamas", + "BT": "Bhutan", + "BV": "Bouvet Island", + "BW": "Botswana", + "BY": "Belarus", + "BZ": "Belize", + "CA": "Canada", + "CC": "Cocos (Keeling) Islands", + "CD": "Congo - Kinshasa", + "CF": "Central African Republic", + "CG": "Congo - Brazzaville", + "CH": "Switzerland", + "CI": "Côte d’Ivoire", + "CK": "Cook Islands", + "CL": "Chile", + "CM": "Cameroon", + "CN": "China", + "CO": "Colombia", + "CR": "Costa Rica", + "CU": "Cuba", + "CV": "Cape Verde", + "CW": "Curaçao", + "CX": "Christmas Island", + "CY": "Cyprus", + "CZ": "Czechia", + "DE": "Germany", + "DJ": "Djibouti", + "DK": "Denmark", + "DM": "Dominica", + "DO": "Dominican Republic", + "DZ": "Algeria", + "EC": "Ecuador", + "EE": "Estonia", + "EG": "Egypt", + "EH": "Western Sahara", + "ER": "Eritrea", + "ES": "Spain", + "ET": "Ethiopia", + "FI": "Finland", + "FJ": "Fiji", + "FK": "Falkland Islands", + "FM": "Micronesia", + "FO": "Faroe Islands", + "FR": "France", + "GA": "Gabon", + "GB": "United Kingdom", + "GD": "Grenada", + "GE": "Georgia", + "GF": "French Guiana", + "GG": "Guernsey", + "GH": "Ghana", + "GI": "Gibraltar", + "GL": "Greenland", + "GM": "Gambia", + "GN": "Guinea", + "GP": "Guadeloupe", + "GQ": "Equatorial Guinea", + "GR": "Greece", + "GS": "South Georgia & South Sandwich Islands", + "GT": "Guatemala", + "GU": "Guam", + "GW": "Guinea-Bissau", + "GY": "Guyana", + "HK": "Hong Kong", + "HM": "Heard & McDonald Islands", + "HN": "Honduras", + "HR": "Croatia", + "HT": "Haiti", + "HU": "Hungary", + "ID": "Indonesia", + "IE": "Ireland", + "IL": "Israel", + "IM": "Isle of Man", + "IN": "India", + "IO": "British Indian Ocean Territory", + "IQ": "Iraq", + "IR": "Iran", + "IS": "Iceland", + "IT": "Italy", + "JE": "Jersey", + "JM": "Jamaica", + "JO": "Jordan", + "JP": "Japan", + "KE": "Kenya", + "KG": "Kyrgyzstan", + "KH": "Cambodia", + "KI": "Kiribati", + "KM": "Comoros", + "KN": "St. Kitts & Nevis", + "KP": "North Korea", + "KR": "South Korea", + "KW": "Kuwait", + "KY": "Cayman Islands", + "KZ": "Kazakhstan", + "LA": "Laos", + "LB": "Lebanon", + "LC": "St. Lucia", + "LI": "Liechtenstein", + "LK": "Sri Lanka", + "LR": "Liberia", + "LS": "Lesotho", + "LT": "Lithuania", + "LU": "Luxembourg", + "LV": "Latvia", + "LY": "Libya", + "MA": "Morocco", + "MC": "Monaco", + "MD": "Moldova", + "ME": "Montenegro", + "MF": "St. Martin", + "MG": "Madagascar", + "MH": "Marshall Islands", + "MK": "North Macedonia", + "ML": "Mali", + "MM": "Myanmar", + "MN": "Mongolia", + "MO": "Macao", + "MP": "Northern Mariana Islands", + "MQ": "Martinique", + "MR": "Mauritania", + "MS": "Montserrat", + "MT": "Malta", + "MU": "Mauritius", + "MV": "Maldives", + "MW": "Malawi", + "MX": "Mexico", + "MY": "Malaysia", + "MZ": "Mozambique", + "NA": "Namibia", + "NC": "New Caledonia", + "NE": "Niger", + "NF": "Norfolk Island", + "NG": "Nigeria", + "NI": "Nicaragua", + "NL": "Netherlands", + "NO": "Norway", + "NP": "Nepal", + "NR": "Nauru", + "NU": "Niue", + "NZ": "New Zealand", + "OM": "Oman", + "PA": "Panama", + "PE": "Peru", + "PF": "French Polynesia", + "PG": "Papua New Guinea", + "PH": "Philippines", + "PK": "Pakistan", + "PL": "Poland", + "PM": "St. Pierre & Miquelon", + "PN": "Pitcairn Islands", + "PR": "Puerto Rico", + "PS": "Palestine", + "PT": "Portugal", + "PW": "Palau", + "PY": "Paraguay", + "QA": "Qatar", + "RE": "Réunion", + "RO": "Romania", + "RS": "Serbia", + "RU": "Russia", + "RW": "Rwanda", + "SA": "Saudi Arabia", + "SB": "Solomon Islands", + "SC": "Seychelles", + "SD": "Sudan", + "SE": "Sweden", + "SG": "Singapore", + "SH": "St. Helena", + "SI": "Slovenia", + "SJ": "Svalbard & Jan Mayen", + "SK": "Slovakia", + "SL": "Sierra Leone", + "SM": "San Marino", + "SN": "Senegal", + "SO": "Somalia", + "SR": "Suriname", + "SS": "South Sudan", + "ST": "São Tomé & Príncipe", + "SV": "El Salvador", + "SX": "Sint Maarten", + "SY": "Syria", + "SZ": "Eswatini", + "TC": "Turks & Caicos Islands", + "TD": "Chad", + "TF": "French Southern Territories", + "TG": "Togo", + "TH": "Thailand", + "TJ": "Tajikistan", + "TK": "Tokelau", + "TL": "Timor-Leste", + "TM": "Turkmenistan", + "TN": "Tunisia", + "TO": "Tonga", + "TR": "Turkey", + "TT": "Trinidad & Tobago", + "TV": "Tuvalu", + "TW": "Taiwan", + "TZ": "Tanzania", + "UA": "Ukraine", + "UG": "Uganda", + "UM": "U.S. Outlying Islands", + "US": "United States", + "UY": "Uruguay", + "UZ": "Uzbekistan", + "VA": "Vatican City", + "VC": "St. Vincent & Grenadines", + "VE": "Venezuela", + "VG": "British Virgin Islands", + "VI": "U.S. Virgin Islands", + "VN": "Vietnam", + "VU": "Vanuatu", + "WF": "Wallis & Futuna", + "WS": "Samoa", + "YE": "Yemen", + "YT": "Mayotte", + "ZA": "South Africa", + "ZM": "Zambia", + "ZW": "Zimbabwe", +} + + +def lookup_country(probe_cc: str) -> str: + """Translate 2-char country code into country name""" + return _countries[probe_cc.upper()] + + +if __name__ == "__main__": + import json + + with open("ooniapi/countries/country-list.json") as f: + d = {e["iso3166_alpha2"].upper(): e["name"] for e in json.load(f)} + print(dict(sorted(d.items()))) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json b/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json new file mode 100644 index 000000000..cb30c2550 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json @@ -0,0 +1 @@ +[{"iso3166_alpha2": "TW", "iso3166_alpha3": "TWN", "iso3166_num": "158", "iso3166_name": "Taiwan", "name": "Taiwan", "languages": ["zh-TW", "zh", "nan", "hak"], "tld": ".tw", "capital": "Taipei", "region_code": "142", "sub_region_code": "156"}, {"iso3166_alpha2": "AF", "iso3166_alpha3": "AFG", "iso3166_num": "004", "iso3166_name": "Afghanistan", "name": "Afghanistan", "languages": ["fa-AF", "ps", "uz-AF", "tk"], "tld": ".af", "capital": "Kabul", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "AL", "iso3166_alpha3": "ALB", "iso3166_num": "008", "iso3166_name": "Albania", "name": "Albania", "languages": ["sq", "el"], "tld": ".al", "capital": "Tirana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "DZ", "iso3166_alpha3": "DZA", "iso3166_num": "012", "iso3166_name": "Algeria", "name": "Algeria", "languages": ["ar-DZ"], "tld": ".dz", "capital": "Algiers", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "AS", "iso3166_alpha3": "ASM", "iso3166_num": "016", "iso3166_name": "American Samoa", "name": "American Samoa", "languages": ["en-AS", "sm", "to"], "tld": ".as", "capital": "Pago Pago", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "AD", "iso3166_alpha3": "AND", "iso3166_num": "020", "iso3166_name": "Andorra", "name": "Andorra", "languages": ["ca"], "tld": ".ad", "capital": "Andorra la Vella", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "AO", "iso3166_alpha3": "AGO", "iso3166_num": "024", "iso3166_name": "Angola", "name": "Angola", "languages": ["pt-AO"], "tld": ".ao", "capital": "Luanda", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AI", "iso3166_alpha3": "AIA", "iso3166_num": "660", "iso3166_name": "Anguilla", "name": "Anguilla", "languages": ["en-AI"], "tld": ".ai", "capital": "The Valley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AQ", "iso3166_alpha3": "ATA", "iso3166_num": "010", "iso3166_name": "Antarctica", "name": "Antarctica", "languages": [""], "tld": ".aq", "capital": "", "region_code": "", "sub_region_code": ""}, {"iso3166_alpha2": "AG", "iso3166_alpha3": "ATG", "iso3166_num": "028", "iso3166_name": "Antigua & Barbuda", "name": "Antigua & Barbuda", "languages": ["en-AG"], "tld": ".ag", "capital": "St. John's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AR", "iso3166_alpha3": "ARG", "iso3166_num": "032", "iso3166_name": "Argentina", "name": "Argentina", "languages": ["es-AR", "en", "it", "de", "fr", "gn"], "tld": ".ar", "capital": "Buenos Aires", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AM", "iso3166_alpha3": "ARM", "iso3166_num": "051", "iso3166_name": "Armenia", "name": "Armenia", "languages": ["hy"], "tld": ".am", "capital": "Yerevan", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "AW", "iso3166_alpha3": "ABW", "iso3166_num": "533", "iso3166_name": "Aruba", "name": "Aruba", "languages": ["nl-AW", "pap", "es", "en"], "tld": ".aw", "capital": "Oranjestad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AU", "iso3166_alpha3": "AUS", "iso3166_num": "036", "iso3166_name": "Australia", "name": "Australia", "languages": ["en-AU"], "tld": ".au", "capital": "Canberra", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "AT", "iso3166_alpha3": "AUT", "iso3166_num": "040", "iso3166_name": "Austria", "name": "Austria", "languages": ["de-AT", "hr", "hu", "sl"], "tld": ".at", "capital": "Vienna", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "AZ", "iso3166_alpha3": "AZE", "iso3166_num": "031", "iso3166_name": "Azerbaijan", "name": "Azerbaijan", "languages": ["az", "ru", "hy"], "tld": ".az", "capital": "Baku", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BS", "iso3166_alpha3": "BHS", "iso3166_num": "044", "iso3166_name": "Bahamas", "name": "Bahamas", "languages": ["en-BS"], "tld": ".bs", "capital": "Nassau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BH", "iso3166_alpha3": "BHR", "iso3166_num": "048", "iso3166_name": "Bahrain", "name": "Bahrain", "languages": ["ar-BH", "en", "fa", "ur"], "tld": ".bh", "capital": "Manama", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BD", "iso3166_alpha3": "BGD", "iso3166_num": "050", "iso3166_name": "Bangladesh", "name": "Bangladesh", "languages": ["bn-BD", "en"], "tld": ".bd", "capital": "Dhaka", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BB", "iso3166_alpha3": "BRB", "iso3166_num": "052", "iso3166_name": "Barbados", "name": "Barbados", "languages": ["en-BB"], "tld": ".bb", "capital": "Bridgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BY", "iso3166_alpha3": "BLR", "iso3166_num": "112", "iso3166_name": "Belarus", "name": "Belarus", "languages": ["be", "ru"], "tld": ".by", "capital": "Minsk", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BE", "iso3166_alpha3": "BEL", "iso3166_num": "056", "iso3166_name": "Belgium", "name": "Belgium", "languages": ["nl-BE", "fr-BE", "de-BE"], "tld": ".be", "capital": "Brussels", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "BZ", "iso3166_alpha3": "BLZ", "iso3166_num": "084", "iso3166_name": "Belize", "name": "Belize", "languages": ["en-BZ", "es"], "tld": ".bz", "capital": "Belmopan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BJ", "iso3166_alpha3": "BEN", "iso3166_num": "204", "iso3166_name": "Benin", "name": "Benin", "languages": ["fr-BJ"], "tld": ".bj", "capital": "Porto-Novo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BM", "iso3166_alpha3": "BMU", "iso3166_num": "060", "iso3166_name": "Bermuda", "name": "Bermuda", "languages": ["en-BM", "pt"], "tld": ".bm", "capital": "Hamilton", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "BT", "iso3166_alpha3": "BTN", "iso3166_num": "064", "iso3166_name": "Bhutan", "name": "Bhutan", "languages": ["dz"], "tld": ".bt", "capital": "Thimphu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BO", "iso3166_alpha3": "BOL", "iso3166_num": "068", "iso3166_name": "Bolivia", "name": "Bolivia", "languages": ["es-BO", "qu", "ay"], "tld": ".bo", "capital": "Sucre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BQ", "iso3166_alpha3": "BES", "iso3166_num": "535", "iso3166_name": "Caribbean Netherlands", "name": "Caribbean Netherlands", "languages": ["nl", "pap", "en"], "tld": ".bq", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BA", "iso3166_alpha3": "BIH", "iso3166_num": "070", "iso3166_name": "Bosnia", "name": "Bosnia", "languages": ["bs", "hr-BA", "sr-BA"], "tld": ".ba", "capital": "Sarajevo", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "BW", "iso3166_alpha3": "BWA", "iso3166_num": "072", "iso3166_name": "Botswana", "name": "Botswana", "languages": ["en-BW", "tn-BW"], "tld": ".bw", "capital": "Gaborone", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BV", "iso3166_alpha3": "BVT", "iso3166_num": "074", "iso3166_name": "Bouvet Island", "name": "Bouvet Island", "languages": [""], "tld": ".bv", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BR", "iso3166_alpha3": "BRA", "iso3166_num": "076", "iso3166_name": "Brazil", "name": "Brazil", "languages": ["pt-BR", "es", "en", "fr"], "tld": ".br", "capital": "Brasilia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "IO", "iso3166_alpha3": "IOT", "iso3166_num": "086", "iso3166_name": "British Indian Ocean Territory", "name": "British Indian Ocean Territory", "languages": ["en-IO"], "tld": ".io", "capital": "Diego Garcia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "VG", "iso3166_alpha3": "VGB", "iso3166_num": "092", "iso3166_name": "British Virgin Islands", "name": "British Virgin Islands", "languages": ["en-VG"], "tld": ".vg", "capital": "Road Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BN", "iso3166_alpha3": "BRN", "iso3166_num": "096", "iso3166_name": "Brunei", "name": "Brunei", "languages": ["ms-BN", "en-BN"], "tld": ".bn", "capital": "Bandar Seri Begawan", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "BG", "iso3166_alpha3": "BGR", "iso3166_num": "100", "iso3166_name": "Bulgaria", "name": "Bulgaria", "languages": ["bg", "tr-BG", "rom"], "tld": ".bg", "capital": "Sofia", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BF", "iso3166_alpha3": "BFA", "iso3166_num": "854", "iso3166_name": "Burkina Faso", "name": "Burkina Faso", "languages": ["fr-BF", "mos"], "tld": ".bf", "capital": "Ouagadougou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BI", "iso3166_alpha3": "BDI", "iso3166_num": "108", "iso3166_name": "Burundi", "name": "Burundi", "languages": ["fr-BI", "rn"], "tld": ".bi", "capital": "Bujumbura", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CV", "iso3166_alpha3": "CPV", "iso3166_num": "132", "iso3166_name": "Cape Verde", "name": "Cape Verde", "languages": ["pt-CV"], "tld": ".cv", "capital": "Praia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KH", "iso3166_alpha3": "KHM", "iso3166_num": "116", "iso3166_name": "Cambodia", "name": "Cambodia", "languages": ["km", "fr", "en"], "tld": ".kh", "capital": "Phnom Penh", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "CM", "iso3166_alpha3": "CMR", "iso3166_num": "120", "iso3166_name": "Cameroon", "name": "Cameroon", "languages": ["en-CM", "fr-CM"], "tld": ".cm", "capital": "Yaounde", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CA", "iso3166_alpha3": "CAN", "iso3166_num": "124", "iso3166_name": "Canada", "name": "Canada", "languages": ["en-CA", "fr-CA", "iu"], "tld": ".ca", "capital": "Ottawa", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "KY", "iso3166_alpha3": "CYM", "iso3166_num": "136", "iso3166_name": "Cayman Islands", "name": "Cayman Islands", "languages": ["en-KY"], "tld": ".ky", "capital": "George Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CF", "iso3166_alpha3": "CAF", "iso3166_num": "140", "iso3166_name": "Central African Republic", "name": "Central African Republic", "languages": ["fr-CF", "sg", "ln", "kg"], "tld": ".cf", "capital": "Bangui", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TD", "iso3166_alpha3": "TCD", "iso3166_num": "148", "iso3166_name": "Chad", "name": "Chad", "languages": ["fr-TD", "ar-TD", "sre"], "tld": ".td", "capital": "N'Djamena", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CL", "iso3166_alpha3": "CHL", "iso3166_num": "152", "iso3166_name": "Chile", "name": "Chile", "languages": ["es-CL"], "tld": ".cl", "capital": "Santiago", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CN", "iso3166_alpha3": "CHN", "iso3166_num": "156", "iso3166_name": "China", "name": "China", "languages": ["zh-CN", "yue", "wuu", "dta", "ug", "za"], "tld": ".cn", "capital": "Beijing", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "HK", "iso3166_alpha3": "HKG", "iso3166_num": "344", "iso3166_name": "Hong Kong", "name": "Hong Kong", "languages": ["zh-HK", "yue", "zh", "en"], "tld": ".hk", "capital": "Hong Kong", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MO", "iso3166_alpha3": "MAC", "iso3166_num": "446", "iso3166_name": "Macau", "name": "Macao", "languages": ["zh", "zh-MO", "pt"], "tld": ".mo", "capital": "Macao", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CX", "iso3166_alpha3": "CXR", "iso3166_num": "162", "iso3166_name": "Christmas Island", "name": "Christmas Island", "languages": ["en", "zh", "ms-CC"], "tld": ".cx", "capital": "Flying Fish Cove", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CC", "iso3166_alpha3": "CCK", "iso3166_num": "166", "iso3166_name": "Cocos (Keeling) Islands", "name": "Cocos (Keeling) Islands", "languages": ["ms-CC", "en"], "tld": ".cc", "capital": "West Island", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CO", "iso3166_alpha3": "COL", "iso3166_num": "170", "iso3166_name": "Colombia", "name": "Colombia", "languages": ["es-CO"], "tld": ".co", "capital": "Bogota", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "KM", "iso3166_alpha3": "COM", "iso3166_num": "174", "iso3166_name": "Comoros", "name": "Comoros", "languages": ["ar", "fr-KM"], "tld": ".km", "capital": "Moroni", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CG", "iso3166_alpha3": "COG", "iso3166_num": "178", "iso3166_name": "Congo - Brazzaville", "name": "Congo - Brazzaville", "languages": ["fr-CG", "kg", "ln-CG"], "tld": ".cg", "capital": "Brazzaville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CK", "iso3166_alpha3": "COK", "iso3166_num": "184", "iso3166_name": "Cook Islands", "name": "Cook Islands", "languages": ["en-CK", "mi"], "tld": ".ck", "capital": "Avarua", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "CR", "iso3166_alpha3": "CRI", "iso3166_num": "188", "iso3166_name": "Costa Rica", "name": "Costa Rica", "languages": ["es-CR", "en"], "tld": ".cr", "capital": "San Jose", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HR", "iso3166_alpha3": "HRV", "iso3166_num": "191", "iso3166_name": "Croatia", "name": "Croatia", "languages": ["hr-HR", "sr"], "tld": ".hr", "capital": "Zagreb", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "CU", "iso3166_alpha3": "CUB", "iso3166_num": "192", "iso3166_name": "Cuba", "name": "Cuba", "languages": ["es-CU", "pap"], "tld": ".cu", "capital": "Havana", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CW", "iso3166_alpha3": "CUW", "iso3166_num": "531", "iso3166_name": "Cura\u00e7ao", "name": "Cura\u00e7ao", "languages": ["nl", "pap"], "tld": ".cw", "capital": " Willemstad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CY", "iso3166_alpha3": "CYP", "iso3166_num": "196", "iso3166_name": "Cyprus", "name": "Cyprus", "languages": ["el-CY", "tr-CY", "en"], "tld": ".cy", "capital": "Nicosia", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "CZ", "iso3166_alpha3": "CZE", "iso3166_num": "203", "iso3166_name": "Czechia", "name": "Czechia", "languages": ["cs", "sk"], "tld": ".cz", "capital": "Prague", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "CI", "iso3166_alpha3": "CIV", "iso3166_num": "384", "iso3166_name": "C\u00f4te d\u2019Ivoire", "name": "C\u00f4te d\u2019Ivoire", "languages": ["fr-CI"], "tld": ".ci", "capital": "Yamoussoukro", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KP", "iso3166_alpha3": "PRK", "iso3166_num": "408", "iso3166_name": "North Korea", "name": "North Korea", "languages": ["ko-KP"], "tld": ".kp", "capital": "Pyongyang", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CD", "iso3166_alpha3": "COD", "iso3166_num": "180", "iso3166_name": "Congo - Kinshasa", "name": "Congo - Kinshasa", "languages": ["fr-CD", "ln", "ktu", "kg", "sw", "lua"], "tld": ".cd", "capital": "Kinshasa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DK", "iso3166_alpha3": "DNK", "iso3166_num": "208", "iso3166_name": "Denmark", "name": "Denmark", "languages": ["da-DK", "en", "fo", "de-DK"], "tld": ".dk", "capital": "Copenhagen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "DJ", "iso3166_alpha3": "DJI", "iso3166_num": "262", "iso3166_name": "Djibouti", "name": "Djibouti", "languages": ["fr-DJ", "ar", "so-DJ", "aa"], "tld": ".dj", "capital": "Djibouti", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DM", "iso3166_alpha3": "DMA", "iso3166_num": "212", "iso3166_name": "Dominica", "name": "Dominica", "languages": ["en-DM"], "tld": ".dm", "capital": "Roseau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "DO", "iso3166_alpha3": "DOM", "iso3166_num": "214", "iso3166_name": "Dominican Republic", "name": "Dominican Republic", "languages": ["es-DO"], "tld": ".do", "capital": "Santo Domingo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EC", "iso3166_alpha3": "ECU", "iso3166_num": "218", "iso3166_name": "Ecuador", "name": "Ecuador", "languages": ["es-EC"], "tld": ".ec", "capital": "Quito", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EG", "iso3166_alpha3": "EGY", "iso3166_num": "818", "iso3166_name": "Egypt", "name": "Egypt", "languages": ["ar-EG", "en", "fr"], "tld": ".eg", "capital": "Cairo", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SV", "iso3166_alpha3": "SLV", "iso3166_num": "222", "iso3166_name": "El Salvador", "name": "El Salvador", "languages": ["es-SV"], "tld": ".sv", "capital": "San Salvador", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GQ", "iso3166_alpha3": "GNQ", "iso3166_num": "226", "iso3166_name": "Equatorial Guinea", "name": "Equatorial Guinea", "languages": ["es-GQ", "fr"], "tld": ".gq", "capital": "Malabo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ER", "iso3166_alpha3": "ERI", "iso3166_num": "232", "iso3166_name": "Eritrea", "name": "Eritrea", "languages": ["aa-ER", "ar", "tig", "kun", "ti-ER"], "tld": ".er", "capital": "Asmara", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "EE", "iso3166_alpha3": "EST", "iso3166_num": "233", "iso3166_name": "Estonia", "name": "Estonia", "languages": ["et", "ru"], "tld": ".ee", "capital": "Tallinn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "ET", "iso3166_alpha3": "ETH", "iso3166_num": "231", "iso3166_name": "Ethiopia", "name": "Ethiopia", "languages": ["am", "en-ET", "om-ET", "ti-ET", "so-ET", "sid"], "tld": ".et", "capital": "Addis Ababa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "FK", "iso3166_alpha3": "FLK", "iso3166_num": "238", "iso3166_name": "Falkland Islands", "name": "Falkland Islands", "languages": ["en-FK"], "tld": ".fk", "capital": "Stanley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FO", "iso3166_alpha3": "FRO", "iso3166_num": "234", "iso3166_name": "Faroe Islands", "name": "Faroe Islands", "languages": ["fo", "da-FO"], "tld": ".fo", "capital": "Torshavn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FJ", "iso3166_alpha3": "FJI", "iso3166_num": "242", "iso3166_name": "Fiji", "name": "Fiji", "languages": ["en-FJ", "fj"], "tld": ".fj", "capital": "Suva", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "FI", "iso3166_alpha3": "FIN", "iso3166_num": "246", "iso3166_name": "Finland", "name": "Finland", "languages": ["fi-FI", "sv-FI", "smn"], "tld": ".fi", "capital": "Helsinki", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FR", "iso3166_alpha3": "FRA", "iso3166_num": "250", "iso3166_name": "France", "name": "France", "languages": ["fr-FR", "frp", "br", "co", "ca", "eu", "oc"], "tld": ".fr", "capital": "Paris", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GF", "iso3166_alpha3": "GUF", "iso3166_num": "254", "iso3166_name": "French Guiana", "name": "French Guiana", "languages": ["fr-GF"], "tld": ".gf", "capital": "Cayenne", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PF", "iso3166_alpha3": "PYF", "iso3166_num": "258", "iso3166_name": "French Polynesia", "name": "French Polynesia", "languages": ["fr-PF", "ty"], "tld": ".pf", "capital": "Papeete", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TF", "iso3166_alpha3": "ATF", "iso3166_num": "260", "iso3166_name": "French Southern Territories", "name": "French Southern Territories", "languages": ["fr"], "tld": ".tf", "capital": "Port-aux-Francais", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GA", "iso3166_alpha3": "GAB", "iso3166_num": "266", "iso3166_name": "Gabon", "name": "Gabon", "languages": ["fr-GA"], "tld": ".ga", "capital": "Libreville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GM", "iso3166_alpha3": "GMB", "iso3166_num": "270", "iso3166_name": "Gambia", "name": "Gambia", "languages": ["en-GM", "mnk", "wof", "wo", "ff"], "tld": ".gm", "capital": "Banjul", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GE", "iso3166_alpha3": "GEO", "iso3166_num": "268", "iso3166_name": "Georgia", "name": "Georgia", "languages": ["ka", "ru", "hy", "az"], "tld": ".ge", "capital": "Tbilisi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "DE", "iso3166_alpha3": "DEU", "iso3166_num": "276", "iso3166_name": "Germany", "name": "Germany", "languages": ["de"], "tld": ".de", "capital": "Berlin", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GH", "iso3166_alpha3": "GHA", "iso3166_num": "288", "iso3166_name": "Ghana", "name": "Ghana", "languages": ["en-GH", "ak", "ee", "tw"], "tld": ".gh", "capital": "Accra", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GI", "iso3166_alpha3": "GIB", "iso3166_num": "292", "iso3166_name": "Gibraltar", "name": "Gibraltar", "languages": ["en-GI", "es", "it", "pt"], "tld": ".gi", "capital": "Gibraltar", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GR", "iso3166_alpha3": "GRC", "iso3166_num": "300", "iso3166_name": "Greece", "name": "Greece", "languages": ["el-GR", "en", "fr"], "tld": ".gr", "capital": "Athens", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GL", "iso3166_alpha3": "GRL", "iso3166_num": "304", "iso3166_name": "Greenland", "name": "Greenland", "languages": ["kl", "da-GL", "en"], "tld": ".gl", "capital": "Nuuk", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "GD", "iso3166_alpha3": "GRD", "iso3166_num": "308", "iso3166_name": "Grenada", "name": "Grenada", "languages": ["en-GD"], "tld": ".gd", "capital": "St. George's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GP", "iso3166_alpha3": "GLP", "iso3166_num": "312", "iso3166_name": "Guadeloupe", "name": "Guadeloupe", "languages": ["fr-GP"], "tld": ".gp", "capital": "Basse-Terre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GU", "iso3166_alpha3": "GUM", "iso3166_num": "316", "iso3166_name": "Guam", "name": "Guam", "languages": ["en-GU", "ch-GU"], "tld": ".gu", "capital": "Hagatna", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "GT", "iso3166_alpha3": "GTM", "iso3166_num": "320", "iso3166_name": "Guatemala", "name": "Guatemala", "languages": ["es-GT"], "tld": ".gt", "capital": "Guatemala City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GG", "iso3166_alpha3": "GGY", "iso3166_num": "831", "iso3166_name": "Guernsey", "name": "Guernsey", "languages": ["en", "nrf"], "tld": ".gg", "capital": "St Peter Port", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "GN", "iso3166_alpha3": "GIN", "iso3166_num": "324", "iso3166_name": "Guinea", "name": "Guinea", "languages": ["fr-GN"], "tld": ".gn", "capital": "Conakry", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GW", "iso3166_alpha3": "GNB", "iso3166_num": "624", "iso3166_name": "Guinea-Bissau", "name": "Guinea-Bissau", "languages": ["pt-GW", "pov"], "tld": ".gw", "capital": "Bissau", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GY", "iso3166_alpha3": "GUY", "iso3166_num": "328", "iso3166_name": "Guyana", "name": "Guyana", "languages": ["en-GY"], "tld": ".gy", "capital": "Georgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HT", "iso3166_alpha3": "HTI", "iso3166_num": "332", "iso3166_name": "Haiti", "name": "Haiti", "languages": ["ht", "fr-HT"], "tld": ".ht", "capital": "Port-au-Prince", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HM", "iso3166_alpha3": "HMD", "iso3166_num": "334", "iso3166_name": "Heard & McDonald Islands", "name": "Heard & McDonald Islands", "languages": [""], "tld": ".hm", "capital": "", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "VA", "iso3166_alpha3": "VAT", "iso3166_num": "336", "iso3166_name": "Vatican City", "name": "Vatican City", "languages": ["la", "it", "fr"], "tld": ".va", "capital": "Vatican City", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "HN", "iso3166_alpha3": "HND", "iso3166_num": "340", "iso3166_name": "Honduras", "name": "Honduras", "languages": ["es-HN", "cab", "miq"], "tld": ".hn", "capital": "Tegucigalpa", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HU", "iso3166_alpha3": "HUN", "iso3166_num": "348", "iso3166_name": "Hungary", "name": "Hungary", "languages": ["hu-HU"], "tld": ".hu", "capital": "Budapest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "IS", "iso3166_alpha3": "ISL", "iso3166_num": "352", "iso3166_name": "Iceland", "name": "Iceland", "languages": ["is", "en", "de", "da", "sv", "no"], "tld": ".is", "capital": "Reykjavik", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IN", "iso3166_alpha3": "IND", "iso3166_num": "356", "iso3166_name": "India", "name": "India", "languages": ["en-IN", "hi", "bn", "te", "mr", "ta", "ur", "gu", "kn", "ml", "or", "pa", "as", "bh", "sat", "ks", "ne", "sd", "kok", "doi", "mni", "sit", "sa", "fr", "lus", "inc"], "tld": ".in", "capital": "New Delhi", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ID", "iso3166_alpha3": "IDN", "iso3166_num": "360", "iso3166_name": "Indonesia", "name": "Indonesia", "languages": ["id", "en", "nl", "jv"], "tld": ".id", "capital": "Jakarta", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "IR", "iso3166_alpha3": "IRN", "iso3166_num": "364", "iso3166_name": "Iran", "name": "Iran", "languages": ["fa-IR", "ku"], "tld": ".ir", "capital": "Tehran", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "IQ", "iso3166_alpha3": "IRQ", "iso3166_num": "368", "iso3166_name": "Iraq", "name": "Iraq", "languages": ["ar-IQ", "ku", "hy"], "tld": ".iq", "capital": "Baghdad", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IE", "iso3166_alpha3": "IRL", "iso3166_num": "372", "iso3166_name": "Ireland", "name": "Ireland", "languages": ["en-IE", "ga-IE"], "tld": ".ie", "capital": "Dublin", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IM", "iso3166_alpha3": "IMN", "iso3166_num": "833", "iso3166_name": "Isle of Man", "name": "Isle of Man", "languages": ["en", "gv"], "tld": ".im", "capital": "Douglas", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IL", "iso3166_alpha3": "ISR", "iso3166_num": "376", "iso3166_name": "Israel", "name": "Israel", "languages": ["he", "ar-IL", "en-IL", ""], "tld": ".il", "capital": "Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IT", "iso3166_alpha3": "ITA", "iso3166_num": "380", "iso3166_name": "Italy", "name": "Italy", "languages": ["it-IT", "de-IT", "fr-IT", "sc", "ca", "co", "sl"], "tld": ".it", "capital": "Rome", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "JM", "iso3166_alpha3": "JAM", "iso3166_num": "388", "iso3166_name": "Jamaica", "name": "Jamaica", "languages": ["en-JM"], "tld": ".jm", "capital": "Kingston", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "JP", "iso3166_alpha3": "JPN", "iso3166_num": "392", "iso3166_name": "Japan", "name": "Japan", "languages": ["ja"], "tld": ".jp", "capital": "Tokyo", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "JE", "iso3166_alpha3": "JEY", "iso3166_num": "832", "iso3166_name": "Jersey", "name": "Jersey", "languages": ["en", "fr", "nrf"], "tld": ".je", "capital": "Saint Helier", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "JO", "iso3166_alpha3": "JOR", "iso3166_num": "400", "iso3166_name": "Jordan", "name": "Jordan", "languages": ["ar-JO", "en"], "tld": ".jo", "capital": "Amman", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KZ", "iso3166_alpha3": "KAZ", "iso3166_num": "398", "iso3166_name": "Kazakhstan", "name": "Kazakhstan", "languages": ["kk", "ru"], "tld": ".kz", "capital": "Nur-Sultan", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "KE", "iso3166_alpha3": "KEN", "iso3166_num": "404", "iso3166_name": "Kenya", "name": "Kenya", "languages": ["en-KE", "sw-KE"], "tld": ".ke", "capital": "Nairobi", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KI", "iso3166_alpha3": "KIR", "iso3166_num": "296", "iso3166_name": "Kiribati", "name": "Kiribati", "languages": ["en-KI", "gil"], "tld": ".ki", "capital": "Tarawa", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "KW", "iso3166_alpha3": "KWT", "iso3166_num": "414", "iso3166_name": "Kuwait", "name": "Kuwait", "languages": ["ar-KW", "en"], "tld": ".kw", "capital": "Kuwait City", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KG", "iso3166_alpha3": "KGZ", "iso3166_num": "417", "iso3166_name": "Kyrgyzstan", "name": "Kyrgyzstan", "languages": ["ky", "uz", "ru"], "tld": ".kg", "capital": "Bishkek", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "LA", "iso3166_alpha3": "LAO", "iso3166_num": "418", "iso3166_name": "Laos", "name": "Laos", "languages": ["lo", "fr", "en"], "tld": ".la", "capital": "Vientiane", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "LV", "iso3166_alpha3": "LVA", "iso3166_num": "428", "iso3166_name": "Latvia", "name": "Latvia", "languages": ["lv", "ru", "lt"], "tld": ".lv", "capital": "Riga", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LB", "iso3166_alpha3": "LBN", "iso3166_num": "422", "iso3166_name": "Lebanon", "name": "Lebanon", "languages": ["ar-LB", "fr-LB", "en", "hy"], "tld": ".lb", "capital": "Beirut", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "LS", "iso3166_alpha3": "LSO", "iso3166_num": "426", "iso3166_name": "Lesotho", "name": "Lesotho", "languages": ["en-LS", "st", "zu", "xh"], "tld": ".ls", "capital": "Maseru", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LR", "iso3166_alpha3": "LBR", "iso3166_num": "430", "iso3166_name": "Liberia", "name": "Liberia", "languages": ["en-LR"], "tld": ".lr", "capital": "Monrovia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LY", "iso3166_alpha3": "LBY", "iso3166_num": "434", "iso3166_name": "Libya", "name": "Libya", "languages": ["ar-LY", "it", "en"], "tld": ".ly", "capital": "Tripoli", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "LI", "iso3166_alpha3": "LIE", "iso3166_num": "438", "iso3166_name": "Liechtenstein", "name": "Liechtenstein", "languages": ["de-LI"], "tld": ".li", "capital": "Vaduz", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "LT", "iso3166_alpha3": "LTU", "iso3166_num": "440", "iso3166_name": "Lithuania", "name": "Lithuania", "languages": ["lt", "ru", "pl"], "tld": ".lt", "capital": "Vilnius", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LU", "iso3166_alpha3": "LUX", "iso3166_num": "442", "iso3166_name": "Luxembourg", "name": "Luxembourg", "languages": ["lb", "de-LU", "fr-LU"], "tld": ".lu", "capital": "Luxembourg", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MG", "iso3166_alpha3": "MDG", "iso3166_num": "450", "iso3166_name": "Madagascar", "name": "Madagascar", "languages": ["fr-MG", "mg"], "tld": ".mg", "capital": "Antananarivo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MW", "iso3166_alpha3": "MWI", "iso3166_num": "454", "iso3166_name": "Malawi", "name": "Malawi", "languages": ["ny", "yao", "tum", "swk"], "tld": ".mw", "capital": "Lilongwe", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MY", "iso3166_alpha3": "MYS", "iso3166_num": "458", "iso3166_name": "Malaysia", "name": "Malaysia", "languages": ["ms-MY", "en", "zh", "ta", "te", "ml", "pa", "th"], "tld": ".my", "capital": "Kuala Lumpur", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MV", "iso3166_alpha3": "MDV", "iso3166_num": "462", "iso3166_name": "Maldives", "name": "Maldives", "languages": ["dv", "en"], "tld": ".mv", "capital": "Male", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ML", "iso3166_alpha3": "MLI", "iso3166_num": "466", "iso3166_name": "Mali", "name": "Mali", "languages": ["fr-ML", "bm"], "tld": ".ml", "capital": "Bamako", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MT", "iso3166_alpha3": "MLT", "iso3166_num": "470", "iso3166_name": "Malta", "name": "Malta", "languages": ["mt", "en-MT"], "tld": ".mt", "capital": "Valletta", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MH", "iso3166_alpha3": "MHL", "iso3166_num": "584", "iso3166_name": "Marshall Islands", "name": "Marshall Islands", "languages": ["mh", "en-MH"], "tld": ".mh", "capital": "Majuro", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MQ", "iso3166_alpha3": "MTQ", "iso3166_num": "474", "iso3166_name": "Martinique", "name": "Martinique", "languages": ["fr-MQ"], "tld": ".mq", "capital": "Fort-de-France", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MR", "iso3166_alpha3": "MRT", "iso3166_num": "478", "iso3166_name": "Mauritania", "name": "Mauritania", "languages": ["ar-MR", "fuc", "snk", "fr", "mey", "wo"], "tld": ".mr", "capital": "Nouakchott", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MU", "iso3166_alpha3": "MUS", "iso3166_num": "480", "iso3166_name": "Mauritius", "name": "Mauritius", "languages": ["en-MU", "bho", "fr"], "tld": ".mu", "capital": "Port Louis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "YT", "iso3166_alpha3": "MYT", "iso3166_num": "175", "iso3166_name": "Mayotte", "name": "Mayotte", "languages": ["fr-YT"], "tld": ".yt", "capital": "Mamoudzou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MX", "iso3166_alpha3": "MEX", "iso3166_num": "484", "iso3166_name": "Mexico", "name": "Mexico", "languages": ["es-MX"], "tld": ".mx", "capital": "Mexico City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FM", "iso3166_alpha3": "FSM", "iso3166_num": "583", "iso3166_name": "Micronesia", "name": "Micronesia", "languages": ["en-FM", "chk", "pon", "yap", "kos", "uli", "woe", "nkr", "kpg"], "tld": ".fm", "capital": "Palikir", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MC", "iso3166_alpha3": "MCO", "iso3166_num": "492", "iso3166_name": "Monaco", "name": "Monaco", "languages": ["fr-MC", "en", "it"], "tld": ".mc", "capital": "Monaco", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MN", "iso3166_alpha3": "MNG", "iso3166_num": "496", "iso3166_name": "Mongolia", "name": "Mongolia", "languages": ["mn", "ru"], "tld": ".mn", "capital": "Ulaanbaatar", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "ME", "iso3166_alpha3": "MNE", "iso3166_num": "499", "iso3166_name": "Montenegro", "name": "Montenegro", "languages": ["sr", "hu", "bs", "sq", "hr", "rom"], "tld": ".me", "capital": "Podgorica", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MS", "iso3166_alpha3": "MSR", "iso3166_num": "500", "iso3166_name": "Montserrat", "name": "Montserrat", "languages": ["en-MS"], "tld": ".ms", "capital": "Plymouth", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MA", "iso3166_alpha3": "MAR", "iso3166_num": "504", "iso3166_name": "Morocco", "name": "Morocco", "languages": ["ar-MA", "ber", "fr"], "tld": ".ma", "capital": "Rabat", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "MZ", "iso3166_alpha3": "MOZ", "iso3166_num": "508", "iso3166_name": "Mozambique", "name": "Mozambique", "languages": ["pt-MZ", "vmw"], "tld": ".mz", "capital": "Maputo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MM", "iso3166_alpha3": "MMR", "iso3166_num": "104", "iso3166_name": "Myanmar", "name": "Myanmar", "languages": ["my"], "tld": ".mm", "capital": "Nay Pyi Taw", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "NA", "iso3166_alpha3": "NAM", "iso3166_num": "516", "iso3166_name": "Namibia", "name": "Namibia", "languages": ["en-NA", "af", "de", "hz", "naq"], "tld": ".na", "capital": "Windhoek", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NR", "iso3166_alpha3": "NRU", "iso3166_num": "520", "iso3166_name": "Nauru", "name": "Nauru", "languages": ["na", "en-NR"], "tld": ".nr", "capital": "Yaren", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NP", "iso3166_alpha3": "NPL", "iso3166_num": "524", "iso3166_name": "Nepal", "name": "Nepal", "languages": ["ne", "en"], "tld": ".np", "capital": "Kathmandu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "NL", "iso3166_alpha3": "NLD", "iso3166_num": "528", "iso3166_name": "Netherlands", "name": "Netherlands", "languages": ["nl-NL", "fy-NL"], "tld": ".nl", "capital": "Amsterdam", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "NC", "iso3166_alpha3": "NCL", "iso3166_num": "540", "iso3166_name": "New Caledonia", "name": "New Caledonia", "languages": ["fr-NC"], "tld": ".nc", "capital": "Noumea", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "NZ", "iso3166_alpha3": "NZL", "iso3166_num": "554", "iso3166_name": "New Zealand", "name": "New Zealand", "languages": ["en-NZ", "mi"], "tld": ".nz", "capital": "Wellington", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "NI", "iso3166_alpha3": "NIC", "iso3166_num": "558", "iso3166_name": "Nicaragua", "name": "Nicaragua", "languages": ["es-NI", "en"], "tld": ".ni", "capital": "Managua", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "NE", "iso3166_alpha3": "NER", "iso3166_num": "562", "iso3166_name": "Niger", "name": "Niger", "languages": ["fr-NE", "ha", "kr", "dje"], "tld": ".ne", "capital": "Niamey", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NG", "iso3166_alpha3": "NGA", "iso3166_num": "566", "iso3166_name": "Nigeria", "name": "Nigeria", "languages": ["en-NG", "ha", "yo", "ig", "ff"], "tld": ".ng", "capital": "Abuja", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NU", "iso3166_alpha3": "NIU", "iso3166_num": "570", "iso3166_name": "Niue", "name": "Niue", "languages": ["niu", "en-NU"], "tld": ".nu", "capital": "Alofi", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "NF", "iso3166_alpha3": "NFK", "iso3166_num": "574", "iso3166_name": "Norfolk Island", "name": "Norfolk Island", "languages": ["en-NF"], "tld": ".nf", "capital": "Kingston", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "MP", "iso3166_alpha3": "MNP", "iso3166_num": "580", "iso3166_name": "Northern Mariana Islands", "name": "Northern Mariana Islands", "languages": ["fil", "tl", "zh", "ch-MP", "en-MP"], "tld": ".mp", "capital": "Saipan", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NO", "iso3166_alpha3": "NOR", "iso3166_num": "578", "iso3166_name": "Norway", "name": "Norway", "languages": ["no", "nb", "nn", "se", "fi"], "tld": ".no", "capital": "Oslo", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "OM", "iso3166_alpha3": "OMN", "iso3166_num": "512", "iso3166_name": "Oman", "name": "Oman", "languages": ["ar-OM", "en", "bal", "ur"], "tld": ".om", "capital": "Muscat", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "PK", "iso3166_alpha3": "PAK", "iso3166_num": "586", "iso3166_name": "Pakistan", "name": "Pakistan", "languages": ["ur-PK", "en-PK", "pa", "sd", "ps", "brh"], "tld": ".pk", "capital": "Islamabad", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PW", "iso3166_alpha3": "PLW", "iso3166_num": "585", "iso3166_name": "Palau", "name": "Palau", "languages": ["pau", "sov", "en-PW", "tox", "ja", "fil", "zh"], "tld": ".pw", "capital": "Melekeok", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "PA", "iso3166_alpha3": "PAN", "iso3166_num": "591", "iso3166_name": "Panama", "name": "Panama", "languages": ["es-PA", "en"], "tld": ".pa", "capital": "Panama City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PG", "iso3166_alpha3": "PNG", "iso3166_num": "598", "iso3166_name": "Papua New Guinea", "name": "Papua New Guinea", "languages": ["en-PG", "ho", "meu", "tpi"], "tld": ".pg", "capital": "Port Moresby", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "PY", "iso3166_alpha3": "PRY", "iso3166_num": "600", "iso3166_name": "Paraguay", "name": "Paraguay", "languages": ["es-PY", "gn"], "tld": ".py", "capital": "Asuncion", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PE", "iso3166_alpha3": "PER", "iso3166_num": "604", "iso3166_name": "Peru", "name": "Peru", "languages": ["es-PE", "qu", "ay"], "tld": ".pe", "capital": "Lima", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PH", "iso3166_alpha3": "PHL", "iso3166_num": "608", "iso3166_name": "Philippines", "name": "Philippines", "languages": ["tl", "en-PH", "fil", "ceb", "tgl", "ilo", "hil", "war", "pam", "bik", "bcl", "pag", "mrw", "tsg", "mdh", "cbk", "krj", "sgd", "msb", "akl", "ibg", "yka", "mta", "abx"], "tld": ".ph", "capital": "Manila", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "PN", "iso3166_alpha3": "PCN", "iso3166_num": "612", "iso3166_name": "Pitcairn Islands", "name": "Pitcairn Islands", "languages": ["en-PN"], "tld": ".pn", "capital": "Adamstown", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "PL", "iso3166_alpha3": "POL", "iso3166_num": "616", "iso3166_name": "Poland", "name": "Poland", "languages": ["pl"], "tld": ".pl", "capital": "Warsaw", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "PT", "iso3166_alpha3": "PRT", "iso3166_num": "620", "iso3166_name": "Portugal", "name": "Portugal", "languages": ["pt-PT", "mwl"], "tld": ".pt", "capital": "Lisbon", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "PR", "iso3166_alpha3": "PRI", "iso3166_num": "630", "iso3166_name": "Puerto Rico", "name": "Puerto Rico", "languages": ["en-PR", "es-PR"], "tld": ".pr", "capital": "San Juan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "QA", "iso3166_alpha3": "QAT", "iso3166_num": "634", "iso3166_name": "Qatar", "name": "Qatar", "languages": ["ar-QA", "es"], "tld": ".qa", "capital": "Doha", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KR", "iso3166_alpha3": "KOR", "iso3166_num": "410", "iso3166_name": "South Korea", "name": "South Korea", "languages": ["ko-KR", "en"], "tld": ".kr", "capital": "Seoul", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MD", "iso3166_alpha3": "MDA", "iso3166_num": "498", "iso3166_name": "Moldova", "name": "Moldova", "languages": ["ro", "ru", "gag", "tr"], "tld": ".md", "capital": "Chisinau", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RO", "iso3166_alpha3": "ROU", "iso3166_num": "642", "iso3166_name": "Romania", "name": "Romania", "languages": ["ro", "hu", "rom"], "tld": ".ro", "capital": "Bucharest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RU", "iso3166_alpha3": "RUS", "iso3166_num": "643", "iso3166_name": "Russia", "name": "Russia", "languages": ["ru", "tt", "xal", "cau", "ady", "kv", "ce", "tyv", "cv", "udm", "tut", "mns", "bua", "myv", "mdf", "chm", "ba", "inh", "tut", "kbd", "krc", "av", "sah", "nog"], "tld": ".ru", "capital": "Moscow", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RW", "iso3166_alpha3": "RWA", "iso3166_num": "646", "iso3166_name": "Rwanda", "name": "Rwanda", "languages": ["rw", "en-RW", "fr-RW", "sw"], "tld": ".rw", "capital": "Kigali", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RE", "iso3166_alpha3": "REU", "iso3166_num": "638", "iso3166_name": "R\u00e9union", "name": "R\u00e9union", "languages": ["fr-RE"], "tld": ".re", "capital": "Saint-Denis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BL", "iso3166_alpha3": "BLM", "iso3166_num": "652", "iso3166_name": "St. Barth\u00e9lemy", "name": "St. Barth\u00e9lemy", "languages": ["fr"], "tld": ".gp", "capital": "Gustavia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SH", "iso3166_alpha3": "SHN", "iso3166_num": "654", "iso3166_name": "St. Helena", "name": "St. Helena", "languages": ["en-SH"], "tld": ".sh", "capital": "Jamestown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KN", "iso3166_alpha3": "KNA", "iso3166_num": "659", "iso3166_name": "St. Kitts & Nevis", "name": "St. Kitts & Nevis", "languages": ["en-KN"], "tld": ".kn", "capital": "Basseterre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "LC", "iso3166_alpha3": "LCA", "iso3166_num": "662", "iso3166_name": "St. Lucia", "name": "St. Lucia", "languages": ["en-LC"], "tld": ".lc", "capital": "Castries", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MF", "iso3166_alpha3": "MAF", "iso3166_num": "663", "iso3166_name": "St. Martin", "name": "St. Martin", "languages": ["fr"], "tld": ".gp", "capital": "Marigot", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PM", "iso3166_alpha3": "SPM", "iso3166_num": "666", "iso3166_name": "St. Pierre & Miquelon", "name": "St. Pierre & Miquelon", "languages": ["fr-PM"], "tld": ".pm", "capital": "Saint-Pierre", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "VC", "iso3166_alpha3": "VCT", "iso3166_num": "670", "iso3166_name": "St. Vincent & Grenadines", "name": "St. Vincent & Grenadines", "languages": ["en-VC", "fr"], "tld": ".vc", "capital": "Kingstown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "WS", "iso3166_alpha3": "WSM", "iso3166_num": "882", "iso3166_name": "Samoa", "name": "Samoa", "languages": ["sm", "en-WS"], "tld": ".ws", "capital": "Apia", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "SM", "iso3166_alpha3": "SMR", "iso3166_num": "674", "iso3166_name": "San Marino", "name": "San Marino", "languages": ["it-SM"], "tld": ".sm", "capital": "San Marino", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "ST", "iso3166_alpha3": "STP", "iso3166_num": "678", "iso3166_name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "languages": ["pt-ST"], "tld": ".st", "capital": "Sao Tome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SA", "iso3166_alpha3": "SAU", "iso3166_num": "682", "iso3166_name": "Saudi Arabia", "name": "Saudi Arabia", "languages": ["ar-SA"], "tld": ".sa", "capital": "Riyadh", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SN", "iso3166_alpha3": "SEN", "iso3166_num": "686", "iso3166_name": "Senegal", "name": "Senegal", "languages": ["fr-SN", "wo", "fuc", "mnk"], "tld": ".sn", "capital": "Dakar", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RS", "iso3166_alpha3": "SRB", "iso3166_num": "688", "iso3166_name": "Serbia", "name": "Serbia", "languages": ["sr", "hu", "bs", "rom"], "tld": ".rs", "capital": "Belgrade", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SC", "iso3166_alpha3": "SYC", "iso3166_num": "690", "iso3166_name": "Seychelles", "name": "Seychelles", "languages": ["en-SC", "fr-SC"], "tld": ".sc", "capital": "Victoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SL", "iso3166_alpha3": "SLE", "iso3166_num": "694", "iso3166_name": "Sierra Leone", "name": "Sierra Leone", "languages": ["en-SL", "men", "tem"], "tld": ".sl", "capital": "Freetown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SG", "iso3166_alpha3": "SGP", "iso3166_num": "702", "iso3166_name": "Singapore", "name": "Singapore", "languages": ["cmn", "en-SG", "ms-SG", "ta-SG", "zh-SG"], "tld": ".sg", "capital": "Singapore", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "SX", "iso3166_alpha3": "SXM", "iso3166_num": "534", "iso3166_name": "Sint Maarten", "name": "Sint Maarten", "languages": ["nl", "en"], "tld": ".sx", "capital": "Philipsburg", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SK", "iso3166_alpha3": "SVK", "iso3166_num": "703", "iso3166_name": "Slovakia", "name": "Slovakia", "languages": ["sk", "hu"], "tld": ".sk", "capital": "Bratislava", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "SI", "iso3166_alpha3": "SVN", "iso3166_num": "705", "iso3166_name": "Slovenia", "name": "Slovenia", "languages": ["sl", "sh"], "tld": ".si", "capital": "Ljubljana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SB", "iso3166_alpha3": "SLB", "iso3166_num": "090", "iso3166_name": "Solomon Islands", "name": "Solomon Islands", "languages": ["en-SB", "tpi"], "tld": ".sb", "capital": "Honiara", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "SO", "iso3166_alpha3": "SOM", "iso3166_num": "706", "iso3166_name": "Somalia", "name": "Somalia", "languages": ["so-SO", "ar-SO", "it", "en-SO"], "tld": ".so", "capital": "Mogadishu", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZA", "iso3166_alpha3": "ZAF", "iso3166_num": "710", "iso3166_name": "South Africa", "name": "South Africa", "languages": ["zu", "xh", "af", "nso", "en-ZA", "tn", "st", "ts", "ss", "ve", "nr"], "tld": ".za", "capital": "Pretoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GS", "iso3166_alpha3": "SGS", "iso3166_num": "239", "iso3166_name": "South Georgia & South Sandwich Islands", "name": "South Georgia & South Sandwich Islands", "languages": ["en"], "tld": ".gs", "capital": "Grytviken", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SS", "iso3166_alpha3": "SSD", "iso3166_num": "728", "iso3166_name": "South Sudan", "name": "South Sudan", "languages": ["en"], "tld": "", "capital": "Juba", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ES", "iso3166_alpha3": "ESP", "iso3166_num": "724", "iso3166_name": "Spain", "name": "Spain", "languages": ["es-ES", "ca", "gl", "eu", "oc"], "tld": ".es", "capital": "Madrid", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "LK", "iso3166_alpha3": "LKA", "iso3166_num": "144", "iso3166_name": "Sri Lanka", "name": "Sri Lanka", "languages": ["si", "ta", "en"], "tld": ".lk", "capital": "Colombo", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PS", "iso3166_alpha3": "PSE", "iso3166_num": "275", "iso3166_name": "Palestine", "name": "Palestine", "languages": ["ar-PS"], "tld": ".ps", "capital": "East Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SD", "iso3166_alpha3": "SDN", "iso3166_num": "729", "iso3166_name": "Sudan", "name": "Sudan", "languages": ["ar-SD", "en", "fia"], "tld": ".sd", "capital": "Khartoum", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SR", "iso3166_alpha3": "SUR", "iso3166_num": "740", "iso3166_name": "Suriname", "name": "Suriname", "languages": ["nl-SR", "en", "srn", "hns", "jv"], "tld": ".sr", "capital": "Paramaribo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SJ", "iso3166_alpha3": "SJM", "iso3166_num": "744", "iso3166_name": "Svalbard & Jan Mayen", "name": "Svalbard & Jan Mayen", "languages": ["no", "ru"], "tld": ".sj", "capital": "Longyearbyen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "SZ", "iso3166_alpha3": "SWZ", "iso3166_num": "748", "iso3166_name": "Swaziland", "name": "Eswatini", "languages": ["en-SZ", "ss-SZ"], "tld": ".sz", "capital": "Mbabane", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SE", "iso3166_alpha3": "SWE", "iso3166_num": "752", "iso3166_name": "Sweden", "name": "Sweden", "languages": ["sv-SE", "se", "sma", "fi-SE"], "tld": ".se", "capital": "Stockholm", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "CH", "iso3166_alpha3": "CHE", "iso3166_num": "756", "iso3166_name": "Switzerland", "name": "Switzerland", "languages": ["de-CH", "fr-CH", "it-CH", "rm"], "tld": ".ch", "capital": "Bern", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "SY", "iso3166_alpha3": "SYR", "iso3166_num": "760", "iso3166_name": "Syria", "name": "Syria", "languages": ["ar-SY", "ku", "hy", "arc", "fr", "en"], "tld": ".sy", "capital": "Damascus", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TJ", "iso3166_alpha3": "TJK", "iso3166_num": "762", "iso3166_name": "Tajikistan", "name": "Tajikistan", "languages": ["tg", "ru"], "tld": ".tj", "capital": "Dushanbe", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TH", "iso3166_alpha3": "THA", "iso3166_num": "764", "iso3166_name": "Thailand", "name": "Thailand", "languages": ["th", "en"], "tld": ".th", "capital": "Bangkok", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MK", "iso3166_alpha3": "MKD", "iso3166_num": "807", "iso3166_name": "Macedonia", "name": "North Macedonia", "languages": ["mk", "sq", "tr", "rmm", "sr"], "tld": ".mk", "capital": "Skopje", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "TL", "iso3166_alpha3": "TLS", "iso3166_num": "626", "iso3166_name": "Timor-Leste", "name": "Timor-Leste", "languages": ["tet", "pt-TL", "id", "en"], "tld": ".tl", "capital": "Dili", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "TG", "iso3166_alpha3": "TGO", "iso3166_num": "768", "iso3166_name": "Togo", "name": "Togo", "languages": ["fr-TG", "ee", "hna", "kbp", "dag", "ha"], "tld": ".tg", "capital": "Lome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TK", "iso3166_alpha3": "TKL", "iso3166_num": "772", "iso3166_name": "Tokelau", "name": "Tokelau", "languages": ["tkl", "en-TK"], "tld": ".tk", "capital": "", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TO", "iso3166_alpha3": "TON", "iso3166_num": "776", "iso3166_name": "Tonga", "name": "Tonga", "languages": ["to", "en-TO"], "tld": ".to", "capital": "Nuku'alofa", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TT", "iso3166_alpha3": "TTO", "iso3166_num": "780", "iso3166_name": "Trinidad & Tobago", "name": "Trinidad & Tobago", "languages": ["en-TT", "hns", "fr", "es", "zh"], "tld": ".tt", "capital": "Port of Spain", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TN", "iso3166_alpha3": "TUN", "iso3166_num": "788", "iso3166_name": "Tunisia", "name": "Tunisia", "languages": ["ar-TN", "fr"], "tld": ".tn", "capital": "Tunis", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "TR", "iso3166_alpha3": "TUR", "iso3166_num": "792", "iso3166_name": "Turkey", "name": "Turkey", "languages": ["tr-TR", "ku", "diq", "az", "av"], "tld": ".tr", "capital": "Ankara", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TM", "iso3166_alpha3": "TKM", "iso3166_num": "795", "iso3166_name": "Turkmenistan", "name": "Turkmenistan", "languages": ["tk", "ru", "uz"], "tld": ".tm", "capital": "Ashgabat", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TC", "iso3166_alpha3": "TCA", "iso3166_num": "796", "iso3166_name": "Turks & Caicos Islands", "name": "Turks & Caicos Islands", "languages": ["en-TC"], "tld": ".tc", "capital": "Cockburn Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TV", "iso3166_alpha3": "TUV", "iso3166_num": "798", "iso3166_name": "Tuvalu", "name": "Tuvalu", "languages": ["tvl", "en", "sm", "gil"], "tld": ".tv", "capital": "Funafuti", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "UG", "iso3166_alpha3": "UGA", "iso3166_num": "800", "iso3166_name": "Uganda", "name": "Uganda", "languages": ["en-UG", "lg", "sw", "ar"], "tld": ".ug", "capital": "Kampala", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UA", "iso3166_alpha3": "UKR", "iso3166_num": "804", "iso3166_name": "Ukraine", "name": "Ukraine", "languages": ["uk", "ru-UA", "rom", "pl", "hu"], "tld": ".ua", "capital": "Kyiv", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "AE", "iso3166_alpha3": "ARE", "iso3166_num": "784", "iso3166_name": "United Arab Emirates", "name": "United Arab Emirates", "languages": ["ar-AE", "fa", "en", "hi", "ur"], "tld": ".ae", "capital": "Abu Dhabi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "GB", "iso3166_alpha3": "GBR", "iso3166_num": "826", "iso3166_name": "UK", "name": "United Kingdom", "languages": ["en-GB", "cy-GB", "gd"], "tld": ".uk", "capital": "London", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "TZ", "iso3166_alpha3": "TZA", "iso3166_num": "834", "iso3166_name": "Tanzania", "name": "Tanzania", "languages": ["sw-TZ", "en", "ar"], "tld": ".tz", "capital": "Dodoma", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UM", "iso3166_alpha3": "UMI", "iso3166_num": "581", "iso3166_name": "U.S. Outlying Islands", "name": "U.S. Outlying Islands", "languages": ["en-UM"], "tld": ".um", "capital": "", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "VI", "iso3166_alpha3": "VIR", "iso3166_num": "850", "iso3166_name": "U.S. Virgin Islands", "name": "U.S. Virgin Islands", "languages": ["en-VI"], "tld": ".vi", "capital": "Charlotte Amalie", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "US", "iso3166_alpha3": "USA", "iso3166_num": "840", "iso3166_name": "US", "name": "United States", "languages": ["en-US", "es-US", "haw", "fr"], "tld": ".us", "capital": "Washington", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "UY", "iso3166_alpha3": "URY", "iso3166_num": "858", "iso3166_name": "Uruguay", "name": "Uruguay", "languages": ["es-UY"], "tld": ".uy", "capital": "Montevideo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "UZ", "iso3166_alpha3": "UZB", "iso3166_num": "860", "iso3166_name": "Uzbekistan", "name": "Uzbekistan", "languages": ["uz", "ru", "tg"], "tld": ".uz", "capital": "Tashkent", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "VU", "iso3166_alpha3": "VUT", "iso3166_num": "548", "iso3166_name": "Vanuatu", "name": "Vanuatu", "languages": ["bi", "en-VU", "fr-VU"], "tld": ".vu", "capital": "Port Vila", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "VE", "iso3166_alpha3": "VEN", "iso3166_num": "862", "iso3166_name": "Venezuela", "name": "Venezuela", "languages": ["es-VE"], "tld": ".ve", "capital": "Caracas", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "VN", "iso3166_alpha3": "VNM", "iso3166_num": "704", "iso3166_name": "Vietnam", "name": "Vietnam", "languages": ["vi", "en", "fr", "zh", "km"], "tld": ".vn", "capital": "Hanoi", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "WF", "iso3166_alpha3": "WLF", "iso3166_num": "876", "iso3166_name": "Wallis & Futuna", "name": "Wallis & Futuna", "languages": ["wls", "fud", "fr-WF"], "tld": ".wf", "capital": "Mata Utu", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "EH", "iso3166_alpha3": "ESH", "iso3166_num": "732", "iso3166_name": "Western Sahara", "name": "Western Sahara", "languages": ["ar", "mey"], "tld": ".eh", "capital": "El-Aaiun", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "YE", "iso3166_alpha3": "YEM", "iso3166_num": "887", "iso3166_name": "Yemen", "name": "Yemen", "languages": ["ar-YE"], "tld": ".ye", "capital": "Sanaa", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "ZM", "iso3166_alpha3": "ZMB", "iso3166_num": "894", "iso3166_name": "Zambia", "name": "Zambia", "languages": ["en-ZM", "bem", "loz", "lun", "lue", "ny", "toi"], "tld": ".zm", "capital": "Lusaka", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZW", "iso3166_alpha3": "ZWE", "iso3166_num": "716", "iso3166_name": "Zimbabwe", "name": "Zimbabwe", "languages": ["en-ZW", "sn", "nr", "nd"], "tld": ".zw", "capital": "Harare", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AX", "iso3166_alpha3": "ALA", "iso3166_num": "248", "iso3166_name": "\u00c5land Islands", "name": "\u00c5land Islands", "languages": ["sv-AX"], "tld": ".ax", "capital": "Mariehamn", "region_code": "150", "sub_region_code": "154"}] diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/data.py b/ooniapi/services/ooniprobe/src/ooniprobe/data.py new file mode 100644 index 000000000..93b72e2d7 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/data.py @@ -0,0 +1,56 @@ +dnscheck_inputs = [ + "https://dns.google/dns-query", + "https://8.8.8.8/dns-query", + "dot://8.8.8.8:853/", + "dot://8.8.4.4:853/", + "https://8.8.4.4/dns-query", + "https://cloudflare-dns.com/dns-query", + "https://1.1.1.1/dns-query", + "https://1.0.0.1/dns-query", + "dot://1.1.1.1:853/", + "dot://1.0.0.1:853/", + "https://dns.quad9.net/dns-query", + "https://9.9.9.9/dns-query", + "dot://9.9.9.9:853/", + "dot://dns.quad9.net/", + "https://family.cloudflare-dns.com/dns-query", + "dot://family.cloudflare-dns.com/dns-query", + "https://dns11.quad9.net/dns-query", + "dot://dns11.quad9.net/dns-query", + "https://dns9.quad9.net/dns-query", + "dot://dns9.quad9.net/dns-query", + "https://dns12.quad9.net/dns-query", + "dot://dns12.quad9.net/dns-query", + "https://1dot1dot1dot1.cloudflare-dns.com/dns-query", + "dot://1dot1dot1dot1.cloudflare-dns.com/dns-query", + "https://dns.adguard.com/dns-query", + "dot://dns.adguard.com/dns-query", + "https://dns-family.adguard.com/dns-query", + "dot://dns-family.adguard.com/dns-query", + "https://dns.cloudflare.com/dns-query", + "https://adblock.doh.mullvad.net/dns-query", + "dot://adblock.doh.mullvad.net/dns-query", + "https://dns.alidns.com/dns-query", + "dot://dns.alidns.com/dns-query", + "https://doh.opendns.com/dns-query", + "https://dns.nextdns.io/dns-query", + "dot://dns.nextdns.io/dns-query", + "https://dns10.quad9.net/dns-query", + "dot://dns10.quad9.net/dns-query", + "https://security.cloudflare-dns.com/dns-query", + "dot://security.cloudflare-dns.com/dns-query", + "https://dns.switch.ch/dns-query", + "dot://dns.switch.ch/dns-query", +] +stunreachability_inputs = [ + "stun://stun.voip.blackberry.com:3478", + "stun://stun.antisip.com:3478", + "stun://stun.bluesip.net:3478", + "stun://stun.dus.net:3478", + "stun://stun.epygi.com:3478", + "stun://stun.sonetel.com:3478", + "stun://stun.sonetel.net:3478", + "stun://stun.uls.co.za:3478", + "stun://stun.voipgate.com:3478", + "stun://stun.voys.nl:3478", +] diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py index c44041e8a..753b99703 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/main.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -22,7 +22,7 @@ from .common.version import get_build_label from .dependencies import PostgresSessionDep, S3ClientDep, get_manifest from .download_geoip import try_update -from .routers import bouncer, prio_crud, reports +from .routers import bouncer, prio_crud, reports, private from .routers.v1 import probe_services from .routers.v2 import vpn @@ -92,6 +92,7 @@ def update_geoip_task(): app.include_router(reports.router) app.include_router(bouncer.router) app.include_router(prio_crud.router, prefix="/api") +app.include_router(private.router, prefix="/api") @app.get("/version") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py new file mode 100644 index 000000000..483a600cb --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -0,0 +1,1147 @@ +""" +prefix: /api/_ + +In here live private API endpoints for use only by OONI services. You should +not rely on these as they are likely to change, break in unexpected ways. Also +there is no versioning on them. +""" +from datetime import date, datetime, timedelta, timezone +from itertools import product + +from urllib.parse import urljoin, urlencode +from typing import Annotated, Dict, Any, Tuple, List, Optional + +import logging +import math + +from sqlalchemy import sql + +from fastapi import APIRouter, Depends, Header, Request, Response, Query +from pydantic_extra_types.country import CountryAlpha2 +from pydantic import AnyUrl, Field + +from .v1.probe_services import probe_geoip, generate_test_helpers_conf +from ..common.clickhouse_utils import query_click, query_click_one_row +from ..common.dependencies import role_required, ClickhouseDep +from ..common.prio import generate_test_list +from ..common.routers import BaseModel +from ..metrics import Metrics +from ..countries import lookup_country +from ..data import dnscheck_inputs, stunreachability_inputs +from ..utils import generate_report_id + + +# The private API is exposed under the prefix /api/_ +# e.g. https://api.ooni.io/api/_/test_names +router = APIRouter(prefix="/_") + +log = logging.getLogger(__name__) + +# TODO: configure tags for HTTP caching across where useful + +TEST_GROUPS = { + "websites": ["web_connectivity"], + "im": ["facebook_messenger", "signal", "telegram", "whatsapp"], + "middlebox": ["http_invalid_request_line", "http_header_field_manipulation"], + "performance": ["ndt", "dash"], + "circumvention": [ + "bridge_reachability", + "meek_fronted_requests_test", + "vanilla_tor", + "tcp_connect", + "psiphon", + "tor", + "torsf", + "riseupvpn", + ], + "legacy": [ + "http_requests", + "dns_consistency", + "http_host", + "multi_protocol_traceroute", + ], + "experimental": [ + "urlgetter", + "dnscheck", + "stunreachability", + ], +} + + +def daterange(start_date, end_date): + for n in range(int((end_date - start_date).days)): + yield start_date + timedelta(n) + + +def expand_dates(li): + """Replaces 'date' key in a list of dict""" + for i in li: + i["date"] = i["date"].strftime("%Y-%m-%dT00:00:00+00:00") + + +class ASNCount(BaseModel): + date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") + value: int = Field(..., description="Count of unique ASN seen.") + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + + +@router.get("/asn_by_month", tags=["private"], response_model=List[ASNCount]) +def api_private_asn_by_month( + clickhouse: ClickhouseDep, +) -> List[ASNCount]: + """Network count by month + """ + + q = """SELECT + COUNT(DISTINCT(probe_asn)) AS value, + toStartOfMonth(measurement_start_time) AS date + FROM fastpath + WHERE measurement_start_time < toStartOfMonth(addMonths(now(), 1)) + AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) + GROUP BY date ORDER BY date + """ + li = list(query_click(clickhouse, q, {})) + expand_dates(li) + return [ASNCount(**item) for item in li] + + +class CountryCount(BaseModel): + date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") + value: int = Field(..., description="Count of unique countries seen.") + + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + + + +@router.get("/countries_by_month", tags=["private"], response_model=List[CountryCount]) +def api_private_countries_by_month( + clickhouse: ClickhouseDep, +) -> List[CountryCount]: + """Countries count by month + """ + q = """SELECT + COUNT(DISTINCT(probe_cc)) AS value, + toStartOfMonth(measurement_start_time) AS date + FROM fastpath + WHERE measurement_start_time < toStartOfMonth(addMonths(now(), 1)) + AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) + GROUP BY date ORDER BY date + """ + li = list(query_click(clickhouse, q, {})) + expand_dates(li) + return [CountryCount(**item) for item in li] + + +class TestName(BaseModel): + id: str = Field(..., description="test id") + name: str = Field(..., description="test name") + + +class TestNameResponse(BaseModel): + test_names: List[TestName] + + +@router.get("/test_names", tags=["private"], response_model=TestNameResponse) +def api_private_test_names() -> TestNameResponse: + """Provides test names and descriptions to Explorer + """ + # TODO: eventually drop this, once we see nobody is using it + TEST_NAMES = { + "bridge_reachability": "Bridge Reachability", + "dash": "DASH", + "dns_consistency": "DNS Consistency", + "dnscheck": "DNS Check", + "facebook_messenger": "Facebook Messenger", + "http_header_field_manipulation": "HTTP Header Field Manipulation", + "http_host": "HTTP Host", + "http_invalid_request_line": "HTTP Invalid Request Line", + "http_requests": "HTTP Requests", + "meek_fronted_requests_test": "Meek Fronted Requests", + "multi_protocol_traceroute": "Multi Protocol Traceroute", + "ndt": "NDT", + "psiphon": "Psiphon", + "riseupvpn": "RiseupVPN", + "signal": "Signal", + "stunreachability": "STUN Reachability", + "tcp_connect": "TCP Connect", + "telegram": "Telegram", + "tor": "Tor", + "torsf": "Tor Snowflake", + "urlgetter": "URL Getter", + "vanilla_tor": "Vanilla Tor", + "web_connectivity": "Web Connectivity", + "whatsapp": "WhatsApp", + } + return TestNameResponse(test_names=[TestName(id=k, name=v) for k, v in TEST_NAMES.items()]) + + +class CountryStat(BaseModel): + alpha_2: CountryAlpha2 = Field(..., description="Country Code") + count: int = Field(..., description="Measurement count") + name: str = Field(..., description="Country Name") + + +class CountryStatResponse(BaseModel): + countries: List[CountryStat] = Field(..., description="List of countries") + + +@router.get("/countries", tags=["private"], response_model=CountryStatResponse) +def api_private_countries( + clickhouse: ClickhouseDep, +) -> CountryStatResponse: + """Summary of countries + """ + q = """ + SELECT probe_cc, COUNT() AS measurement_count + FROM fastpath + GROUP BY probe_cc ORDER BY probe_cc + """ + c = [] + rows = query_click(clickhouse, q, {}) + for r in rows: + try: + name = lookup_country(r["probe_cc"]) + c.append( + CountryStat(alpha_2=r["probe_cc"], name=name, count=r["measurement_count"]) + ) + except KeyError: + pass + + return CountryStatResponse(countries=c) + + +@router.get( + "/quotas_summary", + response_model=List[CountryStat], + tags=["private"], + dependencies=[Depends(role_required(["admin"]))], +) +def api_private_quotas_summary() -> List[CountryStat]: + """Summary on rate-limiting quotas. + [(first ipaddr octet, remaining daily quota), ... ] + """ + # XXX: add limiter to ooniapi + #return nocachejson(current_app.limiter.get_lowest_daily_quotas_summary()) + raise NotImplemented + + +class CheckReportIDResponse(BaseModel): + v: int = Field(..., description="version number of this response") + found: bool = Field(..., description="Report found") + + +@router.get("/check_report_id", + response_model=CheckReportIDResponse, + tags=["private"], +) +def check_report_id() -> CheckReportIDResponse: + """Legacy. Used to check if a report_id existed in the fastpath table. + Used by https://github.com/ooni/probe/issues/1034. Always returns True. + """ + return CheckReportIDResponse(v=0, found=True) + + +def last_30days(begin=31, end=1): + first_day = datetime.now() - timedelta(begin) + first_day = datetime(first_day.year, first_day.month, first_day.day) + + last_day = datetime.now() - timedelta(end) + last_day = datetime(last_day.year, last_day.month, last_day.day) + + for d in daterange(first_day, last_day): + yield d.strftime("%Y-%m-%d") + + +def pivot_test_coverage(rows, test_group_names, days): + # "pivot": create a datapoint for each test group, for each day + # lookup map (tg, day) -> cnt + tmp = {} + for r in rows: + day = r["measurement_start_day"].strftime("%Y-%m-%d") + k = (r["test_group"], day) + tmp[k] = r["msmt_cnt"] + + test_coverage = [ + dict( + count=tmp.get((tg, day), 0), + test_day=day, + test_group=tg, + ) + for tg in test_group_names + for day in days + ] + return test_coverage + + +def get_recent_test_coverage_ch(clickhouse, probe_cc): + """Returns + [{"count": 4888, "test_day": "2021-10-16", "test_group": "websites"}, ... ] + """ + q = "SELECT DISTINCT(test_group) FROM test_groups ORDER BY test_group" + rows = query_click(clickhouse, sql.text(q), {}) + test_group_names = [r["test_group"] for r in rows] + + q = """SELECT + toDate(measurement_start_time) as measurement_start_day, + test_group, + COUNT() as msmt_cnt + FROM fastpath + ANY LEFT JOIN test_groups USING (test_name) + WHERE measurement_start_day >= today() - interval 32 day + AND measurement_start_day < today() - interval 2 day + AND probe_cc = :probe_cc + GROUP BY measurement_start_day, test_group + """ + rows = query_click(clickhouse, sql.text(q), dict(probe_cc=probe_cc)) + rows = tuple(rows) + l30d = tuple(last_30days(32, 2)) + return pivot_test_coverage(rows, test_group_names, l30d) + + +def get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups): + """Count ASNs with at least one measurements, grouped by day, + for a given CC, and filtered by test groups + Return [{"count": 58, "test_day": "2021-10-16" }, ... ]""" + s = """SELECT + toDate(measurement_start_time) AS test_day, + COUNT(DISTINCT probe_asn) as count + FROM fastpath + WHERE test_day >= today() - interval 31 day + AND test_day < today() - interval 1 day + AND probe_cc = :probe_cc + --mark-- + GROUP BY test_day ORDER BY test_day + WITH FILL + FROM today() - interval 31 day + TO today() - interval 1 day + """ + if test_groups: + assert isinstance(test_groups, list) + test_names = set() + for tg in test_groups: + tnames = TEST_GROUPS.get(tg, []) + test_names.update(tnames) + test_names = sorted(test_names) + s = s.replace("--mark--", "AND test_name IN :test_names") + d = {"probe_cc": probe_cc, "test_names": test_names} + + else: + s = s.replace("--mark--", "") + d = {"probe_cc": probe_cc} + + return query_click(clickhouse, sql.text(s), d) + + +class NetworkCoveragePoint(BaseModel): + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)") + count: int = Field(..., description="Count of unique ASNs seen that day") + + +class TestCoveragePoint(BaseModel): + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)") + test_group: str = Field(..., description="Test group name") + count: int = Field(..., description="Number of measurements for this test group on that day") + + +class TestCoverageResponse(BaseModel): + network_coverage: List[NetworkCoveragePoint] = Field(..., description="Daily network coverage (ASNs per day)") + test_coverage: List[TestCoveragePoint] = Field(..., description="Per-test-group coverage per day") + + +@router.get("/test_coverage", response_model=TestCoverageResponse, tags=["private"]) +def api_private_test_coverage( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + test_groups: str = Query(None, description="Comma-separated list of test group keys to filter results") +) -> TestCoverageResponse: + """Return number of measurements per day across test categories + """ + # TODO: merge the two queries into one? + # TODO: remove test categories or move aggregation to the front-end? + if test_groups is not None: + test_groups = test_groups.split(",") + + tc = get_recent_test_coverage_ch(clickhouse, probe_cc) + nc = get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups) + validated = TestCoverageResponse(network_coverage=nc, test_coverage=tc) + return validated + + +class MeasurementsByASN(BaseModel): + count: int = Field(..., description="Number of measurments for eachnetworks in country") + probe_asn: str = Field(..., description="Autonomous System Number") + + +@router.get("/website_networks", response_model=List[MeasurementsByASN], tags=["private"]) +def api_private_website_network_tests( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code") +) -> List[MeasurementsByASN]: + """Daily counts of website measurements per ASN for the past 31 days, returned as a list of (probe_asn, count) ordered by count descending.""" + s = """SELECT + COUNT() AS count, + probe_asn + FROM fastpath + WHERE + measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND probe_cc = :probe_cc + GROUP BY probe_asn + ORDER BY count DESC + """ + results = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) + validated: List[MeasurementsByASN] = [MeasurementsByASN(**x) for x in results] + return validated + + +class DayStats(BaseModel): + test_day: date = Field(..., description="Date for the aggregated counts (YYYY-MM-DD)") + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies on this day") + confirmed_count: int = Field(..., description="Number of anomalies confirmed on this day") + failure_count: int = Field(..., description="Number of measurements that failed on this day") + total_count: int = Field(..., description="Total number of measurements for this day") + + +class WebsiteStatsResponse(BaseModel): + results: List[DayStats] = Field(..., description="Daily aggregated statistics for the queried website, country, and ASN (ordered by day)") + + +@router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) +def api_private_website_stats( + clickhouse: ClickhouseDep, + input: AnyUrl = Query(..., description="Website to query stats"), + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + probe_asn: int = Query(..., description="ASN (integer)"), +) -> WebsiteStatsResponse: + """Daily aggregated website measurement statistics (anomalies, confirmations, failures, and totals) for the past 31 days.""" + # uses_pg_index counters_day_cc_asn_input_idx a BRIN index was not used at + # all, but BTREE on (measurement_start_day, probe_cc, probe_asn, input) + # made queries go from full scan to 50ms + url = input + + s = """SELECT + toDate(measurement_start_time) AS test_day, + countIf(anomaly = 't') as anomaly_count, + countIf(confirmed = 't') as confirmed_count, + countIf(msm_failure = 't') as failure_count, + COUNT() AS total_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + AND input = :input + GROUP BY test_day ORDER BY test_day + """ + d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} + results = query_click(clickhouse, sql.text(s), d) + return WebsiteStatsResponse(result=results) + + +class WebsiteURLItem(BaseModel): + input: AnyUrl = Field(..., description="Tested URL") + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies for this URL in the past 31 days") + confirmed_count: int = Field(..., description="Number of anomalies confirmed for this URL in the past 31 days") + failure_count: int = Field(..., description="Number of measurements that failed for this URL in the past 31 days") + total_count: int = Field(..., description="Total number of measurements for this URL in the past 31 days") + + +class PaginationMetadata(BaseModel): + offset: int = Field(..., description="Current result offset") + limit: int = Field(..., description="Maximum number of results returned") + current_page: int = Field(..., description="Current page number (1-based)") + total_count: int = Field(..., description="Total number of matching URLs") + next_url: Optional[AnyUrl] = Field(None, description="URL for the next page of results, or null if none") + + +class WebsiteURLsResponse(BaseModel): + metadata: PaginationMetadata = Field(..., description="Pagination metadata for the results") + results: List[WebsiteURLItem] = Field(..., description="List of URL statistics for the requested CC/ASN") + + +@router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) +def api_private_website_test_urls( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + probe_asn: str = Query(..., description="ASN, e.g. AS1234"), + limit: int = Query(10, description="Limit results"), + offset: int = Query(0, description="Offset results") +) -> WebsiteURLsResponse: + """Paginated list of tested URLs with per-URL counts (anomalies, confirmations, failures, totals) for the past 31 days.""" + # TODO optimize or remove + if limit <= 0: + limit = 10 + + probe_asn = int(probe_asn.replace("AS", "")) + + # Count how many distinct inputs we have in this CC / ASN / period + s = """ + SELECT COUNT(DISTINCT(input)) as input_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND test_name = 'web_connectivity' + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + """ + q = query_click_one_row(clickhouse, sql.text(s), dict(probe_cc=probe_cc, probe_asn=probe_asn)) + total_count = q["input_count"] if q else 0 + + # Group msmts by CC / ASN / period with LIMIT and OFFSET + s = """SELECT input, + countIf(anomaly = 't') as anomaly_count, + countIf(confirmed = 't') as confirmed_count, + countIf(msm_failure = 't') as failure_count, + COUNT() AS total_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND test_name = 'web_connectivity' + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + GROUP BY input + ORDER BY confirmed_count DESC, total_count DESC, + anomaly_count DESC, input ASC + LIMIT :limit + OFFSET :offset + """ + d = { + "probe_cc": probe_cc, + "probe_asn": probe_asn, + "limit": limit, + "offset": offset, + } + results = query_click(clickhouse, sql.text(s), d) + current_page = math.ceil(offset / limit) + 1 + metadata = { + "offset": offset, + "limit": limit, + "current_page": current_page, + "total_count": total_count, + "next_url": None, + } + + # Create next_url + if len(results) >= limit: + args = dict( + limit=limit, + offset=offset + limit, + probe_asn=probe_asn, + probe_cc=probe_cc, + ) + # TODO: remove BASE_URL? + next_url = urljoin( + current_app.config["BASE_URL"], + "/api/_/website_urls?%s" % urlencode(args), + ) + metadata["next_url"] = next_url + + # validate and return + return WebsiteURLsResponse(metadata=metadata, results=results) + + +class NetworkStat(BaseModel): + failure_count: int = Field(..., description="Number of failed measurements for this ASN") + last_tested: Optional[date] = Field(None, description="Date of the most recent measurement (YYYY-MM-DD)") + probe_asn: int = Field(..., description="Autonomous System Number (integer)") + total_count: int = Field(..., description="Total number of measurements for this ASN") + success_count: int = Field(..., description="Number of successful measurements for this ASN") + test_runtime_avg: Optional[float] = Field(None, description="Average test runtime in seconds for this ASN") + test_runtime_max: Optional[float] = Field(None, description="Maximum test runtime in seconds for this ASN") + test_runtime_min: Optional[float] = Field(None, description="Minimum test runtime in seconds for this ASN") + + +class TorStatsResponse(BaseModel): + last_tested: Optional[date] = Field(None, description="Most recent test date across all networks (YYYY-MM-DD)") + networks: List[NetworkStat] = Field(..., description="List of per-ASN Tor test statistics") + notok_networks: int = Field(..., description="Number of networks considered 'not OK' (low success rate)") + + +@router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) +def api_private_vanilla_tor_stats( + clickhouse: ClickhouseDep, + probe_cc: str = Query(..., description="Country Code") +) -> TorStatsResponse: + """Per-ASN Tor measurement statistics for the given country over the last 6 months, including counts, last-tested date, and a tally of networks with low success rates.""" + try: + CountryAlpha2(probe_cc) + except Exception: + raise + blocked = 0 + nets = [] + s = """SELECT + countIf(msm_failure = 't') as failure_count, + toDate(MAX(measurement_start_time)) AS last_tested, + probe_asn, + COUNT() as total_count, + total_count - countIf(anomaly = 't') AS success_count + FROM fastpath + WHERE measurement_start_time > today() - INTERVAL 6 MONTH + AND probe_cc = :probe_cc + GROUP BY probe_asn + """ + q = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) + extras = { + "test_runtime_avg": None, + "test_runtime_max": None, + "test_runtime_min": None, + } + for n in q: + n.update(extras) + nets.append(n) + if n["total_count"] > 5: + if float(n["success_count"]) / float(n["total_count"]) < 0.6: + blocked += 1 + + if not nets: + return TorStatsResponse( + last_tested=None, + networks=[], + notok_networks=0, + ) + + lt = max(n["last_tested"] for n in nets) + return TorStatsResponse( + last_tested=lt, + networks=nets, + notok_networks=blocked, + ) + + +class NetworkEntry(BaseModel): + asn: int = Field(..., description="Autonomous System Number (integer)") + name: str = Field(..., description="Network/ASN name") + total_count: int = Field(..., description="Total number of measurements for this ASN and test") + last_tested: date = Field(..., description="Date of the most recent measurement (YYYY-MM-DD)") + + +class IMNetworkStats(BaseModel): + anomaly_networks: List[NetworkEntry] = Field(..., description="List of networks showing anomalous behaviour for this test") + ok_networks: List[NetworkEntry] = Field(..., description="List of networks considered OK for this test") + last_tested: date = Field(..., description="Most recent measurement date across networks for this test") + + +@router.get("/im_networks", response_model=Dict[str, IMNetworkStats], tags=["private"]) +def api_private_im_networks( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code") +) -> Dict[str, IMNetworkStats]: + """Per-test instant messaging network statistics (per-ASN totals and last-tested date) for the past 31 days, keyed by test name.""" + s = """SELECT + COUNT() AS total_count, + '' AS name, + toDate(MAX(measurement_start_time)) AS last_tested, + probe_asn, + test_name + FROM fastpath + WHERE measurement_start_time >= today() - interval 31 day + AND measurement_start_time < today() + AND probe_cc = :probe_cc + AND test_name IN :test_names + GROUP BY test_name, probe_asn + ORDER BY test_name ASC, total_count DESC + """ + test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] + q = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) + results: Dict[str, IMNetworkStats] = {} + for r in q: + # get stats for test_name or create a new IMNetworksStats + stats = results.get(r["test_name"], IMNetworkStats(anomaly_networks=[], ok_networks=[], last_tested=None)) + + # create and add a new entry + entry = NetworkEntry(asn=r["probe_asn"], name="", total_count=r["total_count"], last_tested=r["last_tested"]) + stats.ok_networks.append(entry) + + # XXX: anomaly_networks appears unused + + # update last_tested if it is the latest measurement + if stats.last_tested < entry.last_tested: + stats.last_tested = entry.last_tested + + # save the object in results + results[stats.test_name] = stats + + return results + + +def isomid(d) -> str: + """Returns 2020-08-01T00:00:00+00:00""" + return f"{d}T00:00:00+00:00" + + +class IMStatsItem(BaseModel): + anomaly_count: Optional[int] = Field(None, description="Number of measurements flagged as anomalies for that day") + test_day: datetime = Field(..., description="Timestamp for the day (ISO 8601, midnight UTC)") + total_count: int = Field(..., description="Total number of measurements for that day") + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + + +class IMStatsResponse(BaseModel): + results: List[IMStatsItem] = Field(..., description="Daily IM statistics for the requested ASN/CC/test (last 31 days)") + + +@router.get("/im_stats", response_model=IMStatsResponse, tags=["private"]) +def api_private_im_stats( + clickhouse: ClickhouseDep, + probe_asn: str = Query(..., description="ASN, e.g. AS1234"), + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + test_name: str = Query(..., description="Test name") +) -> IMStatsResponse: + """Daily instant messaging measurement totals (and optional anomaly counts) for the past 31 days, for the given ASN, country, and test.""" + test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] + if test_name not in test_names: + raise HTTPException(status_code=400, detail="Invalid test_name") + + probe_asn = int(probe_asn.upper().replace("AS", "")) + + s = """SELECT + COUNT() as total_count, + toDate(measurement_start_time) AS test_day + FROM fastpath + WHERE probe_cc = :probe_cc + AND test_name = :test_name + AND probe_asn = :probe_asn + AND measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + GROUP BY test_day + ORDER BY test_day + """ + query_params = { + "probe_cc": probe_cc, + "probe_asn": probe_asn, + "test_name": test_name, + } + q = query_click(clickhouse, sql.text(s), query_params) + tmp = {r["test_day"]: r for r in q} + items: List[IMStatsItem] = [] + days = [date.today() + timedelta(days=(d - 31)) for d in range(32)] + for d in days: + if d in tmp: + test_day = isomid(tmp[d]["test_day"]) + total_count = tmp[d]["total_count"] + else: + test_day = isomid(d) + total_count = 0 + items.append(IMStatsItem(anomaly_count=None, test_day=test_day, total_count=total_count)) + + response = IMStatsResponse(results=items) + return response + + +class NetworkStats(BaseModel): + asn: int = Field(..., description="ASN (int)") + asn_name: str = Field(..., description="Autonomous System Name") + download_speed_mbps_median: float = Field(description="Median download speed in megabits") + upload_speed_mbps_median: float = Field(description="Median upload speed in megabites") + middlebox_detected: bool = Field(..., description="Middlebox was detected") + msm_count: float = Field(..., description="FIXME: example: 24596.0") + rtt_avg: float = Field(..., description="Round Trip Time Average") + + +class NetworkMetadata(BaseModel): + current_page: int = Field(1, description="Current page") + limit: int = Field(10, description="Result limit") + next_url: Optional[AnyUrl] = Field(None, description="Next URL") + offset: int = Field(0, description="Result offset") + total_count: int = Field(0, description="Total results") + + +class NetworkStatsResponse(BaseModel): + metadata: NetworkMetadata = Field(..., description="Pagination and result metadata") + results: List[NetworkStats] = Field(..., description="List of per-ASN network statistics") + + +@router.get("/network_stats", response_model=NetworkStatsResponse, tags=["private"]) +def api_private_network_stats( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + limit: int = Query(10, description="Limit results"), + offset: int = Query(0, description="Offset results"), +) -> NetworkStatsResponse: + + # TODO: implement the stats from NDT in fastpath and then here + + return NetworkStatsResponse(metadata=NetworkMetadata(), results=[]) + + +class CountryOverviewResponse(BaseModel): + first_bucket_date: date = Field(..., description="First bucket date YYYY-MM-DD") + measurement_count: int = Field(..., description="Number of measurements") + network_count: int = Field(..., description="Number of networks measured") + + +@router.get("/country_overview", response_model=CountryOverviewResponse, tags=["private"]) +def api_private_country_overview( + clickhouse: ClickhouseDep, + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), +) -> CountryOverviewResponse: + """Country-level summary for the requested two-letter code: first available measurement date, total number of measurements since 2012-12-01, and number of distinct ASNs observed (networks).""" + # TODO: add circumvention_tools_blocked im_apps_blocked + # middlebox_detected_networks websites_confirmed_blocked + s = """SELECT + toDate(MIN(measurement_start_time)) AS first_bucket_date, + COUNT() AS measurement_count, + COUNT(DISTINCT probe_asn) AS network_count + FROM fastpath + WHERE probe_cc = :probe_cc + AND measurement_start_time > '2012-12-01' + """ + r = query_click_one_row(clickhouse, sql.text(s), {"probe_cc": probe_cc}) + assert r + result = CountryOverviewResponse( + first_bucket_date=r["first_bucket_date"], + measurement_count=r["measurement_count"], + network_count=r["network_count"]) + return result + + +class GlobalOverviewResponse(BaseModel): + network_count: int = Field(..., description="Number of networks measured") + country_count: int = Field(..., description="Number of countries measured") + measurement_count: int = Field(..., description="Number of total measurements") + + +@router.get("/global_overview", response_model=GlobalOverviewResponse, tags=["private"]) +def api_private_global_overview( + clickhouse: ClickhouseDep, +) -> GlobalOverviewResponse: + """Global summary of measurements across all countries: total distinct networks (ASNs), total countries with measurements, and total measurement count (computed from the fastpath table).""" + q = """SELECT + COUNT(DISTINCT(probe_asn)) AS network_count, + COUNT(DISTINCT probe_cc) AS country_count, + COUNT(*) AS measurement_count + FROM fastpath + """ + r = query_click_one_row(clickhouse, q, {}) + assert r + result = GlobalOverviewResponse( + network_count=r["network_count"], + country_count=r["country_count"], + measurement_count=r["measurement_count"]) + return result + + +class GlobalOverviewStat(BaseModel): + date: datetime = Field(..., description="Month start timestamp (ISO 8601, midnight UTC)") + value: int = Field(..., description="Count value for the month") + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + + +class GlobalOverviewMonthResponse(BaseModel): + networks_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly distinct network (ASN) counts") + countries_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly distinct country counts") + measurements_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly total measurement counts") + + +@router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) +def api_private_global_by_month( + clickhouse: ClickhouseDep, +) -> GlobalOverviewMonthResponse: + """Monthly global time series for the last two years: distinct networks (ASNs), distinct countries, and total measurements per month (month timestamps are start-of-month).""" + q = """SELECT + COUNT(DISTINCT probe_asn) AS networks_by_month, + COUNT(DISTINCT probe_cc) AS countries_by_month, + COUNT() AS measurements_by_month, + toStartOfMonth(measurement_start_time) AS month + FROM fastpath + WHERE measurement_start_time > toStartOfMonth(today() - interval 2 year) + AND measurement_start_time < toStartOfMonth(today() + interval 1 month) + GROUP BY month ORDER BY month + """ + rows = query_click(clickhouse, sql.text(q), {}) + rows = list(rows) + + n = [{"date": r["month"], "value": r["networks_by_month"]} for r in rows] + c = [{"date": r["month"], "value": r["countries_by_month"]} for r in rows] + m = [{"date": r["month"], "value": r["measurements_by_month"]} for r in rows] + expand_dates(n) + expand_dates(c) + expand_dates(m) + validated = GlobalOverviewMonthResponse(networks_by_month=n, countries_by_month=c, measurements_by_month=m) + return validated + + +class CountryCircumventionStat(BaseModel): + cnt: int = Field(..., description="Count of measurements") + probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") + + +class CircumventionStatsResponse(BaseModel): + results: Optional[List[CountryCircumventionStat]] = Field(None, description="List of per-country circumvention tool measurement counts over 6 months") + v: int = Field(..., description="API Response version") + + +@router.get("/circumvention_stats_by_country", response_model=CircumventionStatsResponse, tags=["private"]) +def api_private_circumvention_stats_by_country( + clickhouse: ClickhouseDep, +) -> CircumventionStatsResponse: + """Aggregated statistics on protocols used for circumvention, grouped by country. """ + q = """SELECT probe_cc, COUNT(*) as cnt + FROM fastpath + WHERE measurement_start_time > today() - interval 6 month + AND measurement_start_time < today() - interval 1 day + AND test_name IN ['torsf', 'tor', 'stunreachability', 'psiphon','riseupvpn'] + GROUP BY probe_cc ORDER BY probe_cc + """ + try: + result = query_click(clickhouse, sql.text(q), {}) + return CountryCircumVentionStatsResponse(results=result) + + except Exception as e: + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) + + +class CircumventionRuntimeStat(BaseModel): + test_name: str = Field(..., description="Name of test") + probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") + metric_date: date = Field(..., alias="date", description="Date of metric") + v: Tuple[float, float, int] = Field((), description="(p50, p90, cnt)") + + +def pivot_circumvention_runtime_stats(rows) -> List[CircumventionRuntimeStat]: + # "pivot": create a datapoint for each probe_cc/test_name/date + tmp = {} + test_names = set() + ccs = set() + dates = set() + for r in rows: + k = (r["test_name"], r["probe_cc"], r["date"]) + tmp[k] = (r["p50"], r["p90"], r["cnt"]) + test_names.add(k[0]) + ccs.add(k[1]) + dates.add(k[2]) + + dates = sorted(dates) + ccs = sorted(ccs) + test_names = sorted(test_names) + no_data = () + result = [ + RuntimeStat(test_name=k[0], probe_cc=k[1], date=k[2], v=tmp.get(k, no_data)) + for k in product(test_names, ccs, dates) + ] + return result + + +class CircumventionRuntimeStatsResponse(BaseModel): + results: List[CircumventionRuntimeStat] + v: int = Field(..., description="Version of API response") + + +@router.get("/circumvention_runtime_stats", response_model=CircumventionRuntimeStatsResponse, tags=["private"]) +def api_private_circumvention_runtime_stats( + clickhouse: ClickhouseDep, +) -> CircumventionRuntimeStatsResponse: + """Runtime statistics on protocols used for circumvention, grouped by date, country, test_name. """ + q = """SELECT + toDate(measurement_start_time) AS date, + test_name, + probe_cc, + quantile(.5)(JSONExtractFloat(scores, 'extra', 'test_runtime')) AS p50, + quantile(.9)(JSONExtractFloat(scores, 'extra', 'test_runtime')) AS p90, + count() as cnt + FROM fastpath + WHERE test_name IN ['torsf', 'tor', 'stunreachability', 'psiphon','riseupvpn'] + AND measurement_start_time > today() - interval 6 month + AND measurement_start_time < today() - interval 1 day + AND JSONHas(scores, 'extra', 'test_runtime') + GROUP BY date, probe_cc, test_name + """ + try: + r = query_click(clickhouse, sql.text(q), {}) + return CircumventionRuntimeStatsResponse(results=pivot_circumvention_runtime_stats(r), v=0) + + except Exception as e: + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) + + +DomainStr = Annotated[ + str, + Field( + strip_whitespace=True, + pattern=r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[A-Za-z]{2,}$" + ) +] + + +class DomainMetadataResponse(BaseModel): + canonical_domain: DomainStr = Field(..., description="Domain name") + category_code: str = Field(..., description="Citizenlab Category Code") + + +@router.get("/domain_metadata", response_model=DomainMetadataResponse, tags=["private"]) +def api_private_domain_metadata( + clickhouse: ClickhouseDep, + domain: DomainStr = Query(..., description="Domain Name"), +) -> DomainMetadataResponse: + """Return the primary category code of a certain domain_name and its + canonical representation. + We consider the primary category code to be the category code of whatever is + the shortest URL in the test lists giving higher priority to what is in the + global list (e.g. if we have https://twitter.com/amnesty categorised as HUMR + and https://twitter.com/ as GRP, we will be picking GRP as the canonical + category code). + The canonical representation of a domain is whatever is present in the + global list. If it's not present, then the shortest possible + representation in the other test lists. + Some research related to this problem was done here: + https://gist.github.com/hellais/fab319ae20b0ccca7b548a060ed66e14, where some + notes were taken on what fixes need to be done in the test-lists to ensure + all of this works as expected (ex. moving shortest URL representations from + the country lists into the global list). + """ + category_code = "MISC" + + if domain.startswith("www."): + canonical_domain = domain[4:] + domains = [canonical_domain, domain] + else: + canonical_domain = domain + domains = [canonical_domain, "www." + domain] + + # case 1: domain with or without www is in the global list (cc = 'ZZ') + q = """ + SELECT category_code, domain + FROM citizenlab + WHERE domain IN :domains + AND cc = 'ZZ' + ORDER BY length(url) + LIMIT 1 + """ + res = query_click_one_row(clickhouse, sql.text(q), dict(domains=domains)) + if not res: + # case 2: domain only inside a country list, so we just select the + # shortest domain among the one with and without www + q = """ + SELECT category_code, domain FROM citizenlab + WHERE domain IN :domains + ORDER BY length(url) + LIMIT 1 + """ + res = query_click_one_row(clickhouse, sql.text(q), dict(domains=domains)) + + if res: + category_code = res["category_code"] + canonical_domain = res["domain"] + + return DomainMetadataResponse(category_code=category_code, canonical_domain=canonical_domain) + + +class ASNMetadataResponse(BaseModel): + org_name: str = Field("Unknown", description="ORG Name of ASN") + + +@router.get("/asnmeta", response_model=ASNMetadataResponse, tags=["private"]) +def api_private_asnmeta( + clickhouse: ClickhouseDep, + asn: int = Query(..., description="Autonomous System Number, e.g. 1234"), +) -> ASNMetadataResponse: + """Look up organization name by ASN""" + + q = """SELECT org_name + FROM asnmeta + WHERE asn = :asn + ORDER BY changed DESC + LIMIT 1 + """ + res = query_click_one_row(clickhouse, sql.text(q), dict(asn=asn)) + org_name = res["org_name"] if res else "Unknown" + return ASNMetadataResponse(org_name=org_name) + + +class MeasuredNetworkStat(BaseModel): + cnt: int = Field(..., description="Number of measurements") + org_name: str = Field("", description="Organization name associated with the ASN") + probe_asn: int = Field(..., description="ASN of network (int)") + + +class MeasuredNetworksResponse(BaseModel): + results: List[MeasuredNetworkStat] = Field(..., description="Networks that have measurements") + v: int = Field(..., description="Version of API response") + + +@router.get("/networks", response_model=MeasuredNetworksResponse, tags=["private"]) +def api_private_networks( + clickhouse: ClickhouseDep, +) -> MeasuredNetworksResponse: + """List all networks that have measurements by per-ASN measurement count and associated organization name.""" + q = """ + SELECT probe_asn, cnt, org_name FROM ( + SELECT + COUNT() as cnt, + probe_asn + FROM fastpath + GROUP BY probe_asn + ) as cnts + + LEFT JOIN ( + SELECT + any(org_name) as org_name, + asn + FROM ( + SELECT org_name, asn + FROM asnmeta + ORDER BY changed DESC + ) + GROUP BY asn + ) as asorgs + ON (asorgs.asn = cnts.probe_asn) + """ + try: + results = query_click(clickhouse, sql.text(q), {}) + return MeasuredNetworksResponse(results=results, v=0) + except Exception as e: + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) + + +class MeasuredDomainStat(BaseModel): + category_code: str = Field(..., description="Citizenlab Category Code") + domain_name: DomainStr = Field(..., description="Domain Name") + measurement_count: int = Field(..., description="Number of measurements") + + +class DomainsMeasuredResponse(BaseModel): + results: List[MeasuredDomainStat] = Field(..., description="Domains that have measurements") + v: int = Field(..., description="Version of API response") + + +@router.get("/domains", response_model=DomainsMeasuredResponse, tags=["private"]) +def api_private_domains( + clickhouse: ClickhouseDep, +) -> DomainsMeasuredResponse: + """List all the domains in the test-lists with their measurement count.""" + # The nested ORDER BY lower(cc) puts global entries (cc=ZZ) on top so that + # any(category_code) picks it up as the most meaningful category code. + q = """ + SELECT domain AS domain_name, category_code, measurement_count + FROM ( + SELECT + any(category_code) as category_code, + domain FROM ( + SELECT domain, category_code + FROM citizenlab + ORDER BY lower(cc) != 'zz' + ) + GROUP BY domain + ) AS cz + LEFT JOIN ( + SELECT domain, count() AS measurement_count + FROM fastpath + WHERE test_name = 'web_connectivity' + GROUP BY domain + ) AS fp + ON (fp.domain == cz.domain) + ORDER BY domain_name + """ + try: + results = query_click(clickhouse, sql.text(q), {}) + return DomainsMeasuredResponse(v=0, results=results) + except Exception as e: + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index 44907c7dc..72a7db599 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -1,4 +1,5 @@ import json +import logging import pathlib import time from datetime import datetime @@ -35,6 +36,10 @@ from .utils import setup_user +@pytest.fixture +def log(): + return logging.getLogger(__name__) + def make_override_get_settings(**kw): def override_get_settings(): return Settings(**kw) diff --git a/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql b/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql index 25e6af123..d3ca78a36 100644 --- a/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql +++ b/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql @@ -31,14 +31,15 @@ CREATE TABLE default.fastpath `server_as_name` String, `update_time` DateTime64(3) MATERIALIZED now64(), `test_version` String, - `test_runtime` Float32, `architecture` String, `engine_name` String, `engine_version` String, + `test_runtime` Float32, `blocking_type` String, `test_helper_address` LowCardinality(String), `test_helper_type` LowCardinality(String), - `ooni_run_link_id` Nullable(UInt64) + `ooni_run_link_id` Nullable(UInt64), + `is_verified` LowCardinality(String) DEFAULT 'u' ) ENGINE = ReplacingMergeTree ORDER BY (measurement_start_time, report_id, input) diff --git a/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql b/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql new file mode 100644 index 000000000..912ecf1f4 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql @@ -0,0 +1 @@ +INSERT INTO table default.fastpath (`measurement_uid`, `report_id`, `input`, `probe_cc`, `probe_asn`, `test_name`, `test_start_time`, `measurement_start_time`, `filename`, `scores`, `platform`, `anomaly`, `confirmed`, `msm_failure`, `domain`, `software_name`, `software_version`, `control_failure`, `blocking_general`, `is_ssl_expected`, `page_len`, `page_len_ratio`, `server_cc`, `server_asn`, `server_as_name`, `test_version`, `architecture`, `engine_name`, `engine_version`, `test_runtime`, `blocking_type`, `test_helper_address`, `test_helper_type`, `ooni_run_link_id`, `is_verified`) VALUES ('20260110135447.088775_RS_signal_e7b599ccd49ca171', '20260110T135446Z_signal_RS_8771_n4_kVFlePYMNxHYcbxh', '', 'RS', 8771, 'signal', '2026-01-10 13:54:46', '2026-01-10 13:54:46', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.49980247, '', '', '', NULL, '0'), ('20260123110034.624460_VE_signal_aaeea63fd70b5899', '20260123T110032Z_signal_VE_8048_n4_0aicHCzlncXLgrDf', '', 'VE', 8048, 'signal', '2026-01-23 11:00:32', '2026-01-23 11:00:33', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.24.0', 1.0396768, '', '', '', NULL, '0'), ('20260117141659.519197_IT_signal_6ad8f1c0b89de448', '20260117T141658Z_signal_IT_24608_n4_h0OQyBSdutunwhcj', '', 'IT', 24608, 'signal', '2026-01-17 14:16:58', '2026-01-17 14:16:58', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.6059591, '', '', '', NULL, '0'), ('20260127043315.750799_ES_signal_9dc615c1cb2a4249', '20260127T043315Z_signal_ES_57269_n4_HnUIVcoBr1iaTQwU', '', 'ES', 57269, 'signal', '2026-01-27 04:33:15', '2026-01-27 04:33:15', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.35447666, '', '', '', NULL, '0'), ('20260105215811.053944_AU_signal_143a953edc5475f9', '20260105T215809Z_signal_AU_4764_n4_QcLeZ4AQIiTLM9yO', '', 'AU', 4764, 'signal', '2026-01-05 21:58:09', '2026-01-05 21:58:10', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.67212117, '', '', '', NULL, '0'), ('20260129043446.605662_SV_signal_f0e2dae8ab70b21e', '20260129T043445Z_signal_SV_27773_n4_d2SLo5doCXWMMMe6', '', 'SV', 27773, 'signal', '2026-01-29 04:34:45', '2026-01-29 04:34:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4660445, '', '', '', NULL, '0'), ('20260121064037.490724_ES_signal_b97aa744a3e8b7c9', '20260121T064036Z_signal_ES_57269_n4_qriPsy5OaTcOVIJK', '', 'ES', 57269, 'signal', '2026-01-21 06:40:36', '2026-01-21 06:40:36', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.47944492, '', '', '', NULL, '0'), ('20260120175831.837469_DE_signal_2712804df5faf5b8', '20260120T175831Z_signal_DE_680_n4_ytyvRjo0Ba05PfTN', '', 'DE', 680, 'signal', '2026-01-20 17:58:31', '2026-01-20 17:58:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.1062024, '', '', '', NULL, '0'), ('20260108174143.943125_FR_signal_a93aed855fbbc9bb', '20260108T174142Z_signal_FR_3215_n4_TG5PsH67W6fYJ6Qp', '', 'FR', 3215, 'signal', '2026-01-08 17:42:40', '2026-01-08 17:42:40', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.6', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.25.0', 0.5855959, '', '', '', NULL, '0'), ('20260128182122.005734_DE_signal_bcd5ee367a2525cb', '20260128T182121Z_signal_DE_680_n4_vykMKDVFlM5xyRGB', '', 'DE', 680, 'signal', '2026-01-28 18:21:21', '2026-01-28 18:21:21', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.09559728, '', '', '', NULL, '0'), ('20260110050954.007509_CM_signal_83b2054892b391a3', '20260110T050951Z_signal_CM_15964_n4_TeZN64NZzGN0PoPe', '', 'CM', 15964, 'signal', '2026-01-10 05:09:51', '2026-01-10 05:09:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 2.0773945, '', '', '', NULL, '0'), ('20260127061350.718282_GN_signal_a586a62ab2982dfa', '20260127T061348Z_signal_GN_37461_n4_jSZliomLWIofzQsz', '', 'GN', 37461, 'signal', '2026-01-27 06:13:47', '2026-01-27 06:13:47', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.6756234, '', '', '', NULL, '0'), ('20260102041813.309231_DE_signal_dc028560a210b6c1', '20260102T041812Z_signal_DE_3209_n4_Gu8rwQe9vcmCdKMk', '', 'DE', 3209, 'signal', '2026-01-02 04:18:12', '2026-01-02 04:18:12', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.61336344, '', '', '', NULL, '0'), ('20260120140327.403850_JP_signal_f20d6b7ff13153c1', '20260120T140325Z_signal_JP_45102_n4_My2S5QoD6HDyQRca', '', 'JP', 45102, 'signal', '2026-01-20 14:03:25', '2026-01-20 14:03:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 1.4022309, '', '', '', NULL, '0'), ('20260110160710.218928_DE_signal_f9eb3c4f05d6a772', '20260110T160709Z_signal_DE_3320_n4_4OJLILnklxI63roa', '', 'DE', 3320, 'signal', '2026-01-10 16:07:09', '2026-01-10 16:07:09', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.56549716, '', '', '', NULL, '0'), ('20260119144448.740559_US_signal_1d614b5a411607dc', '20260119T144448Z_signal_US_14615_n4_TpfUn5Zc5XrTKQil', '', 'US', 14615, 'signal', '2026-01-19 14:44:48', '2026-01-19 14:44:48', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 0.1717979, '', '', '', NULL, '0'), ('20260124031352.083113_ES_signal_d8082edf11a991a0', '20260124T031351Z_signal_ES_60203_n4_N7AKwOaSUkWN9Y3i', '', 'ES', 60203, 'signal', '2026-01-24 03:13:51', '2026-01-24 03:13:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4493603, '', '', '', NULL, '0'), ('20260127161132.817453_CA_signal_d98459cdc733a244', '20260127T161131Z_signal_CA_812_n4_muoBlMoRGlOLubog', '', 'CA', 812, 'signal', '2026-01-27 16:11:32', '2026-01-27 16:11:32', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'android', 'f', 'f', 't', '', 'ooniprobe-android-unattended', '3.7.3', '', 0, 0, 0, 0, '', 0, '', '0.2.2', 'arm64', 'ooniprobe-engine', '3.16.7', 0.6236891, '', '', '', NULL, '0'), ('20260101192636.414972_ES_signal_2e95090292761b6a', '20260101T192635Z_signal_ES_3352_n4_v9KQphoYgSyc9CC7', '', 'ES', 3352, 'signal', '2026-01-01 19:26:35', '2026-01-01 19:26:35', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.5347266, '', '', '', NULL, '0'), ('20260102001427.293388_ZW_signal_7ca4c020bfb8c733', '20260102T001425Z_signal_ZW_37332_n4_U3sNsOPWfQENtPbx', '', 'ZW', 37332, 'signal', '2026-01-02 00:14:25', '2026-01-02 00:14:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.9928689, '', '', '', NULL, '0'), ('20260126193945.103317_US_signal_341e23577ab01d08', '20260126T193944Z_signal_US_7922_n4_NKvVQf2foWl0Svoa', '', 'US', 7922, 'signal', '2026-01-26 19:39:44', '2026-01-26 19:39:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.2488854, '', '', '', NULL, '0'), ('20260113081502.171083_US_signal_c493a4cd3b4618b0', '20260113T081500Z_signal_US_11492_n4_aUQDYczPvKxMmlxc', '', 'US', 11492, 'signal', '2026-01-13 08:15:00', '2026-01-13 08:15:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.76373357, '', '', '', NULL, '0'), ('20260130045403.620913_NL_signal_1b76dba3ec160675', '20260130T045402Z_signal_NL_1136_n4_VMJMlf58x8LtrcdL', '', 'NL', 1136, 'signal', '2026-01-30 04:54:03', '2026-01-30 04:54:03', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.51608795, '', '', '', NULL, '0'), ('20260110154809.542690_NZ_signal_77c0e4b1ea472944', '20260110T154807Z_signal_NZ_4771_n4_i5IibPcfcFc60xtX', '', 'NZ', 4771, 'signal', '2026-01-10 15:48:07', '2026-01-10 15:48:07', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.8938294, '', '', '', NULL, '0'), ('20260103074148.111315_FR_signal_e22f2578c70032d8', '20260103T074147Z_signal_FR_12322_n4_BgqUeiKoAgkTHbe8', '', 'FR', 12322, 'signal', '2026-01-03 07:41:47', '2026-01-03 07:41:47', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.44495875, '', '', '', NULL, '0'), ('20260112045926.845756_ZA_signal_5dee9e0dda8a2b6b', '20260112T045924Z_signal_ZA_328471_n4_5gQlMl1mzxj3oGA3', '', 'ZA', 328471, 'signal', '2026-01-12 04:59:24', '2026-01-12 04:59:24', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.9813594, '', '', '', NULL, '0'), ('20260101180538.691977_ET_signal_8814d8efb9dfd565', '20260101T180537Z_signal_ET_24757_n4_vLNoqzjMyUotVNye', '', 'ET', 24757, 'signal', '2026-01-01 18:05:38', '2026-01-01 18:05:38', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.904734, '', '', '', NULL, '0'), ('20260126062200.282691_US_signal_434e3d22004d02a3', '20260126T062159Z_signal_US_5650_n4_xCUw4TgErIyyVVp2', '', 'US', 5650, 'signal', '2026-01-26 06:22:00', '2026-01-26 06:22:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.72052324, '', '', '', NULL, '0'), ('20260131233019.265756_DZ_signal_8337900470d1d647', '20260131T233018Z_signal_DZ_36947_n4_jIBGzDnVQmj0poGW', '', 'DZ', 36947, 'signal', '2026-01-31 23:30:17', '2026-01-31 23:30:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.7189328, '', '', '', NULL, '0'), ('20260118221838.842575_US_signal_0a3512ee77378581', '20260118T221837Z_signal_US_11427_n4_pqw4mf1hB91ubI3J', '', 'US', 11427, 'signal', '2026-01-18 22:18:37', '2026-01-18 22:18:37', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.43586272, '', '', '', NULL, '0'), ('20260121213502.700887_FI_signal_d03ed86cb78733cf', '20260121T213502Z_signal_FI_719_n4_PdCnHJ7kXFWHWAv9', '', 'FI', 719, 'signal', '2026-01-21 21:35:02', '2026-01-21 21:35:02', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.07162474, '', '', '', NULL, '0'), ('20260120234837.045187_FR_signal_d21a8e1e8b757cc6', '20260120T234836Z_signal_FR_3215_n4_G5EQDEXkA27Wx1M0', '', 'FR', 3215, 'signal', '2026-01-20 23:48:36', '2026-01-20 23:48:36', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.19238907, '', '', '', NULL, '0'), ('20260121044125.972617_FI_signal_311cf8bbfdcf8672', '20260121T044125Z_signal_FI_719_n4_TbzDNCzGiqnDLeit', '', 'FI', 719, 'signal', '2026-01-21 04:41:25', '2026-01-21 04:41:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.084080726, '', '', '', NULL, '0'), ('20260121052901.443389_US_signal_1a074c793d403bc7', '20260121T052901Z_signal_US_397005_n4_wIYnABVJEh0orZP2', '', 'US', 397005, 'signal', '2026-01-21 05:29:00', '2026-01-21 05:29:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.14746083, '', '', '', NULL, '0'), ('20260102020535.145478_RU_signal_5ca6f103dd82b676', '20260102T020514Z_signal_RU_25513_n4_5foqozJNdueTiBL8', '', 'RU', 25513, 'signal', '2026-01-02 02:05:14', '2026-01-02 02:05:14', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'windows', 't', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 20.087662, '', '', '', NULL, '0'), ('20260125103153.448150_BR_signal_6d0b131ce71115af', '20260125T103152Z_signal_BR_28210_n4_YnN5X9pxKeiXhDsZ', '', 'BR', 28210, 'signal', '2026-01-25 10:31:52', '2026-01-25 10:31:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.1573689, '', '', '', NULL, '0'), ('20260129222159.563844_IN_signal_a2b24f411322cc53', '20260129T222157Z_signal_IN_17465_n4_rlFpg9LwkMJr4IbF', '', 'IN', 17465, 'signal', '2026-01-29 22:22:00', '2026-01-29 22:22:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.038376, '', '', '', NULL, '0'), ('20260120002609.685381_IT_signal_42648e1be5cb6da5', '20260120T002608Z_signal_IT_3269_n4_5gj1XUhi3GTNhQ2c', '', 'IT', 3269, 'signal', '2026-01-20 00:26:08', '2026-01-20 00:26:08', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.75125754, '', '', '', NULL, '0'), ('20260127033137.903663_US_signal_95796985eb2062eb', '20260127T033137Z_signal_US_20115_n4_CTnZJDdcZ9NNJo9q', '', 'US', 20115, 'signal', '2026-01-27 03:31:37', '2026-01-27 03:31:37', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.35777518, '', '', '', NULL, '0'), ('20260126191421.710076_US_signal_0525c8c13e158738', '20260126T191421Z_signal_US_7922_n4_KYf66iopHyD6ohhY', '', 'US', 7922, 'signal', '2026-01-26 19:14:20', '2026-01-26 19:14:21', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.3128446, '', '', '', NULL, '0'), ('20260118033803.250613_CM_signal_b52936a77ea8b059', '20260118T033800Z_signal_CM_15964_n4_bCoPC05wnzJoHB7x', '', 'CM', 15964, 'signal', '2026-01-18 03:38:00', '2026-01-18 03:38:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.7432516, '', '', '', NULL, '0'), ('20260123090840.650629_CA_signal_71840b6978320000', '20260123T090839Z_signal_CA_577_n4_PIhBTPyLi8PHsoDy', '', 'CA', 577, 'signal', '2026-01-23 09:08:39', '2026-01-23 09:08:40', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.27.0', 0.19444399, '', '', '', NULL, '0'), ('20260112134453.775759_US_signal_785e7b66566d2be1', '20260112T134452Z_signal_US_11090_n4_pxUe58CIjkV9HBiz', '', 'US', 11090, 'signal', '2026-01-12 13:44:52', '2026-01-12 13:44:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.5586467, '', '', '', NULL, '0'), ('20260103103802.609988_FR_signal_0da8cc27f84d5ffd', '20260103T103801Z_signal_FR_12322_n4_DdojMxeEyUYGlzsq', '', 'FR', 12322, 'signal', '2026-01-03 10:37:59', '2026-01-03 10:37:59', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.1015654, '', '', '', NULL, '0'), ('20260106215331.674612_CM_signal_dc63250fe642e00f', '20260106T215328Z_signal_CM_15964_n4_UdbjBXmonpuftMiV', '', 'CM', 15964, 'signal', '2026-01-06 21:53:28', '2026-01-06 21:53:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 2.0551805, '', '', '', NULL, '0'), ('20260110181818.481389_RU_signal_d117815babb1c63b', '20260110T181818Z_signal_RU_39087_n4_kmgsaRh6rfCX8jl1', '', 'RU', 39087, 'signal', '2026-01-06 04:09:49', '2026-01-06 04:09:50', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 10.440062, '', '', '', NULL, '0'), ('20260120122355.499063_TR_signal_c91c358c4e1168dd', '20260120T122354Z_signal_TR_47331_n4_xJeMQAojMLfKZgjL', '', 'TR', 47331, 'signal', '2026-01-20 12:23:54', '2026-01-20 12:23:54', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.60208714, '', '', '', NULL, '0'), ('20260129151311.456862_ES_signal_9c0a9695e0265bf4', '20260129T151310Z_signal_ES_3352_n4_s3RXoTvKt4R2jUav', '', 'ES', 3352, 'signal', '2026-01-29 15:13:11', '2026-01-29 15:13:11', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.56744546, '', '', '', NULL, '0'), ('20260107073559.543536_IT_signal_64816471ddef9ede', '20260107T073558Z_signal_IT_30722_n4_6axwuwALvRqD47Og', '', 'IT', 30722, 'signal', '2026-01-07 07:35:58', '2026-01-07 07:35:58', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 0.8084099, '', '', '', NULL, '0'), ('20260108001413.438299_CM_signal_e1b01bec82c0230b', '20260108T001410Z_signal_CM_15964_n4_8XY0N6z9x8lrzlj2', '', 'CM', 15964, 'signal', '2026-01-08 00:14:09', '2026-01-08 00:14:10', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 2.145543, '', '', '', NULL, '0'), ('20260118061633.900624_PK_signal_f19d350c2a18c773', '20260118T061613Z_signal_PK_135407_n4_oTEQYpvi1viu8ElG', '', 'PK', 135407, 'signal', '2026-01-18 06:16:13', '2026-01-18 06:16:13', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 20.223848, '', '', '', NULL, '0'), ('20260102072053.373246_US_signal_a449924ebfb9e157', '20260102T072051Z_signal_US_33363_n4_UPuo0PoKSfXSELiQ', '', 'US', 33363, 'signal', '2026-01-02 07:20:52', '2026-01-02 07:20:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.5', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.24.0', 1.0205637, '', '', '', NULL, '0'), ('20260129143549.899949_US_signal_1fe8b6ad4eb95bbf', '20260129T143548Z_signal_US_7018_n4_J2zo41KS2WzBfUHg', '', 'US', 7018, 'signal', '2026-01-29 14:35:49', '2026-01-29 14:35:49', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.61999345, '', '', '', NULL, '0'), ('20260105113221.305238_GB_signal_05b461d7c031620a', '20260105T113218Z_signal_GB_328309_n4_CLE8li5t181WljPa', '', 'GB', 328309, 'signal', '2026-01-05 11:32:18', '2026-01-05 11:32:19', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.7276975, '', '', '', NULL, '0'), ('20260110182705.937428_US_signal_5a066278fa5d5fce', '20260110T182705Z_signal_US_7922_n4_UxZt0pglkpSVrn1W', '', 'US', 7922, 'signal', '2026-01-10 18:27:05', '2026-01-10 18:27:05', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.2958476, '', '', '', NULL, '0'), ('20260119082028.968978_US_signal_6c38e75da1910c76', '20260119T082028Z_signal_US_62658_n4_1oIOQ5LY5EabuzRV', '', 'US', 62658, 'signal', '2026-01-19 08:20:28', '2026-01-19 08:20:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.2513034, '', '', '', NULL, '0'), ('20260110211551.852077_AU_signal_9c5540ccf5ab7b76', '20260110T211550Z_signal_AU_4739_n4_DSb1hru8mzKHbOj5', '', 'AU', 4739, 'signal', '2026-01-10 21:15:50', '2026-01-10 21:15:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.6', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.25.0', 0.7667587, '', '', '', NULL, '0'), ('20260123012736.613823_FR_signal_1d12081f301db0bf', '20260123T012735Z_signal_FR_15557_n4_m9AMWnojHrJFlXVd', '', 'FR', 15557, 'signal', '2026-01-23 01:27:39', '2026-01-23 01:27:39', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.28.0', 0.5139983, '', '', '', NULL, '0'), ('20260126214344.619344_ES_signal_ea4412e4cb1b6296', '20260126T214344Z_signal_ES_57269_n4_Oh9Pk4F49VFNS5HD', '', 'ES', 57269, 'signal', '2026-01-26 21:43:44', '2026-01-26 21:43:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.3829573, '', '', '', NULL, '0'), ('20260128193853.330808_BR_signal_191aa206398ad2b2', '20260128T193852Z_signal_BR_28210_n4_vwLcSw36MAdDg6xA', '', 'BR', 28210, 'signal', '2026-01-28 19:38:53', '2026-01-28 19:38:53', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.71647525, '', '', '', NULL, '0'), ('20260125042659.141540_CY_signal_baa784af572c279d', '20260125T042657Z_signal_CY_35432_n4_plK8SnDPLvulz8on', '', 'CY', 35432, 'signal', '2026-01-25 04:27:00', '2026-01-25 04:27:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.1204401, '', '', '', NULL, '0'), ('20260117015457.248827_SI_signal_fc6af267bb729798', '20260117T015456Z_signal_SI_3212_n4_nmRG5b8LnLhWPsQ0', '', 'SI', 3212, 'signal', '2026-01-17 01:54:56', '2026-01-17 01:54:56', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.60261893, '', '', '', NULL, '0'), ('20260131204919.230824_SE_signal_2c4204c6e30ede77', '20260131T204918Z_signal_SE_8473_n4_37HchTGzYHU2oGMk', '', 'SE', 8473, 'signal', '2026-01-31 20:49:18', '2026-01-31 20:49:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.39307818, '', '', '', NULL, '0'), ('20260109230709.648875_RU_signal_083185be8c783b72', '20260109T230658Z_signal_RU_25159_n4_LpoDkzA3ZEz5mtHb', '', 'RU', 25159, 'signal', '2026-01-09 23:06:58', '2026-01-09 23:06:58', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 10.777376, '', '', '', NULL, '0'), ('20260102135516.354188_ES_signal_63305644f94d7800', '20260102T135515Z_signal_ES_3352_n4_cMVCYOuKJPdKyWvs', '', 'ES', 3352, 'signal', '2026-01-02 13:55:14', '2026-01-02 13:55:14', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4341947, '', '', '', NULL, '0'), ('20260109074220.142662_ES_signal_393f01c95fafdd1a', '20260109T074219Z_signal_ES_3352_n4_4g9zQPdSHQZiFon0', '', 'ES', 3352, 'signal', '2026-01-09 07:42:18', '2026-01-09 07:42:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4999763, '', '', '', NULL, '0'), ('20260108162529.001785_US_signal_acd93e5d19dfa896', '20260108T162527Z_signal_US_10796_n4_lNZP314S7LCyQuTo', '', 'US', 10796, 'signal', '2026-01-08 16:25:28', '2026-01-08 16:25:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6204547, '', '', '', NULL, '0'), ('20260129124738.638259_BR_signal_16e194fe04c17935', '20260129T124737Z_signal_BR_266121_n4_bX8sQE36Zbb4IKQO', '', 'BR', 266121, 'signal', '2026-01-29 12:47:38', '2026-01-29 12:47:38', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.5957706, '', '', '', NULL, '0'), ('20260108084022.731865_US_signal_b6756029807297e6', '20260108T084021Z_signal_US_20115_n4_87I8o9k7OwrXwi2g', '', 'US', 20115, 'signal', '2026-01-08 08:40:21', '2026-01-08 08:40:22', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4671578, '', '', '', NULL, '0'), ('20260105060506.962435_US_signal_2b3091d1974a289b', '20260105T060505Z_signal_US_7018_n4_j2yYYu5s0fhAiM22', '', 'US', 7018, 'signal', '2026-01-05 06:05:04', '2026-01-05 06:05:04', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.57331777, '', '', '', NULL, '0'), ('20260115021531.127320_VN_signal_df98957524a319cb', '20260115T021530Z_signal_VN_7552_n4_4QvRK9tJliFJp5xq', '', 'VN', 7552, 'signal', '2026-01-15 02:15:29', '2026-01-15 02:15:29', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8183516, '', '', '', NULL, '0'), ('20260126010331.893272_US_signal_bb73f06817b94957', '20260126T010331Z_signal_US_33387_n4_B3oBoJDmW5wpFqvC', '', 'US', 33387, 'signal', '2026-01-26 01:03:31', '2026-01-26 01:03:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.10474518, '', '', '', NULL, '0'), ('20260128100621.739976_RO_signal_25e202dac905fae5', '20260128T100620Z_signal_RO_8708_n4_Vi17A70VX1exQlXO', '', 'RO', 8708, 'signal', '2026-01-28 10:06:19', '2026-01-28 10:06:19', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.667061, '', '', '', NULL, '0'), ('20260109052451.264065_CA_signal_1eb36573803fa762', '20260109T052450Z_signal_CA_577_n4_J3zYJ0llJEvbDFWb', '', 'CA', 577, 'signal', '2026-01-09 05:24:50', '2026-01-09 05:24:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.6373465, '', '', '', NULL, '0'), ('20260106174916.480115_FR_signal_97df58aec56b5582', '20260106T174915Z_signal_FR_15557_n4_Z0PB4a69L1InxQWO', '', 'FR', 15557, 'signal', '2026-01-06 17:49:15', '2026-01-06 17:49:15', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6112673, '', '', '', NULL, '0'), ('20260130175951.568421_TR_signal_da2133354225608e', '20260130T175950Z_signal_TR_47331_n4_P6wTfiWSRYW9lWVI', '', 'TR', 47331, 'signal', '2026-01-30 17:59:51', '2026-01-30 17:59:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6442853, '', '', '', NULL, '0'), ('20260127044812.840537_GB_signal_61ba50916d8cffac', '20260127T044812Z_signal_GB_6871_n4_8uHbgFez8O2nCsox', '', 'GB', 6871, 'signal', '2026-01-27 04:48:12', '2026-01-27 04:48:12', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.09698934, '', '', '', NULL, '0'), ('20260116085126.865917_TH_signal_2f6e1bb32fc42308', '20260116T085125Z_signal_TH_45758_n4_iPrmlsbjjTozbmYR', '', 'TH', 45758, 'signal', '2026-01-16 08:51:25', '2026-01-16 08:51:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.7706264, '', '', '', NULL, '0'), ('20260124075855.723591_DE_signal_b8d817b2c8c35a59', '20260124T075855Z_signal_DE_31898_n4_IFIclPYPWpPPqoXs', '', 'DE', 31898, 'signal', '2026-01-24 07:58:55', '2026-01-24 07:58:55', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.05214951, '', '', '', NULL, '0'), ('20260130164932.080917_MX_signal_a7891bea830d0fa5', '20260130T164931Z_signal_MX_8151_n4_BfQpxYU7DDbzLKeN', '', 'MX', 8151, 'signal', '2026-01-30 16:49:31', '2026-01-30 16:49:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.27090517, '', '', '', NULL, '0'), ('20260123050626.011753_UA_signal_7781bcc2a8fb0faa', '20260123T050625Z_signal_UA_6876_n4_oZ4fYJJ27Q4wxUCO', '', 'UA', 6876, 'signal', '2026-01-23 05:06:26', '2026-01-23 05:06:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4885918, '', '', '', NULL, '0'), ('20260120045040.224875_DE_signal_b868eac6460f197b', '20260120T045039Z_signal_DE_8881_n4_Htoc7IKmn9n5tu2x', '', 'DE', 8881, 'signal', '2026-01-20 04:50:39', '2026-01-20 04:50:39', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.29948318, '', '', '', NULL, '0'), ('20260129060428.575194_CY_signal_25f09d8c3a288f5c', '20260129T060427Z_signal_CY_15805_n4_gwqonwh2Ax2n4dor', '', 'CY', 15805, 'signal', '2026-01-29 06:04:26', '2026-01-29 06:04:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.779632, '', '', '', NULL, '0'), ('20260115083527.901995_IN_signal_80baaedabdc895de', '20260115T083526Z_signal_IN_24560_n4_czpqbQ7j7Z6tMbwk', '', 'IN', 24560, 'signal', '2026-01-15 08:35:25', '2026-01-15 08:35:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8536811, '', '', '', NULL, '0'), ('20260128060546.823485_US_signal_58a7b8e78ea2f182', '20260128T060545Z_signal_US_203020_n4_KyJQZ7MA7SAvpG3W', '', 'US', 203020, 'signal', '2026-01-28 06:05:44', '2026-01-28 06:05:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.0699235, '', '', '', NULL, '0'), ('20260102010446.359409_ES_signal_151c38de04d7ed1f', '20260102T010445Z_signal_ES_57269_n4_RCxu595agyf2h3l4', '', 'ES', 57269, 'signal', '2026-01-02 01:04:45', '2026-01-02 01:04:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.45330188, '', '', '', NULL, '0'), ('20260114014644.641698_US_signal_110c28b6d721c302', '20260114T014644Z_signal_US_701_n4_KZ2YJ8Nsa5ZIjo81', '', 'US', 701, 'signal', '2026-01-14 01:46:44', '2026-01-14 01:46:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 0.26397383, '', '', '', NULL, '0'), ('20260125214505.275170_ES_signal_9f8649a2dc39dfe4', '20260125T214504Z_signal_ES_57269_n4_sQgWIakYDwIMoPUo', '', 'ES', 57269, 'signal', '2026-01-25 21:45:04', '2026-01-25 21:45:04', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.39790463, '', '', '', NULL, '0'), ('20260112102028.237552_VN_signal_78f9fa738b840749', '20260112T102027Z_signal_VN_45899_n4_WRISDRVfjsS1MVhD', '', 'VN', 45899, 'signal', '2026-01-12 10:20:26', '2026-01-12 10:20:27', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 0.8143061, '', '', '', NULL, '0'), ('20260119235541.383591_GB_signal_9a52a520cbf53453', '20260119T235531Z_signal_GB_6871_n4_Ayw1cubCzQQomBD9', '', 'GB', 6871, 'signal', '2026-01-19 23:55:31', '2026-01-19 23:55:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 10.05995, '', '', '', NULL, '0'), ('20260112213831.467389_KZ_signal_a16ff4187ffd2374', '20260112T213830Z_signal_KZ_60286_n4_4jfMB2x6dRPw1zue', '', 'KZ', 60286, 'signal', '2026-01-12 21:38:09', '2026-01-12 21:38:09', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8840163, '', '', '', NULL, '0'), ('20260129100428.809164_RU_signal_0fd185e6a396f133', '20260129T100408Z_signal_RU_12389_n4_4ckhwMuo9RwCX34e', '', 'RU', 12389, 'signal', '2026-01-29 10:04:08', '2026-01-29 10:04:09', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'windows', 't', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 20.172535, '', '', '', NULL, '0'), ('20260117012950.855475_ES_signal_4b764baa80b700db', '20260117T012949Z_signal_ES_3352_n4_oHfQvBg28Q7SoDou', '', 'ES', 3352, 'signal', '2026-01-17 01:29:49', '2026-01-17 01:29:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6271005, '', '', '', NULL, '0'), ('20260108161611.436103_FR_signal_d5559d8cbfb9f757', '20260108T161610Z_signal_FR_12322_n4_jh7D6RFRDpNxobbn', '', 'FR', 12322, 'signal', '2026-01-08 16:16:08', '2026-01-08 16:16:08', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4508722, '', '', '', NULL, '0'), ('20260122062316.714352_CA_signal_ed9118e794ddd148', '20260122T062315Z_signal_CA_852_n4_vCuP2C3zwUWi9D54', '', 'CA', 852, 'signal', '2026-01-22 06:23:15', '2026-01-22 06:23:16', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.13090481, '', '', '', NULL, '0'), ('20260101045021.148218_ES_signal_c6a0ac5e18361571', '20260101T045020Z_signal_ES_57269_n4_Kj915bUjDJ1q8F7Y', '', 'ES', 57269, 'signal', '2026-01-01 04:50:20', '2026-01-01 04:50:20', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.43395877, '', '', '', NULL, '0'), ('20260117225217.575965_KE_signal_9b2985f63256df7e', '20260117T225216Z_signal_KE_37061_n4_65qgh7tL9ImwL4Eo', '', 'KE', 37061, 'signal', '2026-01-17 22:52:14', '2026-01-17 22:52:14', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 1.1794214, '', '', '', NULL, '0'), ('20260101181803.339683_US_signal_dc3e20af849caa69', '20260101T181802Z_signal_US_395466_n4_RVRxwCzLJANrPAFm', '', 'US', 395466, 'signal', '2026-01-01 18:18:03', '2026-01-01 18:18:03', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.70013964, '', '', '', NULL, '0'), ('20260123204628.537028_US_signal_dbd08de12970f3d9', '20260123T204627Z_signal_US_174_n4_1od1R2Ja8akVVEzM', '', 'US', 174, 'signal', '2026-01-23 20:46:27', '2026-01-23 20:46:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.27568492, '', '', '', NULL, '0'), ('20260114224456.610555_FR_signal_2acdb40bf99cbebf', '20260114T224455Z_signal_FR_16276_n4_wiuEVNNkvDGju0no', '', 'FR', 16276, 'signal', '2026-01-14 22:44:55', '2026-01-14 22:44:55', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.5232244, '', '', '', NULL, '0'); diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py new file mode 100644 index 000000000..624081985 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -0,0 +1,354 @@ +# +# Most of the private API runs statistics on the last N days. As such, tests +# are not deterministic. +# + +import pytest + + +def privapi(client, subpath): + response = client.get(f"/api/_/{subpath}") + assert response.status_code == 200 + return response.json() + + +# TODO: improve tests + + +def test_private_api_asn_by_month(client): + url = "asn_by_month" + response = privapi(client, url) + assert len(response) > 0, response + r = response[0] + assert sorted(r.keys()) == ["date", "value"] + assert r["value"] > 10 + assert r["value"] < 10**6 + assert r["date"].endswith("T00:00:00+00:00") + + +def test_private_api_countries_by_month(client): + url = "countries_by_month" + response = privapi(client, url) + assert len(response) > 0, response + r = response[0] + assert sorted(r.keys()) == ["date", "value"] + assert r["value"] > 10 + assert r["value"] < 1000 + assert r["date"].endswith("T00:00:00+00:00") + + +def test_private_api_test_names(client, log): + url = "test_names" + response = privapi(client, url) + assert response == { + "test_names": [ + {"id": "bridge_reachability", "name": "Bridge Reachability"}, + {"id": "dash", "name": "DASH"}, + {"id": "dns_consistency", "name": "DNS Consistency"}, + {"id": "dnscheck", "name": "DNS Check"}, + {"id": "facebook_messenger", "name": "Facebook Messenger"}, + { + "id": "http_header_field_manipulation", + "name": "HTTP Header Field Manipulation", + }, + {"id": "http_host", "name": "HTTP Host"}, + {"id": "http_invalid_request_line", "name": "HTTP Invalid Request Line"}, + {"id": "http_requests", "name": "HTTP Requests"}, + {"id": "meek_fronted_requests_test", "name": "Meek Fronted Requests"}, + {"id": "multi_protocol_traceroute", "name": "Multi Protocol Traceroute"}, + {"id": "ndt", "name": "NDT"}, + {"id": "psiphon", "name": "Psiphon"}, + {"id": "riseupvpn", "name": "RiseupVPN"}, + {"id": "signal", "name": "Signal"}, + {"id": "stunreachability", "name": "STUN Reachability"}, + {"id": "tcp_connect", "name": "TCP Connect"}, + {"id": "telegram", "name": "Telegram"}, + {"id": "tor", "name": "Tor"}, + {"id": "torsf", "name": "Tor Snowflake"}, + {"id": "urlgetter", "name": "URL Getter"}, + {"id": "vanilla_tor", "name": "Vanilla Tor"}, + {"id": "web_connectivity", "name": "Web Connectivity"}, + {"id": "whatsapp", "name": "WhatsApp"}, + ] + } + + +def test_private_api_countries_total(client, log): + url = "countries" + response = privapi(client, url) + assert "countries" in response + assert len(response["countries"]) >= 20 + for a in response["countries"]: + if a["alpha_2"] == "CA": + assert a["count"] > 3 + assert a["name"] == "Canada" + return + + assert 0, "CA not found" + + +def test_private_api_test_coverage(client, log): + url = "test_coverage?probe_cc=BR" + resp = privapi(client, url) + assert 190 < len(resp["test_coverage"]) < 220 + assert 27 < len(resp["network_coverage"]) < 32 + assert sorted(resp["test_coverage"][0]) == ["count", "test_day", "test_group"] + assert sorted(resp["network_coverage"][0]) == ["count", "test_day"] + + +def test_private_api_test_coverage_with_groups(client, log): + url = "test_coverage?probe_cc=BR&test_groups=websites" + resp = privapi(client, url) + assert len(resp["test_coverage"]) > 10 + assert sorted(resp["test_coverage"][0]) == ["count", "test_day", "test_group"] + assert 27 < len(resp["network_coverage"]) < 32 + + +def test_private_api_domain_metadata1(client): + url = "domain_metadata?domain=facebook.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "www.facebook.com" + + +def test_private_api_domain_metadata2(client): + url = "domain_metadata?domain=www.facebook.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "www.facebook.com" + + +def test_private_api_domain_metadata3(client): + url = "domain_metadata?domain=www.twitter.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "twitter.com" + + +def test_private_api_domain_metadata4(client): + url = "domain_metadata?domain=www.this-domain-is-not-in-the-test-lists-for-sure.com" + resp = privapi(client, url) + assert resp["category_code"] == "MISC" + exp = "this-domain-is-not-in-the-test-lists-for-sure.com" + assert resp["canonical_domain"] == exp + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_networks(client, log): + url = "website_networks?probe_cc=US" + resp = privapi(client, url) + assert len(resp["results"]) > 100 + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_stats(client, log): + url = "website_stats?probe_cc=DE&probe_asn=3320&input=http:%2F%2Fwww.backtrack-linux.org%2F" + resp = privapi(client, url) + assert len(resp["results"]) > 2 + assert sorted(resp["results"][0].keys()) == [ + "anomaly_count", + "confirmed_count", + "failure_count", + "test_day", + "total_count", + ] + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_urls(client, log): + url = "website_urls?probe_cc=US&probe_asn=209" + response = privapi(client, url) + r = response["metadata"] + assert r["total_count"] > 0 + del r["total_count"] + assert r == { + "current_page": 1, + "limit": 10, + "next_url": "https://api.ooni.io/api/_/website_urls?limit=10&offset=10&probe_asn=209&probe_cc=US", + "offset": 0, + } + assert len(response["results"]) == 10 + + +def test_private_api_vanilla_tor_stats(client): + url = "vanilla_tor_stats?probe_cc=BR" + resp = privapi(client, url) + assert "notok_networks" in resp + return # FIXME: implement tests with mocked db + assert resp["notok_networks"] >= 0 + assert len(resp["networks"]) > 10 + assert sorted(resp["networks"][0].keys()) == [ + "failure_count", + "last_tested", + "probe_asn", + "success_count", + "test_runtime_avg", + "test_runtime_max", + "test_runtime_min", + "total_count", + ] + assert resp["last_tested"].startswith("20") + + +def test_private_api_vanilla_tor_stats_empty(client): + url = "vanilla_tor_stats?probe_cc=XY" + resp = privapi(client, url) + assert resp["notok_networks"] == 0 + assert len(resp["networks"]) == 0 + assert resp["last_tested"] is None + + +def test_private_api_im_networks(client): + url = "im_networks?probe_cc=BR" + resp = privapi(client, url) + return # FIXME: implement tests with mocked db + assert len(resp) > 1 + assert len(resp["facebook_messenger"]["ok_networks"]) > 5 + if "telegram" in resp: + assert len(resp["telegram"]["ok_networks"]) > 5 + assert len(resp["signal"]["ok_networks"]) > 5 + # assert len(resp["whatsapp"]["ok_networks"]) > 5 + + +def test_private_api_im_stats_basic(client): + url = "im_stats?probe_cc=CH&probe_asn=3303&test_name=facebook_messenger" + resp = privapi(client, url) + assert 20 < len(resp["results"]) < 34 + assert resp["results"][0]["total_count"] > -1 + assert resp["results"][0]["anomaly_count"] is None + assert len(resp["results"][0]["test_day"]) == 25 + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_im_stats(client): + url = "im_stats?probe_cc=CH&probe_asn=3303&test_name=facebook_messenger" + resp = privapi(client, url) + assert len(resp["results"]) > 10 + assert resp["results"][0]["total_count"] > -1 + assert resp["results"][0]["anomaly_count"] is None + assert len(resp["results"][0]["test_day"]) == 25 + assert sum(e["total_count"] for e in resp["results"]) > 0, resp + + +def test_private_api_network_stats(client): + # TODO: the stats are not implemented + url = "network_stats?probe_cc=GB" + response = privapi(client, url) + assert response == { + "metadata": { + "current_page": 1, + "limit": 10, + "next_url": None, + "offset": 0, + "total_count": 0, + }, + "results": [], + } + + +def test_private_api_country_overview(client): + url = "country_overview?probe_cc=BR" + resp = privapi(client, url) + assert resp["first_bucket_date"].startswith("20"), resp + assert resp["measurement_count"] > 1 + assert resp["network_count"] > 1 + + +def test_private_api_global_overview(client): + url = "global_overview" + response = privapi(client, url) + assert "country_count" in response + assert "measurement_count" in response + assert "network_count" in response + + +def test_private_api_global_overview_by_month(client): + url = "global_overview_by_month" + resp = privapi(client, url) + assert sorted(resp["networks_by_month"][0].keys()) == ["date", "value"] + assert sorted(resp["countries_by_month"][0].keys()) == ["date", "value"] + assert sorted(resp["measurements_by_month"][0].keys()) == ["date", "value"] + assert resp["networks_by_month"][0]["date"].endswith("T00:00:00+00:00") + + +@pytest.mark.skip(reason="cannot be tested") +def test_private_api_quotas_summary(client): + resp = privapi(client, "quotas_summary") + + +def test_private_api_check_report_id(client, log): + rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" + url = f"check_report_id?report_id={rid}" + response = privapi(client, url) + assert response == {"v": 0, "found": True} + + +def test_private_api_check_bogus_report_id_is_found(client, log): + # The API always returns True + url = f"check_report_id?report_id=BOGUS_REPORT_ID" + response = privapi(client, url) + assert response == {"v": 0, "found": True} + + +# # /circumvention_stats_by_country + + +@pytest.mark.skip(reason="depends on fresh data") +def test_private_api_circumvention_stats_by_country(client, log): + url = "circumvention_stats_by_country" + resp = privapi(client, url) + assert resp["v"] == 0 + assert len(resp["results"]) > 3 + + +# # /circumvention_runtime_stats + + +@pytest.mark.skip(reason="depends on fresh data") +def test_private_api_circumvention_runtime_stats(client, log): + url = "circumvention_runtime_stats" + resp = privapi(client, url) + assert resp["v"] == 0 + assert "error" not in resp, resp + assert len(resp["results"]) > 3, resp + + +# # /asnmeta + + +def test_private_api_ansmeta(client, log): + resp = privapi(client, "asnmeta?asn=0") + assert resp == {"org_name": "Unknown"} + + +# # /networks + + +def test_private_api_networks(client, log): + resp = privapi(client, "networks") + assert resp["v"] == 0 + assert len(resp["results"]) > 50 + assert resp["results"][0] + res = resp["results"][0] + for k in ["cnt", "org_name", "probe_asn"]: + assert k in res + assert isinstance(res["probe_asn"], int) + assert res["probe_asn"] > 0 + assert isinstance(res["cnt"], int) + assert res["cnt"] > 0 + assert isinstance(res["org_name"], str) + + +# # /domains + + +def test_private_api_domains(client, log): + resp = privapi(client, "domains") + assert resp["v"] == 0 + rows = resp["results"] + assert len(rows) > 5 + assert sorted(rows[0]) == ["category_code", "domain_name", "measurement_count"] + d = {r["domain_name"]: r["category_code"] for r in rows} + assert d["facebook.com"] == "GRP" + assert d["ncac.org"] == "NEWS" + assert d["twitter.com"] == "GRP"