diff --git a/dev/timing.py b/dev/timing.py index 90c3ad1..88b2436 100644 --- a/dev/timing.py +++ b/dev/timing.py @@ -3,12 +3,11 @@ configure_logging(mode="per-run", console=False, run_tag="SSP2-2050") p = Pathways( - datapackage="remind-SSP2-PkBudg1000.zip", + datapackage="pathways_2025-10-22.zip", # geography_mapping="geo_mapping_remind.yaml", # activities_mapping="act_categories_agg.yaml", ) -print(p.scenarios.coords["variables"].values) vars = [v for v in p.scenarios.coords["variables"].values if v.startswith("FE")] print(f"Calculating {len(vars)} variables") @@ -23,8 +22,8 @@ ], scenarios=p.scenarios.pathway.values.tolist(), years=[ - # 2020, - # 2030, + 2020, + 2030, 2040, 2050, ], diff --git a/pathways/data/topologies/gcam-topology.json b/pathways/data/topologies/gcam-topology.json new file mode 100644 index 0000000..1fb4a22 --- /dev/null +++ b/pathways/data/topologies/gcam-topology.json @@ -0,0 +1,35 @@ +{ + "Africa_Eastern": ["BI", "KM", "DJ", "ER", "ET", "KE", "MG", "MU", "RE", "RW", "SD", "SO", "UG", "SS"], + "Africa_Northern": ["DZ", "EG", "EH", "LY", "MA", "TN"], + "Africa_Southern": ["AO", "BW", "LS", "MZ", "MW", "NA", "SZ", "TZ", "ZM", "ZW"], + "Africa_Western": ["BJ", "BF", "CF", "CI", "CM", "CD", "CG", "CV", "GA", "GH", "GN", "GM", "GW", "GQ", "LR", "ML", "MR", "NE", "NG", "SN", "SL", "ST", "TD", "TG"], + "Argentina": ["AR"], + "Australia_NZ": ["AU", "NZ"], + "Brazil": ["BR"], + "Canada": ["CA"], + "Central America and Caribbean": ["AW", "AI", "AG", "BS", "BZ", "BM", "BB", "CR", "CU", "KY", "DM", "DO", "GP", "GD", "GT", "HN", "HT", "JM", "KN", "LC", "MS", "MQ", "NI", "PA", "SV", "TT", "VC"], + "Central Asia": ["AM", "AZ", "GE", "KZ", "KG", "MN", "TJ", "TM", "UZ"], + "China": ["CN", "HK", "MO"], + "Colombia": ["CO"], + "EU-12": ["BG", "CY", "CZ", "EE", "HU", "LT", "LV", "MT", "PL", "RO", "SK", "SI"], + "EU-15": ["AD", "AT", "BE", "DK", "FI", "FR", "DE", "GR", "GL", "IE", "IT", "LU", "MC", "NL", "PT", "SE", "ES", "GB", "GI"], + "Europe_Eastern": ["BY", "MD", "UA"], + "European Free Trade Association": ["IS", "NO", "CH"], + "Europe_Non_EU": ["AL", "BA", "HR", "MK", "ME", "RS", "TR", "XK"], + "India": ["IN"], + "Indonesia": ["ID"], + "Japan": ["JP"], + "Mexico": ["MX"], + "Middle East": ["AE", "BH", "IR", "IQ", "IL", "JO", "KW", "LB", "OM", "PS", "QA", "SA", "SY", "YE"], + "Pakistan": ["PK"], + "Russia": ["RU"], + "South Africa": ["ZA"], + "South America_Northern": ["GF", "GY", "SR", "VE", "CW"], + "South America_Southern": ["BO", "CL", "EC", "PE", "PY", "UY"], + "South Asia": ["AF", "BD", "BT", "LK", "MV", "NP"], + "Southeast Asia": ["AS", "BN", "CK", "FJ", "FM", "GU", "KH", "KI", "LA", "MH", "MM", "MP", "MY", "YT", "NC", "NF", "NU", "NR", "PN", "PH", "PW", "PG", "KP", "PF", "SG", "SB", "SC", "TH", "TK", "TL", "TO", "TV", "VN", "VU", "WS"], + "South Korea": ["KR"], + "Taiwan": ["TW"], + "USA": ["US"], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/image-topology.json b/pathways/data/topologies/image-topology.json new file mode 100644 index 0000000..1e0beac --- /dev/null +++ b/pathways/data/topologies/image-topology.json @@ -0,0 +1,280 @@ +{ + "RUS": [ + "AM", + "AZ", + "GE", + "RU" + ], + "CHN": [ + "CN", + "HK", + "MN", + "MO", + "TW" + ], + "RSAF": [ + "AO", + "BW", + "LS", + "MW", + "MZ", + "NA", + "SZ", + "TZ", + "ZM", + "ZW" + ], + "MEX": [ + "MX" + ], + "INDO": [ + "ID", + "PG", + "TL" + ], + "JAP": [ + "JP" + ], + "RSAM": [ + "AR", + "BO", + "CL", + "CO", + "EC", + "GF", + "GY", + "PE", + "PY", + "SR", + "UY", + "VE" + ], + "WAF": [ + "BF", + "BJ", + "CF", + "CM", + "CV", + "CD", + "CG", + "CI", + "GA", + "GH", + "GN", + "GQ", + "GM", + "GW", + "LR", + "ML", + "MR", + "NE", + "NG", + "SL", + "SN", + "ST", + "SH", + "TD", + "TG" + ], + "UKR": [ + "BY", + "MD", + "UA" + ], + "INDIA": [ + "IN" + ], + "ME": [ + "AE", + "BH", + "IL", + "IQ", + "IR", + "JO", + "KW", + "LB", + "OM", + "QA", + "SA", + "SY", + "YE" + ], + "WEU": [ + "AD", + "AT", + "BE", + "CH", + "DE", + "DK", + "ES", + "FI", + "FR", + "FO", + "GB", + "GI", + "GR", + "IE", + "IS", + "IT", + "LI", + "LU", + "MC", + "MT", + "NL", + "NO", + "PT", + "SE", + "SM", + "VA" + ], + "NAF": [ + "DZ", + "EG", + "EH", + "LY", + "MA", + "TN" + ], + "KOR": [ + "KP", + "KR" + ], + "EAF": [ + "BI", + "DJ", + "ER", + "ET", + "KE", + "KM", + "MG", + "MU", + "RW", + "RE", + "SC", + "SD", + "SO", + "UG", + "SS" + ], + "STAN": [ + "KZ", + "KG", + "TJ", + "TM", + "UZ" + ], + "CEU": [ + "AL", + "BA", + "BG", + "CS", + "CY", + "CZ", + "EE", + "HR", + "HU", + "LT", + "LV", + "MK", + "PL", + "RO", + "SI", + "SK", + "XK", + "ME" + ], + "RCAM": [ + "AI", + "AW", + "BB", + "BM", + "BZ", + "BS", + "CR", + "DM", + "DO", + "GD", + "GP", + "GT", + "HN", + "HT", + "JM", + "KY", + "MQ", + "MS", + "NI", + "AW", + "CW", + "SX", + "PA", + "US-PR", + "SV", + "KN", + "LC", + "VC", + "TT", + "TC", + "VG", + "VI", + "CU" + ], + "CAN": [ + "CA" + ], + "RSAS": [ + "AF", + "BD", + "BT", + "LK", + "MV", + "NP", + "PK" + ], + "SAF": [ + "ZA" + ], + "BRA": [ + "BR" + ], + "TUR": [ + "TR" + ], + "OCE": [ + "AS", + "AU", + "CK", + "FJ", + "KI", + "MH", + "MP", + "FM", + "NC", + "NR", + "NU", + "NZ", + "PF", + "PW", + "SB", + "TK", + "TO", + "TV", + "VU", + "WS" + ], + "USA": [ + "US", + "PM" + ], + "SEAS": [ + "BN", + "KH", + "LA", + "MM", + "MY", + "PH", + "SG", + "TH", + "VN" + ], + "World": ["GLO", "RoW"] +} diff --git a/pathways/data/topologies/message-topology.json b/pathways/data/topologies/message-topology.json new file mode 100644 index 0000000..b172723 --- /dev/null +++ b/pathways/data/topologies/message-topology.json @@ -0,0 +1,186 @@ +{ + "SAS": ["AF", "BD", "BT", "IN", "MV", "NP", "PK", "LK"], + "EEU": [ + "AL", + "BA", + "BG", + "HR", + "CZ", + "EE", + "HU", + "LV", + "LT", + "PL", + "RO", + "SK", + "SI", + "MK" + ], + "MEA": [ + "DZ", + "BH", + "EG", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "QA", + "SA", + "SD", + "SY", + "TN", + "AE", + "YE" + ], + "PAS": [ + "AS", + "BN", + "FJ", + "PF", + "KI", + "ID", + "MY", + "MM", + "NC", + "PG", + "PH", + "KR", + "SG", + "SB", + "TW", + "TH", + "TO", + "VU", + "WS" + ], + "WEU": [ + "AD", + "AT", + "BE", + "CY", + "DK", + "FO", + "FI", + "FR", + "DE", + "GI", + "GR", + "GL", + "IS", + "IE", + "IM", + "IT", + "LI", + "LU", + "MT", + "MC", + "NL", + "NO", + "PT", + "ES", + "SE", + "CH", + "TR", + "GB" + ], + "AFR": [ + "AO", + "BJ", + "BW", + "IO", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CI", + "DJ", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "MG", + "MW", + "ML", + "MR", + "MU", + "MZ", + "NA", + "NE", + "NG", + "RE", + "RW", + "SH", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SZ", + "TZ", + "TG", + "UG", + "ZM", + "ZW" + ], + "LAM": [ + "AG", + "AR", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BR", + "CL", + "CO", + "CR", + "CU", + "DM", + "DO", + "EC", + "SV", + "GF", + "GD", + "GP", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "MX", + "NI", + "PA", + "PY", + "PE", + "KN", + "VC", + "LC", + "SR", + "TT", + "UY", + "VE" + ], + "FSU": ["AM", "AZ", "BY", "GE", "KZ", "KG", "MD", "RU", "TJ", "TM", "UA", "UZ"], + "PAO": ["AU", "JP", "NZ"], + "RCPA": ["KH", "KP", "LA", "MN", "VN"], + "NAM": ["CA", "GU", "PR", "US", "VI"], + "CHN": ["CN", "HK", "MO"] +} diff --git a/pathways/data/topologies/remind-eu-topology.json b/pathways/data/topologies/remind-eu-topology.json new file mode 100644 index 0000000..0ced79e --- /dev/null +++ b/pathways/data/topologies/remind-eu-topology.json @@ -0,0 +1,294 @@ +{ + "CAZ": [ + "AU", + "CA", + "HM", + "NZ", + "PM" + ], + "CHA": [ + "CN", + "HK", + "MO", + "TW" + ], + "DEU": [ + "DE" + ], + "ECE": [ + "CZ", + "EE", + "LV", + "LT", + "PL", + "SK" + ], + "ECS": [ + "BG", + "HR", + "HU", + "RO", + "SI" + ], + "ENC": [ + "AX", + "DK", + "FO", + "FI", + "SE" + ], + "ESC": [ + "CY", + "GR", + "IT", + "MT" + ], + "ESW": [ + "PT", + "ES" + ], + "EWN": [ + "AT", + "BE", + "LU", + "NL" + ], + "FRA": [ + "FR" + ], + "IND": [ + "IN" + ], + "JPN": [ + "JP" + ], + "LAM": [ + "AI", + "AQ", + "AG", + "AR", + "AW", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BQ", + "BV", + "BR", + "KY", + "CL", + "CO", + "CR", + "CU", + "CW", + "DM", + "DO", + "EC", + "SV", + "FK", + "GF", + "GD", + "GP", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "MX", + "MS", + "NI", + "PA", + "PY", + "PE", + "PR", + "BL", + "KN", + "LC", + "MF", + "VC", + "SX", + "GS", + "SR", + "TT", + "TC", + "UY", + "VE", + "VG", + "VI" + ], + "MEA": [ + "DZ", + "BH", + "EG", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "AE", + "EH", + "YE" + ], + "NEN": [ + "GL", + "IS", + "LI", + "NO", + "SJ", + "CH" + ], + "NES": [ + "AL", + "AD", + "BA", + "VA", + "MK", + "MC", + "ME", + "SM", + "RS", + "TR" + ], + "OAS": [ + "AF", + "AS", + "BD", + "BT", + "IO", + "BN", + "KH", + "CX", + "CC", + "CK", + "FJ", + "PF", + "TF", + "GU", + "ID", + "KI", + "KP", + "KR", + "LA", + "MY", + "MV", + "MH", + "FM", + "MN", + "MM", + "NR", + "NP", + "NC", + "NU", + "NF", + "MP", + "PK", + "PW", + "PG", + "PH", + "PN", + "WS", + "SG", + "SB", + "LK", + "TH", + "TL", + "TK", + "TO", + "TV", + "UM", + "VU", + "VN", + "WF" + ], + "REF": [ + "AM", + "AZ", + "BY", + "GE", + "KZ", + "KG", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + "SSA": [ + "AO", + "BJ", + "BW", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CD", + "CI", + "DJ", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "MG", + "MW", + "ML", + "MR", + "MU", + "YT", + "MZ", + "NA", + "NE", + "NG", + "RE", + "RW", + "SH", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SS", + "SZ", + "TZ", + "TG", + "UG", + "ZM", + "ZW" + ], + "UKI": [ + "GI", + "GG", + "IE", + "IM", + "JE", + "GB" + ], + "USA": [ + "US" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/remind-topology.json b/pathways/data/topologies/remind-topology.json new file mode 100644 index 0000000..774259f --- /dev/null +++ b/pathways/data/topologies/remind-topology.json @@ -0,0 +1,242 @@ +{ + "LAM": [ + "AW", + "AI", + "AR", + "AQ", + "AG", + "BQ", + "BS", + "BZ", + "BM", + "BO", + "BR", + "BB", + "CL", + "CO", + "CR", + "CU", + "CW", + "KY", + "DM", + "DO", + "EC", + "FK", + "GP", + "GD", + "GT", + "GF", + "GY", + "HN", + "HT", + "JM", + "KN", + "LC", + "MF", + "MX", + "MS", + "MQ", + "NI", + "PA", + "PE", + "PR", + "PY", + "GS", + "SV", + "SR", + "SX", + "TC", + "TT", + "UY", + "VC", + "VE", + "VG", + "VI", + "RLA" + ], + "OAS": [ + "AF", + "AS", + "TF", + "BD", + "BN", + "BT", + "CK", + "FJ", + "FM", + "GU", + "ID", + "IO", + "KH", + "KI", + "KR", + "LA", + "LK", + "MV", + "MH", + "MM", + "MN", + "MP", + "MY", + "NC", + "NF", + "NU", + "NP", + "NR", + "PK", + "PN", + "PH", + "PW", + "PG", + "KP", + "PF", + "SG", + "SB", + "TH", + "TL", + "TO", + "TV", + "UM", + "VN", + "VU", + "WF", + "WS", + "MO" + ], + "SSA": [ + "AO", + "BI", + "BJ", + "BF", + "BW", + "CF", + "CI", + "CM", + "CD", + "CG", + "KM", + "CV", + "DJ", + "ER", + "ET", + "GA", + "GH", + "GN", + "GM", + "GW", + "GQ", + "KE", + "LR", + "LS", + "MG", + "ML", + "MZ", + "MR", + "MU", + "MW", + "YT", + "NA", + "NE", + "NG", + "RE", + "RW", + "SN", + "SH", + "SL", + "SO", + "SS", + "ST", + "SZ", + "SC", + "TD", + "TG", + "TZ", + "UG", + "ZA", + "ZM", + "ZW" + ], + "EUR": [ + "AX", + "AT", + "BE", + "BG", + "CY", + "CZ", + "DE", + "DK", + "ES", + "EE", + "FI", + "FR", + "FO", + "GB", + "GI", + "GR", + "HR", + "HU", + "IM", + "IE", + "IT", + "LT", + "LU", + "LV", + "MT", + "NL", + "PL", + "PT", + "RO", + "SK", + "SI", + "SE", + "XK" + ], + "NEU": [ + "AL", + "AD", + "BA", + "CH", + "GL", + "IS", + "LI", + "MC", + "MK", + "ME", + "NO", + "SJ", + "SM", + "RS", + "VA" + ], + "MEA": [ + "AE", + "BH", + "DZ", + "EG", + "EH", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "TR", + "YE" + ], + "REF": ["AM", "AZ", "BY", "GE", "KZ", "KG", "MD", "RU", "TJ", "TM", "UA", "UZ"], + "CAZ": ["AU", "CA", "NZ"], + "CHA": ["CN", "HK", "MO", "TW"], + "IND": ["IN"], + "JPN": ["JP"], + "USA": ["US", "PM"], + "World": ["GLO", "RoW"] +} diff --git a/pathways/data/topologies/tiam-ucl-topology.json b/pathways/data/topologies/tiam-ucl-topology.json new file mode 100644 index 0000000..cdf7d8f --- /dev/null +++ b/pathways/data/topologies/tiam-ucl-topology.json @@ -0,0 +1,251 @@ +{ + "AFR": [ + "DZ", + "AO", + "BJ", + "BW", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CI", + "CD", + "DJ", + "EG", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "LY", + "MG", + "MW", + "ML", + "MR", + "MA", + "MZ", + "NA", + "NE", + "NG", + "RW", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SS", + "SD", + "SZ", + "TG", + "TN", + "UG", + "TZ", + "ZM", + "ZW" + ], + + "AUS": [ + "AU", + "NZ" + ], + + "CAN": [ + "CA" + ], + + "CSA": [ + "AI", + "AG", + "AR", + "AW", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BR", + "KY", + "CL", + "CO", + "CR", + "CU", + "DM", + "DO", + "EC", + "SV", + "FK", + "GD", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "NI", + "PA", + "PY", + "PE", + "KN", + "LC", + "VC", + "SR", + "TT", + "UY", + "VE" + ], + + "CHI": [ + "CN", + "TW" + ], + + "EEU": [ + "BA", + "BG", + "HR", + "CZ", + "HU", + "ME", + "PL", + "RO", + "RS", + "SK", + "SI", + "MK" + ], + + "FSU": [ + "AM", + "AZ", + "BY", + "EE", + "GE", + "KZ", + "KG", + "LV", + "LT", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + + "IND": [ + "IN" + ], + + "JPN": [ + "JP" + ], + + "MEX": [ + "MX" + ], + + "MEA": [ + "BH", + "BN", + "CY", + "IR", + "IL", + "JO", + "KW", + "LB", + "PS", + "OM", + "QA", + "SA", + "SY", + "TR", + "AE", + "YE" + ], + + "ODA": [ + "AF", + "AS", + "BD", + "BT", + "KH", + "KP", + "FJ", + "PF", + "ID", + "KI", + "LA", + "MY", + "MV", + "MU", + "MN", + "MM", + "NP", + "NC", + "PK", + "PG", + "PH", + "WS", + "SG", + "SB", + "LK", + "TH", + "TL", + "TO", + "VU", + "VN" + ], + + "SKO": [ + "KR" + ], + + "UK": [ + "GB" + ], + + "USA": [ + "US" + ], + + "WEU": [ + "AL", + "AD", + "AT", + "BE", + "DK", + "FO", + "FI", + "FR", + "DE", + "GI", + "GR", + "GL", + "IS", + "IE", + "IT", + "LU", + "MT", + "MC", + "NL", + "NO", + "PT", + "SM", + "ES", + "SE", + "CH", + "VA" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/witch-topology.json b/pathways/data/topologies/witch-topology.json new file mode 100644 index 0000000..885f073 --- /dev/null +++ b/pathways/data/topologies/witch-topology.json @@ -0,0 +1,294 @@ +{ + "usa": [ + "US", + "PM" + ], + "te": [ + "AM", + "AZ", + "BY", + "GE", + "KZ", + "KG", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + "ssa": [ + "AO", + "BI", + "BJ", + "BF", + "BW", + "CF", + "CI", + "CM", + "CD", + "CG", + "KM", + "CV", + "DJ", + "ER", + "ET", + "GA", + "GH", + "GN", + "GM", + "GW", + "GQ", + "KE", + "LR", + "LS", + "MG", + "ML", + "MZ", + "MR", + "MU", + "MW", + "NA", + "NE", + "NG", + "RE", + "RW", + "SN", + "SH", + "SL", + "SO", + "SS", + "ST", + "SZ", + "SC", + "TD", + "TG", + "TZ", + "UG", + "ZM", + "ZW" + ], + "southafrica": [ + "ZA" + ], + "seasia": [ + "AS", + "TF", + "BN", + "CK", + "FJ", + "FM", + "GU", + "IO", + "KH", + "KI", + "LA", + "MH", + "MM", + "MN", + "MP", + "MY", + "NC", + "NF", + "NU", + "NR", + "PN", + "PH", + "PW", + "PG", + "KP", + "PF", + "SG", + "SB", + "TH", + "TL", + "TO", + "TV", + "UM", + "VN", + "VU", + "WF", + "WS", + "MO" + ], + "oceania": [ + "AU", + "NZ" + ], + "mexico": [ + "MX" + ], + "mena": [ + "AE", + "BH", + "DZ", + "EG", + "EH", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "TR", + "YE" + ], + "laca": [ + "AW", + "AI", + "AR", + "AQ", + "AG", + "BQ", + "BS", + "BZ", + "BM", + "BO", + "BB", + "CL", + "CO", + "CR", + "CU", + "CW", + "KY", + "DM", + "DO", + "EC", + "FK", + "GP", + "GD", + "GT", + "GF", + "GY", + "HN", + "HT", + "JM", + "KN", + "LC", + "MF", + "MS", + "MQ", + "NI", + "PA", + "PE", + "PR", + "PY", + "GS", + "SV", + "SR", + "SX", + "TC", + "TT", + "UY", + "VC", + "VE", + "VG", + "VI", + "RLA" + ], + "korea": [ + "KR" + ], + "japan": [ + "JP" + ], + "indonesia": [ + "ID" + ], + "india": [ + "IN" + ], + "eu27": [ + "AT", + "BE", + "BG", + "HR", + "CY", + "CZ", + "DK", + "EE", + "FI", + "FR", + "DE", + "GR", + "HU", + "IE", + "IT", + "LV", + "LT", + "LU", + "MT", + "NL", + "PL", + "PT", + "RO", + "SK", + "SI", + "ES", + "SE" + ], + "othereurope": [ + "AX", + "FO", + "GB", + "GI", + "IM", + "IS", + "LI", + "MC", + "NO", + "SM", + "CH", + "AL", + "AD", + "BA", + "XK", + "ME", + "MK", + "RS", + "VA", + "SJ", + "GL" + ], + "china": [ + "CN", + "HK", + "TW" + ], + "canada": [ + "CA" + ], + "brazil": [ + "BR" + ], + "sasia": [ + "AF", + "BD", + "BT", + "NP", + "MV", + "PK", + "LK" + ], + "ccasia": [ + "TM", + "TJ", + "KG", + "KZ", + "UZ", + "MN", + "AM", + "AZ", + "GE" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data_validation.py b/pathways/data_validation.py index 29b38ac..4f9038f 100644 --- a/pathways/data_validation.py +++ b/pathways/data_validation.py @@ -116,7 +116,9 @@ def validate_mapping(resource: datapackage.Resource): mapping = yaml.safe_load(resource.raw_read()) # Check that the data has the required structure - required_keys = ["dataset"] + required_keys = [ + "dataset", + ] for k, v in mapping.items(): if not set(required_keys).issubset(set(v.keys())): raise ValueError(f"Invalid mapping: missing keys for {k}") diff --git a/pathways/edges_matrix.py b/pathways/edges_matrix.py new file mode 100644 index 0000000..1339539 --- /dev/null +++ b/pathways/edges_matrix.py @@ -0,0 +1,213 @@ +""" +This module defines runs Edges, to produce a (regionalized) characterization matrix. +""" + +import bw2calc +import json +from typing import Optional, Dict +from edges import EdgeLCIA +from edges.matrix_builders import build_technosphere_edges_matrix +from edges import setup_package_logging +from bw_processing import Datapackage +from scipy.sparse import csr_matrix, vstack, issparse +import numpy as np +import sparse as spnd +import logging + +from .filesystem_constants import DATA_DIR + +setup_package_logging(level=logging.DEBUG) + + +def fetch_topology(model: str) -> Optional[Dict]: + """ + Find the JSON file containing the topologies of the provided model. + """ + topology_path = DATA_DIR / "topologies" / f"{model.lower()}-topology.json" + if topology_path.exists(): + # load json + return json.loads(topology_path.read_text()) + + raise FileNotFoundError( + f"Geographical definition file for the model '{model.upper()}' not found." + ) + + +def _edge_sets_for_lookup(lca: EdgeLCIA): + # Start empty; we'll OR them depending on which edge family you use + restrict_supplier_positions_bio: set[int] = set() + restrict_supplier_positions_tech: set[int] = set() + restrict_consumer_positions: set[int] = set() + + if getattr(lca, "biosphere_edges", None): + # biosphere_edges: (bio_row, tech_col) + bio_rows = {r for (r, _c) in lca.biosphere_edges} + tech_cols = {c for (_r, c) in lca.biosphere_edges} + restrict_supplier_positions_bio |= bio_rows + restrict_consumer_positions |= tech_cols + + if getattr(lca, "technosphere_edges", None): + # technosphere_edges: (tech_row_supplier, tech_col_consumer) + tech_rows = {r for (r, _c) in lca.technosphere_edges} + tech_cols = {c for (_r, c) in lca.technosphere_edges} + restrict_supplier_positions_tech |= tech_rows + restrict_consumer_positions |= tech_cols + + return ( + restrict_supplier_positions_bio, + restrict_supplier_positions_tech, + restrict_consumer_positions, + ) + + +def _build_position_to_technosphere_lookup( + technosphere_index: dict[int, dict], +) -> dict[int, dict]: + """ + technosphere_index: maps position -> activity metadata (name, location, classifications, etc.) + Return minimal fields Edges uses to enrich consumer/supplier info. + """ + out = {} + for pos, meta in technosphere_index.items(): + out[pos] = { + "location": meta.get("location"), + "classifications": meta.get("classifications"), + "name": meta.get("name"), + "reference product": meta.get("reference product"), + "unit": meta.get("unit"), + } + return out + + +def _ensure_minimal_flows( + lca: EdgeLCIA, + biosphere_index: dict[int, dict], + technosphere_index: dict[int, dict], +): + if not getattr(lca, "biosphere_flows", None): + lca.biosphere_flows = [ + { + "name": f.get("name"), + "categories": list(f.get("categories")), + "unit": f.get("unit"), + "location": f.get("location"), # usually None for biosphere flows + "classifications": f.get("classifications"), # optional + "position": lca.lca.dicts.biosphere[pos], + } + for pos, f in biosphere_index.items() + if pos in lca.lca.dicts.biosphere + ] + + if not getattr(lca, "technosphere_flows", None): + lca.technosphere_flows = [ + { + "name": a.get("name"), + "reference product": a.get("reference product"), + "unit": a.get("unit"), + "location": a.get("location"), + "classifications": a.get("classifications"), + "position": lca.lca.dicts.activity.reversed[pos], + } + for pos, a in technosphere_index.items() + if pos in lca.lca.dicts.activity + ] + + +def _as_row_csr(mat): + """Ensure the characterization is a 2D CSR row matrix.""" + if issparse(mat): + m = mat.tocsr() + # If it's a column/row vector, normalize to (1, n) + if m.ndim == 2 and m.shape[0] == 1: + return m + if m.ndim == 2 and m.shape[1] == 1: + return m.T.tocsr() + return m # already 2D + # Dense / 1D -> make (1, n) + arr = np.atleast_2d(np.asarray(mat)) + if arr.shape[0] != 1 and arr.shape[1] == 1: + arr = arr.T + return csr_matrix(arr) + + +def create_edges_characterization_matrix( + model: str, + multilca_obj: bw2calc.MultiLCA, + methods: list, + indices: dict[str, dict[int, dict]], +): + """ + Run Edges for each method and return a 3D sparse tensor of shape + (n_methods, m, n), where each [i, :, :] is that method's characterization plane. + """ + topology = fetch_topology(model) + + planes = [] + + for method in methods: + # create fake sparse inventory matrix with same SHAPE & DTYPE + first_matrix = next(iter(multilca_obj.inventories.values())) + multilca_obj.inventory = csr_matrix( + first_matrix.shape, dtype=getattr(first_matrix, "dtype", float) + ) + + lca = EdgeLCIA( + demand={}, + method=method, + lca=multilca_obj, + additional_topologies=topology, + ) + + if all( + cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data + ): + lca.technosphere_edges = { + (r, c) + for supply_array in multilca_obj.supply_arrays.values() + for r, c in zip( + *build_technosphere_edges_matrix( + multilca_obj.technosphere_matrix, supply_array + ).nonzero() + ) + } + else: + lca.biosphere_edges = { + (r, c) + for mat in multilca_obj.inventories.values() + for r, c in zip(*mat.nonzero()) + } + + _ensure_minimal_flows( + lca, + biosphere_index=indices["biosphere"], + technosphere_index=indices["technosphere"], + ) + lca.position_to_technosphere_flows_lookup = ( + _build_position_to_technosphere_lookup(indices["technosphere"]) + ) + + rs_bio, rs_tech, rc_cons = _edge_sets_for_lookup(lca) + lca._preprocess_lookups( + restrict_supplier_positions_bio=rs_bio, + restrict_supplier_positions_tech=rs_tech, + restrict_consumer_positions=rc_cons, + ) + lca.apply_strategies() + lca.evaluate_cfs() + + # Each method yields a 2D plane (m x n). Ensure SciPy CSR, then convert to pydata/sparse COO. + plane = lca.characterization_matrix + if not issparse(plane): + plane = csr_matrix(plane) + else: + plane = plane.tocsr() + + planes.append(spnd.COO.from_scipy_sparse(plane)) + + if not planes: + # Return an empty 3D tensor of shape (0, 0, 0) + return spnd.COO(np.zeros((0, 0, 0))).astype(float) + + # Stack along a NEW leading axis -> (n_methods, m, n) + characterization_tensor = spnd.stack(planes, axis=0) + return characterization_tensor diff --git a/pathways/lca.py b/pathways/lca.py index e1c185d..06be086 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -8,7 +8,7 @@ import pickle import uuid from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Dict, List, Tuple, Any import bw2calc as bc import bw_processing as bwp @@ -18,6 +18,11 @@ from bw_processing import Datapackage from premise.geomap import Geomap from scipy import sparse +from scipy.sparse import issparse + +from scipy import sparse as sps +import numpy as np +import sparse as spnd from .filesystem_constants import DIR_CACHED_DB from .lcia import fill_characterization_factors_matrices @@ -26,6 +31,7 @@ find_technology_indices, get_subshares_matrix, ) +from .edges_matrix import create_edges_characterization_matrix from .utils import ( CustomFilter, _group_technosphere_indices, @@ -224,27 +230,25 @@ def create_functional_units( variables, vars_idx, units_map, -) -> [dict, dict]: - """Build functional unit demands per scenario variable for a region. - - :param scenarios: Scenario dataset with multi-dimensional demand trajectories. - :type scenarios: xarray.Dataset - :param region: IAM region to extract. - :type region: str - :param model: Model identifier in the scenario data. - :type model: str - :param scenario: Pathway name within the scenario data. - :type scenario: str - :param year: Year to slice from the scenario dataset. - :type year: int - :param variables: Scenario variables to include. - :type variables: list[str] - :param vars_idx: Mapping with technosphere indices and dataset metadata per variable. - :type vars_idx: dict[str, dict[str, Any]] - :param units_map: Unit conversion dictionary loaded from configuration. - :type units_map: dict - :returns: Tuple of (functional unit mapping, detailed per-variable metadata). - :rtype: tuple[dict[int, float], dict[str, dict[str, Any]]] +) -> tuple[dict[Any, Any], dict[Any, Any]]: + """ + Create functional units for the given region, model, scenario, and year. + The functional units are created based on the demand for each variable in the scenarios dataset. + The demand is converted to the appropriate units using the units_map. + The functional units are returned as a dictionary where the keys are the dataset indices + and the values are the demand for that dataset. + Additionally, a detailed dictionary is returned containing information about each variable, + including its dataset index, demand, and unit conversion vector. + + :param scenarios: xarray.Dataset containing the scenarios data. + :param region: The region for which to create the functional units. + :param model: The model for which to create the functional units. + :param scenario: The scenario for which to create the functional units. + :param year: The year for which to create the functional units. + :param variables: List of variables to include in the functional units. + :param vars_idx: Dictionary mapping variables to their dataset indices and units. + :param units_map: Dictionary mapping units to their conversion factors. + :return: A tuple containing: """ variables_demand = {} @@ -296,7 +300,7 @@ def create_functional_units( ) else: logging.warning( - f"Unit conversion factors not found for {variable}." + f"Alternative unit or conversion factor missing not found for {variable}: {alternative_unit}, conversion_factor: {conversion_factor}." ) unit_vector = 1.0 else: @@ -331,6 +335,43 @@ def create_functional_units( }, variables_demand +def _build_sparse_inventory_results_3d( + lca, + characterization_matrix, + edges_methods: bool, +): + """ + Return a 3D sparse tensor (pydata/sparse COO): + + - edges_methods=True: + slices[i] = characterization_matrix.multiply(v_i) # (n_bio, n_cols) + -> stacked to (n_inv, n_bio, n_cols) + + - edges_methods=False: + slices[i] = (characterization_matrix @ v_i) # (n_methods, n_cols) + -> stacked to (n_inv, n_methods, n_cols) + + In both cases, each slice is built as a SciPy sparse matrix, + then converted to pydata/sparse COO and stacked along axis=0. + """ + if edges_methods: + # elementwise multiply, requires identical shapes + slices = [] + for v in lca.inventories.values(): + assert issparse(v), "inventory matrices must be SciPy sparse." + s = characterization_matrix.multiply(v) # SciPy sparse (n_bio, n_cols) + slices.append(spnd.COO.from_scipy_sparse(s)) + return spnd.stack(slices, axis=0) + else: + # proper matrix multiply + slices = [] + for v in lca.inventories.values(): + assert issparse(v), "inventory matrices must be SciPy sparse." + s = characterization_matrix @ v # SciPy sparse (n_methods, n_cols) + slices.append(spnd.COO.from_scipy_sparse(s)) + return spnd.stack(slices, axis=0) + + def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: """Run LCI/LCIA calculations for one region and persist intermediate arrays. @@ -350,8 +391,9 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: units_map, demand_cutoff, lca, - characterization_matrix, + characterization_matrix, # edges: COO (n_methods, n_bio, n_cols); regular: CSR (n_methods, n_bio) methods, + edges_methods, debug, use_distributions, uncertain_parameters, @@ -359,152 +401,208 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: id_uncertainty_indices_filepath = None id_technosphere_indices_filepath = None - iter_results_files = [] + iter_results_files: List[Path] = [] iter_param_vals_filepath = None - dict_loc_cat = {} - + # Build category × location mapping once + dict_loc_cat: Dict[tuple, np.ndarray] = {} cat_counter = 0 - for cat, act_cat_idx in lca.acts_category_idx_dict.items(): + for _, act_cat_idx in lca.acts_category_idx_dict.items(): loc_counter = 0 - for loc, act_loc_idx in lca.acts_location_idx_dict.items(): - # Find the intersection of indices + for _, act_loc_idx in lca.acts_location_idx_dict.items(): idx = np.intersect1d(act_cat_idx, act_loc_idx) - # Filter out any -1 indices filtered_idx = idx[idx != -1] - if filtered_idx.size > 0: - # Assign the filtered index array - # to the dict_loc_cat with (cat, loc) as key dict_loc_cat[(cat_counter, loc_counter)] = filtered_idx - loc_counter += 1 cat_counter += 1 + # Helper to build sparse 3D tensor: (n_inv, second_dim, n_cols) + # - edges: second_dim = n_methods, using characterization (n_methods, n_bio, n_cols) COO + # inventory v (n_bio, n_cols) -> broadcast multiply, then sum over biosphere (axis=1). + # - regular: second_dim = n_methods, using C (n_methods, n_bio) CSR @ v (n_bio, n_cols) + def _inventory_results_3d_edges(lca, char_coo: spnd.COO): + invs = [mat.tocsr() for mat in lca.inventories.values()] + rows = [] + for v in invs: + v_coo = spnd.COO.from_scipy_sparse(v) # (n_bio, n_cols) + H = char_coo * v_coo # (n_methods, n_bio, n_cols) + S = H.sum(axis=1) # sum over biosphere -> (n_methods, n_cols) + rows.append(S) + return spnd.stack(rows, axis=0) # (n_inv, n_methods, n_cols) + + def _inventory_results_3d_regular(lca, C: sps.csr_matrix): + invs = [mat.tocsr() for mat in lca.inventories.values()] + slices = [] + for v in invs: + M = C @ v # (n_methods, n_cols), SciPy sparse + slices.append(spnd.COO.from_scipy_sparse(M)) + return spnd.stack(slices, axis=0) # (n_inv, n_methods, n_cols) + if use_distributions == 0: # Regular LCA calculations - with CustomFilter("(almost) singular matrix"): - lca.lci() - - if debug: - logging.info(f"Iterations no.: {use_distributions}.") - - # Create a numpy array with the results - inventory_results = np.array( - [ - (characterization_matrix @ value).toarray() - for value in lca.inventories.values() - ] - ) + # with CustomFilter("(almost) singular matrix"): + # lca.lci() if debug: - logging.info(f"Shape of inventory_results: {inventory_results.shape}") + logging.info( + f"Edges methods: {edges_methods}. Monte Carlo iters: {use_distributions}." + ) - if debug: - for fu, inventory in lca.inventories.items(): - logging.info( - f"Functional unit: {fu}. Impact: {(characterization_matrix @ inventory).sum()}" + # --- Build sparse 3D inventory_results: (n_inv, n_methods, n_cols) + if edges_methods: + # characterization_matrix must be a 3D pydata.sparse COO: (n_methods, n_bio, n_cols) + if ( + not isinstance(characterization_matrix, spnd.COO) + or characterization_matrix.ndim != 3 + ): + raise ValueError( + "Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols)." ) - - iter_results = np.zeros( - ( - inventory_results.shape[0], - inventory_results.shape[1], - len(lca.acts_category_idx_dict), - len(lca.acts_location_idx_dict), + inventory_results = _inventory_results_3d_edges( + lca, characterization_matrix + ) + else: + # characterization_matrix must be a 2D SciPy sparse (n_methods, n_bio) + if ( + not issparse(characterization_matrix) + or characterization_matrix.ndim != 2 + ): + raise ValueError( + "Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio)." + ) + inventory_results = _inventory_results_3d_regular( + lca, characterization_matrix.tocsr() ) - ) if debug: - logging.info(f"Shape of iter_results: {iter_results.shape}") + logging.info(f"inventory_results shape: {inventory_results.shape}") + + # Unify downstream aggregation + # inventory_results: (n_inv, second_dim=n_methods, n_cols) + n_inv, second_dim, _ = inventory_results.shape + n_cat = len(lca.acts_category_idx_dict) + n_loc = len(lca.acts_location_idx_dict) + + zeros_block = spnd.zeros((n_inv, second_dim), dtype=inventory_results.dtype) + + cat_stacks = [] + for cat in range(n_cat): + loc_blocks = [] + for loc in range(n_loc): + idx = dict_loc_cat.get((cat, loc)) + if idx is None or idx.size == 0: + block = zeros_block # (n_inv, n_methods) + else: + block = inventory_results[:, :, idx].sum( + axis=2 + ) # (n_inv, n_methods) + loc_blocks.append(block) + # Stack blocks across the location axis -> (n_inv, n_methods, n_loc) + cat_stacks.append(spnd.stack(loc_blocks, axis=2)) - for (cat, loc), idx in dict_loc_cat.items(): - iter_results[:, :, cat, loc] = inventory_results[:, :, idx].sum(axis=2) + # Stack across categories -> (n_inv, n_methods, n_cat, n_loc) + iter_results = spnd.stack(cat_stacks, axis=2) if debug: - for f, fu in enumerate(lca.inventories.keys()): - logging.info(f"Functional unit: {fu}. Impact: {iter_results[f].sum()}") + logging.info(f"iter_results shape: {iter_results.shape}") - # Save iteration results to disk + # Save without densifying iter_results_filepath = DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" - sp.save_npz( - filename=iter_results_filepath, - matrix=sp.COO(iter_results), - compressed=True, + spnd.save_npz( + filename=iter_results_filepath, matrix=iter_results, compressed=True ) iter_results_files.append(iter_results_filepath) else: - # Use distributions for LCA calculations + # Monte Carlo: same sparse flow per iteration iter_param_vals = [] with CustomFilter("(almost) singular matrix"): - for iteration in range(use_distributions): + for _ in range(use_distributions): next(lca) lca.lci() - # Create a numpy array with the results - inventory_results = np.array( - [ - (characterization_matrix @ value).toarray() - for value in lca.inventories.values() - ] - ) - iter_param_vals.append( - [ - -lca.technosphere_matrix[index] - for index in lca.uncertain_parameters - ] - ) - - iter_results = np.zeros( - ( - inventory_results.shape[0], - inventory_results.shape[1], - len(lca.acts_category_idx_dict), - len(lca.acts_location_idx_dict), + if edges_methods: + if ( + not isinstance(characterization_matrix, spnd.COO) + or characterization_matrix.ndim != 3 + ): + raise ValueError( + "Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols)." + ) + inventory_results = _inventory_results_3d_edges( + lca, characterization_matrix ) - ) - - for (cat, loc), idx in dict_loc_cat.items(): - iter_results[:, :, cat, loc] = inventory_results[:, :, idx].sum( - axis=2 + else: + if ( + not issparse(characterization_matrix) + or characterization_matrix.ndim != 2 + ): + raise ValueError( + "Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio)." + ) + inventory_results = _inventory_results_3d_regular( + lca, characterization_matrix.tocsr() ) - # Save iteration results to disk + # Aggregate to (n_inv, n_methods, n_cat, n_loc) + n_inv, second_dim, _ = inventory_results.shape + n_cat = len(lca.acts_category_idx_dict) + n_loc = len(lca.acts_location_idx_dict) + + zeros_block = spnd.zeros( + (n_inv, second_dim), dtype=inventory_results.dtype + ) + + cat_stacks = [] + for cat in range(n_cat): + loc_blocks = [] + for loc in range(n_loc): + idx = dict_loc_cat.get((cat, loc)) + if idx is None or idx.size == 0: + block = zeros_block + else: + block = inventory_results[:, :, idx].sum(axis=2) + loc_blocks.append(block) + cat_stacks.append(spnd.stack(loc_blocks, axis=2)) + + iter_results = spnd.stack(cat_stacks, axis=2) + + # Save per-iteration sparse tensor iter_results_filepath = ( DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" ) - sp.save_npz( - filename=iter_results_filepath, - matrix=sp.COO(iter_results), - compressed=True, + spnd.save_npz( + filename=iter_results_filepath, matrix=iter_results, compressed=True ) iter_results_files.append(iter_results_filepath) - # Save iteration parameter values to disk + # Keep your MC bookkeeping + iter_param_vals.append( + [ + -lca.technosphere_matrix[index] + for index in lca.uncertain_parameters + ] + ) + + # Save MC parameter draws iter_param_vals_filepath = DIR_CACHED_DB / f"iter_param_vals_{uuid.uuid4()}.npy" np.save(file=iter_param_vals_filepath, arr=np.stack(iter_param_vals, axis=-1)) - # Save the uncertainty indices to disk + # Save indices id_uncertainty_indices_filepath = ( DIR_CACHED_DB / f"mc_indices_{uuid.uuid4()}.npy" ) - np.save( - file=id_uncertainty_indices_filepath, - arr=lca.uncertain_parameters, - ) + np.save(file=id_uncertainty_indices_filepath, arr=lca.uncertain_parameters) - # Save the technosphere indices to disk id_technosphere_indices_filepath = ( DIR_CACHED_DB / f"tech_indices_{uuid.uuid4()}.pkl" ) pickle.dump( - lca.technosphere_indices, - open(id_technosphere_indices_filepath, "wb"), + lca.technosphere_indices, open(id_technosphere_indices_filepath, "wb") ) - # Returning a dictionary containing the id_array and the variables - # to be able to fetch them back later + # Return file paths + FU variables d = { "iterations_results": iter_results_files, "variables": {k: v["demand"] for k, v in fus_details.items()}, @@ -515,15 +613,9 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: logging.info(f"FUs: {list(lca.inventories.keys())}") if use_distributions > 0: - d["uncertainty_params"] = [ - str(id_uncertainty_indices_filepath), - ] - d["technosphere_indices"] = [ - str(id_technosphere_indices_filepath), - ] - d["iterations_param_vals"] = [ - str(iter_param_vals_filepath), - ] + d["uncertainty_params"] = [str(id_uncertainty_indices_filepath)] + d["technosphere_indices"] = [str(id_technosphere_indices_filepath)] + d["iterations_param_vals"] = [str(iter_param_vals_filepath)] return d @@ -543,6 +635,7 @@ def _calculate_year(args: tuple): regions, variables, methods, + edges_methods, demand_cutoff, filepaths, mapping, @@ -716,12 +809,40 @@ def _calculate_year(args: tuple): if v in {value for tup in lca.uncertain_parameters for value in tup} } - characterization_matrix = fill_characterization_factors_matrices( - methods=methods, - biosphere_matrix_dict=lca.dicts.biosphere, - biosphere_dict=biosphere_indices, - debug=debug, - ) + if methods: + # regular LCIA methods + characterization_matrix = fill_characterization_factors_matrices( + methods=methods, + biosphere_matrix_dict=lca.dicts.biosphere, + biosphere_dict=biosphere_indices, + debug=debug, + ) + else: + print("Using EDGES' LCIA methods...") + + # EDGES' LCIA methods + formatted_biosphere_index = { + v: {"name": k[0], "categories": k[1:]} + for k, v in biosphere_indices.items() + } + formatted_technosphere_index = { + v: { + "name": k[0], + "reference product": k[1], + "unit": k[2], + "location": k[3], + } + for k, v in technosphere_indices.items() + } + characterization_matrix, lca = create_edges_characterization_matrix( + model=model, + multilca_obj=lca, + methods=edges_methods, + indices={ + "biosphere": formatted_biosphere_index, + "technosphere": formatted_technosphere_index, + }, + ) if debug: logging.info( @@ -745,6 +866,7 @@ def _calculate_year(args: tuple): lca, characterization_matrix, methods, + edges_methods, debug, use_distributions, uncertain_parameters, diff --git a/pathways/pathways.py b/pathways/pathways.py index aab9dbd..2dce18c 100644 --- a/pathways/pathways.py +++ b/pathways/pathways.py @@ -19,6 +19,8 @@ import xarray as xr import yaml +from edges import get_available_methods + from .data_validation import validate_datapackage from .filesystem_constants import DATA_DIR, USER_LOGS_DIR from .lca import _calculate_year, get_lca_matrices @@ -274,6 +276,7 @@ def _get_scenarios(self, scenario_data: pd.DataFrame) -> xr.DataArray: def calculate( self, methods: Optional[List[str]] = None, + edges_methods: Optional[List[str]] = None, models: Optional[List[str]] = None, scenarios: Optional[List[str]] = None, regions: Optional[List[str]] = None, @@ -321,8 +324,32 @@ def calculate( self.scenarios = harmonize_units(self.scenarios, variables) + if methods: + available_methods = get_lcia_method_names() + for m in methods: + if m not in available_methods: + raise ValueError(f"LCIA method {m} not found in available methods.") + + if edges_methods: + available_methods = get_available_methods() + for m in edges_methods: + if m not in available_methods: + raise ValueError( + f"Edge LCIA method {m} not found in available `edges` methods." + ) + + if methods and edges_methods: + raise ValueError( + "Please provide either `methods` or `edges_methods`, not both." + ) + + if methods is None and edges_methods is None: + raise ValueError( + "Please provide at least one of `methods` or `edges_methods`." + ) + # if no methods are provided, use all those available - methods = methods or get_lcia_method_names() + if self.debug: logging.info(f"Using the following LCIA methods: {methods}") @@ -383,7 +410,7 @@ def calculate( self.geography_mapping = {loc: loc for loc in locations} self.lca_results = create_lca_results_array( - methods=methods, + methods=methods or [str(m) for m in edges_methods], years=years, regions=regions, locations=locations, @@ -417,6 +444,7 @@ def calculate( regions, variables, methods, + edges_methods, demand_cutoff, self.filepaths, self.mapping, diff --git a/pathways/utils.py b/pathways/utils.py index ce5e26a..14d08b6 100644 --- a/pathways/utils.py +++ b/pathways/utils.py @@ -549,8 +549,9 @@ def add_lhv(variable, mapping) -> Union[dict, None]: :rtype: dict """ if variable in mapping: - if "lhv" in mapping[variable]: - return mapping[variable]["lhv"] + for ds in mapping[variable]["dataset"]: + if "lhv" in ds: + return ds["lhv"] return {} @@ -610,6 +611,12 @@ def fetch_indices( ) pass + for variable in variables: + if variable not in mapping: + print( + f"Variable '{variable}' not found in mapping. Ensure it is correctly defined." + ) + if idxs is not None: # Map variables to their indices and associated dataset information