From ea336e9a2a7af3a1ca8b8bb6273d621f9405da57 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sun, 2 Nov 2025 18:18:47 +0100 Subject: [PATCH 1/8] Edges implementation --- dev/timing.py | 7 +- pathways/data/topologies/gcam-topology.json | 35 ++ pathways/data/topologies/image-topology.json | 280 +++++++++++++++ .../data/topologies/message-topology.json | 186 ++++++++++ .../data/topologies/remind-eu-topology.json | 294 ++++++++++++++++ pathways/data/topologies/remind-topology.json | 242 +++++++++++++ .../data/topologies/tiam-ucl-topology.json | 251 ++++++++++++++ pathways/data/topologies/witch-topology.json | 294 ++++++++++++++++ pathways/data_validation.py | 2 +- pathways/edges_matrix.py | 203 +++++++++++ pathways/lca.py | 324 +++++++++++------- pathways/pathways.py | 26 +- 12 files changed, 2013 insertions(+), 131 deletions(-) create mode 100644 pathways/data/topologies/gcam-topology.json create mode 100644 pathways/data/topologies/image-topology.json create mode 100644 pathways/data/topologies/message-topology.json create mode 100644 pathways/data/topologies/remind-eu-topology.json create mode 100644 pathways/data/topologies/remind-topology.json create mode 100644 pathways/data/topologies/tiam-ucl-topology.json create mode 100644 pathways/data/topologies/witch-topology.json create mode 100644 pathways/edges_matrix.py diff --git a/dev/timing.py b/dev/timing.py index 90c3ad1..88b2436 100644 --- a/dev/timing.py +++ b/dev/timing.py @@ -3,12 +3,11 @@ configure_logging(mode="per-run", console=False, run_tag="SSP2-2050") p = Pathways( - datapackage="remind-SSP2-PkBudg1000.zip", + datapackage="pathways_2025-10-22.zip", # geography_mapping="geo_mapping_remind.yaml", # activities_mapping="act_categories_agg.yaml", ) -print(p.scenarios.coords["variables"].values) vars = [v for v in p.scenarios.coords["variables"].values if v.startswith("FE")] print(f"Calculating {len(vars)} variables") @@ -23,8 +22,8 @@ ], scenarios=p.scenarios.pathway.values.tolist(), years=[ - # 2020, - # 2030, + 2020, + 2030, 2040, 2050, ], diff --git a/pathways/data/topologies/gcam-topology.json b/pathways/data/topologies/gcam-topology.json new file mode 100644 index 0000000..1fb4a22 --- /dev/null +++ b/pathways/data/topologies/gcam-topology.json @@ -0,0 +1,35 @@ +{ + "Africa_Eastern": ["BI", "KM", "DJ", "ER", "ET", "KE", "MG", "MU", "RE", "RW", "SD", "SO", "UG", "SS"], + "Africa_Northern": ["DZ", "EG", "EH", "LY", "MA", "TN"], + "Africa_Southern": ["AO", "BW", "LS", "MZ", "MW", "NA", "SZ", "TZ", "ZM", "ZW"], + "Africa_Western": ["BJ", "BF", "CF", "CI", "CM", "CD", "CG", "CV", "GA", "GH", "GN", "GM", "GW", "GQ", "LR", "ML", "MR", "NE", "NG", "SN", "SL", "ST", "TD", "TG"], + "Argentina": ["AR"], + "Australia_NZ": ["AU", "NZ"], + "Brazil": ["BR"], + "Canada": ["CA"], + "Central America and Caribbean": ["AW", "AI", "AG", "BS", "BZ", "BM", "BB", "CR", "CU", "KY", "DM", "DO", "GP", "GD", "GT", "HN", "HT", "JM", "KN", "LC", "MS", "MQ", "NI", "PA", "SV", "TT", "VC"], + "Central Asia": ["AM", "AZ", "GE", "KZ", "KG", "MN", "TJ", "TM", "UZ"], + "China": ["CN", "HK", "MO"], + "Colombia": ["CO"], + "EU-12": ["BG", "CY", "CZ", "EE", "HU", "LT", "LV", "MT", "PL", "RO", "SK", "SI"], + "EU-15": ["AD", "AT", "BE", "DK", "FI", "FR", "DE", "GR", "GL", "IE", "IT", "LU", "MC", "NL", "PT", "SE", "ES", "GB", "GI"], + "Europe_Eastern": ["BY", "MD", "UA"], + "European Free Trade Association": ["IS", "NO", "CH"], + "Europe_Non_EU": ["AL", "BA", "HR", "MK", "ME", "RS", "TR", "XK"], + "India": ["IN"], + "Indonesia": ["ID"], + "Japan": ["JP"], + "Mexico": ["MX"], + "Middle East": ["AE", "BH", "IR", "IQ", "IL", "JO", "KW", "LB", "OM", "PS", "QA", "SA", "SY", "YE"], + "Pakistan": ["PK"], + "Russia": ["RU"], + "South Africa": ["ZA"], + "South America_Northern": ["GF", "GY", "SR", "VE", "CW"], + "South America_Southern": ["BO", "CL", "EC", "PE", "PY", "UY"], + "South Asia": ["AF", "BD", "BT", "LK", "MV", "NP"], + "Southeast Asia": ["AS", "BN", "CK", "FJ", "FM", "GU", "KH", "KI", "LA", "MH", "MM", "MP", "MY", "YT", "NC", "NF", "NU", "NR", "PN", "PH", "PW", "PG", "KP", "PF", "SG", "SB", "SC", "TH", "TK", "TL", "TO", "TV", "VN", "VU", "WS"], + "South Korea": ["KR"], + "Taiwan": ["TW"], + "USA": ["US"], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/image-topology.json b/pathways/data/topologies/image-topology.json new file mode 100644 index 0000000..1e0beac --- /dev/null +++ b/pathways/data/topologies/image-topology.json @@ -0,0 +1,280 @@ +{ + "RUS": [ + "AM", + "AZ", + "GE", + "RU" + ], + "CHN": [ + "CN", + "HK", + "MN", + "MO", + "TW" + ], + "RSAF": [ + "AO", + "BW", + "LS", + "MW", + "MZ", + "NA", + "SZ", + "TZ", + "ZM", + "ZW" + ], + "MEX": [ + "MX" + ], + "INDO": [ + "ID", + "PG", + "TL" + ], + "JAP": [ + "JP" + ], + "RSAM": [ + "AR", + "BO", + "CL", + "CO", + "EC", + "GF", + "GY", + "PE", + "PY", + "SR", + "UY", + "VE" + ], + "WAF": [ + "BF", + "BJ", + "CF", + "CM", + "CV", + "CD", + "CG", + "CI", + "GA", + "GH", + "GN", + "GQ", + "GM", + "GW", + "LR", + "ML", + "MR", + "NE", + "NG", + "SL", + "SN", + "ST", + "SH", + "TD", + "TG" + ], + "UKR": [ + "BY", + "MD", + "UA" + ], + "INDIA": [ + "IN" + ], + "ME": [ + "AE", + "BH", + "IL", + "IQ", + "IR", + "JO", + "KW", + "LB", + "OM", + "QA", + "SA", + "SY", + "YE" + ], + "WEU": [ + "AD", + "AT", + "BE", + "CH", + "DE", + "DK", + "ES", + "FI", + "FR", + "FO", + "GB", + "GI", + "GR", + "IE", + "IS", + "IT", + "LI", + "LU", + "MC", + "MT", + "NL", + "NO", + "PT", + "SE", + "SM", + "VA" + ], + "NAF": [ + "DZ", + "EG", + "EH", + "LY", + "MA", + "TN" + ], + "KOR": [ + "KP", + "KR" + ], + "EAF": [ + "BI", + "DJ", + "ER", + "ET", + "KE", + "KM", + "MG", + "MU", + "RW", + "RE", + "SC", + "SD", + "SO", + "UG", + "SS" + ], + "STAN": [ + "KZ", + "KG", + "TJ", + "TM", + "UZ" + ], + "CEU": [ + "AL", + "BA", + "BG", + "CS", + "CY", + "CZ", + "EE", + "HR", + "HU", + "LT", + "LV", + "MK", + "PL", + "RO", + "SI", + "SK", + "XK", + "ME" + ], + "RCAM": [ + "AI", + "AW", + "BB", + "BM", + "BZ", + "BS", + "CR", + "DM", + "DO", + "GD", + "GP", + "GT", + "HN", + "HT", + "JM", + "KY", + "MQ", + "MS", + "NI", + "AW", + "CW", + "SX", + "PA", + "US-PR", + "SV", + "KN", + "LC", + "VC", + "TT", + "TC", + "VG", + "VI", + "CU" + ], + "CAN": [ + "CA" + ], + "RSAS": [ + "AF", + "BD", + "BT", + "LK", + "MV", + "NP", + "PK" + ], + "SAF": [ + "ZA" + ], + "BRA": [ + "BR" + ], + "TUR": [ + "TR" + ], + "OCE": [ + "AS", + "AU", + "CK", + "FJ", + "KI", + "MH", + "MP", + "FM", + "NC", + "NR", + "NU", + "NZ", + "PF", + "PW", + "SB", + "TK", + "TO", + "TV", + "VU", + "WS" + ], + "USA": [ + "US", + "PM" + ], + "SEAS": [ + "BN", + "KH", + "LA", + "MM", + "MY", + "PH", + "SG", + "TH", + "VN" + ], + "World": ["GLO", "RoW"] +} diff --git a/pathways/data/topologies/message-topology.json b/pathways/data/topologies/message-topology.json new file mode 100644 index 0000000..b172723 --- /dev/null +++ b/pathways/data/topologies/message-topology.json @@ -0,0 +1,186 @@ +{ + "SAS": ["AF", "BD", "BT", "IN", "MV", "NP", "PK", "LK"], + "EEU": [ + "AL", + "BA", + "BG", + "HR", + "CZ", + "EE", + "HU", + "LV", + "LT", + "PL", + "RO", + "SK", + "SI", + "MK" + ], + "MEA": [ + "DZ", + "BH", + "EG", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "QA", + "SA", + "SD", + "SY", + "TN", + "AE", + "YE" + ], + "PAS": [ + "AS", + "BN", + "FJ", + "PF", + "KI", + "ID", + "MY", + "MM", + "NC", + "PG", + "PH", + "KR", + "SG", + "SB", + "TW", + "TH", + "TO", + "VU", + "WS" + ], + "WEU": [ + "AD", + "AT", + "BE", + "CY", + "DK", + "FO", + "FI", + "FR", + "DE", + "GI", + "GR", + "GL", + "IS", + "IE", + "IM", + "IT", + "LI", + "LU", + "MT", + "MC", + "NL", + "NO", + "PT", + "ES", + "SE", + "CH", + "TR", + "GB" + ], + "AFR": [ + "AO", + "BJ", + "BW", + "IO", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CI", + "DJ", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "MG", + "MW", + "ML", + "MR", + "MU", + "MZ", + "NA", + "NE", + "NG", + "RE", + "RW", + "SH", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SZ", + "TZ", + "TG", + "UG", + "ZM", + "ZW" + ], + "LAM": [ + "AG", + "AR", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BR", + "CL", + "CO", + "CR", + "CU", + "DM", + "DO", + "EC", + "SV", + "GF", + "GD", + "GP", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "MX", + "NI", + "PA", + "PY", + "PE", + "KN", + "VC", + "LC", + "SR", + "TT", + "UY", + "VE" + ], + "FSU": ["AM", "AZ", "BY", "GE", "KZ", "KG", "MD", "RU", "TJ", "TM", "UA", "UZ"], + "PAO": ["AU", "JP", "NZ"], + "RCPA": ["KH", "KP", "LA", "MN", "VN"], + "NAM": ["CA", "GU", "PR", "US", "VI"], + "CHN": ["CN", "HK", "MO"] +} diff --git a/pathways/data/topologies/remind-eu-topology.json b/pathways/data/topologies/remind-eu-topology.json new file mode 100644 index 0000000..0ced79e --- /dev/null +++ b/pathways/data/topologies/remind-eu-topology.json @@ -0,0 +1,294 @@ +{ + "CAZ": [ + "AU", + "CA", + "HM", + "NZ", + "PM" + ], + "CHA": [ + "CN", + "HK", + "MO", + "TW" + ], + "DEU": [ + "DE" + ], + "ECE": [ + "CZ", + "EE", + "LV", + "LT", + "PL", + "SK" + ], + "ECS": [ + "BG", + "HR", + "HU", + "RO", + "SI" + ], + "ENC": [ + "AX", + "DK", + "FO", + "FI", + "SE" + ], + "ESC": [ + "CY", + "GR", + "IT", + "MT" + ], + "ESW": [ + "PT", + "ES" + ], + "EWN": [ + "AT", + "BE", + "LU", + "NL" + ], + "FRA": [ + "FR" + ], + "IND": [ + "IN" + ], + "JPN": [ + "JP" + ], + "LAM": [ + "AI", + "AQ", + "AG", + "AR", + "AW", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BQ", + "BV", + "BR", + "KY", + "CL", + "CO", + "CR", + "CU", + "CW", + "DM", + "DO", + "EC", + "SV", + "FK", + "GF", + "GD", + "GP", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "MX", + "MS", + "NI", + "PA", + "PY", + "PE", + "PR", + "BL", + "KN", + "LC", + "MF", + "VC", + "SX", + "GS", + "SR", + "TT", + "TC", + "UY", + "VE", + "VG", + "VI" + ], + "MEA": [ + "DZ", + "BH", + "EG", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "AE", + "EH", + "YE" + ], + "NEN": [ + "GL", + "IS", + "LI", + "NO", + "SJ", + "CH" + ], + "NES": [ + "AL", + "AD", + "BA", + "VA", + "MK", + "MC", + "ME", + "SM", + "RS", + "TR" + ], + "OAS": [ + "AF", + "AS", + "BD", + "BT", + "IO", + "BN", + "KH", + "CX", + "CC", + "CK", + "FJ", + "PF", + "TF", + "GU", + "ID", + "KI", + "KP", + "KR", + "LA", + "MY", + "MV", + "MH", + "FM", + "MN", + "MM", + "NR", + "NP", + "NC", + "NU", + "NF", + "MP", + "PK", + "PW", + "PG", + "PH", + "PN", + "WS", + "SG", + "SB", + "LK", + "TH", + "TL", + "TK", + "TO", + "TV", + "UM", + "VU", + "VN", + "WF" + ], + "REF": [ + "AM", + "AZ", + "BY", + "GE", + "KZ", + "KG", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + "SSA": [ + "AO", + "BJ", + "BW", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CD", + "CI", + "DJ", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "MG", + "MW", + "ML", + "MR", + "MU", + "YT", + "MZ", + "NA", + "NE", + "NG", + "RE", + "RW", + "SH", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SS", + "SZ", + "TZ", + "TG", + "UG", + "ZM", + "ZW" + ], + "UKI": [ + "GI", + "GG", + "IE", + "IM", + "JE", + "GB" + ], + "USA": [ + "US" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/remind-topology.json b/pathways/data/topologies/remind-topology.json new file mode 100644 index 0000000..774259f --- /dev/null +++ b/pathways/data/topologies/remind-topology.json @@ -0,0 +1,242 @@ +{ + "LAM": [ + "AW", + "AI", + "AR", + "AQ", + "AG", + "BQ", + "BS", + "BZ", + "BM", + "BO", + "BR", + "BB", + "CL", + "CO", + "CR", + "CU", + "CW", + "KY", + "DM", + "DO", + "EC", + "FK", + "GP", + "GD", + "GT", + "GF", + "GY", + "HN", + "HT", + "JM", + "KN", + "LC", + "MF", + "MX", + "MS", + "MQ", + "NI", + "PA", + "PE", + "PR", + "PY", + "GS", + "SV", + "SR", + "SX", + "TC", + "TT", + "UY", + "VC", + "VE", + "VG", + "VI", + "RLA" + ], + "OAS": [ + "AF", + "AS", + "TF", + "BD", + "BN", + "BT", + "CK", + "FJ", + "FM", + "GU", + "ID", + "IO", + "KH", + "KI", + "KR", + "LA", + "LK", + "MV", + "MH", + "MM", + "MN", + "MP", + "MY", + "NC", + "NF", + "NU", + "NP", + "NR", + "PK", + "PN", + "PH", + "PW", + "PG", + "KP", + "PF", + "SG", + "SB", + "TH", + "TL", + "TO", + "TV", + "UM", + "VN", + "VU", + "WF", + "WS", + "MO" + ], + "SSA": [ + "AO", + "BI", + "BJ", + "BF", + "BW", + "CF", + "CI", + "CM", + "CD", + "CG", + "KM", + "CV", + "DJ", + "ER", + "ET", + "GA", + "GH", + "GN", + "GM", + "GW", + "GQ", + "KE", + "LR", + "LS", + "MG", + "ML", + "MZ", + "MR", + "MU", + "MW", + "YT", + "NA", + "NE", + "NG", + "RE", + "RW", + "SN", + "SH", + "SL", + "SO", + "SS", + "ST", + "SZ", + "SC", + "TD", + "TG", + "TZ", + "UG", + "ZA", + "ZM", + "ZW" + ], + "EUR": [ + "AX", + "AT", + "BE", + "BG", + "CY", + "CZ", + "DE", + "DK", + "ES", + "EE", + "FI", + "FR", + "FO", + "GB", + "GI", + "GR", + "HR", + "HU", + "IM", + "IE", + "IT", + "LT", + "LU", + "LV", + "MT", + "NL", + "PL", + "PT", + "RO", + "SK", + "SI", + "SE", + "XK" + ], + "NEU": [ + "AL", + "AD", + "BA", + "CH", + "GL", + "IS", + "LI", + "MC", + "MK", + "ME", + "NO", + "SJ", + "SM", + "RS", + "VA" + ], + "MEA": [ + "AE", + "BH", + "DZ", + "EG", + "EH", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "TR", + "YE" + ], + "REF": ["AM", "AZ", "BY", "GE", "KZ", "KG", "MD", "RU", "TJ", "TM", "UA", "UZ"], + "CAZ": ["AU", "CA", "NZ"], + "CHA": ["CN", "HK", "MO", "TW"], + "IND": ["IN"], + "JPN": ["JP"], + "USA": ["US", "PM"], + "World": ["GLO", "RoW"] +} diff --git a/pathways/data/topologies/tiam-ucl-topology.json b/pathways/data/topologies/tiam-ucl-topology.json new file mode 100644 index 0000000..cdf7d8f --- /dev/null +++ b/pathways/data/topologies/tiam-ucl-topology.json @@ -0,0 +1,251 @@ +{ + "AFR": [ + "DZ", + "AO", + "BJ", + "BW", + "BF", + "BI", + "CM", + "CV", + "CF", + "TD", + "KM", + "CG", + "CI", + "CD", + "DJ", + "EG", + "GQ", + "ER", + "ET", + "GA", + "GM", + "GH", + "GN", + "GW", + "KE", + "LS", + "LR", + "LY", + "MG", + "MW", + "ML", + "MR", + "MA", + "MZ", + "NA", + "NE", + "NG", + "RW", + "ST", + "SN", + "SC", + "SL", + "SO", + "ZA", + "SS", + "SD", + "SZ", + "TG", + "TN", + "UG", + "TZ", + "ZM", + "ZW" + ], + + "AUS": [ + "AU", + "NZ" + ], + + "CAN": [ + "CA" + ], + + "CSA": [ + "AI", + "AG", + "AR", + "AW", + "BS", + "BB", + "BZ", + "BM", + "BO", + "BR", + "KY", + "CL", + "CO", + "CR", + "CU", + "DM", + "DO", + "EC", + "SV", + "FK", + "GD", + "GT", + "GY", + "HT", + "HN", + "JM", + "MQ", + "NI", + "PA", + "PY", + "PE", + "KN", + "LC", + "VC", + "SR", + "TT", + "UY", + "VE" + ], + + "CHI": [ + "CN", + "TW" + ], + + "EEU": [ + "BA", + "BG", + "HR", + "CZ", + "HU", + "ME", + "PL", + "RO", + "RS", + "SK", + "SI", + "MK" + ], + + "FSU": [ + "AM", + "AZ", + "BY", + "EE", + "GE", + "KZ", + "KG", + "LV", + "LT", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + + "IND": [ + "IN" + ], + + "JPN": [ + "JP" + ], + + "MEX": [ + "MX" + ], + + "MEA": [ + "BH", + "BN", + "CY", + "IR", + "IL", + "JO", + "KW", + "LB", + "PS", + "OM", + "QA", + "SA", + "SY", + "TR", + "AE", + "YE" + ], + + "ODA": [ + "AF", + "AS", + "BD", + "BT", + "KH", + "KP", + "FJ", + "PF", + "ID", + "KI", + "LA", + "MY", + "MV", + "MU", + "MN", + "MM", + "NP", + "NC", + "PK", + "PG", + "PH", + "WS", + "SG", + "SB", + "LK", + "TH", + "TL", + "TO", + "VU", + "VN" + ], + + "SKO": [ + "KR" + ], + + "UK": [ + "GB" + ], + + "USA": [ + "US" + ], + + "WEU": [ + "AL", + "AD", + "AT", + "BE", + "DK", + "FO", + "FI", + "FR", + "DE", + "GI", + "GR", + "GL", + "IS", + "IE", + "IT", + "LU", + "MT", + "MC", + "NL", + "NO", + "PT", + "SM", + "ES", + "SE", + "CH", + "VA" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data/topologies/witch-topology.json b/pathways/data/topologies/witch-topology.json new file mode 100644 index 0000000..885f073 --- /dev/null +++ b/pathways/data/topologies/witch-topology.json @@ -0,0 +1,294 @@ +{ + "usa": [ + "US", + "PM" + ], + "te": [ + "AM", + "AZ", + "BY", + "GE", + "KZ", + "KG", + "MD", + "RU", + "TJ", + "TM", + "UA", + "UZ" + ], + "ssa": [ + "AO", + "BI", + "BJ", + "BF", + "BW", + "CF", + "CI", + "CM", + "CD", + "CG", + "KM", + "CV", + "DJ", + "ER", + "ET", + "GA", + "GH", + "GN", + "GM", + "GW", + "GQ", + "KE", + "LR", + "LS", + "MG", + "ML", + "MZ", + "MR", + "MU", + "MW", + "NA", + "NE", + "NG", + "RE", + "RW", + "SN", + "SH", + "SL", + "SO", + "SS", + "ST", + "SZ", + "SC", + "TD", + "TG", + "TZ", + "UG", + "ZM", + "ZW" + ], + "southafrica": [ + "ZA" + ], + "seasia": [ + "AS", + "TF", + "BN", + "CK", + "FJ", + "FM", + "GU", + "IO", + "KH", + "KI", + "LA", + "MH", + "MM", + "MN", + "MP", + "MY", + "NC", + "NF", + "NU", + "NR", + "PN", + "PH", + "PW", + "PG", + "KP", + "PF", + "SG", + "SB", + "TH", + "TL", + "TO", + "TV", + "UM", + "VN", + "VU", + "WF", + "WS", + "MO" + ], + "oceania": [ + "AU", + "NZ" + ], + "mexico": [ + "MX" + ], + "mena": [ + "AE", + "BH", + "DZ", + "EG", + "EH", + "IR", + "IQ", + "IL", + "JO", + "KW", + "LB", + "LY", + "MA", + "OM", + "PS", + "QA", + "SA", + "SD", + "SY", + "TN", + "TR", + "YE" + ], + "laca": [ + "AW", + "AI", + "AR", + "AQ", + "AG", + "BQ", + "BS", + "BZ", + "BM", + "BO", + "BB", + "CL", + "CO", + "CR", + "CU", + "CW", + "KY", + "DM", + "DO", + "EC", + "FK", + "GP", + "GD", + "GT", + "GF", + "GY", + "HN", + "HT", + "JM", + "KN", + "LC", + "MF", + "MS", + "MQ", + "NI", + "PA", + "PE", + "PR", + "PY", + "GS", + "SV", + "SR", + "SX", + "TC", + "TT", + "UY", + "VC", + "VE", + "VG", + "VI", + "RLA" + ], + "korea": [ + "KR" + ], + "japan": [ + "JP" + ], + "indonesia": [ + "ID" + ], + "india": [ + "IN" + ], + "eu27": [ + "AT", + "BE", + "BG", + "HR", + "CY", + "CZ", + "DK", + "EE", + "FI", + "FR", + "DE", + "GR", + "HU", + "IE", + "IT", + "LV", + "LT", + "LU", + "MT", + "NL", + "PL", + "PT", + "RO", + "SK", + "SI", + "ES", + "SE" + ], + "othereurope": [ + "AX", + "FO", + "GB", + "GI", + "IM", + "IS", + "LI", + "MC", + "NO", + "SM", + "CH", + "AL", + "AD", + "BA", + "XK", + "ME", + "MK", + "RS", + "VA", + "SJ", + "GL" + ], + "china": [ + "CN", + "HK", + "TW" + ], + "canada": [ + "CA" + ], + "brazil": [ + "BR" + ], + "sasia": [ + "AF", + "BD", + "BT", + "NP", + "MV", + "PK", + "LK" + ], + "ccasia": [ + "TM", + "TJ", + "KG", + "KZ", + "UZ", + "MN", + "AM", + "AZ", + "GE" + ], + "World": ["GLO", "RoW"] +} \ No newline at end of file diff --git a/pathways/data_validation.py b/pathways/data_validation.py index 352d1dc..56902a0 100644 --- a/pathways/data_validation.py +++ b/pathways/data_validation.py @@ -128,7 +128,7 @@ def validate_mapping(resource: datapackage.Resource): mapping = yaml.safe_load(resource.raw_read()) # Check that the data has the required structure - required_keys = ["dataset", "scenario variable"] + required_keys = ["dataset", ] for k, v in mapping.items(): if not set(required_keys).issubset(set(v.keys())): raise ValueError(f"Invalid mapping: missing keys for {k}") diff --git a/pathways/edges_matrix.py b/pathways/edges_matrix.py new file mode 100644 index 0000000..90105c5 --- /dev/null +++ b/pathways/edges_matrix.py @@ -0,0 +1,203 @@ +""" +This module defines runs Edges, to produce a (regionalized) characterization matrix. +""" + +import bw2calc +import json +from typing import Optional, Dict +from edges import EdgeLCIA +from edges.matrix_builders import build_technosphere_edges_matrix +from edges import setup_package_logging +from bw_processing import Datapackage +from scipy.sparse import csr_matrix, vstack, issparse +import numpy as np +import sparse as spnd +import logging + +from .filesystem_constants import DATA_DIR + +setup_package_logging(level=logging.DEBUG) + + + +def fetch_topology(model: str) -> Optional[Dict]: + """ + Find the JSON file containing the topologies of the provided model. + """ + topology_path = DATA_DIR / "topologies" / f"{model.lower()}-topology.json" + if topology_path.exists(): + # load json + return json.loads(topology_path.read_text()) + + raise FileNotFoundError( + f"Geographical definition file for the model '{model.upper()}' not found." + ) + +def _edge_sets_for_lookup(lca: EdgeLCIA): + # Start empty; we'll OR them depending on which edge family you use + restrict_supplier_positions_bio: set[int] = set() + restrict_supplier_positions_tech: set[int] = set() + restrict_consumer_positions: set[int] = set() + + if getattr(lca, "biosphere_edges", None): + # biosphere_edges: (bio_row, tech_col) + bio_rows = {r for (r, _c) in lca.biosphere_edges} + tech_cols = {c for (_r, c) in lca.biosphere_edges} + restrict_supplier_positions_bio |= bio_rows + restrict_consumer_positions |= tech_cols + + if getattr(lca, "technosphere_edges", None): + # technosphere_edges: (tech_row_supplier, tech_col_consumer) + tech_rows = {r for (r, _c) in lca.technosphere_edges} + tech_cols = {c for (_r, c) in lca.technosphere_edges} + restrict_supplier_positions_tech |= tech_rows + restrict_consumer_positions |= tech_cols + + return (restrict_supplier_positions_bio, + restrict_supplier_positions_tech, + restrict_consumer_positions) + +def _build_position_to_technosphere_lookup(technosphere_index: dict[int, dict]) -> dict[int, dict]: + """ + technosphere_index: maps position -> activity metadata (name, location, classifications, etc.) + Return minimal fields Edges uses to enrich consumer/supplier info. + """ + out = {} + for pos, meta in technosphere_index.items(): + out[pos] = { + "location": meta.get("location"), + "classifications": meta.get("classifications"), + "name": meta.get("name"), + "reference product": meta.get("reference product"), + "unit": meta.get("unit"), + } + return out + +def _ensure_minimal_flows( + lca: EdgeLCIA, + biosphere_index: dict[int, dict], + technosphere_index: dict[int, dict], +): + if not getattr(lca, "biosphere_flows", None): + lca.biosphere_flows = [ + { + "name": f.get("name"), + "categories": list(f.get("categories")), + "unit": f.get("unit"), + "location": f.get("location"), # usually None for biosphere flows + "classifications": f.get("classifications"), # optional + "position": lca.lca.dicts.biosphere[pos], + } + for pos, f in biosphere_index.items() + if pos in lca.lca.dicts.biosphere + ] + + if not getattr(lca, "technosphere_flows", None): + lca.technosphere_flows = [ + { + "name": a.get("name"), + "reference product": a.get("reference product"), + "unit": a.get("unit"), + "location": a.get("location"), + "classifications": a.get("classifications"), + "position": lca.lca.dicts.activity.reversed[pos], + } + for pos, a in technosphere_index.items() + if pos in lca.lca.dicts.activity + ] + +def _as_row_csr(mat): + """Ensure the characterization is a 2D CSR row matrix.""" + if issparse(mat): + m = mat.tocsr() + # If it's a column/row vector, normalize to (1, n) + if m.ndim == 2 and m.shape[0] == 1: + return m + if m.ndim == 2 and m.shape[1] == 1: + return m.T.tocsr() + return m # already 2D + # Dense / 1D -> make (1, n) + arr = np.atleast_2d(np.asarray(mat)) + if arr.shape[0] != 1 and arr.shape[1] == 1: + arr = arr.T + return csr_matrix(arr) + +def create_edges_characterization_matrix( + model: str, + multilca_obj: bw2calc.MultiLCA, + methods: list, + indices: dict[str, dict[int, dict]], +): + """ + Run Edges for each method and return a 3D sparse tensor of shape + (n_methods, m, n), where each [i, :, :] is that method's characterization plane. + """ + topology = fetch_topology(model) + + planes = [] + + for method in methods: + # create fake sparse inventory matrix with same SHAPE & DTYPE + first_matrix = next(iter(multilca_obj.inventories.values())) + multilca_obj.inventory = csr_matrix(first_matrix.shape, + dtype=getattr(first_matrix, "dtype", float)) + + lca = EdgeLCIA( + demand={}, + method=method, + lca=multilca_obj, + additional_topologies=topology, + ) + + if all(cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data): + lca.technosphere_edges = { + (r, c) + for supply_array in multilca_obj.supply_arrays.values() + for r, c in zip( + *build_technosphere_edges_matrix( + multilca_obj.technosphere_matrix, supply_array + ).nonzero() + ) + } + else: + lca.biosphere_edges = { + (r, c) + for mat in multilca_obj.inventories.values() + for r, c in zip(*mat.nonzero()) + } + + _ensure_minimal_flows( + lca, + biosphere_index=indices["biosphere"], + technosphere_index=indices["technosphere"], + ) + lca.position_to_technosphere_flows_lookup = _build_position_to_technosphere_lookup( + indices["technosphere"] + ) + + rs_bio, rs_tech, rc_cons = _edge_sets_for_lookup(lca) + lca._preprocess_lookups( + restrict_supplier_positions_bio=rs_bio, + restrict_supplier_positions_tech=rs_tech, + restrict_consumer_positions=rc_cons, + ) + lca.apply_strategies() + lca.evaluate_cfs() + + + # Each method yields a 2D plane (m x n). Ensure SciPy CSR, then convert to pydata/sparse COO. + plane = lca.characterization_matrix + if not issparse(plane): + plane = csr_matrix(plane) + else: + plane = plane.tocsr() + + planes.append(spnd.COO.from_scipy_sparse(plane)) + + if not planes: + # Return an empty 3D tensor of shape (0, 0, 0) + return spnd.COO(np.zeros((0, 0, 0))).astype(float) + + # Stack along a NEW leading axis -> (n_methods, m, n) + characterization_tensor = spnd.stack(planes, axis=0) + return characterization_tensor diff --git a/pathways/lca.py b/pathways/lca.py index f6355d8..9dc0751 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -8,7 +8,7 @@ import pickle import uuid from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Any import bw2calc as bc import bw_processing as bwp @@ -18,6 +18,11 @@ from bw_processing import Datapackage from premise.geomap import Geomap from scipy import sparse +from scipy.sparse import issparse + +from scipy import sparse as sps +import numpy as np +import sparse as spnd from .filesystem_constants import DIR_CACHED_DB from .lcia import fill_characterization_factors_matrices @@ -26,6 +31,7 @@ find_technology_indices, get_subshares_matrix, ) +from .edges_matrix import create_edges_characterization_matrix from .utils import ( CustomFilter, _group_technosphere_indices, @@ -215,7 +221,7 @@ def create_functional_units( variables, vars_idx, units_map, -) -> [dict, dict]: +) -> tuple[dict[Any, Any], dict[Any, Any]]: """ Create functional units for the given region, model, scenario, and year. The functional units are created based on the demand for each variable in the scenarios dataset. @@ -320,6 +326,42 @@ def create_functional_units( }, variables_demand +def _build_sparse_inventory_results_3d( + lca, + characterization_matrix, + edges_methods: bool, +): + """ + Return a 3D sparse tensor (pydata/sparse COO): + + - edges_methods=True: + slices[i] = characterization_matrix.multiply(v_i) # (n_bio, n_cols) + -> stacked to (n_inv, n_bio, n_cols) + + - edges_methods=False: + slices[i] = (characterization_matrix @ v_i) # (n_methods, n_cols) + -> stacked to (n_inv, n_methods, n_cols) + + In both cases, each slice is built as a SciPy sparse matrix, + then converted to pydata/sparse COO and stacked along axis=0. + """ + if edges_methods: + # elementwise multiply, requires identical shapes + slices = [] + for v in lca.inventories.values(): + assert issparse(v), "inventory matrices must be SciPy sparse." + s = characterization_matrix.multiply(v) # SciPy sparse (n_bio, n_cols) + slices.append(spnd.COO.from_scipy_sparse(s)) + return spnd.stack(slices, axis=0) + else: + # proper matrix multiply + slices = [] + for v in lca.inventories.values(): + assert issparse(v), "inventory matrices must be SciPy sparse." + s = (characterization_matrix @ v) # SciPy sparse (n_methods, n_cols) + slices.append(spnd.COO.from_scipy_sparse(s)) + return spnd.stack(slices, axis=0) + def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: """ Process the region data. @@ -338,8 +380,9 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: units_map, demand_cutoff, lca, - characterization_matrix, + characterization_matrix, # edges: COO (n_methods, n_bio, n_cols); regular: CSR (n_methods, n_bio) methods, + edges_methods, debug, use_distributions, uncertain_parameters, @@ -347,152 +390,159 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: id_uncertainty_indices_filepath = None id_technosphere_indices_filepath = None - iter_results_files = [] + iter_results_files: List[Path] = [] iter_param_vals_filepath = None - dict_loc_cat = {} - + # Build category × location mapping once + dict_loc_cat: Dict[tuple, np.ndarray] = {} cat_counter = 0 - for cat, act_cat_idx in lca.acts_category_idx_dict.items(): + for _, act_cat_idx in lca.acts_category_idx_dict.items(): loc_counter = 0 - for loc, act_loc_idx in lca.acts_location_idx_dict.items(): - # Find the intersection of indices + for _, act_loc_idx in lca.acts_location_idx_dict.items(): idx = np.intersect1d(act_cat_idx, act_loc_idx) - # Filter out any -1 indices filtered_idx = idx[idx != -1] - if filtered_idx.size > 0: - # Assign the filtered index array - # to the dict_loc_cat with (cat, loc) as key dict_loc_cat[(cat_counter, loc_counter)] = filtered_idx - loc_counter += 1 cat_counter += 1 + # Helper to build sparse 3D tensor: (n_inv, second_dim, n_cols) + # - edges: second_dim = n_methods, using characterization (n_methods, n_bio, n_cols) COO + # inventory v (n_bio, n_cols) -> broadcast multiply, then sum over biosphere (axis=1). + # - regular: second_dim = n_methods, using C (n_methods, n_bio) CSR @ v (n_bio, n_cols) + def _inventory_results_3d_edges(lca, char_coo: spnd.COO): + invs = [mat.tocsr() for mat in lca.inventories.values()] + rows = [] + for v in invs: + v_coo = spnd.COO.from_scipy_sparse(v) # (n_bio, n_cols) + H = char_coo * v_coo # (n_methods, n_bio, n_cols) + S = H.sum(axis=1) # sum over biosphere -> (n_methods, n_cols) + rows.append(S) + return spnd.stack(rows, axis=0) # (n_inv, n_methods, n_cols) + + def _inventory_results_3d_regular(lca, C: sps.csr_matrix): + invs = [mat.tocsr() for mat in lca.inventories.values()] + slices = [] + for v in invs: + M = (C @ v) # (n_methods, n_cols), SciPy sparse + slices.append(spnd.COO.from_scipy_sparse(M)) + return spnd.stack(slices, axis=0) # (n_inv, n_methods, n_cols) + if use_distributions == 0: # Regular LCA calculations with CustomFilter("(almost) singular matrix"): lca.lci() if debug: - logging.info(f"Iterations no.: {use_distributions}.") - - # Create a numpy array with the results - inventory_results = np.array( - [ - (characterization_matrix @ value).toarray() - for value in lca.inventories.values() - ] - ) - - if debug: - logging.info(f"Shape of inventory_results: {inventory_results.shape}") + logging.info(f"Edges methods: {edges_methods}. Monte Carlo iters: {use_distributions}.") + + # --- Build sparse 3D inventory_results: (n_inv, n_methods, n_cols) + if edges_methods: + # characterization_matrix must be a 3D pydata.sparse COO: (n_methods, n_bio, n_cols) + if not isinstance(characterization_matrix, spnd.COO) or characterization_matrix.ndim != 3: + raise ValueError("Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols).") + inventory_results = _inventory_results_3d_edges(lca, characterization_matrix) + else: + # characterization_matrix must be a 2D SciPy sparse (n_methods, n_bio) + if not issparse(characterization_matrix) or characterization_matrix.ndim != 2: + raise ValueError("Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio).") + inventory_results = _inventory_results_3d_regular(lca, characterization_matrix.tocsr()) if debug: - for fu, inventory in lca.inventories.items(): - logging.info( - f"Functional unit: {fu}. Impact: {(characterization_matrix @ inventory).sum()}" - ) - - iter_results = np.zeros( - ( - inventory_results.shape[0], - inventory_results.shape[1], - len(lca.acts_category_idx_dict), - len(lca.acts_location_idx_dict), - ) - ) - - if debug: - logging.info(f"Shape of iter_results: {iter_results.shape}") + logging.info(f"inventory_results shape: {inventory_results.shape}") + + # Unify downstream aggregation + # inventory_results: (n_inv, second_dim=n_methods, n_cols) + n_inv, second_dim, _ = inventory_results.shape + n_cat = len(lca.acts_category_idx_dict) + n_loc = len(lca.acts_location_idx_dict) + + zeros_block = spnd.zeros((n_inv, second_dim), dtype=inventory_results.dtype) + + cat_stacks = [] + for cat in range(n_cat): + loc_blocks = [] + for loc in range(n_loc): + idx = dict_loc_cat.get((cat, loc)) + if idx is None or idx.size == 0: + block = zeros_block # (n_inv, n_methods) + else: + block = inventory_results[:, :, idx].sum(axis=2) # (n_inv, n_methods) + loc_blocks.append(block) + # Stack blocks across the location axis -> (n_inv, n_methods, n_loc) + cat_stacks.append(spnd.stack(loc_blocks, axis=2)) - for (cat, loc), idx in dict_loc_cat.items(): - iter_results[:, :, cat, loc] = inventory_results[:, :, idx].sum(axis=2) + # Stack across categories -> (n_inv, n_methods, n_cat, n_loc) + iter_results = spnd.stack(cat_stacks, axis=2) if debug: - for f, fu in enumerate(lca.inventories.keys()): - logging.info(f"Functional unit: {fu}. Impact: {iter_results[f].sum()}") + logging.info(f"iter_results shape: {iter_results.shape}") - # Save iteration results to disk + # Save without densifying iter_results_filepath = DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" - sp.save_npz( - filename=iter_results_filepath, - matrix=sp.COO(iter_results), - compressed=True, - ) + spnd.save_npz(filename=iter_results_filepath, matrix=iter_results, compressed=True) iter_results_files.append(iter_results_filepath) else: - # Use distributions for LCA calculations + # Monte Carlo: same sparse flow per iteration iter_param_vals = [] with CustomFilter("(almost) singular matrix"): - for iteration in range(use_distributions): + for _ in range(use_distributions): next(lca) lca.lci() - # Create a numpy array with the results - inventory_results = np.array( - [ - (characterization_matrix @ value).toarray() - for value in lca.inventories.values() - ] - ) - iter_param_vals.append( - [ - -lca.technosphere_matrix[index] - for index in lca.uncertain_parameters - ] - ) - - iter_results = np.zeros( - ( - inventory_results.shape[0], - inventory_results.shape[1], - len(lca.acts_category_idx_dict), - len(lca.acts_location_idx_dict), - ) - ) - - for (cat, loc), idx in dict_loc_cat.items(): - iter_results[:, :, cat, loc] = inventory_results[:, :, idx].sum( - axis=2 - ) + if edges_methods: + if not isinstance(characterization_matrix, spnd.COO) or characterization_matrix.ndim != 3: + raise ValueError("Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols).") + inventory_results = _inventory_results_3d_edges(lca, characterization_matrix) + else: + if not issparse(characterization_matrix) or characterization_matrix.ndim != 2: + raise ValueError("Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio).") + inventory_results = _inventory_results_3d_regular(lca, characterization_matrix.tocsr()) + + # Aggregate to (n_inv, n_methods, n_cat, n_loc) + n_inv, second_dim, _ = inventory_results.shape + n_cat = len(lca.acts_category_idx_dict) + n_loc = len(lca.acts_location_idx_dict) + + zeros_block = spnd.zeros((n_inv, second_dim), dtype=inventory_results.dtype) + + cat_stacks = [] + for cat in range(n_cat): + loc_blocks = [] + for loc in range(n_loc): + idx = dict_loc_cat.get((cat, loc)) + if idx is None or idx.size == 0: + block = zeros_block + else: + block = inventory_results[:, :, idx].sum(axis=2) + loc_blocks.append(block) + cat_stacks.append(spnd.stack(loc_blocks, axis=2)) + + iter_results = spnd.stack(cat_stacks, axis=2) + + # Save per-iteration sparse tensor + iter_results_filepath = DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" + spnd.save_npz(filename=iter_results_filepath, matrix=iter_results, compressed=True) + iter_results_files.append(iter_results_filepath) - # Save iteration results to disk - iter_results_filepath = ( - DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" - ) - sp.save_npz( - filename=iter_results_filepath, - matrix=sp.COO(iter_results), - compressed=True, + # Keep your MC bookkeeping + iter_param_vals.append( + [-lca.technosphere_matrix[index] for index in lca.uncertain_parameters] ) - iter_results_files.append(iter_results_filepath) - # Save iteration parameter values to disk + # Save MC parameter draws iter_param_vals_filepath = DIR_CACHED_DB / f"iter_param_vals_{uuid.uuid4()}.npy" np.save(file=iter_param_vals_filepath, arr=np.stack(iter_param_vals, axis=-1)) - # Save the uncertainty indices to disk - id_uncertainty_indices_filepath = ( - DIR_CACHED_DB / f"mc_indices_{uuid.uuid4()}.npy" - ) - np.save( - file=id_uncertainty_indices_filepath, - arr=lca.uncertain_parameters, - ) + # Save indices + id_uncertainty_indices_filepath = DIR_CACHED_DB / f"mc_indices_{uuid.uuid4()}.npy" + np.save(file=id_uncertainty_indices_filepath, arr=lca.uncertain_parameters) - # Save the technosphere indices to disk - id_technosphere_indices_filepath = ( - DIR_CACHED_DB / f"tech_indices_{uuid.uuid4()}.pkl" - ) - pickle.dump( - lca.technosphere_indices, - open(id_technosphere_indices_filepath, "wb"), - ) + id_technosphere_indices_filepath = DIR_CACHED_DB / f"tech_indices_{uuid.uuid4()}.pkl" + pickle.dump(lca.technosphere_indices, open(id_technosphere_indices_filepath, "wb")) - # Returning a dictionary containing the id_array and the variables - # to be able to fetch them back later + # Return file paths + FU variables d = { "iterations_results": iter_results_files, "variables": {k: v["demand"] for k, v in fus_details.items()}, @@ -503,19 +553,14 @@ def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: logging.info(f"FUs: {list(lca.inventories.keys())}") if use_distributions > 0: - d["uncertainty_params"] = [ - str(id_uncertainty_indices_filepath), - ] - d["technosphere_indices"] = [ - str(id_technosphere_indices_filepath), - ] - d["iterations_param_vals"] = [ - str(iter_param_vals_filepath), - ] + d["uncertainty_params"] = [str(id_uncertainty_indices_filepath)] + d["technosphere_indices"] = [str(id_technosphere_indices_filepath)] + d["iterations_param_vals"] = [str(iter_param_vals_filepath)] return d + def _calculate_year(args: tuple): """ Prepares the data for the calculation of LCA results for a given year @@ -534,6 +579,7 @@ def _calculate_year(args: tuple): regions, variables, methods, + edges_methods, demand_cutoff, filepaths, mapping, @@ -707,12 +753,41 @@ def _calculate_year(args: tuple): if v in {value for tup in lca.uncertain_parameters for value in tup} } - characterization_matrix = fill_characterization_factors_matrices( - methods=methods, - biosphere_matrix_dict=lca.dicts.biosphere, - biosphere_dict=biosphere_indices, - debug=debug, - ) + if methods: + # regular LCIA methods + characterization_matrix = fill_characterization_factors_matrices( + methods=methods, + biosphere_matrix_dict=lca.dicts.biosphere, + biosphere_dict=biosphere_indices, + debug=debug, + ) + else: + print("Using EDGES' LCIA methods...") + + # EDGES' LCIA methods + formatted_biosphere_index = { + v: { + "name": k[0], + "categories": k[1:] + } for k, v in biosphere_indices.items() + } + formatted_technosphere_index = { + v: { + "name": k[0], + "reference product": k[1], + "unit": k[2], + "location": k[3] + } for k, v in technosphere_indices.items() + } + characterization_matrix = create_edges_characterization_matrix( + model=model, + multilca_obj=lca, + methods=edges_methods, + indices={ + "biosphere": formatted_biosphere_index, + "technosphere": formatted_technosphere_index, + } + ) if debug: logging.info( @@ -736,6 +811,7 @@ def _calculate_year(args: tuple): lca, characterization_matrix, methods, + edges_methods, debug, use_distributions, uncertain_parameters, diff --git a/pathways/pathways.py b/pathways/pathways.py index 6398730..8f9c4dd 100644 --- a/pathways/pathways.py +++ b/pathways/pathways.py @@ -19,6 +19,8 @@ import xarray as xr import yaml +from edges import get_available_methods + from .data_validation import validate_datapackage from .filesystem_constants import DATA_DIR, USER_LOGS_DIR from .lca import _calculate_year, get_lca_matrices @@ -272,6 +274,7 @@ def _get_scenarios(self, scenario_data: pd.DataFrame) -> xr.DataArray: def calculate( self, methods: Optional[List[str]] = None, + edges_methods: Optional[List[str]] = None, models: Optional[List[str]] = None, scenarios: Optional[List[str]] = None, regions: Optional[List[str]] = None, @@ -322,8 +325,26 @@ def calculate( self.scenarios = harmonize_units(self.scenarios, variables) + if methods: + available_methods = get_lcia_method_names() + for m in methods: + if m not in available_methods: + raise ValueError(f"LCIA method {m} not found in available methods.") + + if edges_methods: + available_methods = get_available_methods() + for m in edges_methods: + if m not in available_methods: + raise ValueError(f"Edge LCIA method {m} not found in available `edges` methods.") + + if methods and edges_methods: + raise ValueError("Please provide either `methods` or `edges_methods`, not both.") + + if methods is None and edges_methods is None: + raise ValueError("Please provide at least one of `methods` or `edges_methods`.") + # if no methods are provided, use all those available - methods = methods or get_lcia_method_names() + if self.debug: logging.info(f"Using the following LCIA methods: {methods}") @@ -384,7 +405,7 @@ def calculate( self.geography_mapping = {loc: loc for loc in locations} self.lca_results = create_lca_results_array( - methods=methods, + methods=methods or [str(m) for m in edges_methods], years=years, regions=regions, locations=locations, @@ -418,6 +439,7 @@ def calculate( regions, variables, methods, + edges_methods, demand_cutoff, self.filepaths, self.mapping, From e8a893764601c950456c773556142b03677b3f54 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sun, 2 Nov 2025 17:19:19 +0000 Subject: [PATCH 2/8] Black reformating --- pathways/data_validation.py | 4 +- pathways/edges_matrix.py | 42 ++++++++----- pathways/lca.py | 122 +++++++++++++++++++++++++----------- pathways/pathways.py | 12 +++- 4 files changed, 123 insertions(+), 57 deletions(-) diff --git a/pathways/data_validation.py b/pathways/data_validation.py index 56902a0..ec9d040 100644 --- a/pathways/data_validation.py +++ b/pathways/data_validation.py @@ -128,7 +128,9 @@ def validate_mapping(resource: datapackage.Resource): mapping = yaml.safe_load(resource.raw_read()) # Check that the data has the required structure - required_keys = ["dataset", ] + required_keys = [ + "dataset", + ] for k, v in mapping.items(): if not set(required_keys).issubset(set(v.keys())): raise ValueError(f"Invalid mapping: missing keys for {k}") diff --git a/pathways/edges_matrix.py b/pathways/edges_matrix.py index 90105c5..1339539 100644 --- a/pathways/edges_matrix.py +++ b/pathways/edges_matrix.py @@ -19,7 +19,6 @@ setup_package_logging(level=logging.DEBUG) - def fetch_topology(model: str) -> Optional[Dict]: """ Find the JSON file containing the topologies of the provided model. @@ -33,31 +32,37 @@ def fetch_topology(model: str) -> Optional[Dict]: f"Geographical definition file for the model '{model.upper()}' not found." ) + def _edge_sets_for_lookup(lca: EdgeLCIA): # Start empty; we'll OR them depending on which edge family you use - restrict_supplier_positions_bio: set[int] = set() + restrict_supplier_positions_bio: set[int] = set() restrict_supplier_positions_tech: set[int] = set() - restrict_consumer_positions: set[int] = set() + restrict_consumer_positions: set[int] = set() if getattr(lca, "biosphere_edges", None): # biosphere_edges: (bio_row, tech_col) bio_rows = {r for (r, _c) in lca.biosphere_edges} tech_cols = {c for (_r, c) in lca.biosphere_edges} restrict_supplier_positions_bio |= bio_rows - restrict_consumer_positions |= tech_cols + restrict_consumer_positions |= tech_cols if getattr(lca, "technosphere_edges", None): # technosphere_edges: (tech_row_supplier, tech_col_consumer) tech_rows = {r for (r, _c) in lca.technosphere_edges} tech_cols = {c for (_r, c) in lca.technosphere_edges} restrict_supplier_positions_tech |= tech_rows - restrict_consumer_positions |= tech_cols + restrict_consumer_positions |= tech_cols - return (restrict_supplier_positions_bio, - restrict_supplier_positions_tech, - restrict_consumer_positions) + return ( + restrict_supplier_positions_bio, + restrict_supplier_positions_tech, + restrict_consumer_positions, + ) -def _build_position_to_technosphere_lookup(technosphere_index: dict[int, dict]) -> dict[int, dict]: + +def _build_position_to_technosphere_lookup( + technosphere_index: dict[int, dict], +) -> dict[int, dict]: """ technosphere_index: maps position -> activity metadata (name, location, classifications, etc.) Return minimal fields Edges uses to enrich consumer/supplier info. @@ -73,6 +78,7 @@ def _build_position_to_technosphere_lookup(technosphere_index: dict[int, dict]) } return out + def _ensure_minimal_flows( lca: EdgeLCIA, biosphere_index: dict[int, dict], @@ -84,7 +90,7 @@ def _ensure_minimal_flows( "name": f.get("name"), "categories": list(f.get("categories")), "unit": f.get("unit"), - "location": f.get("location"), # usually None for biosphere flows + "location": f.get("location"), # usually None for biosphere flows "classifications": f.get("classifications"), # optional "position": lca.lca.dicts.biosphere[pos], } @@ -106,6 +112,7 @@ def _ensure_minimal_flows( if pos in lca.lca.dicts.activity ] + def _as_row_csr(mat): """Ensure the characterization is a 2D CSR row matrix.""" if issparse(mat): @@ -122,6 +129,7 @@ def _as_row_csr(mat): arr = arr.T return csr_matrix(arr) + def create_edges_characterization_matrix( model: str, multilca_obj: bw2calc.MultiLCA, @@ -139,8 +147,9 @@ def create_edges_characterization_matrix( for method in methods: # create fake sparse inventory matrix with same SHAPE & DTYPE first_matrix = next(iter(multilca_obj.inventories.values())) - multilca_obj.inventory = csr_matrix(first_matrix.shape, - dtype=getattr(first_matrix, "dtype", float)) + multilca_obj.inventory = csr_matrix( + first_matrix.shape, dtype=getattr(first_matrix, "dtype", float) + ) lca = EdgeLCIA( demand={}, @@ -149,7 +158,9 @@ def create_edges_characterization_matrix( additional_topologies=topology, ) - if all(cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data): + if all( + cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data + ): lca.technosphere_edges = { (r, c) for supply_array in multilca_obj.supply_arrays.values() @@ -171,8 +182,8 @@ def create_edges_characterization_matrix( biosphere_index=indices["biosphere"], technosphere_index=indices["technosphere"], ) - lca.position_to_technosphere_flows_lookup = _build_position_to_technosphere_lookup( - indices["technosphere"] + lca.position_to_technosphere_flows_lookup = ( + _build_position_to_technosphere_lookup(indices["technosphere"]) ) rs_bio, rs_tech, rc_cons = _edge_sets_for_lookup(lca) @@ -184,7 +195,6 @@ def create_edges_characterization_matrix( lca.apply_strategies() lca.evaluate_cfs() - # Each method yields a 2D plane (m x n). Ensure SciPy CSR, then convert to pydata/sparse COO. plane = lca.characterization_matrix if not issparse(plane): diff --git a/pathways/lca.py b/pathways/lca.py index 9dc0751..232b37d 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -358,10 +358,11 @@ def _build_sparse_inventory_results_3d( slices = [] for v in lca.inventories.values(): assert issparse(v), "inventory matrices must be SciPy sparse." - s = (characterization_matrix @ v) # SciPy sparse (n_methods, n_cols) + s = characterization_matrix @ v # SciPy sparse (n_methods, n_cols) slices.append(spnd.COO.from_scipy_sparse(s)) return spnd.stack(slices, axis=0) + def process_region(data: Tuple) -> Dict[str, str | List[str] | List[int]]: """ Process the region data. @@ -414,19 +415,19 @@ def _inventory_results_3d_edges(lca, char_coo: spnd.COO): invs = [mat.tocsr() for mat in lca.inventories.values()] rows = [] for v in invs: - v_coo = spnd.COO.from_scipy_sparse(v) # (n_bio, n_cols) - H = char_coo * v_coo # (n_methods, n_bio, n_cols) - S = H.sum(axis=1) # sum over biosphere -> (n_methods, n_cols) + v_coo = spnd.COO.from_scipy_sparse(v) # (n_bio, n_cols) + H = char_coo * v_coo # (n_methods, n_bio, n_cols) + S = H.sum(axis=1) # sum over biosphere -> (n_methods, n_cols) rows.append(S) - return spnd.stack(rows, axis=0) # (n_inv, n_methods, n_cols) + return spnd.stack(rows, axis=0) # (n_inv, n_methods, n_cols) def _inventory_results_3d_regular(lca, C: sps.csr_matrix): invs = [mat.tocsr() for mat in lca.inventories.values()] slices = [] for v in invs: - M = (C @ v) # (n_methods, n_cols), SciPy sparse + M = C @ v # (n_methods, n_cols), SciPy sparse slices.append(spnd.COO.from_scipy_sparse(M)) - return spnd.stack(slices, axis=0) # (n_inv, n_methods, n_cols) + return spnd.stack(slices, axis=0) # (n_inv, n_methods, n_cols) if use_distributions == 0: # Regular LCA calculations @@ -434,19 +435,35 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): lca.lci() if debug: - logging.info(f"Edges methods: {edges_methods}. Monte Carlo iters: {use_distributions}.") + logging.info( + f"Edges methods: {edges_methods}. Monte Carlo iters: {use_distributions}." + ) # --- Build sparse 3D inventory_results: (n_inv, n_methods, n_cols) if edges_methods: # characterization_matrix must be a 3D pydata.sparse COO: (n_methods, n_bio, n_cols) - if not isinstance(characterization_matrix, spnd.COO) or characterization_matrix.ndim != 3: - raise ValueError("Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols).") - inventory_results = _inventory_results_3d_edges(lca, characterization_matrix) + if ( + not isinstance(characterization_matrix, spnd.COO) + or characterization_matrix.ndim != 3 + ): + raise ValueError( + "Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols)." + ) + inventory_results = _inventory_results_3d_edges( + lca, characterization_matrix + ) else: # characterization_matrix must be a 2D SciPy sparse (n_methods, n_bio) - if not issparse(characterization_matrix) or characterization_matrix.ndim != 2: - raise ValueError("Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio).") - inventory_results = _inventory_results_3d_regular(lca, characterization_matrix.tocsr()) + if ( + not issparse(characterization_matrix) + or characterization_matrix.ndim != 2 + ): + raise ValueError( + "Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio)." + ) + inventory_results = _inventory_results_3d_regular( + lca, characterization_matrix.tocsr() + ) if debug: logging.info(f"inventory_results shape: {inventory_results.shape}") @@ -467,7 +484,9 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): if idx is None or idx.size == 0: block = zeros_block # (n_inv, n_methods) else: - block = inventory_results[:, :, idx].sum(axis=2) # (n_inv, n_methods) + block = inventory_results[:, :, idx].sum( + axis=2 + ) # (n_inv, n_methods) loc_blocks.append(block) # Stack blocks across the location axis -> (n_inv, n_methods, n_loc) cat_stacks.append(spnd.stack(loc_blocks, axis=2)) @@ -480,7 +499,9 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): # Save without densifying iter_results_filepath = DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" - spnd.save_npz(filename=iter_results_filepath, matrix=iter_results, compressed=True) + spnd.save_npz( + filename=iter_results_filepath, matrix=iter_results, compressed=True + ) iter_results_files.append(iter_results_filepath) else: @@ -492,20 +513,36 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): lca.lci() if edges_methods: - if not isinstance(characterization_matrix, spnd.COO) or characterization_matrix.ndim != 3: - raise ValueError("Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols).") - inventory_results = _inventory_results_3d_edges(lca, characterization_matrix) + if ( + not isinstance(characterization_matrix, spnd.COO) + or characterization_matrix.ndim != 3 + ): + raise ValueError( + "Edges methods require a 3D pydata.sparse COO characterization tensor (n_methods, n_bio, n_cols)." + ) + inventory_results = _inventory_results_3d_edges( + lca, characterization_matrix + ) else: - if not issparse(characterization_matrix) or characterization_matrix.ndim != 2: - raise ValueError("Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio).") - inventory_results = _inventory_results_3d_regular(lca, characterization_matrix.tocsr()) + if ( + not issparse(characterization_matrix) + or characterization_matrix.ndim != 2 + ): + raise ValueError( + "Regular methods require a 2D SciPy sparse characterization matrix (n_methods, n_bio)." + ) + inventory_results = _inventory_results_3d_regular( + lca, characterization_matrix.tocsr() + ) # Aggregate to (n_inv, n_methods, n_cat, n_loc) n_inv, second_dim, _ = inventory_results.shape n_cat = len(lca.acts_category_idx_dict) n_loc = len(lca.acts_location_idx_dict) - zeros_block = spnd.zeros((n_inv, second_dim), dtype=inventory_results.dtype) + zeros_block = spnd.zeros( + (n_inv, second_dim), dtype=inventory_results.dtype + ) cat_stacks = [] for cat in range(n_cat): @@ -522,13 +559,20 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): iter_results = spnd.stack(cat_stacks, axis=2) # Save per-iteration sparse tensor - iter_results_filepath = DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" - spnd.save_npz(filename=iter_results_filepath, matrix=iter_results, compressed=True) + iter_results_filepath = ( + DIR_CACHED_DB / f"iter_results_{uuid.uuid4()}.npz" + ) + spnd.save_npz( + filename=iter_results_filepath, matrix=iter_results, compressed=True + ) iter_results_files.append(iter_results_filepath) # Keep your MC bookkeeping iter_param_vals.append( - [-lca.technosphere_matrix[index] for index in lca.uncertain_parameters] + [ + -lca.technosphere_matrix[index] + for index in lca.uncertain_parameters + ] ) # Save MC parameter draws @@ -536,11 +580,17 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): np.save(file=iter_param_vals_filepath, arr=np.stack(iter_param_vals, axis=-1)) # Save indices - id_uncertainty_indices_filepath = DIR_CACHED_DB / f"mc_indices_{uuid.uuid4()}.npy" + id_uncertainty_indices_filepath = ( + DIR_CACHED_DB / f"mc_indices_{uuid.uuid4()}.npy" + ) np.save(file=id_uncertainty_indices_filepath, arr=lca.uncertain_parameters) - id_technosphere_indices_filepath = DIR_CACHED_DB / f"tech_indices_{uuid.uuid4()}.pkl" - pickle.dump(lca.technosphere_indices, open(id_technosphere_indices_filepath, "wb")) + id_technosphere_indices_filepath = ( + DIR_CACHED_DB / f"tech_indices_{uuid.uuid4()}.pkl" + ) + pickle.dump( + lca.technosphere_indices, open(id_technosphere_indices_filepath, "wb") + ) # Return file paths + FU variables d = { @@ -560,7 +610,6 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): return d - def _calculate_year(args: tuple): """ Prepares the data for the calculation of LCA results for a given year @@ -766,18 +815,17 @@ def _calculate_year(args: tuple): # EDGES' LCIA methods formatted_biosphere_index = { - v: { - "name": k[0], - "categories": k[1:] - } for k, v in biosphere_indices.items() + v: {"name": k[0], "categories": k[1:]} + for k, v in biosphere_indices.items() } formatted_technosphere_index = { v: { "name": k[0], "reference product": k[1], "unit": k[2], - "location": k[3] - } for k, v in technosphere_indices.items() + "location": k[3], + } + for k, v in technosphere_indices.items() } characterization_matrix = create_edges_characterization_matrix( model=model, @@ -786,7 +834,7 @@ def _calculate_year(args: tuple): indices={ "biosphere": formatted_biosphere_index, "technosphere": formatted_technosphere_index, - } + }, ) if debug: diff --git a/pathways/pathways.py b/pathways/pathways.py index 8f9c4dd..e222e9a 100644 --- a/pathways/pathways.py +++ b/pathways/pathways.py @@ -335,13 +335,19 @@ def calculate( available_methods = get_available_methods() for m in edges_methods: if m not in available_methods: - raise ValueError(f"Edge LCIA method {m} not found in available `edges` methods.") + raise ValueError( + f"Edge LCIA method {m} not found in available `edges` methods." + ) if methods and edges_methods: - raise ValueError("Please provide either `methods` or `edges_methods`, not both.") + raise ValueError( + "Please provide either `methods` or `edges_methods`, not both." + ) if methods is None and edges_methods is None: - raise ValueError("Please provide at least one of `methods` or `edges_methods`.") + raise ValueError( + "Please provide at least one of `methods` or `edges_methods`." + ) # if no methods are provided, use all those available From c276c1d2a5c4dad39f8ed43bc32ec7eb7826de95 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sun, 2 Nov 2025 20:40:20 +0100 Subject: [PATCH 3/8] Edges implementation --- pathways/edges_matrix.py | 22 +++++++++++++++------- pathways/lca.py | 6 +++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pathways/edges_matrix.py b/pathways/edges_matrix.py index 90105c5..a8bd7a1 100644 --- a/pathways/edges_matrix.py +++ b/pathways/edges_matrix.py @@ -150,15 +150,23 @@ def create_edges_characterization_matrix( ) if all(cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data): + + multilca_obj.inventories = { + k: build_technosphere_edges_matrix( + multilca_obj.technosphere_matrix, + multilca_obj.supply_arrays[k] + ) + for k in multilca_obj.supply_arrays + } + + + # Collect nonzero positions across all inventories lca.technosphere_edges = { (r, c) - for supply_array in multilca_obj.supply_arrays.values() - for r, c in zip( - *build_technosphere_edges_matrix( - multilca_obj.technosphere_matrix, supply_array - ).nonzero() - ) + for mat in multilca_obj.inventories.values() + for r, c in zip(*mat.nonzero()) } + else: lca.biosphere_edges = { (r, c) @@ -200,4 +208,4 @@ def create_edges_characterization_matrix( # Stack along a NEW leading axis -> (n_methods, m, n) characterization_tensor = spnd.stack(planes, axis=0) - return characterization_tensor + return characterization_tensor, multilca_obj diff --git a/pathways/lca.py b/pathways/lca.py index 9dc0751..4df5f5d 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -430,8 +430,8 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): if use_distributions == 0: # Regular LCA calculations - with CustomFilter("(almost) singular matrix"): - lca.lci() + #with CustomFilter("(almost) singular matrix"): + # lca.lci() if debug: logging.info(f"Edges methods: {edges_methods}. Monte Carlo iters: {use_distributions}.") @@ -779,7 +779,7 @@ def _calculate_year(args: tuple): "location": k[3] } for k, v in technosphere_indices.items() } - characterization_matrix = create_edges_characterization_matrix( + characterization_matrix, lca = create_edges_characterization_matrix( model=model, multilca_obj=lca, methods=edges_methods, From 09e4e515053e0c0b33f659c44f0f2da56606dd6c Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sun, 2 Nov 2025 19:41:11 +0000 Subject: [PATCH 4/8] Black reformating --- pathways/edges_matrix.py | 41 +++++++++++++++++++++++++--------------- pathways/lca.py | 2 +- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/pathways/edges_matrix.py b/pathways/edges_matrix.py index 0bcb1b9..1339539 100644 --- a/pathways/edges_matrix.py +++ b/pathways/edges_matrix.py @@ -32,31 +32,37 @@ def fetch_topology(model: str) -> Optional[Dict]: f"Geographical definition file for the model '{model.upper()}' not found." ) + def _edge_sets_for_lookup(lca: EdgeLCIA): # Start empty; we'll OR them depending on which edge family you use - restrict_supplier_positions_bio: set[int] = set() + restrict_supplier_positions_bio: set[int] = set() restrict_supplier_positions_tech: set[int] = set() - restrict_consumer_positions: set[int] = set() + restrict_consumer_positions: set[int] = set() if getattr(lca, "biosphere_edges", None): # biosphere_edges: (bio_row, tech_col) bio_rows = {r for (r, _c) in lca.biosphere_edges} tech_cols = {c for (_r, c) in lca.biosphere_edges} restrict_supplier_positions_bio |= bio_rows - restrict_consumer_positions |= tech_cols + restrict_consumer_positions |= tech_cols if getattr(lca, "technosphere_edges", None): # technosphere_edges: (tech_row_supplier, tech_col_consumer) tech_rows = {r for (r, _c) in lca.technosphere_edges} tech_cols = {c for (_r, c) in lca.technosphere_edges} restrict_supplier_positions_tech |= tech_rows - restrict_consumer_positions |= tech_cols + restrict_consumer_positions |= tech_cols + + return ( + restrict_supplier_positions_bio, + restrict_supplier_positions_tech, + restrict_consumer_positions, + ) - return (restrict_supplier_positions_bio, - restrict_supplier_positions_tech, - restrict_consumer_positions) -def _build_position_to_technosphere_lookup(technosphere_index: dict[int, dict]) -> dict[int, dict]: +def _build_position_to_technosphere_lookup( + technosphere_index: dict[int, dict], +) -> dict[int, dict]: """ technosphere_index: maps position -> activity metadata (name, location, classifications, etc.) Return minimal fields Edges uses to enrich consumer/supplier info. @@ -72,6 +78,7 @@ def _build_position_to_technosphere_lookup(technosphere_index: dict[int, dict]) } return out + def _ensure_minimal_flows( lca: EdgeLCIA, biosphere_index: dict[int, dict], @@ -83,7 +90,7 @@ def _ensure_minimal_flows( "name": f.get("name"), "categories": list(f.get("categories")), "unit": f.get("unit"), - "location": f.get("location"), # usually None for biosphere flows + "location": f.get("location"), # usually None for biosphere flows "classifications": f.get("classifications"), # optional "position": lca.lca.dicts.biosphere[pos], } @@ -105,6 +112,7 @@ def _ensure_minimal_flows( if pos in lca.lca.dicts.activity ] + def _as_row_csr(mat): """Ensure the characterization is a 2D CSR row matrix.""" if issparse(mat): @@ -121,6 +129,7 @@ def _as_row_csr(mat): arr = arr.T return csr_matrix(arr) + def create_edges_characterization_matrix( model: str, multilca_obj: bw2calc.MultiLCA, @@ -138,8 +147,9 @@ def create_edges_characterization_matrix( for method in methods: # create fake sparse inventory matrix with same SHAPE & DTYPE first_matrix = next(iter(multilca_obj.inventories.values())) - multilca_obj.inventory = csr_matrix(first_matrix.shape, - dtype=getattr(first_matrix, "dtype", float)) + multilca_obj.inventory = csr_matrix( + first_matrix.shape, dtype=getattr(first_matrix, "dtype", float) + ) lca = EdgeLCIA( demand={}, @@ -148,7 +158,9 @@ def create_edges_characterization_matrix( additional_topologies=topology, ) - if all(cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data): + if all( + cf["supplier"].get("matrix") == "technosphere" for cf in lca.raw_cfs_data + ): lca.technosphere_edges = { (r, c) for supply_array in multilca_obj.supply_arrays.values() @@ -170,8 +182,8 @@ def create_edges_characterization_matrix( biosphere_index=indices["biosphere"], technosphere_index=indices["technosphere"], ) - lca.position_to_technosphere_flows_lookup = _build_position_to_technosphere_lookup( - indices["technosphere"] + lca.position_to_technosphere_flows_lookup = ( + _build_position_to_technosphere_lookup(indices["technosphere"]) ) rs_bio, rs_tech, rc_cons = _edge_sets_for_lookup(lca) @@ -183,7 +195,6 @@ def create_edges_characterization_matrix( lca.apply_strategies() lca.evaluate_cfs() - # Each method yields a 2D plane (m x n). Ensure SciPy CSR, then convert to pydata/sparse COO. plane = lca.characterization_matrix if not issparse(plane): diff --git a/pathways/lca.py b/pathways/lca.py index ef5e3fe..005c6a9 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -431,7 +431,7 @@ def _inventory_results_3d_regular(lca, C: sps.csr_matrix): if use_distributions == 0: # Regular LCA calculations - #with CustomFilter("(almost) singular matrix"): + # with CustomFilter("(almost) singular matrix"): # lca.lci() if debug: From 373a454e0e2cb0ad3187efa58570a176105171dc Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Mon, 3 Nov 2025 09:01:50 +0100 Subject: [PATCH 5/8] Edges implementation --- pathways/lca.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pathways/lca.py b/pathways/lca.py index ef5e3fe..ed99ddb 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -280,6 +280,8 @@ def create_functional_units( alternative_unit = vars_idx[variable]["lhv"].get("unit") conversion_factor = vars_idx[variable]["lhv"].get("value") + print(f"alternative unit: {alternative_unit}, conversion_factor: {conversion_factor}") + if alternative_unit and conversion_factor: unit_vector = ( get_unit_conversion_factors( @@ -291,7 +293,7 @@ def create_functional_units( ) else: logging.warning( - f"Unit conversion factors not found for {variable}." + f"Alternative unit or conversion factor missing not found for {variable}: {alternative_unit}, conversion_factor: {conversion_factor}." ) unit_vector = 1.0 else: From 63459e61a1c807dba3ca2d320d63690e0b8625f9 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Mon, 3 Nov 2025 08:02:23 +0000 Subject: [PATCH 6/8] Black reformating --- pathways/lca.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pathways/lca.py b/pathways/lca.py index 44e75fc..59fce14 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -280,7 +280,9 @@ def create_functional_units( alternative_unit = vars_idx[variable]["lhv"].get("unit") conversion_factor = vars_idx[variable]["lhv"].get("value") - print(f"alternative unit: {alternative_unit}, conversion_factor: {conversion_factor}") + print( + f"alternative unit: {alternative_unit}, conversion_factor: {conversion_factor}" + ) if alternative_unit and conversion_factor: unit_vector = ( From 50c936fd7d7d704fc13b33c1790edb44f4c10bf8 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Tue, 4 Nov 2025 08:32:57 +0100 Subject: [PATCH 7/8] Fix LHV issue --- pathways/lca.py | 3 +-- pathways/utils.py | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pathways/lca.py b/pathways/lca.py index 44e75fc..b631bd4 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -280,8 +280,6 @@ def create_functional_units( alternative_unit = vars_idx[variable]["lhv"].get("unit") conversion_factor = vars_idx[variable]["lhv"].get("value") - print(f"alternative unit: {alternative_unit}, conversion_factor: {conversion_factor}") - if alternative_unit and conversion_factor: unit_vector = ( get_unit_conversion_factors( @@ -736,6 +734,7 @@ def _calculate_year(args: tuple): for k in lca_results.coords["location"].values.tolist() } + bar = pyprind.ProgBar(len(regions)) for region in regions: fus, fus_details = create_functional_units( diff --git a/pathways/utils.py b/pathways/utils.py index 94b5a74..cfb97b7 100644 --- a/pathways/utils.py +++ b/pathways/utils.py @@ -511,8 +511,9 @@ def add_lhv(variable, mapping) -> Union[dict, None]: :return: The LHV value if it exists, otherwise None. """ if variable in mapping: - if "lhv" in mapping[variable]: - return mapping[variable]["lhv"] + for ds in mapping[variable]["dataset"]: + if "lhv" in ds: + return ds["lhv"] return {} @@ -573,6 +574,10 @@ def fetch_indices( ) pass + for variable in variables: + if variable not in mapping: + print(f"Variable '{variable}' not found in mapping. Ensure it is correctly defined.") + if idxs is not None: # Map variables to their indices and associated dataset information From 67e9b1329616c27011ad93c7c0bc4f809e664b63 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Tue, 4 Nov 2025 07:33:55 +0000 Subject: [PATCH 8/8] Black reformating --- pathways/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pathways/utils.py b/pathways/utils.py index cfb97b7..bd52441 100644 --- a/pathways/utils.py +++ b/pathways/utils.py @@ -576,7 +576,9 @@ def fetch_indices( for variable in variables: if variable not in mapping: - print(f"Variable '{variable}' not found in mapping. Ensure it is correctly defined.") + print( + f"Variable '{variable}' not found in mapping. Ensure it is correctly defined." + ) if idxs is not None: