diff --git a/docs/src/modules.md b/docs/src/modules.md index da463d18..9de59622 100644 --- a/docs/src/modules.md +++ b/docs/src/modules.md @@ -243,27 +243,41 @@ factors that the impact modules in the next stage multiply the energy estimates ### PWUE Loads both **Power Usage Effectiveness (PUE)** and **Water Usage Effectiveness (WUE)** -factors from a CSV resource file bundled per provider: [`aws-pue-wue.csv`](https://github.com/DigitalPebble/spruce/blob/main/src/main/resources/aws-pue-wue.csv) uses the 2024 data -[published by AWS](https://sustainability.aboutamazon.com/aws-wue-pue.csv), and +factors from a CSV resource file bundled per provider: [`aws-pue-wue.csv`](https://github.com/DigitalPebble/spruce/blob/main/src/main/resources/aws-pue-wue.csv) carries the 2022–2025 figures +[published by AWS](https://sustainability.aboutamazon.com/products-services/aws-cloud), and [`azure-pue-wue.csv`](https://github.com/DigitalPebble/spruce/blob/main/src/main/resources/azure-pue-wue.csv) is sourced from [Microsoft's data centre sustainability pages](https://datacenters.microsoft.com/sustainability/efficiency/). +These factors are published per year and move noticeably from one year to the next, so they +are keyed by region **and** year rather than applied as a blanket value. The year is that of +the line item's usage date, read from whichever of `ChargePeriodStart`, +`line_item_usage_start_date`, `Date` or `BILLING_PERIOD` the report carries. + The lookup logic follows this priority: 1. Exact region match (e.g. `us-east-1`) -2. Regex pattern match (e.g. `us-.+`) -3. Default configured value (fallback to 1.15 for PUE, null for WUE) +2. Regex pattern match (e.g. `eu-.+`), i.e. the geography-level average +3. The provider-wide `GLOBAL` average, where the CSV has one +4. Default configured value (fallback to 1.15 for PUE, null for WUE) + +Within a tier the entry for the usage year is used; when that year is not covered — a +region AWS started reporting on recently, a WUE only published from 2024 onwards, or a row +with no usable date — the closest year available for that region is used instead. + +!!! note "Azure figures are not dated" + Microsoft does not break its PUE and WUE down by year, so the rows in + `azure-pue-wue.csv` leave the year empty and apply to every year. | | | |---|---| | **Class** | `com.digitalpebble.spruce.modules.PWUE` | -| **Reads** | `region` | +| **Reads** | `region`, the usage date of the line item | | **Writes** | `power_usage_effectiveness`, `water_usage_effectiveness` | **Configuration**: | Key | Default | Description | |---|---|---| -| `default` | 1.15 | PUE used when a region matches neither an exact entry nor a pattern | +| `default` | 1.15 | PUE used when a region matches no entry at any tier, global average included | ### AverageCarbonIntensity diff --git a/src/main/java/com/digitalpebble/spruce/CURColumn.java b/src/main/java/com/digitalpebble/spruce/CURColumn.java index b76700b9..e58cadee 100644 --- a/src/main/java/com/digitalpebble/spruce/CURColumn.java +++ b/src/main/java/com/digitalpebble/spruce/CURColumn.java @@ -17,6 +17,8 @@ public class CURColumn extends RowColumn { public static CURColumn LINE_ITEM_PRODUCT_CODE = new CURColumn("line_item_product_code", StringType); public static CURColumn LINE_ITEM_TYPE = new CURColumn("line_item_line_item_type", StringType); public static CURColumn LINE_ITEM_USAGE_TYPE = new CURColumn("line_item_usage_type", StringType); + public static CURColumn LINE_ITEM_USAGE_START_DATE = new CURColumn("line_item_usage_start_date", StringType); + public static CURColumn LINE_ITEM_USAGE_END_DATE = new CURColumn("line_item_usage_end_date", StringType); public static CURColumn PRICING_UNIT= new CURColumn("pricing_unit", StringType); public static CURColumn PRODUCT = new CURColumn("product", MapType.apply(StringType,StringType)); public static CURColumn PRODUCT_INSTANCE_TYPE = new CURColumn("product_instance_type", StringType); diff --git a/src/main/java/com/digitalpebble/spruce/RowColumn.java b/src/main/java/com/digitalpebble/spruce/RowColumn.java index 018772aa..7d6b41ca 100644 --- a/src/main/java/com/digitalpebble/spruce/RowColumn.java +++ b/src/main/java/com/digitalpebble/spruce/RowColumn.java @@ -6,9 +6,13 @@ import org.apache.spark.sql.types.DataType; import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Abstract base class for native column types that work with Spark Row objects. @@ -75,6 +79,44 @@ public LocalDate getDate(Row r) { return null; } + /** Matches a four-digit year not embedded in a longer number, e.g. in 2025-01-01T00:00:00Z, + * 01/15/2025 or the yyyy-MM of a billing period. */ + private static final Pattern YEAR = Pattern.compile("(? * This module centralizes the loading of PUE and WUE values that were previously * loaded separately by the PUE and Water modules. The values are stored in * columns for use by downstream modules. *

+ * Providers publish these factors per year and they move noticeably from one year to the + * next, so the figures are keyed by region and year rather than applied as a blanket + * value. The year comes from the usage date of the line item; when the report carries no + * usable date the most recent figures are used. + *

* The lookup logic follows this priority: *

    *
  1. Exact region match (e.g., "us-east-1")
    2. - *
  2. Regex pattern match (e.g., "us-.+")
    3. + *
  3. Regex pattern match on the region id (e.g., "eu-.+"), i.e. the geography average
    4. + *
  4. The provider-wide "GLOBAL" entry, where the CSV has one
    5. *
  5. Default configured value (fallback to 1.15 for PUE, null for WUE)
    6. *
+ * Within a tier, the entry for the usage year is used; if that year is not covered, the + * closest year available for that region is used instead. **/ public class PWUE implements EnrichmentModule { private double defaultPueValue = 1.15; private static final String DEFAULT_CSV_RESOURCE_PATH = "aws-pue-wue.csv"; - // PUE lookup maps - private final Map pueExactMatches = new HashMap<>(); - private final Map pueRegexMatches = new HashMap<>(); + /** RegionID marking the provider-wide average, used when no region entry matches. */ + private static final String GLOBAL_KEY = "GLOBAL"; + + /** Year given to entries whose CSV row leaves the year empty, i.e. figures that are not + * broken down by year. Below any real year, so it is only picked when nothing else is. */ + private static final int UNDATED = 0; + + /** Year used for rows with no usable usage date: yields the most recent figures. */ + private static final int LATEST = Integer.MAX_VALUE; + + /** + * Columns a usage date can be read from, in the order they are probed. All are optional: a + * report only carries the ones its provider and format define, and the FOCUS columns exist + * but are still null in native reports, where the FOCUS bridge module fills them in later. + */ + private static final RowColumn[] DATE_COLUMNS = { + FOCUSColumn.CHARGE_PERIOD_START, + CURColumn.LINE_ITEM_USAGE_START_DATE, + AzureColumn.DATE, + CURColumn.BILLING_PERIOD + }; + + /** PUE and WUE by year for one CSV key — a region id, or a regex over region ids. */ + private static class Factors implements Serializable { + /** null when the key is an exact region id rather than a pattern. */ + final Pattern pattern; + final NavigableMap pue = new TreeMap<>(); + final NavigableMap wue = new TreeMap<>(); - // WUE lookup maps - private final Map wueExactMatches = new HashMap<>(); - private final Map wueRegexMatches = new HashMap<>(); + Factors(Pattern pattern) { + this.pattern = pattern; + } + } + + private final Map exactMatches = new HashMap<>(); + /** A list rather than a map so the patterns are tried in the order the CSV lists them. */ + private final List regexMatches = new ArrayList<>(); + private Factors globalMatch; @Override public void init(Map params) { @@ -68,44 +115,28 @@ public void init(Map params, Provider provider) { List rows = Utils.loadCSV(csvResourcePath); for (String[] parts : rows) { - // We need at least 3 columns: Geography, RegionID, PUE (WUE is optional) - if (parts.length >= 3) { - String key = parts[1].trim(); - String pueStr = parts[2].trim(); - String wueStr = parts.length >= 4 ? parts[3].trim() : ""; - - // Process PUE value - if (!pueStr.isEmpty()) { - try { - double pueValue = Double.parseDouble(pueStr); - - // Only treat as regex if it contains regex metacharacters - if (key.contains(".") || key.contains("+") || key.contains("*")) { - pueRegexMatches.put(Pattern.compile(key), pueValue); - } else { - pueExactMatches.put(key, pueValue); - } - } catch (NumberFormatException e) { - System.err.println("Invalid PUE format in CSV for key: " + key); - } - } + // We need at least 4 columns: Geography, RegionID, Year, PUE (WUE is optional) + if (parts.length < 4) { + continue; + } + String key = parts[1].trim(); + String yearStr = parts[2].trim(); + String pueStr = parts[3].trim(); + String wueStr = parts.length >= 5 ? parts[4].trim() : ""; - // Process WUE value - if (!wueStr.isEmpty()) { - try { - double wueValue = Double.parseDouble(wueStr); - - // Only treat as regex if it contains regex metacharacters - if (key.contains(".") || key.contains("+") || key.contains("*")) { - wueRegexMatches.put(Pattern.compile(key), wueValue); - } else { - wueExactMatches.put(key, wueValue); - } - } catch (NumberFormatException e) { - System.err.println("Invalid WUE format in CSV for key: " + key); - } + int year = UNDATED; + if (!yearStr.isEmpty()) { + try { + year = Integer.parseInt(yearStr); + } catch (NumberFormatException e) { + System.err.println("Invalid year format in CSV for key: " + key); + continue; } } + + Factors factors = factorsFor(key); + store(factors.pue, key, year, pueStr, "PUE"); + store(factors.wue, key, year, wueStr, "WUE"); } if (params.containsKey("default")) { @@ -122,6 +153,40 @@ public void init(Map params, Provider provider) { } } + /** Returns the entry holding the figures for this CSV key, creating it on first sight. */ + private Factors factorsFor(String key) { + if (GLOBAL_KEY.equals(key)) { + if (globalMatch == null) { + globalMatch = new Factors(null); + } + return globalMatch; + } + // Only treat as regex if it contains regex metacharacters + if (key.contains(".") || key.contains("+") || key.contains("*")) { + for (Factors existing : regexMatches) { + if (existing.pattern.pattern().equals(key)) { + return existing; + } + } + Factors factors = new Factors(Pattern.compile(key)); + regexMatches.add(factors); + return factors; + } + return exactMatches.computeIfAbsent(key, k -> new Factors(null)); + } + + private static void store(NavigableMap byYear, String key, int year, + String value, String label) { + if (value.isEmpty()) { + return; + } + try { + byYear.put(year, Double.parseDouble(value)); + } catch (NumberFormatException e) { + System.err.println("Invalid " + label + " format in CSV for key: " + key); + } + } + @Override public Column[] columnsNeeded() { return new Column[]{REGION}; @@ -135,51 +200,58 @@ public Column[] columnsAdded() { @Override public void enrich(Row row, Map enrichedValues) { String region = REGION.getString(enrichedValues); + int year = usageYear(row); // Get and store PUE value - double pueValue = getPueForRegion(region); - enrichedValues.put(SpruceColumn.PUE, pueValue); + Double pueValue = lookup(region, year, false); + enrichedValues.put(SpruceColumn.PUE, pueValue != null ? pueValue : defaultPueValue); // Get and store WUE value - Double wueValue = getWueForRegion(region); + Double wueValue = lookup(region, year, true); if (wueValue != null) { enrichedValues.put(SpruceColumn.WUE, wueValue); } } - private double getPueForRegion(String region) { - if (region == null || region.isEmpty()) { - return defaultPueValue; - } - - if (pueExactMatches.containsKey(region)) { - return pueExactMatches.get(region); + /** Returns the year the line item was incurred in, or {@link #LATEST} if it has no date. */ + private static int usageYear(Row row) { + for (RowColumn column : DATE_COLUMNS) { + Integer year = column.getYear(row); + if (year != null) { + return year; + } } + return LATEST; + } - for (Map.Entry entry : pueRegexMatches.entrySet()) { - if (entry.getKey().matcher(region).matches()) { - return entry.getValue(); + private Double lookup(String region, int year, boolean water) { + if (region != null && !region.isEmpty()) { + Double value = valueFor(exactMatches.get(region), year, water); + if (value != null) { + return value; + } + for (Factors factors : regexMatches) { + if (factors.pattern.matcher(region).matches()) { + value = valueFor(factors, year, water); + if (value != null) { + return value; + } + } } } - - return defaultPueValue; + return valueFor(globalMatch, year, water); } - private Double getWueForRegion(String region) { - if (region == null || region.isEmpty()) { + /** Returns the figure published for that year, or the closest year available. */ + private static Double valueFor(Factors factors, int year, boolean water) { + if (factors == null) { return null; } - - if (wueExactMatches.containsKey(region)) { - return wueExactMatches.get(region); + NavigableMap byYear = water ? factors.wue : factors.pue; + Map.Entry entry = byYear.floorEntry(year); + if (entry == null) { + entry = byYear.firstEntry(); } - - for (Map.Entry entry : wueRegexMatches.entrySet()) { - if (entry.getKey().matcher(region).matches()) { - return entry.getValue(); - } - } - - return null; + return entry == null ? null : entry.getValue(); } -} \ No newline at end of file +} diff --git a/src/main/java/com/digitalpebble/spruce/modules/aws/FOCUSColumns.java b/src/main/java/com/digitalpebble/spruce/modules/aws/FOCUSColumns.java index 498d5731..f0c79c8d 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/aws/FOCUSColumns.java +++ b/src/main/java/com/digitalpebble/spruce/modules/aws/FOCUSColumns.java @@ -16,6 +16,8 @@ import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_OPERATION; import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_PRODUCT_CODE; import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_TYPE; +import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_USAGE_END_DATE; +import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_USAGE_START_DATE; import static com.digitalpebble.spruce.CURColumn.LINE_ITEM_USAGE_TYPE; import static com.digitalpebble.spruce.CURColumn.PRODUCT_FROM_REGION_CODE; import static com.digitalpebble.spruce.CURColumn.PRODUCT_REGION_CODE; @@ -50,8 +52,6 @@ public class FOCUSColumns implements EnrichmentModule { public static final CURColumn LINE_ITEM_UNBLENDED_COST = new CURColumn("line_item_unblended_cost", DataTypes.DoubleType); public static final CURColumn LINE_ITEM_USAGE_ACCOUNT_ID = new CURColumn("line_item_usage_account_id", DataTypes.StringType); - public static final CURColumn LINE_ITEM_USAGE_START_DATE = new CURColumn("line_item_usage_start_date", DataTypes.StringType); - public static final CURColumn LINE_ITEM_USAGE_END_DATE = new CURColumn("line_item_usage_end_date", DataTypes.StringType); public static final CURColumn RESOURCE_TAGS = new CURColumn("resource_tags", DataTypes.createMapType(DataTypes.StringType, DataTypes.StringType)); public static final CURColumn PRICING_PUBLIC_ON_DEMAND_COST = new CURColumn("pricing_public_on_demand_cost", DataTypes.DoubleType); diff --git a/src/main/resources/aws-pue-wue.csv b/src/main/resources/aws-pue-wue.csv index 6a257006..43388b27 100644 --- a/src/main/resources/aws-pue-wue.csv +++ b/src/main/resources/aws-pue-wue.csv @@ -1,30 +1,126 @@ -# Geography,RegionID,PUE,WUE -Europe,eu-.+,1.11,0.04 -Midde East,me-.+,1.3, -North America,us-.+,1.14,0.13 -North America,mx-.+,1.14, -Asia Pacific (excl. China),ap-.+,1.27,0.98 -China,cn-.+,1.25, -Africa (Cape Town),af-south-1,1.24, -Asia-Pacific (Tokyo),ap-northeast-1,1.27,0.91 -Asia-Pacific (Mumbai),ap-south-1,1.42, -Asia-Pacific (Hyderabad),ap-south-2,1.46, -Asia-Pacific (Singapore),ap-southeast-1,1.32,1.68 -Asia-Pacific (Sydney),ap-southeast-2,,0.12 -Asia-Pacific (Jakarta),ap-southeast-3,1.4,2.75 -Asia-Pacific (Melbourne),ap-southeast-4,,0.02 -Canada (Central),ca-central-1,1.19,0.04 -Canada (West),ca-west-1,1.17,0.08 -China (Ningxia),cn-northwest-1,1.25, -Europe (Frankfurt),eu-central-1,1.35,0.01 -Europe (Stockholm),eu-north-1,1.1,0.02 -Europe (Spain),eu-south-2,1.09,0.24 -Europe (Ireland),eu-west-1,1.11,0.03 -Middle East (UAE),me-central-1,1.27, -Middle East (Bahrain),me-south-1,1.33, -South America (Sao Paulo),sa-east-1,1.17,0.23 -U.S. East (Northern Virginia),us-east-1,1.15,0.12 -U.S. East (Ohio),us-east-2,1.13,0.1 -U.S. West (Northern California),us-west-1,1.18,0.51 -U.S. West (Oregon),us-west-2,1.12,0.16 -Israel,il-central-1,1.3, +# PUE and WUE published by AWS, one row per region and year. +# Source: https://sustainability.aboutamazon.com/products-services/aws-cloud (retrieved 2026-08-26) +# The downloadable https://sustainability.aboutamazon.com/aws-wue-pue.csv only goes up to +# 2024, so the figures below are taken from the tables on the page itself. +# +# Lookup order: exact RegionID, then the geography-level average whose regex matches the +# region id, then the GLOBAL average. Within a tier the row for the usage year is used, +# falling back to the closest year available. An empty PUE or WUE means AWS published no +# figure for that region and year. The GLOBAL WUE for 2022 and 2023 comes from the +# downloadable CSV, which is the only place AWS published it. +# +# Geography,RegionID,Year,PUE,WUE +Global,GLOBAL,2022,1.15,0.19 +Global,GLOBAL,2023,1.15,0.18 +Global,GLOBAL,2024,1.15,0.15 +Global,GLOBAL,2025,1.14,0.12 +Europe,eu-.+,2022,1.11, +Europe,eu-.+,2023,1.11, +Europe,eu-.+,2024,1.11,0.04 +Europe,eu-.+,2025,1.11,0.05 +Middle East,(me|il)-.+,2022,1.38, +Middle East,(me|il)-.+,2023,1.33, +Middle East,(me|il)-.+,2024,1.31, +Middle East,(me|il)-.+,2025,1.29, +Africa,af-.+,2022,1.36, +Africa,af-.+,2023,1.24, +Africa,af-.+,2024,1.24, +Africa,af-.+,2025,1.19, +North America,(us|ca|mx)-.+,2022,1.14, +North America,(us|ca|mx)-.+,2023,1.14, +North America,(us|ca|mx)-.+,2024,1.14,0.13 +North America,(us|ca|mx)-.+,2025,1.14,0.08 +Central/South America,sa-.+,2023,1.18, +Central/South America,sa-.+,2024,1.17,0.23 +Central/South America,sa-.+,2025,1.14,0.09 +Asia Pacific (excl. China),ap-.+,2022,1.26, +Asia Pacific (excl. China),ap-.+,2023,1.28, +Asia Pacific (excl. China),ap-.+,2024,1.27,0.98 +Asia Pacific (excl. China),ap-.+,2025,1.25,1.1 +China,cn-.+,2022,1.28, +China,cn-.+,2023,1.26, +China,cn-.+,2024,1.25, +China,cn-.+,2025,1.25, +Europe (Frankfurt),eu-central-1,2022,1.32, +Europe (Frankfurt),eu-central-1,2023,1.33, +Europe (Frankfurt),eu-central-1,2024,1.35,0.01 +Europe (Frankfurt),eu-central-1,2025,1.24,0.17 +Europe (Ireland),eu-west-1,2022,1.1, +Europe (Ireland),eu-west-1,2023,1.1, +Europe (Ireland),eu-west-1,2024,1.11,0.03 +Europe (Ireland),eu-west-1,2025,1.1,0.02 +Europe (London),eu-west-2,2025,1.23,0.14 +Europe (Paris),eu-west-3,2025,1.35, +Europe (Spain),eu-south-2,2023,1.11, +Europe (Spain),eu-south-2,2024,1.07,0.24 +Europe (Spain),eu-south-2,2025,1.06,0.12 +Europe (Stockholm),eu-north-1,2022,1.12, +Europe (Stockholm),eu-north-1,2023,1.12, +Europe (Stockholm),eu-north-1,2024,1.1,0.02 +Europe (Stockholm),eu-north-1,2025,1.09,0.02 +Middle East (Bahrain),me-south-1,2022,1.38, +Middle East (Bahrain),me-south-1,2023,1.32, +Middle East (Bahrain),me-south-1,2024,1.33, +Middle East (Bahrain),me-south-1,2025,1.33, +Middle East (Tel Aviv),il-central-1,2025,1.27, +Middle East (UAE),me-central-1,2023,1.36, +Middle East (UAE),me-central-1,2024,1.27, +Middle East (UAE),me-central-1,2025,1.25, +Africa (Cape Town),af-south-1,2022,1.36, +Africa (Cape Town),af-south-1,2023,1.24, +Africa (Cape Town),af-south-1,2024,1.24, +Africa (Cape Town),af-south-1,2025,1.19, +Asia-Pacific (Bangkok),ap-southeast-7,2025,1.38, +Asia-Pacific (Hyderabad),ap-south-2,2023,1.5, +Asia-Pacific (Hyderabad),ap-south-2,2024,1.46, +Asia-Pacific (Hyderabad),ap-south-2,2025,1.43, +Asia-Pacific (Jakarta),ap-southeast-3,2022,1.39, +Asia-Pacific (Jakarta),ap-southeast-3,2023,1.35, +Asia-Pacific (Jakarta),ap-southeast-3,2024,1.4,2.75 +Asia-Pacific (Jakarta),ap-southeast-3,2025,1.29,2.85 +Asia-Pacific (Melbourne),ap-southeast-4,2024,,0.02 +Asia-Pacific (Melbourne),ap-southeast-4,2025,1.07,0.06 +Asia-Pacific (Mumbai),ap-south-1,2022,1.43, +Asia-Pacific (Mumbai),ap-south-1,2023,1.44, +Asia-Pacific (Mumbai),ap-south-1,2024,1.42, +Asia-Pacific (Mumbai),ap-south-1,2025,1.4, +Asia-Pacific (Osaka),ap-northeast-3,2025,1.46, +Asia-Pacific (Singapore),ap-southeast-1,2022,1.33, +Asia-Pacific (Singapore),ap-southeast-1,2023,1.3, +Asia-Pacific (Singapore),ap-southeast-1,2024,1.32,1.68 +Asia-Pacific (Singapore),ap-southeast-1,2025,1.3,1.57 +Asia-Pacific (Sydney),ap-southeast-2,2024,,0.12 +Asia-Pacific (Sydney),ap-southeast-2,2025,1.14,0.1 +Asia-Pacific (Tokyo),ap-northeast-1,2022,1.3, +Asia-Pacific (Tokyo),ap-northeast-1,2023,1.32, +Asia-Pacific (Tokyo),ap-northeast-1,2024,1.27,0.91 +Asia-Pacific (Tokyo),ap-northeast-1,2025,1.25,1.22 +China (Ningxia),cn-northwest-1,2022,1.28, +China (Ningxia),cn-northwest-1,2023,1.26, +China (Ningxia),cn-northwest-1,2024,1.25, +China (Ningxia),cn-northwest-1,2025,1.25, +South America (Sao Paulo),sa-east-1,2023,1.18, +South America (Sao Paulo),sa-east-1,2024,1.17,0.23 +South America (Sao Paulo),sa-east-1,2025,1.14,0.09 +U.S. East (Northern Virginia),us-east-1,2022,1.16, +U.S. East (Northern Virginia),us-east-1,2023,1.15, +U.S. East (Northern Virginia),us-east-1,2024,1.15,0.12 +U.S. East (Northern Virginia),us-east-1,2025,1.15,0.06 +U.S. East (Ohio),us-east-2,2022,1.12, +U.S. East (Ohio),us-east-2,2023,1.12, +U.S. East (Ohio),us-east-2,2024,1.13,0.1 +U.S. East (Ohio),us-east-2,2025,1.12,0.06 +U.S. West (Northern California),us-west-1,2022,1.17, +U.S. West (Northern California),us-west-1,2023,1.17, +U.S. West (Northern California),us-west-1,2024,1.18,0.51 +U.S. West (Northern California),us-west-1,2025,1.18,0.39 +U.S. West (Oregon),us-west-2,2022,1.13, +U.S. West (Oregon),us-west-2,2023,1.13, +U.S. West (Oregon),us-west-2,2024,1.12,0.16 +U.S. West (Oregon),us-west-2,2025,1.12,0.12 +Canada (Central),ca-central-1,2022,1.26, +Canada (Central),ca-central-1,2023,1.22, +Canada (Central),ca-central-1,2024,1.19,0.04 +Canada (Central),ca-central-1,2025,1.19,0.06 +Canada (West),ca-west-1,2024,1.17,0.08 +Canada (West),ca-west-1,2025,1.13,0.04 diff --git a/src/main/resources/azure-pue-wue.csv b/src/main/resources/azure-pue-wue.csv index 35d41fb4..e9806bbb 100644 --- a/src/main/resources/azure-pue-wue.csv +++ b/src/main/resources/azure-pue-wue.csv @@ -1,78 +1,80 @@ # https://datacenters.microsoft.com/sustainability/efficiency/ -# Name,RegionID,PUE,WUE -Australia Central (Canberra),australiacentral,1.28,0.25 -Australia Central 2 (Canberra),australiacentral2,1.28,0.25 -Australia Southeast (Melbourne),australiasoutheast,1.28,0.25 -Australia East (Sydney),australiaeast,1.28,0.25 -Austria East (Vienna),austriaeast,1.16,0.03 -Belgium Central,belgiumcentral,1.16,0.03 -Brazil Southeast (Rio),brazilsoutheast,1.16,0.34 -Brazil South (São Paulo),brazilsouth,1.16,0.34 -Canada East (Quebec City),canadaeast,1.16,0.34 -Canada Central (Toronto),canadacentral,1.16,0.34 -Chile Central (Santiago),chilecentral,1.16,0.34 -China North (Beijing),chinanorth,1.28,0.25 -China North 2 (Beijing),chinanorth2,1.28,0.25 -China North 3 (Beijing),chinanorth3,1.28,0.25 -East Asia (Hong Kong),eastasia,1.28,0.25 -China East 3 (Jiangsu),chinaeast3,1.28,0.25 -China East (Shanghai),chinaeast,1.28,0.25 -China East 2 (Shanghai),chinaeast2,1.28,0.25 -Denmark East (Copenhagen),denmarkeast,1.16,0.03 -Finland (Helsinki),finlandcentral,1.16,0.03 -France South (Marseille),francesouth,1.16,0.03 -France Central (Paris),francecentral,1.16,0.03 -Germany North (Berlin),germanynorth,1.16,0.03 -Germany West Central (Frankfurt),germanywestcentral,1.16,0.03 -Greece Central (Athens),greececentral,1.16,0.03 -South India (Chennai),southindia,1.28,0.25 -Southcentral (Hyderabad),indiasouthcentral,1.28,0.25 -West India (Mumbai),westindia,1.28,0.25 -Central India (Pune),centralindia,1.28,0.25 -Indonesia Central (Jakarta),indonesiacentral,1.28,0.25 -North Europe (Dublin),northeurope,1.16,0.03 -Israel Central (Tel Aviv),israelcentral,1.16,0.03 -Italy North (Milan),italynorth,1.16,0.03 -Japan West (Osaka),japanwest,1.28,0.25 -Japan East (Saitama Tokyo),japaneast,1.28,0.25 -Malaysia West (Kuala Lumpur),malaysiawest,1.28,0.25 -Mexico Central (Querétaro),mexicocentral,1.16,0.34 -West Europe (Amsterdam),westeurope,1.16,0.03 -New Zealand North (Auckland),newzealandnorth,1.28,0.25 -Norway East (Oslo),norwayeast,1.16,0.03 -Norway West (Stavanger),norwaywest,1.16,0.03 -Poland Central (Warsaw),polandcentral,1.16,0.03 -Qatar Central (Doha),qatarcentral,1.16,0.03 -Saudi Arabia East,saudiarabiasouthcentral,1.28,0.25 -Southeast Asia (Singapore),southeastasia,1.28,0.25 -South Africa West (Cape Town),southafricawest,1.16,0.03 -South Africa North (Johannesburg),southafricanorth,1.16,0.03 -Korea South (Busan),koreasouth,1.28,0.25 -Korea Central (Seoul),koreacentral,1.28,0.25 -Spain Central (Madrid),spaincentral,1.16,0.03 -Sweden Central (Gävle),swedencentral,1.16,0.03 -Sweden South (Staffanstorp),swedensouth,1.16,0.03 -Switzerland West (Geneva),switzerlandwest,1.16,0.03 -Switzerland North (Zürich),switzerlandnorth,1.16,0.03 -Taiwan North (Taipei),taiwannorth,1.28,0.25 -UAE Central (Abu Dhabi),uaecentral,1.16,0.03 -UAE North (Dubai),uaenorth,1.16,0.03 -UK West (Cardiff),ukwest,1.16,0.03 -UK South (London),uksouth,1.16,0.03 -East US 3 (Atlanta),eastus3,1.16,0.34 -US Gov Texas,usgovtexas,1.16,0.34 -West Central US (Cheyenne),westcentralus,1.16,0.34 -North Central US (Chicago),northcentralus,1.16,0.34 -Central US (Des Moines),centralus,1.16,0.34 -US DoD Central,usdodwest,1.16,0.34 -West US 2 (Quincy),westus2,1.16,0.34 -US Gov Arizona,usgovarizona,1.16,0.34 -West US 3 (Phoenix),westus3,1.16,0.34 -East US 2 (Richmond),eastus2,1.16,0.34 -US DoD East,usdodeast,1.16,0.34 -East US (Ashburn),eastus,1.16,0.34 -US Sec East,usseceast,1.16,0.34 -South Central US (San Antonio),southcentralus,1.16,0.34 -West US (San Francisco),westus,1.16,0.34 -US Sec West,ussecwest,1.16,0.34 -US Gov Virginia ,usgovvirginia,1.16,0.34 +# Microsoft does not break these down by year, so the Year column is left empty, +# meaning the figures apply to every year. +# Name,RegionID,Year,PUE,WUE +Australia Central (Canberra),australiacentral,,1.28,0.25 +Australia Central 2 (Canberra),australiacentral2,,1.28,0.25 +Australia Southeast (Melbourne),australiasoutheast,,1.28,0.25 +Australia East (Sydney),australiaeast,,1.28,0.25 +Austria East (Vienna),austriaeast,,1.16,0.03 +Belgium Central,belgiumcentral,,1.16,0.03 +Brazil Southeast (Rio),brazilsoutheast,,1.16,0.34 +Brazil South (São Paulo),brazilsouth,,1.16,0.34 +Canada East (Quebec City),canadaeast,,1.16,0.34 +Canada Central (Toronto),canadacentral,,1.16,0.34 +Chile Central (Santiago),chilecentral,,1.16,0.34 +China North (Beijing),chinanorth,,1.28,0.25 +China North 2 (Beijing),chinanorth2,,1.28,0.25 +China North 3 (Beijing),chinanorth3,,1.28,0.25 +East Asia (Hong Kong),eastasia,,1.28,0.25 +China East 3 (Jiangsu),chinaeast3,,1.28,0.25 +China East (Shanghai),chinaeast,,1.28,0.25 +China East 2 (Shanghai),chinaeast2,,1.28,0.25 +Denmark East (Copenhagen),denmarkeast,,1.16,0.03 +Finland (Helsinki),finlandcentral,,1.16,0.03 +France South (Marseille),francesouth,,1.16,0.03 +France Central (Paris),francecentral,,1.16,0.03 +Germany North (Berlin),germanynorth,,1.16,0.03 +Germany West Central (Frankfurt),germanywestcentral,,1.16,0.03 +Greece Central (Athens),greececentral,,1.16,0.03 +South India (Chennai),southindia,,1.28,0.25 +Southcentral (Hyderabad),indiasouthcentral,,1.28,0.25 +West India (Mumbai),westindia,,1.28,0.25 +Central India (Pune),centralindia,,1.28,0.25 +Indonesia Central (Jakarta),indonesiacentral,,1.28,0.25 +North Europe (Dublin),northeurope,,1.16,0.03 +Israel Central (Tel Aviv),israelcentral,,1.16,0.03 +Italy North (Milan),italynorth,,1.16,0.03 +Japan West (Osaka),japanwest,,1.28,0.25 +Japan East (Saitama Tokyo),japaneast,,1.28,0.25 +Malaysia West (Kuala Lumpur),malaysiawest,,1.28,0.25 +Mexico Central (Querétaro),mexicocentral,,1.16,0.34 +West Europe (Amsterdam),westeurope,,1.16,0.03 +New Zealand North (Auckland),newzealandnorth,,1.28,0.25 +Norway East (Oslo),norwayeast,,1.16,0.03 +Norway West (Stavanger),norwaywest,,1.16,0.03 +Poland Central (Warsaw),polandcentral,,1.16,0.03 +Qatar Central (Doha),qatarcentral,,1.16,0.03 +Saudi Arabia East,saudiarabiasouthcentral,,1.28,0.25 +Southeast Asia (Singapore),southeastasia,,1.28,0.25 +South Africa West (Cape Town),southafricawest,,1.16,0.03 +South Africa North (Johannesburg),southafricanorth,,1.16,0.03 +Korea South (Busan),koreasouth,,1.28,0.25 +Korea Central (Seoul),koreacentral,,1.28,0.25 +Spain Central (Madrid),spaincentral,,1.16,0.03 +Sweden Central (Gävle),swedencentral,,1.16,0.03 +Sweden South (Staffanstorp),swedensouth,,1.16,0.03 +Switzerland West (Geneva),switzerlandwest,,1.16,0.03 +Switzerland North (Zürich),switzerlandnorth,,1.16,0.03 +Taiwan North (Taipei),taiwannorth,,1.28,0.25 +UAE Central (Abu Dhabi),uaecentral,,1.16,0.03 +UAE North (Dubai),uaenorth,,1.16,0.03 +UK West (Cardiff),ukwest,,1.16,0.03 +UK South (London),uksouth,,1.16,0.03 +East US 3 (Atlanta),eastus3,,1.16,0.34 +US Gov Texas,usgovtexas,,1.16,0.34 +West Central US (Cheyenne),westcentralus,,1.16,0.34 +North Central US (Chicago),northcentralus,,1.16,0.34 +Central US (Des Moines),centralus,,1.16,0.34 +US DoD Central,usdodwest,,1.16,0.34 +West US 2 (Quincy),westus2,,1.16,0.34 +US Gov Arizona,usgovarizona,,1.16,0.34 +West US 3 (Phoenix),westus3,,1.16,0.34 +East US 2 (Richmond),eastus2,,1.16,0.34 +US DoD East,usdodeast,,1.16,0.34 +East US (Ashburn),eastus,,1.16,0.34 +US Sec East,usseceast,,1.16,0.34 +South Central US (San Antonio),southcentralus,,1.16,0.34 +West US (San Francisco),westus,,1.16,0.34 +US Sec West,ussecwest,,1.16,0.34 +US Gov Virginia ,usgovvirginia,,1.16,0.34 diff --git a/src/test/java/com/digitalpebble/spruce/RowColumnTest.java b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java new file mode 100644 index 00000000..e0c276a1 --- /dev/null +++ b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalpebble.spruce; + +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class RowColumnTest { + + private static final String FIELD = "usage_date"; + private static final StructType SCHEMA = new StructType(new StructField[]{ + new StructField(FIELD, DataTypes.StringType, true, Metadata.empty())}); + + private static final CURColumn COLUMN = new CURColumn(FIELD, DataTypes.StringType); + + private static Row row(Object value) { + return new GenericRowWithSchema(new Object[]{value}, SCHEMA); + } + + @Test + void readsTheYearFromTheRepresentationsTheReportsUse() { + // strings, as CSV exports carry them + assertEquals(2025, COLUMN.getYear(row("2025-01-01T00:00:00Z"))); + assertEquals(2024, COLUMN.getYear(row("2024-12-31"))); + assertEquals(2023, COLUMN.getYear(row("12/31/2023"))); + // the yyyy-MM of a billing period + assertEquals(2022, COLUMN.getYear(row("2022-07"))); + + // date and timestamp types, as Parquet reports carry them + assertEquals(2025, COLUMN.getYear(row(java.sql.Timestamp.valueOf("2025-06-15 12:00:00")))); + assertEquals(2025, COLUMN.getYear(row(java.sql.Date.valueOf("2025-06-15")))); + assertEquals(2025, COLUMN.getYear(row(java.time.LocalDate.of(2025, 6, 15)))); + assertEquals(2025, COLUMN.getYear(row(java.time.LocalDateTime.of(2025, 6, 15, 12, 0)))); + assertEquals(2025, COLUMN.getYear(row(java.time.Instant.parse("2025-06-15T12:00:00Z")))); + } + + @Test + void returnsNullWhenThereIsNoYearToRead() { + assertNull(COLUMN.getYear(row(null))); + assertNull(COLUMN.getYear(row("not a date"))); + // a column the report does not carry + assertNull(new CURColumn("absent", DataTypes.StringType).getYear(row("2025-01-01"))); + } +} diff --git a/src/test/java/com/digitalpebble/spruce/modules/PWUETest.java b/src/test/java/com/digitalpebble/spruce/modules/PWUETest.java index 14b85c67..3912ff82 100644 --- a/src/test/java/com/digitalpebble/spruce/modules/PWUETest.java +++ b/src/test/java/com/digitalpebble/spruce/modules/PWUETest.java @@ -2,11 +2,13 @@ package com.digitalpebble.spruce.modules; +import com.digitalpebble.spruce.CURColumn; import com.digitalpebble.spruce.Column; import com.digitalpebble.spruce.Provider; -import com.digitalpebble.spruce.Utils; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,27 +22,40 @@ class PWUETest { private PWUE pwue; - private StructType schema; + + /** Only the usage date is read from the row itself; the region comes from the enriched map. */ + private static final StructType SCHEMA = new StructType(new StructField[]{ + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), + DataTypes.StringType, true, null)}); @BeforeEach void setUp() { pwue = new PWUE(); pwue.init(new HashMap<>(), Provider.AWS); - schema = Utils.getSchema(pwue); } - private Row emptyRow() { - return new GenericRowWithSchema(new Object[schema.length()], schema); + /** A row with no usage date, so the most recent figures apply. */ + private Row undatedRow() { + return new GenericRowWithSchema(new Object[]{null}, SCHEMA); } - @Test - void loadsPUEAndWUEForExactRegionMatch() { - // us-east-1 has PUE = 1.15 and WUE = 0.12 in aws-pue-wue.csv - Row row = emptyRow(); + private Row rowForYear(int year) { + return new GenericRowWithSchema(new Object[]{year + "-06-15T00:00:00Z"}, SCHEMA); + } + + private Map enrich(Row row, String region) { Map enriched = new HashMap<>(); - enriched.put(REGION, "us-east-1"); - + if (region != null) { + enriched.put(REGION, region); + } pwue.enrich(row, enriched); + return enriched; + } + + @Test + void loadsPUEAndWUEForExactRegionMatch() { + // us-east-1 has PUE = 1.15 and WUE = 0.12 for 2024 in aws-pue-wue.csv + Map enriched = enrich(rowForYear(2024), "us-east-1"); assertTrue(enriched.containsKey(PUE)); assertTrue(enriched.containsKey(WUE)); @@ -48,79 +63,109 @@ void loadsPUEAndWUEForExactRegionMatch() { assertEquals(0.12, (Double) enriched.get(WUE), 0.01); } + @Test + void usesTheFiguresPublishedForTheUsageYear() { + // eu-central-1 improved from PUE 1.32 in 2022 to 1.24 in 2025 + assertEquals(1.32, (Double) enrich(rowForYear(2022), "eu-central-1").get(PUE), 0.01); + assertEquals(1.33, (Double) enrich(rowForYear(2023), "eu-central-1").get(PUE), 0.01); + assertEquals(1.35, (Double) enrich(rowForYear(2024), "eu-central-1").get(PUE), 0.01); + assertEquals(1.24, (Double) enrich(rowForYear(2025), "eu-central-1").get(PUE), 0.01); + + // and its WUE went the other way, from 0.01 in 2024 to 0.17 in 2025 + assertEquals(0.01, (Double) enrich(rowForYear(2024), "eu-central-1").get(WUE), 0.01); + assertEquals(0.17, (Double) enrich(rowForYear(2025), "eu-central-1").get(WUE), 0.01); + } + + @Test + void fallsBackToTheClosestYearPublishedForTheRegion() { + // eu-west-2 only has 2025 figures, used for earlier and later years alike + assertEquals(1.23, (Double) enrich(rowForYear(2023), "eu-west-2").get(PUE), 0.01); + assertEquals(1.23, (Double) enrich(rowForYear(2030), "eu-west-2").get(PUE), 0.01); + + // us-east-1 has PUE from 2022 but WUE only from 2024, so 2022 borrows the 2024 WUE + Map enriched = enrich(rowForYear(2022), "us-east-1"); + assertEquals(1.16, (Double) enriched.get(PUE), 0.01); + assertEquals(0.12, (Double) enriched.get(WUE), 0.01); + } + + @Test + void usesTheMostRecentFiguresWhenTheRowHasNoDate() { + // 2025 is the latest year published for us-east-1 + Map enriched = enrich(undatedRow(), "us-east-1"); + assertEquals(1.15, (Double) enriched.get(PUE), 0.01); + assertEquals(0.06, (Double) enriched.get(WUE), 0.01); + } + @Test void loadsPUEAndWUEForRegexRegionMatch() { - // us-gov-west-1 matches regex us-.+ with PUE = 1.14 and WUE = 0.13 - Row row = emptyRow(); - Map enriched = new HashMap<>(); - enriched.put(REGION, "us-gov-west-1"); - - pwue.enrich(row, enriched); + // us-gov-west-1 matches regex (us|ca|mx)-.+, the North America average + Map enriched = enrich(rowForYear(2024), "us-gov-west-1"); - assertTrue(enriched.containsKey(PUE)); - assertTrue(enriched.containsKey(WUE)); assertEquals(1.14, (Double) enriched.get(PUE), 0.01); assertEquals(0.13, (Double) enriched.get(WUE), 0.01); } @Test void loadsPUEAndWUEForRegionWithRegexFallback() { - // ap-south-1 has exact PUE = 1.42 but no exact WUE, so it falls back to regex ap-.+ with WUE = 0.98 - Row row = emptyRow(); - Map enriched = new HashMap<>(); - enriched.put(REGION, "ap-south-1"); - - pwue.enrich(row, enriched); + // ap-south-1 has an exact PUE for 2025 but no WUE at all, so it falls back to the + // Asia Pacific average for the same year + Map enriched = enrich(rowForYear(2025), "ap-south-1"); - assertTrue(enriched.containsKey(PUE)); - assertTrue(enriched.containsKey(WUE)); - assertEquals(1.42, (Double) enriched.get(PUE), 0.01); // Exact match takes priority for PUE - assertEquals(0.98, (Double) enriched.get(WUE), 0.01); // Regex match for WUE + assertEquals(1.4, (Double) enriched.get(PUE), 0.01); + assertEquals(1.1, (Double) enriched.get(WUE), 0.01); } @Test - void usesDefaultPUEForUnknownRegion() { - // Unknown region should get default PUE (1.15) and no WUE - Row row = emptyRow(); - Map enriched = new HashMap<>(); - enriched.put(REGION, "unknown-region-99"); - - pwue.enrich(row, enriched); + void fallsBackToTheGlobalAverageForUnknownRegion() { + Map enriched = enrich(rowForYear(2024), "unknown-region-99"); - assertTrue(enriched.containsKey(PUE)); - assertFalse(enriched.containsKey(WUE)); assertEquals(1.15, (Double) enriched.get(PUE), 0.01); + assertEquals(0.15, (Double) enriched.get(WUE), 0.01); + + // 2025 lowered the global average + enriched = enrich(rowForYear(2025), "unknown-region-99"); + assertEquals(1.14, (Double) enriched.get(PUE), 0.01); + assertEquals(0.12, (Double) enriched.get(WUE), 0.01); } @Test - void usesCustomDefaultPUEWhenConfigured() { - // Test with custom default PUE value - PWUE customLoader = new PWUE(); - Map config = new HashMap<>(); - config.put("default", 1.20); - customLoader.init(config, Provider.AWS); - - Row row = emptyRow(); - Map enriched = new HashMap<>(); - enriched.put(REGION, "unknown-region-99"); - - customLoader.enrich(row, enriched); + void handlesNullRegion() { + // no region: the global average for the usage year + Map enriched = enrich(rowForYear(2024), null); assertTrue(enriched.containsKey(PUE)); - assertEquals(1.20, (Double) enriched.get(PUE), 0.01); + assertEquals(1.15, (Double) enriched.get(PUE), 0.01); } @Test - void handlesNullRegion() { - Row row = emptyRow(); + void undatedCsvEntriesApplyToEveryYear() { + // Microsoft does not publish these by year, so the same figures apply throughout + PWUE azure = new PWUE(); + azure.init(new HashMap<>(), Provider.AZURE); + + for (int year : new int[]{2022, 2025}) { + Map enriched = new HashMap<>(); + enriched.put(REGION, "westeurope"); + azure.enrich(rowForYear(year), enriched); + assertEquals(1.16, (Double) enriched.get(PUE), 0.01); + assertEquals(0.03, (Double) enriched.get(WUE), 0.01); + } + } + + @Test + void usesDefaultPUEWhenNothingMatches() { + // the Azure CSV has no global entry, so an unknown region falls through to the default + PWUE azure = new PWUE(); + Map config = new HashMap<>(); + config.put("default", 1.20); + azure.init(config, Provider.AZURE); + Map enriched = new HashMap<>(); - // No region set - - pwue.enrich(row, enriched); + enriched.put(REGION, "unknown-region-99"); + azure.enrich(undatedRow(), enriched); - assertTrue(enriched.containsKey(PUE)); + assertEquals(1.20, (Double) enriched.get(PUE), 0.01); assertFalse(enriched.containsKey(WUE)); - assertEquals(1.15, (Double) enriched.get(PUE), 0.01); } @Test @@ -137,4 +182,4 @@ void columnsNeeded() { assertEquals(1, columns.length); assertEquals(REGION, columns[0]); } -} \ No newline at end of file +}