Merge master april 26

.github/pull_request_template.md

@@ -1,8 +1,16 @@
 Before asking for a review for this PR make sure to complete the following checklist:
 
-- [ ] Workflow with target rule `ariadne_all` completes without errors
-- [ ] The logic of `export_ariadne_variables` has been adapted to the changes
-- [ ] One or several figures that validate the changes in the PR have been posted as a comment
-- [ ] A brief description of the changes has been added to `Changelog.md`
-- [ ] The latest `main` has been merged into the PR
-- [ ] The config has a new prefix of the format `YYYYMMDDdescriptive_title`
+## Changes proposed in this Pull Request
+
+
+## Checklist
+
+**Required:**
+- [ ] Changes are tested locally and behave as expected.
+- [ ] Code and workflow changes are documented.
+- [ ] A brief description of the changes has been added to `Changelog.md`.
+
+**If applicable:**
+- [ ] Changes in configuration options are reflected in `scripts/lib/validation`.
+- [ ] For new data sources or versions, [these instructions](https://pypsa-eur.readthedocs.io/en/latest/data_sources.html) have been followed.
+- [ ] New rules are documented in the appropriate `doc/*.rst` files.

.github/pypsa-bot.yml

@@ -0,0 +1,30 @@
+tasks:
+  sync-locks:
+    triggers:
+    - "pixi.toml"
+    - "pixi.lock"
+    stage:
+    - "pixi.lock"
+    - "envs/"
+
+  generate-config:
+    triggers:
+    - "scripts/lib/validation/config/**"
+    stage:
+    - "config/"
+    - "doc/"
+    - "*.yaml"
+    - "*.yml"
+
+app:
+  trusted_users:
+  - daniel-rdt
+  runs:
+  - name: "config.validator.yaml"
+    configfile: "config/test/config.validator.yaml"
+    triggers: [main, pr]
+    cache:
+      key: "validator-{YYYY}-{MM}"
+      dirs: ["data"]
+    import_networks:
+    - "results/validation/networks/base_s_50___2050.nc"

.github/workflows/push-images.yaml

@@ -18,9 +18,9 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: 'Get last commit that changed pixi env'
+    - name: 'Get last commit that changed pixi env or Dockerfile'
       run: |
-        hash_last_changed=$(git log -1 --pretty=format:%H -- pixi.toml pixi.lock)
+        hash_last_changed=$(git log -1 --pretty=format:%H -- pixi.toml pixi.lock docker/dev-env/Dockerfile)
         echo "hash_last_changed=$hash_last_changed" >> $GITHUB_ENV
 
     - name: 'Login to GitHub Container Registry'

.github/workflows/update-lockfile.yaml

@@ -1,14 +1,11 @@
 name: Update locked envs
 on:
-  push:
-    paths:
-    - pixi.toml
   schedule:
   - cron: "0 8 1,16 * *" # Bi-weekly
   workflow_dispatch:
 
 jobs:
-  update-locked-environment:
+  update-lockfiles:
     if: ${{ github.ref == 'refs/heads/master' }}
     name: Update lockfiles
     runs-on: ubuntu-latest
@@ -23,37 +20,19 @@ jobs:
       with:
         pixi-version: v0.59.0
 
-    - name: Update lockfile
+    - name: Full resolve (exclude packages newer than 7 days)
+      # Exclude recently published packages to avoid pulling in broken or
+      # yanked releases before upstream has had time to react
       run: |
+        EXCLUDE_DATE=$(date -u -d '7 days ago' +%Y-%m-%dT00:00:00Z)
+        sed -i "/^\[workspace\]/a exclude-newer = \"$EXCLUDE_DATE\"" pixi.toml
         rm pixi.lock
         pixi install --all
+        git checkout pixi.toml
 
-    - name: Create new conda lock files
-      run: |
-        pixi workspace export conda-explicit-spec -e default envs
-        for f in envs/*_conda_spec*; do mv "$f" "${f/_conda_spec/.pin}"; done
-
-    - name: Create new conda environment file
-      run: |
-        pixi workspace export conda-environment -e default envs/environment.yaml -n pypsa-eur
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v7
-      with:
-        name: lockfiles
-        path: |
-          pixi.lock
-          envs/
-
-  create-pull-request:
-    needs: update-locked-environment
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v6
-    - name: Download all artifacts
-      uses: actions/download-artifact@v8
-      with:
-        name: lockfiles
+    - name: Export conda environment files
+      if: ${{ !vars.PYPSA_BOT_ID }}
+      run: pixi run sync-locks
 
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v8
@@ -64,5 +43,5 @@ jobs:
         body: |
           Automatically generated PR to update the pixi lockfile.
 
-          **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
+          **Note: Do not merge without manual test execution, unless pypsa-bot has already exported the environment files and triggered tests.**
         commit-message: "Update locked environment files for all platforms"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: CC0-1.0
-exclude: "^LICENSES|^config/schema\\.json$|^config/config\\.default\\.yaml$"
+exclude: "^LICENSES|^config/schema\\.default[\\.]*.*\\.json$|^config/config\\.default[\\.]*.*\\.yaml$"
 
 ci:
   autoupdate_schedule: quarterly

README.md

@@ -114,9 +114,9 @@ SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pyps
 SPDX-License-Identifier: CC-BY-4.0
 -->
 
-![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases)
-[![Test workflows](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg)](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml)
+[![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases)](https://github.com/PyPSA/pypsa-eur/releases)
 [![Documentation](https://readthedocs.org/projects/pypsa-eur/badge/?version=latest)](https://pypsa-eur.readthedocs.io/en/latest/?badge=latest)
+[![Test workflows](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg)](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml)
 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur)
 [![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)
 [![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042)

Snakefile

@@ -2,13 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
+import copy
 from pathlib import Path
 import yaml
 import sys
 from os.path import normpath, exists, join
 from shutil import copyfile, move, rmtree, unpack_archive
 from dotenv import load_dotenv
-from snakemake.utils import min_version
+from snakemake.utils import min_version, update_config
 
 load_dotenv()
 
@@ -33,6 +34,17 @@ validate_config(config)
 
 run = config["run"]
 scenarios = get_scenarios(run)
+
+for scenario_name, scenario_overrides in scenarios.items():
+    merged = copy.deepcopy(config)
+    update_config(merged, scenario_overrides)
+    try:
+        validate_config(merged)
+    except Exception as e:
+        raise ValueError(
+            f"Scenario '{scenario_name}' failed config validation: {e}"
+        ) from e
+
 RDIR = get_rdir(run)
 shadow_config = get_shadow(run)
 

config/config.de.yaml

@@ -425,7 +425,6 @@ solving:
     transmission_losses: true
     custom_extra_functionality: "../scripts/pypsa-de/additional_functionality.py"
     assign_all_duals: true
-    load_shedding: false
     skip_iterations: true # settings for post-discretization: false
     min_iterations: 1 # settings for post-discretization: 1
     max_iterations: 1 # settings for post-discretization: 1

config/config.default.yaml

@@ -187,7 +187,7 @@ electricity:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
   default_cutout: "europe-2013-sarah3-era5"
-  nprocesses: 16
+  nprocesses: 1
   show_progress: false
   plot_availability_matrix: false
   cutouts:
@@ -582,7 +582,7 @@ solar_thermal:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing_capacities
 existing_capacities:
   grouping_years_power:
-  - 1920
+  - 1900
   - 1950
   - 1955
   - 1960
@@ -1147,7 +1147,16 @@ adjustments:
 solving:
   options:
     clip_p_max_pu: 0.01
-    load_shedding: false
+    load_shedding:
+      enable: false
+      default_cost: 100000
+      all_carriers: true
+      carriers: {}
+    load_sinks:
+      enable: false
+      default_cost: 100000
+      all_carriers: false
+      carriers: {}
     curtailment_mode: false
     noisy_costs: true
     skip_iterations: true

config/examples/config.validation.yaml

@@ -81,7 +81,8 @@ clustering:
 
 solving:
   options:
-    load_shedding: true
+    load_shedding:
+      enable: true
     rolling_horizon: false
     horizon: 1000
     overlap: 48