add dependency-update workflow template (fixes #683)

.github/workflows/dependency-update.yml

@@ -0,0 +1,115 @@
+name: Dependency Update
+
+on:
+  schedule:
+    # Every Monday at 03:00 UTC
+    - cron: "0 3 * * 1"
+  workflow_dispatch:
+
+jobs:
+  dependency-update:
+    name: Dependency Update
+    runs-on: "ubuntu-24.04"
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Check out Repository
+        id: check-out-repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Fail if not running on the default branch
+        id: check-branch
+        if: github.ref != format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: actions/github-script@v8
+        with:
+          script: |
+            core.setFailed('Not running on the default branch. github.ref is ${{ github.ref }}')
+
+      - name: Set up Python & Poetry Environment
+        id: set-up-python-and-poetry-environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v6
+        with:
+          python-version: "3.10"
+          poetry-version: "2.3.0"
+
+      - name: Audit Dependencies
+        id: audit-dependencies
+        run: |
+          poetry run -- nox -s dependency:audit | tee vulnerabilities.json
+          LENGTH=$(jq 'length' vulnerabilities.json)
+          echo "count=$LENGTH" >> "$GITHUB_OUTPUT"
+
+      - name: Update Dependencies
+        id: update-dependencies
+        if: steps.audit-dependencies.outputs.count > 0
+        run: poetry update
+
+      - name: Check for poetry.lock Changes
+        id: check-for-poetry-lock-changes
+        if: steps.audit-dependencies.outputs.count > 0
+        run: |
+          if git diff --quiet -- poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Configure git
+        id: configure-git
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          git config --global user.email "opensource@exasol.com"
+          git config --global user.name "Automatic Dependency Updater"
+
+      - name: Create branch
+        id: create-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name="dependency-update/$(date "+%Y-%m-%d_%H:%M:%S")"
+          echo "Creating branch $branch_name"
+          git switch -C "$branch_name"
+
+      - name: Commit Changes & Push
+        id: publish-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name=$(git rev-parse --abbrev-ref HEAD)
+          git add poetry.lock
+          git commit --message "Updated poetry.lock"
+          git push --set-upstream origin "$branch_name"
+
+      - name: Create Pull Request
+        id: create-pr
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          BASE_BRANCH=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name)
+
+          PR_BODY="Automated dependency update for \`poetry.lock\`.
+          This PR was created by the dependency update workflow after running:
+          - \`poetry run -- nox -s dependency:audit\`
+          - \`poetry update\`"
+
+          PR_URL=$(gh pr create \
+                      --base "$BASE_BRANCH" \
+                      --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \
+                      --body "$PR_BODY")
+
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+
+      - name: Report New Pull Request to Slack Channel
+        id: report-pr-slack
+        if: ${{ steps.create-pr.outputs.pr_url }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: '${{ job.status }}'
+          token: '${{ secrets.GITHUB_TOKEN }}'
+          notification_title: 'Dependency update for {repo} created a Pull Request'
+          message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}'
+        env:
+          SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SECURITY_UPDATES_WEBHOOK }}'

.github/workflows/merge-gate.yml

@@ -48,14 +48,6 @@ jobs:
     permissions:
       contents: read
 
-  test-python-environment:
-    name: Test python-environment Action
-    needs:
-      - approve-run-slow-tests
-    uses: ./.github/workflows/test-python-environment.yml
-    permissions:
-      contents: read
-
   # This job ensures inputs have been executed successfully.
   allow-merge:
     name: Allow Merge
@@ -70,6 +62,7 @@ jobs:
       - test-python-environment
     # To prevent accidentally merges, this step is required. For more details
     # see: https://github.com/exasol/python-toolbox/issues/563
+    # Each job requires a step, so we added this dummy step.
     steps:
       - name: Branch Protection - failure if any ancestor failed or was cancelled
         if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}

doc/changes/unreleased.md

@@ -2,6 +2,9 @@
 
 ## Summary
 
+## Features
+
+* #756: Added `dependency-update.yml` to automate resolving vulnerabilities with a generated pull request
 ## Bugfix
 
 * #563: Fixed merge-gate to prevent auto-merges from happening when integration tests failed

doc/user_guide/features/github_workflows/index.rst

@@ -59,6 +59,9 @@ Workflows
      - Pull request and monthly
      - Executes the continuous integration suite by calling ``merge-gate.yml`` and
        ``report.yml``. See :ref:`ci_yml` for a graph of workflow calls.
+   * - ``dependency-update.yml``
+     - Weekly and manual
+     - Audits project dependencies for known vulnerabilities, updates them with Poetry when needed, and creates a pull request if the ``poetry.lock`` was changed.
    * - ``gh-pages.yml``
      - Workflow call
      - Builds the documentation and deploys it to GitHub Pages.
@@ -97,6 +100,17 @@ Workflows
 CI Actions
 ----------
 
+Dependency Update
+^^^^^^^^^^^^^^^^^
+
+The ``dependency-update.yml`` workflow is used to resolve vulnerabilities by updating our project dependencies.
+
+It can be triggered manually and is also scheduled to run weekly.
+
+The workflow first audits dependencies for known vulnerabilities. If vulnerabilities
+are detected, it updates the dependencies using Poetry. When the ``poetry.lock`` is changed,
+then it creates a pull request with the update.
+
 .. _ci_yml:
 
 Pull Request

exasol/toolbox/templates/github/workflows/dependency-update.yml

@@ -0,0 +1,126 @@
+name: Dependency Update
+
+on:
+  schedule:
+    # Every Monday at 03:00 UTC
+    - cron: "0 3 * * 1"
+  workflow_dispatch:
+
+jobs:
+  dependency-update:
+    name: Dependency Update
+    runs-on: "(( os_version ))"
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Check out Repository
+        id: check-out-repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Fail if not running on the default branch
+        id: check-branch
+        if: github.ref != format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: actions/github-script@v8
+        with:
+          script: |
+            core.setFailed('Not running on the default branch. github.ref is ${{ github.ref }}')
+
+      - name: Set up Python & Poetry Environment
+        id: set-up-python-and-poetry-environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v6
+        with:
+          python-version: "(( minimum_python_version ))"
+          poetry-version: "(( dependency_manager_version ))"
+
+      - name: Audit Dependencies
+        id: audit-dependencies
+        run: |
+          poetry run -- nox -s dependency:audit | tee vulnerabilities.json
+          LENGTH=$(jq 'length' vulnerabilities.json)
+          echo "count=$LENGTH" >> "$GITHUB_OUTPUT"
+
+      - name: Update Dependencies
+        id: update-dependencies
+        if: steps.audit-dependencies.outputs.count > 0
+        run: poetry update
+
+      - name: Check for poetry.lock Changes
+        id: check-for-poetry-lock-changes
+        if: steps.audit-dependencies.outputs.count > 0
+        run: |
+          if git diff --quiet -- poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Configure git
+        id: configure-git
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          git config --global user.email "opensource@exasol.com"
+          git config --global user.name "Automatic Dependency Updater"
+
+      - name: Create branch
+        id: create-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name="dependency-update/$(date "+%Y-%m-%d_%H:%M:%S")"
+          echo "Creating branch $branch_name"
+          git switch -C "$branch_name"
+
+      - name: Commit Changes & Push
+        id: publish-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name=$(git rev-parse --abbrev-ref HEAD)
+          git add poetry.lock
+          git commit --message "Updated poetry.lock"
+          git push --set-upstream origin "$branch_name"
+
+      - name: Create Pull Request
+        id: create-pr
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          BASE_BRANCH=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name)
+
+          PR_BODY="Automated dependency update for \`poetry.lock\`.
+          This PR was created by the dependency update workflow after running:
+          - \`poetry run -- nox -s dependency:audit\`
+          - \`poetry update\`"
+
+          PR_URL=$(gh pr create \
+                      --base "$BASE_BRANCH" \
+                      --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \
+                      --body "$PR_BODY")
+
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+
+          pr_url=$(gh pr create \
+            --base "$BASE_BRANCH" \
+            --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \
+            --body "Automated dependency update for \`poetry.lock\`.)
+          echo "pr_url=$pr_url" >> "$GITHUB_OUTPUT"
+          This PR was created by the dependency update workflow after running:
+          - \`poetry run -- nox -s dependency:audit\`
+          - \`poetry update\`"
+      - name: Report New Pull Request to Slack Channel
+        id: report-pr-slack
+        if: ${{ steps.create-pr.outputs.pr_url }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: '${{ job.status }}'
+          token: '${{ secrets.GITHUB_TOKEN }}'
+          notification_title: 'Dependency update for {repo} created a Pull Request'
+          status: '${{ job.status }}',
+          token: '${{ secrets.GITHUB_TOKEN }}',
+          notification_title: 'Dependency update for {repo} created a Pull Request',
+          message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}'
+        env:
+          SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SECURITY_UPDATES_WEBHOOK }}'

exasol/toolbox/util/dependencies/audit.py

@@ -260,7 +260,7 @@ def load_from_pip_audit(cls, working_directory: Path) -> Vulnerabilities:
 
         vulnerabilities = []
         for entry in audit_dict["dependencies"]:
-            for vuln_entry in entry["vulns"]:
+            for vuln_entry in entry.get("vulns", []):
                 vulnerabilities.append(
                     Vulnerability.from_audit_entry(
                         package_name=entry["name"],

test/integration/project-template/nox_test.py

@@ -83,4 +83,4 @@ def test_install_github_workflows(self, poetry_path, run_command):
         assert output.returncode == 0
 
         file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines()
-        assert len(file_list) == 13
+        assert len(file_list) == 14

test/unit/nox/_workflow_test.py

@@ -35,7 +35,7 @@ class TestGenerateWorkflow:
     @staticmethod
     @pytest.mark.parametrize(
         "nox_session_runner_posargs, expected_count",
-        [(ALL, 13), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]],
+        [(ALL, 14), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]],
         indirect=["nox_session_runner_posargs"],
     )
     def test_works_as_expected(

test/unit/util/dependencies/audit_test.py

@@ -240,7 +240,13 @@ class TestVulnerabilities:
     @staticmethod
     def test_with_no_vulnerabilities():
         pip_audit_dict = {
-            "dependencies": [{"name": "alabaster", "version": "0.7.16", "vulns": []}]
+            "dependencies": [
+                {
+                    "name": "exasol-toolbox",
+                    "skip_reason": "Dependency not found on PyPI and could not be audited: exasol-toolbox (7.0.0)",
+                },
+                {"name": "alabaster", "version": "0.7.16", "vulns": []},
+            ]
         }
         pip_audit_json = json.dumps(pip_audit_dict)
 

test/unit/util/workflows/templates_test.py

@@ -11,6 +11,7 @@ def test_get_workflow_templates(project_config):
         "check-release-tag",
         "checks",
         "ci",
+        "dependency-update",
         "gh-pages",
         "matrix-all",
         "matrix-exasol",