diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..ce543c356a7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,20 @@ +* text=auto eol=lf + +# Windows batch scripts: keep CRLF on checkout so they run on Windows +*.bat text eol=crlf +*.cmd text eol=crlf + +# Binary files: do not normalize or diff +*.jar binary +*.zip binary +*.png binary +*.jpg binary +*.jpeg binary +*.ico binary +*.gif binary +*.exe binary +*.pdf binary +*.woff binary +*.woff2 binary +*.hprof binary +*.zst binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 1af912a7328..3d70cc4dc88 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -19,19 +19,24 @@ A clear and concise description of what you expected to happen. **To Reproduce** > The *best* way is to provide an [SSCCE (Short, Self Contained, Correct (Compilable), Example)](http://sscce.org/). One type of SSCCE could be a small test which reproduces the issue and can be run without dependencies. The [XQSuite - Annotation-based Test Framework for XQuery](http://exist-db.org/exist/apps/doc/xqsuite.xml) makes it very easy for you to create tests. These tests can be executed from the [eXide editor](http://exist-db.org/exist/apps/eXide/index.html) (XQuery - Run as Test) -```Xquery +```xquery xquery version "3.1"; module namespace t="http://exist-db.org/xquery/test"; declare namespace test="http://exist-db.org/xquery/xqsuite"; +declare namespace xmldb="http://exist-db.org/xquery/xmldb"; - +(:~ + : Replace with minimal data that reproduces your issue. + :) declare variable $t:XML := document { }; - +(:~ + : Replace index config if needed for the reported bug. + :) declare variable $t:xconf := @@ -39,35 +44,49 @@ declare variable $t:xconf := ; - +(:~ + : Use a unique collection name to avoid collisions with other test runs. + : Keep tests self-contained and avoid mutating shared collections. + :) +declare variable $t:COLL := "/db/test-" || translate(string(current-dateTime()), "-:TZ.+", ""); +declare variable $t:CONF_COLL := "/db/system/config/db/" || substring-after($t:COLL, "/db/"); + +(:~ + : setUp should be idempotent and safe to run repeatedly. + :) declare %test:setUp function t:setup() { - let $testCol := xmldb:create-collection("/db", "test") - let $indexCol := xmldb:create-collection("/db/system/config/db", "test") + let $_ := xmldb:create-collection("/db/system", "config") + let $_ := xmldb:create-collection("/db/system/config", "db") + let $_ := xmldb:create-collection("/db", substring-after($t:COLL, "/db/")) + let $_ := xmldb:create-collection("/db/system/config/db", substring-after($t:COLL, "/db/")) return ( - xmldb:store("/db/test", "test.xml", $t:XML), - xmldb:store("/db/system/config/db/test", "collection.xconf", $t:xconf), - xmldb:reindex("/db/test") + xmldb:store($t:COLL, "test.xml", $t:XML), + xmldb:store($t:CONF_COLL, "collection.xconf", $t:xconf), + xmldb:reindex($t:COLL) ) }; +(:~ + : tearDown should be idempotent: guard removals so partial setup does not fail cleanup. + :) declare %test:tearDown function t:tearDown() { - xmldb:remove("/db/test"), - xmldb:remove("/db/system/config/db/test") + if (xmldb:collection-available($t:COLL)) then xmldb:remove($t:COLL) else (), + if (xmldb:collection-available($t:CONF_COLL)) then xmldb:remove($t:CONF_COLL) else () }; -<-- Adjust to your reported issue --> +(:~ + : Adjust this test body to your reported issue. + : Prefer exact assertions (assertEquals/assertEqualsPermutation) over broad assertTrue when possible. + :) declare - %test:assertTrue + %test:assertEquals(1) function t:test() { - let $test-data := collection('/db/test') - for $result in $test-data//root - return - count($result) eq 1 + count(collection($t:COLL)//root) }; ``` @@ -83,9 +102,9 @@ If applicable, add screenshots to help explain your problem. **Context (please always complete the following information)** One option is to use [xst](https://www.npmjs.com/package/@existdb/xst), and copy and paste the output produced by running `xst info` here:** - - Build: [eXist-6.1.0] - - Java: [1.8.0_352] - - OS: [Mac OS X 12.6.2] + - Build: [eXist-6.4.1] + - Java: [11.0.30+7] + - OS: [Mac OS X 26.3.1] **Additional context** - How is eXist-db installed? [e.g. JAR installer, DMG, … ] diff --git a/.github/actions/install-mvnd/action.yml b/.github/actions/install-mvnd/action.yml index a0dc3bdf1ff..cc64f79d9bb 100644 --- a/.github/actions/install-mvnd/action.yml +++ b/.github/actions/install-mvnd/action.yml @@ -4,7 +4,7 @@ inputs: version: description: 'The version of the Maven Daemon to install' required: true - default: '0.9.0' + default: '1.0.6' file-version-suffix: description: 'A suffix to append to the version of the download file of Maven Daemon to install' required: false @@ -48,7 +48,7 @@ runs: - name: Cache mvnd if: inputs.cache == 'true' id: cache-mvnd - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip @@ -61,11 +61,14 @@ runs: run: | curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip.sha256 - - name: Install sha256sum (macOS) - if: ${{ runner.os == 'macOS' }} - shell: bash - run: brew install coreutils + # see #5842 + # brew install mvndaemon/homebrew-mvnd/mvnd@1 + # - name: Install coreutils (macOS) + # if: ${{ runner.os == 'macOS' }} + # shell: bash + # run: brew install coreutils - name: Verify mvnd sha256 checksum + if: ${{ runner.os != 'macOS' }} shell: bash run: echo "$(cat ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256) ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip" | sha256sum --check - name: Unzip mvnd diff --git a/.github/actions/maven-cache/action.yml b/.github/actions/maven-cache/action.yml new file mode 100644 index 00000000000..27e100b551d --- /dev/null +++ b/.github/actions/maven-cache/action.yml @@ -0,0 +1,21 @@ +# Shared Maven cache: restore only. Caller adds a separate Save step at end of job when appropriate (see plans/caching.md). +name: Maven cache +description: Restore Maven repository cache (one cache per branch, PRs read-only). Add actions/cache/save at end of job for the single job that should persist. +inputs: + java-version: + description: 'Java version for cache key (e.g. matrix.java-version); omit for jobs with single JDK' + required: false + default: '' +runs: + using: 'composite' + steps: + - name: Restore Maven cache + uses: actions/cache/restore@v5 + with: + path: ~/.m2 + key: maven-${{ github.event.pull_request.base.ref || github.ref_name }}-${{ inputs.java-version && format('java{0}-', inputs.java-version) || '' }}${{ hashFiles('**/pom.xml') }} + enableCrossOsArchive: true + restore-keys: | + maven-${{ github.event.pull_request.base.ref || github.ref_name }}-${{ inputs.java-version && format('java{0}-', inputs.java-version) || '' }} + maven-${{ github.event.pull_request.base.ref || github.ref_name }}- + maven- diff --git a/.github/actions/maven-github-settings/action.yml b/.github/actions/maven-github-settings/action.yml new file mode 100644 index 00000000000..25264d54d1a --- /dev/null +++ b/.github/actions/maven-github-settings/action.yml @@ -0,0 +1,61 @@ +# Creates Maven settings.xml with auth for the eXist-db org's GitHub Packages repos +# and optionally for Sonatype Central Portal publishing. +# Required for resolving artifacts from maven.pkg.github.com/eXist-db/{exist, exist-xqts-runner, jackrabbit-webdav-jakarta}. +name: Maven GitHub Packages settings +description: Create settings.xml with GitHub Packages and (optionally) Sonatype Central Portal servers +inputs: + token: + description: 'GitHub token for package authentication' + required: true + central-token-username: + description: 'Sonatype Central Portal user token username (release jobs only)' + required: false + default: '' + central-token-password: + description: 'Sonatype Central Portal user token password (release jobs only)' + required: false + default: '' +runs: + using: 'composite' + steps: + - name: Create Maven settings for GitHub Packages + shell: bash + env: + CENTRAL_TOKEN_USERNAME: ${{ inputs.central-token-username }} + CENTRAL_TOKEN_PASSWORD: ${{ inputs.central-token-password }} + run: | + mkdir -p ~/.m2 + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + + # Build optional Central Portal server block + CENTRAL_SERVER="" + if [ -n "$CENTRAL_TOKEN_USERNAME" ]; then + CENTRAL_SERVER=" + + central + ${CENTRAL_TOKEN_USERNAME} + ${CENTRAL_TOKEN_PASSWORD} + " + fi + + cat > ~/.m2/settings.xml << EOF + + + + github + ${OWNER} + ${{ inputs.token }} + + + github-xqts-runner + ${OWNER} + ${{ inputs.token }} + + + github-jackrabbit-webdav-jakarta + ${OWNER} + ${{ inputs.token }} + ${CENTRAL_SERVER} + + + EOF diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9f75c20a307..a048222f89b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,35 +4,120 @@ updates: directory: "/" schedule: interval: "weekly" + groups: + actions: + patterns: + - "*" + - package-ecosystem: "maven" directory: "/" schedule: interval: "daily" time: "03:00" open-pull-requests-limit: 10 - ignore: - - dependency-name: org.apache.lucene:lucene-analyzers-common - versions: - - ">= 8.a, < 9" - - dependency-name: org.apache.lucene:lucene-analyzers-icu - versions: - - ">= 8.a, < 9" - - dependency-name: org.apache.lucene:lucene-core - versions: - - ">= 8.a, < 9" - - dependency-name: org.apache.lucene:lucene-facet - versions: - - ">= 8.a, < 9" - - dependency-name: org.apache.lucene:lucene-queries - versions: - - ">= 8.a, < 9" - - dependency-name: org.apache.lucene:lucene-queryparser - versions: - - ">= 8.a, < 9" - - dependency-name: com.ibm.icu:icu4j - versions: - - "69.1" - - dependency-name: com.mycila:license-maven-plugin - versions: - - "4.0" - - "4.0.rc2" + groups: + # Major bumps stay separate for QA; patch/minor trains bundle here. + jetty: + patterns: + - "org.eclipse.jetty:*" + - "org.eclipse.jetty.websocket:*" + update-types: + - patch + - minor + lucene: + patterns: + - "org.apache.lucene:*" + update-types: + - patch + - minor + logging: + patterns: + - "org.apache.logging.log4j:*" + - "org.slf4j:*" + update-types: + - patch + - minor + junit-jupiter-bom: + patterns: + - "org.junit:junit-bom" + - "org.junit.jupiter:*" + - "org.junit.platform:*" + - "org.junit.vintage:*" + update-types: + - patch + - minor + junit4-test-support: + patterns: + - "junit:junit" + - "org.hamcrest:*" + - "org.assertj:*" + - "org.easymock:*" + - "org.objenesis:*" + - "org.awaitility:*" + - "com.googlecode.junit-toolbox:*" + - "org.xmlunit:*" + update-types: + - patch + - minor + httpcomponents: + patterns: + - "org.apache.httpcomponents:*" + update-types: + - patch + - minor + jaxb-angus: + patterns: + - "jakarta.xml.bind:*" + - "org.glassfish.jaxb:*" + - "org.eclipse.angus:*" + update-types: + - patch + - minor + apache-commons: + patterns: + - "commons-codec:*" + - "commons-io:*" + - "org.apache.commons:*" + update-types: + - patch + - minor + jackson: + patterns: + - "com.fasterxml.jackson.core:*" + update-types: + - patch + - minor + jline: + patterns: + - "org.jline:*" + update-types: + - patch + - minor + exificient: + patterns: + - "com.siemens.ct.exi:*" + update-types: + - patch + - minor + exquery: + patterns: + - "org.exquery:*" + update-types: + - patch + - minor + maven-plugins-apache: + patterns: + - "org.apache.maven.plugins:*" + update-types: + - patch + - minor + maven-plugins-tools: + patterns: + - "org.codehaus.mojo:*" + - "org.jacoco:*" + - "org.owasp:*" + - "io.github.git-commit-id:*" + - "software.xdev:*" + update-types: + - patch + - minor diff --git a/.github/scripts/prepare-governance-context.sh b/.github/scripts/prepare-governance-context.sh new file mode 100755 index 00000000000..43b48759644 --- /dev/null +++ b/.github/scripts/prepare-governance-context.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Pure git plumbing for schema governance — everything else (pairing, +# version comparison, the GitHub annotations, failing the build) happens in +# a single Saxon XSLT 2.0 transform (schema/governance.xsl) driven by +# `mvn xml:transform@schema-governance`. This script's only job is to put +# the git state that transform needs onto disk as plain files/XML, so the +# stylesheet never has to shell out itself. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +WORKSPACE="${1:?workspace required}" +OUT="${2:-${WORKSPACE}/target/governance}" +mkdir -p "${OUT}/base" + +if [[ -n "${GITHUB_BASE_REF:-}" ]]; then + git fetch --depth=1 origin "${GITHUB_BASE_REF}" 2>/dev/null || true + BASE="$(git merge-base HEAD "origin/${GITHUB_BASE_REF}")" +elif [[ -n "${GITHUB_EVENT_BEFORE:-}" && "${GITHUB_EVENT_BEFORE}" != "0000000000000000000000000000000000000000" ]]; then + BASE="${GITHUB_EVENT_BEFORE}" +else + BASE="$(git rev-parse HEAD~1 2>/dev/null || git rev-parse HEAD)" +fi + +# Every tracked schema/*.xsd's content as it existed at BASE, one file per +# schema named after its basename. A missing file at $OUT/base/ means +# "didn't exist at BASE" (new schema) — governance.xsl checks for that with +# doc-available() rather than this script trying to distinguish "new file" +# from "tool failure" itself. +git ls-tree -r --name-only HEAD -- schema | grep '\.xsd$' | while read -r f; do + out="${OUT}/base/$(basename "${f}")" + git show "${BASE}:${f}" > "${out}" 2>/dev/null || rm -f "${out}" +done + +# One path per line; governance.xsl reads this with unparsed-text() + tokenize(), +# so no XML-escaping of path characters is needed anywhere in this pipeline. +git diff --name-only "${BASE}" -- \ + schema \ + exist-distribution/src/main/config \ + exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml \ + exist-core/src/main/resources/org/exist/util/mime-types.xml \ + exist-core/src/main/java/org/exist/util/SchemaVersion.java \ + > "${OUT}/changed.txt" 2>/dev/null || true + +cat > "${OUT}/context.xml" < + +EOF + +echo "Governance context: ${OUT}/context.xml (base ${BASE})" diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml new file mode 100644 index 00000000000..71e8e49d631 --- /dev/null +++ b/.github/workflows/ci-benchmarks.yml @@ -0,0 +1,100 @@ +name: JMH Benchmarks +on: + workflow_dispatch: + schedule: + # Weekly, Sunday 04:00 UTC - distinct from ci-container.yml (daily 06:00) + # and prethink.yml (Monday 06:00) so runners aren't all queued together. + - cron: '0 4 * * 0' + +permissions: + contents: read # default; the benchmarks job declares write explicitly + +# Only one publish to gh-pages at a time; a manual dispatch overlapping the +# scheduled run would otherwise race pushing the benchmark-data commit. +concurrency: + group: jmh-benchmarks-gh-pages + cancel-in-progress: false + +env: + DEV_JDK: '21' + # Kept small enough that both benchmark suites fit in the job timeout on a + # shared GitHub-hosted runner; not intended to produce publishable numbers, + # only a consistent trend line. See exist-core-jmh/README.md and + # exist-indexes-jmh/README.md for slower, more precise local invocations. + JMH_ARGS: '-wi 2 -i 5 -f 1' + +jobs: + benchmarks: + name: Run JMH benchmarks + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: write # github-action-benchmark pushes the data series to gh-pages + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + + - uses: ./.github/actions/maven-cache + + - name: Build exist-core-jmh + run: > + mvn -B install -pl exist-core-jmh -am -DskipTests + -Ddependency-check.skip=true -Ddocker=false + + - name: Run exist-core-jmh benchmarks + run: > + java -jar exist-core-jmh/target/exist-core-jmh-*-benchmarks.jar + -rf json -rff exist-core-jmh/target/jmh-result.json + ${{ env.JMH_ARGS }} + + - name: Track exist-core-jmh results on gh-pages + if: github.repository == 'eXist-db/exist' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'exist-core-jmh' + tool: 'jmh' + output-file-path: exist-core-jmh/target/jmh-result.json + benchmark-data-dir-path: dev/bench/core + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Report-only: flag regressions, never fail the run. + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: false + summary-always: true + + - name: Build exist-indexes-jmh + run: > + mvn -B install -pl exist-indexes-jmh -am -DskipTests + -Ddependency-check.skip=true -Ddocker=false + + - name: Run exist-indexes-jmh benchmarks + run: > + mvn -B exec:exec -pl exist-indexes-jmh + -Dbenchmark.args="-rf json -rff target/jmh-result.json ${{ env.JMH_ARGS }}" + + - name: Track exist-indexes-jmh results on gh-pages + if: github.repository == 'eXist-db/exist' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'exist-indexes-jmh' + tool: 'jmh' + output-file-path: exist-indexes-jmh/target/jmh-result.json + benchmark-data-dir-path: dev/bench/indexes + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: false + summary-always: true + + - name: Save Maven cache + if: github.event_name == 'schedule' + uses: actions/cache/save@v6 + with: + path: ~/.m2 + key: maven-${{ github.ref_name }}-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/ci-container.yml b/.github/workflows/ci-container.yml index 24ae8af927e..dfb47d0fbad 100644 --- a/.github/workflows/ci-container.yml +++ b/.github/workflows/ci-container.yml @@ -1,73 +1,237 @@ name: Publish Container -on: [push, pull_request] +on: + push: + branches: + - develop + - master + tags: + - 'eXist-*' + - '!eXist-*-oidc-smoke' + pull_request: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + +# Prevent race for two pushes to develop on the shared tag. +# Build B could overwrite A's staging image between A's push and A's promote. +# Do not cancel builds in progress and let it finish and publish before the next starts. +concurrency: + group: ci-container-${{ github.ref }} + cancel-in-progress: false + jobs: build: name: Test and Publish Container Images runs-on: ubuntu-latest - # NOTE (DP): Publish on develop and master, test on PRs against these - # TODO(DP) Reinstate CRONed release builds to update stock apps regularly - if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' || github.base_ref == 'develop' || github.base_ref == 'master' + timeout-minutes: 90 + # Publish on develop (latest) and eXist-* tags (versioned + release); test on PRs against these branches. + if: > + github.ref == 'refs/heads/develop' || + github.ref == 'refs/heads/master' || + startsWith(github.ref, 'refs/tags/eXist-') || + github.base_ref == 'develop' || + github.base_ref == 'master' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 1 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: '21' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/amd64,linux/arm64 - - name: Make buildkit default - uses: docker/setup-buildx-action@v3 - id: buildx + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 with: - install: true - - name: Cache Maven packages - uses: actions/cache@v4 + platforms: linux/amd64,linux/arm64 + - name: Verify containerd snapshotter is active + run: docker info -f '{{.DriverStatus}}' + - name: Make BuildKit default + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - uses: ./.github/actions/maven-cache + - uses: ./.github/actions/maven-github-settings with: - path: ~/.m2 - key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: deploy-${{ runner.os }}-maven + token: ${{ secrets.GITHUB_TOKEN }} - name: Install bats run: sudo apt-get install bats - # Hack around #5450 - - name: pull base image + + # Decide, once, whether this run publishes and under which channel. + # publish=true only on the upstream repo for develop (latest) and eXist-* tags (release); + # every other run (PRs, forks) just builds a local amd64 image and tests it. + - name: Determine publish target + id: meta + run: | + publish=false + channel="" + version="" + if [[ "${{ github.repository }}" == "eXist-db/exist" ]]; then + if [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then + publish=true + channel=latest + elif [[ "${{ github.ref }}" == refs/tags/eXist-* ]]; then + publish=true + channel=release + version="${{ github.ref_name }}" + version="${version#eXist-}" + fi + fi + echo "publish=$publish" >> "$GITHUB_OUTPUT" + echo "channel=$channel" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "Resolved: publish=$publish channel=$channel version=$version" + + # Assemble the distribution + Dockerfile context and build the local + # single-arch amd64 test image (fabric8 `build test image` execution). + - name: Build distribution and local test image + timeout-minutes: 35 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + REVISION_ARG="" + if [[ -n "${{ steps.meta.outputs.version }}" ]]; then + REVISION_ARG="-Drevision=${{ steps.meta.outputs.version }}" + fi + mvn -V -B --no-transfer-progress \ + -Pdocker,skip-build-dist-archives \ + -DskipTests \ + $REVISION_ARG \ + clean package + + # The exact version string used for the immutable version tag at promote time. + - name: Resolve project version + id: ver run: | - docker pull --platform linux/amd64 --platform linux/arm64 gcr.io/distroless/java21-debian12:latest - - name: Build images - run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives clean package + v="${{ steps.meta.outputs.version }}" + if [[ -z "$v" ]]; then + v=$(mvn -q -N \ + help:evaluate -Dexpression=project.version -DforceStdout) + fi + echo "version=$v" >> "$GITHUB_OUTPUT" + echo "Project version: $v" + - name: Check local images run: docker image ls - name: Check license headers - run: mvn license:check - working-directory: exist-docker + run: mvn --no-transfer-progress license:check + working-directory: exist-docker + + # --- Non-publish (PR/fork): test the locally-built amd64 image --- + - name: Tag local image for tests + if: steps.meta.outputs.publish != 'true' + run: docker tag existdb/existdb:latest existdb/existdb:ci + + # --- Publish: build & push the multi-arch, attested staging images --- + # Pushed as -staging (+ -debug-staging) so nothing + # un-suffixed reaches Docker Hub until the tests below pass. + # + # The build context and both Dockerfiles are produced by the Maven + # `package` above (assembly + shade + resource filtering resolve the + # OCI LABELs and the version-stamped context dir). docker/build-push-action + # only builds & pushes that pre-assembled context — it does not re-run Maven. + - name: Log in to Docker Hub + if: steps.meta.outputs.publish == 'true' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build & push staging image (main) + if: steps.meta.outputs.publish == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: exist-docker/target/exist-docker-${{ steps.ver.outputs.version }}-docker-dir + file: exist-docker/target/classes/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + provenance: mode=max + sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: existdb/existdb:${{ steps.meta.outputs.channel }}-staging + + - name: Build & push staging image (DEBUG) + if: steps.meta.outputs.publish == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: exist-docker/target/exist-docker-${{ steps.ver.outputs.version }}-docker-dir + file: exist-docker/target/classes/Dockerfile-DEBUG + platforms: linux/amd64,linux/arm64 + push: true + provenance: mode=max + sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: existdb/existdb:${{ steps.meta.outputs.channel }}-debug-staging + + - name: Pull & tag staging image for tests + if: steps.meta.outputs.publish == 'true' + run: | + docker pull existdb/existdb:${{ steps.meta.outputs.channel }}-staging + docker tag existdb/existdb:${{ steps.meta.outputs.channel }}-staging existdb/existdb:ci + + # --- Common: run the bats suite against the amd64 image under test (:ci) --- - name: Start exist-ci container run: | - docker run -dit -p 8080:8080 --name exist-ci --rm existdb/existdb:latest + docker run -dit -p 8080:8080 --name exist-ci --rm existdb/existdb:ci sleep 35s - name: Run tests run: bats --tap exist-docker/src/test/bats/*.bats - # NOTE (DP): When on master push release, when on develop push latest: Version is included automatically - # TODO (DP): Confirm that releases triggered from maven publish images with the non SNAPSHOT version - - name: Publish latest images - if: github.ref == 'refs/heads/develop' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: mvn -q -Ddocker.tag=latest -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push - working-directory: ./exist-docker - - name: Publish release images - if: github.ref == 'refs/heads/master' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: mvn -q -Ddocker.tag=release -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push - working-directory: ./exist-docker + + - name: debug logs + if: failure() + run: docker logs exist-ci | grep 'ERROR' + + - name: Capture container logs on failure + if: failure() + run: | + echo "Capturing container stdout/stderr..." + docker logs exist-ci > exist.log 2>&1 || true + + - name: Upload logs artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: exist-core-failed-log + path: exist.log + + # --- Test: Dropped testing of an image running in emulation. The outcome is meaningless at best according to DP + + # --- Publish: promote the tested staging digest to the public tags --- + # imagetools copies the whole OCI index (both architectures + attestation manifests) + # server-side, no rebuild. The `-staging` + # tag is deliberately left in place as the inspectable "last CI build". + # (Docker Hub login already happened before the staging build & push above.) + - name: Promote staging images to public tags + if: steps.meta.outputs.publish == 'true' + run: | + CH="${{ steps.meta.outputs.channel }}" + V="${{ steps.ver.outputs.version }}" + docker buildx imagetools create \ + -t existdb/existdb:${CH} \ + -t existdb/existdb:${V} \ + existdb/existdb:${CH}-staging + docker buildx imagetools create \ + -t existdb/existdb:debug \ + -t existdb/existdb:${V}-DEBUG \ + existdb/existdb:${CH}-debug-staging + + # --- Publish: verify the promoted image carries both arches + attestations --- + - name: Verify published manifest (arch + attestations) + if: steps.meta.outputs.publish == 'true' + run: | + CH="${{ steps.meta.outputs.channel }}" + for ref in "existdb/existdb:${CH}" "existdb/existdb:debug"; do + echo "== $ref ==" + raw=$(docker buildx imagetools inspect "$ref" --raw) + archs=$(echo "$raw" | jq -r '.manifests[].platform.architecture' | sort -u | tr '\n' ' ') + for a in amd64 arm64; do + echo "$archs" | grep -qw "$a" || { echo "::error::$ref is missing arch $a (found: $archs)"; exit 1; } + done + att=$(echo "$raw" | jq '[.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest")] | length') + [[ "$att" -ge 1 ]] || { echo "::error::$ref has no attestation manifests (SBOM/provenance missing)"; exit 1; } + echo "OK: arches=[$archs] attestation-manifests=$att" + done + # NOTE (DP): This is for debugging, publishes an experimental image from inside PRs against develop # - name: Publish experimental images # if: github.base_ref == 'develop' @@ -76,5 +240,4 @@ jobs: # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} # run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push - # working-directory: ./exist-docker - + # working-directory: ./exist-docker diff --git a/.github/workflows/ci-oidc-smoke.yml b/.github/workflows/ci-oidc-smoke.yml new file mode 100644 index 00000000000..8d8285aeb48 --- /dev/null +++ b/.github/workflows/ci-oidc-smoke.yml @@ -0,0 +1,213 @@ +name: Release signing smoke (Azure OIDC + Key Vault JAR sign) + +# Exercises the Azure federated credential, Key Vault JCA jarsigner path, +# and AzureSignTool Authenticode path without dispatching a full release. +# Triggered by tags matching `eXist-*-oidc-smoke` — a pattern that +# ci-release.yml does NOT match, so the full release pipeline never starts. +# +# Validates: +# 1. Federated credential expression accepts eXist-* tag subject (OIDC login). +# 2. App Registration can read the signing cert from Key Vault (az CLI check). +# 3. Key Vault JCA provider initialises + signs a JAR (jarsigner path). +# 4. TSA timestamping reachable (Sectigo URL responds). +# 5. AzureSignTool can sign a .exe via DefaultAzureCredential / AzureCliCredential. +# +# Usage: +# git tag -a eXist-99.0.0-oidc-smoke -m "release signing smoke" +# git push upstream eXist-99.0.0-oidc-smoke +# # watch Actions → Release signing smoke (~5 min) +# git push upstream :eXist-99.0.0-oidc-smoke # cleanup +# +# Runs on windows-latest to mirror ci-release.yml build-windows exactly, +# including the shell (bash via Git Bash) and Python path behaviour. + +on: + push: + tags: + - 'eXist-*-oidc-smoke' + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + smoke: + name: OIDC login + Key Vault JAR + .exe sign + runs-on: windows-latest + timeout-minutes: 15 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '21' + + - name: Azure login (OIDC) + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Confirm token + tenant + run: | + az account show --query "{tenantId:tenantId,user:user.name}" -o table + echo "OIDC login succeeded — federated credential expression matches eXist-* tag subject." + + - name: Check Key Vault cert accessible via az CLI + env: + AZURE_KEYVAULT_URI: ${{ vars.AZURE_KEYVAULT_URI }} + AZURE_KEYVAULT_CERT_NAME: ${{ vars.AZURE_KEYVAULT_CERT_NAME }} + run: | + # Distinct from signing permission: this checks the App Registration + # can READ the certificate (needs Key Vault Certificate User or higher). + # A failure here means an RBAC problem, not a code problem. + az keyvault certificate show \ + --id "${AZURE_KEYVAULT_URI}certificates/${AZURE_KEYVAULT_CERT_NAME}" \ + --query "{name:name,enabled:attributes.enabled,thumbprint:x509Thumbprint}" \ + -o table + echo "Key Vault cert is accessible — Certificate read access confirmed." + + - name: Build throwaway JAR to sign + run: | + mkdir -p /tmp/smoke && cd /tmp/smoke + echo "smoke canary" > canary.txt + jar cf smoke.jar canary.txt + ls -la smoke.jar + + - name: Start KV token mock IMDS + run: | + # azure-security-keyvault-jca 2.10.0 AccessTokenUtil has no workload + # identity path — only App Service (MSI_SECRET), Container App + # (IDENTITY_ENDPOINT), and bare IMDS (169.254.169.254, unavailable + # on GitHub runners). We mock the Container App endpoint using a real + # KV token from the az CLI (authenticated via azure/login above). + # + # Token is passed via env var — NOT written to /tmp — because native + # Windows Python open('/tmp/...') fails (no MSYS2 path translation). + KV_TOKEN=$(az account get-access-token \ + --resource https://vault.azure.net --query accessToken -o tsv) + [ -n "$KV_TOKEN" ] || { echo "Failed to get Key Vault access token"; exit 1; } + export KV_TOKEN_VALUE="$KV_TOKEN" + export KV_TOKEN_EXPIRES=$(python3 -c "import time; print(int(time.time()) + 3600)") + python3 -c " + import http.server, os, json + token_bytes = json.dumps({ + 'access_token': os.environ['KV_TOKEN_VALUE'], + 'expires_on': os.environ['KV_TOKEN_EXPIRES'], + 'token_type': 'Bearer', + 'resource': 'https://vault.azure.net' + }).encode() + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type','application/json') + self.end_headers() + self.wfile.write(token_bytes) + def log_message(self,*a): pass + http.server.HTTPServer(('127.0.0.1',19999),H).serve_forever() + " & + echo "MOCK_IMDS_PID=$!" >> "$GITHUB_ENV" + echo "IDENTITY_ENDPOINT=http://127.0.0.1:19999/" >> "$GITHUB_ENV" + echo "IDENTITY_HEADER=mock-header" >> "$GITHUB_ENV" + sleep 1 + curl -sf http://127.0.0.1:19999/ \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print('Mock IMDS OK, token_type:', d.get('token_type'))" + + - name: Write signing cert chain + env: + EXISTDB_WIN_SIGNING_CERTCHAIN: ${{ secrets.EXISTDB_WIN_SIGNING_CERTCHAIN }} + run: | + # Cert expires 2027-12-03 — update EXISTDB_WIN_SIGNING_CERTCHAIN secret and + # exist-db-win-signing in Key Vault when renewing. + [ -n "$EXISTDB_WIN_SIGNING_CERTCHAIN" ] || { echo "EXISTDB_WIN_SIGNING_CERTCHAIN secret not set"; exit 1; } + echo "$EXISTDB_WIN_SIGNING_CERTCHAIN" | base64 --decode > /tmp/certchain.pem + chmod 600 /tmp/certchain.pem + + - name: Sign smoke JAR with Azure Key Vault JCA + env: + AZURE_KEYVAULT_URI: ${{ vars.AZURE_KEYVAULT_URI }} + AZURE_KEYVAULT_CERT_NAME: ${{ vars.AZURE_KEYVAULT_CERT_NAME }} + # IDENTITY_ENDPOINT + IDENTITY_HEADER flow in via GITHUB_ENV from prior step + run: | + # cygpath -w converts MSYS2 paths to Windows-native paths for the JVM. + # On Linux/macOS cygpath is absent so we fall back to the path as-is. + native() { command -v cygpath &>/dev/null && cygpath -w "$1" || echo "$1"; } + + KV_JCA_JAR="$HOME/.m2/repository/com/azure/azure-security-keyvault-jca/2.10.0/azure-security-keyvault-jca-2.10.0.jar" + if [ ! -f "$KV_JCA_JAR" ]; then + mvn -q dependency:get \ + -Dartifact=com.azure:azure-security-keyvault-jca:2.10.0:jar + fi + + echo "=== Path diagnostics ===" + echo "KV_JCA_JAR (bash): $KV_JCA_JAR" + echo "KV_JCA_JAR (native): $(native "$KV_JCA_JAR")" + echo "smoke.jar (bash): /tmp/smoke/smoke.jar" + echo "smoke.jar (native): $(native /tmp/smoke/smoke.jar)" + echo "IDENTITY_ENDPOINT: $IDENTITY_ENDPOINT" + echo "========================" + + jarsigner \ + -J-cp -J"$(native "$KV_JCA_JAR")" \ + -J-Dazure.keyvault.uri="${AZURE_KEYVAULT_URI}" \ + -keystore NONE \ + -storetype AzureKeyVault \ + -storepass "" \ + -providerClass com.azure.security.keyvault.jca.KeyVaultJcaProvider \ + -certchain "$(native /tmp/certchain.pem)" \ + -tsa http://timestamp.sectigo.com/ \ + "$(native /tmp/smoke/smoke.jar)" "$AZURE_KEYVAULT_CERT_NAME" + + - name: Stop KV token mock IMDS + if: always() + run: | + [ -n "$MOCK_IMDS_PID" ] && kill "$MOCK_IMDS_PID" 2>/dev/null || true + rm -f /tmp/certchain.pem + + - name: Verify smoke JAR signature + show signing cert + run: | + native() { command -v cygpath &>/dev/null && cygpath -w "$1" || echo "$1"; } + jarsigner -verify -strict -verbose "$(native /tmp/smoke/smoke.jar)" | tail -20 + echo "--- Signing certificate ---" + keytool -printcert -jarfile "$(native /tmp/smoke/smoke.jar)" \ + | grep -E "(Owner|Issuer|Serial|Valid|SHA)" + echo "Key Vault JCA signing succeeded — Crypto User RBAC + cert + TSA all working." + + - name: Sign test .exe with AzureSignTool (Authenticode) + shell: pwsh + env: + AZURE_KEYVAULT_URI: ${{ vars.AZURE_KEYVAULT_URI }} + AZURE_KEYVAULT_CERT_NAME: ${{ vars.AZURE_KEYVAULT_CERT_NAME }} + run: | + # AzureSignTool uses DefaultAzureCredential, which includes AzureCliCredential + # as a fallback. After azure/login the CLI credential cache is populated, + # so no managed identity or service principal secret is needed. + dotnet tool install --global AzureSignTool --version 7.0.1 + + # Copy a small Windows binary to a temp location as our test target. + $testExe = "$env:RUNNER_TEMP\smoke-sign.exe" + Copy-Item "$env:SystemRoot\System32\where.exe" $testExe + + AzureSignTool sign ` + --azure-key-vault-url $env:AZURE_KEYVAULT_URI ` + --azure-key-vault-certificate $env:AZURE_KEYVAULT_CERT_NAME ` + --azure-key-vault-managed-identity ` + --timestamp-rfc3161 http://timestamp.sectigo.com/ ` + --description "eXist-db signing smoke" ` + $testExe + + $sig = Get-AuthenticodeSignature $testExe + Write-Host "Authenticode status: $($sig.Status)" + Write-Host "Signer subject: $($sig.SignerCertificate.Subject)" + Write-Host "Signer issuer: $($sig.SignerCertificate.Issuer)" + if ($sig.Status -ne 'Valid') { + Write-Error "Authenticode signature invalid: $($sig.Status)" + exit 1 + } + Write-Host "AzureSignTool succeeded — .exe Authenticode signing works." diff --git a/.github/workflows/ci-release-prepare.yml b/.github/workflows/ci-release-prepare.yml new file mode 100644 index 00000000000..4fddd7455d1 --- /dev/null +++ b/.github/workflows/ci-release-prepare.yml @@ -0,0 +1,63 @@ +name: Prepare Release + +# Replaces mvn release:prepare. Updates CITATION.cff, commits, creates the +# annotated tag, and pushes — which then triggers ci-release.yml. +# +# Requires a fine-grained PAT (RELEASE_PAT) with contents:write on this repo. +# GITHUB_TOKEN pushes do not trigger downstream tag workflows. + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g. 7.0.0)' + required: true + type: string + +permissions: + contents: write + +jobs: + prepare: + name: Prepare eXist-${{ inputs.version }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # PAT required so the tag push triggers ci-release.yml. + # GITHUB_TOKEN pushes are intentionally blocked from triggering workflows. + token: ${{ secrets.RELEASE_PAT }} + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '21' + + - uses: ./.github/actions/maven-cache + + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update CITATION.cff + run: | + mvn -B --no-transfer-progress \ + -Pcitation-release-metadata \ + -DupdateCff=true \ + -Drevision=${{ inputs.version }} \ + validate + + - name: Commit and tag + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add CITATION.cff + # Only commit if CITATION.cff actually changed + git diff --staged --quiet || \ + git commit -m "[release] Prepare eXist-${{ inputs.version }}" + git tag -a "eXist-${{ inputs.version }}" \ + -m "eXist-db ${{ inputs.version }}" + git push origin HEAD + git push origin "eXist-${{ inputs.version }}" diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml new file mode 100644 index 00000000000..99b7eb19140 --- /dev/null +++ b/.github/workflows/ci-release.yml @@ -0,0 +1,375 @@ +name: Release + +# Triggered by an eXist-* tag (created by ci-release-prepare.yml). +# Four jobs run in parallel then converge on a single GitHub Release: +# build-linux — Maven Central deploy + zip/tar archives (Ubuntu) +# build-mac — signed + notarized DMG (macOS; installer excluded) +# build-windows — jarsigner + Authenticode signed installer JAR + .exe (Windows) +# publish-github-release — collects all artifacts, creates the GitHub Release +# +# The installer JAR is built and deployed to Maven Central from build-linux (GPG-signed). +# The Windows job re-builds and jarsigner-signs it for the GitHub Release download. + +on: + push: + tags: + - 'eXist-*' + - '!eXist-*-oidc-smoke' + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to re-release (e.g. eXist-7.0.0)' + required: false + type: string + +permissions: + contents: read # default; jobs that need more declare it explicitly + +env: + JAVA_VERSION: '21' + +# ─── Shared helper ──────────────────────────────────────────────────────────── +# Extract the semantic version from the tag name (strips the "eXist-" prefix). +# Usage in run steps: TAG="${{ github.event.inputs.tag || github.ref_name }}" +# REVISION="${TAG#eXist-}" + +jobs: + + # ── Job 1: Build and publish to Maven Central ──────────────────────────────── + build-linux: + name: Build and publish to Maven Central + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + CENTRAL_TOKEN_USERNAME: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + CENTRAL_TOKEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + + - uses: ./.github/actions/maven-cache + + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + central-token-username: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + central-token-password: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + + - name: Deploy to Maven Central + env: + EXISTDB_RELEASE_KEY: ${{ secrets.EXISTDB_RELEASE_KEY }} + EXISTDB_RELEASE_KEY_ID: ${{ secrets.EXISTDB_RELEASE_KEY_ID }} + EXISTDB_RELEASE_KEY_PASSPHRASE: ${{ secrets.EXISTDB_RELEASE_KEY_PASSPHRASE }} + run: | + # maven-gpg-plugin's BouncyCastle signer reads the armored key from + # MAVEN_GPG_KEY (not from gnupg's keyring). Decode the base64 secret + # in-place so the key never lands on disk. + export MAVEN_GPG_KEY="$(echo "$EXISTDB_RELEASE_KEY" | base64 --decode)" + TAG="${{ github.event.inputs.tag || github.ref_name }}" + REVISION="${TAG#eXist-}" + mvn -V -B --no-transfer-progress \ + -Prelease-build \ + -Drevision="$REVISION" \ + -Drelease.preflight=true \ + -DskipTests \ + -Ddependency-check.skip=true \ + -Dlicense.skip=true \ + -Dexistdb.release.key="$EXISTDB_RELEASE_KEY_ID" \ + clean deploy + + - name: Collect distribution archives + run: | + mkdir -p /tmp/release-assets + cp exist-distribution/target/*.zip /tmp/release-assets/ 2>/dev/null || true + cp exist-distribution/target/*.tar.bz2 /tmp/release-assets/ 2>/dev/null || true + + - uses: actions/upload-artifact@v7 + with: + name: release-linux + path: /tmp/release-assets/ + retention-days: 3 + + # ── Job 2: Build signed + notarized macOS DMG ──────────────────────────────── + build-mac: + name: Build macOS DMG + runs-on: macos-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + + - uses: ./.github/actions/maven-cache + + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Import Developer ID certificate + env: + EXISTDB_MAC_CERTIFICATE: ${{ secrets.EXISTDB_MAC_CERTIFICATE }} + EXISTDB_MAC_CERTIFICATE_PASSWORD: ${{ secrets.EXISTDB_MAC_CERTIFICATE_PASSWORD }} + run: | + KEYCHAIN_PASS=$(openssl rand -base64 32) + security create-keychain -p "$KEYCHAIN_PASS" release.keychain + security default-keychain -s release.keychain + security unlock-keychain -p "$KEYCHAIN_PASS" release.keychain + security set-keychain-settings -lut 21600 release.keychain + echo "$EXISTDB_MAC_CERTIFICATE" | base64 --decode > /tmp/cert.p12 + security import /tmp/cert.p12 -k release.keychain \ + -P "$EXISTDB_MAC_CERTIFICATE_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list \ + -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASS" release.keychain + rm /tmp/cert.p12 + + - name: Write Apple API key + env: + EXISTDB_APPLE_API_KEY: ${{ secrets.EXISTDB_APPLE_API_KEY }} + run: | + echo "$EXISTDB_APPLE_API_KEY" | base64 --decode > /tmp/notarytool.p8 + chmod 600 /tmp/notarytool.p8 + + - name: Build, sign, notarize and staple DMG + env: + EXISTDB_APPLE_API_KEY_ID: ${{ secrets.EXISTDB_APPLE_API_KEY_ID }} + EXISTDB_APPLE_API_ISSUER_ID: ${{ secrets.EXISTDB_APPLE_API_ISSUER_ID }} + run: | + TAG="${{ github.event.inputs.tag || github.ref_name }}" + REVISION="${TAG#eXist-}" + mvn -V -B --no-transfer-progress \ + -Prelease-build \ + -Dmac-signing=true \ + -Drevision="$REVISION" \ + -DskipTests \ + -Ddependency-check.skip=true \ + -Dlicense.skip=true \ + -Dmac.codesign.identity="${{ vars.EXISTDB_MAC_CODESIGN_IDENTITY }}" \ + -Dexistdb.release.notarize.key-path=/tmp/notarytool.p8 \ + -Dexistdb.release.notarize.key-id="$EXISTDB_APPLE_API_KEY_ID" \ + -Dexistdb.release.notarize.issuer-id="$EXISTDB_APPLE_API_ISSUER_ID" \ + --projects '!exist-installer' \ + clean package + + - name: Remove Apple API key + if: always() + run: rm -f /tmp/notarytool.p8 + + - name: Verify DMG notarization + run: | + DMG=$(find exist-distribution/target -name "*.dmg" | head -1) + xcrun stapler validate "$DMG" + + - name: Collect DMG + run: | + mkdir -p /tmp/release-assets + cp exist-distribution/target/*.dmg /tmp/release-assets/ 2>/dev/null || true + + - uses: actions/upload-artifact@v7 + with: + name: release-mac + path: /tmp/release-assets/ + retention-days: 3 + + # ── Job 3: Build signed Windows installer ─────────────────────────────────── + build-windows: + name: Build Windows installer + runs-on: windows-latest + timeout-minutes: 60 + permissions: + id-token: write # OIDC for Azure Key Vault signing + contents: read + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + + - uses: ./.github/actions/maven-cache + + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Azure login (OIDC) + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Build installer + shell: bash + run: | + TAG="${{ github.event.inputs.tag || github.ref_name }}" + REVISION="${TAG#eXist-}" + mvn -V -B --no-transfer-progress \ + -Prelease-build \ + -Drevision="$REVISION" \ + -DskipTests \ + -Ddependency-check.skip=true \ + -Dlicense.skip=true \ + clean package + + - name: Start KV token mock IMDS + shell: bash + run: | + KV_TOKEN=$(az account get-access-token \ + --resource https://vault.azure.net --query accessToken -o tsv) + [ -n "$KV_TOKEN" ] || { echo "Failed to get Key Vault access token"; exit 1; } + export KV_TOKEN_VALUE="$KV_TOKEN" + export KV_TOKEN_EXPIRES=$(python3 -c "import time; print(int(time.time()) + 3600)") + python3 -c " + import http.server, os, json + token_bytes = json.dumps({ + 'access_token': os.environ['KV_TOKEN_VALUE'], + 'expires_on': os.environ['KV_TOKEN_EXPIRES'], + 'token_type': 'Bearer', + 'resource': 'https://vault.azure.net' + }).encode() + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type','application/json') + self.end_headers() + self.wfile.write(token_bytes) + def log_message(self,*a): pass + http.server.HTTPServer(('127.0.0.1',19999),H).serve_forever() + " & + echo "MOCK_IMDS_PID=$!" >> "$GITHUB_ENV" + echo "IDENTITY_ENDPOINT=http://127.0.0.1:19999/" >> "$GITHUB_ENV" + echo "IDENTITY_HEADER=mock-header" >> "$GITHUB_ENV" + sleep 1 + curl -sf http://127.0.0.1:19999/ \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print('Mock IMDS OK, token_type:', d.get('token_type'))" + + - name: Write signing cert chain + shell: bash + env: + EXISTDB_WIN_SIGNING_CERTCHAIN: ${{ secrets.EXISTDB_WIN_SIGNING_CERTCHAIN }} + run: | + # Cert expires 2027-12-03 — update EXISTDB_WIN_SIGNING_CERTCHAIN secret and + # exist-db-win-signing in Key Vault when renewing. + [ -n "$EXISTDB_WIN_SIGNING_CERTCHAIN" ] || { echo "EXISTDB_WIN_SIGNING_CERTCHAIN secret not set"; exit 1; } + echo "$EXISTDB_WIN_SIGNING_CERTCHAIN" | base64 --decode > /tmp/certchain.pem + chmod 600 /tmp/certchain.pem + + - name: Sign installer JAR (Azure Key Vault JCA) + shell: bash + env: + AZURE_KEYVAULT_URI: ${{ vars.AZURE_KEYVAULT_URI }} + # IDENTITY_ENDPOINT + IDENTITY_HEADER flow in via GITHUB_ENV from prior step + run: | + native() { command -v cygpath &>/dev/null && cygpath -w "$1" || echo "$1"; } + + JAR=$(find exist-installer/target -name "*.jar" \ + -not -name "*sources*" -not -name "*javadoc*" | head -1) + [ -n "$JAR" ] || { echo "No installer JAR found in exist-installer/target"; exit 1; } + KV_JCA_JAR="$HOME/.m2/repository/com/azure/azure-security-keyvault-jca/2.10.0/azure-security-keyvault-jca-2.10.0.jar" + if [ ! -f "$KV_JCA_JAR" ]; then + mvn -q dependency:get \ + -Dartifact=com.azure:azure-security-keyvault-jca:2.10.0:jar + fi + CERT_NAME="${{ vars.AZURE_KEYVAULT_CERT_NAME }}" + jarsigner \ + -J-cp -J"$(native "$KV_JCA_JAR")" \ + -J-Dazure.keyvault.uri="${AZURE_KEYVAULT_URI}" \ + -keystore NONE \ + -storetype AzureKeyVault \ + -storepass "" \ + -providerClass com.azure.security.keyvault.jca.KeyVaultJcaProvider \ + -certchain "$(native /tmp/certchain.pem)" \ + -tsa http://timestamp.sectigo.com/ \ + "$(native "$JAR")" "$CERT_NAME" + jarsigner -verify -strict "$(native "$JAR")" + + - name: Stop KV token mock IMDS + if: always() + shell: bash + run: | + [ -n "$MOCK_IMDS_PID" ] && kill "$MOCK_IMDS_PID" 2>/dev/null || true + rm -f /tmp/certchain.pem + + - name: Sign .exe with Authenticode (AzureSignTool) + shell: pwsh + env: + AZURE_KEYVAULT_URI: ${{ vars.AZURE_KEYVAULT_URI }} + run: | + $exe = Get-ChildItem -Path exist-installer/target -Filter "*.exe" | + Select-Object -First 1 + if (-not $exe) { + Write-Host "No .exe produced on this runner; skipping Authenticode signing" + exit 0 + } + dotnet tool install --global AzureSignTool --version 7.0.1 + $certName = '${{ vars.AZURE_KEYVAULT_CERT_NAME }}' + AzureSignTool sign ` + --azure-key-vault-url $env:AZURE_KEYVAULT_URI ` + --azure-key-vault-certificate $certName ` + --azure-key-vault-managed-identity ` + --timestamp-rfc3161 http://timestamp.sectigo.com/ ` + --description "eXist-db Installer" ` + $exe.FullName + $sig = Get-AuthenticodeSignature $exe.FullName + if ($sig.Status -ne 'Valid') { + Write-Error "Authenticode signature invalid: $($sig.Status)" + exit 1 + } + + - name: Collect installer artifacts + shell: bash + env: + # Use the runner-provided temp dir; Git Bash and Node actions disagree on /tmp. + RELEASE_ASSETS: ${{ runner.temp }}/release-assets + run: | + mkdir -p "$RELEASE_ASSETS" + find exist-installer/target \ + -name "*.jar" -not -name "*sources*" -not -name "*javadoc*" \ + -exec cp {} "$RELEASE_ASSETS/" \; + find exist-installer/target -name "*.exe" \ + -exec cp {} "$RELEASE_ASSETS/" \; 2>/dev/null || true + ls -lh "$RELEASE_ASSETS/" + + - uses: actions/upload-artifact@v7 + with: + name: release-windows + path: ${{ runner.temp }}/release-assets/ + retention-days: 3 + + # ── Job 4: Publish GitHub Release ─────────────────────────────────────────── + publish-github-release: + name: Publish GitHub Release + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [build-linux, build-mac, build-windows] + permissions: + contents: write # create GitHub Release + steps: + - uses: actions/download-artifact@v8 + with: + pattern: release-* + merge-multiple: true + path: /tmp/release-assets + + - name: List release assets + run: ls -lh /tmp/release-assets/ + + - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + with: + tag_name: ${{ github.event.inputs.tag || github.ref_name }} + files: /tmp/release-assets/* + generate_release_notes: true diff --git a/.github/workflows/ci-schema-checks.yml b/.github/workflows/ci-schema-checks.yml new file mode 100644 index 00000000000..09b3aa156ab --- /dev/null +++ b/.github/workflows/ci-schema-checks.yml @@ -0,0 +1,60 @@ +name: Schema checks + +on: + pull_request: + paths: + - 'schema/**' + - 'exist-distribution/src/main/config/**' + - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml' + - 'exist-core/src/main/resources/org/exist/util/mime-types.xml' + - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java' + push: + branches: [develop] + paths: + - 'schema/**' + - 'exist-distribution/src/main/config/**' + - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml' + - 'exist-core/src/main/resources/org/exist/util/mime-types.xml' + - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java' + workflow_dispatch: + +permissions: + contents: read + +env: + MAVEN_OPTS: -DtrimStackTrace=false + DEV_JDK: '21' + +jobs: + schema: + name: Native XSD checks + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + + - uses: ./.github/actions/maven-cache + + - name: Validate canonical templates against XSD + run: mvn -V -B --no-transfer-progress validate -Ddependency-check.skip=true -Ddocker=false + + - name: Prepare governance context + env: + GITHUB_BASE_REF: ${{ github.base_ref }} + GITHUB_EVENT_BEFORE: ${{ github.event.before }} + run: | + chmod +x .github/scripts/prepare-governance-context.sh + .github/scripts/prepare-governance-context.sh "${{ github.workspace }}" + + - name: Run schema governance (XSLT 2.0 / Saxon) + run: mvn -N -B --no-transfer-progress xml:transform@schema-governance -Ddependency-check.skip=true -Ddocker=false + + - name: Verify SchemaVersion.java matches XSD versions + run: mvn -B --no-transfer-progress test -pl exist-core -Dtest=org.exist.util.SchemaVersionSyncTest -Ddependency-check.skip=true -Ddocker=false diff --git a/.github/workflows/ci-snapshots.yml b/.github/workflows/ci-snapshots.yml index f88d0c5727a..bdd6bbe8f99 100644 --- a/.github/workflows/ci-snapshots.yml +++ b/.github/workflows/ci-snapshots.yml @@ -10,29 +10,26 @@ jobs: publish-snapshots: name: Deploy Snapshots runs-on: ubuntu-latest + timeout-minutes: 45 if: github.ref == 'refs/heads/develop' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 1 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: '21' - server-id: github - settings-path: ${{ github.workspace }} - - name: Cache Maven packages - uses: actions/cache@v4 + - uses: ./.github/actions/maven-cache + - uses: ./.github/actions/maven-github-settings with: - path: ~/.m2 - key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: deploy-${{ runner.os }}-maven + token: ${{ secrets.GITHUB_TOKEN }} - name: Deploy SNAPSHOT maven artefacts + timeout-minutes: 40 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - mvn -V -B -q -DskipTests -Ddependency-check.skip=true \ - -s $GITHUB_WORKSPACE/settings.xml \ - -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives \ + mvn -V -B --no-transfer-progress -q -DskipTests -Ddependency-check.skip=true \ + -P skip-build-dist-archives \ clean deploy diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 337075825ea..83973bfce5b 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,86 +1,142 @@ name: Test & documentation -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: contents: read env: - MAVEN_OPTS: -DtrimStackTrace=false -D'maven.resolver.transport=wagon' + MAVEN_OPTS: -DtrimStackTrace=false DEV_JDK: '21' jobs: license: name: License check runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: ${{ env.DEV_JDK }} - cache: 'maven' - - run: mvn -V -B license:check - timeout-minutes: 60 + - uses: ./.github/actions/maven-cache + - run: mvn -V -B --no-transfer-progress license:check + timeout-minutes: 10 dependencies: name: Dependency checks if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/develop' }} runs-on: ubuntu-latest + timeout-minutes: 95 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: ${{ env.DEV_JDK }} - cache: 'maven' + - uses: ./.github/actions/maven-cache + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Set week for OWASP cache key + run: echo "WEEK=$(date +%Y-%U)" >> $GITHUB_ENV + - name: Restore OWASP dependency-check cache + uses: actions/cache/restore@v6 + with: + path: ~/.dependency-check-data + key: owasp-dc-${{ env.WEEK }} + restore-keys: owasp-dc- - name: OWASP dependency check env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NVD_API_KEY: ${{ secrets.NVD_API_KEY }} - run: mvn -V -B dependency-check:check - timeout-minutes: 60 + run: mvn -V -B --no-transfer-progress dependency-check:check + timeout-minutes: 90 + - name: Save OWASP dependency-check cache + if: github.event_name == 'push' + uses: actions/cache/save@v6 + with: + path: ~/.dependency-check-data + key: owasp-dc-${{ env.WEEK }} test: - name: ${{ matrix.os }} Test + name: ${{ matrix.os }} ${{ matrix.test-type }} (Java ${{ matrix.java-version }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 75 strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - jvm: ['21'] - runs-on: ${{ matrix.os }} + test-type: [unit, integration] + java-version: ['21'] + exclude: + - os: windows-latest + test-type: unit + - os: macOS-latest + test-type: unit steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: - distribution: liberica - java-version: ${{ matrix.jvm }} - cache: 'maven' + distribution: temurin + java-version: ${{ matrix.java-version }} + - uses: ./.github/actions/maven-cache + with: + java-version: ${{ matrix.java-version }} + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Install Maven Daemon id: install-mvnd uses: ./.github/actions/install-mvnd with: - version: '1.0.2' + version: '1.0.6' file-version-suffix: '' cache: 'true' - name: Maven Build timeout-minutes: 30 - run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -T 1C compile test-compile -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' - - name: Maven Test - timeout-minutes: 60 - run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B verify -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000' - - name: Javadoc (Linux only) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make - - name: Maven Code Coverage (Develop branch on Linux only) - if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MAVEN_OPTS: ${{ env.MAVEN_OPTS }} -D'aether.connector.basic.connectTimeout=10000' -D'aether.connector.basic.requestTimeout=30000' + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress -T 1C compile test-compile -Ponnx-model -D'dependency-check.skip' -D'license.skip' + - name: Maven Unit Tests + id: unit-tests + if: matrix.test-type == 'unit' + timeout-minutes: 45 + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress test -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000' + - name: Dependency tree on unit test failure + if: always() && matrix.test-type == 'unit' && steps.unit-tests.outcome == 'failure' + run: | + ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress dependency:tree -pl extensions/indexes/lucene -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' | tee dependency-tree-lucene.txt + echo "--- Lucene-related dependencies ---" + grep -i lucene dependency-tree-lucene.txt || true + - name: Maven Integration Tests + if: matrix.test-type == 'integration' + timeout-minutes: 45 + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress verify "-Ponnx-model,!mac-dmg-on-mac" -DskipUnitTests=true -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000' + - name: Javadoc (ubuntu unit only) + if: matrix.os == 'ubuntu-latest' && matrix.test-type == 'unit' + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution' --also-make + - name: Maven Code Coverage (Develop branch, ubuntu unit only) + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' && matrix.test-type == 'unit' env: CI_NAME: github BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }} CI_BUILD_NUMBER: ${{ github.run_id }} CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B jacoco:report coveralls:report + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B --no-transfer-progress jacoco:report coveralls:report + - name: Save Maven cache + if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.test-type == 'unit' + uses: actions/cache/save@v6 + with: + path: ~/.m2 + key: maven-${{ github.ref_name }}-java${{ matrix.java-version }}-${{ hashFiles('**/pom.xml') }} + enableCrossOsArchive: true - name: Archive build logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: ${{ runner.os }}-${{ matrix.jvm }}-build-logs + name: ${{ runner.os }}-${{ matrix.test-type }}-java${{ matrix.java-version }}-build-logs retention-days: 5 path: | **/hs_err_pid*.log **/target/surefire-reports/* + **/target/failsafe-reports/* + dependency-tree-lucene.txt diff --git a/.github/workflows/ci-xqts.yml b/.github/workflows/ci-xqts.yml index 883e50bf4ae..5828a82e24d 100644 --- a/.github/workflows/ci-xqts.yml +++ b/.github/workflows/ci-xqts.yml @@ -1,5 +1,5 @@ name: XQTS -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: contents: read @@ -7,21 +7,23 @@ jobs: xqts: name: W3C XQuery Test Suite runs-on: ubuntu-latest + timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: '21' - - name: Cache Maven packages - uses: actions/cache@v4 + - uses: ./.github/actions/maven-cache + - uses: ./.github/actions/maven-github-settings with: - path: ~/.m2 - key: xqts-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: xqts-${{ runner.os }}-maven + token: ${{ secrets.GITHUB_TOKEN }} - name: Maven XQTS Build - run: mvn -V -B clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make + timeout-minutes: 25 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: mvn -V -B --no-transfer-progress clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make - name: Run XQTS timeout-minutes: 60 env: @@ -37,29 +39,55 @@ jobs: run: zstd --rm -9 --progress -T0 /tmp/*.hprof - name: Attach HeapDump artifact if: steps.check_heapdump.outputs.files_exists == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: exist-xqts-runner-hprof retention-days: 1 path: /tmp/*.hprof.zst + - name: Find Latest Successful XQTS Run on Develop + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RUN_ID=$(gh run list \ + --repo eXist-db/exist \ + --workflow ci-xqts.yml \ + --branch develop \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "::error::Could not find a previous successful XQTS run on develop" + exit 1 + fi + + echo "Found develop run: ${RUN_ID}" + echo "$RUN_ID" > /tmp/previous_run_id.txt + - name: Download Previous XQTS Logs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh run download \ + --repo eXist-db/exist \ + "$(cat /tmp/previous_run_id.txt)" \ + --name xqts-logs \ + --dir /tmp/previous-xqts-output + - name: Compare Previous and Current XQTS Logs + run: | + java \ + -jar ~/.m2/repository/net/sf/saxon/Saxon-HE/9.9.1-8/Saxon-HE-9.9.1-8.jar \ + -xsl:exist-xqts/src/main/xslt/compare-results.xslt \ + -it:compare-results \ + -o:/tmp/comparison-results.xml \ + xqts.previous.junit-data-path=/tmp/previous-xqts-output/junit/data \ + xqts.current.junit-data-path=/tmp/xqts-output/junit/data + - name: Show Comparison Results + run: cat /tmp/comparison-results.xml - name: Archive XQTS Logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: xqts-logs retention-days: 14 path: /tmp/xqts-output - - name: Get Previous XQTS Logs Artifacts JSON - run: 'curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/exist-db/exist/actions/artifacts?name=xqts-logs > /tmp/previous-xqts-logs-artifacts.json' - - name: Extract Previous XQTS Logs Artifact JSON - run: cat /tmp/previous-xqts-logs-artifacts.json | jq -r "[.artifacts[] | select(.workflow_run.head_branch == \"develop\")][1].archive_download_url" > /tmp/previous-xqts-logs-artifact.json - - name: Get Previous XQTS Logs Artifact - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: 'cat /tmp/previous-xqts-logs-artifact.json | xargs curl -H "Authorization: Bearer ${GITHUB_TOKEN}" --location --output /tmp/previous-xqts-output.zip' - - name: Extract Previous XQTS Logs Artifact - run: mkdir /tmp/previous-xqts-output && unzip /tmp/previous-xqts-output.zip -d /tmp/previous-xqts-output - - name: Compare Previous and Current XQTS Logs - run: java -jar ~/.m2/repository/net/sf/saxon/Saxon-HE/9.9.1-8/Saxon-HE-9.9.1-8.jar -xsl:exist-xqts/src/main/xslt/compare-results.xslt -it:compare-results -o:/tmp/comparison-results.xml xqts.previous.junit-data-path=/tmp/previous-xqts-output/junit/data xqts.current.junit-data-path=/tmp/xqts-output/junit/data - - name: Show Comparison Results - run: cat /tmp/comparison-results.xml diff --git a/.github/workflows/citation-cff.yml b/.github/workflows/citation-cff.yml new file mode 100644 index 00000000000..8715864291f --- /dev/null +++ b/.github/workflows/citation-cff.yml @@ -0,0 +1,26 @@ +name: CITATION.cff validation + +on: + push: + paths: + - CITATION.cff + - .github/workflows/citation-cff.yml + pull_request: + paths: + - CITATION.cff + - .github/workflows/citation-cff.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate-citation-cff: + name: Validate CITATION.cff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Validate CITATION.cff schema + uses: citation-file-format/cffconvert-github-action@2.0.0 + with: + args: "--validate" diff --git a/.github/workflows/prethink.yml b/.github/workflows/prethink.yml new file mode 100644 index 00000000000..6e3ceccf406 --- /dev/null +++ b/.github/workflows/prethink.yml @@ -0,0 +1,77 @@ +name: Update Prethink Context + +on: + schedule: + - cron: '0 6 * * 1' # Weekly Monday 6am UTC + workflow_dispatch: # Manual trigger + +permissions: + contents: write + +jobs: + prethink: + if: github.repository == 'eXist-db/exist' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + java-version: '21' + distribution: 'temurin' + + - name: Install Moderne CLI + env: + # Pinned to avoid the Maven Central propagation race observed on 2026-05-08, where + # the parent moderne-cli metadata advertised a release before the platform-specific + # moderne-cli-linux binary had CDN-synced. Bump intentionally as new releases stabilize. + MODERNE_CLI_VERSION: '4.2.3' + run: | + curl -sSL https://app.moderne.io/cli | bash + echo "$HOME/.moderne/cli/bin" >> $GITHUB_PATH + # The installer writes 'version=RELEASE' so every `mod` invocation re-resolves to whatever + # is currently latest on Maven Central. Replace with the pinned version so subsequent + # `mod` calls reuse the dist downloaded once and never race a fresh release. + sed -i "s/^version=RELEASE$/version=${MODERNE_CLI_VERSION}/" \ + "$HOME/.moderne/cli/dist/moderne-wrapper.properties" + + - name: Configure Moderne + run: | + mod config moderne edit https://app.moderne.io --token ${{ secrets.MODERNE_TOKEN }} + mod config recipes jar install io.moderne.recipe:rewrite-prethink:LATEST + + - uses: ./.github/actions/maven-github-settings + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build LSTs + run: mod build . + + - name: Run Prethink (deterministic, no AI) + run: mod run . --recipe io.moderne.prethink.UpdatePrethinkContextStarter + + - name: Apply changes + run: mod git apply . --last-recipe-run + + - name: Commit and push + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add .moderne/context/ AGENTS.md + if git diff --staged --quiet; then + echo "No Prethink context changes to commit." + exit 0 + fi + git commit -m "[ci] Update Prethink context" + # Concurrent commits to develop between checkout and push caused the 2026-05-04 + # scheduled run to fail with a non-fast-forward rejection. Rebase + bounded retry + # handles that without forcing or losing the new context updates. + for attempt in 1 2 3; do + if git pull --rebase --autostash origin develop && git push origin HEAD:develop; then + exit 0 + fi + echo "Push attempt $attempt failed; retrying after backoff..." + sleep $((attempt * 10)) + done + echo "Push failed after 3 attempts" >&2 + exit 1 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index cfd886ece17..084f470a386 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -4,35 +4,30 @@ on: branches: - develop - master - pull_request: - types: [opened, synchronize, reopened] jobs: build: name: SonarCloud Analysis runs-on: ubuntu-latest + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: - distribution: liberica + distribution: temurin java-version: 21 - name: Cache SonarCloud packages - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.sonar/cache key: sonarcloud-${{ runner.os }}-cache-${{ hashFiles('**/pom.xml') }} restore-keys: sonarcloud-${{ runner.os }}-cache - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: sonarcloud-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: sonarcloud-${{ runner.os }}-maven + - uses: ./.github/actions/maven-cache - name: Analyze + timeout-minutes: 55 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: mvn -V -B -Dsurefire.useFile=false -DtrimStackTrace=false -Ddependency-check.skip=true -Ddocker=false -P \!mac-dmg-on-mac,\!codesign-mac-dmg,\!mac-dmg-on-unix,\!installer,\!concurrency-stress-tests,\!micro-benchmarks,\!build-dist-archives verify site org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + run: mvn -V -B --no-transfer-progress -Dsurefire.useFile=false -DtrimStackTrace=false -Ddependency-check.skip=true -Ddocker=false -Pskip-build-dist-archives verify site org.sonarsource.scanner.maven:sonar-maven-plugin:sonar diff --git a/.gitignore b/.gitignore index 7937befa7c4..415ea9dec4f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,12 +8,34 @@ target/ # quartz artefacts dependency-reduced-pom.xml +# Maven generated +effective-pom.xml +.flattened-pom.xml + +# XQTS test artefacts +work/ + # IDE specific files .idea/* !/.idea/runConfigurations *.iml -.vscode +.vscode/ +.cursor/ +.zed/ # OS specific files .DS_Store +# Claude planning files +plans/ +.codacy/ +.moderne/* +!.moderne/context/ +!.moderne/moderne.yml +.github/instructions/ + +# Debug logs (e.g. from reindex investigation) +reindex-dbg.log + +# Temporary files occasionally created by Ant replaceregexp +/tmp*tmp diff --git a/.idea/runConfigurations/Java_Admin_Client.xml b/.idea/runConfigurations/Java_Admin_Client.xml index a6e09e6508b..9fa7df632bb 100644 --- a/.idea/runConfigurations/Java_Admin_Client.xml +++ b/.idea/runConfigurations/Java_Admin_Client.xml @@ -5,7 +5,7 @@