diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 9c85126..356d2d9 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -6,14 +6,20 @@ on: branches: - main pull_request: - + jobs: run-scripts: strategy: fail-fast: false matrix: os: [ubuntu-latest] - pgver: [16, 17] + pgver: [16, 17, 18, 19] + include: + # Spock PG 19 support is merged but not yet released; build Spock + # from this branch for PG 19 and drop the override once a Spock + # release includes it + - pgver: 19 + spock_ref: main runs-on: ${{ matrix.os }} @@ -33,12 +39,64 @@ jobs: - name: Set up docker-compose uses: docker/setup-compose-action@2fe291b7677a45ee1269ec56a42604c143505e7e # v1 - - name: Build and run docker images + # Resolve the exact source tags on the runner and pass them in as build + # args: the docker layer cache then reuses the compiled PostgreSQL and + # Spock until a new release is tagged. + - name: Resolve PostgreSQL and Spock release tags + id: tags + run: | + PG_TAGS=$(git ls-remote --tags https://github.com/postgres/postgres.git "REL_${{ matrix.pgver }}_*" \ + | awk -F/ '{print $NF}' | grep -v '\^{}') + PG_TAG=$(echo "$PG_TAGS" | grep -E "^REL_${{ matrix.pgver }}_[0-9]+$" | sort -t_ -k3,3n | tail -n 1) + # A pre-GA major has no stable minor tag yet: latest RC, else BETA + [ -n "$PG_TAG" ] || PG_TAG=$(echo "$PG_TAGS" | grep -E "^REL_${{ matrix.pgver }}_RC[0-9]+$" | sort -V | tail -n 1) + [ -n "$PG_TAG" ] || PG_TAG=$(echo "$PG_TAGS" | grep -E "^REL_${{ matrix.pgver }}_BETA[0-9]+$" | sort -V | tail -n 1) + if [ -n "${{ matrix.spock_ref }}" ]; then + # Pin the moving branch to its current SHA so the docker layer + # cache is invalidated by every Spock merge + SPOCK_TAG=$(git ls-remote https://github.com/pgEdge/spock.git "refs/heads/${{ matrix.spock_ref }}" | cut -f1) + else + SPOCK_TAG=$(git ls-remote --tags https://github.com/pgEdge/spock.git \ + | awk -F/ '{print $NF}' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V | tail -n 1) + fi + if [ -z "$PG_TAG" ] || [ -z "$SPOCK_TAG" ]; then + echo "::error::Failed to resolve tags (PG_TAG='$PG_TAG', SPOCK_TAG='$SPOCK_TAG')" + exit 1 + fi + echo "Using PostgreSQL $PG_TAG and Spock $SPOCK_TAG" + echo "pg_tag=$PG_TAG" >> $GITHUB_OUTPUT + echo "spock_tag=$SPOCK_TAG" >> $GITHUB_OUTPUT + + - name: Build node image + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: docker + file: docker/Dockerfile.el9 + tags: lolor + load: true + build-args: | + PGVER=${{ matrix.pgver }} + PG_TAG=${{ steps.tags.outputs.pg_tag }} + SPOCK_TAG=${{ steps.tags.outputs.spock_tag }} + cache-from: type=gha,scope=pg${{ matrix.pgver }} + cache-to: type=gha,mode=max,scope=pg${{ matrix.pgver }} + + - name: Build tester image + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: docker + file: docker/Dockerfile.tester + tags: lolor-tester + load: true + cache-from: type=gha,scope=tester + cache-to: type=gha,scope=tester + + - name: Run docker images run: | cd docker echo PG_VER=${{ matrix.pgver }} >> pgedge.env - docker build --build-arg PGVER=${{ matrix.pgver }} -t lolor -f Dockerfile.el9 . - docker build -t lolor-tester -f Dockerfile.tester . docker compose up -d env: PG_VER: ${{ matrix.pgver }} diff --git a/docker/Dockerfile.el9 b/docker/Dockerfile.el9 index bc98d74..e019b73 100644 --- a/docker/Dockerfile.el9 +++ b/docker/Dockerfile.el9 @@ -1,29 +1,89 @@ -FROM rockylinux:9 -ARG PGVER -# I need this to persist for use in entrypoint as well -ENV PG_VER=$PGVER - -RUN dnf -y --allowerasing install \ - epel-release \ - curl sudo python3 \ - openssh-server openssh-clients \ - make gcc clang llvm +# Build a pgEdge-style node from source: PostgreSQL (with the Spock +# patches applied) plus the Spock extension. The pgedge CLI installer is +# not used because it is deprecated (it also only supports PostgreSQL +# 15-17). +# +# The base image is Rocky Linux 9 with all build dependencies preinstalled +# (built from pgEdge/spock tests/docker/Dockerfile-base.el9). +# +# PG_TAG and SPOCK_TAG are normally resolved by the CI workflow and passed +# in as build args, so docker layer caching rebuilds only when a new +# release is tagged. When left empty (e.g. a local build), they resolve +# here to the latest release: for PostgreSQL the newest REL_$PGVER_x minor +# tag (falling back to the newest RC, then BETA, for a pre-GA major), for +# Spock the newest vMAJOR.MINOR.PATCH tag (numeric version order, +# pre-releases excluded). +ARG BASE_IMAGE=ghcr.io/pgedge/base-test-image:latest +FROM ${BASE_IMAGE} -RUN useradd -m pgedge -s /bin/bash && \ - echo pgedge:asdf | chpasswd && \ - echo "pgedge ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/pgedge +ARG PGVER=18 +ARG PG_TAG= +ARG SPOCK_TAG= +# PG_VER must persist for use in entrypoint as well +ENV PG_VER=$PGVER USER pgedge WORKDIR /home/pgedge -RUN ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_ed25519 && \ - cat ~/.ssh/*.pub >> ~/.ssh/authorized_keys +# SPOCK_TAG may be a release tag, a branch name or a commit SHA +RUN set -eux && \ + if [ -z "${SPOCK_TAG}" ]; then \ + SPOCK_TAG=$(git ls-remote --tags https://github.com/pgEdge/spock.git | \ + awk -F/ '{print $NF}' | \ + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | \ + sort -V | tail -n 1); \ + fi && \ + echo "Building Spock ${SPOCK_TAG}" && \ + mkdir /home/pgedge/spock && \ + cd /home/pgedge/spock && \ + git init -q && \ + git remote add origin https://github.com/pgEdge/spock.git && \ + git fetch --depth 1 origin "${SPOCK_TAG}" && \ + git checkout -q FETCH_HEAD + +RUN set -eux && \ + if [ -z "${PG_TAG}" ]; then \ + PG_TAGS=$(git ls-remote --tags https://github.com/postgres/postgres.git "REL_${PGVER}_*" | \ + awk -F/ '{print $NF}' | grep -v '\^{}'); \ + PG_TAG=$(echo "${PG_TAGS}" | grep -E "^REL_${PGVER}_[0-9]+$" | sort -t_ -k3,3n | tail -n 1); \ + [ -n "${PG_TAG}" ] || PG_TAG=$(echo "${PG_TAGS}" | grep -E "^REL_${PGVER}_RC[0-9]+$" | sort -V | tail -n 1); \ + [ -n "${PG_TAG}" ] || PG_TAG=$(echo "${PG_TAGS}" | grep -E "^REL_${PGVER}_BETA[0-9]+$" | sort -V | tail -n 1); \ + fi && \ + echo "Building PostgreSQL ${PG_TAG}" && \ + git clone --branch "${PG_TAG}" --depth 1 \ + https://github.com/postgres/postgres.git /home/pgedge/postgres && \ + cd /home/pgedge/postgres && \ + PATCH_DIR="/home/pgedge/spock/patches/${PGVER}" && \ + if [ -d "${PATCH_DIR}" ] && [ -n "$(ls -A "${PATCH_DIR}")" ]; then \ + for patchfile in "${PATCH_DIR}"/*; do \ + echo "Applying $(basename "${patchfile}")"; \ + patch -p1 < "${patchfile}"; \ + done; \ + fi + +# Install into the same layout the pgedge CLI used +# (/home/pgedge/pgedge/pg$PGVER) so the entrypoint, compose healthchecks +# and tests keep working unchanged. +RUN set -eux && \ + cd /home/pgedge/postgres && \ + ./configure --prefix=/home/pgedge/pgedge/pg${PGVER} --with-icu > /dev/null && \ + make -j"$(nproc)" > /dev/null && \ + make install > /dev/null + +RUN set -eux && \ + cd /home/pgedge/spock && \ + make -j"$(nproc)" PG_CONFIG=/home/pgedge/pgedge/pg${PGVER}/bin/pg_config > /dev/null && \ + make install PG_CONFIG=/home/pgedge/pgedge/pg${PGVER}/bin/pg_config > /dev/null -RUN curl -fsSL https://pgedge-download.s3.amazonaws.com/REPO/install.py > /home/pgedge/install.py -RUN sudo -u pgedge python3 /home/pgedge/install.py -RUN cd pgedge && ./pgedge setup -U admin -P password -d demo --pg_ver=$PG_VER +# Environment file at the path the pgedge CLI used to generate it; the +# entrypoint and the compose healthchecks source it. +RUN set -eux && \ + { \ + echo "export PATH=/home/pgedge/pgedge/pg${PGVER}/bin:\$PATH"; \ + echo "export LD_LIBRARY_PATH=/home/pgedge/pgedge/pg${PGVER}/lib:\$LD_LIBRARY_PATH"; \ + echo "export PGDATA=/home/pgedge/pgedge/data/pg${PGVER}"; \ + } > /home/pgedge/pgedge/pg${PGVER}/pg${PGVER}.env -COPY entrypoint.sh /home/pgedge -RUN sudo chmod +x /home/pgedge/entrypoint.sh +COPY --chown=pgedge:pgedge --chmod=755 entrypoint.sh /home/pgedge/entrypoint.sh -CMD /home/pgedge/entrypoint.sh +CMD ["/home/pgedge/entrypoint.sh"] diff --git a/docker/Dockerfile.tester b/docker/Dockerfile.tester index 930d52b..206d16c 100644 --- a/docker/Dockerfile.tester +++ b/docker/Dockerfile.tester @@ -16,4 +16,4 @@ COPY run-tests.sh /home/pgedge RUN sudo chmod +x /home/pgedge/run-tests.sh WORKDIR /home/pgedge/lolor/tests -CMD /home/pgedge/run-tests.sh +CMD ["/home/pgedge/run-tests.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 978931c..ac12080 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,22 +3,51 @@ set -e cd /home/pgedge/pgedge . pg${PG_VER}/pg${PG_VER}.env -echo 'export LD_LIBRARY_PATH=/home/pgedge/pgedge/pg${PG_VER}/lib:$LD_LIBRARY_PATH' >> /home/pgedge/.bashrc -echo 'export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH' >> /home/pgedge/.bashrc -echo 'export PATH=/home/pgedge/pgedge/pg${PG_VER}/bin:$PATH' >> /home/pgedge/.bashrc -. /home/pgedge/.bashrc -sudo ldconfig +echo ". /home/pgedge/pgedge/pg${PG_VER}/pg${PG_VER}.env" >> /home/pgedge/.bashrc -./pgedge start +# Initialize the cluster. This replaces `pgedge setup` from the +# deprecated pgedge CLI. +initdb -D "$PGDATA" -U admin --encoding=UTF8 --locale=C + +cat >> "$PGDATA/postgresql.conf" <<_EOF_ +listen_addresses = '*' +wal_level = logical +track_commit_timestamp = on +max_worker_processes = 32 +max_replication_slots = 32 +max_wal_senders = 32 +shared_preload_libraries = 'spock' +spock.conflict_resolution = 'last_update_wins' +spock.save_resolutions = on +_EOF_ + +cat >> "$PGDATA/pg_hba.conf" <<_EOF_ +# Trust connections from the peer nodes and the tester +host all all 0.0.0.0/0 trust +_EOF_ + +pg_ctl -D "$PGDATA" -l /home/pgedge/logfile.log -o "-k /tmp" -w start while ! pg_isready -h /tmp; do echo "Waiting for PostgreSQL to become ready..." sleep 1 done +# The admin user is what the tests connect as; the pgedge user is used for +# the spock node and subscription DSNs (and matches the OS user, so plain +# psql on the nodes works). +psql -U admin -d postgres -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ +ALTER USER admin PASSWORD 'password'; +CREATE ROLE pgedge SUPERUSER LOGIN; +CREATE DATABASE demo OWNER admin; +_EOF_ + echo "==========Creating tables and repsets==========" -./pgedge spock node-create $HOSTNAME "host=$HOSTNAME user=pgedge dbname=demo" demo -./pgedge spock repset-create demo_replication_set demo +psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ +CREATE EXTENSION spock; +SELECT spock.node_create('$HOSTNAME', 'host=$HOSTNAME user=pgedge dbname=demo'); +SELECT spock.repset_create('demo_replication_set'); +_EOF_ IFS=',' read -r -a peer_names <<< "$PEER_NAMES" @@ -38,10 +67,35 @@ do done done -./pgedge spock sub-create sub_${peer_names[0]}$HOSTNAME "host=${peer_names[0]} port=5432 user=pgedge dbname=demo" demo -./pgedge spock sub-create sub_${peer_names[1]}$HOSTNAME "host=${peer_names[1]} port=5432 user=pgedge dbname=demo" demo -./pgedge spock sub-add-repset sub_${peer_names[0]}$HOSTNAME demo_replication_set demo -./pgedge spock sub-add-repset sub_${peer_names[1]}$HOSTNAME demo_replication_set demo +# spock.sub_create connects to the provider synchronously, and the peer +# restarts into its final foreground postgres at the end of its own setup, +# so a connection can land in the peer's stop/start window. Retry until +# the peer is actually up. +create_sub() { + local sub_name=$1 + local provider_dsn=$2 + + for attempt in $(seq 1 60); do + # A previous attempt may have already created the subscription + if [ "$(psql -U admin -d demo -h /tmp -t -A -c "SELECT count(*) FROM spock.subscription WHERE sub_name = '$sub_name';")" = "1" ]; then + return 0 + fi + psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 \ + -c "SELECT spock.sub_create('$sub_name', '$provider_dsn');" && return 0 + echo "Retrying sub_create $sub_name..." + sleep 2 + done + echo "Failed to create subscription $sub_name" + return 1 +} + +create_sub sub_${peer_names[0]}$HOSTNAME "host=${peer_names[0]} port=5432 user=pgedge dbname=demo" +create_sub sub_${peer_names[1]}$HOSTNAME "host=${peer_names[1]} port=5432 user=pgedge dbname=demo" + +psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ +SELECT spock.sub_add_repset('sub_${peer_names[0]}$HOSTNAME', 'demo_replication_set'); +SELECT spock.sub_add_repset('sub_${peer_names[1]}$HOSTNAME', 'demo_replication_set'); +_EOF_ # Build out of the bind-mounted source tree. The mount may not be writable # by this user (host ownership / SELinux labeling), so copy to a writable @@ -55,16 +109,16 @@ cd /tmp/lolor-build make USE_PGXS=1 with_llvm=no make USE_PGXS=1 with_llvm=no install -psql -U admin -d demo -h /tmp <<_EOF_ -drop extension lolor; -create extension lolor; +psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ +create extension lolor; alter system set lolor.node to ${HOSTNAME: -1}; _EOF_ -cd /home/pgedge/pgedge -./pgedge spock repset-add-table demo_replication_set 'lolor.pg_largeobject' demo -./pgedge spock repset-add-table demo_replication_set 'lolor.pg_largeobject_metadata' demo +psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ +SELECT spock.repset_add_table('demo_replication_set', 'lolor.pg_largeobject'); +SELECT spock.repset_add_table('demo_replication_set', 'lolor.pg_largeobject_metadata'); +_EOF_ -./pgedge stop +pg_ctl -D "$PGDATA" -m fast -w stop -/home/pgedge/pgedge/pg${PG_VER}/bin/postgres -D /home/pgedge/pgedge/data/pg${PG_VER} 2>&1 +exec /home/pgedge/pgedge/pg${PG_VER}/bin/postgres -D "$PGDATA" 2>&1 diff --git a/docker/pgedge.env b/docker/pgedge.env index 794aa5a..aa66ab6 100644 --- a/docker/pgedge.env +++ b/docker/pgedge.env @@ -1,4 +1,3 @@ -REPO=https://pgedge-upstream.s3.amazonaws.com/REPO DBUSER=admin DBPASSWD=password DBNAME=demo