diff --git a/.github/workflows/dockertests-par.yml b/.github/workflows/dockertests-par.yml index ff6e237dc..aad8c23ce 100644 --- a/.github/workflows/dockertests-par.yml +++ b/.github/workflows/dockertests-par.yml @@ -1,6 +1,6 @@ name: Docker tests -on: [ push, pull_request, workflow_dispatch ] +on: [ pull_request, workflow_dispatch ] jobs: parallel: @@ -8,89 +8,8 @@ jobs: with: command-json-array: > [ - "make TEST=\"pg_ready_rename_test\" pg_integration_test", - "make TEST=\"pg_backup_mark_impermanent_test\" pg_integration_test", - "make TEST=\"pg_backup_mark_permanent_test\" pg_integration_test", - "make TEST=\"pg_config_test\" pg_integration_test", - "make TEST=\"pg_crypto_test\" pg_integration_test", - "make TEST=\"pg_copy_all_test\" pg_integration_test", - "make TEST=\"pg_copy_backup_test\" pg_integration_test", - "make TEST=\"pg_delta_backup_wal_delta_test\" pg_integration_test", - "make TEST=\"pg_full_backup_test\" pg_integration_test", - "make TEST=\"pg_full_backup_streamed_test\" pg_integration_test", - "make TEST=\"pg_partial_restore_test\" pg_integration_test", - "make TEST=\"pg_remote_backup_test\" pg_integration_test", - "make TEST=\"pg_remote_backup_without_files_metadata_test\" pg_integration_test", - "make TEST=\"pg_ssh_backup_test\" pg_integration_test", - "make TEST=\"pg_transfer_backup_test\" pg_integration_test", - "make TEST=\"pg_receive_wal_test\" pg_integration_test", - "make TEST=\"pg_full_backup_copy_composer_test\" pg_integration_test", - "make TEST=\"pg_full_backup_rating_composer_test\" pg_integration_test", - "make TEST=\"pg_full_backup_database_composer_test\" pg_integration_test", - "make TEST=\"pg_full_backup_failover_storage_test\" pg_integration_test", - "make TEST=\"pg_delete_before_name_find_full_test\" pg_integration_test", - "make TEST=\"pg_delete_before_permanent_delta_multist_test\" pg_integration_test", - "make TEST=\"pg_delete_before_permanent_delta_test\" pg_integration_test", - "make TEST=\"pg_delete_before_permanent_full_test\" pg_integration_test", - "make TEST=\"pg_delete_before_time_find_full_test\" pg_integration_test", - "make TEST=\"pg_delete_end_to_end_test\" pg_integration_test", - "make TEST=\"pg_delete_garbage_multist_test\" pg_integration_test", - "make TEST=\"pg_delete_garbage_test\" pg_integration_test", - "make TEST=\"pg_delete_retain_find_full_test\" pg_integration_test", - "make TEST=\"pg_delete_retain_full_multist_test\" pg_integration_test", - "make TEST=\"pg_delete_retain_full_test\" pg_integration_test", - "make TEST=\"pg_delete_retain_after_test\" pg_integration_test", - "make TEST=\"pg_delete_target_delta_find_full_test\" pg_integration_test", - "make TEST=\"pg_delete_target_delta_multist_test\" pg_integration_test", - "make TEST=\"pg_delete_target_delta_test\" pg_integration_test", - "make TEST=\"pg_delete_target_test\" pg_integration_test", - "make TEST=\"pg_delete_without_confirm_test\" pg_integration_test", - "make TEST=\"pg_delta_backup_fullscan_test\" pg_integration_test", - "make TEST=\"pg_ghost_table_test\" pg_integration_test", - "make TEST=\"pg_several_delta_backups_test\" pg_integration_test", - "make TEST=\"pg_several_delta_backups_reverse_test\" pg_integration_test", - "make TEST=\"pg_storage_swift_test\" pg_integration_test", - "make TEST=\"pg_storage_ssh_test\" pg_integration_test", - "make TEST=\"pg_wale_compatibility_test\" pg_integration_test", - "make TEST=\"pg_backup_mark_permanent_no_error_test\" pg_integration_test", - "make TEST=\"pg_daemon_test\" pg_integration_test", - "make TEST=\"pg_daemon_backup_and_restore_test\" pg_integration_test", - "make TEST=\"pg_daemon_client_test\" pg_integration_test", - "make TEST=\"prefetch_test\" pg_integration_test", - "make mongo_test", - "make MONGO_VERSION=\"8.0.3\" MONGO_REPO=\"repo.mongodb.org\" MONGO_PACKAGE=\"mongodb-org\" mongo_catch_up_features", - "make MONGO_VERSION=\"8.0.3\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_catch_up_features", - "make MONGO_VERSION=\"8.0.3\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_binary_features", - "make MONGO_VERSION=\"8.0.3\" MONGO_REPO=\"repo.mongodb.org\" MONGO_PACKAGE=\"mongodb-org\" mongo_logical_features", - "make MONGO_VERSION=\"8.0.3\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_partial_features", - "make MONGO_VERSION=\"7.0.15\" MONGO_REPO=\"repo.mongodb.org\" MONGO_PACKAGE=\"mongodb-org\" mongo_catch_up_features", - "make MONGO_VERSION=\"7.0.15\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_catch_up_features", - "make MONGO_VERSION=\"7.0.15\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_binary_features", - "make MONGO_VERSION=\"7.0.15\" MONGO_REPO=\"repo.mongodb.org\" MONGO_PACKAGE=\"mongodb-org\" mongo_logical_features", - "make MONGO_VERSION=\"7.0.15\" MONGO_REPO=\"repo.mongodb.com\" MONGO_PACKAGE=\"mongodb-enterprise\" mongo_partial_features", "make redis_test", - "make FEATURE=\"aof_backup\" IMAGE_TYPE=\"aof\" REDIS_VERSION=\"7.0.15\" redis_features", - "make FEATURE=\"aof_backup\" IMAGE_TYPE=\"aof\" REDIS_VERSION=\"7.2.4\" redis_features", "make FEATURE=\"rdb_backup\" IMAGE_TYPE=\"rdb\" REDIS_VERSION=\"6.2.4\" redis_features", - "make FEATURE=\"rdb_backup\" IMAGE_TYPE=\"rdb\" REDIS_VERSION=\"7.0.15\" redis_features", - "make FEATURE=\"rdb_backup\" IMAGE_TYPE=\"rdb\" REDIS_VERSION=\"7.2.4\" redis_features", - "make TEST=\"pg_wale_tablespace_compatibility_test\" pg_integration_test", - "make TEST=\"pg_tablespace_support_test\" pg_integration_test", - "make TEST=\"pg_walg_tablespace_restore_spec_test\" pg_integration_test", - "make TEST=\"pg_wal_perftest\" pg_integration_test", - "make TEST=\"pg_wal_perftest_with_throttling\" pg_integration_test", - "make TEST=\"pg_backup_perftest\" pg_integration_test", - "make TEST=\"pg_catchup_test\" pg_integration_test", - "make TEST=\"pg_wal_restore_test\" pg_integration_test", - "make MYSQL_TEST=mysql_base_tests mysql_integration_test", - "make MYSQL_TEST=mysql_delete_tests mysql_integration_test", - "make MYSQL_TEST=mysql_copy_tests mysql_integration_test", - "make USE_BROTLI= USE_LZO= USE_LIBSODIUM= MYSQL_TEST=mysql8_tests mysql8_integration_test", - "make mariadb_test", - "make gp_test", - "make cloudberry_test", - "make st_test", - "make TEST=\"pg_pgbackrest_backup_fetch_test\" pg_integration_test", - "make etcd_test", - "make orioledb_integration_test", + "make REDIS_VERSION=\"7.0.15\" redis_combined_features", + "make REDIS_VERSION=\"7.2.4\" redis_combined_features", ] diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index dce6cf2b8..000000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: golangci-lint -on: [ push, pull_request ] - -env: - GO_VERSION: "1.25" - GOEXPERIMENT: jsonv2 - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - name: Install deps - run: sudo apt-get install liblzo2-dev brotli libsodium-dev - - uses: actions/checkout@v6 - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - id: go - - name: golangci-lint - uses: golangci/golangci-lint-action@v9.2.0 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v2.4 diff --git a/.github/workflows/golvulncheck.yml b/.github/workflows/golvulncheck.yml deleted file mode 100644 index 363ee2f62..000000000 --- a/.github/workflows/golvulncheck.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Run govulncheck -on: - push: - pull_request: - -env: - GO_VERSION: "1.25" - USE_BROTLI: 1 - USE_LIBSODIUM: 1 - USE_LZO: 1 - GOEXPERIMENT: jsonv2 - -jobs: - scan: - runs-on: ubuntu-latest - steps: - - - name: Checkout code - uses: actions/checkout@v6 - - - name: Install deps - run: sudo apt-get install liblzo2-dev brotli libsodium-dev - - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - id: go - - - name: Get dependencies - run: | - go get -v -t -d ./... - - id: govulncheck - uses: golang/govulncheck-action@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 0c5c1f8f2..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Release workflow for tagged versions -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events to matching v*, i.e. v0.2.19, v0.2.14a - -env: - GO_VERSION: "1.25" - USE_BROTLI: 1 - USE_LIBSODIUM: 1 - USE_LZO: 1 - DEBIAN_FRONTEND: noninteractive - -jobs: - # Prevents race conditions by creating the release only once before uploads. - create-release: - runs-on: ubuntu-latest - steps: - - name: Create GitHub release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ github.ref_name }} - name: ${{ github.ref_name }} - draft: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - release-ubuntu: - needs: create-release - strategy: - matrix: - os: [20.04, 22.04, 24.04] - db: [pg, mysql, sqlserver, redis, mongo, fdb, gp] - runner: [ubuntu-24.04, ubuntu-24.04-arm] - fail-fast: false - runs-on: ${{ matrix.runner }} - name: wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ contains(matrix.runner, 'arm') &&'aarch64' || 'amd64' }} - container: - image: ubuntu:${{ matrix.os }} - steps: - - - name: Determine arch - id: determine-arch - run: | - case $(uname -m) in - 'aarch64') arch='aarch64' ;; - 'x86_64') arch='amd64' ;; - *) arch='unknown' ;; - esac - echo "arch=$arch" >> $GITHUB_OUTPUT - - - name: Install build dependencies - run: | - apt-get update - apt-get install -y liblzo2-dev brotli libsodium-dev curl git cmake build-essential tzdata - - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - id: go - - - name: Get dependencies - run: go get -v -t ./... - - - name: Mark repo and submodule as safe - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - git config --global --add safe.directory $GITHUB_WORKSPACE/submodules/brotli - - - name: Make deps - run: make deps - - - name: Build WAL-G - run: make ${{ matrix.db }}_build - - - name: Rename WAL-G binary - run: mv main/${{ matrix.db }}/wal-g wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }} - - - name: Compress WAL-G binary - run: tar --owner=0 --group=0 -zcvf wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.tar.gz wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }} - - - name: Calculate checksum - run: sha256sum wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }} > wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.sha256 - - - name: Calculate checksum for the compressed binary - run: sha256sum wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.tar.gz > wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.tar.gz.sha256 - - - name: Upload WAL-G binary - uses: softprops/action-gh-release@v2 - with: - files: | - wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }} - wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.tar.gz - wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.sha256 - wal-g-${{ matrix.db }}-${{ matrix.os }}-${{ steps.determine-arch.outputs.arch }}.tar.gz.sha256 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml deleted file mode 100644 index fee2eb669..000000000 --- a/.github/workflows/unittests.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Unit tests - -on: [ push, pull_request, workflow_dispatch ] - -env: - GO_VERSION: "1.25" - USE_BROTLI: 1 - -jobs: - unittest: - name: all_unittests - runs-on: ubuntu-latest - steps: - - name: Install deps - run: | - sudo apt-get update - sudo apt-get install -y liblzo2-dev - - - name: Check out code into the Go module directory - uses: actions/checkout@v6 - - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - id: go - - - name: Get dependencies - run: | - go get -v -t ./... - - - name: Test - run: make all_unittests - env: - USE_BROTLI: 1 - USE_LIBSODIUM: 1 - USE_LZO: 1 - TEST_MODIFIER: -race - diff --git a/.github/workflows/windows-native.yml b/.github/workflows/windows-native.yml deleted file mode 100644 index 0be2549f8..000000000 --- a/.github/workflows/windows-native.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: windows native build - -on: [ push, pull_request, workflow_dispatch] - -env: - GO_VERSION: "1.25" - LIBSODIUM_VERSION: "1.0.20" - -defaults: - run: - shell: pwsh - -jobs: - build: - name: windows build smoke test - runs-on: windows-latest - steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v6 - - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - id: go - - - name: Build dependencies - run: | - go get -v -t ./... - curl --retry 5 --retry-delay 0 -sL "https://github.com/jedisct1/libsodium/releases/download/${{ env.LIBSODIUM_VERSION }}-RELEASE/libsodium-${{ env.LIBSODIUM_VERSION }}-mingw.tar.gz" -o "libsodium-${{ env.LIBSODIUM_VERSION }}-mingw.tar.gz" - tar -xf .\libsodium-${{ env.LIBSODIUM_VERSION }}-mingw.tar.gz - mkdir tmp - mv libsodium-win64 ./tmp/libsodium - make deps - <# - Some tests create hardlinks and Windows cannot create hardlinks across volumes. - So, set TMP to RUNNER_TEMP; ideally avoiding volume crossing like: - https://github.com/actions/runner-images/issues/12386 - https://github.com/actions/runner-images/issues/12416 - https://github.com/actions/runner-images/issues/12744 - #> - echo "TMP=$env:RUNNER_TEMP" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Run unittests - run: | - make unittest - - - name: Build wal-g for SQL Server and PostgreSQL - run: | - make sqlserver_build pg_build diff --git a/Makefile b/Makefile index c9599e4c8..7dc746ef7 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,15 @@ clean_redis_features: set -e cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) go test -v -count=1 -timeout 5m --tf.test=false --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=redis +# Run both AOF and RDB tests sequentially to save infrastructure setup time +# Note: If AOF test fails, cleanup runs with --tf.test=false to only clean up (no tests) +redis_combined_features: + set -e + make go_deps + cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) IMAGE_TYPE=aof FEATURE=aof_backup go test -v -count=1 -timeout 20m --tf.test=true --tf.debug=false --tf.clean=false --tf.stop=false --tf.database=redis || \ + (cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) go test -v -count=1 -timeout 5m --tf.test=false --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=redis; exit 1) + cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) IMAGE_TYPE=rdb FEATURE=rdb_backup go test -v -count=1 -timeout 20m --tf.test=true --tf.debug=false --tf.clean=true --tf.stop=true --tf.database=redis + etcd_test: deps etcd_build unlink_brotli etcd_integration_test etcd_build: $(CMD_FILES) $(PKG_FILES)