From df4fdf3f111a307196f904a4d963a4e64d31d379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 27 Jan 2022 12:39:39 +0100 Subject: [PATCH 1/7] DISPATCH-2144 Use python3-dbg in the Debug GHA build --- .github/workflows/build.yaml | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ee9548b15..d14807b58 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,6 +53,7 @@ jobs: protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 + python: [/usr/bin/python3-dbg] env: BuildType: ${{matrix.buildType}} ProtonBuildDir: ${{github.workspace}}/qpid-proton/build @@ -69,6 +70,7 @@ jobs: -DENABLE_FUZZ_TESTING=OFF -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DBUILD_TLS=ON + -DPython_EXECUTABLE=${{matrix.python}} RouterCMakeExtraArgs: > -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -77,6 +79,7 @@ jobs: -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DSANITIZE_PYTHON=OFF -DBUILD_BENCHMARKS=ON + -DPython_EXECUTABLE=${{matrix.python}} CCACHE_BASEDIR: ${{github.workspace}} CCACHE_DIR: ${{github.workspace}}/.ccache @@ -127,18 +130,12 @@ jobs: - name: Create Build and Install directories run: mkdir -p "${ProtonBuildDir}" "${RouterBuildDir}" "${InstallPrefix}" - - name: Setup python - uses: actions/setup-python@v3 - with: - python-version: 3.6 - architecture: x64 - - - name: Install Python build dependencies - run: python -m pip install setuptools wheel tox - - name: Install Linux build dependencies run: | - sudo apt update; sudo apt install -y libdw-dev swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev + sudo apt update; sudo apt install -y libdw-dev swig python3-dbg libpython3-dbg libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev + + - name: Install Python build dependencies + run: ${{matrix.python}} -m pip install setuptools wheel tox - name: Zero ccache stats run: ccache -z @@ -207,6 +204,7 @@ jobs: protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 + python: [/usr/bin/python3-dbg] shard: [1, 2] shards: [2] env: @@ -218,6 +216,7 @@ jobs: LD_LIBRARY_PATH: ${{github.workspace}}/install/lib QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True + PYTHONTRACEMALLOC: 5 # this enables colored output FORCE_COLOR: 1 COLUMNS: 160 @@ -240,12 +239,6 @@ jobs: with: name: skupper_router_wrk_${{env.JOB_IDENTIFIER}} - - name: Setup python - uses: actions/setup-python@v3 - with: - python-version: 3.6 - architecture: x64 - - name: Install Linux runtime/test dependencies run: | sudo apt update; sudo apt install -y libdw1 libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl ncat gdb elfutils findutils file python3-dbg @@ -254,10 +247,10 @@ jobs: run: tar -I pixz -xf archive.tar.xz - name: Install Python runtime/test dependencies - run: python -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt + run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt - - name: install qpid-proton python wheel - run: python -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl + - name: Install qpid-proton python wheel + run: ${{matrix.python}} -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl - name: CTest working-directory: ${{env.RouterBuildDir}} @@ -460,12 +453,10 @@ jobs: - name: Zero ccache stats run: ccache -z - # PROTON-2473: either install `openssl1.1-devel` compat version, or build with `-Wno-error=deprecated-declarations` - name: qpid-proton cmake configure working-directory: ${{env.ProtonBuildDir}} run: > cmake "${{github.workspace}}/qpid-proton" \ - '-DCMAKE_C_FLAGS="-Wno-error=deprecated-declarations"' \ "-DCMAKE_INSTALL_PREFIX=${InstallPrefix}" \ "-DCMAKE_BUILD_TYPE=${BuildType}" \ ${ProtonCMakeExtraArgs} From 6253141b5c539fef0ca43edfb668ed6fa0e6d5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Mon, 2 May 2022 23:00:18 +0200 Subject: [PATCH 2/7] default ubuntu python has old pip --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d14807b58..725891082 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -246,6 +246,9 @@ jobs: - name: Unpack archive run: tar -I pixz -xf archive.tar.xz + - name: Upgrade pip + run: python3 -m pip install --upgrade pip + - name: Install Python runtime/test dependencies run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt From d4c78de93b4e9a9149f0aa3654645136387f4fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 30 Apr 2022 21:02:17 +0200 Subject: [PATCH 3/7] do note skipe any teste --- .github/workflows/build.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 725891082..8943fd7fd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -85,8 +85,6 @@ jobs: CCACHE_DIR: ${{github.workspace}}/.ccache CCACHE_COMPRESS: 'true' CCACHE_MAXSIZE: '400MB' - SKUPPER_SYSTEM_TEST_SKIP_POLICY: True - SKUPPER_SYSTEM_TEST_SKIP_DELIVERY_ABORT: True steps: # The option to enable + in sed regexps differs by OS so we avoid it, https://github.com/actions/upload-artifact/issues/22 @@ -393,8 +391,6 @@ jobs: CCACHE_MAXSIZE: '400MB' QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True - SKUPPER_SYSTEM_TEST_SKIP_POLICY: True - SKUPPER_SYSTEM_TEST_SKIP_DELIVERY_ABORT: True VERBOSE: 1 steps: From f671c7f6c1a72acb1404066e374cfc4dab021a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 27 Jan 2022 14:29:36 +0100 Subject: [PATCH 4/7] fixup leak:^_PyMem_RawRealloc$ fixup qdmanage runs with system python, --- .github/workflows/build.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8943fd7fd..c31b368d0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -214,6 +214,10 @@ jobs: LD_LIBRARY_PATH: ${{github.workspace}}/install/lib QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True + # the PyMalloc mechanism is incompatible with Valgrind, different mechanism must be set here + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONMALLOC + # https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_python.html#debug-version-of-python-memory-alloc-label + PYTHONMALLOC: malloc_debug PYTHONTRACEMALLOC: 5 # this enables colored output FORCE_COLOR: 1 @@ -250,7 +254,10 @@ jobs: - name: Install Python runtime/test dependencies run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt - - name: Install qpid-proton python wheel + - name: Replace /usr/bin/python3 with ${{matrix.python}}, for tools such as qdmanage + run: sudo ln -sf ${{matrix.python}} /usr/bin/python3 + + - name: Install qpid-proton python wheel (python3-dbg) run: ${{matrix.python}} -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl - name: CTest From ef59101650016b9be6864ac65e22e4575669d87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sun, 1 May 2022 01:22:35 +0200 Subject: [PATCH 5/7] PYTHONTRACEMALLOC: 20 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c31b368d0..e14ee72d4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -218,7 +218,7 @@ jobs: # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONMALLOC # https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_python.html#debug-version-of-python-memory-alloc-label PYTHONMALLOC: malloc_debug - PYTHONTRACEMALLOC: 5 + PYTHONTRACEMALLOC: 20 # this enables colored output FORCE_COLOR: 1 COLUMNS: 160 From 314364b76521eba05dff44c1fa37ec006c8618a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sun, 1 May 2022 01:53:39 +0200 Subject: [PATCH 6/7] j6 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e14ee72d4..f6969f8fa 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -266,7 +266,7 @@ jobs: sudo sysctl -w kernel.core_pattern="coredump.%e.%p" ulimit -c unlimited - ctest --timeout 1200 -V --output-junit=Testing/Test.xml --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j100 ${{env.RouterCTestExtraArgs}} + ctest --timeout 1200 -V --output-junit=Testing/Test.xml --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j6 ${{env.RouterCTestExtraArgs}} - name: Report coredump stacktraces (if any tests failed) if: ${{ failure() }} From 482bf98fe1eebb7d1268ed08edbea8be2510bbea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 26 May 2022 11:36:31 +0200 Subject: [PATCH 7/7] enable python-debug for centos as well can't reproduce ubuntu fail on fedora locally enable python-debug for centos as well can't reproduce ubuntu fail on fedora locally enable python-debug for fedora as well can't reproduce ubuntu fail on fedora locally enable python-debug for fedora as well can't reproduce ubuntu fail on fedora locally enable python-debug for fedora as well can't reproduce ubuntu fail on fedora locally --- .github/workflows/build.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f6969f8fa..2c3973dd8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -311,6 +311,7 @@ jobs: containerTag: ['37'] buildType: [RelWithDebInfo] runtimeCheck: [asan, tsan] + python: [/usr/bin/python3.10d] protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 @@ -322,6 +323,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }} shard: 1 shards: 2 @@ -329,6 +331,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }} shard: 2 shards: 2 @@ -336,6 +339,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: 0.38.0 shard: 1 shards: 2 @@ -343,6 +347,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: 0.38.0 shard: 2 shards: 2 @@ -386,10 +391,11 @@ jobs: -DENABLE_FUZZ_TESTING=OFF -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DBUILD_TLS=ON + -DPython_EXECUTABLE=${{matrix.python}} RouterCMakeExtraArgs: > -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DPython_EXECUTABLE=/usr/bin/python3 + -DPython_EXECUTABLE=${{matrix.python}} -DRUNTIME_CHECK=${{matrix.runtimeCheck}} CCACHE_BASEDIR: ${{github.workspace}} @@ -431,10 +437,10 @@ jobs: - name: Install Linux build dependencies run: | - dnf install -y gcc gcc-c++ elfutils-devel cmake libuuid-devel openssl openssl-devel cyrus-sasl-devel cyrus-sasl-plain swig python3-devel python3-pip make libwebsockets-devel libnghttp2-devel ccache libasan libubsan libtsan + dnf install -y gcc gcc-c++ elfutils-devel cmake libuuid-devel openssl openssl-devel cyrus-sasl-devel cyrus-sasl-plain swig python3-devel python3-debug python3-pip make libwebsockets-devel libnghttp2-devel ccache libasan libubsan libtsan - name: Install Python build dependencies - run: python3 -m pip install setuptools wheel tox + run: ${{matrix.python}} -m pip install setuptools wheel tox # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - name: Prepare ccache timestamp @@ -507,16 +513,24 @@ jobs: - name: Upgrade pip run: python3 -m pip install --upgrade pip + - name: CentOS 8 python3.6-debug needs this to build pip packages from source in next step + if: ${{ matrix.container == 'centos' }} + run: | + dnf install -y libxml2-devel libxslt-devel + - name: Install Python runtime/test dependencies - run: python3 -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt + run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt - name: Install Linux runtime/test dependencies run: | dnf install -y curl nmap-ncat gdb binutils elfutils elfutils-debuginfod-client findutils file dnf debuginfo-install -y python3 + - name: Replace /usr/bin/python3 with ${{matrix.python}}, for tools such as skmanage + run: sudo ln -sf ${{matrix.python}} /usr/bin/python3 + - name: install qpid-proton python wheel - run: python3 -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl + run: ${{matrix.python}} -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl - name: CTest working-directory: ${{env.RouterBuildDir}}