diff --git a/.azure-pipelines/build-sairedis-template.yml b/.azure-pipelines/build-sairedis-template.yml index a88922f2b..88546894c 100644 --- a/.azure-pipelines/build-sairedis-template.yml +++ b/.azure-pipelines/build-sairedis-template.yml @@ -101,6 +101,8 @@ jobs: patterns: | target/debs/${{ parameters.debian_version }}/libyang-*_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/python3-libyang_*.deb displayName: "Download libyang from common lib" - script: | set -ex diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index 306629f10..939f8863e 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -104,9 +104,13 @@ jobs: target/debs/${{ parameters.debian_version }}/libnl-nf*.deb target/debs/${{ parameters.debian_version }}/libyang-*_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/python3-libyang_*.deb target/debs/${{ parameters.debian_version }}/libprotobuf*.deb target/debs/${{ parameters.debian_version }}/libprotoc*.deb target/debs/${{ parameters.debian_version }}/protobuf-compiler*.deb + target/debs/${{ parameters.debian_version }}/libnexthopgroup_*.deb + target/debs/${{ parameters.debian_version }}/libnexthopgroup-dev_*.deb displayName: "Download common libs" - task: DownloadPipelineArtifact@2 inputs: diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 52a58be1d..825e4ccf8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -83,6 +83,9 @@ jobs: target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb + target/debs/${{ parameters.debian_version }}/python3-libyang_*.deb displayName: "Download libyang from ${{ parameters.arch }} common lib" condition: ne('${{ parameters.debian_version }}', 'trixie') - task: DownloadPipelineArtifact@2 @@ -102,12 +105,50 @@ jobs: target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb + target/debs/${{ parameters.debian_version }}/python3-libyang_*.deb target/debs/${{ parameters.debian_version }}/libpcre*.deb displayName: "Download libyang from ${{ parameters.arch }} common lib" condition: eq('${{ parameters.debian_version }}', 'trixie') - script: | set -ex - sudo dpkg -i $(find ./download -name *.deb) + # Exclude libyang-dev_3*.deb from the dpkg install: it has the same + # package name (``libyang-dev``) as the libyang1 libyang-dev that the + # C++ compile still needs. We'll instead extract it into a private + # sysroot below for the Python binding build. + find ./download -name '*.deb' ! -name 'libyang-dev_3*.deb' -print0 \ + | xargs -0 sudo dpkg -i + # The trixie common-lib pin (pipelineId 926659) predates + # sonic-net/sonic-buildimage#26584 and doesn't ship + # python3-libyang_*.deb. trixie's own apt repo doesn't have a + # ``python3-libyang`` package either, so fall back to building the + # CESNET libyang Python binding from PyPI sdist. We build against a + # local sysroot extracted from libyang3 + libyang-dev_3*.deb because + # the libyang1 ``libyang-dev`` we just installed has a clashing + # ``libyang.so`` symlink that would otherwise make the linker pick up + # the v1 ABI -- which is missing symbols like ``lyplg_ext_parse_log`` + # that the v3 Python binding needs. + if ! python3 -c 'import libyang' 2>/dev/null; then + DEBS=./download/target/debs/${{ parameters.debian_version }} + LIBYANG3_DEV_DEB=$(ls $DEBS/libyang-dev_3*.deb 2>/dev/null | head -n1 || true) + LIBYANG3_DEB=$(ls $DEBS/libyang3_3*.deb 2>/dev/null | head -n1 || true) + if [ -n "$LIBYANG3_DEV_DEB" ] && [ -n "$LIBYANG3_DEB" ]; then + SYSROOT=/tmp/libyang3-sysroot + sudo rm -rf $SYSROOT + sudo mkdir -p $SYSROOT + sudo dpkg-deb -x $LIBYANG3_DEB $SYSROOT + sudo dpkg-deb -x $LIBYANG3_DEV_DEB $SYSROOT + sudo apt-get install -y python3-cffi python3-setuptools python3-wheel python3-dev pkg-config dpkg-dev libpcre2-dev || true + MULTIARCH=$(dpkg-architecture -qDEB_HOST_MULTIARCH 2>/dev/null || gcc -print-multiarch) + sudo CFLAGS="-I$SYSROOT/usr/include" \ + LDFLAGS="-L$SYSROOT/usr/lib/$MULTIARCH -Wl,-rpath,/usr/lib/$MULTIARCH" \ + pip3 install --no-build-isolation 'libyang==3.1.0' + else + sudo apt-get install -y python3-cffi python3-setuptools python3-wheel python3-dev pkg-config || true + sudo pip3 install --no-build-isolation 'libyang==3.1.0' + fi + fi workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install libyang from common lib" - task: DownloadPipelineArtifact@2 diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 6dc9476b9..5bf3d14c3 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -57,6 +57,8 @@ jobs: target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb displayName: "Download libyang from common lib" - task: DownloadPipelineArtifact@2 inputs: @@ -94,10 +96,31 @@ jobs: sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh # Install libyang packages from downloaded artifacts - sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-*_1.0*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + DEBS=$(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }} + sudo dpkg -i $DEBS/libyang-*_1.0*.deb \ + $DEBS/libyang_1.0*.deb \ + $DEBS/libyang-cpp_*.deb \ + $DEBS/python3-yang_*.deb + # libyang3 system library is needed because sonic-yang-mgmt + # (sonic-net/sonic-buildimage#26584) was ported from libyang1 to libyang3. + sudo dpkg -i $DEBS/libyang3_3*.deb + # Build the CESNET libyang Python binding from sdist against a local + # sysroot extracted from libyang3 + libyang-dev_3*.deb. We can't dpkg- + # install python3-libyang_3*.deb directly (bookworm-built; requires + # Python >= 3.11 while this pool may run Python 3.10), nor can we + # install libyang-dev_3*.deb system-wide (same package name as the + # libyang1 libyang-dev that other components still need; its + # `libyang.so` symlink would otherwise mask v1's). PyPI doesn't publish + # a binary wheel for libyang, so a from-source build is the only option. + SYSROOT=/tmp/libyang3-sysroot + sudo rm -rf $SYSROOT + sudo mkdir -p $SYSROOT + sudo dpkg-deb -x $DEBS/libyang3_3*.deb $SYSROOT + sudo dpkg-deb -x $DEBS/libyang-dev_3*.deb $SYSROOT + sudo apt-get install -y python3-cffi python3-setuptools python3-wheel python3-dev pkg-config libpcre2-dev + sudo CFLAGS="-I$SYSROOT/usr/include" \ + LDFLAGS="-L$SYSROOT/usr/lib/x86_64-linux-gnu -Wl,-rpath,/usr/lib/x86_64-linux-gnu" \ + pip3 install --no-build-isolation 'libyang==3.1.0' sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb diff --git a/azure-pipelines.yml b/azure-pipelines.yml index afc94b712..74b08d402 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -146,13 +146,40 @@ stages: target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + target/debs/${{ parameters.debian_version }}/libyang3_3*.deb + target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb displayName: "Download yang deb from amd64 common lib" - script: | set -ex - sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang_1.0*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-*_1.0*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-cpp_*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/python3-yang_*.deb + DEBS=$(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }} + sudo dpkg -i $DEBS/libyang_1.0*.deb \ + $DEBS/libyang-*_1.0*.deb \ + $DEBS/libyang-cpp_*.deb \ + $DEBS/python3-yang_*.deb + # libyang3 system library is needed because sonic-yang-mgmt + # (sonic-net/sonic-buildimage#26584) was ported from libyang1 to libyang3. + sudo dpkg -i $DEBS/libyang3_3*.deb + # We can't dpkg-install python3-libyang_3*.deb here: the bookworm-built + # deb requires Python >= 3.11, but Microsoft-hosted ubuntu-22.04 ships + # Python 3.10. PyPI doesn't publish a binary wheel for libyang either, + # so we build the CESNET libyang Python binding from sdist against a + # local sysroot extracted from libyang3 + libyang-dev_3*.deb (we can't + # install libyang-dev_3*.deb system-wide because it has the same + # package name as the libyang1 libyang-dev that our C++ compile still + # needs, and its `libyang.so` symlink would otherwise mask v1's). + SYSROOT=/tmp/libyang3-sysroot + sudo rm -rf $SYSROOT + sudo mkdir -p $SYSROOT + sudo dpkg-deb -x $DEBS/libyang3_3*.deb $SYSROOT + sudo dpkg-deb -x $DEBS/libyang-dev_3*.deb $SYSROOT + # Use --no-build-isolation so the sdist's cffi build picks up the + # system python3-cffi (1.15) instead of pulling an incompatible + # cffi 2.0 into an isolated overlay (where it would mismatch the + # system _cffi_backend.so). + sudo apt-get install -y python3-cffi python3-setuptools python3-wheel python3-dev pkg-config libpcre2-dev + sudo CFLAGS="-I$SYSROOT/usr/include" \ + LDFLAGS="-L$SYSROOT/usr/lib/x86_64-linux-gnu -Wl,-rpath,/usr/lib/x86_64-linux-gnu" \ + pip3 install --no-build-isolation 'libyang==3.1.0' workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install yang deb from common lib" - task: DownloadPipelineArtifact@2