Add CI retry logic to setup scripts

cmake/cose_openssl.cmake

@@ -60,11 +60,12 @@ add_custom_target(
   COMMAND "${CMAKE_COMMAND}" -E make_directory "${COSE_RS_CARGO_TARGET_DIR}"
   COMMAND
     "${CMAKE_COMMAND}" -E env --unset=CARGO_BUILD_TARGET
-    "RUSTFLAGS=${COSE_RS_RUSTFLAGS}" "CC=${CMAKE_C_COMPILER}"
-    "CXX=${CMAKE_CXX_COMPILER}" "AR=${CMAKE_AR}" "CARGO_BUILD_RUSTC=${RUSTC}"
-    "${CARGO}" build --lib --package "${COSE_RS_PACKAGE}" --manifest-path
-    "${COSE_RS_MANIFEST_PATH}" --target-dir "${COSE_RS_CARGO_TARGET_DIR}"
-    ${COSE_RS_CARGO_PROFILE_FLAG} --locked
+    "RUSTFLAGS=${COSE_RS_RUSTFLAGS}" "CARGO_NET_RETRY=10"
+    "CARGO_HTTP_TIMEOUT=60" "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}"
+    "AR=${CMAKE_AR}" "CARGO_BUILD_RUSTC=${RUSTC}" "${CARGO}" build --lib
+    --package "${COSE_RS_PACKAGE}" --manifest-path "${COSE_RS_MANIFEST_PATH}"
+    --target-dir "${COSE_RS_CARGO_TARGET_DIR}" ${COSE_RS_CARGO_PROFILE_FLAG}
+    --locked
   COMMAND
     "${CMAKE_COMMAND}" -E copy_if_different "${COSE_RS_CARGO_LIB_PATH}"
     "${CMAKE_BINARY_DIR}"

scripts/setup-ci.sh

@@ -2,66 +2,121 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the Apache 2.0 License.
 
-set -ex
+set -exo pipefail
 
 H2SPEC_VERSION="v2.6.0"
 
 export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(date +%s)}
 echo "Using SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
 
-# Source control
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
-    git  \
-    ca-certificates
-
-# To build CCF
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
-    build-essential  \
-    clang  \
-    cmake  \
-    ninja-build  \
-    which  \
-    openssl-devel  \
-    libuv-devel  \
-    nghttp2-devel  \
-    curl-devel  \
-    libarrow-devel  \
-    parquet-libs-devel  \
-    doxygen  \
-    clang-tools-extra-devel
-
-# To run standard tests
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
-    lldb  \
-    expect  \
-    npm  \
-    jq
-
-# Extra-dependency for CDDL schema checker
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rubygems
-gem install cddl
-
-# Release (extended) tests
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install procps
-
-# protocoltest
-tdnf --snapshottime=$SOURCE_DATE_EPOCH install -y bind-utils
-curl -L --output h2spec_linux_amd64.tar.gz https://github.com/summerwind/h2spec/releases/download/$H2SPEC_VERSION/h2spec_linux_amd64.tar.gz
-tar -xvf h2spec_linux_amd64.tar.gz
-mkdir -p /opt/h2spec
-mv h2spec /opt/h2spec/h2spec
-rm h2spec_linux_amd64.tar.gz
-
-# partitions test
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install iptables
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install strace
-
-# For packaging
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rpm-build
-
-# For end to end tests and scripts
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install python3-pip
-pip install uv==0.10.8
-
-# Rust
-tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rust
+retry() {
+    local description=$1
+    shift
+
+    if [[ -z ${CI+x} ]]; then
+        "$@"
+        return
+    fi
+
+    local attempt=1
+    local delay
+    while true; do
+        if "$@"; then
+            return
+        fi
+
+        if (( attempt == 3 )); then
+            echo "'$description' failed after 3 attempts"
+            return 1
+        fi
+
+        if (( attempt == 1 )); then
+            delay=5
+        else
+            delay=30
+        fi
+
+        echo "'$description' failed on attempt $attempt. Retrying in ${delay}s..."
+        sleep "$delay"
+        attempt=$(( attempt + 1 ))
+    done
+}
+
+install_source_control() {
+    # Source control
+    tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
+        git  \
+        ca-certificates
+}
+
+install_build_dependencies() {
+    # To build CCF
+    tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
+        build-essential  \
+        clang  \
+        cmake  \
+        ninja-build  \
+        which  \
+        openssl-devel  \
+        libuv-devel  \
+        nghttp2-devel  \
+        curl-devel  \
+        libarrow-devel  \
+        parquet-libs-devel  \
+        doxygen  \
+        clang-tools-extra-devel
+}
+
+install_test_dependencies() {
+    {
+        # To run standard tests
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install  \
+            lldb  \
+            expect  \
+            npm  \
+            jq &&
+        # Extra-dependency for CDDL schema checker
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rubygems &&
+        gem install cddl &&
+        # Release (extended) tests
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install procps &&
+        # protocoltest
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH install -y bind-utils &&
+        # partitions test
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install iptables &&
+        tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install strace
+    }
+}
+
+install_h2spec() {
+    if ! curl -L --output h2spec_linux_amd64.tar.gz https://github.com/summerwind/h2spec/releases/download/$H2SPEC_VERSION/h2spec_linux_amd64.tar.gz; then
+        echo "Failed to download h2spec"
+        return 1
+    fi
+
+    tar -xvf h2spec_linux_amd64.tar.gz &&
+    mkdir -p /opt/h2spec &&
+    mv h2spec /opt/h2spec/h2spec &&
+    rm h2spec_linux_amd64.tar.gz
+}
+
+install_packaging_and_python() {
+    # For packaging
+    tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rpm-build &&
+
+    # For end to end tests and scripts
+    tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install python3-pip &&
+    pip install uv==0.10.8
+}
+
+install_rust() {
+    # Rust
+    tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rust
+}
+
+retry "Source control dependencies" install_source_control
+retry "Build dependencies" install_build_dependencies
+retry "Test dependencies" install_test_dependencies
+retry "h2spec installation" install_h2spec
+retry "Packaging and Python dependencies" install_packaging_and_python
+retry "Rust dependencies" install_rust

scripts/setup-dev.sh

@@ -2,25 +2,75 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the Apache 2.0 License.
 
-set -ex
-
-tdnf -y install  \
-    clang-tools-extra  \
-    python-pip \
-    jq \
-    tar \
-    npm \
-    build-essential
-
-# For LTS test to extract binaries from rpms
-tdnf -y install cpio
-
-pip install gersemi
-
-# For shellcheck
-curl -L https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz  \
-    --output shellcheck.tar.gz
-mkdir -p shellcheck-dir
-tar -xvf shellcheck.tar.gz -C shellcheck-dir
-mv shellcheck-dir/shellcheck-stable/shellcheck /usr/local/bin/shellcheck
-rm -rf shellcheck-dir shellcheck.tar.gz
+set -exo pipefail
+
+retry() {
+    local description=$1
+    shift
+
+    if [[ -z ${CI+x} ]]; then
+        "$@"
+        return
+    fi
+
+    local attempt=1
+    local delay
+    while true; do
+        if "$@"; then
+            return
+        fi
+
+        if (( attempt == 3 )); then
+            echo "'$description' failed after 3 attempts"
+            return 1
+        fi
+
+        if (( attempt == 1 )); then
+            delay=5
+        else
+            delay=30
+        fi
+
+        echo "'$description' failed on attempt $attempt. Retrying in ${delay}s..."
+        sleep "$delay"
+        attempt=$(( attempt + 1 ))
+    done
+}
+
+install_dev_dependencies() {
+    tdnf -y install  \
+        clang-tools-extra  \
+        python-pip \
+        jq \
+        tar \
+        npm \
+        build-essential
+}
+
+install_lts_test_dependencies() {
+    # For LTS test to extract binaries from rpms
+    tdnf -y install cpio
+}
+
+install_python_tools() {
+    pip install gersemi
+}
+
+install_shellcheck() {
+    # For shellcheck
+    if ! curl -L https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz  \
+        --output shellcheck.tar.gz; then
+        echo "Failed to download shellcheck"
+        return 1
+    fi
+
+    mkdir -p shellcheck-dir &&
+    tar -xvf shellcheck.tar.gz -C shellcheck-dir &&
+    mv shellcheck-dir/shellcheck-stable/shellcheck /usr/local/bin/shellcheck &&
+    rm -rf shellcheck-dir shellcheck.tar.gz
+}
+
+retry "Development dependencies" install_dev_dependencies
+retry "LTS test dependencies" install_lts_test_dependencies
+retry "Python tools" install_python_tools
+retry "shellcheck installation" install_shellcheck

tests/infra/consortium.py

@@ -272,7 +272,7 @@ def get_member_by_service_id(self, service_id):
         )
 
     def vote_using_majority(
-        self, remote_node, proposal, ballot, wait_for_commit=True, timeout=5
+        self, remote_node, proposal, ballot, wait_for_commit=True, timeout=10
     ):
         response = None