diff --git a/cmake/cose_openssl.cmake b/cmake/cose_openssl.cmake index b9c0c70f2c4..367a48378e4 100644 --- a/cmake/cose_openssl.cmake +++ b/cmake/cose_openssl.cmake @@ -60,11 +60,12 @@ add_custom_target( COMMAND "${CMAKE_COMMAND}" -E make_directory "${COSE_RS_CARGO_TARGET_DIR}" COMMAND "${CMAKE_COMMAND}" -E env --unset=CARGO_BUILD_TARGET - "RUSTFLAGS=${COSE_RS_RUSTFLAGS}" "CC=${CMAKE_C_COMPILER}" - "CXX=${CMAKE_CXX_COMPILER}" "AR=${CMAKE_AR}" "CARGO_BUILD_RUSTC=${RUSTC}" - "${CARGO}" build --lib --package "${COSE_RS_PACKAGE}" --manifest-path - "${COSE_RS_MANIFEST_PATH}" --target-dir "${COSE_RS_CARGO_TARGET_DIR}" - ${COSE_RS_CARGO_PROFILE_FLAG} --locked + "RUSTFLAGS=${COSE_RS_RUSTFLAGS}" "CARGO_NET_RETRY=10" + "CARGO_HTTP_TIMEOUT=60" "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" + "AR=${CMAKE_AR}" "CARGO_BUILD_RUSTC=${RUSTC}" "${CARGO}" build --lib + --package "${COSE_RS_PACKAGE}" --manifest-path "${COSE_RS_MANIFEST_PATH}" + --target-dir "${COSE_RS_CARGO_TARGET_DIR}" ${COSE_RS_CARGO_PROFILE_FLAG} + --locked COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${COSE_RS_CARGO_LIB_PATH}" "${CMAKE_BINARY_DIR}" diff --git a/scripts/setup-ci.sh b/scripts/setup-ci.sh index af34de9e0ea..8bd689efce2 100755 --- a/scripts/setup-ci.sh +++ b/scripts/setup-ci.sh @@ -2,66 +2,121 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -set -ex +set -exo pipefail H2SPEC_VERSION="v2.6.0" export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(date +%s)} echo "Using SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" -# Source control -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ - git \ - ca-certificates - -# To build CCF -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ - build-essential \ - clang \ - cmake \ - ninja-build \ - which \ - openssl-devel \ - libuv-devel \ - nghttp2-devel \ - curl-devel \ - libarrow-devel \ - parquet-libs-devel \ - doxygen \ - clang-tools-extra-devel - -# To run standard tests -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ - lldb \ - expect \ - npm \ - jq - -# Extra-dependency for CDDL schema checker -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rubygems -gem install cddl - -# Release (extended) tests -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install procps - -# protocoltest -tdnf --snapshottime=$SOURCE_DATE_EPOCH install -y bind-utils -curl -L --output h2spec_linux_amd64.tar.gz https://github.com/summerwind/h2spec/releases/download/$H2SPEC_VERSION/h2spec_linux_amd64.tar.gz -tar -xvf h2spec_linux_amd64.tar.gz -mkdir -p /opt/h2spec -mv h2spec /opt/h2spec/h2spec -rm h2spec_linux_amd64.tar.gz - -# partitions test -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install iptables -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install strace - -# For packaging -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rpm-build - -# For end to end tests and scripts -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install python3-pip -pip install uv==0.10.8 - -# Rust -tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rust +retry() { + local description=$1 + shift + + if [[ -z ${CI+x} ]]; then + "$@" + return + fi + + local attempt=1 + local delay + while true; do + if "$@"; then + return + fi + + if (( attempt == 3 )); then + echo "'$description' failed after 3 attempts" + return 1 + fi + + if (( attempt == 1 )); then + delay=5 + else + delay=30 + fi + + echo "'$description' failed on attempt $attempt. Retrying in ${delay}s..." + sleep "$delay" + attempt=$(( attempt + 1 )) + done +} + +install_source_control() { + # Source control + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + git \ + ca-certificates +} + +install_build_dependencies() { + # To build CCF + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + build-essential \ + clang \ + cmake \ + ninja-build \ + which \ + openssl-devel \ + libuv-devel \ + nghttp2-devel \ + curl-devel \ + libarrow-devel \ + parquet-libs-devel \ + doxygen \ + clang-tools-extra-devel +} + +install_test_dependencies() { + { + # To run standard tests + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + lldb \ + expect \ + npm \ + jq && + # Extra-dependency for CDDL schema checker + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rubygems && + gem install cddl && + # Release (extended) tests + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install procps && + # protocoltest + tdnf --snapshottime=$SOURCE_DATE_EPOCH install -y bind-utils && + # partitions test + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install iptables && + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install strace + } +} + +install_h2spec() { + if ! curl -L --output h2spec_linux_amd64.tar.gz https://github.com/summerwind/h2spec/releases/download/$H2SPEC_VERSION/h2spec_linux_amd64.tar.gz; then + echo "Failed to download h2spec" + return 1 + fi + + tar -xvf h2spec_linux_amd64.tar.gz && + mkdir -p /opt/h2spec && + mv h2spec /opt/h2spec/h2spec && + rm h2spec_linux_amd64.tar.gz +} + +install_packaging_and_python() { + # For packaging + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rpm-build && + + # For end to end tests and scripts + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install python3-pip && + pip install uv==0.10.8 +} + +install_rust() { + # Rust + tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install rust +} + +retry "Source control dependencies" install_source_control +retry "Build dependencies" install_build_dependencies +retry "Test dependencies" install_test_dependencies +retry "h2spec installation" install_h2spec +retry "Packaging and Python dependencies" install_packaging_and_python +retry "Rust dependencies" install_rust diff --git a/scripts/setup-dev.sh b/scripts/setup-dev.sh index 18b36925d32..7c5616cc7fa 100755 --- a/scripts/setup-dev.sh +++ b/scripts/setup-dev.sh @@ -2,25 +2,75 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -set -ex - -tdnf -y install \ - clang-tools-extra \ - python-pip \ - jq \ - tar \ - npm \ - build-essential - -# For LTS test to extract binaries from rpms -tdnf -y install cpio - -pip install gersemi - -# For shellcheck -curl -L https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz \ - --output shellcheck.tar.gz -mkdir -p shellcheck-dir -tar -xvf shellcheck.tar.gz -C shellcheck-dir -mv shellcheck-dir/shellcheck-stable/shellcheck /usr/local/bin/shellcheck -rm -rf shellcheck-dir shellcheck.tar.gz +set -exo pipefail + +retry() { + local description=$1 + shift + + if [[ -z ${CI+x} ]]; then + "$@" + return + fi + + local attempt=1 + local delay + while true; do + if "$@"; then + return + fi + + if (( attempt == 3 )); then + echo "'$description' failed after 3 attempts" + return 1 + fi + + if (( attempt == 1 )); then + delay=5 + else + delay=30 + fi + + echo "'$description' failed on attempt $attempt. Retrying in ${delay}s..." + sleep "$delay" + attempt=$(( attempt + 1 )) + done +} + +install_dev_dependencies() { + tdnf -y install \ + clang-tools-extra \ + python-pip \ + jq \ + tar \ + npm \ + build-essential +} + +install_lts_test_dependencies() { + # For LTS test to extract binaries from rpms + tdnf -y install cpio +} + +install_python_tools() { + pip install gersemi +} + +install_shellcheck() { + # For shellcheck + if ! curl -L https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz \ + --output shellcheck.tar.gz; then + echo "Failed to download shellcheck" + return 1 + fi + + mkdir -p shellcheck-dir && + tar -xvf shellcheck.tar.gz -C shellcheck-dir && + mv shellcheck-dir/shellcheck-stable/shellcheck /usr/local/bin/shellcheck && + rm -rf shellcheck-dir shellcheck.tar.gz +} + +retry "Development dependencies" install_dev_dependencies +retry "LTS test dependencies" install_lts_test_dependencies +retry "Python tools" install_python_tools +retry "shellcheck installation" install_shellcheck diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 4a17b228055..5c213fe083a 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -272,7 +272,7 @@ def get_member_by_service_id(self, service_id): ) def vote_using_majority( - self, remote_node, proposal, ballot, wait_for_commit=True, timeout=5 + self, remote_node, proposal, ballot, wait_for_commit=True, timeout=10 ): response = None