diff --git a/.github/actions/install-native-demo-deps/action.yml b/.github/actions/install-native-demo-deps/action.yml new file mode 100644 index 00000000000..e0eb40060bf --- /dev/null +++ b/.github/actions/install-native-demo-deps/action.yml @@ -0,0 +1,23 @@ +name: Install native demo deps +description: Install anvil (foundry), KeyDB, and the postgres server for the native demo + +runs: + using: composite + steps: + - uses: ./.github/actions/install-foundry + - shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y postgresql + # stop the auto-started cluster so the demo can bind 5432/5433 + sudo systemctl stop postgresql || true + # postgres server binaries (initdb, postgres) are not on PATH on Debian + echo "$(ls -d /usr/lib/postgresql/*/bin | sort -V | tail -1)" >> "$GITHUB_PATH" + + # keydb: extract the prebuilt server (no apt repo for 24.04) + its runtime libs + sudo apt-get install -y libsnappy1v5 libzstd1 libuuid1 libcurl4t64 libssl3t64 + curl -fsSL "https://download.keydb.dev/pkg/open_source/deb/ubuntu22.04_jammy/amd64/keydb-latest/keydb-tools_6.3.4-1~jammy1_amd64.deb" -o /tmp/keydb-tools.deb + dpkg-deb -x /tmp/keydb-tools.deb /tmp/keydb + echo "/tmp/keydb/usr/bin" >> "$GITHUB_PATH" + /tmp/keydb/usr/bin/keydb-server --version diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2f7de0b113..6c5c95434b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -373,6 +373,7 @@ jobs: - uses: ./.github/actions/install-process-compose with: version: v1.75.2 + - uses: ./.github/actions/install-native-demo-deps - name: Download archive uses: actions/download-artifact@v8 @@ -433,6 +434,7 @@ jobs: - uses: ./.github/actions/install-process-compose with: version: v1.75.2 + - uses: ./.github/actions/install-native-demo-deps - uses: ./.github/actions/free-disk-space # TODO: this downloads all (available) artifacts, which is a bit wasteful but artifact diff --git a/flake.nix b/flake.nix index 98347835f6f..c19ff930150 100644 --- a/flake.nix +++ b/flake.nix @@ -97,6 +97,7 @@ }) ]; pkgs = import nixpkgs { inherit system overlays; }; + keydb = pkgs.callPackage ./nix/keydb.nix { }; myShell = pkgs.mkShellNoCC.override (pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { # The mold linker is around 50% faster on Linux than the default linker. stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.clangStdenv; @@ -199,6 +200,8 @@ }; }; }; + packages.keydb = keydb; + devShells.default = let stableToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; @@ -237,9 +240,8 @@ pup process-compose lazydocker # a docker compose TUI - # `postgresql` defaults to an older version (15), so we select the latest version (16) - # explicitly. - postgresql_16 + keydb + postgresql_18 # Figures graphviz diff --git a/nix/keydb.nix b/nix/keydb.nix new file mode 100644 index 00000000000..81dd5fb95b6 --- /dev/null +++ b/nix/keydb.nix @@ -0,0 +1,99 @@ +# Prebuilt KeyDB (not in nixpkgs): Linux keydb-tools .deb, macOS Homebrew bottle. +{ lib +, stdenv +, fetchurl +, autoPatchelfHook +, dpkg +, openssl +, snappy +, zstd +, bzip2 +, lz4 +, zlib +, systemd +, curl +, util-linux +, darwin +}: + +let + version = "6.3.4"; + debBase = "https://download.keydb.dev/pkg/open_source/deb/ubuntu22.04_jammy"; + + linuxSrcs = { + x86_64-linux = fetchurl { + url = "${debBase}/amd64/keydb-latest/keydb-tools_${version}-1~jammy1_amd64.deb"; + sha256 = "8653a2631858d6e58106e8607de61d4031b2d0c7bba0a8040859df65ea98724b"; + }; + aarch64-linux = fetchurl { + url = "${debBase}/arm64/keydb-latest/keydb-tools_${version}-1~jammy1_arm64.deb"; + sha256 = "5496d9d116cf2449b7be1b12da866f6d882ccb531ace45c8a3d679fcd0ae98af"; + }; + }; + + # sha256 == ghcr blob digest. + darwinBottleSha = "eefed6df2c14cfbab28ac8ce65f888d011bed8a1edec7095b891ba2b418ea733"; + darwinSrc = fetchurl { + url = "https://ghcr.io/v2/homebrew/core/keydb/blobs/sha256:${darwinBottleSha}"; + sha256 = darwinBottleSha; + curlOptsList = [ "-H" "Authorization: Bearer QQ==" ]; + name = "keydb-bottle-${version}.tar.gz"; + }; + + src = + if stdenv.isDarwin then darwinSrc + else linuxSrcs.${stdenv.hostPlatform.system} + or (throw "keydb.nix: unsupported system ${stdenv.hostPlatform.system}"); +in +stdenv.mkDerivation { + pname = "keydb"; + inherit version src; + + nativeBuildInputs = + lib.optionals stdenv.isLinux [ autoPatchelfHook dpkg ] + ++ lib.optionals stdenv.isDarwin [ darwin.autoSignDarwinBinariesHook ]; + + buildInputs = + lib.optionals stdenv.isLinux [ + (lib.getLib stdenv.cc.cc) + openssl + snappy + zstd + bzip2 + lz4 + zlib + (lib.getLib systemd) + curl + (lib.getLib util-linux) + ] + ++ lib.optionals stdenv.isDarwin [ openssl ]; + + unpackPhase = + if stdenv.isLinux + then "dpkg-deb -x $src ." + else "tar xzf $src"; + + dontBuild = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + '' + (if stdenv.isLinux then '' + cp usr/bin/keydb-server $out/bin/ + '' else '' + cp keydb/${version}/bin/keydb-server $out/bin/ + install_name_tool \ + -change @@HOMEBREW_PREFIX@@/opt/openssl@3/lib/libcrypto.3.dylib ${lib.getLib openssl}/lib/libcrypto.3.dylib \ + -change @@HOMEBREW_PREFIX@@/opt/openssl@3/lib/libssl.3.dylib ${lib.getLib openssl}/lib/libssl.3.dylib \ + $out/bin/keydb-server + '') + '' + runHook postInstall + ''; + + meta = { + description = "KeyDB server, prebuilt, for the native demo CDN discovery store"; + homepage = "https://keydb.dev"; + platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; + mainProgram = "keydb-server"; + }; +} diff --git a/process-compose.yaml b/process-compose.yaml index b0781ae81cc..5e6148876ae 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -1,7 +1,7 @@ version: "3" -# When adding/removing native processes (non-docker commands), update the NATIVE_PROCESSES -# array in scripts/cleanup-process-compose to ensure proper cleanup on test failures. +# When adding/removing processes, update the NATIVE_PROCESSES array in +# scripts/cleanup-process-compose to ensure proper cleanup on test failures. x-availability: # terminate process compose if process exits @@ -21,13 +21,12 @@ environment: - ESPRESSO_L1_PROVIDER=http://localhost:${ESPRESSO_L1_PORT} - ESPRESSO_NODE_GENESIS_FILE=${ESPRESSO_NODE_GENESIS_FILE:-data/genesis/demo.toml} - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:${ESPRESSO_STATE_RELAY_SERVER_PORT} - - QUERY_SERVICE_URI=http://localhost:${ESPRESSO_NODE_1_API_PORT}/v0/ - - NODE_VALIDATOR_URI=ws://localhost:${ESPRESSO_NODE_VALIDATOR_PORT}/v0/ processes: - # Cheating a bit here but since we don't usually have to debug go-ethereum - # it's using the docker compose service which is a bit easier. demo-l1-network: - command: docker compose up demo-l1-network --force-recreate --renew-anon-volumes + command: >- + anvil --host 127.0.0.1 --port ${ESPRESSO_L1_PORT} --chain-id 31337 + --accounts 20 --balance 1000000000 --block-time 1 + --slots-in-an-epoch 32 readiness_probe: exec: command: >- @@ -203,16 +202,6 @@ processes: path: /healthcheck availability: *run-forever - telemetry-collector: - command: docker compose up vector --force-recreate --renew-anon-volumes - log_location: logs/vector.log - # Telemetry is not load-bearing: a failure (e.g. Docker Hub pull timeout in CI) - # must not tear down the whole demo like *run-forever would. - availability: - restart: on_failure - backoff_seconds: 5 - max_restarts: 5 - prover-one-shot: # "nice" this compute heavy process to reduce load command: nice state-prover @@ -304,7 +293,7 @@ processes: command: espresso-node -- storage-sql -- http -- config -- query -- submit -- explorer -- catchup environment: - ESPRESSO_NODE_TELEMETRY_METRICS_ENABLE=true - - ESPRESSO_NODE_TELEMETRY_ENDPOINT=http://localhost:${ESPRESSO_DEMO_TELEMETRY_PRW_PORT} + - ESPRESSO_NODE_TELEMETRY_ENDPOINT=http://intentionally-unreachable-telemetry-endpoint - ESPRESSO_NODE_API_PORT=${ESPRESSO_NODE_1_API_PORT} - ESPRESSO_NODE_AXUM_PORT=${ESPRESSO_NODE_1_AXUM_PORT} - ESPRESSO_NODE_TONIC_PORT=${ESPRESSO_NODE_1_TONIC_PORT} @@ -334,7 +323,7 @@ processes: - ESPRESSO_NODE_IDENTITY_LATITUDE=39.0742 - ESPRESSO_NODE_IDENTITY_LONGITUDE=21.8243 - ESPRESSO_NODE_PUBLIC_API_URL=http://localhost:${ESPRESSO_NODE_1_API_PORT}/ - - ESPRESSO_L1_WS_PROVIDER=ws://localhost:${ESPRESSO_L1_WS_PORT} + - ESPRESSO_L1_WS_PROVIDER=ws://localhost:${ESPRESSO_L1_PORT} depends_on: orchestrator: condition: process_healthy @@ -423,7 +412,7 @@ processes: command: espresso-node -- http -- config -- query -- storage-fs environment: - ESPRESSO_NODE_TELEMETRY_LOGS_ENABLE=true - - ESPRESSO_NODE_TELEMETRY_ENDPOINT=http://localhost:${ESPRESSO_DEMO_TELEMETRY_OTLP_PORT} + - ESPRESSO_NODE_TELEMETRY_ENDPOINT=http://intentionally-unreachable-telemetry-endpoint - ESPRESSO_NODE_TELEMETRY_LOG=info - ESPRESSO_NODE_API_PORT=${ESPRESSO_NODE_3_API_PORT} - ESPRESSO_NODE_AXUM_PORT=${ESPRESSO_NODE_3_AXUM_PORT} @@ -449,7 +438,7 @@ processes: - ESPRESSO_NODE_IDENTITY_LATITUDE=35.8617 - ESPRESSO_NODE_IDENTITY_LONGITUDE=104.1954 - ESPRESSO_NODE_PUBLIC_API_URL=http://localhost:${ESPRESSO_NODE_3_API_PORT}/ - - ESPRESSO_L1_WS_PROVIDER=ws://localhost:${ESPRESSO_L1_WS_PORT} + - ESPRESSO_L1_WS_PROVIDER=ws://localhost:${ESPRESSO_L1_PORT} depends_on: orchestrator: condition: process_healthy @@ -564,13 +553,8 @@ processes: failure_threshold: 100 availability: *run-forever - # We use KeyDB (a Redis variant) to maintain consistency between - # different parts of the CDN - # Cheating a bit here too, but KeyDB is not available as a Nix package. - # Could do local (SQLite) discovery, but removes some of the spirit - # from the local demo. keydb: - command: docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb --requirepass changeme! + command: keydb-server --bind 127.0.0.1 --port 6379 --requirepass 'changeme!' readiness_probe: exec: command: nc -zv localhost 6379 @@ -719,48 +703,25 @@ processes: availability: *run-forever espresso-node-db-0: - command: - docker run -e POSTGRES_PASSWORD -e POSTGRES_USER -e POSTGRES_DB -p ${ESPRESSO_NODE_0_DB_PORT}:5432 postgres - environment: - - POSTGRES_PASSWORD=password - - POSTGRES_USER=root - - POSTGRES_DB=espresso + command: scripts/run-postgres ${ESPRESSO_NODE_0_DB_PORT} ${ESPRESSO_BASE_STORAGE_PATH}/pg0 readiness_probe: exec: - command: pg_isready -h localhost -p ${ESPRESSO_NODE_0_DB_PORT} + # gate on the espresso db, not just the server accepting connections + command: psql -h localhost -p ${ESPRESSO_NODE_0_DB_PORT} -U root -d espresso -c 'SELECT 1' initial_delay_seconds: 5 period_seconds: 5 timeout_seconds: 4 - # Postgres can be falsely "ready" once before running init scripts. - # See https://github.com/docker-library/postgres/issues/146 for discussion. - success_threshold: 2 failure_threshold: 20 availability: *run-forever espresso-node-db-1: - command: - docker run -e POSTGRES_PASSWORD -e POSTGRES_USER -e POSTGRES_DB -p ${ESPRESSO_NODE_1_DB_PORT}:5432 postgres - environment: - - POSTGRES_PASSWORD=password - - POSTGRES_USER=root - - POSTGRES_DB=espresso + command: scripts/run-postgres ${ESPRESSO_NODE_1_DB_PORT} ${ESPRESSO_BASE_STORAGE_PATH}/pg1 readiness_probe: exec: - command: pg_isready -h localhost -p ${ESPRESSO_NODE_1_DB_PORT} + # gate on the espresso db, not just the server accepting connections + command: psql -h localhost -p ${ESPRESSO_NODE_1_DB_PORT} -U root -d espresso -c 'SELECT 1' initial_delay_seconds: 5 period_seconds: 5 timeout_seconds: 4 - # Postgres can be falsely "ready" once before running init scripts. - # See https://github.com/docker-library/postgres/issues/146 for discussion. - success_threshold: 2 failure_threshold: 20 availability: *run-forever - - block-explorer: - command: - docker run --rm -p ${ESPRESSO_BLOCK_EXPLORER_PORT}:3000 -e QUERY_SERVICE_URI -e NODE_VALIDATOR_URI - ghcr.io/espressosystems/espresso-block-explorer:main - depends_on: - espresso-node-1: - condition: process_healthy - availability: *run-forever diff --git a/scripts/cleanup-process-compose b/scripts/cleanup-process-compose index 9703ea282a2..a328e9501f4 100755 --- a/scripts/cleanup-process-compose +++ b/scripts/cleanup-process-compose @@ -2,12 +2,11 @@ # # Cleanup script for espresso-network demo # -# This script safely terminates all processes and containers started by the native demo. +# This script safely terminates all processes started by the native demo. # It's called automatically by scripts/demo-native when the demo exits (via trap). # # Key features: # - Only kills processes running from this project directory (checks /proc/$pid/cwd) -# - Cleans up Docker containers and networks # - Safe to run multiple times # set -e @@ -56,6 +55,9 @@ NATIVE_PROCESSES=( state-prover espresso-bridge staking-cli + anvil + postgres + keydb-server ) # Helper function to safely kill processes only if they're running from this project @@ -86,11 +88,7 @@ kill_project_processes "process-compose" TERM sleep 2 kill_project_processes "process-compose" KILL -# 2. Stop all docker containers via docker compose -echo "Stopping docker services..." -docker compose down --remove-orphans --volumes 2>/dev/null || echo "docker-compose not running or already stopped" - -# 3. Kill lingering native processes (only if running from this project directory) +# 2. Kill lingering native processes (only if running from this project directory) echo "Killing native processes from $PROJECT_DIR..." for proc in "${NATIVE_PROCESSES[@]}"; do kill_project_processes "$proc" TERM diff --git a/scripts/run-postgres b/scripts/run-postgres new file mode 100755 index 00000000000..8f257305b17 --- /dev/null +++ b/scripts/run-postgres @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Native postgres for the demo: initdb, create the espresso db, run in foreground. +set -euo pipefail + +readonly SCRIPT_NAME="${0##*/}" + +usage() { + cat <&2 +} + +main() { + if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 + fi + + if [[ $# -ne 2 ]]; then + err "Expected 2 arguments, got $#" + usage >&2 + exit 1 + fi + + local port="$1" + local datadir="$2" + local pg_pid + + initdb -D "${datadir}" -U root --auth=trust >/dev/null + + postgres -D "${datadir}" -p "${port}" -k "${datadir}" -h 127.0.0.1 & + pg_pid=$! + + # SIGINT = fast shutdown (SIGTERM waits for clients). Flag lets the readiness + # loop exit if signalled before postgres is ready. + local shutting_down=0 + trap 'shutting_down=1; kill -INT "${pg_pid}" 2>/dev/null' TERM INT + + until pg_isready -h 127.0.0.1 -p "${port}" >/dev/null 2>&1; do + [[ "${shutting_down}" -eq 1 ]] && break + sleep 0.5 + done + + if [[ "${shutting_down}" -eq 0 ]]; then + createdb -h 127.0.0.1 -p "${port}" -U root espresso + fi + + # first wait returns on trap; second blocks until postgres actually exits. + wait "${pg_pid}" || true + wait "${pg_pid}" 2>/dev/null || true +} + +main "$@"