diff --git a/Cargo.lock b/Cargo.lock index a5a1a3d0223..eda6f407535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2626,6 +2626,7 @@ dependencies = [ "anyhow", "async-broadcast", "async-lock 3.4.2", + "axum 0.8.9", "clap 4.6.1", "committable", "espresso-node", @@ -2642,7 +2643,10 @@ dependencies = [ "hotshot-types", "jf-signature 0.4.2", "rand 0.8.6", + "serde", + "serde_json", "surf-disco", + "tagged-base64", "tempfile", "test-log", "test-utils", @@ -4780,10 +4784,11 @@ version = "0.1.0" dependencies = [ "alloy", "anyhow", - "async-trait", + "axum 0.8.9", "clap 4.6.1", "committable", "escargot", + "espresso-api", "espresso-contract-deployer", "espresso-node", "espresso-types", @@ -4806,7 +4811,6 @@ dependencies = [ "test-utils", "tide-disco", "tokio", - "toml", "tracing", "url", "vbs", @@ -4854,6 +4858,7 @@ dependencies = [ "async-lock 3.4.2", "async-once-cell", "async-trait", + "axum 0.8.9", "base64 0.22.1", "base64-bytes", "bincode", @@ -6554,6 +6559,7 @@ dependencies = [ "alloy", "anyhow", "async-lock 3.4.2", + "axum 0.8.9", "blake3", "clap 4.6.1", "csv", @@ -6562,6 +6568,7 @@ dependencies = [ "libp2p-identity", "multiaddr", "serde", + "serde_json", "surf-disco", "tide-disco", "tokio", @@ -6677,6 +6684,7 @@ dependencies = [ "ark-serialize 0.5.0", "ark-srs", "ark-std 0.5.0", + "axum 0.8.9", "clap 4.6.1", "displaydoc", "espresso-contract-deployer", @@ -6702,12 +6710,12 @@ dependencies = [ "reqwest 0.12.28", "rstest", "serde", + "serde_json", "surf-disco", "test-log", "tide-disco", "time 0.3.51", "tokio", - "toml", "tracing", "url", "vbs", @@ -8673,20 +8681,24 @@ name = "light-client-query-service" version = "0.1.0" dependencies = [ "anyhow", + "axum 0.8.9", "clap 4.6.1", "espresso-node", "espresso-types", + "futures", "hotshot-query-service", "hotshot-types", "light-client", "log-panics", - "semver 1.0.28", + "serde", + "serde_json", "surf-disco", "tide-disco", "tokio", "toml", "tracing", "tracing-subscriber 0.3.23", + "url", "vbs", ] @@ -9206,6 +9218,7 @@ dependencies = [ "anyhow", "async-lock 3.4.2", "async-trait", + "axum 0.8.9", "bincode", "bitvec 1.1.0", "circular-buffer", @@ -9225,7 +9238,6 @@ dependencies = [ "serde_json", "surf-disco", "test-log", - "tide-disco", "time 0.3.51", "tokio", "toml", diff --git a/Cargo.toml b/Cargo.toml index ea1aa31fe5c..88840addb6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -215,6 +215,7 @@ ed25519-compact = "2.2.0" either = "1" es-version = { git = "https://github.com/EspressoSystems/es-version.git", branch = "main" } escargot = "0.5.10" +espresso-api = { path = "crates/espresso/api" } espresso-contract-deployer = { path = "contracts/rust/deployer" } espresso-keyset = { path = "crates/espresso/keyset" } espresso-macros = { git = "https://github.com/EspressoSystems/espresso-macros.git", tag = "0.1.0" } diff --git a/crates/builder/Cargo.toml b/crates/builder/Cargo.toml index 7d9d4f911d3..477112ae2da 100644 --- a/crates/builder/Cargo.toml +++ b/crates/builder/Cargo.toml @@ -10,6 +10,7 @@ alloy = { workspace = true } anyhow = { workspace = true } async-broadcast = { workspace = true } async-lock = { workspace = true } +axum = { workspace = true } clap = { workspace = true } committable = { workspace = true } espresso-node = { workspace = true, default-features = false } @@ -25,7 +26,10 @@ hotshot-example-types = { workspace = true } hotshot-state-prover = { workspace = true } hotshot-types = { workspace = true } rand = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } surf-disco = { workspace = true } +tagged-base64 = { workspace = true } test-utils = { workspace = true } tide-disco = { workspace = true } tokio = { workspace = true } diff --git a/crates/builder/src/api.rs b/crates/builder/src/api.rs new file mode 100644 index 00000000000..02215b6f211 --- /dev/null +++ b/crates/builder/src/api.rs @@ -0,0 +1,382 @@ +//! Axum port of the builder's `block_info` and `txn_submit` tide-disco modules +//! (`hotshot_builder_api::v0_1::builder::{define_api, submit_api}`). +//! +//! Route paths, status codes and the wire error type (`BuilderApiError`) are taken directly from +//! `hotshot-builder-api`'s handler bodies, so the node's `BuilderClient` +//! (`crates/hotshot/task-impls/src/builder.rs`), which is unmodified, keeps working against this +//! server unchanged. + +use std::sync::Arc; + +use axum::{ + Json, Router, + body::Bytes, + extract::{Path, State}, + http::{HeaderMap, StatusCode as AxumStatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use committable::Committable; +use hotshot_builder_api::v0_1::{ + block_info::{ + AvailableBlockData, AvailableBlockHeaderInputV1, AvailableBlockHeaderInputV2, + AvailableBlockInfo, + }, + builder::{Error as BuilderApiError, RequestError, TransactionStatus}, + data_source::{AcceptsTxnSubmits, BuilderDataSource}, +}; +use hotshot_builder_legacy::service::ProxyGlobalState; +use hotshot_types::{ + data::VidCommitment, + traits::{node_implementation::NodeType, signature_key::SignatureKey}, + utils::BuilderCommitment, +}; +use serde::{Serialize, de::DeserializeOwned}; +use surf_disco::{Error as _, StatusCode}; +use tagged_base64::TaggedBase64; +use tide_disco::healthcheck::HealthStatus; +use vbs::{BinarySerializer, Serializer, version::StaticVersion}; + +/// Binary framing version for VBS-negotiated responses, matching `hotshot_builder_api::v0_1`'s +/// framing, which is what `BuilderClient` sends/expects. +type WireVersion = StaticVersion<0, 1>; + +type SharedState = Arc>; + +fn wants_binary(headers: &HeaderMap) -> bool { + headers + .get(header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("application/octet-stream")) +} + +/// Maps tide-disco's `StatusCode` (what `BuilderApiError::status()` returns) onto axum's. +fn wire_status(status: StatusCode) -> AxumStatusCode { + AxumStatusCode::from_u16(u16::from(status)).unwrap_or(AxumStatusCode::INTERNAL_SERVER_ERROR) +} + +fn encode_ok(headers: &HeaderMap, value: T) -> Response { + if wants_binary(headers) { + match Serializer::::serialize(&value) { + Ok(bytes) => { + ([(header::CONTENT_TYPE, "application/octet-stream")], bytes).into_response() + }, + Err(err) => encode_err( + headers, + BuilderApiError::Custom { + message: err.to_string(), + status: StatusCode::INTERNAL_SERVER_ERROR, + }, + ), + } + } else { + Json(value).into_response() + } +} + +fn encode_err(headers: &HeaderMap, err: BuilderApiError) -> Response { + let status = wire_status(err.status()); + if wants_binary(headers) { + match Serializer::::serialize(&err) { + Ok(bytes) => ( + status, + [(header::CONTENT_TYPE, "application/octet-stream")], + bytes, + ) + .into_response(), + Err(_) => (status, Json(err)).into_response(), + } + } else { + (status, Json(err)).into_response() + } +} + +fn respond(headers: &HeaderMap, result: Result) -> Response { + match result { + Ok(v) => encode_ok(headers, v), + Err(e) => encode_err(headers, e), + } +} + +/// Decodes a request body the way tide-disco's `body_auto` did: VBS for +/// `application/octet-stream`, JSON for `application/json`. +fn decode_body(headers: &HeaderMap, body: &[u8]) -> Result { + match headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + { + Some("application/json") => serde_json::from_slice(body).map_err(|_| RequestError::Json), + Some("application/octet-stream") => { + Serializer::::deserialize(body).map_err(|_| RequestError::Binary) + }, + _ => Err(RequestError::UnsupportedContentType), + } +} + +fn tb64_request_error(field: &str) -> BuilderApiError { + BuilderApiError::Request(RequestError::TaggedBase64 { + reason: format!("invalid tagged base64 for {field}"), + }) +} + +/// Parses a hash-type path parameter (`parent_hash`, `block_hash`, `transaction_hash`). Any +/// failure is an `Error::Request`, which `BuilderApiError::status()` maps to 400, like tide's +/// `blob_param` path did for these params. +fn parse_hash_param(value: &str, field: &str) -> Result +where + T: for<'a> TryFrom<&'a TaggedBase64>, +{ + let tb64: TaggedBase64 = value.parse().map_err(|_| tb64_request_error(field))?; + T::try_from(&tb64).map_err(|_| tb64_request_error(field)) +} + +/// Parses a key/signature path parameter, mirroring `try_extract_param`: a wrong-type value is a +/// `Custom` error carrying 422. Note `BuilderApiError::status()` returns 500 for every `Custom`, +/// so the wire status is 500, as it was under tide. +fn parse_key_param(value: &str, field: &str) -> Result +where + T: for<'a> TryFrom<&'a TaggedBase64>, +{ + let tb64: TaggedBase64 = value.parse().map_err(|_| tb64_request_error(field))?; + T::try_from(&tb64).map_err(|_| BuilderApiError::Custom { + message: format!("Invalid {field}"), + status: StatusCode::UNPROCESSABLE_ENTITY, + }) +} + +type Sender = ::SignatureKey; +type Signature = ::PureAssembledSignatureType; + +fn parse_sender_signature( + sender: &str, + signature: &str, +) -> Result<(Sender, Signature), BuilderApiError> { + let sender = parse_key_param::(sender, "sender")?; + let signature = parse_key_param::(signature, "signature")?; + Ok((sender, signature)) +} + +/// Tide-disco-compatible singleton-app healthcheck: a bare [`HealthStatus`], so +/// `BuilderClient::connect` can decode it in both JSON and binary form. +async fn healthcheck(headers: HeaderMap) -> Response { + encode_ok(&headers, HealthStatus::Available) +} + +// --- block_info ----------------------------------------------------------------------------- + +async fn available_blocks( + State(state): State, + headers: HeaderMap, + Path((parent_hash, view_number, sender, signature)): Path<(String, u64, String, String)>, +) -> Response { + let result: Result>, BuilderApiError> = async { + let hash = parse_hash_param::(&parent_hash, "parent_hash")?; + let (sender, signature) = parse_sender_signature(&sender, &signature)?; + state + .available_blocks(&hash, view_number, sender, &signature) + .await + .map_err(|source| BuilderApiError::BlockAvailable { + source, + resource: hash.to_string(), + }) + } + .await; + respond(&headers, result) +} + +async fn claim_block( + State(state): State, + headers: HeaderMap, + Path((block_hash, view_number, sender, signature)): Path<(String, u64, String, String)>, +) -> Response { + let result: Result, BuilderApiError> = async { + let hash = parse_hash_param::(&block_hash, "block_hash")?; + let (sender, signature) = parse_sender_signature(&sender, &signature)?; + state + .claim_block(&hash, view_number, sender, &signature) + .await + .map_err(|source| BuilderApiError::BlockClaim { + source, + resource: hash.to_string(), + }) + } + .await; + respond(&headers, result) +} + +async fn claim_block_with_num_nodes( + State(state): State, + headers: HeaderMap, + Path((block_hash, view_number, sender, signature, num_nodes)): Path<( + String, + u64, + String, + String, + usize, + )>, +) -> Response { + let result: Result, BuilderApiError> = async { + let hash = parse_hash_param::(&block_hash, "block_hash")?; + let (sender, signature) = parse_sender_signature(&sender, &signature)?; + state + .claim_block_with_num_nodes(&hash, view_number, sender, &signature, num_nodes) + .await + .map_err(|source| BuilderApiError::BlockClaim { + source, + resource: hash.to_string(), + }) + } + .await; + respond(&headers, result) +} + +async fn claim_header_input( + State(state): State, + headers: HeaderMap, + Path((block_hash, view_number, sender, signature)): Path<(String, u64, String, String)>, +) -> Response { + let result: Result, BuilderApiError> = async { + let hash = parse_hash_param::(&block_hash, "block_hash")?; + let (sender, signature) = parse_sender_signature(&sender, &signature)?; + state + .claim_block_header_input(&hash, view_number, sender, &signature) + .await + .map_err(|source| BuilderApiError::BlockClaim { + source, + resource: hash.to_string(), + }) + } + .await; + respond(&headers, result) +} + +async fn claim_header_input_v2( + State(state): State, + headers: HeaderMap, + Path((block_hash, view_number, sender, signature)): Path<(String, u64, String, String)>, +) -> Response { + let result: Result, BuilderApiError> = + async { + let hash = parse_hash_param::(&block_hash, "block_hash")?; + let (sender, signature) = parse_sender_signature(&sender, &signature)?; + let input = state + .claim_block_header_input(&hash, view_number, sender, &signature) + .await + .map_err(|source| BuilderApiError::BlockClaim { + source, + resource: hash.to_string(), + })?; + Ok(AvailableBlockHeaderInputV2 { + fee_signature: input.fee_signature, + sender: input.sender, + }) + } + .await; + respond(&headers, result) +} + +async fn builder_address(State(state): State, headers: HeaderMap) -> Response { + let result = state.builder_address().await.map_err(BuilderApiError::from); + respond(&headers, result) +} + +// --- txn_submit ------------------------------------------------------------------------------- + +async fn submit_txn(State(state): State, headers: HeaderMap, body: Bytes) -> Response { + let result = async { + let tx: ::Transaction = + decode_body(&headers, &body).map_err(BuilderApiError::TxnUnpack)?; + let hash = tx.commit(); + state + .submit_txns(vec![tx]) + .await + .map_err(BuilderApiError::TxnSubmit)?; + Ok(hash) + } + .await; + respond(&headers, result) +} + +async fn submit_batch( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Response { + let result = async { + let txns: Vec<::Transaction> = + decode_body(&headers, &body).map_err(BuilderApiError::TxnUnpack)?; + let hashes = txns.iter().map(|tx| tx.commit()).collect::>(); + state + .submit_txns(txns) + .await + .map_err(BuilderApiError::TxnSubmit)?; + Ok(hashes) + } + .await; + respond(&headers, result) +} + +async fn get_status( + State(state): State, + headers: HeaderMap, + Path(transaction_hash): Path, +) -> Response { + let result: Result = async { + let hash = parse_hash_param(&transaction_hash, "transaction_hash")?; + state + .txn_status(hash) + .await + .map_err(BuilderApiError::TxnStat) + } + .await; + respond(&headers, result) +} + +fn block_info_router(state: SharedState) -> Router { + Router::new() + .route( + "/availableblocks/{parent_hash}/{view_number}/{sender}/{signature}", + get(available_blocks), + ) + .route( + "/claimblock/{block_hash}/{view_number}/{sender}/{signature}", + get(claim_block), + ) + .route( + "/claimblockwithnumnodes/{block_hash}/{view_number}/{sender}/{signature}/{num_nodes}", + get(claim_block_with_num_nodes), + ) + .route( + "/claimheaderinput/{block_hash}/{view_number}/{sender}/{signature}", + get(claim_header_input), + ) + .route( + "/claimheaderinput/v2/{block_hash}/{view_number}/{sender}/{signature}", + get(claim_header_input_v2), + ) + .route("/builderaddress", get(builder_address)) + .with_state(state) +} + +fn txn_submit_router(state: SharedState) -> Router { + Router::new() + .route("/submit", post(submit_txn)) + .route("/batch", post(submit_batch)) + .route("/status/{transaction_hash}", get(get_status)) + .with_state(state) +} + +/// Builds the full router: `healthcheck`, plus `block_info` and `txn_submit`, served both +/// unversioned and under `/v0` (both modules were registered with API version major `0`, tide's +/// convention for the module's only registered major version). +pub fn router(state: ProxyGlobalState) -> Router { + let state: SharedState = Arc::new(state); + let block_info = block_info_router(state.clone()); + let txn_submit = txn_submit_router(state); + Router::new() + .route("/healthcheck", get(healthcheck)) + .nest("/block_info", block_info.clone()) + .nest("/block_info/v0", block_info) + .nest("/txn_submit", txn_submit.clone()) + .nest("/txn_submit/v0", txn_submit) +} diff --git a/crates/builder/src/lib.rs b/crates/builder/src/lib.rs index e8cdf0c7d1b..14a42f727c5 100755 --- a/crates/builder/src/lib.rs +++ b/crates/builder/src/lib.rs @@ -1,41 +1,26 @@ -use espresso_node::SequencerApiVersion; use espresso_types::SeqTypes; -use hotshot_builder_api::v0_1::builder::{ - Error as BuilderApiError, Options as HotshotBuilderApiOptions, -}; use hotshot_builder_legacy::service::ProxyGlobalState; -use tide_disco::{App, Url}; -use tokio::spawn; -use vbs::version::{StaticVersion, StaticVersionType}; +use tokio::{net::TcpListener, spawn}; +use url::Url; +pub mod api; pub mod non_permissioned; -// It runs the api service for the builder +/// Runs the builder's `block_info` and `txn_submit` API service in the background. pub fn run_builder_api_service(url: Url, source: ProxyGlobalState) { - // it is to serve hotshot - let builder_api = hotshot_builder_api::v0_1::builder::define_api::< - ProxyGlobalState, - SeqTypes, - >(&HotshotBuilderApiOptions::default()) - .expect("Failed to construct the builder APIs"); - - // it enables external clients to submit txn to the builder's private mempool - let private_mempool_api = hotshot_builder_api::v0_1::builder::submit_api::< - ProxyGlobalState, - SeqTypes, - StaticVersion<0, 1>, - >(&HotshotBuilderApiOptions::default()) - .expect("Failed to construct the builder API for private mempool txns"); - - let mut app: App, BuilderApiError> = App::with_state(source); - - app.register_module("block_info", builder_api) - .expect("Failed to register the builder API"); - - app.register_module("txn_submit", private_mempool_api) - .expect("Failed to register the private mempool API"); - - spawn(app.serve(url, SequencerApiVersion::instance())); + let router = api::router(source); + spawn(async move { + let host = url.host_str().expect("builder API url missing host"); + let port = url + .port_or_known_default() + .expect("builder API url missing port"); + let listener = TcpListener::bind((host, port)) + .await + .expect("failed to bind builder API port"); + axum::serve(listener, router) + .await + .expect("builder API server failed"); + }); } #[cfg(test)] @@ -84,8 +69,9 @@ pub mod testing { }, }; use surf_disco::Client; + use tide_disco::App; use tokio::time::sleep; - use vbs::version::{StaticVersion, Version}; + use vbs::version::{StaticVersion, StaticVersionType, Version}; use super::*; use crate::non_permissioned::BuilderConfig; diff --git a/crates/builder/src/non_permissioned.rs b/crates/builder/src/non_permissioned.rs index c3cc51d21a5..70545590a83 100644 --- a/crates/builder/src/non_permissioned.rs +++ b/crates/builder/src/non_permissioned.rs @@ -22,8 +22,8 @@ use hotshot_types::{ epoch_membership::EpochMembershipCoordinator, traits::{EncodeBytes, block_contents::GENESIS_VID_NUM_STORAGE_NODES, metrics::NoMetrics}, }; -use tide_disco::Url; use tokio::spawn; +use url::Url; use vbs::version::Version; use crate::run_builder_api_service; diff --git a/crates/espresso/dev-node/Cargo.toml b/crates/espresso/dev-node/Cargo.toml index cc470c2cc84..a94bb4e543f 100644 --- a/crates/espresso/dev-node/Cargo.toml +++ b/crates/espresso/dev-node/Cargo.toml @@ -12,8 +12,9 @@ path = "src/main.rs" alloy = { workspace = true } anyhow = { workspace = true } -async-trait = { workspace = true } +axum = { workspace = true } clap = { workspace = true } +espresso-api = { workspace = true } espresso-contract-deployer = { workspace = true } espresso-node = { path = "../node", features = ["testing", "embedded-db"] } espresso-types = { workspace = true, features = ["node"] } @@ -28,9 +29,7 @@ serde_json = { workspace = true } staking-cli = { workspace = true } tempfile = { workspace = true } test-utils = { workspace = true } -tide-disco = { workspace = true } tokio = { workspace = true } -toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } vbs = { workspace = true } @@ -45,6 +44,7 @@ rand = { workspace = true } rstest = { workspace = true } surf-disco = { workspace = true } test-log = { workspace = true } +tide-disco = { workspace = true } tokio = { workspace = true } [lints] diff --git a/crates/espresso/dev-node/src/main.rs b/crates/espresso/dev-node/src/main.rs index 4e0ba69e0e3..5c8daca158d 100644 --- a/crates/espresso/dev-node/src/main.rs +++ b/crates/espresso/dev-node/src/main.rs @@ -1,14 +1,15 @@ use std::{ collections::{BTreeMap, HashMap}, - io::{self, Read}, + io::Read, iter::{self, once}, + sync::Arc, time::Duration, }; use alloy::{ network::EthereumWallet, node_bindings::Anvil, - primitives::{Address, Bytes, U256}, + primitives::{Address, Bytes as AlloyBytes, U256}, providers::{Provider, ProviderBuilder, WalletProvider}, rpc::client::RpcClient, signers::{ @@ -17,7 +18,14 @@ use alloy::{ }, }; use anyhow::Context; -use async_trait::async_trait; +use axum::{ + Json, Router, + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; use clap::{Parser, ValueEnum}; use espresso_contract_deployer::{ self as deployer, Contract, Contracts, DEFAULT_EXIT_ESCROW_PERIOD_SECONDS, DeployedContracts, @@ -42,11 +50,7 @@ use espresso_types::{ L1ClientOptions, SeqTypes, ValidatedState, parse_duration, v0_3::ChainConfig, }; use espresso_utils::logging; -use futures::{ - FutureExt, StreamExt, - future::{BoxFuture, join_all}, - stream::FuturesUnordered, -}; +use futures::{StreamExt, future::join_all, stream::FuturesUnordered}; use hotshot_contract_adapter::sol_types::LightClientV2Mock::{self, LightClientV2MockInstance}; use hotshot_state_prover::{StateProverConfig, v2::service::run_prover_service}; use hotshot_types::{ @@ -58,7 +62,6 @@ use serde::{Deserialize, Serialize}; use staking_cli::demo::{DelegationConfig, StakingTransactions}; use tempfile::NamedTempFile; use test_utils::reserve_tcp_port; -use tide_disco::{Api, Error, StatusCode, error::ServerError, method::ReadState}; use tokio::spawn; use url::Url; use vbs::version::StaticVersionType; @@ -776,7 +779,7 @@ async fn async_main(migrated_envs: Vec<(&str, &str)>) -> anyhow::Result<()> { Ok(()) } -// ApiState is passed to the tide disco app so avoid cloning the contracts for each endpoint +// ApiState is passed to the axum router state, so avoid cloning the contracts for each endpoint #[derive(Clone)] pub struct ApiState { /// all light client proxy addresses indexed by chain_id @@ -802,21 +805,21 @@ impl Default for ApiState { impl ApiState { /// Return a client/handle to the light client proxy on `chain_id` - pub fn light_client_instance( + pub(crate) fn light_client_instance( &self, chain_id: Option, - ) -> Result, ServerError> { + ) -> Result, DevNodeError> { // if chain id is not provided, primary L1 light client is used let id = chain_id.unwrap_or(self.l1_chain_id); let proxy_addr = self.lc_proxy_addr.get(&id).ok_or_else(|| { - ServerError::catch_all( + DevNodeError::catch_all( StatusCode::INTERNAL_SERVER_ERROR, "LightClientProxy address not found for chain id {chain_id}".to_string(), ) })?; let provider_url = self.provider_urls.get(&id).ok_or_else(|| { - ServerError::catch_all( + DevNodeError::catch_all( StatusCode::INTERNAL_SERVER_ERROR, "Provider URL not found for chain id {chain_id}".to_string(), ) @@ -830,94 +833,145 @@ impl ApiState { } } -#[async_trait] -impl ReadState for ApiState { - type State = ApiState; - async fn read( - &self, - op: impl Send + for<'a> FnOnce(&'a Self::State) -> BoxFuture<'a, T> + 'async_trait, - ) -> T { - op(self).await - } +#[derive(Clone)] +struct DevNodeState { + api: ApiState, + dev_info: Arc, } -async fn run_dev_node_server( - port: u16, - client_states: ApiState, - dev_info: DevInfo, - bind_version: ApiVer, -) -> anyhow::Result<()> { - let mut app = tide_disco::App::<_, ServerError>::with_state(client_states); - let toml = toml::from_str::(include_str!("../api/espresso_dev_node.toml")) - .map_err(io::Error::other)?; - - let mut api = Api::<_, ServerError, ApiVer>::new(toml).map_err(io::Error::other)?; - api.get("devinfo", move |_, _| { - let info = dev_info.clone(); - async move { Ok(info.clone()) }.boxed() - }) - .map_err(io::Error::other)? - .at("sethotshotdown", move |req, state: &ApiState| { - async move { - let body = req - .body_auto::(ApiVer::instance()) - .map_err(ServerError::from_request_error)?; - let contract = state.light_client_instance(body.chain_id)?; - - contract - .setHotShotDownSince(U256::from(body.height)) - .send() - .await - .map_err(|err| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - })? - .watch() - .await - .map_err(|err| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - })?; - Ok(()) - } - .boxed() - }) - .map_err(io::Error::other)? - .at("sethotshotup", move |req, state| { - async move { - let chain_id = req - .body_auto::, ApiVer>(ApiVer::instance()) - .map_err(ServerError::from_request_error)? - .map(|b| b.chain_id); - let contract = state.light_client_instance(chain_id)?; - - contract - .setHotShotUp() - .send() - .await - .map_err(|err| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - })? - .watch() - .await - .map_err(|err| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - })?; - Ok(()) +/// Error envelope matching tide-disco's `ServerError`: `{"status": , "message": }`, +/// with the HTTP status set to the same code. +#[derive(Debug, Serialize)] +pub(crate) struct DevNodeError { + status: u16, + message: String, +} + +impl DevNodeError { + fn catch_all(status: StatusCode, message: String) -> Self { + Self { + status: status.as_u16(), + message, } - .boxed() - }) - .map_err(io::Error::other)?; + } +} + +impl IntoResponse for DevNodeError { + fn into_response(self) -> Response { + let status = StatusCode::from_u16(self.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + (status, Json(self)).into_response() + } +} + +/// Decode a JSON request body, matching tide-disco's `body_auto` behavior for the +/// `application/json` content type (the only one this server needs to support). +fn json_body( + headers: &HeaderMap, + body: &[u8], +) -> Result { + let content_type = headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()); + match content_type { + Some("application/json") => serde_json::from_slice(body).map_err(|_| { + DevNodeError::catch_all( + StatusCode::BAD_REQUEST, + "Unable to deserialize from JSON".to_string(), + ) + }), + _ => Err(DevNodeError::catch_all( + StatusCode::BAD_REQUEST, + "Content type not specified or type not supported".to_string(), + )), + } +} - app.register_module("api", api).map_err(io::Error::other)?; +async fn get_dev_info(State(state): State) -> Json { + Json((*state.dev_info).clone()) +} - tracing::info!("Starting dev-node API on http://0.0.0.0:{port}"); +async fn set_hotshot_down( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result, DevNodeError> { + let body: SetHotshotDownReqBody = json_body(&headers, &body)?; + let contract = state.api.light_client_instance(body.chain_id)?; + + contract + .setHotShotDownSince(U256::from(body.height)) + .send() + .await + .map_err(|err| DevNodeError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))? + .watch() + .await + .map_err(|err| { + DevNodeError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + })?; + Ok(Json(())) +} - app.serve(format!("0.0.0.0:{port}"), bind_version) +async fn set_hotshot_up( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result, DevNodeError> { + let body: Option = json_body(&headers, &body)?; + let contract = state.api.light_client_instance(body.map(|b| b.chain_id))?; + + contract + .setHotShotUp() + .send() + .await + .map_err(|err| DevNodeError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))? + .watch() .await .map_err(|err| { - // If we get an "Address in use" during startup, make it a bit easier to find the cause. - tracing::error!("Failed to start dev-node API on http://0.0.0.0:{port} : {err}"); - err + DevNodeError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) })?; + Ok(Json(())) +} + +async fn healthcheck(headers: HeaderMap) -> Response { + espresso_api::healthcheck_response(&headers) +} + +/// Serves the dev-info/set-hotshot-down/set-hotshot-up routes at both the `/v0/api/...` forms +/// tide-disco served directly and the unversioned `/api/...` forms it served via a redirect +/// (used by the Go SDK and surf-disco clients, respectively). +fn dev_node_router(state: DevNodeState) -> Router { + let api = Router::new() + .route("/dev-info", get(get_dev_info)) + .route("/set-hotshot-down", post(set_hotshot_down)) + .route("/set-hotshot-up", post(set_hotshot_up)) + .with_state(state); + + Router::new() + .nest("/api", api.clone()) + .nest("/v0/api", api) + .route("/healthcheck", get(healthcheck)) +} + +async fn run_dev_node_server( + port: u16, + client_states: ApiState, + dev_info: DevInfo, + _bind_version: ApiVer, +) -> anyhow::Result<()> { + let state = DevNodeState { + api: client_states, + dev_info: Arc::new(dev_info), + }; + let router = dev_node_router(state); + + let addr = format!("0.0.0.0:{port}"); + tracing::info!("Starting dev-node API on http://{addr}"); + let listener = tokio::net::TcpListener::bind(&addr).await.map_err(|err| { + // If we get an "Address in use" during startup, make it a bit easier to find the cause. + tracing::error!("Failed to start dev-node API on http://{addr} : {err}"); + err + })?; + axum::serve(listener, router).await?; Ok(()) } @@ -925,7 +979,7 @@ async fn run_dev_node_server( #[derive(Default, Debug, Serialize, Deserialize, PartialEq, Eq)] struct AnvilAccount { nonce: u64, - code: Bytes, + code: AlloyBytes, storage: HashMap, balance: U256, /// Name of the contract at this account, filled diff --git a/crates/espresso/node/Cargo.toml b/crates/espresso/node/Cargo.toml index 66a9658270d..55b4340c61d 100644 --- a/crates/espresso/node/Cargo.toml +++ b/crates/espresso/node/Cargo.toml @@ -35,6 +35,7 @@ async-channel = { workspace = true } async-lock = { workspace = true } async-once-cell = { workspace = true } async-trait = { workspace = true } +axum = { workspace = true } base64 = { workspace = true } base64-bytes = { workspace = true } bincode = { workspace = true } diff --git a/crates/espresso/node/src/bin/nasty-client.rs b/crates/espresso/node/src/bin/nasty-client.rs index c8b5267255f..b16d1bfcb19 100644 --- a/crates/espresso/node/src/bin/nasty-client.rs +++ b/crates/espresso/node/src/bin/nasty-client.rs @@ -13,7 +13,6 @@ //! count of various types of actions performed and the number of open streams. use std::{ - borrow::Cow, cmp::max, collections::{BTreeMap, HashMap}, fmt::Debug, @@ -23,7 +22,12 @@ use std::{ }; use anyhow::{Context, bail, ensure}; -use async_lock::RwLock; +use axum::{ + extract::State, + http::{HeaderMap, StatusCode as AxumStatusCode, header}, + response::{IntoResponse, Response}, + routing::get, +}; use clap::Parser; use committable::Committable; use derivative::Derivative; @@ -50,16 +54,14 @@ use hotshot_types::traits::{ use jf_merkle_tree_compat::{ ForgetableMerkleTreeScheme, MerkleTreeScheme, UniversalMerkleTreeScheme, }; +use prometheus::{Encoder, TextEncoder}; use rand::{RngCore, seq::SliceRandom}; use serde::de::DeserializeOwned; use strum::{EnumDiscriminants, VariantArray}; use surf_disco::{Error, StatusCode, Url, error::ClientError, socket}; -use tide_disco::{App, error::ServerError}; use time::OffsetDateTime; -use tokio::{task::spawn, time::sleep}; -use toml::toml; +use tokio::{net::TcpListener, task::spawn, time::sleep}; use tracing::info_span; -use vbs::version::StaticVersionType; /// An adversarial stress test for sequencer APIs. #[derive(Clone, Debug, Parser)] @@ -1378,26 +1380,47 @@ impl Client { } async fn serve(port: u16, metrics: PrometheusMetrics) { - let api = toml! { - [route.metrics] - PATH = ["/metrics"] - METHOD = "METRICS" + let metrics = Arc::new(metrics); + let app = axum::Router::new() + .route("/healthcheck", get(healthcheck)) + .route("/status/metrics", get(status_metrics)) + .route("/v0/status/metrics", get(status_metrics)) + .with_state(metrics); + + let addr = format!("0.0.0.0:{port}"); + let listener = match TcpListener::bind(&addr).await { + Ok(listener) => listener, + Err(err) => { + tracing::error!("failed to bind web server to {addr}: {err:#}"); + return; + }, }; - let mut app = App::<_, ServerError>::with_state(RwLock::new(metrics)); - app.module::("status", api) - .unwrap() - .metrics("metrics", |_req, state| { - async move { Ok(Cow::Borrowed(state)) }.boxed() - }) - .unwrap(); - if let Err(err) = app - .serve(format!("0.0.0.0:{port}"), SequencerApiVersion::instance()) - .await - { + if let Err(err) = axum::serve(listener, app).await { tracing::error!("web server exited unexpectedly: {err:#}"); } } +async fn healthcheck(headers: HeaderMap) -> Response { + espresso_api::healthcheck_response(&headers) +} + +/// Prometheus text exposition of `metrics`, matching the `text/plain; charset=utf-8` content type +/// tide-disco's `METHOD = "METRICS"` route used. +async fn status_metrics(State(metrics): State>) -> impl IntoResponse { + let encoder = TextEncoder::new(); + let families = metrics.registry().gather(); + let mut buffer = Vec::new(); + match encoder.encode(&families, &mut buffer).and_then(|()| { + String::from_utf8(buffer).map_err(|err| prometheus::Error::Msg(err.to_string())) + }) { + Ok(text) => ([(header::CONTENT_TYPE, "text/plain; charset=utf-8")], text).into_response(), + Err(err) => { + tracing::error!("failed to export metrics: {err:#}"); + AxumStatusCode::INTERNAL_SERVER_ERROR.into_response() + }, + } +} + fn main() { let migrated_envs = espresso_utils::env_compat::migrate_legacy_env_vars(); tokio::runtime::Runtime::new().unwrap().block_on(async { diff --git a/crates/espresso/node/src/bin/submit-transactions.rs b/crates/espresso/node/src/bin/submit-transactions.rs index 38df82ab04e..33482b220c7 100644 --- a/crates/espresso/node/src/bin/submit-transactions.rs +++ b/crates/espresso/node/src/bin/submit-transactions.rs @@ -9,6 +9,7 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; +use axum::{http::HeaderMap, response::Response, routing::get}; use clap::Parser; use committable::{Commitment, Committable}; #[cfg(feature = "benchmarking")] @@ -26,8 +27,7 @@ use rand::{Rng, RngCore, SeedableRng}; use rand_chacha::ChaChaRng; use rand_distr::Distribution; use surf_disco::{Client, Url, reexports::WebSocketConfig}; -use tide_disco::{App, error::ServerError}; -use tokio::{task::spawn, time::sleep}; +use tokio::{net::TcpListener, task::spawn, time::sleep}; use vbs::version::StaticVersionType; /// Submit random transactions to an Espresso Sequencer. @@ -201,7 +201,11 @@ async fn async_main(migrated_envs: Vec<(&str, &str)>) { // Subscribe to block stream so we can check that our transactions are getting sequenced. let client = Client::::new(opt.urls[0].clone()); - let block_height: usize = client.get("status/block-height").send().await.unwrap(); + let block_height: usize = client + .get(&espresso_api::routes::v1::status_block_height()) + .send() + .await + .unwrap(); // Create a new [`WebSocketConfig`]. We trust the events service on our nodes to not // send us malicious messages. @@ -213,10 +217,7 @@ async fn async_main(migrated_envs: Vec<(&str, &str)>) { let mut blocks = client .socket_with_config( - &format!( - "availability/stream/blocks/{}", - block_height.saturating_sub(1) - ), + &espresso_api::routes::v1::stream_blocks(block_height.saturating_sub(1)), websocket_config, ) .subscribe() @@ -236,7 +237,7 @@ async fn async_main(migrated_envs: Vec<(&str, &str)>) { // Start healthcheck endpoint once tasks are running. if let Some(port) = opt.port { - spawn(server(port, SequencerApiVersion::instance())); + spawn(server(port)); } // Keep track of the results. @@ -508,15 +509,25 @@ async fn submit_transactions( } } -async fn server(port: u16, bind_version: ApiVer) { - if let Err(err) = App::<(), ServerError>::with_state(()) - .serve(format!("0.0.0.0:{port}"), bind_version) - .await - { +async fn server(port: u16) { + let app = axum::Router::new().route("/healthcheck", get(healthcheck)); + let addr = format!("0.0.0.0:{port}"); + let listener = match TcpListener::bind(&addr).await { + Ok(listener) => listener, + Err(err) => { + tracing::error!("failed to bind web server to {addr}: {err}"); + return; + }, + }; + if let Err(err) = axum::serve(listener, app).await { tracing::error!("web server exited: {err}"); } } +async fn healthcheck(headers: HeaderMap) -> Response { + espresso_api::healthcheck_response(&headers) +} + fn random_transaction(opt: &Options, rng: &mut ChaChaRng) -> Transaction { // TODO instead use NamespaceId::random, but that does not allow us to // enforce `gen_range(opt.min_namespace..=opt.max_namespace)` diff --git a/crates/espresso/node/src/state_signature/relay_server.rs b/crates/espresso/node/src/state_signature/relay_server.rs index 1a5c53b5b43..ca8a30fe6db 100644 --- a/crates/espresso/node/src/state_signature/relay_server.rs +++ b/crates/espresso/node/src/state_signature/relay_server.rs @@ -1,35 +1,134 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{fmt, sync::Arc}; use async_lock::RwLock; -use clap::Args; -use futures::FutureExt; +use axum::{ + Json, Router, + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; use hotshot_types::{ light_client::{ - LCV1StateSignatureRequestBody, LCV1StateSignaturesBundle, LCV2StateSignaturesBundle, - LCV3StateSignatureRequestBody, LCV3StateSignaturesBundle, + LCV1StateSignatureRequestBody, LCV1StateSignaturesBundle, LCV2StateSignatureRequestBody, + LCV2StateSignaturesBundle, LCV3StateSignatureRequestBody, LCV3StateSignaturesBundle, }, traits::signature_key::LCV1StateSignatureKey, }; use lcv1_relay::{LCV1StateRelayServerDataSource, LCV1StateRelayServerState}; use lcv2_relay::{LCV2StateRelayServerDataSource, LCV2StateRelayServerState}; use lcv3_relay::{LCV3StateRelayServerDataSource, LCV3StateRelayServerState}; -use tide_disco::{ - Api, App, Error as _, StatusCode, - api::ApiError, - error::ServerError, - method::{ReadState, WriteState}, -}; -use tokio::sync::oneshot; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use tide_disco::healthcheck::HealthStatus; +use tokio::{net::TcpListener, sync::oneshot}; use url::Url; -use vbs::version::StaticVersionType; - -use super::LCV2StateSignatureRequestBody; +use vbs::{ + BinarySerializer, Serializer, + version::{StaticVersion, StaticVersionType}, +}; pub mod lcv1_relay; pub mod lcv2_relay; pub mod lcv3_relay; pub mod stake_table_tracker; +/// Binary framing version used by `state_signature.rs` and `hotshot-state-prover`, whose +/// surf-disco clients default to `Accept`/`Content-Type: application/octet-stream`. +type WireVersion = StaticVersion<0, 1>; + +/// Wire-compatible error envelope: mirrors `tide_disco::error::ServerError`'s `{status, message}` +/// JSON/VBS shape, since production clients (`state_signature.rs`, `hotshot-state-prover`) +/// deserialize error responses into that type. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RelayError { + pub status: u16, + pub message: String, +} + +impl RelayError { + pub fn catch_all(status: StatusCode, message: impl Into) -> Self { + Self { + status: status.as_u16(), + message: message.into(), + } + } +} + +impl fmt::Display for RelayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Error {}: {}", self.status, self.message) + } +} + +impl std::error::Error for RelayError {} + +fn wants_binary(headers: &HeaderMap) -> bool { + headers + .get(header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("application/octet-stream")) +} + +/// Encode a successful response body, negotiating VBS binary vs JSON from the `Accept` header, +/// matching tide-disco's content negotiation for the real (default-binary) surf-disco clients. +fn encode_ok(headers: &HeaderMap, value: T) -> Response { + if wants_binary(headers) { + match Serializer::::serialize(&value) { + Ok(bytes) => { + ([(header::CONTENT_TYPE, "application/octet-stream")], bytes).into_response() + }, + Err(err) => encode_err( + headers, + RelayError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + ), + } + } else { + Json(value).into_response() + } +} + +/// Encode an error response using the same content negotiation as [`encode_ok`]. +fn encode_err(headers: &HeaderMap, err: RelayError) -> Response { + let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + if wants_binary(headers) { + match Serializer::::serialize(&err) { + Ok(bytes) => ( + status, + [(header::CONTENT_TYPE, "application/octet-stream")], + bytes, + ) + .into_response(), + Err(_) => (status, Json(err)).into_response(), + } + } else { + (status, Json(err)).into_response() + } +} + +/// Decode a request body, matching tide-disco's `body_auto` (VBS for +/// `application/octet-stream`, JSON for `application/json`). +fn decode_body(headers: &HeaderMap, body: &[u8]) -> Result { + let content_type = headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + if content_type.starts_with("application/octet-stream") { + Serializer::::deserialize(body).map_err(|e| { + RelayError::catch_all(StatusCode::BAD_REQUEST, format!("invalid binary body: {e}")) + }) + } else if content_type.starts_with("application/json") { + serde_json::from_slice(body).map_err(|e| { + RelayError::catch_all(StatusCode::BAD_REQUEST, format!("invalid json body: {e}")) + }) + } else { + Err(RelayError::catch_all( + StatusCode::BAD_REQUEST, + "missing or unsupported Content-Type", + )) + } +} + /// State that checks the light client state update and the signature collection pub struct StateRelayServerState { /// Handling LCV1 state signatures @@ -69,227 +168,380 @@ impl StateRelayServerState { #[async_trait::async_trait] impl LCV1StateRelayServerDataSource for StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.lcv1_state.get_latest_signature_bundle() } async fn post_signature( &mut self, req: LCV1StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { self.lcv1_state.post_signature(req).await } } #[async_trait::async_trait] impl LCV2StateRelayServerDataSource for StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.lcv2_state.get_latest_signature_bundle() } async fn post_signature( &mut self, req: LCV2StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { self.lcv2_state.post_signature(req).await } } #[async_trait::async_trait] impl LCV3StateRelayServerDataSource for StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.lcv3_state.get_latest_signature_bundle() } async fn post_signature( &mut self, req: LCV3StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { self.lcv3_state.post_signature(req).await } } -/// configurability options for the web server -#[derive(Args, Default)] -pub struct Options { - #[arg( - long = "state-relay-server-api-path", - env = "STATE_RELAY_SERVER_API_PATH" - )] - /// path to API - pub api_path: Option, -} - -/// Set up APIs for relay server -fn define_api( - options: &Options, - bind_version: BindVer, - api_ver: semver::Version, -) -> Result, ApiError> -where - State: 'static + Send + Sync + ReadState + WriteState, - ::State: Send - + Sync - + LCV1StateRelayServerDataSource - + LCV2StateRelayServerDataSource - + LCV3StateRelayServerDataSource, -{ - let mut api = match &options.api_path { - Some(path) => Api::::from_file(path)?, - None => { - let toml: toml::Value = toml::from_str(include_str!( - "../../api/state_relay_server.toml" - )) - .map_err(|err| ApiError::CannotReadToml { - reason: err.to_string(), - })?; - Api::::new(toml)? - }, - }; +/// Shared, lock-guarded server state, cloned into every axum handler via `State`. +type SharedState = Arc>; - api.with_version(api_ver.clone()); - - api.post("postlegacystatesignature", move |req, state| { - async move { - let req = match req.body_auto::(bind_version) { - Ok(req) => req, - Err(_) => { - match req.body_auto::(bind_version) { - Ok(req) => req.into(), - Err(_) => { - return Err(ServerError::catch_all( - StatusCode::BAD_REQUEST, - "Invalid request body".to_string(), - )); - }, - } - }, - }; - LCV1StateRelayServerDataSource::post_signature(state, req).await?; - Ok(()) - } - .boxed() - })? - .post("poststatesignature", move |req, state| { - async move { - if let Ok(req) = req.body_auto::(bind_version) { - tracing::debug!("Received LCV3 state signature: {req}"); - if let Err(e) = - LCV2StateRelayServerDataSource::post_signature(state, req.clone().into()).await - { - tracing::error!("Failed to post downgraded LCV2 state signature: {}", e); - } - LCV3StateRelayServerDataSource::post_signature(state, req).await - } else if let Ok(req) = - req.body_auto::(bind_version) - { - tracing::debug!("Received LCV2 state signature: {req}"); - if LCV1StateSignatureKey::verify_state_sig(&req.key, &req.signature, &req.state) { - LCV1StateRelayServerDataSource::post_signature(state, req.into()).await - } else { - LCV2StateRelayServerDataSource::post_signature(state, req).await - } - } else if let Ok(req) = - req.body_auto::(bind_version) - { - tracing::debug!("Received LCV1 state signature: {req}"); - LCV1StateRelayServerDataSource::post_signature(state, req).await - } else { - Err(ServerError::catch_all( - StatusCode::BAD_REQUEST, - "Invalid request body".to_string(), - )) - } +/// Handle a `POST` to `state`/`api/state`: tries LCV3, then LCV2 (auto-downgrading to LCV1 if the +/// signature verifies against the legacy scheme), then LCV1. Mirrors tide's `poststatesignature`. +async fn post_state_signature( + state: &SharedState, + headers: &HeaderMap, + body: &[u8], +) -> Result<(), RelayError> { + if let Ok(req) = decode_body::(headers, body) { + tracing::debug!("Received LCV3 state signature: {req}"); + let mut state = state.write().await; + if let Err(e) = + LCV2StateRelayServerDataSource::post_signature(&mut *state, req.clone().into()).await + { + tracing::error!("Failed to post downgraded LCV2 state signature: {}", e); } - .boxed() - })? - .get("getlatestlegacystate", |_req, state| { - async move { - LCV1StateRelayServerDataSource::get_latest_signature_bundle(state) - .map(LCV2StateSignaturesBundle::from_v1) + LCV3StateRelayServerDataSource::post_signature(&mut *state, req).await + } else if let Ok(req) = decode_body::(headers, body) { + tracing::debug!("Received LCV2 state signature: {req}"); + let mut state = state.write().await; + if LCV1StateSignatureKey::verify_state_sig(&req.key, &req.signature, &req.state) { + LCV1StateRelayServerDataSource::post_signature(&mut *state, req.into()).await + } else { + LCV2StateRelayServerDataSource::post_signature(&mut *state, req).await } - .boxed() - })? - .get("getlateststate", |_req, state| { - async move { LCV2StateRelayServerDataSource::get_latest_signature_bundle(state) }.boxed() - })?; - - if api_ver.major == 1 { - api.get("lateststate", |_req, state| { - async move { LCV1StateRelayServerDataSource::get_latest_signature_bundle(state) } - .boxed() - })?; - } else if api_ver.major == 2 { - api.get("lateststate", |_req, state| { - async move { LCV2StateRelayServerDataSource::get_latest_signature_bundle(state) } - .boxed() - })?; + } else if let Ok(req) = decode_body::(headers, body) { + tracing::debug!("Received LCV1 state signature: {req}"); + let mut state = state.write().await; + LCV1StateRelayServerDataSource::post_signature(&mut *state, req).await } else { - api.get("lateststate", |_req, state| { - async move { LCV3StateRelayServerDataSource::get_latest_signature_bundle(state) } - .boxed() - })?; + Err(RelayError::catch_all( + StatusCode::BAD_REQUEST, + "Invalid request body", + )) } - Ok(api) } -pub async fn run_relay_server( - shutdown_listener: Option>, - sequencer_url: Url, - url: Url, - bind_version: BindVer, -) -> anyhow::Result<()> { - let options = Options::default(); +/// Handle a `POST` to `legacy-state`: tries LCV1, then LCV2 (downgraded to LCV1). Mirrors tide's +/// `postlegacystatesignature`. +async fn post_legacy_state_signature( + state: &SharedState, + headers: &HeaderMap, + body: &[u8], +) -> Result<(), RelayError> { + let req = if let Ok(req) = decode_body::(headers, body) { + req + } else if let Ok(req) = decode_body::(headers, body) { + req.into() + } else { + return Err(RelayError::catch_all( + StatusCode::BAD_REQUEST, + "Invalid request body", + )); + }; + let mut state = state.write().await; + LCV1StateRelayServerDataSource::post_signature(&mut *state, req).await +} + +/// `GET state` (deprecated): mirrors tide's `getlateststate`, always the LCV2 bundle regardless +/// of version, since all three registered API versions shared this handler. +async fn get_latest_state(state: &SharedState) -> Result { + let state = state.read().await; + LCV2StateRelayServerDataSource::get_latest_signature_bundle(&*state) +} + +/// `GET legacy-state`: mirrors tide's `getlatestlegacystate`. +async fn get_latest_legacy_state( + state: &SharedState, +) -> Result { + let state = state.read().await; + LCV1StateRelayServerDataSource::get_latest_signature_bundle(&*state) + .map(LCV2StateSignaturesBundle::from_v1) +} + +async fn get_latest_state_v1(state: &SharedState) -> Result { + let state = state.read().await; + LCV1StateRelayServerDataSource::get_latest_signature_bundle(&*state) +} + +async fn get_latest_state_v2(state: &SharedState) -> Result { + let state = state.read().await; + LCV2StateRelayServerDataSource::get_latest_signature_bundle(&*state) +} + +async fn get_latest_state_v3(state: &SharedState) -> Result { + let state = state.read().await; + LCV3StateRelayServerDataSource::get_latest_signature_bundle(&*state) +} - let state = RwLock::new( - StateRelayServerState::new(sequencer_url).with_shutdown_signal(shutdown_listener), - ); - let mut app = App::, ServerError>::with_state(state); +/// Tide-disco-compatible singleton-app healthcheck: a bare [`HealthStatus`], negotiated JSON or +/// vbs binary from `Accept`. +async fn healthcheck(headers: HeaderMap) -> Response { + encode_ok(&headers, HealthStatus::Available) +} + +async fn post_state(State(state): State, headers: HeaderMap, body: Bytes) -> Response { + match post_state_signature(&state, &headers, &body).await { + Ok(()) => encode_ok(&headers, ()), + Err(e) => encode_err(&headers, e), + } +} + +async fn get_state(State(state): State, headers: HeaderMap) -> Response { + match get_latest_state(&state).await { + Ok(bundle) => encode_ok(&headers, bundle), + Err(e) => encode_err(&headers, e), + } +} + +async fn post_legacy_state( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Response { + match post_legacy_state_signature(&state, &headers, &body).await { + Ok(()) => encode_ok(&headers, ()), + Err(e) => encode_err(&headers, e), + } +} + +async fn get_legacy_state(State(state): State, headers: HeaderMap) -> Response { + match get_latest_legacy_state(&state).await { + Ok(bundle) => encode_ok(&headers, bundle), + Err(e) => encode_err(&headers, e), + } +} + +async fn get_lateststate_v1(State(state): State, headers: HeaderMap) -> Response { + match get_latest_state_v1(&state).await { + Ok(bundle) => encode_ok(&headers, bundle), + Err(e) => encode_err(&headers, e), + } +} + +async fn get_lateststate_v2(State(state): State, headers: HeaderMap) -> Response { + match get_latest_state_v2(&state).await { + Ok(bundle) => encode_ok(&headers, bundle), + Err(e) => encode_err(&headers, e), + } +} + +async fn get_lateststate_v3(State(state): State, headers: HeaderMap) -> Response { + match get_latest_state_v3(&state).await { + Ok(bundle) => encode_ok(&headers, bundle), + Err(e) => encode_err(&headers, e), + } +} - let v1_api = define_api(&options, bind_version, "1.0.0".parse().unwrap()).unwrap(); - let v2_api = define_api(&options, bind_version, "2.0.0".parse().unwrap()).unwrap(); - let v3_api = define_api(&options, bind_version, "3.0.0".parse().unwrap()).unwrap(); - app.register_module("api", v1_api)? - .register_module("api", v2_api)? - .register_module("api", v3_api)?; +const STATE_PATH: &str = "/api/state"; +const LEGACY_STATE_PATH: &str = "/api/legacy-state"; +const LATEST_STATE_PATH: &str = "/api/lateststate"; - let app_future = app.serve(url.clone(), bind_version); - app_future.await?; +/// Build the relay server router. Tide-disco registered `api` under three stacked major API +/// versions (v1, v2, v3) with identical handlers for every route except `lateststate`; requests +/// with no version prefix were redirected to the latest (v3). We reproduce that by serving the +/// same handlers at the unversioned and all three `/v{1,2,3}` paths, and only special-casing +/// `lateststate` per version. +fn router(state: SharedState) -> Router { + let mut router = Router::::new() + .route("/healthcheck", get(healthcheck)) + .route(STATE_PATH, post(post_state).get(get_state)) + .route( + LEGACY_STATE_PATH, + post(post_legacy_state).get(get_legacy_state), + ) + .route(LATEST_STATE_PATH, get(get_lateststate_v3)) + .route(&format!("/v1{LATEST_STATE_PATH}"), get(get_lateststate_v1)) + .route(&format!("/v2{LATEST_STATE_PATH}"), get(get_lateststate_v2)) + .route(&format!("/v3{LATEST_STATE_PATH}"), get(get_lateststate_v3)); + for v in 1..=3 { + router = router + .route( + &format!("/v{v}{STATE_PATH}"), + post(post_state).get(get_state), + ) + .route( + &format!("/v{v}{LEGACY_STATE_PATH}"), + post(post_legacy_state).get(get_legacy_state), + ); + } + router.with_state(state) +} - tracing::info!(%url, "Relay server starts serving at "); +async fn serve(server_url: Url, state: StateRelayServerState) -> anyhow::Result<()> { + let host = server_url + .host_str() + .ok_or_else(|| anyhow::anyhow!("relay server url missing host: {server_url}"))?; + let port = server_url + .port_or_known_default() + .ok_or_else(|| anyhow::anyhow!("relay server url missing port: {server_url}"))?; + let listener = TcpListener::bind((host, port)).await?; + tracing::info!(%server_url, "Relay server starts serving at "); + axum::serve(listener, router(Arc::new(RwLock::new(state)))).await?; Ok(()) } +pub async fn run_relay_server( + shutdown_listener: Option>, + sequencer_url: Url, + url: Url, + // Kept only for compatibility with the binary's call site; the axum server no longer needs a + // binary framing version for its own top-level endpoints. + _bind_version: BindVer, +) -> anyhow::Result<()> { + let state = StateRelayServerState::new(sequencer_url).with_shutdown_signal(shutdown_listener); + serve(url, state).await +} + pub async fn run_relay_server_with_state( server_url: Url, - bind_version: BindVer, + _bind_version: BindVer, state: StateRelayServerState, ) -> anyhow::Result<()> { - let options = Options::default(); - - let mut app = App::, ServerError>::with_state(RwLock::new(state)); - - app.register_module( - "api", - define_api(&options, bind_version, "1.0.0".parse().unwrap()).unwrap(), - ) - .unwrap(); - app.register_module( - "api", - define_api(&options, bind_version, "2.0.0".parse().unwrap()).unwrap(), - ) - .unwrap(); - app.register_module( - "api", - define_api(&options, bind_version, "3.0.0".parse().unwrap()).unwrap(), - ) - .unwrap(); - - let app_future = app.serve(server_url.clone(), bind_version); - app_future.await?; + serve(server_url, state).await +} - tracing::info!(%server_url, "Relay server starts serving at "); +#[cfg(test)] +mod test { + use alloy::primitives::{FixedBytes, U256}; + use espresso_types::SeqTypes; + use hotshot::types::SchnorrPubKey; + use hotshot_contract_adapter::light_client::derive_signed_state_digest; + use hotshot_types::{ + PeerConfig, ValidatorConfig, + light_client::{LightClientState, StakeTableState}, + traits::signature_key::{LCV2StateSignatureKey, LCV3StateSignatureKey}, + }; + use surf_disco::Client; + use tide_disco::error::ServerError; + use vbs::version::StaticVersion; - Ok(()) + use super::*; + + type TestApiVer = StaticVersion<0, 1>; + + /// Fake sequencer serving just enough of `config/hotshot` for the relay's + /// [`stake_table_tracker::StakeTableTracker`] to bootstrap a genesis stake table with a + /// single validator, with `epoch_height: 0` so every lookup takes the genesis path. + async fn spawn_fake_sequencer(peer: PeerConfig) -> Url { + let config = serde_json::json!({ + "config": { + "known_nodes_with_stake": [peer], + "epoch_height": 0, + "epoch_start_block": 0, + } + }); + let app = Router::new().route( + "/config/hotshot", + get(move || { + let config = config.clone(); + async move { Json(config) } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}").parse().unwrap() + } + + /// Spins the axum relay on an ephemeral port and returns its URL. + async fn spawn_relay(state: StateRelayServerState) -> Url { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router(Arc::new(RwLock::new(state)))) + .await + .unwrap(); + }); + format!("http://{addr}").parse().unwrap() + } + + /// Posts a signature the way `state_signature.rs` does (unversioned path, VBS-binary body), + /// then fetches it back the way `hotshot-state-prover`'s v3 service does (unversioned path, + /// default VBS-binary `Accept`). + #[tokio::test] + async fn post_and_fetch_lcv3_state_signature() { + let validator = ValidatorConfig::::generated_from_seed_indexed( + [7u8; 32], + 0, + U256::from(1), + true, + ); + let sequencer_url = spawn_fake_sequencer(validator.public_config()).await; + let relay_url = spawn_relay(StateRelayServerState::new(sequencer_url)).await; + + let light_client_state = LightClientState { + view_number: 1, + block_height: 1, + block_comm_root: Default::default(), + }; + let next_stake = StakeTableState::default(); + let auth_root = FixedBytes::<32>::default(); + let digest = derive_signed_state_digest(&light_client_state, &next_stake, &auth_root); + let signature = ::sign_state( + &validator.state_private_key, + digest, + ) + .unwrap(); + let v2_signature = ::sign_state( + &validator.state_private_key, + &light_client_state, + &next_stake, + ) + .unwrap(); + let request_body = LCV3StateSignatureRequestBody { + key: validator.state_public_key.clone(), + state: light_client_state, + next_stake, + auth_root, + signature, + v2_signature, + }; + + let client = Client::::new(relay_url); + client + .post::<()>("api/state") + .body_binary(&request_body) + .unwrap() + .send() + .await + .unwrap(); + + let bundle = client + .get::("api/lateststate") + .send() + .await + .unwrap(); + assert_eq!(bundle.state, light_client_state); + assert_eq!(bundle.signatures.len(), 1); + assert!(bundle.signatures.contains_key(&validator.state_public_key)); + } } diff --git a/crates/espresso/node/src/state_signature/relay_server/lcv1_relay.rs b/crates/espresso/node/src/state_signature/relay_server/lcv1_relay.rs index 70f15918faf..902dd55ccc8 100644 --- a/crates/espresso/node/src/state_signature/relay_server/lcv1_relay.rs +++ b/crates/espresso/node/src/state_signature/relay_server/lcv1_relay.rs @@ -4,22 +4,22 @@ use std::{ }; use alloy::primitives::{U256, map::HashMap}; +use axum::http::StatusCode; use hotshot_types::{ light_client::{ LCV1StateSignatureRequestBody, LCV1StateSignaturesBundle, LightClientState, StateVerKey, }, traits::signature_key::LCV1StateSignatureKey, }; -use tide_disco::{Error, StatusCode, error::ServerError}; -use super::stake_table_tracker::StakeTableTracker; +use super::{RelayError, stake_table_tracker::StakeTableTracker}; #[async_trait::async_trait] pub trait LCV1StateRelayServerDataSource { /// Get the latest available signatures bundle. /// # Errors /// Errors if there's no available signatures bundle. - fn get_latest_signature_bundle(&self) -> Result; + fn get_latest_signature_bundle(&self) -> Result; /// Post a signature to the relay server /// # Errors @@ -27,7 +27,7 @@ pub trait LCV1StateRelayServerDataSource { async fn post_signature( &mut self, req: LCV1StateSignatureRequestBody, - ) -> Result<(), ServerError>; + ) -> Result<(), RelayError>; } /// Server state that tracks the light client V1 state and signatures @@ -49,10 +49,10 @@ pub struct LCV1StateRelayServerState { #[async_trait::async_trait] impl LCV1StateRelayServerDataSource for LCV1StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.latest_available_bundle .clone() - .ok_or(ServerError::catch_all( + .ok_or(RelayError::catch_all( StatusCode::NOT_FOUND, "The light client V1 state signatures are not ready.".to_owned(), )) @@ -61,7 +61,7 @@ impl LCV1StateRelayServerDataSource for LCV1StateRelayServerState { async fn post_signature( &mut self, req: LCV1StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { let block_height = req.state.block_height; if block_height <= self.latest_block_height.unwrap_or(0) { // This signature is no longer needed @@ -71,12 +71,10 @@ impl LCV1StateRelayServerDataSource for LCV1StateRelayServerState { .stake_table_tracker .genesis_stake_table_info() .await - .map_err(|e| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) - })?; + .map_err(|e| RelayError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; let Some(weight) = stake_table.known_nodes.get(&req.key) else { tracing::warn!("Received LCV1 signature from unknown node: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::UNAUTHORIZED, "LCV1 signature posted by nodes not on the stake table".to_owned(), )); @@ -89,7 +87,7 @@ impl LCV1StateRelayServerDataSource for LCV1StateRelayServerState { &req.state, ) { tracing::warn!("Couldn't verify the received LCV1 signature: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "The posted LCV1 signature is not valid.".to_owned(), )); @@ -113,7 +111,7 @@ impl LCV1StateRelayServerDataSource for LCV1StateRelayServerState { match bundle.signatures.entry(req.key) { Entry::Occupied(_) => { // A signature is already posted for this key with this state - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "A LCV1 signature of this light client state is already posted at this block \ height for this key." diff --git a/crates/espresso/node/src/state_signature/relay_server/lcv2_relay.rs b/crates/espresso/node/src/state_signature/relay_server/lcv2_relay.rs index fbf0ef4944b..bbfae1ac848 100644 --- a/crates/espresso/node/src/state_signature/relay_server/lcv2_relay.rs +++ b/crates/espresso/node/src/state_signature/relay_server/lcv2_relay.rs @@ -4,22 +4,22 @@ use std::{ }; use alloy::primitives::U256; +use axum::http::StatusCode; use hotshot_types::{ light_client::{ LCV2StateSignatureRequestBody, LCV2StateSignaturesBundle, LightClientState, StateVerKey, }, traits::signature_key::LCV2StateSignatureKey, }; -use tide_disco::{Error, StatusCode, error::ServerError}; -use super::stake_table_tracker::StakeTableTracker; +use super::{RelayError, stake_table_tracker::StakeTableTracker}; #[async_trait::async_trait] pub trait LCV2StateRelayServerDataSource { /// Get the latest available signatures bundle. /// # Errors /// Errors if there's no available signatures bundle. - fn get_latest_signature_bundle(&self) -> Result; + fn get_latest_signature_bundle(&self) -> Result; /// Post a signature to the relay server /// # Errors @@ -27,7 +27,7 @@ pub trait LCV2StateRelayServerDataSource { async fn post_signature( &mut self, req: LCV2StateSignatureRequestBody, - ) -> Result<(), ServerError>; + ) -> Result<(), RelayError>; } /// Server state that tracks the light client V2 state and signatures @@ -49,10 +49,10 @@ pub struct LCV2StateRelayServerState { #[async_trait::async_trait] impl LCV2StateRelayServerDataSource for LCV2StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.latest_available_bundle .clone() - .ok_or(ServerError::catch_all( + .ok_or(RelayError::catch_all( StatusCode::NOT_FOUND, "The light client V2 state signatures are not ready.".to_owned(), )) @@ -61,7 +61,7 @@ impl LCV2StateRelayServerDataSource for LCV2StateRelayServerState { async fn post_signature( &mut self, req: LCV2StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { let block_height = req.state.block_height; if block_height <= self.latest_block_height.unwrap_or(0) { // This signature is no longer needed @@ -71,12 +71,10 @@ impl LCV2StateRelayServerDataSource for LCV2StateRelayServerState { .stake_table_tracker .stake_table_info_for_block(block_height) .await - .map_err(|e| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) - })?; + .map_err(|e| RelayError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; let Some(weight) = stake_table.known_nodes.get(&req.key) else { tracing::warn!("Received LCV2 signature from unknown node: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::UNAUTHORIZED, "LCV2 signature posted by nodes not on the stake table".to_owned(), )); @@ -90,7 +88,7 @@ impl LCV2StateRelayServerDataSource for LCV2StateRelayServerState { &req.next_stake, ) { tracing::warn!("Couldn't verify the received LCV2 signature: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "The posted LCV2 signature is not valid.".to_owned(), )); @@ -115,7 +113,7 @@ impl LCV2StateRelayServerDataSource for LCV2StateRelayServerState { match bundle.signatures.entry(req.key) { Entry::Occupied(_) => { // A signature is already posted for this key with this state - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "A LCV2 signature of this light client state is already posted at this block \ height for this key." diff --git a/crates/espresso/node/src/state_signature/relay_server/lcv3_relay.rs b/crates/espresso/node/src/state_signature/relay_server/lcv3_relay.rs index 9bb6f4d2ea4..904271554d7 100644 --- a/crates/espresso/node/src/state_signature/relay_server/lcv3_relay.rs +++ b/crates/espresso/node/src/state_signature/relay_server/lcv3_relay.rs @@ -4,6 +4,7 @@ use std::{ }; use alloy::primitives::U256; +use axum::http::StatusCode; use hotshot_contract_adapter::light_client::derive_signed_state_digest; use hotshot_types::{ light_client::{ @@ -11,16 +12,15 @@ use hotshot_types::{ }, traits::signature_key::LCV3StateSignatureKey, }; -use tide_disco::{Error, StatusCode, error::ServerError}; -use super::stake_table_tracker::StakeTableTracker; +use super::{RelayError, stake_table_tracker::StakeTableTracker}; #[async_trait::async_trait] pub trait LCV3StateRelayServerDataSource { /// Get the latest available signatures bundle. /// # Errors /// Errors if there's no available signatures bundle. - fn get_latest_signature_bundle(&self) -> Result; + fn get_latest_signature_bundle(&self) -> Result; /// Post a signature to the relay server /// # Errors @@ -28,7 +28,7 @@ pub trait LCV3StateRelayServerDataSource { async fn post_signature( &mut self, req: LCV3StateSignatureRequestBody, - ) -> Result<(), ServerError>; + ) -> Result<(), RelayError>; } /// Server state that tracks the light client V3 state and signatures @@ -50,10 +50,10 @@ pub struct LCV3StateRelayServerState { #[async_trait::async_trait] impl LCV3StateRelayServerDataSource for LCV3StateRelayServerState { - fn get_latest_signature_bundle(&self) -> Result { + fn get_latest_signature_bundle(&self) -> Result { self.latest_available_bundle .clone() - .ok_or(ServerError::catch_all( + .ok_or(RelayError::catch_all( StatusCode::NOT_FOUND, "The light client V3 state signatures are not ready.".to_owned(), )) @@ -62,7 +62,7 @@ impl LCV3StateRelayServerDataSource for LCV3StateRelayServerState { async fn post_signature( &mut self, req: LCV3StateSignatureRequestBody, - ) -> Result<(), ServerError> { + ) -> Result<(), RelayError> { let block_height = req.state.block_height; if block_height <= self.latest_block_height.unwrap_or(0) { // This signature is no longer needed @@ -72,12 +72,10 @@ impl LCV3StateRelayServerDataSource for LCV3StateRelayServerState { .stake_table_tracker .stake_table_info_for_block(block_height) .await - .map_err(|e| { - ServerError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) - })?; + .map_err(|e| RelayError::catch_all(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; let Some(weight) = stake_table.known_nodes.get(&req.key) else { tracing::warn!("Received LCV3 signature from unknown node: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::UNAUTHORIZED, "LCV3 signature posted by nodes not on the stake table".to_owned(), )); @@ -92,7 +90,7 @@ impl LCV3StateRelayServerDataSource for LCV3StateRelayServerState { signed_state_digest, ) { tracing::warn!("Couldn't verify the received LCV3 signature: {req}"); - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "The posted LCV3 signature is not valid.".to_owned(), )); @@ -118,7 +116,7 @@ impl LCV3StateRelayServerDataSource for LCV3StateRelayServerState { match bundle.signatures.entry(req.key) { Entry::Occupied(_) => { // A signature is already posted for this key with this state - return Err(ServerError::catch_all( + return Err(RelayError::catch_all( StatusCode::BAD_REQUEST, "A LCV3 signature of this light client state is already posted at this block \ height for this key." diff --git a/crates/hotshot/builder-api/src/v0_1/builder.rs b/crates/hotshot/builder-api/src/v0_1/builder.rs index 35f0664987a..71e847efb65 100644 --- a/crates/hotshot/builder-api/src/v0_1/builder.rs +++ b/crates/hotshot/builder-api/src/v0_1/builder.rs @@ -13,7 +13,11 @@ use hotshot_types::{traits::node_implementation::NodeType, utils::BuilderCommitm use serde::{Deserialize, Serialize}; use tagged_base64::TaggedBase64; use thiserror::Error; -use tide_disco::{Api, RequestError, RequestParams, StatusCode, api::ApiError, method::ReadState}; +// `RequestError` is re-exported because it is embedded in this module's wire error type +// (`Error::Request`/`Error::TxnUnpack`); servers reimplementing this API against a different +// HTTP framework need to construct it without a direct tide-disco dependency. +pub use tide_disco::RequestError; +use tide_disco::{Api, RequestParams, StatusCode, api::ApiError, method::ReadState}; use vbs::version::StaticVersionType; use super::{ diff --git a/crates/hotshot/orchestrator/Cargo.toml b/crates/hotshot/orchestrator/Cargo.toml index 522cc1e65d8..5740d20cc8b 100644 --- a/crates/hotshot/orchestrator/Cargo.toml +++ b/crates/hotshot/orchestrator/Cargo.toml @@ -8,6 +8,7 @@ license = "MIT" alloy = { workspace = true } anyhow = { workspace = true } async-lock = { workspace = true } +axum = { workspace = true } blake3 = { workspace = true } clap = { workspace = true } csv = { workspace = true } @@ -16,6 +17,10 @@ hotshot-types = { workspace = true } libp2p-identity = { workspace = true } multiaddr = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } +# Only used for its `error::ServerError` wire-error type and `Error` trait: `surf_disco::Client` +# is generic over `E: tide_disco::Error`, so `OrchestratorClient` and the builder-pinging client +# in `post_builder` both still name it. The HTTP server itself is axum. surf-disco = { workspace = true } tide-disco = { workspace = true } tokio = { workspace = true } diff --git a/crates/hotshot/orchestrator/src/client.rs b/crates/hotshot/orchestrator/src/client.rs index c0a62b7450f..6b1c0ba2604 100644 --- a/crates/hotshot/orchestrator/src/client.rs +++ b/crates/hotshot/orchestrator/src/client.rs @@ -15,8 +15,7 @@ use hotshot_types::{ }; use libp2p_identity::PeerId; use multiaddr::Multiaddr; -use surf_disco::{Client, error::ClientError}; -use tide_disco::Url; +use surf_disco::{Client, Url, error::ClientError}; use tokio::time::sleep; use tracing::{info, instrument}; use vbs::BinarySerializer; diff --git a/crates/hotshot/orchestrator/src/lib.rs b/crates/hotshot/orchestrator/src/lib.rs index 1f3703968bf..0a6ba4d32c1 100644 --- a/crates/hotshot/orchestrator/src/lib.rs +++ b/crates/hotshot/orchestrator/src/lib.rs @@ -14,14 +14,23 @@ use std::{ fs, fs::OpenOptions, io, + sync::Arc, time::Duration, }; use alloy::primitives::U256; use async_lock::RwLock; +use axum::{ + Json, Router, + body::Bytes, + extract::{Path, State}, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; use client::{BenchResults, BenchResultsDownloadConfig}; use csv::Writer; -use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; +use futures::{StreamExt, stream::FuturesUnordered}; use hotshot_types::{ PeerConfig, network::{BuilderType, NetworkConfig, PublicKeysFile}, @@ -35,17 +44,11 @@ use libp2p_identity::{ ed25519::{Keypair as EdKeypair, SecretKey}, }; use multiaddr::Multiaddr; +use serde::{Serialize, de::DeserializeOwned}; use surf_disco::Url; -use tide_disco::{ - Api, App, RequestError, - api::ApiError, - error::ServerError, - method::{ReadState, WriteState}, -}; -use vbs::{ - BinarySerializer, - version::{StaticVersion, StaticVersionType}, -}; +use tide_disco::{error::ServerError, healthcheck::HealthStatus}; +use tokio::net::TcpListener; +use vbs::{BinarySerializer, Serializer, version::StaticVersion}; /// Orchestrator is not, strictly speaking, bound to the network; it can have its own versioning. /// Orchestrator Version (major) @@ -668,176 +671,278 @@ where } } -/// Sets up all API routes -#[allow(clippy::too_many_lines)] -fn define_api() -> Result, ApiError> -where - TYPES: NodeType, - State: 'static + Send + Sync + ReadState + WriteState, - ::State: Send + Sync + OrchestratorApi, - TYPES::SignatureKey: serde::Serialize, - VER: StaticVersionType + 'static, -{ - let api_toml = toml::from_str::(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/api.toml" - ))) - .expect("API file is not valid toml"); - let mut api = Api::::new(api_toml)?; - api.post("post_identity", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - // Call our state function to process the request - state.post_identity(libp2p_address, libp2p_public_key) - } - .boxed() - })? - .post("post_getconfig", |req, state| { - async move { - let node_index = req.integer_param("node_index")?; - state.post_getconfig(node_index) - } - .boxed() - })? - .post("get_tmp_node_index", |_req, state| { - async move { state.get_tmp_node_index() }.boxed() - })? - .post("post_pubkey", |req, state| { - async move { - let is_da = req.boolean_param("is_da")?; - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((mut pubkey, libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - state.register_public_key(&mut pubkey, is_da, libp2p_address, libp2p_public_key) - } - .boxed() - })? - .get("peer_pubconfig_ready", |_req, state| { - async move { state.peer_pub_ready() }.boxed() - })? - .post("post_config_after_peer_collected", |_req, state| { - async move { state.post_config_after_peer_collected() }.boxed() - })? - .post( - "post_ready", - |req, state: &mut ::State| { - async move { - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - // Decode the payload-supplied pubkey - let Some(pubkey) = PeerConfig::::from_bytes(&body_bytes) else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - state.post_ready(&pubkey) - } - .boxed() - }, - )? - .post( - "post_manual_start", - |req, state: &mut ::State| { - async move { - let password = req.body_bytes(); - state.post_manual_start(password) - } - .boxed() - }, - )? - .get("get_start", |_req, state| { - async move { state.get_start() }.boxed() - })? - .post("post_results", |req, state| { - async move { - let metrics: Result = req.body_json(); - state.post_run_results(metrics.unwrap()) - } - .boxed() - })? - .post("post_builder", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - let Ok(urls) = - vbs::Serializer::::deserialize::>(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; +/// Shared, lock-guarded orchestrator state, cloned into every axum handler via `State`. +type SharedOrchestratorState = Arc>>; + +fn wants_binary(headers: &HeaderMap) -> bool { + headers + .get(header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("application/octet-stream")) +} + +fn axum_status(status: tide_disco::StatusCode) -> StatusCode { + StatusCode::from_u16(u16::from(status)).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) +} + +/// Encode a successful response body, negotiating VBS binary vs JSON from the `Accept` header, to +/// match tide-disco's content negotiation for `OrchestratorClient`, whose surf-disco client +/// defaults to `Accept: application/octet-stream`. +fn encode_ok(headers: &HeaderMap, value: T) -> Response { + if wants_binary(headers) { + match Serializer::::serialize(&value) { + Ok(bytes) => { + ([(header::CONTENT_TYPE, "application/octet-stream")], bytes).into_response() + }, + Err(err) => encode_err( + headers, + ServerError { + status: tide_disco::StatusCode::INTERNAL_SERVER_ERROR, + message: err.to_string(), + }, + ), + } + } else { + Json(value).into_response() + } +} + +/// Encode an error response using the same content negotiation as [`encode_ok`]. +fn encode_err(headers: &HeaderMap, err: ServerError) -> Response { + let status = axum_status(err.status); + if wants_binary(headers) { + match Serializer::::serialize(&err) { + Ok(bytes) => ( + status, + [(header::CONTENT_TYPE, "application/octet-stream")], + bytes, + ) + .into_response(), + Err(_) => (status, Json(err)).into_response(), + } + } else { + (status, Json(err)).into_response() + } +} + +fn respond(headers: &HeaderMap, result: Result) -> Response { + match result { + Ok(value) => encode_ok(headers, value), + Err(err) => encode_err(headers, err), + } +} + +fn malformed_body() -> ServerError { + ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "Malformed body".to_string(), + } +} + +/// Decode a POST body from `OrchestratorClient`. +/// +/// `client.rs` builds these bodies by calling `body_binary` on an already vbs-serialized +/// `Vec`, so the wire body is `[4-byte version][8-byte bincode `Vec` length][inner +/// vbs-serialized payload]`; the first 12 bytes are an artifact of that double encoding and must +/// be stripped before decoding the inner payload. +fn decode_wrapped_body(body: &[u8]) -> Result { + body.get(12..) + .and_then(|inner| Serializer::::deserialize(inner).ok()) + .ok_or_else(malformed_body) +} + +/// Like [`decode_wrapped_body`], but for the `ready` route, whose inner payload is a raw +/// [`PeerConfig::to_bytes`] blob rather than a second vbs-serialized value. +fn decode_wrapped_peer_config( + body: &[u8], +) -> Result, ServerError> { + body.get(12..) + .and_then(PeerConfig::::from_bytes) + .ok_or_else(malformed_body) +} + +/// Tide-disco-compatible module healthcheck: a bare [`HealthStatus`], negotiated JSON or vbs +/// binary from `Accept`. +async fn healthcheck(headers: HeaderMap) -> Response { + encode_ok(&headers, HealthStatus::Available) +} - let mut futures = urls +async fn post_identity( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let result = match decode_wrapped_body::<(Option, Option)>(&body) { + Ok((libp2p_address, libp2p_public_key)) => state + .write() + .await + .post_identity(libp2p_address, libp2p_public_key), + Err(err) => Err(err), + }; + respond(&headers, result) +} + +async fn post_getconfig( + State(state): State>, + Path(node_index): Path, + headers: HeaderMap, +) -> Response { + let result = state.write().await.post_getconfig(node_index); + respond(&headers, result) +} + +async fn get_tmp_node_index( + State(state): State>, + headers: HeaderMap, +) -> Response { + let result = state.write().await.get_tmp_node_index(); + respond(&headers, result) +} + +async fn post_pubkey( + State(state): State>, + Path(is_da): Path, + headers: HeaderMap, + body: Bytes, +) -> Response { + let result = match decode_wrapped_body::<(Vec, Option, Option)>(&body) { + Ok((mut pubkey, libp2p_address, libp2p_public_key)) => state + .write() + .await + .register_public_key(&mut pubkey, is_da, libp2p_address, libp2p_public_key), + Err(err) => Err(err), + }; + respond(&headers, result) +} + +async fn peer_pubconfig_ready( + State(state): State>, + headers: HeaderMap, +) -> Response { + let result = state.read().await.peer_pub_ready(); + respond(&headers, result) +} + +async fn post_config_after_peer_collected( + State(state): State>, + headers: HeaderMap, +) -> Response { + let result = state.write().await.post_config_after_peer_collected(); + respond(&headers, result) +} + +async fn post_ready( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let result = match decode_wrapped_peer_config::(&body) { + Ok(peer_config) => state.write().await.post_ready(&peer_config), + Err(err) => Err(err), + }; + respond(&headers, result) +} + +async fn post_manual_start( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + // Mirrors tide-disco's handler: the raw body is used as-is, with no framing. + let result = state.write().await.post_manual_start(body.to_vec()); + respond(&headers, result) +} + +async fn get_start( + State(state): State>, + headers: HeaderMap, +) -> Response { + let result = state.read().await.get_start(); + respond(&headers, result) +} + +async fn post_results( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + // Mirrors tide-disco's handler, which also panics on a malformed body: the only production + // caller (`client.rs`'s `post_bench_results`) always sends well-formed JSON. + let metrics: BenchResults = serde_json::from_slice(&body).unwrap(); + let result = state.write().await.post_run_results(metrics); + respond(&headers, result) +} + +async fn post_builder( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let result = match decode_wrapped_body::>(&body) { + Ok(urls) => { + let mut reachable = urls .into_iter() .map(|url| async { let client: surf_disco::Client = - surf_disco::client::Client::builder(url.clone()).build(); - if client.connect(Some(Duration::from_secs(2))).await { - Some(url) - } else { - None - } + surf_disco::Client::builder(url.clone()).build(); + client + .connect(Some(Duration::from_secs(2))) + .await + .then_some(url) }) .collect::>() .filter_map(futures::future::ready); - - if let Some(url) = futures.next().await { - state.post_builder(url) - } else { - Err(ServerError { + match reachable.next().await { + Some(url) => state.write().await.post_builder(url), + None => Err(ServerError { status: tide_disco::StatusCode::BAD_REQUEST, message: "No reachable addresses".to_string(), - }) + }), } - } - .boxed() - })? - .get("get_builders", |_req, state| { - async move { state.get_builders() }.boxed() - })?; - Ok(api) + }, + Err(err) => Err(err), + }; + respond(&headers, result) +} + +async fn get_builders( + State(state): State>, + headers: HeaderMap, +) -> Response { + let result = state.read().await.get_builders(); + respond(&headers, result) +} + +/// Builds the `api` module's routes. tide-disco served these both directly (e.g. `api/identity`) +/// and under a major-version prefix (`v0/api/identity`), redirecting the former to the latter; we +/// serve both forms directly instead, by nesting this router at both `/api` and `/v0/api`. +fn api_router() -> Router> { + Router::new() + .route("/healthcheck", get(healthcheck)) + .route("/identity", post(post_identity::)) + .route("/config/{node_index}", post(post_getconfig::)) + .route("/get_tmp_node_index", post(get_tmp_node_index::)) + .route("/pubkey/{is_da}", post(post_pubkey::)) + .route("/peer_pub_ready", get(peer_pubconfig_ready::)) + .route( + "/post_config_after_peer_collected", + post(post_config_after_peer_collected::), + ) + .route("/ready", post(post_ready::)) + .route("/start", get(get_start::)) + .route("/results", post(post_results::)) + .route("/manual_start", post(post_manual_start::)) + .route("/builders", get(get_builders::)) + .route("/builder", post(post_builder::)) } /// Runs the orchestrator /// # Errors -/// This errors if tide disco runs into an issue during serving -/// # Panics -/// This panics if unable to register the api with tide disco +/// This errors if axum runs into an issue during serving pub async fn run_orchestrator( mut network_config: NetworkConfig, url: Url, -) -> io::Result<()> -where - TYPES::SignatureKey: 'static + serde::Serialize, -{ +) -> io::Result<()> { let env_password = std::env::var("ORCHESTRATOR_MANUAL_START_PASSWORD"); if env_password.is_ok() { @@ -892,14 +997,27 @@ where }) .collect(); - let web_api = define_api().map_err(|_e| io::Error::other("Failed to define api")); - - let state: RwLock> = - RwLock::new(OrchestratorState::new(network_config)); + let state: SharedOrchestratorState = + Arc::new(RwLock::new(OrchestratorState::new(network_config))); + + let api = api_router::(); + let app = Router::new() + // tide-disco's app-level `healthcheck` isn't actually reachable for a singleton app like + // this one (it only registers the `api` module's own `healthcheck`), but we serve it + // anyway for parity with other axum conversions in this repo; no client depends on it. + .route("/healthcheck", get(healthcheck)) + .nest("/api", api.clone()) + .nest("/v0/api", api) + .with_state(state); + + let host = url + .host_str() + .ok_or_else(|| io::Error::other(format!("orchestrator url missing host: {url}")))?; + let port = url + .port_or_known_default() + .ok_or_else(|| io::Error::other(format!("orchestrator url missing port: {url}")))?; + let listener = TcpListener::bind((host, port)).await?; - let mut app = App::>, ServerError>::with_state(state); - app.register_module::("api", web_api.unwrap()) - .expect("Error registering api"); tracing::error!("listening on {url:?}"); - app.serve(url, ORCHESTRATOR_VERSION).await + axum::serve(listener, app).await } diff --git a/hotshot-state-prover/Cargo.toml b/hotshot-state-prover/Cargo.toml index 5c40cc411dd..9287890ef35 100644 --- a/hotshot-state-prover/Cargo.toml +++ b/hotshot-state-prover/Cargo.toml @@ -20,6 +20,7 @@ ark-ff = { workspace = true } ark-serialize = { workspace = true } ark-srs = { workspace = true } ark-std = { workspace = true } +axum = { workspace = true } clap = { workspace = true } displaydoc = { workspace = true } espresso-contract-deployer = { workspace = true } @@ -45,11 +46,11 @@ jf-signature-compat = { workspace = true } jf-utils = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } surf-disco = { workspace = true } tide-disco = { workspace = true } time = { workspace = true } tokio = { workspace = true } -toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } vbs = { workspace = true } diff --git a/hotshot-state-prover/src/http.rs b/hotshot-state-prover/src/http.rs new file mode 100644 index 00000000000..933b6fbc787 --- /dev/null +++ b/hotshot-state-prover/src/http.rs @@ -0,0 +1,41 @@ +//! Minimal HTTP server exposing the light client contract address, shared by the v1/v2/v3 +//! prover services. + +use alloy::primitives::Address; +use axum::{Json, Router, routing::get}; + +/// Serve the light client contract address at the paths tide-disco used to expose it: +/// `/v0/api/lightclient_contract` directly, and `/api/lightclient_contract` (which tide-disco +/// served via a redirect to the versioned path). Also serves `/healthcheck`. Runs until the +/// process exits; bind failures are logged, not propagated, since this server only provides a +/// healthcheck ahead of the prover's (fallible) main loop. +pub(crate) fn start_light_client_contract_server(port: u16, light_client_address: Address) { + let router = Router::new() + .route( + "/api/lightclient_contract", + get(move || async move { Json(light_client_address) }), + ) + .route( + "/v0/api/lightclient_contract", + get(move || async move { Json(light_client_address) }), + ) + .route("/healthcheck", get(healthcheck)); + + tokio::spawn(async move { + let addr = format!("0.0.0.0:{port}"); + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(listener) => listener, + Err(err) => { + tracing::error!("Failed to start prover http server on http://{addr} : {err}"); + return; + }, + }; + if let Err(err) = axum::serve(listener, router).await { + tracing::error!("Prover http server on http://{addr} stopped: {err}"); + } + }); +} + +async fn healthcheck() -> Json { + Json(serde_json::json!({ "status": "Available" })) +} diff --git a/hotshot-state-prover/src/lib.rs b/hotshot-state-prover/src/lib.rs index 701e243e9d9..5337d44f851 100644 --- a/hotshot-state-prover/src/lib.rs +++ b/hotshot-state-prover/src/lib.rs @@ -28,6 +28,7 @@ pub mod v3; pub mod utils; +mod http; mod test_utils; /// Configuration/Parameters used for hotshot state prover diff --git a/hotshot-state-prover/src/v1/service.rs b/hotshot-state-prover/src/v1/service.rs index 7b0ef84f434..5bfe4b2b724 100644 --- a/hotshot-state-prover/src/v1/service.rs +++ b/hotshot-state-prover/src/v1/service.rs @@ -9,7 +9,6 @@ use alloy::{ rpc::types::TransactionReceipt, }; use anyhow::{Context, Result, anyhow}; -use futures::FutureExt; use hotshot_contract_adapter::{ field_to_u256, sol_types::{LightClient, LightClientStateSol, PlonkProofSol, StakeTableStateSol}, @@ -24,9 +23,9 @@ use hotshot_types::{ use jf_pcs::prelude::UnivariateUniversalParams; use jf_relation_compat::Circuit as _; use surf_disco::Client; -use tide_disco::{Api, error::ServerError}; +use tide_disco::error::ServerError; use time::ext::InstantExt; -use tokio::{io, spawn, task::spawn_blocking, time::sleep}; +use tokio::{task::spawn_blocking, time::sleep}; use vbs::version::StaticVersionType; use crate::{ @@ -305,31 +304,10 @@ pub async fn sync_state( Ok(()) } -fn start_http_server( - port: u16, - light_client_address: Address, - bind_version: ApiVer, -) -> io::Result<()> { - let mut app = tide_disco::App::<_, ServerError>::with_state(()); - let toml = toml::from_str::(include_str!("../../api/prover-service.toml")) - .map_err(io::Error::other)?; - - let mut api = Api::<_, ServerError, ApiVer>::new(toml).map_err(io::Error::other)?; - - api.get("getlightclientcontract", move |_, _| { - async move { Ok(light_client_address) }.boxed() - }) - .map_err(io::Error::other)?; - app.register_module("api", api).map_err(io::Error::other)?; - - spawn(app.serve(format!("0.0.0.0:{port}"), bind_version)); - Ok(()) -} - /// Run prover in daemon mode pub async fn run_prover_service( config: StateProverConfig, - bind_version: ApiVer, + _bind_version: ApiVer, ) -> Result<()> { let mut state = ProverServiceState::new_genesis(config).await?; @@ -346,10 +324,8 @@ pub async fn run_prover_service( )); // Start the HTTP server to get a functioning healthcheck before any heavy computations. - if let Some(port) = state.config.port - && let Err(err) = start_http_server(port, state.config.light_client_address, bind_version) - { - tracing::error!("Error starting http server: {}", err); + if let Some(port) = state.config.port { + crate::http::start_light_client_contract_server(port, state.config.light_client_address); } let proving_key = diff --git a/hotshot-state-prover/src/v2/service.rs b/hotshot-state-prover/src/v2/service.rs index 59bb1d7cd22..e60b1061231 100644 --- a/hotshot-state-prover/src/v2/service.rs +++ b/hotshot-state-prover/src/v2/service.rs @@ -10,7 +10,6 @@ use alloy::{ }; use anyhow::{Context, Result, anyhow}; use espresso_types::{SeqTypes, StateCertQueryDataV1}; -use futures::FutureExt; use hotshot_contract_adapter::{ field_to_u256, sol_types::{LightClientStateSol, LightClientV2, PlonkProofSol, StakeTableStateSol}, @@ -30,9 +29,9 @@ use hotshot_types::{ use jf_pcs::prelude::UnivariateUniversalParams; use jf_relation_compat::Circuit as _; use surf_disco::Client; -use tide_disco::{Api, error::ServerError}; +use tide_disco::error::ServerError; use time::ext::InstantExt; -use tokio::{io, spawn, task::spawn_blocking, time::sleep}; +use tokio::{task::spawn_blocking, time::sleep}; use url::Url; use vbs::version::{StaticVersion, StaticVersionType}; @@ -481,31 +480,10 @@ pub async fn sync_state( Ok(()) } -fn start_http_server( - port: u16, - light_client_address: Address, - bind_version: ApiVer, -) -> io::Result<()> { - let mut app = tide_disco::App::<_, ServerError>::with_state(()); - let toml = toml::from_str::(include_str!("../../api/prover-service.toml")) - .map_err(io::Error::other)?; - - let mut api = Api::<_, ServerError, ApiVer>::new(toml).map_err(io::Error::other)?; - - api.get("getlightclientcontract", move |_, _| { - async move { Ok(light_client_address) }.boxed() - }) - .map_err(io::Error::other)?; - app.register_module("api", api).map_err(io::Error::other)?; - - spawn(app.serve(format!("0.0.0.0:{port}"), bind_version)); - Ok(()) -} - /// Run prover in daemon mode pub async fn run_prover_service( config: StateProverConfig, - bind_version: ApiVer, + _bind_version: ApiVer, ) -> Result<()> { let mut state = ProverServiceState::new_genesis(config).await?; @@ -522,10 +500,8 @@ pub async fn run_prover_service( )); // Start the HTTP server to get a functioning healthcheck before any heavy computations. - if let Some(port) = state.config.port - && let Err(err) = start_http_server(port, state.config.light_client_address, bind_version) - { - tracing::error!("Error starting http server: {}", err); + if let Some(port) = state.config.port { + crate::http::start_light_client_contract_server(port, state.config.light_client_address); } let proving_key = diff --git a/hotshot-state-prover/src/v3/service.rs b/hotshot-state-prover/src/v3/service.rs index 9e2a667f3d6..b1b06321498 100644 --- a/hotshot-state-prover/src/v3/service.rs +++ b/hotshot-state-prover/src/v3/service.rs @@ -10,7 +10,6 @@ use alloy::{ }; use anyhow::{Context, Result, anyhow}; use espresso_types::{SeqTypes, StateCertQueryDataV2}; -use futures::FutureExt; use hotshot_contract_adapter::{ field_to_u256, light_client::derive_signed_state_digest, @@ -31,9 +30,9 @@ use hotshot_types::{ use jf_pcs::prelude::UnivariateUniversalParams; use jf_relation::Circuit as _; use surf_disco::Client; -use tide_disco::{Api, error::ServerError}; +use tide_disco::error::ServerError; use time::ext::InstantExt; -use tokio::{io, spawn, task::spawn_blocking, time::sleep}; +use tokio::{task::spawn_blocking, time::sleep}; use url::Url; use vbs::version::{StaticVersion, StaticVersionType}; @@ -528,31 +527,10 @@ pub async fn sync_state( Ok(()) } -fn start_http_server( - port: u16, - light_client_address: Address, - bind_version: ApiVer, -) -> io::Result<()> { - let mut app = tide_disco::App::<_, ServerError>::with_state(()); - let toml = toml::from_str::(include_str!("../../api/prover-service.toml")) - .map_err(io::Error::other)?; - - let mut api = Api::<_, ServerError, ApiVer>::new(toml).map_err(io::Error::other)?; - - api.get("getlightclientcontract", move |_, _| { - async move { Ok(light_client_address) }.boxed() - }) - .map_err(io::Error::other)?; - app.register_module("api", api).map_err(io::Error::other)?; - - spawn(app.serve(format!("0.0.0.0:{port}"), bind_version)); - Ok(()) -} - /// Run prover in daemon mode pub async fn run_prover_service( config: StateProverConfig, - bind_version: ApiVer, + _bind_version: ApiVer, ) -> Result<()> { let mut state = ProverServiceState::new_genesis(config).await?; @@ -569,10 +547,8 @@ pub async fn run_prover_service( )); // Start the HTTP server to get a functioning healthcheck before any heavy computations. - if let Some(port) = state.config.port - && let Err(err) = start_http_server(port, state.config.light_client_address, bind_version) - { - tracing::error!("Error starting http server: {}", err); + if let Some(port) = state.config.port { + crate::http::start_light_client_contract_server(port, state.config.light_client_address); } let proving_key = diff --git a/light-client-query-service/Cargo.toml b/light-client-query-service/Cargo.toml index 99ed8a81820..c2e261aa2a6 100644 --- a/light-client-query-service/Cargo.toml +++ b/light-client-query-service/Cargo.toml @@ -12,26 +12,28 @@ required-features = ["genesis"] [features] default = ["genesis"] -genesis = ["hotshot-types", "surf-disco"] +genesis = [] [dependencies] anyhow = { workspace = true } +axum = { workspace = true } clap = { workspace = true } espresso-node = { workspace = true, default-features = false } -espresso-types = { workspace = true, features = ["node"] } +espresso-types = { workspace = true } +futures = { workspace = true } hotshot-query-service = { workspace = true } - -# Optional dependencies for genesis utility. -hotshot-types = { workspace = true, optional = true } -light-client = { workspace = true, features = ["client", "rlp"] } +hotshot-types = { workspace = true } +light-client = { workspace = true } log-panics = { workspace = true } -semver = { workspace = true } -surf-disco = { workspace = true, optional = true } +serde = { workspace = true } +serde_json = { workspace = true } +surf-disco = { workspace = true } tide-disco = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } vbs = { workspace = true } [lints] diff --git a/light-client-query-service/src/api.rs b/light-client-query-service/src/api.rs new file mode 100644 index 00000000000..cf5a6fc680d --- /dev/null +++ b/light-client-query-service/src/api.rs @@ -0,0 +1,1230 @@ +//! Axum port of the `availability` and `node` API modules that this service used to serve via +//! `hotshot_query_service::{availability, node}::define_api` on a `tide_disco::App`. +//! +//! Route paths, status codes and the wire error type are all taken directly from +//! `hotshot-query-service` (see `availability.rs`/`node.rs` there and their handler bodies) so +//! that clients built against the old tide-disco server keep working unmodified. + +use std::{ops::Bound, time::Duration}; + +use axum::{ + Json, Router, + extract::{ + Path, State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::get, +}; +use espresso_node::api::sql::DataSource; +use espresso_types::SeqTypes; +use futures::{StreamExt as _, TryStreamExt as _, stream::BoxStream}; +use hotshot_query_service::{ + Error as ApiError, Header, + availability::{ + self, AvailabilityDataSource as _, BlockHash, BlockId, BlockQueryData, + BlockSummaryQueryData, BlockWithTransaction, LeafHash, LeafId, LeafQueryData, + Limits as AvailabilityLimits, PayloadQueryData, QueryableHeader as _, + QueryablePayload as _, TransactionHash, TransactionQueryData, + TransactionWithProofQueryData, VidCommonQueryData, + }, + node::{self, Limits as NodeLimits, NodeDataSource as _, WindowStart}, + types::HeightIndexed as _, +}; +use hotshot_types::data::VidCommitment; +use serde::Serialize; +use surf_disco::Error as _; +use tide_disco::healthcheck::HealthStatus; +use vbs::{BinarySerializer, Serializer, version::StaticVersion}; + +/// Binary framing version for VBS-negotiated responses, matching the wire version this service +/// used under tide-disco. +type WireVersion = StaticVersion<0, 1>; + +/// Mirrors `hotshot_query_service::availability::Options::default()` and +/// `node::Options::default()`, which is what this service's tide-disco setup used. +const FETCH_TIMEOUT: Duration = Duration::from_millis(500); +const SMALL_OBJECT_RANGE_LIMIT: usize = 500; +const LARGE_OBJECT_RANGE_LIMIT: usize = 100; +const WINDOW_LIMIT: usize = 500; + +fn wants_binary(headers: &HeaderMap) -> bool { + headers + .get(header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("application/octet-stream")) +} + +/// Maps `hotshot_query_service`'s wrapped `reqwest`-based status code onto axum's. +fn wire_status(status: surf_disco::StatusCode) -> StatusCode { + StatusCode::from_u16(u16::from(status)).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) +} + +/// Encode a successful response, negotiating VBS binary vs JSON from the `Accept` header, just +/// as tide-disco did for `surf-disco` clients (which default to `application/octet-stream`). +fn encode_ok(headers: &HeaderMap, value: T) -> Response { + if wants_binary(headers) { + match Serializer::::serialize(&value) { + Ok(bytes) => { + ([(header::CONTENT_TYPE, "application/octet-stream")], bytes).into_response() + }, + Err(err) => encode_err(headers, ApiError::internal(err)), + } + } else { + Json(value).into_response() + } +} + +/// Encode an error response using the same content negotiation as [`encode_ok`]. `err` is +/// `hotshot_query_service::Error`, the exact type the old tide-disco `App` used, so both its +/// status mapping and its wire shape (externally tagged enum) match byte-for-byte. +fn encode_err(headers: &HeaderMap, err: ApiError) -> Response { + let status = wire_status(err.status()); + if wants_binary(headers) { + match Serializer::::serialize(&err) { + Ok(bytes) => ( + status, + [(header::CONTENT_TYPE, "application/octet-stream")], + bytes, + ) + .into_response(), + Err(_) => (status, Json(err)).into_response(), + } + } else { + (status, Json(err)).into_response() + } +} + +fn respond(headers: &HeaderMap, result: Result) -> Response { + match result { + Ok(v) => encode_ok(headers, v), + Err(e) => encode_err(headers, e), + } +} + +/// Parses a path parameter the way tide-disco's `TaggedBase64`/`Integer` param types did, +/// reporting failures the same way tide-disco's own request-parsing errors are surfaced: as a +/// 400 with a descriptive message. +fn parse_availability_param( + value: &str, + field: &str, +) -> Result +where + T::Err: std::fmt::Display, +{ + value.parse().map_err(|e| availability::Error::Custom { + message: format!("invalid {field}: {e}"), + status: surf_disco::StatusCode::BAD_REQUEST, + }) +} + +/// Same as [`parse_availability_param`], for handlers in the `node` module. +fn parse_node_param(value: &str, field: &str) -> Result +where + T::Err: std::fmt::Display, +{ + value.parse().map_err(|e| node::Error::Custom { + message: format!("invalid {field}: {e}"), + status: surf_disco::StatusCode::BAD_REQUEST, + }) +} + +fn enforce_range_limit(from: usize, until: usize, limit: usize) -> Result<(), availability::Error> { + if until.saturating_sub(from) > limit { + return Err(availability::Error::RangeLimit { from, until, limit }); + } + Ok(()) +} + +/// Tide-disco-compatible singleton-app healthcheck: a bare [`HealthStatus`], negotiated JSON or +/// vbs binary from `Accept`. +async fn healthcheck(headers: HeaderMap) -> Response { + encode_ok(&headers, HealthStatus::Available) +} + +async fn drive_ws_stream( + mut socket: WebSocket, + mut stream: BoxStream<'static, T>, + binary: bool, +) { + while let Some(item) = stream.next().await { + let msg = if binary { + match Serializer::::serialize(&item) { + Ok(bytes) => Message::Binary(bytes.into()), + Err(_) => break, + } + } else { + match serde_json::to_string(&item) { + Ok(text) => Message::Text(text.into()), + Err(_) => break, + } + }; + if socket.send(msg).await.is_err() { + break; + } + } +} + +// --- availability ----------------------------------------------------------------------------- + +async fn fetch_leaf( + ds: &DataSource, + id: LeafId, +) -> Result, availability::Error> { + ds.get_leaf(id) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchLeaf { + resource: id.to_string(), + }) +} + +async fn fetch_leaf_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, SMALL_OBJECT_RANGE_LIMIT)?; + ds.get_leaf_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchLeaf { + resource: (index + from).to_string(), + }) + }) + .try_collect() + .await +} + +async fn fetch_header( + ds: &DataSource, + id: BlockId, +) -> Result, availability::Error> { + ds.get_header(id) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchHeader { + resource: id.to_string(), + }) +} + +async fn fetch_header_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, LARGE_OBJECT_RANGE_LIMIT)?; + ds.get_header_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch.with_timeout(FETCH_TIMEOUT).await.ok_or_else(|| { + availability::Error::FetchHeader { + resource: (index + from).to_string(), + } + }) + }) + .try_collect() + .await +} + +async fn fetch_block( + ds: &DataSource, + id: BlockId, +) -> Result, availability::Error> { + ds.get_block(id) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: id.to_string(), + }) +} + +async fn fetch_block_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, LARGE_OBJECT_RANGE_LIMIT)?; + ds.get_block_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: (index + from).to_string(), + }) + }) + .try_collect() + .await +} + +async fn fetch_payload( + ds: &DataSource, + id: BlockId, +) -> Result, availability::Error> { + // Matches tide: payloads are keyed by `BlockId` and report `FetchBlock` on a miss, there is + // no separate `FetchPayload` variant. + ds.get_payload(id) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: id.to_string(), + }) +} + +async fn fetch_payload_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, LARGE_OBJECT_RANGE_LIMIT)?; + ds.get_payload_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: (index + from).to_string(), + }) + }) + .try_collect() + .await +} + +async fn fetch_vid_common( + ds: &DataSource, + id: BlockId, +) -> Result, availability::Error> { + ds.get_vid_common(id) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: id.to_string(), + }) +} + +async fn fetch_vid_common_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, SMALL_OBJECT_RANGE_LIMIT)?; + ds.get_vid_common_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: (index + from).to_string(), + }) + }) + .try_collect() + .await +} + +async fn fetch_transaction_by_position( + ds: &DataSource, + height: u64, + index: u64, +) -> Result, availability::Error> { + let block = fetch_block(ds, BlockId::Number(height as usize)).await?; + let ix = block + .payload() + .nth(block.metadata(), index as usize) + .ok_or(availability::Error::InvalidTransactionIndex { height, index })?; + let transaction = block + .transaction(&ix) + .ok_or(availability::Error::InvalidTransactionIndex { height, index })?; + let transaction = TransactionQueryData::new(transaction, &block, &ix, index) + .ok_or(availability::Error::InvalidTransactionIndex { height, index })?; + Ok(BlockWithTransaction { + block, + transaction, + index: ix, + }) +} + +async fn fetch_transaction_by_hash( + ds: &DataSource, + hash: &str, +) -> Result, availability::Error> { + let hash = parse_availability_param::>(hash, "hash")?; + ds.get_block_containing_transaction(hash) + .await + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchTransaction { + resource: hash.to_string(), + }) +} + +async fn fetch_transaction_with_proof( + ds: &DataSource, + bwt: BlockWithTransaction, +) -> Result, availability::Error> { + let height = bwt.block.height(); + let vid = fetch_vid_common(ds, BlockId::Number(height as usize)).await?; + let proof = bwt.block.transaction_proof(&vid, &bwt.index).ok_or( + availability::Error::InvalidTransactionIndex { + height, + index: bwt.transaction.index(), + }, + )?; + Ok(TransactionWithProofQueryData::new(bwt.transaction, proof)) +} + +async fn fetch_block_summary( + ds: &DataSource, + height: usize, +) -> Result, availability::Error> { + fetch_block(ds, BlockId::Number(height)) + .await + .map(BlockSummaryQueryData::from) +} + +async fn fetch_block_summary_range( + ds: &DataSource, + from: usize, + until: usize, +) -> Result>, availability::Error> { + enforce_range_limit(from, until, LARGE_OBJECT_RANGE_LIMIT)?; + ds.get_block_range(from..until) + .await + .enumerate() + .then(|(index, fetch)| async move { + fetch + .with_timeout(FETCH_TIMEOUT) + .await + .ok_or_else(|| availability::Error::FetchBlock { + resource: (index + from).to_string(), + }) + }) + .map(|result| result.map(BlockSummaryQueryData::from)) + .try_collect() + .await +} + +/// Dispatches a `BlockId`-keyed fetch once the id has been parsed from the path, matching the +/// `respond`/error-conversion boilerplate every `availability` route needs. +async fn respond_block_resource( + headers: &HeaderMap, + id: Result, availability::Error>, + fetch: F, +) -> Response +where + T: Serialize, + F: AsyncFnOnce(BlockId) -> Result, +{ + let result = match id { + Ok(id) => fetch(id).await, + Err(e) => Err(e), + }; + respond(headers, result.map_err(ApiError::from)) +} + +async fn get_leaf_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let result = fetch_leaf(&ds, LeafId::Number(height as usize)).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_leaf_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let result = match parse_availability_param::>(&hash, "hash") { + Ok(hash) => fetch_leaf(&ds, LeafId::Hash(hash)).await, + Err(e) => Err(e), + }; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_leaf_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_leaf_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_leaves( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = ds.subscribe_leaves(height).await; + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn get_header_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + respond_block_resource(&headers, Ok(BlockId::Number(height as usize)), async |id| { + fetch_header(&ds, id).await + }) + .await +} + +async fn get_header_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let id = parse_availability_param::>(&hash, "hash").map(BlockId::Hash); + respond_block_resource(&headers, id, async |id| fetch_header(&ds, id).await).await +} + +async fn get_header_by_payload_hash( + State(ds): State, + headers: HeaderMap, + Path(payload_hash): Path, +) -> Response { + let id = parse_availability_param::(&payload_hash, "payload-hash") + .map(BlockId::PayloadHash); + respond_block_resource(&headers, id, async |id| fetch_header(&ds, id).await).await +} + +async fn get_header_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_header_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_headers( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = ds.subscribe_headers(height).await; + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn get_block_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + respond_block_resource(&headers, Ok(BlockId::Number(height as usize)), async |id| { + fetch_block(&ds, id).await + }) + .await +} + +async fn get_block_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let id = parse_availability_param::>(&hash, "hash").map(BlockId::Hash); + respond_block_resource(&headers, id, async |id| fetch_block(&ds, id).await).await +} + +async fn get_block_by_payload_hash( + State(ds): State, + headers: HeaderMap, + Path(payload_hash): Path, +) -> Response { + let id = parse_availability_param::(&payload_hash, "payload-hash") + .map(BlockId::PayloadHash); + respond_block_resource(&headers, id, async |id| fetch_block(&ds, id).await).await +} + +async fn get_block_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_block_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_blocks( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = ds.subscribe_blocks(height).await; + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn get_payload_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + respond_block_resource(&headers, Ok(BlockId::Number(height as usize)), async |id| { + fetch_payload(&ds, id).await + }) + .await +} + +async fn get_payload_by_payload_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let id = parse_availability_param::(&hash, "hash").map(BlockId::PayloadHash); + respond_block_resource(&headers, id, async |id| fetch_payload(&ds, id).await).await +} + +async fn get_payload_by_block_hash( + State(ds): State, + headers: HeaderMap, + Path(block_hash): Path, +) -> Response { + let id = parse_availability_param::>(&block_hash, "block-hash") + .map(BlockId::Hash); + respond_block_resource(&headers, id, async |id| fetch_payload(&ds, id).await).await +} + +async fn get_payload_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_payload_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_payloads( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = ds.subscribe_payloads(height).await; + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn get_vid_common_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + respond_block_resource(&headers, Ok(BlockId::Number(height as usize)), async |id| { + fetch_vid_common(&ds, id).await + }) + .await +} + +async fn get_vid_common_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let id = parse_availability_param::>(&hash, "hash").map(BlockId::Hash); + respond_block_resource(&headers, id, async |id| fetch_vid_common(&ds, id).await).await +} + +async fn get_vid_common_by_payload_hash( + State(ds): State, + headers: HeaderMap, + Path(payload_hash): Path, +) -> Response { + let id = parse_availability_param::(&payload_hash, "payload-hash") + .map(BlockId::PayloadHash); + respond_block_resource(&headers, id, async |id| fetch_vid_common(&ds, id).await).await +} + +async fn get_vid_common_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_vid_common_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_vid_common( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = ds.subscribe_vid_common(height).await; + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn get_transaction_by_position( + State(ds): State, + headers: HeaderMap, + Path((height, index)): Path<(u64, u64)>, +) -> Response { + let result = fetch_transaction_by_position(&ds, height, index) + .await + .map(|bwt| bwt.transaction); + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_transaction_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let result = fetch_transaction_by_hash(&ds, &hash) + .await + .map(|bwt| bwt.transaction); + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_transaction_proof_by_position( + State(ds): State, + headers: HeaderMap, + Path((height, index)): Path<(u64, u64)>, +) -> Response { + let result = async { + let bwt = fetch_transaction_by_position(&ds, height, index).await?; + fetch_transaction_with_proof(&ds, bwt).await + } + .await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_transaction_proof_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let result = async { + let bwt = fetch_transaction_by_hash(&ds, &hash).await?; + fetch_transaction_with_proof(&ds, bwt).await + } + .await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn stream_transactions( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = transactions_stream(ds.subscribe_blocks(height).await, None); + drive_ws_stream(socket, stream, binary).await; + }) +} + +async fn stream_transactions_ns( + ws: WebSocketUpgrade, + State(ds): State, + headers: HeaderMap, + Path((height, namespace)): Path<(usize, i64)>, +) -> Response { + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| async move { + let stream = transactions_stream(ds.subscribe_blocks(height).await, Some(namespace)); + drive_ws_stream(socket, stream, binary).await; + }) +} + +/// Mirrors the filtering closure in tide's `stream_transactions` handler: pulls every +/// transaction out of each block, optionally restricted to a single namespace. +fn transactions_stream( + blocks: BoxStream<'static, BlockQueryData>, + namespace: Option, +) -> BoxStream<'static, TransactionQueryData> { + blocks + .flat_map(move |block| { + let header = block.header().clone(); + let filtered: Vec<_> = block + .enumerate() + .enumerate() + .filter_map(|(i, (index, _tx))| { + if let Some(ns) = namespace { + let ns_id = header.namespace_id(&index.ns_index)?; + if i64::from(ns_id) != ns { + return None; + } + } + let tx = block.transaction(&index)?; + TransactionQueryData::new(tx, &block, &index, i as u64) + }) + .collect(); + futures::stream::iter(filtered) + }) + .boxed() +} + +async fn get_block_summary_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let result = fetch_block_summary(&ds, height).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_block_summary_range( + State(ds): State, + headers: HeaderMap, + Path((from, until)): Path<(usize, usize)>, +) -> Response { + let result = fetch_block_summary_range(&ds, from, until).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_cert2( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let result = ds + .get_cert2(height) + .await + .map_err(availability::Error::from); + respond(&headers, result.map_err(ApiError::from)) +} + +async fn get_availability_limits(headers: HeaderMap) -> Response { + encode_ok( + &headers, + AvailabilityLimits { + small_object_range_limit: SMALL_OBJECT_RANGE_LIMIT, + large_object_range_limit: LARGE_OBJECT_RANGE_LIMIT, + }, + ) +} + +fn availability_router(ds: DataSource) -> Router { + Router::new() + .route("/leaf/{height}", get(get_leaf_by_height)) + .route("/leaf/hash/{hash}", get(get_leaf_by_hash)) + .route("/leaf/{from}/{until}", get(get_leaf_range)) + .route("/stream/leaves/{height}", get(stream_leaves)) + .route("/header/{height}", get(get_header_by_height)) + .route("/header/hash/{hash}", get(get_header_by_hash)) + .route( + "/header/payload-hash/{payload_hash}", + get(get_header_by_payload_hash), + ) + .route("/header/{from}/{until}", get(get_header_range)) + .route("/stream/headers/{height}", get(stream_headers)) + .route("/block/{height}", get(get_block_by_height)) + .route("/block/hash/{hash}", get(get_block_by_hash)) + .route( + "/block/payload-hash/{payload_hash}", + get(get_block_by_payload_hash), + ) + .route("/block/{from}/{until}", get(get_block_range)) + .route("/stream/blocks/{height}", get(stream_blocks)) + .route("/payload/{height}", get(get_payload_by_height)) + .route("/payload/hash/{hash}", get(get_payload_by_payload_hash)) + .route( + "/payload/block-hash/{block_hash}", + get(get_payload_by_block_hash), + ) + .route("/payload/{from}/{until}", get(get_payload_range)) + .route("/stream/payloads/{height}", get(stream_payloads)) + .route("/vid/common/{height}", get(get_vid_common_by_height)) + .route("/vid/common/hash/{hash}", get(get_vid_common_by_hash)) + .route( + "/vid/common/payload-hash/{payload_hash}", + get(get_vid_common_by_payload_hash), + ) + .route("/vid/common/{from}/{until}", get(get_vid_common_range)) + .route("/stream/vid/common/{height}", get(stream_vid_common)) + .route( + "/transaction/{height}/{index}/noproof", + get(get_transaction_by_position), + ) + .route( + "/transaction/hash/{hash}/noproof", + get(get_transaction_by_hash), + ) + .route( + "/transaction/{height}/{index}", + get(get_transaction_proof_by_position), + ) + .route( + "/transaction/hash/{hash}", + get(get_transaction_proof_by_hash), + ) + .route( + "/transaction/{height}/{index}/proof", + get(get_transaction_proof_by_position), + ) + .route( + "/transaction/hash/{hash}/proof", + get(get_transaction_proof_by_hash), + ) + .route( + "/stream/transactions/{height}/namespace/{namespace}", + get(stream_transactions_ns), + ) + .route("/stream/transactions/{height}", get(stream_transactions)) + .route("/block/summary/{height}", get(get_block_summary_by_height)) + .route( + "/block/summaries/{from}/{until}", + get(get_block_summary_range), + ) + .route("/cert2/{height}", get(get_cert2)) + .route("/limits", get(get_availability_limits)) + .with_state(ds) +} + +// --- node --------------------------------------------------------------------------------------- + +fn range_bounds(from: Option, to: Option) -> (Bound, Bound) { + ( + from.map_or(Bound::Unbounded, |f| Bound::Included(f as usize)), + to.map_or(Bound::Unbounded, |t| Bound::Included(t as usize)), + ) +} + +async fn fetch_count_transactions( + ds: &DataSource, + from: Option, + to: Option, + namespace: Option, +) -> Result { + ds.count_transactions_in_range(range_bounds(from, to), namespace.map(Into::into)) + .await + .map_err(Into::into) +} + +async fn fetch_payload_size( + ds: &DataSource, + from: Option, + to: Option, + namespace: Option, +) -> Result { + ds.payload_size_in_range(range_bounds(from, to), namespace.map(Into::into)) + .await + .map_err(Into::into) +} + +async fn fetch_vid_share( + ds: &DataSource, + id: BlockId, +) -> Result { + ds.vid_share(id) + .await + .map_err(|source| node::Error::QueryVid { + source, + block: id.to_string(), + }) +} + +async fn fetch_header_window( + ds: &DataSource, + start: WindowStart, + end: u64, +) -> Result>, node::Error> { + ds.get_header_window(start, end, WINDOW_LIMIT) + .await + .map_err(|source| node::Error::QueryWindow { + source, + start: format!("{start:?}"), + end, + }) +} + +async fn node_block_height(State(ds): State, headers: HeaderMap) -> Response { + let result = ds.block_height().await.map_err(node::Error::from); + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_count_transactions( + State(ds): State, + headers: HeaderMap, + from: Option, + to: Option, + namespace: Option, +) -> Response { + let result = fetch_count_transactions(&ds, from, to, namespace).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_count_transactions_all(state: State, headers: HeaderMap) -> Response { + node_count_transactions(state, headers, None, None, None).await +} + +async fn node_count_transactions_to( + state: State, + headers: HeaderMap, + Path(to): Path, +) -> Response { + node_count_transactions(state, headers, None, Some(to), None).await +} + +async fn node_count_transactions_from_to( + state: State, + headers: HeaderMap, + Path((from, to)): Path<(u64, u64)>, +) -> Response { + node_count_transactions(state, headers, Some(from), Some(to), None).await +} + +async fn node_count_transactions_ns( + state: State, + headers: HeaderMap, + Path(namespace): Path, +) -> Response { + node_count_transactions(state, headers, None, None, Some(namespace)).await +} + +async fn node_count_transactions_ns_to( + state: State, + headers: HeaderMap, + Path((namespace, to)): Path<(i64, u64)>, +) -> Response { + node_count_transactions(state, headers, None, Some(to), Some(namespace)).await +} + +async fn node_count_transactions_ns_from_to( + state: State, + headers: HeaderMap, + Path((namespace, from, to)): Path<(i64, u64, u64)>, +) -> Response { + node_count_transactions(state, headers, Some(from), Some(to), Some(namespace)).await +} + +async fn node_payload_size( + State(ds): State, + headers: HeaderMap, + from: Option, + to: Option, + namespace: Option, +) -> Response { + let result = fetch_payload_size(&ds, from, to, namespace).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_payload_size_all(state: State, headers: HeaderMap) -> Response { + node_payload_size(state, headers, None, None, None).await +} + +async fn node_payload_size_to( + state: State, + headers: HeaderMap, + Path(to): Path, +) -> Response { + node_payload_size(state, headers, None, Some(to), None).await +} + +async fn node_payload_size_from_to( + state: State, + headers: HeaderMap, + Path((from, to)): Path<(u64, u64)>, +) -> Response { + node_payload_size(state, headers, Some(from), Some(to), None).await +} + +async fn node_payload_size_ns( + state: State, + headers: HeaderMap, + Path(namespace): Path, +) -> Response { + node_payload_size(state, headers, None, None, Some(namespace)).await +} + +async fn node_payload_size_ns_to( + state: State, + headers: HeaderMap, + Path((namespace, to)): Path<(i64, u64)>, +) -> Response { + node_payload_size(state, headers, None, Some(to), Some(namespace)).await +} + +async fn node_payload_size_ns_from_to( + state: State, + headers: HeaderMap, + Path((namespace, from, to)): Path<(i64, u64, u64)>, +) -> Response { + node_payload_size(state, headers, Some(from), Some(to), Some(namespace)).await +} + +async fn node_vid_share_by_height( + State(ds): State, + headers: HeaderMap, + Path(height): Path, +) -> Response { + let result = fetch_vid_share(&ds, BlockId::Number(height as usize)).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_vid_share_by_hash( + State(ds): State, + headers: HeaderMap, + Path(hash): Path, +) -> Response { + let result = match parse_node_param::>(&hash, "hash") { + Ok(hash) => fetch_vid_share(&ds, BlockId::Hash(hash)).await, + Err(e) => Err(e), + }; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_vid_share_by_payload_hash( + State(ds): State, + headers: HeaderMap, + Path(payload_hash): Path, +) -> Response { + let result = match parse_node_param::(&payload_hash, "payload-hash") { + Ok(hash) => fetch_vid_share(&ds, BlockId::PayloadHash(hash)).await, + Err(e) => Err(e), + }; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_sync_status(State(ds): State, headers: HeaderMap) -> Response { + let result = ds.sync_status().await.map_err(node::Error::from); + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_header_window( + State(ds): State, + headers: HeaderMap, + Path((start, end)): Path<(u64, u64)>, +) -> Response { + let result = fetch_header_window(&ds, WindowStart::Time(start), end).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_header_window_from_height( + State(ds): State, + headers: HeaderMap, + Path((height, end)): Path<(u64, u64)>, +) -> Response { + let result = fetch_header_window(&ds, WindowStart::Height(height), end).await; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_header_window_from_hash( + State(ds): State, + headers: HeaderMap, + Path((hash, end)): Path<(String, u64)>, +) -> Response { + let result = match parse_node_param::>(&hash, "hash") { + Ok(hash) => fetch_header_window(&ds, WindowStart::Hash(hash), end).await, + Err(e) => Err(e), + }; + respond(&headers, result.map_err(ApiError::from)) +} + +async fn node_limits(headers: HeaderMap) -> Response { + encode_ok( + &headers, + NodeLimits { + window_limit: WINDOW_LIMIT, + }, + ) +} + +fn node_router(ds: DataSource) -> Router { + Router::new() + .route("/block-height", get(node_block_height)) + .route("/transactions/count", get(node_count_transactions_all)) + .route("/transactions/count/{to}", get(node_count_transactions_to)) + .route( + "/transactions/count/{from}/{to}", + get(node_count_transactions_from_to), + ) + .route( + "/transactions/count/namespace/{namespace}", + get(node_count_transactions_ns), + ) + .route( + "/transactions/count/namespace/{namespace}/{to}", + get(node_count_transactions_ns_to), + ) + .route( + "/transactions/count/namespace/{namespace}/{from}/{to}", + get(node_count_transactions_ns_from_to), + ) + .route("/payloads/size", get(node_payload_size_all)) + .route("/payloads/total-size", get(node_payload_size_all)) + .route("/payloads/size/{to}", get(node_payload_size_to)) + .route("/payloads/size/{from}/{to}", get(node_payload_size_from_to)) + .route( + "/payloads/size/namespace/{namespace}", + get(node_payload_size_ns), + ) + .route( + "/payloads/size/namespace/{namespace}/{to}", + get(node_payload_size_ns_to), + ) + .route( + "/payloads/size/namespace/{namespace}/{from}/{to}", + get(node_payload_size_ns_from_to), + ) + .route("/vid/share/{height}", get(node_vid_share_by_height)) + .route("/vid/share/hash/{hash}", get(node_vid_share_by_hash)) + .route( + "/vid/share/payload-hash/{payload_hash}", + get(node_vid_share_by_payload_hash), + ) + .route("/sync-status", get(node_sync_status)) + .route("/header/window/{start}/{end}", get(node_header_window)) + .route( + "/header/window/from/{height}/{end}", + get(node_header_window_from_height), + ) + .route( + "/header/window/from/hash/{hash}/{end}", + get(node_header_window_from_hash), + ) + .route("/limits", get(node_limits)) + .with_state(ds) +} + +/// Builds the full router: `healthcheck`, plus the `availability` and `node` modules served both +/// unversioned and under `/v1`, matching the paths tide-disco exposed for this service (which +/// only ever registered API version `1.0.0`). +pub fn router(ds: DataSource) -> Router { + let availability = availability_router(ds.clone()); + let node = node_router(ds); + Router::new() + .route("/healthcheck", get(healthcheck)) + .nest("/availability", availability.clone()) + .nest("/v1/availability", availability) + .nest("/node", node.clone()) + .nest("/v1/node", node) +} diff --git a/light-client-query-service/src/lib.rs b/light-client-query-service/src/lib.rs index 9a4a1fc0375..4103b0a4523 100644 --- a/light-client-query-service/src/lib.rs +++ b/light-client-query-service/src/lib.rs @@ -2,6 +2,8 @@ use clap::ValueEnum; use log_panics::BacktraceMode; use tracing_subscriber::{EnvFilter, fmt::format::FmtSpan}; +pub mod api; + /// Controls how logs are displayed and how backtraces are logged on panic. /// /// The values here match the possible values of `RUST_LOG_FORMAT`, and their corresponding behavior diff --git a/light-client-query-service/src/main.rs b/light-client-query-service/src/main.rs index 71d1157a1b6..b62fe1013ce 100644 --- a/light-client-query-service/src/main.rs +++ b/light-client-query-service/src/main.rs @@ -3,16 +3,13 @@ use std::{cmp::max, fs, path::PathBuf, process::ExitCode, sync::Arc, time::Durat use anyhow::{Context, Result}; use clap::Parser; use espresso_node::{ - SequencerApiVersion, api::{data_source::SequencerDataSource, sql::DataSource}, persistence::sql, }; use espresso_types::parse_duration; use hotshot_query_service::{ - ApiState, - availability::{self, BlockInfo, LeafId, UpdateAvailabilityData}, + availability::{BlockInfo, LeafId, UpdateAvailabilityData}, fetching::provider::AnyProvider, - node, }; use light_client::{ LightClient, @@ -20,12 +17,10 @@ use light_client::{ state, storage::{LightClientSqliteOptions, Storage}, }; -use light_client_query_service::{LogFormat, init_logging}; -use semver::Version; -use tide_disco::{App, Url}; -use tokio::{spawn, time::sleep}; +use light_client_query_service::{LogFormat, api, init_logging}; +use tokio::{net::TcpListener, spawn, time::sleep}; use tracing::instrument; -use vbs::version::StaticVersionType; +use url::Url; /// Run an Espresso query service. /// @@ -188,19 +183,10 @@ async fn run() -> Result<()> { spawn(update(lc, ds.clone(), opt.poll_opt)); // Run server. - let mut app = App::<_, hotshot_query_service::Error>::with_state(ApiState::from(ds)); - let ver = SequencerApiVersion::instance(); - let api_ver: Version = "1.0.0".parse().unwrap(); - app.register_module( - "availability", - availability::define_api(&Default::default(), ver, api_ver.clone())?, - )? - .register_module("node", node::define_api(&Default::default(), ver, api_ver)?)?; - app.serve( - format!("0.0.0.0:{}", opt.api_port), - SequencerApiVersion::instance(), - ) - .await?; + let listener = TcpListener::bind(("0.0.0.0", opt.api_port)) + .await + .context("binding query service API port")?; + axum::serve(listener, api::router(ds)).await?; Ok(()) } diff --git a/node-metrics/Cargo.toml b/node-metrics/Cargo.toml index 3482b190d67..d0c19bb3510 100644 --- a/node-metrics/Cargo.toml +++ b/node-metrics/Cargo.toml @@ -13,6 +13,7 @@ alloy = { workspace = true } anyhow = { workspace = true } async-lock = { workspace = true } async-trait = { workspace = true } +axum = { workspace = true } bincode = { workspace = true } bitvec = { workspace = true } circular-buffer = { workspace = true } @@ -32,7 +33,6 @@ reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } surf-disco = { workspace = true } -tide-disco = { workspace = true } time = { workspace = true } tokio = { workspace = true } toml = { workspace = true } diff --git a/node-metrics/src/api/node_validator/v0/mod.rs b/node-metrics/src/api/node_validator/v0/mod.rs index cb7e4bd8608..171d47caafe 100644 --- a/node-metrics/src/api/node_validator/v0/mod.rs +++ b/node-metrics/src/api/node_validator/v0/mod.rs @@ -4,6 +4,16 @@ use std::{fmt, future::Future, pin::Pin, str::FromStr, time::Duration}; use alloy::primitives::Address; use anyhow::Context; +use axum::{ + Router, + extract::{ + State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + http::HeaderMap, + response::Response, + routing::get, +}; use espresso_types::{BackoffParams, SeqTypes, v0_3::AuthenticatedValidator}; use futures::{ FutureExt, Sink, SinkExt, Stream, StreamExt, @@ -18,11 +28,13 @@ use hotshot_query_service::{ use hotshot_types::{PeerConfig, signature_key::BLSPubKey}; use indexmap::IndexMap; use prometheus_parse::{Sample, Scrape}; -use serde::{Deserialize, Serialize}; -use tide_disco::{Api, api::ApiError, socket::Connection}; +use serde::Deserialize; use tokio::{spawn, task::JoinHandle, time::timeout}; use url::Url; -use vbs::version::{StaticVersion, StaticVersionType, Version}; +use vbs::{ + BinarySerializer, Serializer, + version::{StaticVersion, Version}, +}; use crate::service::{ client_message::{ClientMessage, InternalClientMessage}, @@ -50,247 +62,192 @@ pub type Version01 = StaticVersion; // Static instance of the Version01 type pub const STATIC_VER_0_1: Version01 = StaticVersion {}; -#[derive(Debug, Serialize, Deserialize)] -pub enum Error { - UnhandledTideDisco(tide_disco::StatusCode, String), - UnhandledSurfDisco(surf_disco::StatusCode, String), -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::UnhandledSurfDisco(status, msg) => { - write!(f, "Unhandled Surf Disco Error: {status} - {msg}") - }, - - Self::UnhandledTideDisco(status, msg) => { - write!(f, "Unhandled Tide Disco Error: {status} - {msg}") - }, - } - } -} - -impl std::error::Error for Error {} - -impl tide_disco::Error for Error { - fn catch_all(status: tide_disco::StatusCode, msg: String) -> Self { - Self::UnhandledTideDisco(status, msg) - } - - fn status(&self) -> tide_disco::StatusCode { - tide_disco::StatusCode::INTERNAL_SERVER_ERROR - } -} - -#[derive(Debug)] -pub enum LoadApiError { - Toml(toml::de::Error), - Api(ApiError), -} - -impl From for LoadApiError { - fn from(err: toml::de::Error) -> Self { - LoadApiError::Toml(err) - } -} - -impl From for LoadApiError { - fn from(err: ApiError) -> Self { - LoadApiError::Api(err) - } -} - -pub(crate) fn load_api( - default: &str, -) -> Result, LoadApiError> { - let toml: toml::Value = toml::from_str(default)?; - Ok(Api::new(toml)?) +/// [StateClientMessageSender] allows for the retrieval of a [Sender] for sending +/// messages received from the client to the Server for request processing. +pub trait StateClientMessageSender { + fn sender(&self) -> Sender>; } #[derive(Debug)] -pub enum LoadTomlError { - Io(std::io::Error), - Toml(toml::de::Error), - Utf8(std::str::Utf8Error), -} +pub enum EndpointError {} -impl From for LoadTomlError { - fn from(err: std::io::Error) -> Self { - LoadTomlError::Io(err) - } +fn wants_binary(headers: &HeaderMap) -> bool { + headers + .get(axum::http::header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("application/octet-stream")) } -impl From for LoadTomlError { - fn from(err: toml::de::Error) -> Self { - LoadTomlError::Toml(err) +/// Decode a message received from the `details` socket. Mirrors tide-disco's `Connection` stream +/// impl: binary frames are vbs, text frames are JSON, decoded according to the frame actually +/// received (not the negotiated response type). Returns `None` for anything that isn't a decoded +/// [`ClientMessage`]; the caller treats control frames (ping/pong/close) as ignorable and anything +/// else as a fatal decode error, closing the connection, matching tide-disco's behavior. +fn decode_client_message(message: &Message) -> Option { + match message { + Message::Text(text) => serde_json::from_str(text).ok(), + Message::Binary(bytes) => Serializer::::deserialize(bytes).ok(), + Message::Ping(_) | Message::Pong(_) | Message::Close(_) => None, } } -impl From for LoadTomlError { - fn from(err: std::str::Utf8Error) -> Self { - LoadTomlError::Utf8(err) +/// Encode a message to send on the `details` socket, negotiating VBS binary vs JSON text from the +/// `Accept` header of the upgrade request, matching tide-disco's content negotiation. The +/// dashboard client is a browser `WebSocket`, which cannot set an `Accept` header, so this +/// defaults to JSON (mirroring tide-disco's negotiation order, which also prefers JSON on a +/// wildcard `Accept`). +fn encode_server_message(message: &ServerMessage, binary: bool) -> Option { + if binary { + Serializer::::serialize(message) + .map(Message::binary) + .inspect_err(|err| tracing::error!("failed to serialize server message: {err}")) + .ok() + } else { + serde_json::to_string(message) + .map(Message::text) + .inspect_err(|err| tracing::error!("failed to serialize server message: {err}")) + .ok() } } -#[derive(Debug)] -pub enum DefineApiError { - LoadApiError(LoadApiError), - LoadTomlError(LoadTomlError), - ApiError(ApiError), -} - -impl From for DefineApiError { - fn from(err: LoadApiError) -> Self { - DefineApiError::LoadApiError(err) - } -} +/// Relays messages between the `details` socket and the internal client-message channel. Mirrors +/// tide-disco's socket handler: register with the server, wait for our assigned [`ClientId`], then +/// forward client requests and server responses until either side disconnects. +async fn handle_details_socket(socket: WebSocket, state: S, binary: bool) +where + S: StateClientMessageSender>, +{ + // Split into independent halves, matching tide-disco's `Connection`, which was `Clone` for + // exactly this reason: polling the next client message must not block sending server + // messages, and vice versa. + let (mut socket_sink, mut socket_stream) = socket.split(); -impl From for DefineApiError { - fn from(err: LoadTomlError) -> Self { - DefineApiError::LoadTomlError(err) - } -} + let mut internal_client_message_sender = state.sender(); + let (server_message_sender, mut server_message_receiver) = mpsc::channel(32); -impl From for DefineApiError { - fn from(err: ApiError) -> Self { - DefineApiError::ApiError(err) + // Let's register ourselves with the Server + if let Err(err) = internal_client_message_sender + .send(InternalClientMessage::Connected(server_message_sender)) + .await + { + // This means that the client_message_sender is closed + // we need to exit the stream. + tracing::info!( + "client message sender is closed before first message: {}", + err + ); + return; } -} -/// [StateClientMessageSender] allows for the retrieval of a [Sender] for sending -/// messages received from the client to the Server for request processing. -pub trait StateClientMessageSender { - fn sender(&self) -> Sender>; -} + // We should receive a response from the server that identifies us uniquely. + let client_id = + if let Some(ServerMessage::YouAre(client_id)) = server_message_receiver.next().await { + client_id + } else { + // The channel is closed, and this client should be removed we need to exit the stream + tracing::info!("server message receiver closed before first message"); + return; + }; -#[derive(Debug)] -pub enum EndpointError {} + // We want to start these futures outside of the loop. If we don't do this then every + // iteration will not be guaranteed to not skip a message. + let mut next_client_message = socket_stream.next(); + let mut next_server_message = server_message_receiver.next(); + + loop { + match futures::future::select(next_client_message, next_server_message).await { + Either::Left((client_frame, remaining_server_message)) => { + let Some(client_frame) = client_frame else { + // The client has disconnected, we need to exit the stream + tracing::info!("client message has disconnected"); + break; + }; -pub fn define_api() -> Result, DefineApiError> -where - State: StateClientMessageSender> + Send + Sync + 'static, -{ - let mut api = load_api::(include_str!("./node_validator.toml"))?; + let Ok(client_frame) = client_frame else { + // This indicates that there was a more specific error with the socket + // message. This error can be various, and may be recoverable depending on + // the actual nature of the error. We will treat it as unrecoverable for now. + break; + }; - api.with_version("0.0.1".parse().unwrap()).socket( - "details", - move |_req, socket: Connection, state| { - async move { - let mut socket_stream = socket.clone(); - let mut socket_sink = socket; + if matches!(client_frame, Message::Ping(_) | Message::Pong(_)) { + // Not a client request; keep waiting without disturbing `client_id`. + next_client_message = socket_stream.next(); + next_server_message = remaining_server_message; + continue; + } - let mut internal_client_message_sender = state.sender(); - let (server_message_sender, mut server_message_receiver) = mpsc::channel(32); + let Some(client_message) = decode_client_message(&client_frame) else { + break; + }; - // Let's register ourselves with the Server + let internal_client_message = client_message.to_internal_with_client_id(client_id); if let Err(err) = internal_client_message_sender - .send(InternalClientMessage::Connected(server_message_sender)) + .send(internal_client_message) .await { // This means that the client_message_sender is closed - // we need to exit the stream. - tracing::info!( - "client message sender is closed before first message: {}", - err - ); - return Ok(()); + tracing::info!("client message sender is closed: {}", err); + break; } - // We should receive a response from the server that identifies us - // uniquely. - let client_id = if let Some(ServerMessage::YouAre(client_id)) = - server_message_receiver.next().await - { - client_id - } else { - // The channel is closed, and this client should be removed - // we need to exit the stream - tracing::info!("server message receiver closed before first message",); - return Ok(()); + // let's queue up the next client message to receive + next_client_message = socket_stream.next(); + next_server_message = remaining_server_message; + }, + Either::Right((server_message, remaining_client_message)) => { + // Alright, we have a server message, we want to forward it to the down-stream + // client. + let Some(server_message) = server_message else { + // The server has disconnected, we need to exit the stream + break; }; - // We want to start these futures outside of the loop. If we - // don't do this then every iteration will not be guaranteed - // to not skip a message. - let mut next_client_message = socket_stream.next(); - let mut next_server_message = server_message_receiver.next(); - - loop { - match futures::future::select(next_client_message, next_server_message).await { - Either::Left((client_request, remaining_server_message)) => { - let client_request = if let Some(client_request) = client_request { - client_request - } else { - // The client has disconnected, we need to exit the stream - tracing::info!("client message has disconnected"); - break; - }; - - let client_request = if let Ok(client_request) = client_request { - client_request - } else { - // This indicates that there was a more - // specific error with the socket message. - // This error can be various, and may be - // recoverable depending on the actual nature - // of the error. We will treat it as - // unrecoverable for now. - break; - }; - - let internal_client_message = - client_request.to_internal_with_client_id(client_id); - if let Err(err) = internal_client_message_sender - .send(internal_client_message) - .await - { - // This means that the client_message_sender is closed - tracing::info!("client message sender is closed: {}", err); - break; - } - - // let's queue up the next client message to receive - next_client_message = socket_stream.next(); - next_server_message = remaining_server_message; - }, - Either::Right((server_message, remaining_client_message)) => { - // Alright, we have a server message, we want to forward it - // to the down-stream client. - - let server_message = if let Some(server_message) = server_message { - server_message - } else { - // The server has disconnected, we need to exit the stream - break; - }; - - // We want to forward the message to the client - if let Err(err) = socket_sink.send(&server_message).await { - // This means that the socket is closed - tracing::info!("socket is closed: {}", err); - break; - } - - // let's queue up the next server message to receive - next_server_message = server_message_receiver.next(); - next_client_message = remaining_client_message; - }, - } + let Some(frame) = encode_server_message(&server_message, binary) else { + break; + }; + // We want to forward the message to the client + if socket_sink.send(frame).await.is_err() { + // This means that the socket is closed + tracing::info!("socket is closed"); + break; } - // We don't actually care if this fails or not, as we're exiting - // this function anyway, and these Senders and Receivers will - // automatically be dropped. - _ = internal_client_message_sender - .send(InternalClientMessage::Disconnected(client_id)) - .await; + // let's queue up the next server message to receive + next_server_message = server_message_receiver.next(); + next_client_message = remaining_client_message; + }, + } + } + + // We don't actually care if this fails or not, as we're exiting this function anyway, and + // these Senders and Receivers will automatically be dropped. + _ = internal_client_message_sender + .send(InternalClientMessage::Disconnected(client_id)) + .await; +} - Ok(()) - } - .boxed() - }, - )?; - Ok(api) +async fn details_handler( + ws: WebSocketUpgrade, + headers: HeaderMap, + State(state): State, +) -> Response +where + S: StateClientMessageSender> + Send + 'static, +{ + let binary = wants_binary(&headers); + ws.on_upgrade(move |socket| handle_details_socket(socket, state, binary)) +} + +/// Builds the `node-validator` module's routes. tide-disco served `details` both directly and +/// under a major-version prefix (`v0/details`), redirecting the former to the latter; we serve +/// both forms directly instead, by nesting this router at both `/node-validator` and +/// `/v0/node-validator`. +pub fn router() -> Router +where + S: StateClientMessageSender> + Clone + Send + Sync + 'static, +{ + Router::new().route("/details", get(details_handler::)) } /// [get_config_stake_table_from_sequencer] retrieves the stake table from the @@ -423,9 +380,9 @@ pub async fn get_node_identity_from_url(url: url::Url) -> anyhow::Result>>, } @@ -215,16 +215,14 @@ pub async fn run_standalone_service(options: Options) { internal_client_message_sender, }; - let mut app: App<_, api::node_validator::v0::Error> = App::with_state(state); - let node_validator_api = - api::node_validator::v0::define_api().expect("error defining node validator api"); - - match app.register_module("node-validator", node_validator_api) { - Ok(_) => {}, - Err(err) => { - panic!("error registering node validator api: {err:?}"); - }, - } + // tide-disco served `details` both directly (e.g. `node-validator/details`) and under a + // major-version prefix (`v0/node-validator/details`), redirecting the former to the latter; + // we serve both forms directly instead. + let node_validator_api = api::node_validator::v0::router::(); + let app = Router::new() + .nest("/node-validator", node_validator_api.clone()) + .nest("/v0/node-validator", node_validator_api) + .with_state(state); let (leaf_and_block_pair_sender, leaf_and_block_pair_receiver) = mpsc::channel(10); @@ -293,7 +291,14 @@ pub async fn run_standalone_service(options: Options) { let port = options.port(); // We would like to wait until being signaled let app_serve_handle = spawn(async move { - let app_serve_result = app.serve(format!("0.0.0.0:{port}"), STATIC_VER_0_1).await; + let listener = match TcpListener::bind(("0.0.0.0", port)).await { + Ok(listener) => listener, + Err(err) => { + tracing::error!("failed to bind node-validator server: {err}"); + return; + }, + }; + let app_serve_result = axum::serve(listener, app).await; tracing::info!("app serve result: {:?}", app_serve_result); }); diff --git a/node-metrics/src/service/client_message/mod.rs b/node-metrics/src/service/client_message/mod.rs index e347aa3b3ec..e3910964771 100644 --- a/node-metrics/src/service/client_message/mod.rs +++ b/node-metrics/src/service/client_message/mod.rs @@ -25,11 +25,11 @@ pub enum ClientMessage { RequestStakeTableSnapshot, /// This allows the use-case of a user sending a message that is not a - /// valid recognized request to be handled explicitly by the server rather - /// than the underlying mechanism of tide-disco. - /// If not for this case [tide_disco] would silently ignore the error and - /// drop the client connection, resulting in the client receiving no - /// response / feedback about the mistake in the request. + /// valid recognized request to be handled explicitly by the server, + /// rather than by the underlying websocket handling mechanism. + /// If not for this case, an unrecognized message would fail to + /// deserialize and drop the client connection, resulting in the client + /// receiving no response / feedback about the mistake in the request. #[serde(untagged)] UnrecognizedCommand(serde_json::Value), }