Skip to content

feat: benchmark proxy payload scaling #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
franciscojavierarceo merged 2 commits into from
Jul 9, 2026
+121 −62
Merged

feat: benchmark proxy payload scaling #88

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9652179
feat: benchmark proxy payload scaling
harivilasp Jul 7, 2026
df1f463
clarify proxy benchmark prompt sizing
harivilasp Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 82 additions & 60 deletions crates/agentic-server/benches/proxy_bench.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
use std::convert::Infallible;
use std::sync::Arc;

use axum::body::Body;
use axum::extract::Request;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Router, serve};
use bytes::Bytes;
use criterion::{Criterion, criterion_group};
use criterion::{BenchmarkId, Criterion, criterion_group};
use futures::stream;
use http::StatusCode;
use tokio::net::TcpListener;
Expand All @@ -18,7 +19,9 @@ use agentic_core::executor::{ConversationHandler, ExecutionContext, ResponseHand
use agentic_core::proxy::ProxyState;
use agentic_core::storage::{ConversationStore, ResponseStore};
use agentic_server::app::{AppState, ServerConfig, build_router};
use std::sync::Arc;

const CONTENT_TYPE_JSON: &str = "application/json";
const PAYLOAD_SIZES: [usize; 3] = [1024, 10 * 1024, 100 * 1024];
Comment thread
harivilasp marked this conversation as resolved.
Outdated

fn bench_config(llm_url: &str) -> Config {
Config {
Expand Down Expand Up @@ -58,7 +61,7 @@ async fn responses_handler(req: Request) -> Response {
}

let out = r#"{"id":"resp_bench","object":"response","status":"completed"}"#;
(StatusCode::OK, [("content-type", "application/json")], out).into_response()
(StatusCode::OK, [("content-type", CONTENT_TYPE_JSON)], out).into_response()
Comment thread
harivilasp marked this conversation as resolved.
}

async fn spawn_llm() -> String {
Expand Down Expand Up @@ -102,91 +105,110 @@ async fn spawn_gateway(config: Config) -> String {
format!("http://{addr}")
}

fn proxy_benchmarks(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
fn responses_url(base_url: &str) -> String {
format!("{base_url}/v1/responses")
}

let (llm_url, gateway_url) = rt.block_on(async {
let llm_url = spawn_llm().await;
let config = bench_config(&llm_url);
let gateway_url = spawn_gateway(config).await;
(llm_url, gateway_url)
fn request_body(prompt_bytes: usize, stream: bool) -> Bytes {
let prompt = "x".repeat(prompt_bytes);
let body = serde_json::json!({
"model": "bench-model",
"input": [{"role": "user", "content": prompt}],
"store": false,
"stream": stream
});
Bytes::from(serde_json::to_vec(&body).expect("benchmark request body serializes"))
}

let client = reqwest::Client::new();
async fn post_response(client: reqwest::Client, url: String, body: Bytes) -> usize {
let resp = client
.post(url)
.header("content-type", CONTENT_TYPE_JSON)
.body(body)
.send()
.await
.expect("benchmark request succeeds");

let non_stream_body = serde_json::json!({
"model": "bench-model",
"input": [{"role": "user", "content": "hello"}]
});
let stream_body = serde_json::json!({
"model": "bench-model",
"input": [{"role": "user", "content": "hello"}],
"stream": true
});
resp.bytes().await.expect("benchmark response body").len()
}

fn bench_single_request(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) {
let non_stream_body = request_body(5, false);
let stream_body = request_body(5, true);

let mut group = c.benchmark_group("non_stream");

group.bench_function("direct", |b| {
let url = format!("{llm_url}/v1/responses");
let url = responses_url(llm_url);
let body = non_stream_body.clone();
b.to_async(&rt).iter(|| {
let client = client.clone();
let url = url.clone();
let body = body.clone();
async move {
let resp = client.post(&url).json(&body).send().await.unwrap();
resp.bytes().await.unwrap()
}
});
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});

group.bench_function("proxied", |b| {
let url = format!("{gateway_url}/v1/responses");
let url = responses_url(gateway_url);
let body = non_stream_body.clone();
b.to_async(&rt).iter(|| {
let client = client.clone();
let url = url.clone();
let body = body.clone();
async move {
let resp = client.post(&url).json(&body).send().await.unwrap();
resp.bytes().await.unwrap()
}
});
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});

group.finish();

let mut group = c.benchmark_group("stream");

group.bench_function("direct", |b| {
let url = format!("{llm_url}/v1/responses");
let url = responses_url(llm_url);
let body = stream_body.clone();
b.to_async(&rt).iter(|| {
let client = client.clone();
let url = url.clone();
let body = body.clone();
async move {
let resp = client.post(&url).json(&body).send().await.unwrap();
resp.bytes().await.unwrap()
}
});
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});

group.bench_function("proxied", |b| {
let url = format!("{gateway_url}/v1/responses");
let url = responses_url(gateway_url);
let body = stream_body.clone();
b.to_async(&rt).iter(|| {
let client = client.clone();
let url = url.clone();
let body = body.clone();
async move {
let resp = client.post(&url).json(&body).send().await.unwrap();
resp.bytes().await.unwrap()
}
});
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});

group.finish();
}

fn bench_body_size(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) {
let mut group = c.benchmark_group("non_stream/body_size");

for size in PAYLOAD_SIZES {
group.bench_with_input(BenchmarkId::new("direct", size), &size, |b, &size| {
let url = responses_url(llm_url);
let body = request_body(size, false);
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});

group.bench_with_input(BenchmarkId::new("proxied", size), &size, |b, &size| {
Comment thread
harivilasp marked this conversation as resolved.
Outdated
let url = responses_url(gateway_url);
let body = request_body(size, false);
b.to_async(rt)
.iter(|| post_response(client.clone(), url.clone(), body.clone()));
});
}

group.finish();
}

fn proxy_benchmarks(c: &mut Criterion) {
let rt = Runtime::new().unwrap();

let (llm_url, gateway_url) = rt.block_on(async {
let llm_url = spawn_llm().await;
let config = bench_config(&llm_url);
let gateway_url = spawn_gateway(config).await;
(llm_url, gateway_url)
});

let client = reqwest::Client::new();

bench_single_request(c, &rt, &client, &llm_url, &gateway_url);
bench_body_size(c, &rt, &client, &llm_url, &gateway_url);
}

criterion_group!(proxy_benches, proxy_benchmarks);
32 changes: 30 additions & 2 deletions crates/agentic-server/tests/responses_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
mod common;

use std::sync::Arc;

use axum::Router;
use axum::body::Bytes;
use axum::response::IntoResponse;
use axum::routing::post;
use http::StatusCode;
use std::sync::Arc;
use tokio::net::TcpListener;
use tokio::sync::Mutex;

Expand Down Expand Up @@ -133,6 +132,35 @@ async fn test_store_false_with_web_search_reaches_executor() {
assert_eq!(requests[0]["tools"][0]["name"], "web_search");
}

#[tokio::test]
async fn test_store_false_proxies_large_json_body_to_vllm() {
// Arrange
let (llm_url, requests, _h1) = spawn_mock_vllm_json_capture().await;
let (gw_url, _h2) = spawn_gateway(test_state(&test_config(&llm_url))).await;
let prompt = "x".repeat(100 * 1024);

// Act
let resp = reqwest::Client::new()
.post(format!("{gw_url}/v1/responses"))
.json(&serde_json::json!({
"model": "test",
"input": [{"type": "message", "role": "user", "content": prompt}],
"store": false,
"stream": false
}))
.send()
.await
.unwrap();

// Assert — the gateway keeps this below-limit request on the proxy path.
assert_eq!(resp.status(), 200);
let requests = requests.lock().await;
assert_eq!(requests.len(), 1);
assert_eq!(requests[0]["store"], false);
assert_eq!(requests[0]["stream"], false);
assert_eq!(requests[0]["input"][0]["content"].as_str().unwrap().len(), 100 * 1024);
}

#[tokio::test]
async fn test_store_false_proxies_sse_to_vllm() {
// Arrange
Expand Down