Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ unicode-segmentation = "1.12.0"
secrecy = { version = "0.10", features = ["serde"] }
globset = { version = "0.4.18", default-features = false }
tower_governor = { version = "0.8.0", default-features = false, features = ["axum", "tracing"] }
http-body-util = "0.1.3"
http = "1.4.0"

[dependencies.serde]
version = "1"
Expand Down
13 changes: 13 additions & 0 deletions src/gha_logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,19 @@ pub async fn gha_logs(

let logs = match logs {
Ok(logs) => logs,
Err(err)
if err
.downcast_ref::<http_body_util::LengthLimitError>()
.is_some() =>
{
// Return a friendly error message for no logs too big.
tracing::info!("gha_logs: raw logs too big (over 50 mib) for {log_uuid}");
return Ok((
StatusCode::BAD_REQUEST,
HeaderMap::new(),
"The requested logs are too large (over 50 Mib).\n\nTry download the raw logs from GitHub instead.".to_string(),
));
}
Err(err) if matches!(err.downcast_ref::<reqwest::Error>(), Some(err) if err.status() == Some(StatusCode::GONE)) =>
{
// Return a friendly error message for no longer available logs.
Expand Down
70 changes: 58 additions & 12 deletions src/github/client.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
use anyhow::Context;
use async_trait::async_trait;
use futures::{FutureExt, future::BoxFuture};
use http_body_util::BodyExt;
use http_body_util::Limited;
use itertools::Itertools;
use reqwest::Body;
use reqwest::header::{AUTHORIZATION, USER_AGENT};
use reqwest::{Client, Request, RequestBuilder, Response, StatusCode};
use secrecy::{ExposeSecret, SecretString};
Expand Down Expand Up @@ -98,36 +101,79 @@ impl GithubClient {
}

pub async fn send_req(&self, req: RequestBuilder) -> anyhow::Result<(Bytes, String)> {
const MAX_DEFAULT_RESPONSE_SIZE: usize = 1 * 1024 * 1024; // 1 Mib

self.send_req_with_limit(req, MAX_DEFAULT_RESPONSE_SIZE)
.await
}

pub async fn send_req_with_limit(
&self,
req: RequestBuilder,
max_response_size: usize,
) -> anyhow::Result<(Bytes, String)> {
const MAX_ATTEMPTS: u32 = 2;

log::debug!("send_req with {:?}", req);

let req_dbg = format!("{req:?}");

let req = req
.build()
.with_context(|| format!("building reqwest {req_dbg}"))?;

let req_url = req.url().to_string();

let mut resp = self.client.execute(req.try_clone().unwrap()).await?;
if self.retry_rate_limit
&& let Some(sleep) = Self::needs_retry(&resp).await
{
resp = self.retry(req, sleep, MAX_ATTEMPTS).await?;
}

let maybe_err = resp.error_for_status_ref().err();
let github_request_id = resp.headers().get("x-github-request-id").cloned();
let body = resp
.bytes()
.await
.with_context(|| format!("failed to read response body {req_dbg}"))?;

let resp: http::Response<Body> = resp.into();
let limited = Limited::new(resp, max_response_size);

let body = match limited.collect().await {
Ok(body) => body.to_bytes(),
Err(e) => match e.downcast::<http_body_util::LengthLimitError>() {
Ok(e) => {
return Err(anyhow::Error::new(*e)).with_context(|| {
format!(
"req={req_url} (x-github-request-id: {}): lenght exceeded (over {max_response_size} bytes)",
github_request_id
.as_ref()
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown")
)
});
}
Err(e) => {
return Err(anyhow::Error::from_boxed(e)).with_context(|| {
format!(
"req={req_url} (x-github-request-id: {}): unable to complete the request",
github_request_id
.as_ref()
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown")
)
});
}
},
};

if let Some(e) = maybe_err {
return Err(anyhow::Error::new(e)).with_context(|| {
format!(
"response (x-github-request-id: {}): {}",
String::from_utf8_lossy(
github_request_id
.as_ref()
.map(|id| id.as_bytes())
.unwrap_or_default()
),
String::from_utf8_lossy(&body)
"req={req_url} (x-github-request-id: {}): {:.500}",
github_request_id
.as_ref()
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown"),
String::from_utf8_lossy(&body),
)
});
}
Expand Down
4 changes: 3 additions & 1 deletion src/github/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,11 @@ impl GithubClient {
repo: &IssueRepository,
job_id: u128,
) -> anyhow::Result<String> {
const MAX_LOG_SIZE_IN_MB: usize = 50 * 1024 * 1024; // 50 Mib

let url = format!("{}/actions/jobs/{job_id}/logs", repo.url(self));
let (body, _req_dbg) = self
.send_req(self.get(&url))
.send_req_with_limit(self.get(&url), MAX_LOG_SIZE_IN_MB)
.await
.context("failed to retrieve job logs")?;
Ok(String::from_utf8_lossy(&body).to_string())
Expand Down