diff --git a/Cargo.lock b/Cargo.lock index fb288e5c..01b0eb28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,9 +389,8 @@ dependencies = [ [[package]] name = "clubcard-crlite" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fefdd645e5cb901cf302e9a581a63ae96dab748cf209168e87da35d980dcafa" +version = "0.3.3" +source = "git+https://github.com/mozilla/clubcard-crlite?rev=83d4afe61d511fe88e8bf22e8a71c3b582536e69#83d4afe61d511fe88e8bf22e8a71c3b582536e69" dependencies = [ "base64 0.21.7", "bincode", diff --git a/Cargo.toml b/Cargo.toml index 4c03e85f..90a9f573 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ aws-lc-rs = "1.15.2" base64 = "0.22.1" chrono = { version = "0.4.42", features = ["alloc"], default-features = false } clap = { version = "4.5", features = ["derive"] } -clubcard-crlite = "0.3.2" +clubcard-crlite = "0.3.3" criterion = "0.8" directories = "6" eyre = "0.6" @@ -62,3 +62,6 @@ unreachable_pub = "warn" unused_extern_crates = "warn" unused_import_braces = "warn" unused_qualifications = "warn" + +[patch.crates-io] +clubcard-crlite = { git = "https://github.com/mozilla/clubcard-crlite", rev = "83d4afe61d511fe88e8bf22e8a71c3b582536e69" } diff --git a/revoke-test/benches/bench.rs b/revoke-test/benches/bench.rs index 896b6961..9e3a67ad 100644 --- a/revoke-test/benches/bench.rs +++ b/revoke-test/benches/bench.rs @@ -10,7 +10,7 @@ use codspeed_criterion_compat::{Criterion, criterion_group, criterion_main}; use criterion::{Criterion, criterion_group, criterion_main}; use revoke_test::RevocationTestSites; use rustls_pki_types::CertificateDer; -use upki::revocation::{Manifest, RevocationCheckInput, RevocationStatus}; +use upki::revocation::{Index, Manifest, RevocationCheckInput, RevocationStatus}; use upki::{Config, ConfigPath}; fn revocation(c: &mut Criterion) { @@ -44,13 +44,27 @@ fn revocation(c: &mut Criterion) { c.bench_function("revocation-check", |b| { let config = Config::from_file_or_default(&ConfigPath::new(None).unwrap()).unwrap(); + let index = Index::from_cache(&config).unwrap(); let revoked_certs = certificates_for_test_site(BENCHMARK_CASE); b.iter(|| { - let manifest = Manifest::from_config(&config).unwrap(); let input = RevocationCheckInput::from_certificates(&revoked_certs).unwrap(); assert_eq!( - manifest.check(&input, &config).unwrap(), + index.check(&input).unwrap(), + RevocationStatus::CertainlyRevoked + ); + }) + }); + + c.bench_function("e2e-revocation-check", |b| { + let config = Config::from_file_or_default(&ConfigPath::new(None).unwrap()).unwrap(); + let revoked_certs = certificates_for_test_site(BENCHMARK_CASE); + + b.iter(|| { + let index = Index::from_cache(&config).unwrap(); + let input = RevocationCheckInput::from_certificates(&revoked_certs).unwrap(); + assert_eq!( + index.check(&input).unwrap(), RevocationStatus::CertainlyRevoked ); }) diff --git a/rustls-upki/src/lib.rs b/rustls-upki/src/lib.rs index 75df3220..a5c48a3a 100644 --- a/rustls-upki/src/lib.rs +++ b/rustls-upki/src/lib.rs @@ -14,7 +14,8 @@ use rustls::{ ExtendedKeyPurpose, RootCertStore, SignatureScheme, SupportedCipherSuite, }; use upki::revocation::{ - CertSerial, CtTimestamp, IssuerSpkiHash, Manifest, RevocationCheckInput, RevocationStatus, + CertSerial, CtTimestamp, Index, IssuerSpkiHash, Manifest, RevocationCheckInput, + RevocationStatus, }; use upki::{self, Config, ConfigPath}; use webpki::{EndEntityCert, ExtendedKeyUsage, InvalidNameContext, VerifiedPath}; @@ -128,9 +129,7 @@ impl ServerVerifier { sct_timestamps, }; - match Manifest::from_config(&self.config) - .and_then(|manifest| manifest.check(&input, &self.config)) - { + match Index::from_cache(&self.config).and_then(|index| index.check(&input)) { Ok(rs) => Ok(rs), Err(e) => Err(rustls::Error::General(e.to_string())), } diff --git a/upki-ffi/src/lib.rs b/upki-ffi/src/lib.rs index 2dcfe428..6965b13f 100644 --- a/upki-ffi/src/lib.rs +++ b/upki-ffi/src/lib.rs @@ -8,7 +8,7 @@ use std::path::Path; use std::slice; use rustls_pki_types::CertificateDer; -use upki::revocation::{self, Manifest, RevocationCheckInput, RevocationStatus}; +use upki::revocation::{self, Index, RevocationCheckInput, RevocationStatus}; use upki::{Config, Error}; /// Check the revocation status of a certificate. @@ -47,12 +47,12 @@ pub unsafe extern "C" fn upki_check_revocation( Err(err) => return Error::Revocation(err).into(), }; - let manifest = match Manifest::from_config(config) { - Ok(manifest) => manifest, + let index = match Index::from_cache(config) { + Ok(index) => index, Err(err) => return Error::Revocation(err).into(), }; - match manifest.check(&input, config) { + match index.check(&input) { Ok(status) => match status { RevocationStatus::NotCoveredByRevocationData => { upki_result::UPKI_REVOCATION_NOT_COVERED diff --git a/upki/src/main.rs b/upki/src/main.rs index bdb7db44..b2e66342 100644 --- a/upki/src/main.rs +++ b/upki/src/main.rs @@ -8,7 +8,7 @@ use clap::{Parser, Subcommand}; use eyre::{Context, Report}; use rustls_pki_types::CertificateDer; use rustls_pki_types::pem::PemObject; -use upki::revocation::{Manifest, RevocationCheckInput, fetch}; +use upki::revocation::{Index, Manifest, RevocationCheckInput, fetch}; use upki::{Config, ConfigPath}; #[tokio::main(flavor = "current_thread")] @@ -51,8 +51,8 @@ async fn main() -> Result { } let input = RevocationCheckInput::from_certificates(&certs)?; - Manifest::from_config(&config)? - .check(&input, &config)? + Index::from_cache(&config)? + .check(&input)? .to_cli() } }) diff --git a/upki/src/revocation/fetch.rs b/upki/src/revocation/fetch.rs index 0d5df543..d90554dc 100644 --- a/upki/src/revocation/fetch.rs +++ b/upki/src/revocation/fetch.rs @@ -11,7 +11,7 @@ use core::time::Duration; use std::collections::HashSet; use std::env; use std::fs::{self, File, Permissions}; -use std::io::{self, Read}; +use std::io::{self, Read, Write}; #[cfg(target_family = "unix")] use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; @@ -20,7 +20,8 @@ use std::process::ExitCode; use aws_lc_rs::digest; use tracing::{debug, info}; -use super::{Error, Filter, Manifest}; +use super::index::INDEX_BIN; +use super::{Error, Filter, Index, Manifest}; use crate::Config; /// Update the local revocation cache by fetching updates over the network. @@ -157,6 +158,11 @@ impl Plan { steps.push(PlanStep::download(filter, remote_url, local)); } + steps.push(PlanStep::SaveIndex { + manifest: manifest.clone(), + local_dir: local.to_owned(), + }); + steps.push(PlanStep::SaveManifest { manifest: manifest.clone(), local_dir: local.to_owned(), @@ -197,6 +203,12 @@ enum PlanStep { /// Delete the given single local file. Delete(PathBuf), + /// Build and save the index from filter universe metadata. + SaveIndex { + manifest: Manifest, + local_dir: PathBuf, + }, + /// Save the manifest structure SaveManifest { manifest: Manifest, @@ -266,6 +278,46 @@ impl PlanStep { path: target, })?; } + Self::SaveIndex { + manifest, + local_dir, + } => { + debug!("building index"); + let Some(buf) = Index::write(&manifest, &local_dir) else { + return Ok(()); + }; + + #[cfg(target_family = "unix")] + let temp = tempfile::Builder::new() + .permissions(Permissions::from_mode(0o644)) + .suffix(".new") + .tempfile_in(&local_dir); + #[cfg(not(target_family = "unix"))] + let temp = tempfile::Builder::new() + .suffix(".new") + .tempfile_in(&local_dir); + + let mut local_temp = temp.map_err(|error| Error::FileWrite { + error, + path: local_dir.clone(), + })?; + + local_temp + .as_file_mut() + .write_all(&buf) + .map_err(|error| Error::FileWrite { + error, + path: local_temp.path().to_owned(), + })?; + + let path = local_dir.join(INDEX_BIN); + local_temp + .persist(&path) + .map_err(|error| Error::FileWrite { + error: error.error, + path, + })?; + } Self::SaveManifest { manifest, local_dir, @@ -329,6 +381,9 @@ impl fmt::Display for PlanStep { filter.size ), Self::Delete(path) => write!(f, "delete stale file {path:?}"), + Self::SaveIndex { local_dir, .. } => { + write!(f, "build index from filters into {local_dir:?}") + } Self::SaveManifest { local_dir, .. } => { write!(f, "save new manifest into {local_dir:?}") } diff --git a/upki/src/revocation/index.rs b/upki/src/revocation/index.rs new file mode 100644 index 00000000..aeb75bb6 --- /dev/null +++ b/upki/src/revocation/index.rs @@ -0,0 +1,544 @@ +use core::{fmt, str}; +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{BufReader, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use clubcard_crlite::{CRLiteClubcard, CRLiteStatus}; + +use super::{Error, Manifest, RevocationCheckInput, RevocationStatus}; +use crate::Config; + +/// Binary-encoded index of universe metadata for all filters in a manifest. +/// +/// This allows the check path to identify which filter file covers a given certificate +/// without loading every filter. Written atomically during fetch, this is the single +/// source of truth for revocation checks. +/// +/// # Encoding +/// +/// All integers are big-endian. +/// +/// ```text +/// HEADER (read by from_cache): +/// magic: [u8; 8] "upkiidx0" +/// num_filenames: u8 +/// Per filename: +/// filename_len: u16 +/// filename: [u8; filename_len] UTF-8 +/// num_log_ids: u32 +/// Per log_id (sorted lexicographically): +/// log_id: [u8; 32] +/// offset: u64 byte offset from file start +/// num_entries: u16 +/// +/// ENTRY SECTIONS (read by check via seek): +/// Per entry: +/// filter_index: u8 +/// min_timestamp: u64 +/// max_timestamp: u64 +/// ``` +pub struct Index { + cache_dir: PathBuf, + index_path: PathBuf, + data: Vec, + num_filenames: u8, + num_log_ids: u32, + log_dir_offset: usize, +} + +const LOG_DIR_ENTRY_SIZE: usize = 32 + 8 + 2; +const ENTRY_SIZE: usize = 1 + 8 + 8; + +impl Index { + /// Read the index header from the cache directory specified in `config`. + /// + /// Only the header (filename table and log-ID directory) is loaded into memory. + /// Entry sections are read on demand during [`check`](Self::check) via seeking. + pub fn from_cache(config: &Config) -> Result { + let cache_dir = config.revocation_cache_dir(); + let index_path = cache_dir.join(INDEX_BIN); + let file = File::open(&index_path).map_err(|error| Error::FilterRead { + error, + path: Some(index_path.clone()), + })?; + let mut reader = BufReader::new(file); + + let mut magic = [0u8; 8]; + reader + .read_exact(&mut magic) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + if magic != *INDEX_MAGIC { + return Err(Error::IndexDecode("invalid index magic".into())); + } + + let num_filenames = u8::read_from(&mut reader)?; + + let mut data = Vec::new(); + for _ in 0..num_filenames { + let len = u16::read_from(&mut reader)?; + data.extend_from_slice(&len.to_be_bytes()); + let start = data.len(); + data.resize(start + len as usize, 0); + reader + .read_exact(&mut data[start..]) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + } + + let log_dir_offset = data.len(); + let num_log_ids = u32::read_from(&mut reader)?; + + let dir_bytes = num_log_ids as usize * LOG_DIR_ENTRY_SIZE; + let start = data.len(); + data.resize(start + dir_bytes, 0); + reader + .read_exact(&mut data[start..]) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + + Ok(Self { + cache_dir, + index_path, + data, + num_filenames, + num_log_ids, + log_dir_offset, + }) + } + + /// Build index bytes by reading filter files from `dir` and extracting universe metadata. + /// + /// Returns `None` if any filter file cannot be read or decoded. + pub(super) fn write(manifest: &Manifest, dir: &Path) -> Option> { + let mut by_log_id: BTreeMap<[u8; 32], Vec<(u8, u64, u64)>> = BTreeMap::new(); + + for (filter_idx, filter) in manifest.filters.iter().enumerate() { + let path = dir.join(&filter.filename); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) => { + tracing::warn!("skipping index: cannot read {path:?}: {error}"); + return None; + } + }; + + let clubcard = match CRLiteClubcard::from_bytes(&bytes) { + Ok(c) => c, + Err(error) => { + tracing::warn!("skipping index: cannot decode {path:?}: {error:?}"); + return None; + } + }; + + for (log_id, (min_ts, max_ts)) in clubcard.universe().iter() { + by_log_id + .entry(*log_id) + .or_default() + .push((filter_idx as u8, *min_ts, *max_ts)); + } + } + + // Compute header size to determine entry section offsets + let mut header_size: usize = 8 + 1; // magic + num_filenames + for filter in &manifest.filters { + header_size += 2 + filter.filename.len(); // filename_len + filename + } + header_size += 4; // num_log_ids + header_size += by_log_id.len() * LOG_DIR_ENTRY_SIZE; + + // Write header + let mut buf = Vec::new(); + buf.extend_from_slice(INDEX_MAGIC); + buf.push(manifest.filters.len() as u8); + for filter in &manifest.filters { + let filename = filter.filename.as_bytes(); + buf.extend_from_slice(&(filename.len() as u16).to_be_bytes()); + buf.extend_from_slice(filename); + } + + buf.extend_from_slice(&(by_log_id.len() as u32).to_be_bytes()); + + // Pre-compute offsets for each log_id's entry section + let mut current_offset = header_size; + let mut dir_entries: Vec<(&[u8; 32], u64, u16)> = Vec::new(); + for (log_id, entries) in &by_log_id { + dir_entries.push((log_id, current_offset as u64, entries.len() as u16)); + current_offset += entries.len() * ENTRY_SIZE; + } + + // Write log_id directory + for (log_id, offset, count) in &dir_entries { + buf.extend_from_slice(*log_id); + buf.extend_from_slice(&offset.to_be_bytes()); + buf.extend_from_slice(&count.to_be_bytes()); + } + + // Write entry sections + for entries in by_log_id.values() { + for (filter_idx, min_ts, max_ts) in entries { + buf.push(*filter_idx); + buf.extend_from_slice(&min_ts.to_be_bytes()); + buf.extend_from_slice(&max_ts.to_be_bytes()); + } + } + + Some(buf) + } + + /// Perform a revocation check using the index. + /// + /// Uses binary search over the log-ID directory to find relevant entries, + /// then seeks into the index file to read only those entries. Finally loads + /// the matching filter file and checks the certificate against it. + pub fn check(&self, input: &RevocationCheckInput) -> Result { + let key = input.key(); + let dir_data = &self.data[self.log_dir_offset..]; + + let covering_filter = 'outer: { + for sct in &input.sct_timestamps { + // Binary search the sorted log_id directory (stride LOG_DIR_ENTRY_SIZE) + let found = binary_search_log_dir(dir_data, self.num_log_ids, &sct.log_id); + let Some(entry_offset) = found else { + continue; + }; + + let mut entry_data = &dir_data[entry_offset + 32..]; + let offset = u64::read_be(&mut entry_data)?; + let count = u16::read_be(&mut entry_data)?; + + // Seek into the index file to read entries for this log_id + let mut file = + File::open(&self.index_path).map_err(|error| Error::FilterRead { + error, + path: Some(self.index_path.clone()), + })?; + + file.seek(SeekFrom::Start(offset)) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + + let mut buf = vec![0u8; count as usize * ENTRY_SIZE]; + file.read_exact(&mut buf) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + + let mut data = &buf[..]; + for _ in 0..count { + let filter_index = u8::read_be(&mut data)? as usize; + let min_ts = u64::read_be(&mut data)?; + let max_ts = u64::read_be(&mut data)?; + if min_ts <= sct.timestamp && sct.timestamp <= max_ts { + break 'outer Some(filter_index); + } + } + } + None + }; + + let Some(filter_index) = covering_filter else { + return Ok(RevocationStatus::NotCoveredByRevocationData); + }; + + let filename = self.filename(filter_index)?; + let path = self.cache_dir.join(filename); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) => { + return Err(Error::FilterRead { + error, + path: Some(path), + }); + } + }; + + let filter = CRLiteClubcard::from_bytes(&bytes).map_err(|error| Error::FilterDecode { + error: format!("cannot decode crlite filter: {error:?}").into(), + path, + })?; + + Ok( + match filter.contains( + &key, + input + .sct_timestamps + .iter() + .map(|ct_ts| (&ct_ts.log_id, ct_ts.timestamp)), + ) { + CRLiteStatus::Revoked => RevocationStatus::CertainlyRevoked, + CRLiteStatus::Good => RevocationStatus::NotRevoked, + CRLiteStatus::NotEnrolled | CRLiteStatus::NotCovered => { + RevocationStatus::NotCoveredByRevocationData + } + }, + ) + } + + /// Look up the `i`-th filename in the filename table by scanning raw bytes. + fn filename(&self, index: usize) -> Result<&str, Error> { + if index >= self.num_filenames as usize { + return Err(Error::IndexDecode("filter index out of bounds".into())); + } + + let mut data = &self.data[..self.log_dir_offset]; + for _ in 0..index { + let len = u16::read_be(&mut data)? as usize; + let _ = try_split_at(&mut data, len)?; + } + + let len = u16::read_be(&mut data)? as usize; + let name = try_split_at(&mut data, len)?; + str::from_utf8(name).map_err(|e| Error::IndexDecode(Box::new(e))) + } +} + +impl fmt::Debug for Index { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Index") + .field("cache_dir", &self.cache_dir) + .field("filenames", &self.num_filenames) + .field("log_ids", &self.num_log_ids) + .finish_non_exhaustive() + } +} + +/// Binary search the log-ID directory for `target`. +/// +/// Returns the byte offset within `dir_data` of the matching entry, or `None`. +fn binary_search_log_dir(dir_data: &[u8], count: u32, target: &[u8; 32]) -> Option { + let mut lo = 0u32; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let offset = mid as usize * LOG_DIR_ENTRY_SIZE; + let log_id = &dir_data[offset..offset + 32]; + match log_id.cmp(target.as_slice()) { + core::cmp::Ordering::Less => lo = mid + 1, + core::cmp::Ordering::Equal => return Some(offset), + core::cmp::Ordering::Greater => hi = mid, + } + } + None +} + +fn try_split_at<'a>(data: &mut &'a [u8], mid: usize) -> Result<&'a [u8], Error> { + match data.split_at_checked(mid) { + Some((left, right)) => { + *data = right; + Ok(left) + } + None => Err(Error::IndexDecode("unexpected end of index data".into())), + } +} + +impl FromBeBytes<8> for u64 { + fn from_be_bytes(bytes: &[u8; 8]) -> Self { + Self::from_be_bytes(*bytes) + } +} + +impl FromBeBytes<4> for u32 { + fn from_be_bytes(bytes: &[u8; 4]) -> Self { + Self::from_be_bytes(*bytes) + } +} + +impl FromBeBytes<2> for u16 { + fn from_be_bytes(bytes: &[u8; 2]) -> Self { + Self::from_be_bytes(*bytes) + } +} + +impl FromBeBytes<1> for u8 { + fn from_be_bytes(bytes: &[u8; 1]) -> Self { + bytes[0] + } +} + +trait FromBeBytes: Sized { + fn read_be(data: &mut &[u8]) -> Result { + match data.split_first_chunk::() { + Some((chunk, rest)) => { + *data = rest; + Ok(Self::from_be_bytes(chunk)) + } + None => Err(Error::IndexDecode("unexpected end of index data".into())), + } + } + + fn read_from(reader: &mut impl Read) -> Result { + let mut buf = [0u8; N]; + reader + .read_exact(&mut buf) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + Ok(Self::from_be_bytes(&buf)) + } + + fn from_be_bytes(bytes: &[u8; N]) -> Self; +} + +pub(super) const INDEX_BIN: &str = "index.bin"; +const INDEX_MAGIC: &[u8; 8] = b"upkiidx0"; + +#[cfg(test)] +mod tests { + use super::*; + use crate::revocation::{CertSerial, CtTimestamp, IssuerSpkiHash, RevocationConfig}; + + #[test] + fn check_empty_index() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + write_index(dir.path(), &build_index(&[])); + assert_eq!( + Index::from_cache(&config) + .unwrap() + .check(&test_input()) + .unwrap(), + RevocationStatus::NotCoveredByRevocationData, + ); + } + + #[test] + fn check_no_matching_log_id() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + // Input has log_id [0xbb; 32], index has [0xcc; 32] + let data = build_index(&[("test.filter", &[([0xcc; 32], 500, 1500)])]); + write_index(dir.path(), &data); + assert_eq!( + Index::from_cache(&config) + .unwrap() + .check(&test_input()) + .unwrap(), + RevocationStatus::NotCoveredByRevocationData, + ); + } + + #[test] + fn check_no_matching_timestamp_range() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + // Input has timestamp 1000, index range is 2000..3000 + let data = build_index(&[("test.filter", &[([0xbb; 32], 2000, 3000)])]); + write_index(dir.path(), &data); + assert_eq!( + Index::from_cache(&config) + .unwrap() + .check(&test_input()) + .unwrap(), + RevocationStatus::NotCoveredByRevocationData, + ); + } + + #[test] + fn invalid_magic() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + write_index(dir.path(), b"wrongmag\x00\x00\x00\x00"); + let err = Index::from_cache(&config).unwrap_err(); + assert!(matches!(err, Error::IndexDecode(_))); + } + + #[test] + fn truncated_after_magic() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + write_index(dir.path(), INDEX_MAGIC); + let err = Index::from_cache(&config).unwrap_err(); + assert!(matches!(err, Error::IndexDecode(_))); + } + + #[test] + fn truncated_before_magic() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + write_index(dir.path(), b"upki"); + let err = Index::from_cache(&config).unwrap_err(); + assert!(matches!(err, Error::IndexDecode(_))); + } + + #[test] + fn missing_index() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + let err = Index::from_cache(&config).unwrap_err(); + assert!(matches!(err, Error::FilterRead { .. })); + } + + fn test_config(dir: &Path) -> Config { + Config { + cache_dir: dir.to_owned(), + revocation: RevocationConfig::default(), + } + } + + fn test_input() -> RevocationCheckInput { + RevocationCheckInput { + cert_serial: CertSerial(vec![1, 2, 3]), + issuer_spki_hash: IssuerSpkiHash([0xaa; 32]), + sct_timestamps: vec![CtTimestamp { + log_id: [0xbb; 32], + timestamp: 1000, + }], + } + } + + #[expect(clippy::type_complexity)] + fn build_index(filters: &[(&str, &[([u8; 32], u64, u64)])]) -> Vec { + // Aggregate by log_id using BTreeMap for sorted order + let mut by_log_id: BTreeMap<[u8; 32], Vec<(u8, u64, u64)>> = BTreeMap::new(); + for (filter_idx, (_, entries)) in filters.iter().enumerate() { + for (log_id, min_ts, max_ts) in *entries { + by_log_id + .entry(*log_id) + .or_default() + .push((filter_idx as u8, *min_ts, *max_ts)); + } + } + + // Compute header size + let mut header_size: usize = 8 + 1; // magic + num_filenames + for (filename, _) in filters { + header_size += 2 + filename.len(); + } + header_size += 4; // num_log_ids + header_size += by_log_id.len() * LOG_DIR_ENTRY_SIZE; + + // Write header + let mut buf = Vec::new(); + buf.extend_from_slice(INDEX_MAGIC); + buf.push(filters.len() as u8); + for (filename, _) in filters { + let bytes = filename.as_bytes(); + buf.extend_from_slice(&(bytes.len() as u16).to_be_bytes()); + buf.extend_from_slice(bytes); + } + + buf.extend_from_slice(&(by_log_id.len() as u32).to_be_bytes()); + + // Compute offsets and write directory + let mut current_offset = header_size; + let mut entry_counts: Vec = Vec::new(); + for (log_id, entries) in &by_log_id { + buf.extend_from_slice(log_id); + buf.extend_from_slice(&(current_offset as u64).to_be_bytes()); + buf.extend_from_slice(&(entries.len() as u16).to_be_bytes()); + current_offset += entries.len() * ENTRY_SIZE; + entry_counts.push(entries.len()); + } + + // Write entry sections + for entries in by_log_id.values() { + for (filter_idx, min_ts, max_ts) in entries { + buf.push(*filter_idx); + buf.extend_from_slice(&min_ts.to_be_bytes()); + buf.extend_from_slice(&max_ts.to_be_bytes()); + } + } + + buf + } + + fn write_index(dir: &Path, data: &[u8]) { + let revocation_dir = dir.join("revocation"); + fs::create_dir_all(&revocation_dir).unwrap(); + fs::write(revocation_dir.join(INDEX_BIN), data).unwrap(); + } +} diff --git a/upki/src/revocation/mod.rs b/upki/src/revocation/mod.rs index 8794014e..0641da3e 100644 --- a/upki/src/revocation/mod.rs +++ b/upki/src/revocation/mod.rs @@ -1,7 +1,7 @@ use core::error::Error as StdError; -use core::fmt; use core::str::FromStr; -use std::fs::{self, File}; +use core::{fmt, str}; +use std::fs::File; use std::io::{self, BufReader}; use std::path::PathBuf; use std::process::ExitCode; @@ -10,7 +10,7 @@ use aws_lc_rs::digest; use base64::Engine; use base64::prelude::BASE64_STANDARD; use chrono::{DateTime, Utc}; -use clubcard_crlite::{CRLiteClubcard, CRLiteKey, CRLiteStatus}; +use clubcard_crlite::CRLiteKey; use rustls_pki_types::{CertificateDer, TrustAnchor}; use serde::{Deserialize, Serialize}; use tracing::info; @@ -21,6 +21,9 @@ mod fetch; use fetch::Plan; pub use fetch::fetch; +mod index; +pub use index::Index; + /// The structure contained in a manifest.json #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Manifest { @@ -58,54 +61,6 @@ impl Manifest { }) } - /// This function does a low-level revocation check. - /// - /// It is assumed the caller has already done a path verification, and now wants to - /// check the revocation status of the end-entity certificate. - /// - /// On success, this returns a [`RevocationStatus`] saying whether the certificate - /// is revoked, not revoked, or whether the data set cannot make that determination. - pub fn check( - &self, - input: &RevocationCheckInput, - config: &Config, - ) -> Result { - let key = input.key(); - let cache_dir = config.revocation_cache_dir(); - for f in &self.filters { - let path = cache_dir.join(&f.filename); - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(error) => { - return Err(Error::FilterRead { - error, - path: Some(path), - }); - } - }; - - let filter = - CRLiteClubcard::from_bytes(&bytes).map_err(|error| Error::FilterDecode { - error: format!("cannot decode crlite filter: {error:?}").into(), - path, - })?; - - match filter.contains( - &key, - input - .sct_timestamps - .iter() - .map(|ct_ts| (&ct_ts.log_id, ct_ts.timestamp)), - ) { - CRLiteStatus::Revoked => return Ok(RevocationStatus::CertainlyRevoked), - CRLiteStatus::Good => return Ok(RevocationStatus::NotRevoked), - CRLiteStatus::NotEnrolled | CRLiteStatus::NotCovered => continue, - } - } - - Ok(RevocationStatus::NotCoveredByRevocationData) - } - /// Verify the current contents of the cache against this manifest. /// /// This performs disk IO but does not perform network IO. @@ -399,6 +354,8 @@ pub enum Error { }, /// A downloaded file did not match the expected hash. HashMismatch(PathBuf), + /// Failed to decode the index file. + IndexDecode(Box), /// Failed to fetch a file over HTTP. HttpFetch { /// Underlying error. @@ -500,6 +457,7 @@ impl fmt::Display for Error { None => write!(f, "cannot read filter file"), }, Self::HashMismatch(path) => write!(f, "hash mismatch for file {path:?}"), + Self::IndexDecode(_) => write!(f, "cannot decode index file"), Self::HttpFetch { url, .. } => write!(f, "HTTP fetch error for URL {url}"), Self::InvalidBase64 { context, .. } => { write!(f, "invalid base64 for {context}") @@ -555,6 +513,7 @@ impl StdError for Error { Self::FilterDecode { error, .. } => Some(&**error), Self::FilterRead { error, .. } => Some(error), Self::HashMismatch(_) => None, + Self::IndexDecode(error) => Some(&**error), Self::HttpFetch { error, .. } => Some(&**error), Self::InvalidBase64 { error, .. } => Some(&**error), Self::InvalidEndEntityCertificate(error) => Some(&**error), diff --git a/upki/tests/integration.rs b/upki/tests/integration.rs index 97e4c487..c6c87187 100644 --- a/upki/tests/integration.rs +++ b/upki/tests/integration.rs @@ -161,7 +161,7 @@ fn fetch_of_empty_manifest() { ); assert_eq!( list_dir(&temp.path().join("revocation")), - vec!["manifest.json"], + vec!["index.bin", "manifest.json"], ); } @@ -365,8 +365,9 @@ fn typical_incremental_fetch_dry_run() { success: true exit_code: 0 ----- stdout ----- - 2 steps required (14 bytes to download) + 3 steps required (14 bytes to download) - download 14 bytes from http://127.0.0.1:[PORT]/filter2.delta to "[TEMPDIR]/revocation/filter2.delta" + - build index from filters into "[TEMPDIR]/revocation" - save new manifest into "[TEMPDIR]/revocation" ----- stderr -----