Prototype of intermediate preloading

.github/workflows/mirror.yml

@@ -27,7 +27,10 @@ jobs:
       - name: Generate backend files
         run: |
           mkdir tmp/
-          cargo run --locked --release --bin upki-mirror -- tmp/ production --manifest-comment="$GITHUB_REPOSITORY run $GITHUB_RUN_ID"
+          cargo run --locked --release --bin mozilla-crlite -- tmp/ production --manifest-comment="$GITHUB_REPOSITORY run $GITHUB_RUN_ID"
+          # backwards compatible name
+          cp tmp/v1-revocation-manifest.json tmp/manifest.json
+          cargo run --locked --release --bin intermediates -- tmp/ --manifest-comment="$GITHUB_REPOSITORY run $GITHUB_RUN_ID"
 
       - name: Package and upload artifact
         uses: actions/upload-pages-artifact@v4

Cargo.toml

@@ -15,6 +15,7 @@ chrono = { version = "0.4.42", features = ["alloc"], default-features = false }
 clap = { version = "4.5", features = ["derive"] }
 clubcard-crlite = "0.3.2"
 criterion = "0.8"
+csv = "1.4"
 directories = "6"
 eyre = "0.6"
 hex = { version = "0.4", features = ["serde"] }

upki-mirror/Cargo.toml

@@ -9,9 +9,11 @@ repository.workspace = true
 [dependencies]
 aws-lc-rs.workspace = true
 clap.workspace = true
+csv.workspace = true
 eyre.workspace = true
 hex.workspace = true
 reqwest.workspace = true
+rustls-pki-types.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true

upki-mirror/src/bin/intermediates.rs

@@ -0,0 +1,156 @@
+use core::time::Duration;
+use std::collections::HashMap;
+use std::fs;
+use std::path::PathBuf;
+use std::time::SystemTime;
+
+use aws_lc_rs::digest::{SHA256, digest};
+use clap::Parser;
+use eyre::{Context, Report, anyhow};
+use rustls_pki_types::CertificateDer;
+use rustls_pki_types::pem::PemObject;
+use serde::Deserialize;
+use upki::data::{Manifest, ManifestFile};
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<(), Report> {
+    let opts = Opts::try_parse()?;
+
+    let client = reqwest::Client::builder()
+        .use_rustls_tls()
+        .timeout(Duration::from_secs(opts.http_timeout_secs))
+        .user_agent(format!(
+            "{} v{} ({})",
+            env!("CARGO_PKG_NAME"),
+            env!("CARGO_PKG_VERSION"),
+            env!("CARGO_PKG_REPOSITORY")
+        ))
+        .build()
+        .wrap_err("failed to create HTTP client")?;
+
+    let response = client
+        .get("https://ccadb.my.salesforce-sites.com/mozilla/MozillaIntermediateCertsCSVReport")
+        .send()
+        .await
+        .wrap_err("records request failed")?;
+
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "HTTP error for records request: {}",
+            response.status()
+        ));
+    }
+
+    let csv_bytes = response
+        .bytes()
+        .await
+        .wrap_err("failed to receive CSV body")?;
+
+    let intermediates = csv::ReaderBuilder::new()
+        .has_headers(true)
+        .from_reader(&mut csv_bytes.as_ref())
+        .into_deserialize::<IntermediateData>()
+        .collect::<Result<Vec<_>, _>>()
+        .wrap_err("failed to parse CSV")?;
+
+    println!("we have {} intermediates", intermediates.len());
+
+    // we bucket intermediates into up to 256 files, by the first byte of the
+    // sha256-hash of their DER value.
+    //
+    // that means the manifest contains up to 256 items, and the filenames are small.
+    let mut buckets: HashMap<u8, Vec<IntermediateData>> = HashMap::new();
+
+    for i in intermediates {
+        let der = CertificateDer::from_pem_slice(i.pem.as_bytes()).wrap_err("cannot parse PEM")?;
+
+        // check hash matches
+        let actual_hash = digest(&SHA256, &der);
+        if i.sha256 != actual_hash.as_ref() {
+            return Err(anyhow!("cert {i:?} does not have correct hash"));
+        }
+
+        let bucket = i.sha256[0];
+        buckets
+            .entry(bucket)
+            .or_default()
+            .push(i);
+    }
+
+    let mut files = Vec::new();
+    for (bucket, certs) in buckets {
+        let filename = format!("{bucket:02x?}.pem",);
+
+        let mut contents = String::new();
+        for inter in certs {
+            contents.push_str(&inter.pem);
+            contents.push('\n');
+        }
+
+        fs::write(opts.output_dir.join(&filename), &contents).wrap_err("cannot write PEM file")?;
+        let hash = digest(&SHA256, contents.as_bytes());
+
+        files.push(ManifestFile {
+            filename,
+            size: contents.len(),
+            hash: hash.as_ref().to_vec(),
+        });
+    }
+
+    let manifest = Manifest {
+        generated_at: SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+        comment: opts.manifest_comment.clone(),
+        files,
+    };
+    let output_filename = opts
+        .output_dir
+        .join("v1-intermediates-manifest.json");
+    fs::write(
+        output_filename,
+        serde_json::to_string(&manifest)
+            .wrap_err("cannot encode JSON manifest")?
+            .as_bytes(),
+    )
+    .wrap_err_with(|| "cannot write manifest to {output_filename:?}")?;
+
+    Ok(())
+}
+
+#[derive(Debug, Parser)]
+struct Opts {
+    /// Where to write output files.  This must exist.
+    output_dir: PathBuf,
+
+    /// Timeout in seconds for all HTTP requests.
+    #[clap(long, default_value_t = 10)]
+    http_timeout_secs: u64,
+
+    /// Comment included in output manifest.
+    #[clap(long, default_value = "")]
+    manifest_comment: String,
+}
+
+#[non_exhaustive]
+#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize)]
+pub struct IntermediateData {
+    #[serde(rename = "Subject")]
+    pub subject: String,
+
+    #[serde(rename = "Issuer")]
+    pub issuer: String,
+
+    #[serde(rename = "SHA256", with = "hex::serde")]
+    pub sha256: [u8; 32],
+
+    #[serde(rename = "Full CRL Issued By This CA")]
+    pub full_crl: String,
+
+    #[serde(rename = "PEM")]
+    pub pem: String,
+
+    #[serde(rename = "JSON Array of Partitioned CRLs")]
+    pub json_crls: String,
+}

upki-mirror/src/main.rs → upki-mirror/src/bin/mozilla-crlite.rs

@@ -7,9 +7,7 @@ use std::time::SystemTime;
 use aws_lc_rs::digest::{SHA256, digest};
 use clap::{Parser, ValueEnum};
 use eyre::{Context, Report, anyhow};
-use upki::revocation::{Filter, Manifest};
-
-mod mozilla;
+use upki::data::{Manifest, ManifestFile};
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<(), Report> {
@@ -68,7 +66,7 @@ async fn main() -> Result<(), Report> {
         download_plan.push(item);
     }
 
-    let mut filters = Vec::new();
+    let mut files = Vec::new();
 
     for p in download_plan {
         let attachment_url = source.attachment_host.to_string() + &p.attachment.location;
@@ -100,7 +98,7 @@ async fn main() -> Result<(), Report> {
         fs::write(&output_filename, bytes)
             .wrap_err_with(|| format!("cannot write filter data to {output_filename:?}",))?;
 
-        filters.push(Filter {
+        files.push(ManifestFile {
             filename: p.attachment.filename.clone(),
             size: p.attachment.size,
             hash: p.attachment.hash.clone(),
@@ -113,9 +111,11 @@ async fn main() -> Result<(), Report> {
             .unwrap()
             .as_secs(),
         comment: opts.manifest_comment.clone(),
-        filters,
+        files,
     };
-    let output_filename = opts.output_dir.join("manifest.json");
+    let output_filename = opts
+        .output_dir
+        .join("v1-revocation-manifest.json");
     fs::write(
         output_filename,
         serde_json::to_string(&manifest)
@@ -168,3 +168,38 @@ const MOZILLA_PROD: Source = Source {
     records_url: "https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/cert-revocations/records",
     attachment_host: "https://firefox-settings-attachments.cdn.mozilla.net/",
 };
+
+/// JSON structures used in the Mozilla preferences service.
+mod mozilla {
+    use serde::Deserialize;
+
+    #[derive(Debug, Deserialize)]
+    pub(crate) struct Manifest {
+        pub(crate) data: Vec<Item>,
+    }
+
+    #[derive(Clone, Debug, Deserialize)]
+    pub(crate) struct Item {
+        pub(crate) attachment: Attachment,
+        pub(crate) channel: Channel,
+        pub(crate) id: String,
+        pub(crate) incremental: bool,
+        pub(crate) parent: Option<String>,
+    }
+
+    #[derive(Clone, Debug, Deserialize, PartialEq)]
+    #[serde(rename_all = "snake_case")]
+    pub(crate) enum Channel {
+        Default,
+        Compat,
+    }
+
+    #[derive(Clone, Debug, Deserialize)]
+    pub(crate) struct Attachment {
+        #[serde(with = "hex::serde")]
+        pub hash: Vec<u8>,
+        pub size: usize,
+        pub filename: String,
+        pub location: String,
+    }
+}