diff --git a/Cargo.lock b/Cargo.lock index fa9c4a5..95a1227 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,6 +373,7 @@ dependencies = [ "anyhow", "base64", "cbc", + "chat4n6-fs", "chat4n6-plugin-api", "chat4n6-sqlite-forensics", "chrono", diff --git a/crates/chat4n6-plugin-api/src/types.rs b/crates/chat4n6-plugin-api/src/types.rs index 4d5829a..0ce0e57 100644 --- a/crates/chat4n6-plugin-api/src/types.rs +++ b/crates/chat4n6-plugin-api/src/types.rs @@ -323,6 +323,17 @@ pub enum ForensicWarning { SelectiveDeletion { suspect_jid: String, deletion_rate_pct: u8 }, /// Timestamp order violated: a later ROWID has an earlier timestamp. TimestampAnomaly { message_row_id: i64, description: String }, + /// The message timestamp distribution is implausible in bulk — a large share + /// of messages predate the app's existence. A property of the whole set, + /// which no per-message check can express. + TimestampDistributionAnomaly { + total_messages: u32, + implausible_count: u32, + ratio_pct: u8, + /// The most frequent implausible instant, and how often it occurs. + modal_utc: DateTime, + modal_occurrences: u32, + }, /// Backup crypt14/15 HMAC does not match payload — file may have been tampered. HmacMismatch, /// PRAGMA user_version inconsistent with claimed app version. @@ -382,6 +393,11 @@ impl fmt::Display for ForensicWarning { Self::TimestampAnomaly { message_row_id, description } => { write!(f, "Timestamp anomaly at row {message_row_id}: {description}") } + Self::TimestampDistributionAnomaly { + total_messages, implausible_count, ratio_pct, modal_utc, modal_occurrences + } => { + write!(f, "Implausible timestamp distribution: {implausible_count} of {total_messages} messages ({ratio_pct}%) predate 2009-01-01; most frequent is {modal_utc} ({modal_occurrences}×)") + } Self::HmacMismatch => write!(f, "HMAC mismatch — backup integrity check FAILED"), Self::SchemaVersionMismatch { db_version, app_version } => { write!(f, "Schema v{db_version} incompatible with app version {app_version}") diff --git a/crates/plugins/chat4n6-whatsapp/Cargo.toml b/crates/plugins/chat4n6-whatsapp/Cargo.toml index f595c2d..5740d67 100644 --- a/crates/plugins/chat4n6-whatsapp/Cargo.toml +++ b/crates/plugins/chat4n6-whatsapp/Cargo.toml @@ -31,6 +31,7 @@ regex = "1" base64 = { workspace = true } [dev-dependencies] +chat4n6-fs = { path = "../../chat4n6-fs", version = "0.1" } tempfile = "3" rusqlite = { version = "0.31", features = ["backup"] } proptest = "1" diff --git a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs index c7e0491..9a75902 100644 --- a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs +++ b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs @@ -3,6 +3,7 @@ //! Detects evidence of tampering, selective deletion, timestamp anomalies, //! and SQLite VACUUM operations that destroy deleted record remnants. +use crate::schema_gate::EARLIEST_PLAUSIBLE_MS; use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicWarning}; use chrono::{DateTime, Utc}; use std::collections::HashMap; @@ -24,7 +25,9 @@ pub fn detect_vacuum(db_bytes: &[u8]) -> Vec { } let free_pages = u32::from_be_bytes([db_bytes[36], db_bytes[37], db_bytes[38], db_bytes[39]]); if free_pages > 0 { - vec![ForensicWarning::DatabaseVacuumed { freelist_page_count: free_pages }] + vec![ForensicWarning::DatabaseVacuumed { + freelist_page_count: free_pages, + }] } else { vec![] } @@ -47,8 +50,9 @@ pub fn detect_header_tamper(db_bytes: &[u8]) -> Vec { // Check 1: write_counter vs read_counter (bytes 92–95 vs 96–99). let write_version = db_bytes[18]; // 1=journal, 2=WAL - let write_counter = u32::from_be_bytes([db_bytes[92], db_bytes[93], db_bytes[94], db_bytes[95]]); - let read_counter = u32::from_be_bytes([db_bytes[96], db_bytes[97], db_bytes[98], db_bytes[99]]); + let write_counter = + u32::from_be_bytes([db_bytes[92], db_bytes[93], db_bytes[94], db_bytes[95]]); + let read_counter = u32::from_be_bytes([db_bytes[96], db_bytes[97], db_bytes[98], db_bytes[99]]); if write_counter != read_counter && write_version != 2 { warnings.push(ForensicWarning::HeaderTampered { change_counter: write_counter, @@ -58,8 +62,13 @@ pub fn detect_header_tamper(db_bytes: &[u8]) -> Vec { // Check 2: declared page_size × page_count vs actual file length. let raw_page_size = u16::from_be_bytes([db_bytes[16], db_bytes[17]]); - let page_size: u64 = if raw_page_size == 1 { 65536 } else { raw_page_size as u64 }; - let page_count = u32::from_be_bytes([db_bytes[28], db_bytes[29], db_bytes[30], db_bytes[31]]) as u64; + let page_size: u64 = if raw_page_size == 1 { + 65536 + } else { + raw_page_size as u64 + }; + let page_count = + u32::from_be_bytes([db_bytes[28], db_bytes[29], db_bytes[30], db_bytes[31]]) as u64; let expected_size = page_size * page_count; let actual_size = db_bytes.len() as u64; if actual_size >= 100 && actual_size != expected_size { @@ -122,15 +131,71 @@ pub fn detect_timestamp_anomalies(result: &ExtractionResult) -> Vec Vec { + let mut total: u32 = 0; + let mut occurrences: HashMap, u32> = HashMap::new(); + for msg in chats.iter().flat_map(|c| c.messages.iter()) { + total = total.saturating_add(1); + // Compared in milliseconds so the bound needs no fallible conversion. + if msg.timestamp.utc.timestamp_millis() < EARLIEST_PLAUSIBLE_MS { + *occurrences.entry(msg.timestamp.utc).or_insert(0) += 1; + } + } + + let implausible_count: u32 = occurrences.values().sum(); + if total == 0 || implausible_count == 0 { + return vec![]; + } + + // u64 avoids overflowing the ×100 on a large extraction. + let ratio_pct = (u64::from(implausible_count) * 100 / u64::from(total)).min(100) as u8; + if ratio_pct <= IMPLAUSIBLE_SHARE_PCT { + return vec![]; + } + + // Ties broken by the earlier instant so the finding is reproducible. + let Some((&modal_utc, &modal_occurrences)) = occurrences + .iter() + .max_by_key(|(instant, count)| (**count, std::cmp::Reverse(**instant))) + else { + return vec![]; + }; + + vec![ForensicWarning::TimestampDistributionAnomaly { + total_messages: total, + implausible_count, + ratio_pct, + modal_utc, + modal_occurrences, + }] +} + // ── New detectors (§2.6) ───────────────────────────────────────────────────── /// Detect duplicate XMPP stanza IDs (key_id column) in the message table. /// /// Takes a map of key_id → list of message row_ids built from raw SQLite records. /// Any key_id appearing more than once emits `DuplicateStanzaId`. -pub fn detect_duplicate_stanza_ids( - key_id_map: &HashMap>, -) -> Vec { +pub fn detect_duplicate_stanza_ids(key_id_map: &HashMap>) -> Vec { key_id_map .iter() .filter(|(_, rows)| rows.len() > 1) @@ -238,7 +303,9 @@ mod header_tamper_tests { let header = valid_header(4096, 1, 5, 3); let warnings = detect_header_tamper(&header); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), "write/read counter mismatch must emit HeaderTampered, got: {warnings:?}" ); } @@ -249,7 +316,9 @@ mod header_tamper_tests { let header = valid_header(4096, 2, 1, 1); let warnings = detect_header_tamper(&header); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), "page size * count != file size must emit HeaderTampered, got: {warnings:?}" ); } @@ -278,14 +347,18 @@ mod header_tamper_tests { buf[36..40].copy_from_slice(&5u32.to_be_bytes()); // free_pages=5 let warnings = detect_vacuum(&buf); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::DatabaseVacuumed { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::DatabaseVacuumed { .. })), "non-zero free_pages must emit DatabaseVacuumed" ); } #[test] fn selective_deletion_detected() { - use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent}; + use chat4n6_plugin_api::{ + Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent, + }; let make_msg = |id: i64| Message { id, chat_id: 1, @@ -310,8 +383,12 @@ mod header_tamper_tests { name: None, is_group: false, messages: vec![ - make_msg(1), make_msg(2), make_msg(3), - make_msg(100), make_msg(101), make_msg(102), // gap 3→100 + make_msg(1), + make_msg(2), + make_msg(3), + make_msg(100), + make_msg(101), + make_msg(102), // gap 3→100 ], archived: false, }; @@ -324,20 +401,24 @@ mod header_tamper_tests { schema_version: 200, forensic_warnings: vec![], group_participant_events: vec![], - extraction_started_at: None, - extraction_finished_at: None, - wal_snapshots: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], }; let warnings = detect_selective_deletion(&result); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::SelectiveDeletion { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::SelectiveDeletion { .. })), "gap 3→100 should trigger SelectiveDeletion, got: {warnings:?}" ); } #[test] fn timestamp_anomaly_pre_whatsapp() { - use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent}; + use chat4n6_plugin_api::{ + Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent, + }; let make_msg = |id: i64, ts: i64| Message { id, chat_id: 1, @@ -376,13 +457,15 @@ mod header_tamper_tests { schema_version: 200, forensic_warnings: vec![], group_participant_events: vec![], - extraction_started_at: None, - extraction_finished_at: None, - wal_snapshots: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], }; let warnings = detect_timestamp_anomalies(&result); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::TimestampAnomaly { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::TimestampAnomaly { .. })), "reversed timestamps should emit TimestampAnomaly, got: {warnings:?}" ); } @@ -404,7 +487,8 @@ mod new_detector_tests { use crate::schema::SchemaVersion; let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL); @@ -423,9 +507,12 @@ mod new_detector_tests { INSERT INTO message VALUES (1, 1, NULL, 0, 1710513127000, 'hello', 0, 'ABC123'); INSERT INTO message VALUES (2, 1, NULL, 1, 1710513128000, 'world', 0, 'ABC123'); INSERT INTO message VALUES (3, 1, NULL, 0, 1710513129000, 'foo', 0, 'XYZ999'); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); @@ -451,7 +538,8 @@ mod new_detector_tests { use crate::schema::SchemaVersion; let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL); @@ -488,9 +576,12 @@ mod new_detector_tests { INSERT INTO message_thumbnails VALUES (13, x'ff'); INSERT INTO message_thumbnails VALUES (14, x'ff'); INSERT INTO message_thumbnails VALUES (15, x'ff'); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); @@ -498,7 +589,11 @@ mod new_detector_tests { assert!( result.forensic_warnings.iter().any(|w| matches!( w, - ForensicWarning::ThumbnailOrphanHigh { orphan_thumbnails: 5, total_messages: 10, ratio_pct: 50 } + ForensicWarning::ThumbnailOrphanHigh { + orphan_thumbnails: 5, + total_messages: 10, + ratio_pct: 50 + } )), "expected ThumbnailOrphanHigh with 5 orphans / 10 messages = 50%, got: {:?}", result.forensic_warnings @@ -572,3 +667,172 @@ mod new_detector_tests { ); } } + +// ── T3: distribution-level timestamp detection ─────────────────────────────── + +#[cfg(test)] +mod timestamp_distribution_tests { + use super::*; + use chat4n6_plugin_api::{EvidenceSource, ForensicTimestamp, Message, MessageContent}; + + /// 2022-09-15T12:00:00Z. + const GOOD_MS: i64 = 1_663_243_200_000; + + fn msg(id: i64, ts_ms: i64) -> Message { + Message { + id, + chat_id: 1, + sender_jid: None, + from_me: false, + timestamp: ForensicTimestamp::from_millis(ts_ms, 0), + content: MessageContent::Text(String::new()), + reactions: vec![], + quoted_message: None, + source: EvidenceSource::Live, + row_offset: 0, + starred: false, + forward_score: None, + is_forwarded: false, + edit_history: vec![], + receipts: vec![], + forwarded_from: None, + } + } + + fn chat_of(messages: Vec) -> Vec { + vec![Chat { + id: 1, + jid: "a@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages, + archived: false, + }] + } + + fn distribution_warning(chats: &[Chat]) -> Option { + detect_timestamp_distribution_anomaly(chats) + .into_iter() + .find(|w| matches!(w, ForensicWarning::TimestampDistributionAnomaly { .. })) + } + + /// The headline failure: every message at epoch zero, in perfect order, so + /// the pairwise detector sees nothing wrong. + #[test] + fn mass_epoch_zero_timestamps_raise_a_warning() { + let chats = chat_of((1..=100).map(|i| msg(i, 0)).collect()); + assert!( + distribution_warning(&chats).is_some(), + "100 messages dated 1970-01-01 must not pass unremarked" + ); + assert!( + detect_timestamp_anomalies(&ExtractionResult { + chats: chats.clone(), + contacts: vec![], + calls: vec![], + wal_deltas: vec![], + timezone_offset_seconds: Some(0), + schema_version: 1, + forensic_warnings: vec![], + group_participant_events: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], + }) + .is_empty(), + "the pairwise detector cannot see this — which is why the set-level one exists" + ); + } + + #[test] + fn warning_reports_the_count_share_and_modal_instant() { + let mut messages: Vec = (1..=90).map(|i| msg(i, GOOD_MS + i)).collect(); + messages.extend((91..=100).map(|i| msg(i, 0))); + let warning = distribution_warning(&chat_of(messages)).expect("10% is over the threshold"); + match warning { + ForensicWarning::TimestampDistributionAnomaly { + total_messages, + implausible_count, + ratio_pct, + modal_utc, + modal_occurrences, + } => { + assert_eq!(total_messages, 100); + assert_eq!(implausible_count, 10); + assert_eq!(ratio_pct, 10); + assert_eq!( + modal_utc.timestamp_millis(), + 0, + "epoch zero is the modal instant" + ); + assert_eq!(modal_occurrences, 10); + } + other => panic!("expected TimestampDistributionAnomaly, got {other:?}"), + } + } + + #[test] + fn a_healthy_distribution_raises_nothing() { + let chats = chat_of((1..=100).map(|i| msg(i, GOOD_MS + i * 1000)).collect()); + assert!( + distribution_warning(&chats).is_none(), + "plausible timestamps must not be flagged" + ); + } + + #[test] + fn a_handful_of_bad_rows_stays_below_the_threshold() { + let mut messages: Vec = (1..=98).map(|i| msg(i, GOOD_MS + i * 1000)).collect(); + messages.push(msg(99, 0)); + messages.push(msg(100, 1)); + assert!( + distribution_warning(&chat_of(messages)).is_none(), + "2% of implausible rows is an outlier, not a distribution anomaly" + ); + } + + #[test] + fn an_empty_extraction_raises_nothing() { + assert!(detect_timestamp_distribution_anomaly(&[]).is_empty()); + assert!(distribution_warning(&chat_of(vec![])).is_none()); + } + + #[test] + fn the_share_is_measured_across_all_chats_not_per_chat() { + // 10 implausible messages sitting in their own chat are 10% of the + // extraction, not 100% of one chat. + let good = Chat { + id: 1, + jid: "a@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages: (1..=90).map(|i| msg(i, GOOD_MS + i * 1000)).collect(), + archived: false, + }; + let bad = Chat { + id: 2, + jid: "b@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages: (91..=100).map(|i| msg(i, 0)).collect(), + archived: false, + }; + let warning = distribution_warning(&[good, bad]).expect("10% overall"); + if let ForensicWarning::TimestampDistributionAnomaly { ratio_pct, .. } = warning { + assert_eq!(ratio_pct, 10); + } + } + + #[test] + fn the_warning_renders_the_numbers_it_carries() { + let chats = chat_of((1..=20).map(|i| msg(i, 0)).collect()); + let rendered = distribution_warning(&chats) + .expect("all implausible") + .to_string(); + assert!(rendered.contains("20"), "must state the counts: {rendered}"); + assert!( + rendered.contains("1970"), + "must state the modal instant: {rendered}" + ); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/src/columns.rs b/crates/plugins/chat4n6-whatsapp/src/columns.rs new file mode 100644 index 0000000..28f6aaa --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/src/columns.rs @@ -0,0 +1,528 @@ +//! Name-based column resolution for msgstore.db tables. +//! +//! WhatsApp reorders, inserts and removes `message`/`chat`/`jid`/`call_log` +//! columns between app releases, so a fixed ordinal is only ever right for the +//! one schema it was written against. Reading the wrong column produces a +//! structurally complete report full of wrong values — the failure class that +//! is indistinguishable from a correct one. +//! +//! Every ordinal used by the extractor is therefore derived at run time from +//! the database's own `CREATE TABLE` SQL (read out of `sqlite_master`), and a +//! column that the schema does not declare resolves to `None` rather than to a +//! neighbouring column's data. +//! +//! # Record layout invariant +//! +//! SQLite stores an `INTEGER PRIMARY KEY` column as a serial-type-0 NULL **at +//! its declared position** in the record body; the real value lives in the +//! cell's rowid varint. The btree walker decodes serial types in order, so +//! `values[i]` is the column declared at DDL position `i` — including when the +//! rowid alias is not the first declared column. + +use std::collections::HashMap; + +/// Zero-based column positions for one table, keyed by lowercased column name. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TableColumns { + by_name: HashMap, +} + +impl TableColumns { + /// Parse a `CREATE TABLE` statement into a name → position map. + /// + /// Table-level constraints (`PRIMARY KEY (…)`, `UNIQUE (…)`, `CHECK (…)`, + /// `FOREIGN KEY (…)`, `CONSTRAINT …`) do not occupy a column position and + /// are skipped. Malformed input yields an empty map rather than a panic. + pub fn from_ddl(ddl: &str) -> Self { + let Some(body) = column_body(ddl) else { + return Self::default(); + }; + let mut by_name = HashMap::new(); + let mut position = 0usize; + for part in split_top_level(body) { + let Some(name) = column_name(part) else { + continue; // table-level constraint: occupies no column position + }; + by_name.entry(name).or_insert(position); + position += 1; + } + Self { by_name } + } + + /// Zero-based position of `name`, or `None` when the table has no such column. + pub fn get(&self, name: &str) -> Option { + self.by_name.get(&name.to_ascii_lowercase()).copied() + } + + /// Position of the first of `names` that the table declares. + /// + /// Used where one logical field is spelled differently across schema + /// generations; the order of `names` is the preference order. + pub fn first_of(&self, names: &[&str]) -> Option { + names.iter().find_map(|n| self.get(n)) + } + + /// Number of declared columns. + pub fn len(&self) -> usize { + self.by_name.len() + } + + pub fn is_empty(&self) -> bool { + self.by_name.is_empty() + } +} + +/// Column maps for every table in a database, built from `ForensicEngine::table_ddl`. +#[derive(Debug, Clone, Default)] +pub struct SchemaColumns { + tables: HashMap, + empty: TableColumns, +} + +impl SchemaColumns { + /// Build from a `table name → CREATE TABLE SQL` map. + pub fn from_ddl_map(ddl: &HashMap) -> Self { + Self { + tables: ddl + .iter() + .map(|(name, sql)| (name.to_ascii_lowercase(), TableColumns::from_ddl(sql))) + .collect(), + empty: TableColumns::default(), + } + } + + /// Columns of `table`; an empty map when the database has no such table. + /// + /// Returning an empty map (rather than `Option`) keeps every downstream + /// lookup on the same degrade-to-`None` path. + pub fn table(&self, name: &str) -> &TableColumns { + self.tables + .get(&name.to_ascii_lowercase()) + .unwrap_or(&self.empty) + } + + /// Every table name, sorted — for diagnostics that must show what was found. + pub fn table_names(&self) -> Vec<&str> { + let mut names: Vec<&str> = self.tables.keys().map(String::as_str).collect(); + names.sort_unstable(); + names + } + + /// Whether any table DDL was resolved at all. + /// + /// A database whose schema could not be read is a bootstrap failure, not an + /// empty database — callers must fail loudly rather than report no results. + pub fn is_empty(&self) -> bool { + self.tables.is_empty() + } +} + +// ── DDL parsing ────────────────────────────────────────────────────────────── + +/// Closing delimiter for an opening quote/bracket, or `None` if `b` doesn't open one. +fn closing_delimiter(b: u8) -> Option { + match b { + b'\'' | b'"' | b'`' => Some(b), + b'[' => Some(b']'), + _ => None, + } +} + +/// The text between a `CREATE TABLE`'s outermost parentheses. +/// +/// Returns `None` when the statement has no balanced parenthesised body. +fn column_body(ddl: &str) -> Option<&str> { + let start = ddl.find('(')?; + let mut depth = 0usize; + let mut closing: Option = None; + for (i, &b) in ddl.as_bytes().iter().enumerate().skip(start) { + if let Some(c) = closing { + if b == c { + closing = None; + } + continue; + } + if let Some(c) = closing_delimiter(b) { + closing = Some(c); + } else if b == b'(' { + depth += 1; + } else if b == b')' { + depth = depth.saturating_sub(1); + if depth == 0 { + // Both delimiters are ASCII, so these are char boundaries. + return Some(&ddl[start + 1..i]); + } + } + } + None +} + +/// Split a column body on commas that are neither nested nor quoted. +fn split_top_level(body: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut depth = 0usize; + let mut closing: Option = None; + let mut start = 0usize; + for (i, &b) in body.as_bytes().iter().enumerate() { + if let Some(c) = closing { + if b == c { + closing = None; + } + continue; + } + if let Some(c) = closing_delimiter(b) { + closing = Some(c); + } else if b == b'(' { + depth += 1; + } else if b == b')' { + depth = depth.saturating_sub(1); + } else if b == b',' && depth == 0 { + parts.push(&body[start..i]); + start = i + 1; + } + } + parts.push(&body[start..]); + parts +} + +/// The lowercased column name a body part declares, or `None` when the part is +/// a table-level constraint. +/// +/// SQLite reserves `PRIMARY`, `UNIQUE`, `CHECK`, `FOREIGN` and `CONSTRAINT`, so +/// a column that really carries one of those names has to be quoted — and a +/// quoted name skips the keyword test. +fn column_name(part: &str) -> Option { + const CONSTRAINT_KEYWORDS: [&str; 5] = ["primary", "unique", "check", "foreign", "constraint"]; + + let s = part.trim(); + let first = s.chars().next()?; + if let Some(close) = closing_delimiter(first as u8) { + let rest = s.get(first.len_utf8()..)?; + let end = rest.find(close as char)?; + let name = rest.get(..end)?; + return (!name.is_empty()).then(|| name.to_ascii_lowercase()); + } + + let end = s + .find(|c: char| c.is_whitespace() || c == '(' || c == ',') + .unwrap_or(s.len()); + let name = s.get(..end)?.to_ascii_lowercase(); + if name.is_empty() || CONSTRAINT_KEYWORDS.contains(&name.as_str()) { + return None; + } + Some(name) +} + +/// Resolved `message` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MessageColumns { + pub chat_row_id: Option, + pub sender_jid_row_id: Option, + pub from_me: Option, + pub timestamp: Option, + pub text_data: Option, + pub message_type: Option, + pub media_mime_type: Option, + pub media_name: Option, + pub starred: Option, + pub edit_version: Option, + pub key_id: Option, +} + +impl MessageColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + chat_row_id: cols.get("chat_row_id"), + sender_jid_row_id: cols.get("sender_jid_row_id"), + from_me: cols.get("from_me"), + timestamp: cols.get("timestamp"), + text_data: cols.get("text_data"), + message_type: cols.get("message_type"), + media_mime_type: cols.get("media_mime_type"), + media_name: cols.get("media_name"), + starred: cols.get("starred"), + edit_version: cols.get("edit_version"), + key_id: cols.get("key_id"), + } + } +} + +/// Resolved `jid` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct JidColumns { + pub raw_string: Option, +} + +impl JidColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + raw_string: cols.get("raw_string"), + } + } +} + +/// Resolved `chat` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ChatColumns { + pub jid_row_id: Option, + pub subject: Option, + pub archived: Option, +} + +impl ChatColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + jid_row_id: cols.get("jid_row_id"), + subject: cols.get("subject"), + archived: cols.get("archived"), + } + } +} + +/// Resolved `call_log` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CallLogColumns { + pub jid_row_id: Option, + pub from_me: Option, + pub video_call: Option, + pub duration: Option, + pub timestamp: Option, + pub call_result: Option, + pub call_row_id: Option, + pub call_creator_device_jid_row_id: Option, +} + +impl CallLogColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + jid_row_id: cols.get("jid_row_id"), + from_me: cols.get("from_me"), + video_call: cols.get("video_call"), + duration: cols.get("duration"), + timestamp: cols.get("timestamp"), + call_result: cols.get("call_result"), + call_row_id: cols.get("call_row_id"), + call_creator_device_jid_row_id: cols.get("call_creator_device_jid_row_id"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The `message` DDL read out of a real 2023-era Android device. + const REAL_MESSAGE_DDL: &str = "CREATE TABLE message (\ + _id INTEGER PRIMARY KEY AUTOINCREMENT, chat_row_id INTEGER NOT NULL, \ + from_me INTEGER NOT NULL, key_id TEXT NOT NULL, sender_jid_row_id INTEGER, \ + status INTEGER, broadcast INTEGER, recipient_count INTEGER, \ + participant_hash TEXT, origination_flags INTEGER, origin INTEGER, \ + timestamp INTEGER, received_timestamp INTEGER, receipt_server_timestamp INTEGER, \ + message_type INTEGER, text_data TEXT, starred INTEGER, lookup_tables INTEGER, \ + sort_id INTEGER, message_add_on_flags INTEGER, view_mode INTEGER)"; + + const REAL_JID_DDL: &str = "CREATE TABLE jid (_id INTEGER PRIMARY KEY AUTOINCREMENT, \ + user TEXT NOT NULL, server TEXT NOT NULL, agent INTEGER, type INTEGER, \ + raw_string TEXT, device INTEGER)"; + + const REAL_CHAT_DDL: &str = "CREATE TABLE chat (_id INTEGER PRIMARY KEY AUTOINCREMENT, \ + jid_row_id INTEGER UNIQUE, hidden INTEGER, subject TEXT, created_timestamp INTEGER, \ + display_message_row_id INTEGER, last_message_row_id INTEGER, \ + last_read_message_row_id INTEGER, last_read_receipt_sent_message_row_id INTEGER, \ + last_important_message_row_id INTEGER, archived INTEGER, sort_timestamp INTEGER)"; + + const REAL_CALL_LOG_DDL: &str = "CREATE TABLE call_log (\ + _id INTEGER PRIMARY KEY AUTOINCREMENT, jid_row_id INTEGER, from_me INTEGER, \ + call_id TEXT, transaction_id INTEGER, timestamp INTEGER, video_call INTEGER, \ + duration INTEGER, call_result INTEGER, is_dnd_mode_on INTEGER)"; + + #[test] + fn resolves_declared_column_positions() { + let c = + TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, b TEXT, c INTEGER)"); + assert_eq!(c.get("_id"), Some(0)); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("c"), Some(2)); + assert_eq!(c.len(), 3); + } + + #[test] + fn lookup_is_case_insensitive() { + let c = TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, Text_Data TEXT)"); + assert_eq!(c.get("text_data"), Some(1)); + assert_eq!(c.get("TEXT_DATA"), Some(1)); + } + + #[test] + fn absent_column_resolves_to_none() { + let c = TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, b TEXT)"); + assert_eq!( + c.get("media_mime_type"), + None, + "a column the schema does not declare must not resolve to a neighbour" + ); + } + + #[test] + fn table_level_constraints_do_not_occupy_positions() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER, b INTEGER, c TEXT, \ + PRIMARY KEY (a, b), UNIQUE (c), \ + FOREIGN KEY (b) REFERENCES other (id), \ + CHECK (a > 0), CONSTRAINT uq1 UNIQUE (a, c))", + ); + assert_eq!(c.get("a"), Some(0)); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("c"), Some(2)); + assert_eq!( + c.len(), + 3, + "table constraints must not be counted as columns" + ); + } + + #[test] + fn commas_inside_parentheses_do_not_split_columns() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER PRIMARY KEY, b NUMERIC(10, 2), d TEXT)", + ); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("d"), Some(2)); + } + + #[test] + fn commas_inside_string_defaults_do_not_split_columns() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER PRIMARY KEY, b TEXT DEFAULT 'x,y', d TEXT)", + ); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("d"), Some(2)); + } + + #[test] + fn quoted_identifiers_are_unwrapped() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (`_id` INTEGER PRIMARY KEY, \"text_data\" TEXT, [starred] INTEGER)", + ); + assert_eq!(c.get("_id"), Some(0)); + assert_eq!(c.get("text_data"), Some(1)); + assert_eq!(c.get("starred"), Some(2)); + } + + #[test] + fn quoted_table_name_and_if_not_exists_are_tolerated() { + let c = TableColumns::from_ddl( + "CREATE TABLE IF NOT EXISTS \"message\" (_id INTEGER PRIMARY KEY, timestamp INTEGER)", + ); + assert_eq!(c.get("timestamp"), Some(1)); + } + + #[test] + fn malformed_ddl_yields_empty_map_without_panicking() { + for ddl in ["", "CREATE TABLE t", "not sql at all", "CREATE TABLE t ("] { + let c = TableColumns::from_ddl(ddl); + assert!( + c.is_empty(), + "malformed DDL {ddl:?} must resolve to no columns" + ); + } + } + + #[test] + fn first_of_returns_the_first_declared_alias() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (_id INTEGER PRIMARY KEY, parent_message_row_id INTEGER)", + ); + assert_eq!( + c.first_of(&["message_row_id", "parent_message_row_id"]), + Some(1) + ); + assert_eq!(c.first_of(&["nope", "also_nope"]), None); + } + + // ── Real-device DDL: the positions the extractor must resolve ──────────── + + #[test] + fn real_message_ddl_resolves_confirmed_positions() { + let c = TableColumns::from_ddl(REAL_MESSAGE_DDL); + assert_eq!(c.get("chat_row_id"), Some(1)); + assert_eq!(c.get("from_me"), Some(2)); + assert_eq!(c.get("key_id"), Some(3)); + assert_eq!(c.get("sender_jid_row_id"), Some(4)); + assert_eq!(c.get("timestamp"), Some(11)); + assert_eq!(c.get("message_type"), Some(14)); + assert_eq!(c.get("text_data"), Some(15)); + assert_eq!(c.get("starred"), Some(16)); + } + + #[test] + fn real_message_ddl_has_no_media_columns() { + let c = TableColumns::from_ddl(REAL_MESSAGE_DDL); + let m = MessageColumns::resolve(&c); + assert_eq!( + m.media_mime_type, None, + "modern schema moved media to message_media" + ); + assert_eq!(m.media_name, None); + assert_eq!(m.edit_version, None); + assert_eq!(m.timestamp, Some(11)); + assert_eq!(m.text_data, Some(15)); + } + + #[test] + fn real_jid_ddl_resolves_raw_string_not_user() { + let c = TableColumns::from_ddl(REAL_JID_DDL); + let j = JidColumns::resolve(&c); + assert_eq!( + j.raw_string, + Some(5), + "raw_string is at 5; `user` at 1 is the bare number" + ); + } + + #[test] + fn real_chat_ddl_resolves_subject_and_archived() { + let c = TableColumns::from_ddl(REAL_CHAT_DDL); + let ch = ChatColumns::resolve(&c); + assert_eq!(ch.jid_row_id, Some(1)); + assert_eq!(ch.subject, Some(3), "subject is at 3; `hidden` sits at 2"); + assert_eq!(ch.archived, Some(10)); + } + + #[test] + fn real_call_log_ddl_resolves_duration_and_video_call() { + let c = TableColumns::from_ddl(REAL_CALL_LOG_DDL); + let cl = CallLogColumns::resolve(&c); + assert_eq!(cl.jid_row_id, Some(1)); + assert_eq!(cl.from_me, Some(2)); + assert_eq!(cl.timestamp, Some(5)); + assert_eq!(cl.video_call, Some(6)); + assert_eq!(cl.duration, Some(7)); + assert_eq!(cl.call_result, Some(8)); + assert_eq!( + cl.call_row_id, None, + "no call_row_id column: calls must not be grouped by a neighbouring value" + ); + } + + // ── SchemaColumns ──────────────────────────────────────────────────────── + + #[test] + fn schema_columns_resolves_per_table() { + let mut ddl = HashMap::new(); + ddl.insert("message".to_string(), REAL_MESSAGE_DDL.to_string()); + ddl.insert("jid".to_string(), REAL_JID_DDL.to_string()); + let sc = SchemaColumns::from_ddl_map(&ddl); + assert!(!sc.is_empty()); + assert_eq!(sc.table("message").get("timestamp"), Some(11)); + assert_eq!(sc.table("jid").get("raw_string"), Some(5)); + } + + #[test] + fn schema_columns_unknown_table_is_empty_not_a_panic() { + let sc = SchemaColumns::from_ddl_map(&HashMap::new()); + assert!(sc.is_empty()); + assert!(sc.table("message").is_empty()); + assert_eq!(sc.table("message").get("timestamp"), None); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index 7d98713..c52a6a0 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -1,7 +1,14 @@ -use crate::anti_forensics::{detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans}; -use crate::schema::{cols, SchemaVersion}; +use crate::anti_forensics::{ + detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans, + detect_timestamp_distribution_anomaly, +}; +use crate::columns::{ + CallLogColumns, ChatColumns, JidColumns, MessageColumns, SchemaColumns, TableColumns, +}; +use crate::schema::SchemaVersion; pub use crate::schema::{default_mime_for_type, is_media_type, msg_type_label}; -use anyhow::{Context, Result}; +use crate::schema_gate::validate_message_columns; +use anyhow::{bail, Context, Result}; use chat4n6_plugin_api::{ CallRecord, CallResult, Chat, Contact, EditHistoryEntry, ExtractionResult, ForensicTimestamp, GroupParticipantEvent, MediaRef, Message, MessageContent, MessageReceipt, ParticipantAction, @@ -12,6 +19,7 @@ use chat4n6_sqlite_forensics::{ partition_by_table, record::{RecoveredRecord, SqlValue}, }; +use chrono::Utc; use rayon::prelude::*; use std::collections::{HashMap, HashSet}; @@ -19,18 +27,10 @@ use std::collections::{HashMap, HashSet}; /// /// `tz_offset_secs` is seconds east of UTC for local time display. /// -/// NOTE on column indices: the btree walker stores INTEGER PRIMARY KEY as -/// SqlValue::Null at values[0]. Real column data starts at values[1]. -/// Schema layout (zero-based values[] index, after the leading Null): -/// -/// jid: [0]=Null(_id), [1]=raw_string -/// chat: [0]=Null(_id), [1]=jid_row_id, [2]=subject -/// message: [0]=Null(_id), [1]=chat_row_id, [2]=sender_jid_row_id, -/// [3]=from_me, [4]=timestamp, [5]=text_data, [6]=message_type, -/// [7]=media_mime_type, [8]=media_name, [9]=starred, -/// [10]=edit_version (newer schemas; 5=deleted-for-me, 7=deleted-for-all) -/// call_log: [0]=Null(_id), [1]=jid_row_id, [2]=from_me, -/// [3]=video_call,[4]=duration, [5]=timestamp +/// Every column position is resolved by name from the database's own +/// `CREATE TABLE` SQL (see [`crate::columns`]). WhatsApp reorders these tables +/// between releases, so a fixed ordinal is only ever right for one schema — +/// reading the wrong one yields a complete report full of wrong values. pub fn extract_from_msgstore( db_bytes: &[u8], tz_offset_secs: i32, @@ -39,8 +39,39 @@ pub fn extract_from_msgstore( let engine = ForensicEngine::new(db_bytes, Some(tz_offset_secs)) .context("failed to open msgstore.db")?; - // Read DDL map for schema-aware column index resolution (e.g. key_id position). + // Resolve every column position from the database's own schema. A database + // whose sqlite_master cannot be read is a bootstrap failure, not an empty + // database: report it rather than returning a plausible-looking empty result. let ddl_map = engine.table_ddl(); + if ddl_map.is_empty() { + bail!( + "no CREATE TABLE statement could be read from sqlite_master (page 1 of \ + {} bytes); refusing to extract with unresolved column positions", + db_bytes.len() + ); + } + let schema_cols = SchemaColumns::from_ddl_map(&ddl_map); + if schema_cols.table("message").is_empty() { + // Reporting zero messages here would be indistinguishable from a clean + // device, so name what was actually found instead. + if !schema_cols.table("messages").is_empty() { + bail!( + "msgstore.db declares the legacy `messages` table and no modern `message` \ + table; this extractor reads the modern generation (message/jid/chat) only. \ + Refusing to report zero messages for a database it cannot read." + ); + } + bail!( + "msgstore.db declares no `message` table, and no legacy `messages` table \ + either. Tables found ({} total): [{}]", + schema_cols.table_names().len(), + summarise_table_names(&schema_cols.table_names()) + ); + } + let msg_cols = MessageColumns::resolve(schema_cols.table("message")); + let jid_cols = JidColumns::resolve(schema_cols.table("jid")); + let chat_cols = ChatColumns::resolve(schema_cols.table("chat")); + let call_cols = CallLogColumns::resolve(schema_cols.table("call_log")); let records = engine.recover_layer1().context("Layer 1 recovery failed")?; @@ -48,19 +79,22 @@ pub fn extract_from_msgstore( let by_table = partition_by_table(&records); // Build JID lookup: id → raw_string - let jid_map = build_jid_map(tbl(&by_table, "jid")); + let jid_map = build_jid_map(tbl(&by_table, "jid"), &jid_cols); // Build chat map: chat_id → Chat (populated with messages below) - let mut chats = build_chats( - tbl(&by_table, "chat"), - &jid_map, - ); + let mut chats = build_chats(tbl(&by_table, "chat"), &jid_map, &chat_cols); + + // Validate the resolved map against the records before interpreting any of + // them. A wrong map still yields a complete report, so this must fail + // loudly rather than degrade. + let msg_records = tbl(&by_table, "message"); + validate_message_columns(msg_records, &msg_cols, Utc::now().timestamp_millis()) + .context("msgstore.db schema validation failed")?; // Map messages into chats. If the chat record was deleted/unrecovered, // create a stub so forensically-recovered messages are never silently dropped. - let msg_records = tbl(&by_table, "message"); for rec in msg_records { - if let Some(msg) = record_to_message(rec, &jid_map, tz_offset_secs) { + if let Some(msg) = record_to_message(rec, &jid_map, tz_offset_secs, &msg_cols) { chats .entry(msg.chat_id) .or_insert_with(|| Chat { @@ -77,20 +111,20 @@ pub fn extract_from_msgstore( } // Build key_id → Vec map for DuplicateStanzaId detection. - // Uses DDL-parsed column index to be robust to varying schema column order. - let key_id_map = build_key_id_map(msg_records, &ddl_map); + let key_id_map = build_key_id_map(msg_records, msg_cols.key_id); // ── Quoted messages ────────────────────────────────────────────────── // Build a map of message_row_id → (text, sender_jid, from_me, timestamp) // from the message_quoted table, then attach to parent messages. + let quoted_cols = schema_cols.table("message_quoted"); let quoted_records = tbl(&by_table, "message_quoted"); - let quoted_map = build_quoted_map(quoted_records, &jid_map, tz_offset_secs); + let quoted_map = build_quoted_map(quoted_records, &jid_map, tz_offset_secs, quoted_cols); // Build ghost map: message_row_id → text_data for ghost recovery. // When a deleted/tombstone message (msg_type=15) is later quoted, the // message_quoted table preserves its original text. We index that here // so we can upgrade Deleted/Unknown(15) messages to GhostRecovered. - let ghost_map = build_ghost_map(quoted_records); + let ghost_map = build_ghost_map(quoted_records, quoted_cols); for chat in chats.values_mut() { for msg in &mut chat.messages { @@ -100,7 +134,10 @@ pub fn extract_from_msgstore( } // Ghost recovery: upgrade tombstone messages whose original text // was preserved in message_quoted. - if matches!(&msg.content, MessageContent::Unknown(15) | MessageContent::Deleted) { + if matches!( + &msg.content, + MessageContent::Unknown(15) | MessageContent::Deleted + ) { if let Some(ghost_text) = ghost_map.get(&msg.id) { msg.content = MessageContent::GhostRecovered(ghost_text.clone()); } @@ -111,58 +148,53 @@ pub fn extract_from_msgstore( // ── Reactions (message_add_on type=56) ────────────────────────────────── // Build map: message_row_id → Vec let mut reactions_map: HashMap> = HashMap::new(); + let add_on_cols = schema_cols.table("message_add_on"); + let add_on_msg_row_id = add_on_cols.get("message_row_id"); + let add_on_sender = add_on_cols.get("sender_jid_row_id"); + let add_on_ts = add_on_cols.get("timestamp"); + let add_on_type_col = add_on_cols.get("type"); + let add_on_text = add_on_cols.get("text_data"); let add_on_records = tbl(&by_table, "message_add_on"); for r in add_on_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, add_on_msg_row_id) else { + continue; }; - let add_on_type = match r.values.get(5) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(add_on_type) = int_at(r, add_on_type_col) else { + continue; }; if add_on_type == 56 { // Reaction: emoji in text_data - let emoji = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => s.clone(), - _ => continue, - }; - let ts_ms = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(emoji) = text_at(r, add_on_text) else { + continue; }; - let reactor_jid = match r.values.get(3) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned().unwrap_or_default(), - _ => String::new(), - }; - reactions_map - .entry(msg_row_id) - .or_default() - .push(Reaction { - emoji, - reactor_jid, - timestamp: ForensicTimestamp::from_millis(ts_ms, tz_offset_secs), - source: r.source.clone(), - }); + let ts_ms = int_at(r, add_on_ts).unwrap_or(0); + let reactor_jid = int_at(r, add_on_sender) + .and_then(|n| jid_map.get(&n).cloned()) + .unwrap_or_default(); + reactions_map.entry(msg_row_id).or_default().push(Reaction { + emoji, + reactor_jid, + timestamp: ForensicTimestamp::from_millis(ts_ms, tz_offset_secs), + source: r.source.clone(), + }); } } // ── Edit history (message_edit_info) ──────────────────────────────────── // Build map: message_row_id → Vec let mut edits_map: HashMap> = HashMap::new(); + let edit_cols = schema_cols.table("message_edit_info"); + let edit_msg_row_id = edit_cols.get("message_row_id"); + let edit_ts = edit_cols.get("edited_timestamp"); + let edit_text = edit_cols.get("original_text"); let edit_records = tbl(&by_table, "message_edit_info"); for r in edit_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let edited_ts = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(msg_row_id) = int_at(r, edit_msg_row_id) else { + continue; }; - let original_text = match r.values.get(3) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, + let edited_ts = int_at(r, edit_ts).unwrap_or(0); + let Some(original_text) = text_at(r, edit_text) else { + continue; }; edits_map .entry(msg_row_id) @@ -177,24 +209,23 @@ pub fn extract_from_msgstore( // ── Receipts (receipt_user) ───────────────────────────────────────────── // Build map: message_row_id → Vec let mut receipts_map: HashMap> = HashMap::new(); + let receipt_cols = schema_cols.table("receipt_user"); + let receipt_msg_row_id = receipt_cols.get("message_row_id"); + let receipt_jid_row_id = receipt_cols.get("receipt_user_jid_row_id"); + let receipt_status = receipt_cols.get("status"); + let receipt_ts = receipt_cols.get("timestamp"); let receipt_records = tbl(&by_table, "receipt_user"); for r in receipt_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let jid_row_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, receipt_msg_row_id) else { + continue; }; - let status = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, receipt_jid_row_id) else { + continue; }; - let ts_ms = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(status) = int_at(r, receipt_status) else { + continue; }; + let ts_ms = int_at(r, receipt_ts).unwrap_or(0); let receipt_type = match status { 5 => ReceiptType::Delivered, 13 => ReceiptType::Read, @@ -216,17 +247,18 @@ pub fn extract_from_msgstore( // ── Forwarded messages (message_forwarded) ─────────────────────────────── // Build map: message_row_id → forward_score let mut forwarded_map: HashMap = HashMap::new(); + let fwd_cols = schema_cols.table("message_forwarded"); + let fwd_msg_row_id = fwd_cols.get("message_row_id"); + let fwd_score = fwd_cols.get("forward_score"); let fwd_records = tbl(&by_table, "message_forwarded"); for r in fwd_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, fwd_msg_row_id) else { + continue; }; - let score = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n as u32, - _ => continue, + let Some(score) = int_at(r, fwd_score) else { + continue; }; - forwarded_map.insert(msg_row_id, score); + forwarded_map.insert(msg_row_id, score as u32); } // Attach reactions, edit history, receipts, and forwarding to messages @@ -257,14 +289,18 @@ pub fn extract_from_msgstore( let call_records = tbl(&by_table, "call_log"); let raw_calls: Vec<(CallRecord, Option)> = call_records .iter() - .filter_map(|r| record_to_call(r, &jid_map, tz_offset_secs)) + .filter_map(|r| record_to_call(r, &jid_map, tz_offset_secs, &call_cols)) .collect(); let calls = merge_group_calls(raw_calls); // ── Group participant events (group_participant_user) ─────────────────── let gpe_records = tbl(&by_table, "group_participant_user"); - let group_participant_events = - build_group_participant_events(gpe_records, &jid_map, tz_offset_secs); + let group_participant_events = build_group_participant_events( + gpe_records, + &jid_map, + tz_offset_secs, + schema_cols.table("group_participant_user"), + ); // WAL deltas (placeholder — WAL integration in CLI layer) let wal_deltas: Vec = Vec::new(); @@ -280,17 +316,15 @@ pub fn extract_from_msgstore( // ── §2.6 Anti-forensics detectors ──────────────────────────────────────── // Collect live message IDs for thumbnail orphan detection. - let live_message_ids: HashSet = chats.values() + let live_message_ids: HashSet = chats + .values() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); let total_messages = live_message_ids.len() as u32; // Collect message_thumbnails row IDs. let thumbnail_records = tbl(&by_table, "message_thumbnails"); - let thumbnail_row_ids: Vec = thumbnail_records - .iter() - .filter_map(|r| r.row_id) - .collect(); + let thumbnail_row_ids: Vec = thumbnail_records.iter().filter_map(|r| r.row_id).collect(); let chats_vec: Vec<_> = chats.into_values().collect(); @@ -302,6 +336,7 @@ pub fn extract_from_msgstore( total_messages, )); forensic_warnings.extend(detect_rowid_reuse(&chats_vec)); + forensic_warnings.extend(detect_timestamp_distribution_anomaly(&chats_vec)); Ok(ExtractionResult { chats: chats_vec, @@ -362,179 +397,154 @@ pub fn extract_parallel( /// Look up a table name in a `partition_by_table` map and return its records as a slice. /// Returns an empty slice when the table is absent. -fn tbl<'a>(by: &'a HashMap>, name: &str) -> &'a [&'a RecoveredRecord] { +fn tbl<'a>( + by: &'a HashMap>, + name: &str, +) -> &'a [&'a RecoveredRecord] { by.get(name).map(|v| v.as_slice()).unwrap_or_default() } +/// Render a table-name list for a diagnostic. +/// +/// A real msgstore declares well over a hundred tables, so whole names beyond +/// the first 20 are omitted — and the omission is stated, with the full count +/// alongside. Every name shown is verbatim. +fn summarise_table_names(names: &[&str]) -> String { + const SHOWN: usize = 20; + let head = names + .iter() + .take(SHOWN) + .copied() + .collect::>() + .join(", "); + if names.len() > SHOWN { + format!("{head}, and {} more", names.len() - SHOWN) + } else { + head + } +} + +/// Read the integer at a resolved column position. +/// +/// `None` when the schema does not declare the column, the record is shorter +/// than that position, or the stored value is not an integer — never a +/// neighbouring column's value. +fn int_at(r: &RecoveredRecord, idx: Option) -> Option { + match r.values.get(idx?) { + Some(SqlValue::Int(n)) => Some(*n), + _ => None, + } +} + +/// Read the non-empty text at a resolved column position. +fn text_at(r: &RecoveredRecord, idx: Option) -> Option { + match r.values.get(idx?) { + Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), + _ => None, + } +} + +/// Read a resolved column as a boolean flag (any non-zero integer is true). +fn flag_at(r: &RecoveredRecord, idx: Option) -> bool { + int_at(r, idx).is_some_and(|n| n != 0) +} + /// Build a map of key_id (XMPP stanza ID) → list of message row_ids. /// -/// Parses the CREATE TABLE DDL for the `message` table to determine the -/// zero-based values[] index of the `key_id` column (accounting for the -/// leading Null at index 0 for the INTEGER PRIMARY KEY). Returns an empty -/// map if the column doesn't exist in this schema. +/// Empty when the schema declares no `key_id` column. fn build_key_id_map( msg_records: &[&RecoveredRecord], - ddl_map: &HashMap, + key_id_idx: Option, ) -> HashMap> { - // Find the values[] index for key_id by parsing the DDL column list. - // The btree walker puts INTEGER PRIMARY KEY at values[0] as Null, - // so real column n (1-based in DDL order after _id) → values[n]. - let key_id_idx = match ddl_map.get("message") { - Some(ddl) => key_id_column_index(ddl), - None => return HashMap::new(), - }; - let Some(idx) = key_id_idx else { - return HashMap::new(); - }; - let mut map: HashMap> = HashMap::new(); + if key_id_idx.is_none() { + return map; + } for rec in msg_records { - let row_id = match rec.row_id { - Some(id) => id, - None => continue, + let Some(row_id) = rec.row_id else { + continue; }; - if let Some(SqlValue::Text(kid)) = rec.values.get(idx) { - if !kid.is_empty() { - map.entry(kid.clone()).or_default().push(row_id); - } + if let Some(kid) = text_at(rec, key_id_idx) { + map.entry(kid).or_default().push(row_id); } } map } -/// Parse a CREATE TABLE DDL string and return the 1-based values[] index -/// for the `key_id` column (i.e. column position counting from 1, since -/// index 0 is the INTEGER PRIMARY KEY alias Null). +/// jid table: row_id is `_id`; the JID string lives in `raw_string`. /// -/// Returns `None` if the column is not found. -fn key_id_column_index(ddl: &str) -> Option { - // Strip everything up to the first '(' and after the last ')'. - let start = ddl.find('(')?; - let end = ddl.rfind(')')?; - let cols_str = &ddl[start + 1..end]; - - // Split on commas (naive but sufficient for well-formed SQLite DDL). - // Column 0 in values[] is the INTEGER PRIMARY KEY (always first column). - // Real columns start at index 1 in values[]. - let mut idx = 0usize; - for col_def in cols_str.split(',') { - let col_def = col_def.trim(); - // Extract first token as column name (may be quoted with `backticks` or plain). - let col_name = col_def - .split_whitespace() - .next() - .unwrap_or("") - .trim_matches('`') - .trim_matches('"'); - if col_name.eq_ignore_ascii_case("key_id") { - return Some(idx); - } - idx += 1; - } - None -} - -/// jid table: row_id=_id, values[0]=Null(_id alias), values[1]=raw_string -fn build_jid_map(records: &[&RecoveredRecord]) -> HashMap { +/// Reading the neighbouring `user` column instead yields a bare phone number, +/// which looks plausible enough to hide the mistake. +fn build_jid_map(records: &[&RecoveredRecord], cols: &JidColumns) -> HashMap { let mut map = HashMap::new(); for r in records { - let id = match r.row_id { - Some(id) => id, - None => continue, + let Some(id) = r.row_id else { + continue; }; - let raw = match r.values.get(1) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, + let Some(raw) = text_at(r, cols.raw_string) else { + continue; }; map.insert(id, raw); } map } -/// chat table: row_id=_id, values[0]=Null, [1]=jid_row_id, [2]=subject -fn build_chats(records: &[&RecoveredRecord], jid_map: &HashMap) -> HashMap { +/// chat table: row_id is `_id`; `jid_row_id` links to `jid`. +fn build_chats( + records: &[&RecoveredRecord], + jid_map: &HashMap, + cols: &ChatColumns, +) -> HashMap { let mut map = HashMap::new(); for r in records { - let id = match r.row_id { - Some(id) => id, - None => continue, + let Some(id) = r.row_id else { + continue; }; - let jid_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, cols.jid_row_id) else { + continue; }; let jid = jid_map.get(&jid_row_id).cloned().unwrap_or_default(); - let subject = match r.values.get(2) { - Some(SqlValue::Text(s)) => Some(s.clone()), - _ => None, - }; let is_group = jid.ends_with("@g.us"); map.insert( id, Chat { id, jid, - name: subject, + name: text_at(r, cols.subject), is_group, messages: Vec::new(), - archived: matches!(r.values.get(3), Some(SqlValue::Int(1))), + archived: flag_at(r, cols.archived), }, ); } map } -/// message table: row_id=_id, values[0]=Null, [1]=chat_row_id, -/// [2]=sender_jid_row_id, [3]=from_me, [4]=timestamp, [5]=text_data, [6]=message_type, -/// [7]=media_mime_type, [8]=media_name, [9]=starred, [10]=edit_version (newer schemas) +/// Map one `message` record using resolved column positions. +/// +/// A record without a resolvable `chat_row_id` or `timestamp` is skipped rather +/// than attributed to a guessed column. fn record_to_message( r: &RecoveredRecord, jid_map: &HashMap, tz_offset_secs: i32, + cols: &MessageColumns, ) -> Option { - use cols::message as col; let id = r.row_id?; - let chat_id = match r.values.get(col::CHAT_ROW_ID)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let sender_jid = match r.values.get(col::SENDER_JID_ROW_ID) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; - let from_me = match r.values.get(col::FROM_ME) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let ts_ms = match r.values.get(col::TIMESTAMP)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let msg_type = match r.values.get(col::MESSAGE_TYPE) { - Some(SqlValue::Int(n)) => *n as i32, - _ => 0, - }; - let media_mime = match r.values.get(col::MEDIA_MIME_TYPE) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let media_name = match r.values.get(col::MEDIA_NAME) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let starred = matches!(r.values.get(col::STARRED), Some(SqlValue::Int(n)) if *n != 0); + let chat_id = int_at(r, cols.chat_row_id)?; + let sender_jid = int_at(r, cols.sender_jid_row_id).and_then(|n| jid_map.get(&n).cloned()); + let from_me = flag_at(r, cols.from_me); + let ts_ms = int_at(r, cols.timestamp)?; + let msg_type = int_at(r, cols.message_type).unwrap_or(0) as i32; + let media_mime = text_at(r, cols.media_mime_type); + let media_name = text_at(r, cols.media_name); + let starred = flag_at(r, cols.starred); // edit_version column added in newer schema versions. // 5 = deleted-for-me (local deletion only) // 7 = deleted-for-all (sender deleted from everyone's view) // Both cases override the content to Deleted regardless of msg_type or text_data. - let edit_version = match r.values.get(col::EDIT_VERSION) { - Some(SqlValue::Int(n)) => *n, - _ => 0, - }; - let text_data = match r.values.get(col::TEXT_DATA) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; + let edit_version = int_at(r, cols.edit_version).unwrap_or(0); + let text_data = text_at(r, cols.text_data); let content = if edit_version == 5 || edit_version == 7 { // Message deleted: override any msg_type or text content. // edit_version=5 → deleted-for-me; edit_version=7 → deleted-for-all. @@ -542,7 +552,12 @@ fn record_to_message( } else if msg_type == 53 || msg_type == 54 { // View-once image (53) or video (54) — media key/CDN URL may survive device deletion let mime = media_mime.unwrap_or_else(|| { - if msg_type == 54 { "video/mp4" } else { "image/jpeg" }.to_string() + if msg_type == 54 { + "video/mp4" + } else { + "image/jpeg" + } + .to_string() }); MessageContent::ViewOnce(MediaRef { file_path: media_name.unwrap_or_default(), @@ -600,50 +615,30 @@ fn record_to_message( }) } -/// call_log: row_id=_id, values[0]=Null, [1]=jid_row_id, [2]=from_me, -/// [3]=video_call, [4]=duration, [5]=timestamp, [6]=call_result, -/// [7]=call_row_id (optional grouping key), [8]=call_creator_device_jid_row_id (optional) +/// Map one `call_log` record using resolved column positions. /// -/// Returns `(CallRecord, call_row_id)` where call_row_id groups group-call participants. +/// Returns `(CallRecord, call_row_id)`, where `call_row_id` groups the +/// participants of one group call. Schemas that declare no such column yield +/// `None`, so unrelated calls are never merged on a neighbouring value — the +/// modern schema has no `call_row_id`, and reading `duration` in its place +/// collapses every pair of calls that happened to run the same length. fn record_to_call( r: &RecoveredRecord, jid_map: &HashMap, tz_offset_secs: i32, + cols: &CallLogColumns, ) -> Option<(CallRecord, Option)> { let id = r.row_id?; - let jid_row_id = match r.values.get(1)? { - SqlValue::Int(n) => *n, - _ => return None, - }; + let jid_row_id = int_at(r, cols.jid_row_id)?; let participant = jid_map.get(&jid_row_id).cloned().unwrap_or_default(); - let from_me = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let video = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let duration = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n as u32, - _ => 0, - }; - let ts_ms = match r.values.get(5)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let call_result = match r.values.get(6) { - Some(SqlValue::Int(n)) => CallResult::from(*n), - _ => CallResult::Unknown, - }; - let call_row_id = match r.values.get(7) { - Some(SqlValue::Int(n)) => Some(*n), - _ => None, - }; - let call_creator_device_jid = match r.values.get(8) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; + let from_me = flag_at(r, cols.from_me); + let video = flag_at(r, cols.video_call); + let duration = int_at(r, cols.duration).unwrap_or(0) as u32; + let ts_ms = int_at(r, cols.timestamp)?; + let call_result = int_at(r, cols.call_result).map_or(CallResult::Unknown, CallResult::from); + let call_row_id = int_at(r, cols.call_row_id); + let call_creator_device_jid = + int_at(r, cols.call_creator_device_jid_row_id).and_then(|n| jid_map.get(&n).cloned()); Some(( CallRecord { call_id: id, @@ -698,48 +693,37 @@ fn merge_group_calls(raw: Vec<(CallRecord, Option)>) -> Vec { // ── Quoted message support ─────────────────────────────────────────────────── -/// message_quoted: [0]=Null(_id), [1]=message_row_id, [2]=chat_row_id, -/// [3]=sender_jid_row_id, [4]=from_me, [5]=timestamp, [6]=text_data, [7]=message_type +/// message_quoted: the snapshot a reply keeps of the message it quotes. fn build_quoted_map( records: &[&RecoveredRecord], jid_map: &HashMap, tz_offset_secs: i32, + cols: &TableColumns, ) -> HashMap { + let msg_row_id_col = cols.get("message_row_id"); + let chat_row_id_col = cols.get("chat_row_id"); + let sender_col = cols.get("sender_jid_row_id"); + let from_me_col = cols.get("from_me"); + let ts_col = cols.get("timestamp"); + let text_col = cols.get("text_data"); + let type_col = cols.get("message_type"); + let mut map = HashMap::new(); for r in records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let chat_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let sender_jid = match r.values.get(3) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; - let from_me = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let ts_ms = match r.values.get(5) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(msg_row_id) = int_at(r, msg_row_id_col) else { + continue; }; - let msg_type = match r.values.get(7) { - Some(SqlValue::Int(n)) => *n as i32, - _ => 0, + let Some(chat_id) = int_at(r, chat_row_id_col) else { + continue; }; - let content = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => MessageContent::Text(s.clone()), - _ => { - if msg_type == 0 { - MessageContent::Deleted - } else { - MessageContent::Unknown(msg_type) - } - } + let sender_jid = int_at(r, sender_col).and_then(|n| jid_map.get(&n).cloned()); + let from_me = flag_at(r, from_me_col); + let ts_ms = int_at(r, ts_col).unwrap_or(0); + let msg_type = int_at(r, type_col).unwrap_or(0) as i32; + let content = match text_at(r, text_col) { + Some(text) => MessageContent::Text(text), + None if msg_type == 0 => MessageContent::Deleted, + None => MessageContent::Unknown(msg_type), }; map.insert( @@ -769,31 +753,30 @@ fn build_quoted_map( // ── Group participant events ───────────────────────────────────────────────── -/// group_participant_user: [0]=Null(_id), [1]=group_jid_row_id, [2]=jid_row_id, -/// [3]=user_action, [4]=action_ts, [5]=actor_jid_row_id +/// group_participant_user: membership changes for group chats. fn build_group_participant_events( records: &[&RecoveredRecord], jid_map: &HashMap, tz_offset_secs: i32, + cols: &TableColumns, ) -> Vec { + let group_col = cols.get("group_jid_row_id"); + let jid_col = cols.get("jid_row_id"); + let action_col = cols.get("user_action"); + let action_ts_col = cols.get("action_ts"); + let mut events = Vec::new(); for r in records { - let group_jid_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let jid_row_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(group_jid_row_id) = int_at(r, group_col) else { + continue; }; - let user_action = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, jid_col) else { + continue; }; - let action_ts = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(user_action) = int_at(r, action_col) else { + continue; }; + let action_ts = int_at(r, action_ts_col).unwrap_or(0); let action = match user_action { 0 => ParticipantAction::Added, 1 => ParticipantAction::Removed, @@ -820,16 +803,17 @@ fn build_group_participant_events( /// Used to recover the original text of deleted/tombstone messages (msg_type=15) /// that were later quoted by another message. The quoting message preserves /// the original text in message_quoted.text_data. -fn build_ghost_map(records: &[&RecoveredRecord]) -> HashMap { +fn build_ghost_map(records: &[&RecoveredRecord], cols: &TableColumns) -> HashMap { + let msg_row_id_col = cols.get("message_row_id"); + let text_col = cols.get("text_data"); + let mut map = HashMap::new(); for r in records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, msg_row_id_col) else { + continue; }; - let text = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => s.clone(), - _ => continue, + let Some(text) = text_at(r, text_col) else { + continue; }; map.insert(msg_row_id, text); } @@ -839,28 +823,27 @@ fn build_ghost_map(records: &[&RecoveredRecord]) -> HashMap { // ── wa.db contact extraction ───────────────────────────────────────────────── /// Extract contacts from wa.db bytes using the forensic B-tree walker. -/// wa_contacts table: [0]=Null(_id), [1]=jid, [2]=display_name, [3]=status, [4]=number +/// +/// `wa_contacts` column positions are resolved by name, as for msgstore.db. pub fn extract_contacts(wa_db_bytes: &[u8]) -> Result> { - let engine = ForensicEngine::new(wa_db_bytes, None) - .context("failed to open wa.db")?; + let engine = ForensicEngine::new(wa_db_bytes, None).context("failed to open wa.db")?; + let schema_cols = SchemaColumns::from_ddl_map(&engine.table_ddl()); + let cols = schema_cols.table("wa_contacts"); + let jid_col = cols.get("jid"); + let display_name_col = cols.get("display_name"); + let number_col = cols.get("number"); + let records = engine.recover_layer1().context("wa.db layer 1 recovery")?; let by_table = partition_by_table(&records); let contact_records = tbl(&by_table, "wa_contacts"); let mut contacts = Vec::new(); for r in contact_records { - let jid = match r.values.get(1) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, - }; - let display_name = match r.values.get(2) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let phone_number = match r.values.get(4) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, + let Some(jid) = text_at(r, jid_col) else { + continue; }; + let display_name = text_at(r, display_name_col); + let phone_number = text_at(r, number_col); contacts.push(Contact { jid, display_name, @@ -962,7 +945,10 @@ mod tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); assert_eq!(result.calls.len(), 1); // Fixture inserts call_result=1 (Connected) - assert_eq!(result.calls[0].call_result, chat4n6_plugin_api::CallResult::Connected); + assert_eq!( + result.calls[0].call_result, + chat4n6_plugin_api::CallResult::Connected + ); } // ── E5: quoted message tests ───────────────────────────────────────── @@ -1021,7 +1007,10 @@ mod tests { fn test_extract_contacts_from_wa_db() { let wa_db = make_wa_db(); let contacts = extract_contacts(&wa_db).unwrap(); - assert!(contacts.len() >= 2, "should have at least 2 contacts with names"); + assert!( + contacts.len() >= 2, + "should have at least 2 contacts with names" + ); let alice = contacts .iter() .find(|c| c.jid == "4155550100@s.whatsapp.net") @@ -1108,9 +1097,16 @@ mod tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let chat1 = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); // Message 6 has message_type=15 (tombstone, no message_quoted entry) → Deleted - let msg6 = chat1.messages.iter().find(|m| m.id == 6).expect("msg 6 (tombstone)"); + let msg6 = chat1 + .messages + .iter() + .find(|m| m.id == 6) + .expect("msg 6 (tombstone)"); assert!( - matches!(&msg6.content, MessageContent::Deleted | MessageContent::GhostRecovered(_)), + matches!( + &msg6.content, + MessageContent::Deleted | MessageContent::GhostRecovered(_) + ), "msg_type=15 tombstone should produce Deleted or GhostRecovered, got {:?}", msg6.content ); @@ -1133,14 +1129,18 @@ mod tests { #[test] fn schema_version_is_read_from_pragma_user_version() { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch("PRAGMA user_version = 215;").unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - assert_eq!(result.schema_version, 215, - "schema_version must be read from PRAGMA user_version, not hardcoded"); + assert_eq!( + result.schema_version, 215, + "schema_version must be read from PRAGMA user_version, not hardcoded" + ); } // ── C8: ghost message recovery from message_quoted ─────────────────── @@ -1148,30 +1148,39 @@ mod tests { #[test] fn ghost_message_recovered_from_message_quoted() { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch( "INSERT INTO message_quoted VALUES (99, 6, 1, NULL, 1, 1710513600000, 'Secret deleted message', 0, NULL, NULL);", ).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let ghost = result.chats.iter() + let ghost = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .find(|m| matches!(&m.content, MessageContent::GhostRecovered(_))); - assert!(ghost.is_some(), "msg_type=15 with message_quoted entry must produce GhostRecovered"); + assert!( + ghost.is_some(), + "msg_type=15 with message_quoted entry must produce GhostRecovered" + ); if let Some(MessageContent::GhostRecovered(text)) = ghost.map(|m| &m.content) { - assert!(text.contains("Secret deleted message"), - "GhostRecovered text must contain the quoted text_data"); + assert!( + text.contains("Secret deleted message"), + "GhostRecovered text must contain the quoted text_data" + ); } } #[test] fn test_is_media_type_helper() { - assert!(is_media_type(1)); // image - assert!(is_media_type(2)); // audio - assert!(is_media_type(3)); // video - assert!(is_media_type(8)); // document + assert!(is_media_type(1)); // image + assert!(is_media_type(2)); // audio + assert!(is_media_type(3)); // video + assert!(is_media_type(8)); // document assert!(is_media_type(13)); // gif assert!(is_media_type(20)); // sticker assert!(!is_media_type(0)); // text @@ -1216,7 +1225,8 @@ mod tests { INSERT INTO call_log VALUES (1, 1, 0, 0, 90, 1710513500000, 1, NULL, 2); "#).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1273,7 +1283,8 @@ mod tests { INSERT INTO call_log VALUES (3, 1, 1, 0, 60, 1710513500000, 1, NULL, NULL); "#).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1282,7 +1293,11 @@ mod tests { let db = make_msgstore_with_group_call(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let group_calls: Vec<_> = result.calls.iter().filter(|c| c.group_call).collect(); - assert_eq!(group_calls.len(), 1, "two rows with same call_row_id → 1 merged record"); + assert_eq!( + group_calls.len(), + 1, + "two rows with same call_row_id → 1 merged record" + ); assert_eq!(group_calls[0].participants.len(), 2); } @@ -1290,8 +1305,14 @@ mod tests { fn test_group_call_participants_contain_both_jids() { let db = make_msgstore_with_group_call(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let gc = result.calls.iter().find(|c| c.group_call).expect("no group call"); - assert!(gc.participants.contains(&"alice@s.whatsapp.net".to_string())); + let gc = result + .calls + .iter() + .find(|c| c.group_call) + .expect("no group call"); + assert!(gc + .participants + .contains(&"alice@s.whatsapp.net".to_string())); assert!(gc.participants.contains(&"bob@s.whatsapp.net".to_string())); } @@ -1487,7 +1508,10 @@ mod features_i1_i8_tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let chat1 = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); let msg2 = chat1.messages.iter().find(|m| m.id == 2).expect("msg 2"); - assert!(msg2.starred, "msg 2 starred=1 in DB must produce Message.starred=true"); + assert!( + msg2.starred, + "msg 2 starred=1 in DB must produce Message.starred=true" + ); let msg1 = chat1.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!(!msg1.starred, "msg 1 starred=0 must be false"); } @@ -1519,7 +1543,10 @@ mod features_i1_i8_tests { Some(8), "msg 5 forward_score should be Some(8)" ); - assert!(msg5.is_forwarded, "msg 5 with forward_score=8 must be is_forwarded=true"); + assert!( + msg5.is_forwarded, + "msg 5 with forward_score=8 must be is_forwarded=true" + ); } } @@ -1530,10 +1557,12 @@ mod proptest_redo_tests { fn make_db_with_sql(extra_sql: &str) -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch(extra_sql).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1545,11 +1574,16 @@ mod proptest_redo_tests { VALUES (1, 1, 0, 1710514000000, NULL, 53, 'image/jpeg', 'Media/WhatsApp View Once/VIEW-001.jpg');", ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let view_once_count = result.chats.iter() + let view_once_count = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .filter(|m| matches!(&m.content, MessageContent::ViewOnce(_))) .count(); - assert_eq!(view_once_count, 1, "msg_type 53 must produce ViewOnce, not Unknown"); + assert_eq!( + view_once_count, 1, + "msg_type 53 must produce ViewOnce, not Unknown" + ); } #[test] @@ -1560,7 +1594,9 @@ mod proptest_redo_tests { VALUES (1, 1, 0, 1710514100000, NULL, 54, 'video/mp4', 'Media/WhatsApp View Once/VIEW-002.mp4');", ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let view_once_count = result.chats.iter() + let view_once_count = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .filter(|m| matches!(&m.content, MessageContent::ViewOnce(_))) .count(); @@ -1576,7 +1612,10 @@ mod proptest_redo_tests { ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let archived_chats: Vec<_> = result.chats.iter().filter(|c| c.archived).collect(); - assert!(!archived_chats.is_empty(), "archived=1 in DB must produce Chat.archived=true"); + assert!( + !archived_chats.is_empty(), + "archived=1 in DB must produce Chat.archived=true" + ); } #[test] @@ -1691,7 +1730,8 @@ mod media_type_extended_tests { INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, NULL, {}, NULL, NULL, 0, 0, NULL); "#, msg_type_val)).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1703,7 +1743,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(mr) if mr.mime_type.contains("geo")), - "msg_type=5 (location) should produce Media with geo mime, got: {:?}", msg.content + "msg_type=5 (location) should produce Media with geo mime, got: {:?}", + msg.content ); } @@ -1715,7 +1756,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(_)), - "msg_type=42 (live location) should produce Media, got: {:?}", msg.content + "msg_type=42 (live location) should produce Media, got: {:?}", + msg.content ); } @@ -1727,7 +1769,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(mr) if mr.mime_type == "text/vcard"), - "msg_type=64 (contact card) should produce Media with text/vcard, got: {:?}", msg.content + "msg_type=64 (contact card) should produce Media with text/vcard, got: {:?}", + msg.content ); } @@ -1754,7 +1797,8 @@ mod streaming_extraction_tests { fn make_multi_message_db(msg_count: usize) -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); @@ -1774,15 +1818,23 @@ mod streaming_extraction_tests { ); INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); INSERT INTO chat VALUES (1, 1, NULL); - "#).unwrap(); + "#, + ) + .unwrap(); for i in 1..=msg_count { conn.execute( "INSERT INTO message VALUES (?, 1, NULL, 0, ?, ?, 0, NULL, NULL, 0, 0, NULL)", - rusqlite::params![i as i64, (1710000000000i64 + i as i64 * 1000), format!("msg {}", i)], - ).unwrap(); + rusqlite::params![ + i as i64, + (1710000000000i64 + i as i64 * 1000), + format!("msg {}", i) + ], + ) + .unwrap(); } let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1793,7 +1845,8 @@ mod streaming_extraction_tests { let count_clone = Arc::clone(&count); extract_streaming(&db, 0, SchemaVersion::Modern, |_msg| { count_clone.fetch_add(1, Ordering::SeqCst); - }).expect("extract_streaming should succeed"); + }) + .expect("extract_streaming should succeed"); assert_eq!( count.load(Ordering::SeqCst), 10, @@ -1807,7 +1860,9 @@ mod streaming_extraction_tests { // Batch extraction let batch = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let mut batch_ids: Vec = batch.chats.iter() + let mut batch_ids: Vec = batch + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); batch_ids.sort(); @@ -1816,7 +1871,8 @@ mod streaming_extraction_tests { let mut streaming_ids = Vec::new(); extract_streaming(&db, 0, SchemaVersion::Modern, |msg| { streaming_ids.push(msg.id); - }).expect("extract_streaming should succeed"); + }) + .expect("extract_streaming should succeed"); streaming_ids.sort(); assert_eq!( @@ -1830,13 +1886,17 @@ mod streaming_extraction_tests { let db = make_multi_message_db(20); let serial = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let mut serial_ids: Vec = serial.chats.iter() + let mut serial_ids: Vec = serial + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); serial_ids.sort(); let parallel = extract_parallel(&db, 0, SchemaVersion::Modern).unwrap(); - let mut parallel_ids: Vec = parallel.chats.iter() + let mut parallel_ids: Vec = parallel + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); parallel_ids.sort(); @@ -1854,9 +1914,11 @@ mod fts5_tests { fn make_fts5_db() -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/fts5_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/fts5_schema.sql")) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1865,10 +1927,14 @@ mod fts5_tests { let db = make_fts5_db(); let fragments = extract_fts5_content(&db).unwrap(); // Should find at least one _content table with text fragments - let content_tables: Vec<_> = fragments.keys() + let content_tables: Vec<_> = fragments + .keys() .filter(|k| k.ends_with("_content")) .collect(); - assert!(!content_tables.is_empty(), "must find at least one FTS5 _content table"); + assert!( + !content_tables.is_empty(), + "must find at least one FTS5 _content table" + ); let all_texts: Vec<_> = fragments.values().flatten().collect(); assert!( all_texts.iter().any(|t| t.contains("forensics")), @@ -1961,7 +2027,8 @@ mod edit_version_tests { INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, 'original text', 0, NULL, NULL, 0, {}); "#, edit_version_val, edit_version_val)).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -2043,7 +2110,11 @@ mod tbl_helper_tests { let mut map: HashMap> = HashMap::new(); map.insert("message".to_string(), vec![&record]); let result = tbl(&map, "message"); - assert_eq!(result.len(), 1, "known key must return the inserted records"); + assert_eq!( + result.len(), + 1, + "known key must return the inserted records" + ); } } @@ -2061,7 +2132,8 @@ mod truncated_record_tests { let conn = rusqlite::Connection::open_in_memory().unwrap(); // Deliberately omit media_mime_type, media_name, starred, edit_version // so that column indices 7, 8, 9, 10 are absent from the physical row. - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 100; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); @@ -2077,9 +2149,12 @@ mod truncated_record_tests { INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); INSERT INTO chat VALUES (1, 1, NULL); INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, 'hello short', 0); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -2105,3 +2180,379 @@ mod truncated_record_tests { ); } } + +// ── T1: extraction against the real 2023-era device schema ─────────────────── + +/// These exercise the same code path as the simplified fixtures, but against +/// the `CREATE TABLE` shapes read out of a real device. Every assertion here +/// is one the hardcoded-ordinal reader gets wrong: it lands on +/// `sender_jid_row_id` instead of `timestamp`, `user` instead of `raw_string`, +/// `hidden` instead of `subject`, and `duration` instead of `call_row_id`. +#[cfg(test)] +mod real_schema_tests { + use super::*; + use chrono::Datelike; + + fn make_real_msgstore() -> Vec { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/real_modern_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + std::fs::read(tmp.path()).unwrap() + } + + fn all_messages(result: &ExtractionResult) -> Vec<&Message> { + result + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .collect() + } + + #[test] + fn message_timestamps_are_2022_not_epoch_zero() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + assert_eq!(msgs.len(), 3, "all three messages must be extracted"); + for m in &msgs { + assert_eq!( + m.timestamp.utc.year(), + 2022, + "message {} timestamp resolved to {} — the reader is on the wrong column", + m.id, + m.timestamp.utc + ); + } + } + + #[test] + fn message_text_comes_from_text_data() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let texts: Vec<&str> = all_messages(&result) + .iter() + .filter_map(|m| match &m.content { + MessageContent::Text(s) => Some(s.as_str()), + _ => None, + }) + .collect(); + assert!( + texts.contains(&"Incoming real-schema message"), + "text_data must be read by name, got: {texts:?}" + ); + } + + #[test] + fn sender_resolves_via_jid_raw_string_not_bare_user() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let received = all_messages(&result) + .into_iter() + .find(|m| m.id == 2) + .expect("message 2"); + assert_eq!( + received.sender_jid.as_deref(), + Some("4155550100@s.whatsapp.net"), + "sender must come from jid.raw_string, not the bare `user` column" + ); + } + + #[test] + fn from_me_is_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + let outgoing = msgs.iter().find(|m| m.id == 1).expect("message 1"); + let incoming = msgs.iter().find(|m| m.id == 2).expect("message 2"); + assert!(outgoing.from_me, "message 1 has from_me = 1"); + assert!(!incoming.from_me, "message 2 has from_me = 0"); + } + + #[test] + fn starred_flag_is_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + assert!( + msgs.iter().find(|m| m.id == 2).expect("message 2").starred, + "message 2 has starred = 1 at DDL position 16" + ); + assert!( + !msgs.iter().find(|m| m.id == 1).expect("message 1").starred, + "message 1 has starred = 0" + ); + } + + #[test] + fn chat_subject_and_archived_are_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let group = result.chats.iter().find(|c| c.id == 2).expect("chat 2"); + assert_eq!( + group.name.as_deref(), + Some("Real Schema Group"), + "subject sits at position 3; position 2 is `hidden`" + ); + assert!(group.archived, "chat 2 has archived = 1 at position 10"); + let direct = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); + assert!(!direct.archived, "chat 1 has archived = 0"); + } + + #[test] + fn chat_jid_resolves_and_group_is_flagged() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let group = result.chats.iter().find(|c| c.id == 2).expect("chat 2"); + assert_eq!(group.jid, "120363001234567890@g.us"); + assert!(group.is_group); + } + + #[test] + fn calls_are_not_collapsed_by_a_mistaken_grouping_key() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + assert_eq!( + result.calls.len(), + 3, + "three call_log rows must yield three records; the real schema has no \ + call_row_id column, so nothing may be merged" + ); + assert!( + result.calls.iter().all(|c| !c.group_call), + "no call may be flagged as a group call" + ); + } + + #[test] + fn call_duration_and_video_flag_are_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let video = result + .calls + .iter() + .find(|c| c.call_id == 2) + .expect("call 2"); + assert!(video.video, "call 2 has video_call = 1 at position 6"); + assert_eq!(video.duration_secs, 42, "duration sits at position 7"); + let voice = result + .calls + .iter() + .find(|c| c.call_id == 1) + .expect("call 1"); + assert!(!voice.video); + assert_eq!(voice.duration_secs, 137); + } + + fn make_shuffled_msgstore() -> Vec { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/shuffled_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + std::fs::read(tmp.path()).unwrap() + } + + /// Two databases that differ only in column declaration order must extract + /// identically. Any ordinal creeping back into the extractor breaks this. + #[test] + fn column_order_does_not_change_the_extraction() { + let straight = extract_from_msgstore(&make_real_msgstore(), 0, SchemaVersion::Modern) + .expect("real-order fixture"); + let shuffled = extract_from_msgstore(&make_shuffled_msgstore(), 0, SchemaVersion::Modern) + .expect("permuted-order fixture"); + + let summarise = |r: &ExtractionResult| { + let mut chats: Vec<(i64, String, Option, bool)> = r + .chats + .iter() + .map(|c| (c.id, c.jid.clone(), c.name.clone(), c.archived)) + .collect(); + chats.sort(); + let mut msgs: Vec<(i64, i64, Option, bool, i64, bool)> = r + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .map(|m| { + ( + m.id, + m.chat_id, + m.sender_jid.clone(), + m.from_me, + m.timestamp.utc.timestamp_millis(), + m.starred, + ) + }) + .collect(); + msgs.sort(); + let mut calls: Vec<(i64, bool, u32, i64, bool)> = r + .calls + .iter() + .map(|c| { + ( + c.call_id, + c.video, + c.duration_secs, + c.timestamp.utc.timestamp_millis(), + c.group_call, + ) + }) + .collect(); + calls.sort(); + (chats, msgs, calls) + }; + + assert_eq!( + summarise(&straight), + summarise(&shuffled), + "extraction must depend on column names, not on their declaration order" + ); + // Guard against the comparison passing because both sides are empty. + assert_eq!(summarise(&straight).1.len(), 3); + } + + /// A legacy-generation database has no `message` table at all. Reporting + /// zero messages would be indistinguishable from a clean device. + #[test] + fn a_legacy_schema_database_is_refused_rather_than_reported_empty() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/legacy_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Legacy) + .expect_err("a legacy database must not yield a silently empty report"); + let msg = format!("{err:#}"); + assert!( + msg.contains("messages"), + "error must name the legacy table it found: {msg}" + ); + assert!( + msg.contains("message") && msg.contains("legacy"), + "error must say which generation it recognised: {msg}" + ); + } + + /// A database with neither table is likewise a refusal, not an empty report. + #[test] + fn a_database_with_no_message_table_is_refused() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE TABLE unrelated (_id INTEGER PRIMARY KEY, x TEXT);") + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("a database with no message table is not a msgstore.db"); + assert!( + format!("{err:#}").contains("message"), + "error must name what it looked for: {err:#}" + ); + } + + /// A database whose `message.timestamp` column does not hold epoch + /// milliseconds is a bootstrap failure: extraction must stop rather than + /// emit a report whose every date is wrong. + #[test] + fn implausible_timestamps_abort_extraction_naming_the_values() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); + CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); + CREATE TABLE message (_id INTEGER PRIMARY KEY, chat_row_id INTEGER NOT NULL, + sender_jid_row_id INTEGER, from_me INTEGER, timestamp INTEGER, + text_data TEXT, message_type INTEGER); + INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); + INSERT INTO chat VALUES (1, 1, NULL); + INSERT INTO message VALUES (1, 1, NULL, 1, 5243, 'a', 0); + INSERT INTO message VALUES (2, 1, NULL, 1, 5244, 'b', 0); + INSERT INTO message VALUES (3, 1, NULL, 1, 5245, 'c', 0);", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("must refuse to report on timestamps that are not epoch millis"); + let msg = format!("{err:#}"); + assert!(msg.contains("timestamp"), "must name the column: {msg}"); + assert!(msg.contains("5243"), "must quote an offending value: {msg}"); + } + + /// A schema that declares no `timestamp` at all must be named, not silently + /// treated as a database with no messages. + #[test] + fn message_table_without_a_timestamp_column_aborts_extraction() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); + CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); + CREATE TABLE message (_id INTEGER PRIMARY KEY, chat_row_id INTEGER NOT NULL, + text_data TEXT); + INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); + INSERT INTO chat VALUES (1, 1, NULL); + INSERT INTO message VALUES (1, 1, 'orphan');", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("an unrecognisable message schema must be reported"); + assert!( + format!("{err:#}").contains("timestamp"), + "error must name the missing column: {err:#}" + ); + } + + /// The invariant every resolved ordinal rests on: SQLite writes the + /// `INTEGER PRIMARY KEY` column as a NULL **at its declared position**, so + /// `values[i]` is the column at DDL position `i` — even when the rowid + /// alias is not declared first. + #[test] + fn rowid_alias_is_null_at_its_declared_position() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t (a TEXT, _id INTEGER PRIMARY KEY, c INTEGER); + INSERT INTO t VALUES ('first', 7, 42);", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let engine = ForensicEngine::new(&db, None).unwrap(); + let records = engine.recover_layer1().unwrap(); + let r = records.iter().find(|r| r.table == "t").expect("table t"); + + assert_eq!(r.row_id, Some(7), "rowid comes from the cell header"); + assert!( + matches!(r.values.first(), Some(SqlValue::Text(s)) if s == "first"), + "position 0 is the declared first column, got {:?}", + r.values.first() + ); + assert!( + matches!(r.values.get(1), Some(SqlValue::Null)), + "the rowid alias reads as Null at its own declared position, got {:?}", + r.values.get(1) + ); + assert!( + matches!(r.values.get(2), Some(SqlValue::Int(42))), + "position 2 is the declared third column, got {:?}", + r.values.get(2) + ); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/src/lib.rs b/crates/plugins/chat4n6-whatsapp/src/lib.rs index 600607b..569ea4d 100644 --- a/crates/plugins/chat4n6-whatsapp/src/lib.rs +++ b/crates/plugins/chat4n6-whatsapp/src/lib.rs @@ -2,25 +2,27 @@ pub mod anti_forensics; +pub mod album; pub mod cdn; +pub mod columns; pub mod contact_report; pub mod decrypt; pub mod extractor; -pub mod orphaned_media; -pub mod platform; -pub mod system_event; -pub mod schema; -pub mod timezone; -pub mod album; -pub mod poll; pub mod group_metadata; +pub mod link; pub mod location; pub mod mention; +pub mod orphaned_media; pub mod pin; +pub mod platform; +pub mod poll; +pub mod schema; +pub mod schema_gate; pub mod status; -pub mod link; +pub mod system_event; +pub mod timezone; -use crate::extractor::{extract_contacts, extract_from_msgstore, build_contact_names}; +use crate::extractor::{build_contact_names, extract_contacts, extract_from_msgstore}; use crate::schema::detect_schema_version; use anyhow::{Context, Result}; use chat4n6_plugin_api::{ExtractionResult, ForensicFs, ForensicPlugin}; @@ -110,13 +112,20 @@ impl ForensicPlugin for WhatsAppPlugin { let tz = local_offset_seconds.unwrap_or_else(|| detect_timezone(fs)); - // Detect schema version + // Detect the schema generation from the tables the database actually + // declares. PRAGMA user_version is app-defined (a real 2023-era device + // reports 1), so it cannot stand in for the table list. let user_version = if db_bytes.len() >= 64 { u32::from_be_bytes([db_bytes[60], db_bytes[61], db_bytes[62], db_bytes[63]]) } else { 0 }; - let schema = detect_schema_version(user_version, &[]); + let table_names: Vec = + chat4n6_sqlite_forensics::db::ForensicEngine::new(&db_bytes, None) + .map(|e| e.table_ddl().into_keys().collect()) + .unwrap_or_default(); + let table_refs: Vec<&str> = table_names.iter().map(String::as_str).collect(); + let schema = detect_schema_version(user_version, &table_refs); let mut result = extract_from_msgstore(&db_bytes, tz, schema)?; @@ -156,10 +165,9 @@ fn detect_timezone(fs: &dyn ForensicFs) -> i32 { impl WhatsAppPlugin { fn try_decrypt(&self, bytes: &[u8]) -> Result> { - let key = self - .key - .as_ref() - .ok_or_else(|| anyhow::anyhow!("database is not SQLite and no decryption key provided"))?; + let key = self.key.as_ref().ok_or_else(|| { + anyhow::anyhow!("database is not SQLite and no decryption key provided") + })?; // Try crypt15 first, then crypt14 if let Ok(plain) = decrypt::decrypt_db(bytes, key, decrypt::CryptVersion::Crypt15) { return Ok(plain); @@ -299,7 +307,8 @@ mod tests { fn db_to_bytes(conn: &rusqlite::Connection) -> Vec { let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -332,7 +341,8 @@ mod tests { jid TEXT, display_name TEXT, status TEXT, number TEXT); INSERT INTO wa_contacts VALUES (1, '{jid}', '{display_name}', NULL, NULL); " - )).unwrap(); + )) + .unwrap(); db_to_bytes(&conn) } @@ -340,9 +350,7 @@ mod tests { fn wa_db_contact_names_applied_to_chats() { let msgstore = minimal_msgstore("alice@s.whatsapp.net"); let wa_db = minimal_wa_db("alice@s.whatsapp.net", "Alice"); - let fs = MockFs::new() - .add(DB_PATH, msgstore) - .add(WA_DB_PATH, wa_db); + let fs = MockFs::new().add(DB_PATH, msgstore).add(WA_DB_PATH, wa_db); let result = WhatsAppPlugin::new().extract(&fs, None).unwrap(); let chat = result.chats.first().expect("expected at least one chat"); assert_eq!( @@ -355,9 +363,10 @@ mod tests { #[test] fn timezone_autodetect_from_property_file() { let msgstore = minimal_msgstore("test@s.whatsapp.net"); - let fs = MockFs::new() - .add(DB_PATH, msgstore) - .add("data/property/persist.sys.timezone", b"Asia/Manila\n".to_vec()); + let fs = MockFs::new().add(DB_PATH, msgstore).add( + "data/property/persist.sys.timezone", + b"Asia/Manila\n".to_vec(), + ); let result = WhatsAppPlugin::new().extract(&fs, None).unwrap(); assert_eq!( result.timezone_offset_seconds, diff --git a/crates/plugins/chat4n6-whatsapp/src/schema.rs b/crates/plugins/chat4n6-whatsapp/src/schema.rs index 35d3042..fa30dd3 100644 --- a/crates/plugins/chat4n6-whatsapp/src/schema.rs +++ b/crates/plugins/chat4n6-whatsapp/src/schema.rs @@ -4,12 +4,15 @@ pub enum SchemaVersion { Modern, } -/// Detect the WhatsApp msgstore.db schema version. -/// - Modern: user_version >= 100 OR has both "message" and "jid" tables +/// Detect the WhatsApp msgstore.db schema version from the tables present. +/// - Modern: has both "message" and "jid" tables /// - Legacy: otherwise ("messages" + "wa_contacts" era) -pub fn detect_schema_version(user_version: u32, tables: &[&str]) -> SchemaVersion { - let has_modern = tables.contains(&"message") && tables.contains(&"jid"); - if has_modern || user_version >= 100 { +/// +/// `user_version` carries no schema-generation signal: a real 2023-era device +/// reports 1, so a `>= 100` test classified legacy databases as modern and +/// modern ones as legacy depending on nothing but the app's own counter. +pub fn detect_schema_version(_user_version: u32, tables: &[&str]) -> SchemaVersion { + if tables.contains(&"message") && tables.contains(&"jid") { SchemaVersion::Modern } else { SchemaVersion::Legacy @@ -26,16 +29,16 @@ pub fn detect_schema_version(user_version: u32, tables: &[&str]) -> SchemaVersio /// 15 = Deleted (deleted-for-all tombstone placeholder; NOT "ProductSingle") pub fn msg_type_label(n: i32) -> &'static str { match n { - 0 => "Text", - 1 => "Image", - 2 => "Audio", - 3 => "Video", - 4 => "Contact", - 5 => "Location", - 6 => "MediaOmitted", - 7 => "StatusUpdate", - 8 => "VoiceNote", - 9 => "Document", + 0 => "Text", + 1 => "Image", + 2 => "Audio", + 3 => "Video", + 4 => "Contact", + 5 => "Location", + 6 => "MediaOmitted", + 7 => "StatusUpdate", + 8 => "VoiceNote", + 9 => "Document", 10 => "MissedVoiceCall", 11 => "MissedVideoCall", 12 => "MediaCiphertextUnknown", @@ -44,7 +47,7 @@ pub fn msg_type_label(n: i32) -> &'static str { 15 => "Deleted", 16 => "LiveLocation", 20 => "Sticker", - _ => "Unknown", + _ => "Unknown", } } @@ -68,54 +71,28 @@ pub fn default_mime_for_type(msg_type: i32) -> &'static str { } } -/// Named column index constants for msgstore.db tables. -/// -/// The btree walker stores INTEGER PRIMARY KEY as SqlValue::Null at values[0]. -/// Real column data starts at values[1] (matching the DDL column order after _id). -pub mod cols { - /// message table (modern schema, post-2021) - pub mod message { - pub const CHAT_ROW_ID: usize = 1; - pub const SENDER_JID_ROW_ID: usize = 2; - pub const FROM_ME: usize = 3; - pub const TIMESTAMP: usize = 4; - pub const TEXT_DATA: usize = 5; - pub const MESSAGE_TYPE: usize = 6; - pub const MEDIA_MIME_TYPE: usize = 7; - pub const MEDIA_NAME: usize = 8; - pub const STARRED: usize = 9; - pub const EDIT_VERSION: usize = 10; - } - - /// jid table - pub mod jid { - pub const RAW_STRING: usize = 1; - } - - /// chat table - pub mod chat { - pub const JID_ROW_ID: usize = 1; - pub const SUBJECT: usize = 2; - pub const ARCHIVED: usize = 3; - } - - /// call_log table - pub mod call_log { - pub const JID_ROW_ID: usize = 1; - pub const FROM_ME: usize = 2; - pub const VIDEO_CALL: usize = 3; - pub const DURATION: usize = 4; - pub const TIMESTAMP: usize = 5; - pub const CALL_RESULT: usize = 6; - pub const CALL_ROW_ID: usize = 7; - pub const CALL_CREATOR_DEVICE_JID_ROW_ID: usize = 8; - } -} - #[cfg(test)] mod tests { use super::*; + /// Classification of the committed legacy fixture, so the legacy branch is + /// pinned to a concrete schema rather than a hand-written table list. + #[test] + fn legacy_fixture_is_classified_legacy() { + let ddl = include_str!("../tests/fixtures/legacy_schema.sql"); + let tables: Vec<&str> = ddl + .lines() + .filter_map(|l| l.trim().strip_prefix("CREATE TABLE ")) + .filter_map(|l| l.split_whitespace().next()) + .filter_map(|t| t.split('(').next()) + .collect(); + assert!( + tables.contains(&"messages"), + "fixture must declare `messages`" + ); + assert_eq!(detect_schema_version(1, &tables), SchemaVersion::Legacy); + } + #[test] fn test_schema_legacy_detection() { assert_eq!( @@ -124,14 +101,27 @@ mod tests { ); } + /// A real 2023-era device reports `user_version = 1`, so the version number + /// carries no schema-generation signal. Table presence is the reliable one. #[test] - fn test_schema_modern_by_user_version() { + fn real_device_user_version_1_with_modern_tables_is_modern() { assert_eq!( - detect_schema_version(200, &["messages"]), + detect_schema_version(1, &["message", "jid", "chat", "call_log"]), SchemaVersion::Modern ); } + /// `user_version` alone must not promote a legacy-table database to Modern: + /// the number is app-defined and unrelated to the schema generation. + #[test] + fn high_user_version_alone_does_not_imply_modern() { + assert_eq!( + detect_schema_version(200, &["messages", "wa_contacts"]), + SchemaVersion::Legacy, + "classification must follow the tables that are actually present" + ); + } + #[test] fn test_schema_modern_by_tables() { assert_eq!( diff --git a/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs new file mode 100644 index 0000000..ed0aec9 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs @@ -0,0 +1,302 @@ +//! Bootstrap validation for the resolved `message` column map. +//! +//! Resolving a column by name is only as good as the DDL it was read from. If +//! the map is wrong the extractor still produces a structurally complete +//! report — one that is indistinguishable from a correct one until somebody +//! notices every message is dated 1970. That is the worst failure class a +//! forensic tool has, so the map is checked against the data before any of it +//! is interpreted, and a failed check is an error rather than a report. + +use crate::columns::MessageColumns; +use anyhow::{bail, Result}; +use chat4n6_sqlite_forensics::record::{RecoveredRecord, SqlValue}; + +/// 2009-01-01T00:00:00Z in epoch milliseconds. +/// +/// WhatsApp shipped in 2009, so no genuine message timestamp precedes it. +pub const EARLIEST_PLAUSIBLE_MS: i64 = 1_230_768_000_000; + +/// Grace added to the acquisition time before a timestamp counts as future-dated. +pub const FUTURE_GRACE_MS: i64 = 24 * 60 * 60 * 1000; + +/// Rows examined. Enough to be decisive, small enough to stay free on a +/// quarter-million-row database. +const SAMPLE_SIZE: usize = 512; + +/// Share of sampled timestamps that must be plausible. +/// +/// Deliberately not 1.0: a genuine database can carry a handful of corrupt or +/// out-of-range rows, and rejecting real evidence over one bad row would be a +/// worse failure than the one this gate exists to catch. A misresolved column +/// is wrong for essentially every row, so it fails this bar easily. +const MIN_PLAUSIBLE_RATIO: f64 = 0.90; + +/// Offending values quoted in the error. +const OFFENDERS_SHOWN: usize = 3; + +/// Check that the resolved `message` map actually describes the records. +/// +/// `now_ms` is the acquisition time in epoch milliseconds; it is passed in +/// rather than read from the clock so the bounds are testable. +pub fn validate_message_columns( + records: &[&RecoveredRecord], + cols: &MessageColumns, + now_ms: i64, +) -> Result<()> { + let Some(ts_idx) = cols.timestamp else { + bail!( + "msgstore `message` table declares no `timestamp` column; the schema is \ + unrecognised and its records will not be interpreted" + ); + }; + if cols.chat_row_id.is_none() { + bail!( + "msgstore `message` table declares no `chat_row_id` column; messages \ + cannot be attributed to a chat and will not be interpreted" + ); + } + if records.is_empty() { + return Ok(()); + } + + let latest_ms = now_ms.saturating_add(FUTURE_GRACE_MS); + let mut sampled = 0usize; + let mut plausible = 0usize; + let mut offenders: Vec = Vec::new(); + let mut short_rows = 0usize; + + for r in records.iter().take(SAMPLE_SIZE) { + let value = r.values.get(ts_idx); + if value.is_none() { + short_rows += 1; + } + sampled += 1; + if matches!(value, Some(SqlValue::Int(ms)) if *ms >= EARLIEST_PLAUSIBLE_MS && *ms <= latest_ms) + { + plausible += 1; + } else if offenders.len() < OFFENDERS_SHOWN { + let row = r + .row_id + .map_or_else(|| "?".to_string(), |id| id.to_string()); + offenders.push(format!("_id={row} -> {}", render_value(value))); + } + } + + #[allow(clippy::cast_precision_loss)] // sample is at most SAMPLE_SIZE rows + let ratio = plausible as f64 / sampled as f64; + if ratio >= MIN_PLAUSIBLE_RATIO { + return Ok(()); + } + + let detail = if short_rows == sampled { + format!("no sampled record is long enough to carry column {ts_idx}") + } else { + format!( + "{plausible} of {sampled} sampled rows carry a plausible epoch-millisecond \ + value (need {:.0}%)", + MIN_PLAUSIBLE_RATIO * 100.0 + ) + }; + + bail!( + "msgstore `message`.`timestamp` resolved to column {ts_idx}, but {detail}. \ + Offending values: [{}]. Expected epoch milliseconds between \ + {EARLIEST_PLAUSIBLE_MS} (2009-01-01) and {latest_ms} (acquisition + 1 day). \ + Refusing to report on a column map the data contradicts.", + offenders.join("; ") + ) +} + +/// Render a stored value for a diagnostic, verbatim where it is bounded. +pub fn render_value(value: Option<&SqlValue>) -> String { + match value { + None => "".to_string(), + Some(SqlValue::Null) => "NULL".to_string(), + Some(SqlValue::Int(n)) => n.to_string(), + Some(SqlValue::Real(f)) => f.to_string(), + Some(SqlValue::Text(s)) => format!("TEXT {s:?}"), + Some(SqlValue::Blob(b)) => { + const SHOWN: usize = 16; + let head: String = b.iter().take(SHOWN).map(|x| format!("{x:02x}")).collect(); + if b.len() > SHOWN { + format!("BLOB {} bytes, first {SHOWN}: {head}", b.len()) + } else { + format!("BLOB {} bytes: {head}", b.len()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chat4n6_plugin_api::EvidenceSource; + + /// 2024-03-15T14:32:07Z. + const GOOD_MS: i64 = 1_710_513_127_000; + /// A little after the newest fixture timestamp, standing in for acquisition. + const NOW_MS: i64 = 1_760_000_000_000; + + fn cols_at(timestamp: Option, chat_row_id: Option) -> MessageColumns { + MessageColumns { + chat_row_id, + timestamp, + ..MessageColumns::default() + } + } + + fn record(row_id: i64, values: Vec) -> RecoveredRecord { + RecoveredRecord { + table: "message".to_string(), + row_id: Some(row_id), + values, + source: EvidenceSource::Live, + offset: 0, + confidence: 1.0, + } + } + + /// `n` records whose column 1 holds `ts`. + fn records_with(ts: impl Fn(usize) -> SqlValue, n: usize) -> Vec { + (0..n) + .map(|i| record(i as i64 + 1, vec![SqlValue::Null, ts(i)])) + .collect() + } + + fn refs(v: &[RecoveredRecord]) -> Vec<&RecoveredRecord> { + v.iter().collect() + } + + #[test] + fn plausible_timestamps_pass() { + let recs = records_with(|i| SqlValue::Int(GOOD_MS + i as i64 * 1000), 20); + assert!(validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok()); + } + + #[test] + fn no_message_rows_is_not_an_error() { + assert!( + validate_message_columns(&[], &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "an empty message table is a real state, not a bootstrap failure" + ); + } + + #[test] + fn missing_timestamp_column_is_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 5); + let err = validate_message_columns(&refs(&recs), &cols_at(None, Some(0)), NOW_MS) + .expect_err("a message table with no resolvable timestamp must not be extracted"); + let msg = err.to_string(); + assert!( + msg.contains("timestamp"), + "error must name the column: {msg}" + ); + assert!(msg.contains("message"), "error must name the table: {msg}"); + } + + #[test] + fn missing_chat_row_id_column_is_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 5); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), None), NOW_MS) + .expect_err("messages cannot be attributed without chat_row_id"); + assert!( + err.to_string().contains("chat_row_id"), + "error must name the column: {err}" + ); + } + + #[test] + fn epoch_zero_timestamps_are_rejected_and_the_values_shown() { + // The real failure: sender_jid_row_id read as a timestamp — small ints. + let recs = records_with(|i| SqlValue::Int(5243 + i as i64), 50); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("small integers are not epoch milliseconds"); + let msg = err.to_string(); + assert!(msg.contains("timestamp"), "must name the column: {msg}"); + assert!(msg.contains('1'), "must name the resolved ordinal: {msg}"); + assert!(msg.contains("5243"), "must quote an offending value: {msg}"); + } + + #[test] + fn a_few_implausible_rows_do_not_trip_the_gate() { + // A real database carries the odd corrupt row; rejecting good evidence + // over one of them would be the worse failure. + let mut recs = records_with(|i| SqlValue::Int(GOOD_MS + i as i64 * 1000), 199); + recs.push(record(200, vec![SqlValue::Null, SqlValue::Int(0)])); + assert!( + validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "1 bad row in 200 is an outlier, not a misresolved column" + ); + } + + #[test] + fn future_dated_timestamps_are_rejected() { + let beyond = NOW_MS + FUTURE_GRACE_MS + 1; + let recs = records_with(|i| SqlValue::Int(beyond + i as i64), 20); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("timestamps past the acquisition time cannot be genuine"); + assert!(err.to_string().contains(&beyond.to_string())); + } + + #[test] + fn one_day_of_clock_skew_is_tolerated() { + let recs = records_with(|_| SqlValue::Int(NOW_MS + FUTURE_GRACE_MS - 1), 20); + assert!( + validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "a day of grace absorbs device/host clock skew" + ); + } + + #[test] + fn non_integer_timestamps_are_rejected_and_shown() { + let recs = records_with(|_| SqlValue::Text("AAAA1111".to_string()), 20); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("a text column is not a timestamp column"); + assert!( + err.to_string().contains("AAAA1111"), + "the offending value must be shown verbatim: {err}" + ); + } + + #[test] + fn records_too_short_for_the_resolved_ordinal_are_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 20); + // Resolve timestamp past the end of every record. + let err = validate_message_columns(&refs(&recs), &cols_at(Some(9), Some(0)), NOW_MS) + .expect_err("no row carries the resolved column"); + assert!( + err.to_string().contains('9'), + "error must name the resolved ordinal: {err}" + ); + } + + // ── render_value ───────────────────────────────────────────────────────── + + #[test] + fn render_value_shows_each_kind() { + assert_eq!(render_value(Some(&SqlValue::Int(-5))), "-5"); + assert_eq!(render_value(Some(&SqlValue::Null)), "NULL"); + assert_eq!( + render_value(Some(&SqlValue::Text("hi".into()))), + "TEXT \"hi\"" + ); + assert_eq!( + render_value(Some(&SqlValue::Blob(vec![0xde, 0xad]))), + "BLOB 2 bytes: dead" + ); + assert!(render_value(None).contains("absent")); + } + + #[test] + fn render_value_labels_a_truncated_blob() { + let rendered = render_value(Some(&SqlValue::Blob(vec![0xab; 40]))); + assert!( + rendered.contains("40 bytes"), + "the full length must be stated: {rendered}" + ); + assert!( + rendered.contains("first 16"), + "the elision must be labelled: {rendered}" + ); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql new file mode 100644 index 0000000..abbd4ed --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql @@ -0,0 +1,80 @@ +-- Legacy WhatsApp Android msgstore schema (pre-2018 `messages` + `chat_list` era). +-- +-- PROVENANCE +-- Column names and order follow the legacy schema published in the WhatsApp +-- forensic literature. Unlike real_modern_schema.sql, this was NOT read off +-- the case device — nothing in the case is of this generation. It exists so +-- the legacy branch has something concrete to classify and refuse, rather +-- than being reasoned about in the abstract. +-- +-- A trimmed but faithful subset: the columns below sit in their published +-- relative order. The extractor reads none of them, because it has no +-- legacy path. +-- +-- WHAT THIS FIXTURE IS FOR +-- 1. detect_schema_version must classify it Legacy. +-- 2. Extraction must refuse it loudly. A database of this generation has no +-- `message`, `jid` or `chat` table, so a modern extractor finds nothing — +-- and "nothing" is indistinguishable from a clean device. + +PRAGMA user_version = 1; + +CREATE TABLE messages ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + key_remote_jid TEXT NOT NULL, + key_from_me INTEGER, + key_id TEXT NOT NULL, + status INTEGER, + needs_push INTEGER, + data TEXT, + timestamp INTEGER, + media_url TEXT, + media_mime_type TEXT, + media_wa_type TEXT, + media_size INTEGER, + media_name TEXT, + media_caption TEXT, + media_hash TEXT, + media_duration INTEGER, + origin INTEGER, + latitude REAL, + longitude REAL, + thumb_image TEXT, + remote_resource TEXT, + received_timestamp INTEGER, + send_timestamp INTEGER, + starred INTEGER, + quoted_row_id INTEGER, + edit_version INTEGER, + forwarded INTEGER +); + +CREATE TABLE chat_list ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + key_remote_jid TEXT UNIQUE, + message_table_id INTEGER, + subject TEXT, + creation INTEGER, + archived INTEGER, + sort_timestamp INTEGER +); + +CREATE TABLE wa_contacts ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid TEXT UNIQUE NOT NULL, + display_name TEXT, + status TEXT, + number TEXT +); + +-- Synthetic rows. Timestamps are 2016-06-01T12:00:00Z (1464782400000). +INSERT INTO chat_list (_id, key_remote_jid, message_table_id, subject, creation, archived) +VALUES (1, '4155550100@s.whatsapp.net', 2, NULL, 1464782400000, 0); + +INSERT INTO messages (_id, key_remote_jid, key_from_me, key_id, data, timestamp, media_wa_type) +VALUES + (1, '4155550100@s.whatsapp.net', 1, 'LEGACY0001', 'Outgoing legacy message', 1464782400000, '0'), + (2, '4155550100@s.whatsapp.net', 0, 'LEGACY0002', 'Incoming legacy message', 1464782460000, '0'); + +INSERT INTO wa_contacts (_id, jid, display_name, status, number) +VALUES (1, '4155550100@s.whatsapp.net', 'Legacy Contact', NULL, '+14155550100'); diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql index f24fd80..6daac50 100644 --- a/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql @@ -1,5 +1,25 @@ --- Minimal modern WhatsApp Android msgstore schema + sample data --- Column order matches what ForensicEngine/btree walker will emit +-- Simplified WhatsApp Android msgstore schema + sample data. +-- +-- READ THIS BEFORE ADDING A TEST HERE +-- These CREATE TABLE shapes are a simplification, not a device's schema. They +-- were written to the same column layout the extractor's old hardcoded +-- ordinals assumed, so for as long as those ordinals existed this fixture +-- agreed with the code while both were wrong about real data. It is kept for +-- two reasons, neither of which is realism: +-- +-- 1. It carries `media_mime_type` and `media_name` on the `message` table. +-- Those columns are genuine in older modern-generation schemas but the +-- 2023-era device moved media metadata to `message_media`, which the +-- extractor does not yet read. This fixture is what exercises the +-- media-in-message path. +-- 2. Its aux tables (message_quoted, message_add_on, receipt_user, …) have +-- not been checked against a device, so the column names here are the +-- only ones those code paths are known against. +-- +-- For anything about column POSITIONS, use real_modern_schema.sql (real +-- device DDL) or shuffled_schema.sql (same names, permuted order). A test +-- that could pass here purely because the layout happens to match the reader +-- proves nothing. PRAGMA user_version = 200; diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql new file mode 100644 index 0000000..37db26e --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql @@ -0,0 +1,140 @@ +-- Real-device WhatsApp Android msgstore.db schema (2023-era handset). +-- +-- PROVENANCE +-- The CREATE TABLE shapes below reproduce the DDL read out of a real +-- `msgstore.db` acquired in a live forensic case. DDL is not case data — +-- no identifiers, names, numbers, paths or message content from that device +-- appear here. Every INSERT below is synthetic. +-- +-- Column positions confirmed against that device: +-- message : chat_row_id(1) from_me(2) key_id(3) sender_jid_row_id(4) +-- timestamp(11) message_type(14) text_data(15) starred(16) +-- jid : raw_string(5) +-- chat : jid_row_id(1) hidden(2) subject(3) created_timestamp(4) archived(10) +-- call_log: jid_row_id(1) from_me(2) call_id(3) transaction_id(4) +-- timestamp(5) video_call(6) duration(7) call_result(8) +-- Columns at the intervening positions carry the names published in the +-- WhatsApp-schema forensic literature; they are position-preserving and are +-- never read by the extractor. Correctness of this fixture depends only on +-- the confirmed names above sitting at the confirmed positions. +-- +-- WHY THIS FIXTURE EXISTS +-- `modern_schema.sql` was authored to the same simplified layout the +-- extractor's hardcoded ordinals assumed, so it could never falsify them. +-- This fixture can: the values a positional reader picks up here are the +-- wrong columns, exactly as on the real device. +-- +-- PRAGMA user_version is 1 on the real device — the modern schema is NOT +-- identified by a high user_version. + +PRAGMA user_version = 1; + +CREATE TABLE jid ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + user TEXT NOT NULL, + server TEXT NOT NULL, + agent INTEGER, + type INTEGER, + raw_string TEXT, + device INTEGER +); + +CREATE TABLE chat ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid_row_id INTEGER UNIQUE, + hidden INTEGER, + subject TEXT, + created_timestamp INTEGER, + display_message_row_id INTEGER, + last_message_row_id INTEGER, + last_read_message_row_id INTEGER, + last_read_receipt_sent_message_row_id INTEGER, + last_important_message_row_id INTEGER, + archived INTEGER, + sort_timestamp INTEGER, + mod_tag INTEGER, + gen TEXT, + spam_detection INTEGER, + unseen_earliest_message_received_time INTEGER, + unseen_message_count INTEGER, + unseen_missed_calls_count INTEGER, + unseen_row_count INTEGER, + plaintext_disabled INTEGER, + vcard_ui_dismissed INTEGER, + change_number_notified_message_row_id INTEGER, + show_group_description INTEGER, + ephemeral_expiration INTEGER, + last_read_ephemeral_message_row_id INTEGER, + ephemeral_setting_timestamp INTEGER, + unseen_important_message_count INTEGER +); + +CREATE TABLE message ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_row_id INTEGER NOT NULL, + from_me INTEGER NOT NULL, + key_id TEXT NOT NULL, + sender_jid_row_id INTEGER, + status INTEGER, + broadcast INTEGER, + recipient_count INTEGER, + participant_hash TEXT, + origination_flags INTEGER, + origin INTEGER, + timestamp INTEGER, + received_timestamp INTEGER, + receipt_server_timestamp INTEGER, + message_type INTEGER, + text_data TEXT, + starred INTEGER, + lookup_tables INTEGER, + sort_id INTEGER, + message_add_on_flags INTEGER, + view_mode INTEGER +); + +CREATE TABLE call_log ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid_row_id INTEGER, + from_me INTEGER, + call_id TEXT, + transaction_id INTEGER, + timestamp INTEGER, + video_call INTEGER, + duration INTEGER, + call_result INTEGER, + is_dnd_mode_on INTEGER, + bytes_transferred INTEGER, + group_jid_row_id INTEGER, + is_joinable_group_call INTEGER, + call_creator_device_jid_row_id INTEGER, + call_random_id TEXT +); + +-- ── Synthetic rows ─────────────────────────────────────────────────────────── +-- All timestamps are on 2022-09-15 UTC (1663243200000 = 2022-09-15T12:00:00Z). + +INSERT INTO jid (_id, user, server, agent, type, raw_string, device) VALUES + (1, '4155550100', 's.whatsapp.net', 0, 0, '4155550100@s.whatsapp.net', 0), + (2, '120363001234567890', 'g.us', 0, 1, '120363001234567890@g.us', 0); + +INSERT INTO chat (_id, jid_row_id, hidden, subject, created_timestamp, archived) VALUES + (1, 1, 0, NULL, 1663200000000, 0), + (2, 2, 0, 'Real Schema Group', 1663200000000, 1); + +INSERT INTO message + (_id, chat_row_id, from_me, key_id, sender_jid_row_id, timestamp, message_type, text_data, starred) +VALUES + (1, 1, 1, 'AAAA1111', NULL, 1663243200000, 0, 'Outgoing real-schema message', 0), + (2, 1, 0, 'BBBB2222', 1, 1663243260000, 0, 'Incoming real-schema message', 1), + (3, 2, 0, 'CCCC3333', 1, 1663243320000, 0, 'Group real-schema message', 0); + +-- Three distinct calls. Calls 1 and 3 share a duration (137 s) but are +-- unrelated: a reader that mistakes `duration` for a group-call grouping key +-- collapses them into one record. +INSERT INTO call_log + (_id, jid_row_id, from_me, call_id, transaction_id, timestamp, video_call, duration, call_result) +VALUES + (1, 1, 1, 'CALL-AAAA', 11, 1663243400000, 0, 137, 1), + (2, 1, 0, 'CALL-BBBB', 12, 1663243500000, 1, 42, 1), + (3, 1, 1, 'CALL-CCCC', 13, 1663243600000, 0, 137, 1); diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql new file mode 100644 index 0000000..6d7e091 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql @@ -0,0 +1,96 @@ +-- real_modern_schema.sql with every column order permuted. +-- +-- Same table names, same column names, same rows, same values — only the +-- declaration order differs, and `_id` is deliberately not declared first. +-- +-- WHY +-- A fixture built to the layout the code assumes cannot falsify that +-- assumption; that is how the ordinal bug shipped green. This fixture makes +-- position-independence a property under test rather than a claim: paired +-- with real_modern_schema.sql it asserts that two databases differing only in +-- column order extract identically. Any ordinal creeping back into the +-- extractor breaks one of the pair. +-- +-- Declaring `_id` mid-table also exercises the record-layout invariant the +-- resolver rests on: SQLite writes the INTEGER PRIMARY KEY as a NULL at its +-- own declared position, not at position 0. +-- +-- This is a deliberately synthetic permutation. No device ships these orders. + +PRAGMA user_version = 1; + +CREATE TABLE jid ( + raw_string TEXT, + device INTEGER, + _id INTEGER PRIMARY KEY, + server TEXT NOT NULL, + type INTEGER, + user TEXT NOT NULL, + agent INTEGER +); + +CREATE TABLE chat ( + subject TEXT, + sort_timestamp INTEGER, + archived INTEGER, + created_timestamp INTEGER, + hidden INTEGER, + display_message_row_id INTEGER, + _id INTEGER PRIMARY KEY, + jid_row_id INTEGER UNIQUE, + mod_tag INTEGER +); + +CREATE TABLE message ( + text_data TEXT, + timestamp INTEGER, + view_mode INTEGER, + sender_jid_row_id INTEGER, + sort_id INTEGER, + from_me INTEGER NOT NULL, + _id INTEGER PRIMARY KEY, + status INTEGER, + message_type INTEGER, + key_id TEXT NOT NULL, + received_timestamp INTEGER, + starred INTEGER, + chat_row_id INTEGER NOT NULL, + origin INTEGER +); + +CREATE TABLE call_log ( + duration INTEGER, + call_result INTEGER, + _id INTEGER PRIMARY KEY, + video_call INTEGER, + call_id TEXT, + jid_row_id INTEGER, + bytes_transferred INTEGER, + timestamp INTEGER, + transaction_id INTEGER, + from_me INTEGER +); + +-- Identical rows to real_modern_schema.sql, inserted by name. + +INSERT INTO jid (_id, user, server, agent, type, raw_string, device) VALUES + (1, '4155550100', 's.whatsapp.net', 0, 0, '4155550100@s.whatsapp.net', 0), + (2, '120363001234567890', 'g.us', 0, 1, '120363001234567890@g.us', 0); + +INSERT INTO chat (_id, jid_row_id, hidden, subject, created_timestamp, archived) VALUES + (1, 1, 0, NULL, 1663200000000, 0), + (2, 2, 0, 'Real Schema Group', 1663200000000, 1); + +INSERT INTO message + (_id, chat_row_id, from_me, key_id, sender_jid_row_id, timestamp, message_type, text_data, starred) +VALUES + (1, 1, 1, 'AAAA1111', NULL, 1663243200000, 0, 'Outgoing real-schema message', 0), + (2, 1, 0, 'BBBB2222', 1, 1663243260000, 0, 'Incoming real-schema message', 1), + (3, 2, 0, 'CCCC3333', 1, 1663243320000, 0, 'Group real-schema message', 0); + +INSERT INTO call_log + (_id, jid_row_id, from_me, call_id, transaction_id, timestamp, video_call, duration, call_result) +VALUES + (1, 1, 1, 'CALL-AAAA', 11, 1663243400000, 0, 137, 1), + (2, 1, 0, 'CALL-BBBB', 12, 1663243500000, 1, 42, 1), + (3, 1, 1, 'CALL-CCCC', 13, 1663243600000, 0, 137, 1); diff --git a/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs b/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs new file mode 100644 index 0000000..49c4085 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs @@ -0,0 +1,359 @@ +//! Tier-1 validation of the WhatsApp extractor against a real msgstore.db. +//! +//! Ground truth comes from `sqlite3` reading the same file — an independent +//! implementation, not a fixture we authored. Fixtures can only prove the code +//! agrees with itself; this proves it agrees with SQLite about a database +//! neither of them was written for. +//! +//! # Running +//! +//! Both inputs are supplied by the operator and never recorded here: +//! +//! ```text +//! CHAT4N6_REAL_MSGSTORE=/path/to/msgstore.db \ +//! CHAT4N6_REAL_INPUT_DIR=/path/to/extraction-root \ +//! cargo test -p chat4n6-whatsapp --test real_msgstore_validation -- --nocapture +//! ``` +//! +//! Absent the variables every test skips cleanly, so CI stays green without the +//! evidence. `sqlite3` must be on PATH for the oracle comparisons; without it +//! the reconciling tests skip and only the self-contained ones run. +//! +//! # Case hygiene +//! +//! Nothing here prints or asserts on message content, phone numbers, JIDs, +//! subjects or paths. Failures report counts, shapes and years only. Keep it +//! that way: this file is committed, the evidence is not. + +use chat4n6_plugin_api::ExtractionResult; +use chat4n6_whatsapp::extractor::extract_from_msgstore; +use chat4n6_whatsapp::schema::SchemaVersion; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const MSGSTORE_ENV: &str = "CHAT4N6_REAL_MSGSTORE"; +const INPUT_DIR_ENV: &str = "CHAT4N6_REAL_INPUT_DIR"; + +/// The evidence path, or `None` with a skip note. +fn msgstore_path() -> Option { + match std::env::var(MSGSTORE_ENV) { + Ok(p) if !p.is_empty() => { + let path = PathBuf::from(p); + if path.is_file() { + Some(path) + } else { + // A variable pointing at nothing is operator error, not absence + // of evidence — say so rather than skipping quietly. + panic!("{MSGSTORE_ENV} is set but is not a readable file"); + } + } + _ => { + eprintln!("SKIP: {MSGSTORE_ENV} not set — Tier-1 validation not run"); + None + } + } +} + +/// Run one scalar query through the `sqlite3` oracle, read-only and immutable. +fn oracle(db: &Path, sql: &str) -> Option { + let uri = format!("file:{}?mode=ro&immutable=1", db.display()); + let out = Command::new("sqlite3").arg(uri).arg(sql).output().ok()?; + if !out.status.success() { + eprintln!( + "SKIP: sqlite3 oracle failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + return None; + } + Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +fn oracle_count(db: &Path, table: &str) -> Option { + oracle(db, &format!("SELECT COUNT(*) FROM {table};"))? + .parse() + .ok() +} + +/// Extract, or skip. Returns the result and the evidence path. +fn extract() -> Option<(ExtractionResult, PathBuf)> { + let path = msgstore_path()?; + let bytes = std::fs::read(&path).expect("reading the evidence file"); + let result = extract_from_msgstore(&bytes, 0, SchemaVersion::Modern) + .expect("extraction must succeed on a real msgstore.db"); + Some((result, path)) +} + +fn message_count(r: &ExtractionResult) -> usize { + r.chats.iter().map(|c| c.messages.len()).sum() +} + +// ── Reconciliation against the sqlite3 oracle ──────────────────────────────── + +#[test] +fn message_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "message") else { + return; + }; + let actual = message_count(&result) as u64; + + if actual != expected { + // Turn a bare mismatch into a diagnosis: extras are either duplicate + // row_ids or records attributed to a layer other than the live btree. + let mut by_source: BTreeMap = BTreeMap::new(); + let mut ids: Vec = Vec::new(); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + *by_source.entry(format!("{:?}", m.source)).or_insert(0) += 1; + ids.push(m.id); + } + let unique = { + let mut v = ids.clone(); + v.sort_unstable(); + v.dedup(); + v.len() + }; + panic!( + "extracted {actual} messages, sqlite3 reports {expected} in `message` \ + (delta {}). Distinct row_ids: {unique} of {}. By source: {by_source:?}", + actual as i64 - expected as i64, + ids.len() + ); + } +} + +#[test] +fn chat_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "chat") else { + return; + }; + // Chats carrying messages whose own row was not recovered appear as stubs, + // so our count may exceed the table's; it must never fall short. + let stubs = result.chats.iter().filter(|c| c.jid.is_empty()).count(); + assert!( + result.chats.len() as u64 >= expected, + "extracted {} chats, sqlite3 reports {expected} in `chat` ({stubs} stubs)", + result.chats.len() + ); + assert_eq!( + result.chats.len() as u64 - stubs as u64, + expected, + "non-stub chats must reconcile exactly with the `chat` table" + ); +} + +#[test] +fn call_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "call_log") else { + return; + }; + // Group calls merge several rows into one record; without a call_row_id + // column nothing merges, so the counts are equal. + let participants: usize = result.calls.iter().map(|c| c.participants.len()).sum(); + assert_eq!( + participants as u64, + expected, + "one participant per call_log row: got {participants} across {} records", + result.calls.len() + ); +} + +#[test] +fn year_histogram_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(raw) = oracle( + &path, + "SELECT strftime('%Y', timestamp/1000, 'unixepoch'), COUNT(*) \ + FROM message GROUP BY 1 ORDER BY 1;", + ) else { + return; + }; + + let mut expected: BTreeMap = BTreeMap::new(); + for line in raw.lines() { + let (year, count) = line.split_once('|').unwrap_or(("", "")); + if let Ok(n) = count.parse::() { + // A row whose year does not render (NULL or out of range) keys on + // the empty string; it must be carried, not dropped. + expected.insert(year.to_string(), n); + } + } + if expected.is_empty() { + eprintln!("SKIP: oracle returned no year histogram"); + return; + } + + let mut actual: BTreeMap = BTreeMap::new(); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + *actual + .entry(m.timestamp.utc.format("%Y").to_string()) + .or_insert(0) += 1; + } + + // Compare only years both sides can express: SQLite renders an + // unrepresentable timestamp as an empty year, which has no counterpart in a + // parsed DateTime. Report it rather than letting it vanish. + let unrenderable = expected.get("").copied().unwrap_or(0); + if unrenderable > 0 { + eprintln!( + "NOTE: sqlite3 renders no year for {unrenderable} row(s); \ + extractor placed them in: {:?}", + actual + .iter() + .filter(|(y, _)| !expected.contains_key(*y)) + .collect::>() + ); + } + let comparable: BTreeMap<&String, &u64> = + expected.iter().filter(|(y, _)| !y.is_empty()).collect(); + for (year, count) in comparable { + assert_eq!( + actual.get(year).copied().unwrap_or(0), + *count, + "year {year}: extractor and sqlite3 disagree (full histogram: {actual:?})" + ); + } +} + +// ── Self-contained assertions (no oracle needed) ───────────────────────────── + +#[test] +fn no_message_is_dated_1970() { + let Some((result, _)) = extract() else { + return; + }; + let epoch_year: Vec = result + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .filter(|m| m.timestamp.utc.format("%Y").to_string() == "1970") + .map(|m| m.id) + .take(5) + .collect(); + assert!( + epoch_year.is_empty(), + "messages dated 1970 indicate a misread timestamp column; first row_ids: {epoch_year:?}" + ); +} + +#[test] +fn senders_resolve_to_jids_not_bare_numbers() { + let Some((result, _)) = extract() else { + return; + }; + let (mut resolved, mut bare) = (0usize, 0usize); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + match m.sender_jid.as_deref() { + Some(s) if s.contains('@') => resolved += 1, + Some(_) => bare += 1, + None => {} + } + } + assert!(resolved > 0, "no sender resolved to a JID at all"); + assert_eq!( + bare, 0, + "{bare} senders carry no '@' — that is jid.user, not jid.raw_string \ + ({resolved} resolved correctly)" + ); +} + +#[test] +fn hourly_distribution_is_not_concentrated_in_one_bucket() { + let Some((result, _)) = extract() else { + return; + }; + let mut hours = [0u64; 24]; + let mut total = 0u64; + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + let h = m + .timestamp + .utc + .format("%H") + .to_string() + .parse::() + .unwrap_or(0); + if let Some(slot) = hours.get_mut(h) { + *slot += 1; + } + total += 1; + } + // A spread is a property of volume: a handful of messages can legitimately + // sit in one hour, so below this the check would report noise as a finding. + const MIN_FOR_SPREAD: u64 = 1000; + if total < MIN_FOR_SPREAD { + eprintln!("SKIP: {total} messages is too few to judge hourly spread"); + return; + } + let occupied = hours.iter().filter(|&&n| n > 0).count(); + let busiest = hours.iter().max().copied().unwrap_or(0); + assert!( + occupied >= 12, + "only {occupied} of 24 hourly buckets occupied — human messaging spreads \ + across the day; a single bucket means every timestamp collapsed to one instant" + ); + assert!( + busiest * 100 / total < 50, + "busiest hour holds {}% of {total} messages", + busiest * 100 / total + ); +} + +#[test] +fn no_implausible_timestamp_distribution_warning_is_raised() { + let Some((result, _)) = extract() else { + return; + }; + let anomalies: Vec = result + .forensic_warnings + .iter() + .filter(|w| { + matches!( + w, + chat4n6_plugin_api::ForensicWarning::TimestampDistributionAnomaly { .. } + ) + }) + .map(std::string::ToString::to_string) + .collect(); + assert!( + anomalies.is_empty(), + "the detector reports the extraction's own dates as implausible: {anomalies:?}" + ); +} + +// ── Pipeline-level run over the extraction tree ────────────────────────────── + +#[test] +fn plugin_detects_and_extracts_from_the_extraction_tree() { + let Ok(root) = std::env::var(INPUT_DIR_ENV) else { + eprintln!("SKIP: {INPUT_DIR_ENV} not set — pipeline-level run not performed"); + return; + }; + if root.is_empty() { + eprintln!("SKIP: {INPUT_DIR_ENV} is empty"); + return; + } + let fs = + chat4n6_fs::PlaintextDirFs::new(Path::new(&root)).expect("opening the extraction tree"); + let plugin = chat4n6_whatsapp::WhatsAppPlugin::new(); + assert!( + chat4n6_plugin_api::ForensicPlugin::detect(&plugin, &fs), + "the WhatsApp plugin must detect its database under {INPUT_DIR_ENV}" + ); + let result = chat4n6_plugin_api::ForensicPlugin::extract(&plugin, &fs, Some(0)) + .expect("pipeline extraction must succeed"); + assert!( + message_count(&result) > 0, + "pipeline extraction produced no messages" + ); +} diff --git a/docs/user-stories/platforms/schema-resolution-audit.json b/docs/user-stories/platforms/schema-resolution-audit.json new file mode 100644 index 0000000..49c5090 --- /dev/null +++ b/docs/user-stories/platforms/schema-resolution-audit.json @@ -0,0 +1,17 @@ +[ + { + "description": "iOS WhatsApp, Signal and Telegram resolve column positions by name instead of hardcoded ordinals", + "steps": [ + "Audit chat4n6-ios-whatsapp, chat4n6-signal and chat4n6-telegram for the pattern the Android WhatsApp plugin carried: record fields read at a fixed values[] index (helpers::cols::* in Signal, cols::* in Telegram, CoreData attribute positions in iOS)", + "For each plugin, record the real CREATE TABLE DDL from a genuine database of that platform (DDL only, no case content) and check it against the ordinals the code assumes", + "Add a real-DDL fixture per plugin whose column layout differs from the simplified fixture, so a positional reader fails by construction", + "Add a column-order-permuted fixture per plugin, as tests/fixtures/shuffled_schema.sql does for Android WhatsApp, and assert both extract identically", + "Reuse chat4n6-whatsapp's columns module for name-to-ordinal resolution rather than reimplementing it per plugin; promote it to a shared crate if three plugins need it", + "Apply the equivalent of schema_gate::validate_message_columns per platform, so a contradicted column map aborts instead of producing a complete but wrong report", + "Note the iOS timestamp base differs (seconds since 2001-01-01), so the plausibility bounds need converting rather than copying", + "Update the 'other plugins are unaudited' gap in docs/validation.md", + "cargo test --workspace passes" + ], + "passes": false + } +] diff --git a/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json b/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json new file mode 100644 index 0000000..52b1df9 --- /dev/null +++ b/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json @@ -0,0 +1,32 @@ +[ + { + "description": "A raw decrypted partition image is an accepted input, so unallocated space can be carved", + "steps": [ + "Accepted inputs today are a plaintext directory, a .dar archive and an iOS backup; a raw decrypted userdata partition image cannot be supplied at all", + "Add a raw-image input type to chat4n6-fs that opens an ext4 partition image and exposes it through the ForensicFs trait", + "Wire it into the CLI input auto-detection alongside the existing three", + "Implement unallocated_regions() for it from the ext4 block bitmap: PlaintextDirFs returns an empty vec by design, which is why layer 8 has nothing to carve on this evidence class", + "Build a small synthetic ext4 image as a committed fixture so the path is exercised without a multi-gigabyte artifact, keeping the coverage gate satisfiable from committed bytes alone", + "Assert files under data/data/com.whatsapp/databases are readable through the raw-image filesystem", + "Assert unallocated_regions() returns non-empty regions for an image with freed blocks, and that layer 8 carving runs over them", + "Validate against an independent oracle on a real image: reconcile the carved region list against what The Sleuth Kit reports as unallocated", + "Update the 'unallocated carving needs a raw-image input' gap in docs/validation.md", + "cargo test -p chat4n6-fs passes" + ], + "passes": false + }, + { + "description": "Recovery layers beyond the live btree are mapped into messages by the WhatsApp plugin", + "steps": [ + "extract_from_msgstore consumes recover_layer1 only, so WAL deltas, freelist, journal, intra-page carve and unallocated carve are recovered by the engine but never interpreted as messages", + "Feed recover_all() records through the same resolved column maps the live layer uses, so no layer reinvents a column position", + "Confirm carvers that pattern-match record shapes against the message schema use the resolved shape rather than a fixture shape", + "Assert a message present only in a WAL frame appears in the extraction with source WalHistoric", + "Assert a message present only in a freelist page appears with source Freelist", + "Assert records recovered from a non-live layer still pass through the schema gate, so a wrong map cannot enter through a side door", + "Re-verify WAL parsing against a real database after the column-resolution fix, since the WAL path shared the ordinals that were wrong", + "cargo test -p chat4n6-whatsapp passes" + ], + "passes": false + } +] diff --git a/docs/user-stories/whatsapp/message-media-metadata.json b/docs/user-stories/whatsapp/message-media-metadata.json new file mode 100644 index 0000000..88d6de7 --- /dev/null +++ b/docs/user-stories/whatsapp/message-media-metadata.json @@ -0,0 +1,29 @@ +[ + { + "description": "Media metadata for the modern schema is read from message_media, which replaced the media columns on the message table", + "steps": [ + "Extend tests/fixtures/real_modern_schema.sql with a message_media table (message_row_id, mime_type, file_path, media_name, file_length, media_duration, file_hash, enc_file_hash, media_key, direct_path) and a row joined to a media message", + "Resolve every message_media column by name via columns::TableColumns, as message/jid/chat/call_log already are", + "Call extract_from_msgstore on that fixture", + "Assert the media message produces MessageContent::Media with the mime_type and file_path from message_media, not None", + "Assert file_size, duration_secs, file_hash and encrypted_hash are populated from the joined row", + "Assert a modern-schema database with no message_media row for a media message still extracts the message, with media fields absent rather than wrong", + "Assert modern_schema.sql (media columns on message) keeps working, so both schema generations are covered", + "cargo test -p chat4n6-whatsapp -- message_media passes" + ], + "passes": false + }, + { + "description": "Auxiliary msgstore table column names are confirmed against real device DDL", + "steps": [ + "Record the CREATE TABLE DDL for message_quoted, message_add_on, message_add_on_reaction, message_edit_info, receipt_user, message_forwarded and group_participant_user from a real device (DDL only, no case content)", + "Compare each against the names the extractor resolves today, which come from the simplified modern_schema.sql and are unverified", + "Where a real schema spells a field differently (for example parent_message_row_id for message_row_id, or message_add_on_type for type), resolve both spellings via TableColumns::first_of", + "Add a real-DDL fixture for those tables alongside real_modern_schema.sql", + "Assert reactions, edits and receipts extract from the real-DDL fixture rather than resolving to None", + "Update the 'auxiliary table column names are unverified' gap in docs/validation.md", + "cargo test -p chat4n6-whatsapp -- real_schema passes" + ], + "passes": false + } +] diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..7efd656 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,165 @@ +# Validation — WhatsApp Android (msgstore.db) + +## Summary + +The WhatsApp extractor resolves every msgstore.db column position by name from +the database's own `CREATE TABLE` SQL, and refuses to report on a database whose +data contradicts the resolved map. This document records how that is checked, +by what, and how far each check can be trusted. + +Three things carry the correctness claim: + +1. **Real-device DDL under test.** A fixture built from the `CREATE TABLE` + shapes of a real 2023-era handset, where the columns a positional reader + would land on are the wrong ones. +2. **Position-independence as a property.** A second fixture with identical + names and data but permuted column order. Both must extract identically. +3. **Reconciliation with `sqlite3`.** An independent implementation reading the + same evidence file, comparing counts and the timestamp year histogram. + +Item 3 is Tier 1 and requires the evidence, which lives only in the operator's +case workspace. **It has not been executed in this repository.** Items 1 and 2 +run in CI on every commit. + +## Evidence tiers + +Tiering is by *who confirms the result*, not by whether the input is synthetic. + +| Check | Tier | Confirmed by | +|---|---|---| +| Counts and year histogram vs. `sqlite3` on the case database | 1 | Independent implementation, real-world artifact | +| Extraction over real-device DDL (`real_modern_schema.sql`) | 2 | Real SQLite engine writes the file; expected values follow from the documented construction | +| Column order permutation (`shuffled_schema.sql`) | 2 | Differential: two constructions must agree, neither one's answer authored by hand | +| DDL parser unit tests (`columns.rs`) | 3 | Rules authored here; correctness is defined by the SQLite grammar they implement | +| Schema-gate behaviour (`schema_gate.rs`) | 3 | Detection rules — the fixture specifies behaviour rather than vouching for a value | + +Tier 3 is legitimate where it sits: those cases define detection rules and +robustness properties, not values an external oracle could adjudicate. The +value-producing paths — timestamps, senders, subjects, call durations — are +covered at Tier 2 by construction and at Tier 1 by `sqlite3`. + +## Method + +### Tier-1 reconciliation + +Ground truth is `sqlite3` reading the evidence read-only and immutable: + +``` +sqlite3 "file:$CHAT4N6_REAL_MSGSTORE?mode=ro&immutable=1" +``` + +The test derives every expected value from that oracle at run time rather than +from transcribed constants, so it reconciles against whatever database it is +pointed at. It asserts: + +- extracted message count equals `SELECT COUNT(*) FROM message` +- non-stub chat count equals `SELECT COUNT(*) FROM chat` +- call participants equal `SELECT COUNT(*) FROM call_log` +- the per-year message histogram matches + `strftime('%Y', timestamp/1000, 'unixepoch')` bucket for bucket +- no message is dated 1970 +- every resolved sender is a JID, not a bare subscriber number +- messages spread across at least 12 hourly buckets, with no single hour + holding half of them +- the extraction raises no `TimestampDistributionAnomaly` against itself + +A count mismatch fails with a breakdown by evidence source and a distinct-rowid +tally, so an over-count is diagnosed rather than merely reported. + +### Running it + +Both paths come from the operator's case workspace and are never recorded in +this repository: + +``` +CHAT4N6_REAL_MSGSTORE=/path/to/msgstore.db \ +CHAT4N6_REAL_INPUT_DIR=/path/to/extraction-root \ + cargo test -p chat4n6-whatsapp --test real_msgstore_validation -- --nocapture +``` + +Without the variables every test skips and prints why; without `sqlite3` on +PATH the reconciling tests skip and the self-contained ones still run. A +variable that is set but points at no readable file fails loudly rather than +skipping, so a typo cannot masquerade as "no evidence available". + +The test file asserts on counts, shapes and years only. No message content, +JID, subject, phone number or path is printed or embedded. + +### Expected values + +Recorded from the oracle on 2026-07-29 against this case's database. Aggregate +counts only. + +| Query | Value | +|---|---| +| `SELECT COUNT(*) FROM message` | 245,227 | +| `SELECT COUNT(*) FROM chat` | 2,309 | +| `SELECT COUNT(*) FROM call_log` | 166 | +| `PRAGMA freelist_count` | 0 | +| Messages dated 1970 | 0 | + +One `message` row renders an empty year under `strftime`, meaning its timestamp +is NULL or outside the representable range. The year-histogram test reports +that row and where the extractor placed it rather than dropping it silently. + +## What is checked in CI + +`cargo test -p chat4n6-whatsapp` covers, without any evidence present: + +- DDL parsing: nested parentheses, commas inside string defaults, quoted and + bracketed identifiers, table-level constraints occupying no column position, + malformed input yielding an empty map rather than a panic +- resolution of the confirmed real-device positions for `message`, `jid`, + `chat` and `call_log` +- extraction over real-device DDL: 2022 timestamps, senders via + `jid.raw_string`, subject via `chat.subject`, `archived`, per-call duration + and video flag, and three call rows staying three records +- identical extraction from the column-permuted fixture +- the record-layout invariant the resolver rests on: SQLite writes an + `INTEGER PRIMARY KEY` as a NULL at its *declared* position, so `values[i]` is + the column at DDL position `i` even when `_id` is declared mid-table +- the schema gate rejecting an unresolvable or contradicted column map, and + accepting an empty message table and a handful of outlier rows +- refusal of a legacy-generation database instead of an empty report + +## Known gaps + +These are real and unaddressed; none is masked by a passing test. + +- **Media metadata on the modern schema.** The 2023-era `message` table has no + `media_mime_type` or `media_name`; that metadata moved to `message_media`, + which the extractor does not read. Media fields resolve to `None` on a modern + database and media messages carry no path or MIME type. The + media-in-message path is still exercised by `modern_schema.sql`, which + represents an older modern-generation schema. +- **Auxiliary table column names are unverified.** `message_quoted`, + `message_add_on`, `message_edit_info`, `receipt_user`, `message_forwarded` + and `group_participant_user` resolve by the names in `modern_schema.sql`, + which were not read off a device. Where a real schema spells a column + differently the field resolves to `None`, so reactions, edits and receipts + may be absent rather than wrong. Auditing those names against the case + database is outstanding. +- **Only the live btree layer is interpreted.** WAL deltas, freelist, journal, + intra-page carve and unallocated carve are not mapped into messages by this + plugin. Unallocated carving additionally needs a raw-image input: the + plaintext-directory filesystem reports no unallocated regions by design. +- **`detect_timestamp_anomalies` cannot fire from this plugin.** Messages are + sorted by timestamp before the detectors run, so the pairwise out-of-order + check has nothing to find. `detect_selective_deletion` is likewise not wired + into extraction. The distribution-level detector is wired and does fire. +- **Other plugins are unaudited.** `chat4n6-ios-whatsapp`, `chat4n6-signal` and + `chat4n6-telegram` still read records by hardcoded ordinals and have not been + checked against real DDL. + +## Fixtures + +| File | What it is | Provenance | +|---|---|---| +| `real_modern_schema.sql` | Real 2023-era device DDL, synthetic rows | Confirmed positions read off the case device; intervening columns from published WhatsApp schema documentation, position-preserving and never read | +| `shuffled_schema.sql` | The same names, data and values with every column order permuted | Synthetic permutation; no device ships these orders | +| `legacy_schema.sql` | Pre-2018 `messages` + `chat_list` generation | Published legacy schema documentation, not read off the case device | +| `modern_schema.sql` | Simplified shapes, retained for the media-in-message path and for the aux tables | Authored here; explicitly not a device schema | + +`real_modern_schema.sql` and `shuffled_schema.sql` exist because a fixture +written to the layout the reader assumes cannot falsify that assumption. Those +two can.