From 1eed77515701f3239bd706ccbd8cc3ddd9bd0b68 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:11:35 +0800 Subject: [PATCH 01/11] test(red): name-based column resolution against real device DDL (24 tests fail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WhatsApp extractor reads msgstore.db records by hardcoded ordinals that only match our own simplified fixtures. Against a real 2023-era Android msgstore.db every ordinal lands on the wrong column: values[4] is sender_jid_row_id rather than timestamp (every message dated 1970-01-01), jid[1] is the bare `user` rather than raw_string, chat[2] is `hidden` rather than subject, and call_log[7] is `duration` rather than call_row_id (so unrelated calls sharing a duration merge into one record). The fixtures could never catch this: modern_schema.sql was authored to the same layout the ordinals assume, so code and fixture agree while both are wrong about real data. Adds: - tests/fixtures/real_modern_schema.sql — the CREATE TABLE shapes read out of a real device (DDL only; no case identifiers, contents or paths), with synthetic rows whose timestamps are a known 2022 value. - src/columns.rs — API and unit tests for DDL-driven name -> ordinal resolution, with a null implementation so the tests fail on assertions. - extractor::real_schema_tests — extraction assertions over the real DDL, plus a characterisation test proving the invariant the resolver rests on: SQLite writes the INTEGER PRIMARY KEY as a NULL at its *declared* position, so values[i] is the column at DDL position i even when _id is not declared first. That one passes today. - schema: user_version = 1 on the real device, so the `>= 100` arm classifies a legacy-table database as Modern. Co-Authored-By: Claude Opus 5 (1M context) --- crates/plugins/chat4n6-whatsapp/src/album.rs | 5 +- .../chat4n6-whatsapp/src/anti_forensics.rs | 96 ++-- crates/plugins/chat4n6-whatsapp/src/cdn.rs | 119 +++-- .../plugins/chat4n6-whatsapp/src/columns.rs | 409 ++++++++++++++++ .../chat4n6-whatsapp/src/contact_report.rs | 40 +- .../plugins/chat4n6-whatsapp/src/extractor.rs | 457 +++++++++++++++--- .../chat4n6-whatsapp/src/group_metadata.rs | 77 ++- crates/plugins/chat4n6-whatsapp/src/lib.rs | 43 +- crates/plugins/chat4n6-whatsapp/src/link.rs | 11 +- .../plugins/chat4n6-whatsapp/src/location.rs | 10 +- .../chat4n6-whatsapp/src/orphaned_media.rs | 72 +-- .../plugins/chat4n6-whatsapp/src/platform.rs | 29 +- crates/plugins/chat4n6-whatsapp/src/poll.rs | 12 +- crates/plugins/chat4n6-whatsapp/src/schema.rs | 39 +- crates/plugins/chat4n6-whatsapp/src/status.rs | 40 +- .../chat4n6-whatsapp/src/system_event.rs | 225 ++++++--- .../tests/fixtures/real_modern_schema.sql | 140 ++++++ 17 files changed, 1499 insertions(+), 325 deletions(-) create mode 100644 crates/plugins/chat4n6-whatsapp/src/columns.rs create mode 100644 crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql diff --git a/crates/plugins/chat4n6-whatsapp/src/album.rs b/crates/plugins/chat4n6-whatsapp/src/album.rs index e7332cf..7b2e2f0 100644 --- a/crates/plugins/chat4n6-whatsapp/src/album.rs +++ b/crates/plugins/chat4n6-whatsapp/src/album.rs @@ -75,7 +75,10 @@ mod tests { note.contains("WhatsApp expected"), "note format: got {note}" ); - assert!(note.contains("possible evidence gap"), "note format: got {note}"); + assert!( + note.contains("possible evidence gap"), + "note format: got {note}" + ); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs index c7e0491..5ef9aed 100644 --- a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs +++ b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs @@ -24,7 +24,9 @@ pub fn detect_vacuum(db_bytes: &[u8]) -> Vec { } let free_pages = u32::from_be_bytes([db_bytes[36], db_bytes[37], db_bytes[38], db_bytes[39]]); if free_pages > 0 { - vec![ForensicWarning::DatabaseVacuumed { freelist_page_count: free_pages }] + vec![ForensicWarning::DatabaseVacuumed { + freelist_page_count: free_pages, + }] } else { vec![] } @@ -47,8 +49,9 @@ pub fn detect_header_tamper(db_bytes: &[u8]) -> Vec { // Check 1: write_counter vs read_counter (bytes 92–95 vs 96–99). let write_version = db_bytes[18]; // 1=journal, 2=WAL - let write_counter = u32::from_be_bytes([db_bytes[92], db_bytes[93], db_bytes[94], db_bytes[95]]); - let read_counter = u32::from_be_bytes([db_bytes[96], db_bytes[97], db_bytes[98], db_bytes[99]]); + let write_counter = + u32::from_be_bytes([db_bytes[92], db_bytes[93], db_bytes[94], db_bytes[95]]); + let read_counter = u32::from_be_bytes([db_bytes[96], db_bytes[97], db_bytes[98], db_bytes[99]]); if write_counter != read_counter && write_version != 2 { warnings.push(ForensicWarning::HeaderTampered { change_counter: write_counter, @@ -58,8 +61,13 @@ pub fn detect_header_tamper(db_bytes: &[u8]) -> Vec { // Check 2: declared page_size × page_count vs actual file length. let raw_page_size = u16::from_be_bytes([db_bytes[16], db_bytes[17]]); - let page_size: u64 = if raw_page_size == 1 { 65536 } else { raw_page_size as u64 }; - let page_count = u32::from_be_bytes([db_bytes[28], db_bytes[29], db_bytes[30], db_bytes[31]]) as u64; + let page_size: u64 = if raw_page_size == 1 { + 65536 + } else { + raw_page_size as u64 + }; + let page_count = + u32::from_be_bytes([db_bytes[28], db_bytes[29], db_bytes[30], db_bytes[31]]) as u64; let expected_size = page_size * page_count; let actual_size = db_bytes.len() as u64; if actual_size >= 100 && actual_size != expected_size { @@ -128,9 +136,7 @@ pub fn detect_timestamp_anomalies(result: &ExtractionResult) -> Vec>, -) -> Vec { +pub fn detect_duplicate_stanza_ids(key_id_map: &HashMap>) -> Vec { key_id_map .iter() .filter(|(_, rows)| rows.len() > 1) @@ -238,7 +244,9 @@ mod header_tamper_tests { let header = valid_header(4096, 1, 5, 3); let warnings = detect_header_tamper(&header); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), "write/read counter mismatch must emit HeaderTampered, got: {warnings:?}" ); } @@ -249,7 +257,9 @@ mod header_tamper_tests { let header = valid_header(4096, 2, 1, 1); let warnings = detect_header_tamper(&header); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::HeaderTampered { .. })), "page size * count != file size must emit HeaderTampered, got: {warnings:?}" ); } @@ -278,14 +288,18 @@ mod header_tamper_tests { buf[36..40].copy_from_slice(&5u32.to_be_bytes()); // free_pages=5 let warnings = detect_vacuum(&buf); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::DatabaseVacuumed { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::DatabaseVacuumed { .. })), "non-zero free_pages must emit DatabaseVacuumed" ); } #[test] fn selective_deletion_detected() { - use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent}; + use chat4n6_plugin_api::{ + Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent, + }; let make_msg = |id: i64| Message { id, chat_id: 1, @@ -310,8 +324,12 @@ mod header_tamper_tests { name: None, is_group: false, messages: vec![ - make_msg(1), make_msg(2), make_msg(3), - make_msg(100), make_msg(101), make_msg(102), // gap 3→100 + make_msg(1), + make_msg(2), + make_msg(3), + make_msg(100), + make_msg(101), + make_msg(102), // gap 3→100 ], archived: false, }; @@ -324,20 +342,24 @@ mod header_tamper_tests { schema_version: 200, forensic_warnings: vec![], group_participant_events: vec![], - extraction_started_at: None, - extraction_finished_at: None, - wal_snapshots: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], }; let warnings = detect_selective_deletion(&result); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::SelectiveDeletion { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::SelectiveDeletion { .. })), "gap 3→100 should trigger SelectiveDeletion, got: {warnings:?}" ); } #[test] fn timestamp_anomaly_pre_whatsapp() { - use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent}; + use chat4n6_plugin_api::{ + Chat, ExtractionResult, ForensicTimestamp, Message, MessageContent, + }; let make_msg = |id: i64, ts: i64| Message { id, chat_id: 1, @@ -376,13 +398,15 @@ mod header_tamper_tests { schema_version: 200, forensic_warnings: vec![], group_participant_events: vec![], - extraction_started_at: None, - extraction_finished_at: None, - wal_snapshots: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], }; let warnings = detect_timestamp_anomalies(&result); assert!( - warnings.iter().any(|w| matches!(w, ForensicWarning::TimestampAnomaly { .. })), + warnings + .iter() + .any(|w| matches!(w, ForensicWarning::TimestampAnomaly { .. })), "reversed timestamps should emit TimestampAnomaly, got: {warnings:?}" ); } @@ -404,7 +428,8 @@ mod new_detector_tests { use crate::schema::SchemaVersion; let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL); @@ -423,9 +448,12 @@ mod new_detector_tests { INSERT INTO message VALUES (1, 1, NULL, 0, 1710513127000, 'hello', 0, 'ABC123'); INSERT INTO message VALUES (2, 1, NULL, 1, 1710513128000, 'world', 0, 'ABC123'); INSERT INTO message VALUES (3, 1, NULL, 0, 1710513129000, 'foo', 0, 'XYZ999'); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); @@ -451,7 +479,8 @@ mod new_detector_tests { use crate::schema::SchemaVersion; let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL); @@ -488,9 +517,12 @@ mod new_detector_tests { INSERT INTO message_thumbnails VALUES (13, x'ff'); INSERT INTO message_thumbnails VALUES (14, x'ff'); INSERT INTO message_thumbnails VALUES (15, x'ff'); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); @@ -498,7 +530,11 @@ mod new_detector_tests { assert!( result.forensic_warnings.iter().any(|w| matches!( w, - ForensicWarning::ThumbnailOrphanHigh { orphan_thumbnails: 5, total_messages: 10, ratio_pct: 50 } + ForensicWarning::ThumbnailOrphanHigh { + orphan_thumbnails: 5, + total_messages: 10, + ratio_pct: 50 + } )), "expected ThumbnailOrphanHigh with 5 orphans / 10 messages = 50%, got: {:?}", result.forensic_warnings diff --git a/crates/plugins/chat4n6-whatsapp/src/cdn.rs b/crates/plugins/chat4n6-whatsapp/src/cdn.rs index 12db901..fa3515d 100644 --- a/crates/plugins/chat4n6-whatsapp/src/cdn.rs +++ b/crates/plugins/chat4n6-whatsapp/src/cdn.rs @@ -18,12 +18,12 @@ pub enum CdnError { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CdnAcquisitionRecord { - pub url_hash: String, // SHA-256 hex of the URL (NOT the URL itself) - pub media_key_hash: String, // SHA-256 hex of the raw media key bytes - pub timestamp_utc: String, // ISO 8601 UTC timestamp of download attempt + pub url_hash: String, // SHA-256 hex of the URL (NOT the URL itself) + pub media_key_hash: String, // SHA-256 hex of the raw media key bytes + pub timestamp_utc: String, // ISO 8601 UTC timestamp of download attempt pub file_hash_result: Option, // SHA-256 hex of plaintext bytes (None if download failed) pub file_size_bytes: Option, - pub examiner: Option, // examiner identifier for chain of custody + pub examiner: Option, // examiner identifier for chain of custody pub success: bool, } @@ -43,7 +43,7 @@ pub fn decrypt_whatsapp_media( media_key_bytes: &[u8], encrypted_bytes: &[u8], ) -> Result, CdnError> { - use aes::cipher::{BlockDecryptMut, KeyIvInit, block_padding::Pkcs7}; + use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, KeyIvInit}; type Aes256CbcDec = cbc::Decryptor; // Minimum: need at least 1 AES block (16 bytes) of ciphertext — check blob first @@ -64,7 +64,7 @@ pub fn decrypt_whatsapp_media( let mut buf = encrypted_bytes.to_vec(); let plaintext_len = Aes256CbcDec::new(aes_key.into(), iv.into()) .decrypt_padded_mut::(&mut buf) - .map_err(|_| CdnError::HmacMismatch)? // unpad failure treated as decrypt error + .map_err(|_| CdnError::HmacMismatch)? // unpad failure treated as decrypt error .len(); buf.truncate(plaintext_len); Ok(buf) @@ -136,7 +136,10 @@ mod tests { assert!(!record.url_hash.is_empty(), "url_hash must not be empty"); // Verify it's the expected SHA-256 hex let expected = hex::encode(Sha256::digest(url.as_bytes())); - assert_eq!(record.url_hash, expected, "url_hash should be SHA-256 of the URL"); + assert_eq!( + record.url_hash, expected, + "url_hash should be SHA-256 of the URL" + ); } #[test] @@ -145,18 +148,30 @@ mod tests { let key = b"0123456789abcdef0123456789abcdef"; // 32 bytes let record = build_acquisition_record(url, key, None, None); let expected = hex::encode(Sha256::digest(key)); - assert_eq!(record.media_key_hash, expected, "media_key_hash should be SHA-256 of key bytes"); + assert_eq!( + record.media_key_hash, expected, + "media_key_hash should be SHA-256 of key bytes" + ); // Also verify it doesn't leak the actual key let key_hex = hex::encode(key); - assert_ne!(record.media_key_hash, key_hex, "media_key_hash should be SHA-256, not hex of key"); + assert_ne!( + record.media_key_hash, key_hex, + "media_key_hash should be SHA-256, not hex of key" + ); } #[test] fn test_build_acquisition_record_success_false_when_no_plaintext() { let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); - assert!(!record.success, "success should be false when plaintext is None"); - assert!(record.file_hash_result.is_none(), "file_hash_result should be None when no plaintext"); + assert!( + !record.success, + "success should be false when plaintext is None" + ); + assert!( + record.file_hash_result.is_none(), + "file_hash_result should be None when no plaintext" + ); assert!(record.file_size_bytes.is_none()); } @@ -165,8 +180,14 @@ mod tests { let key = vec![0u8; 32]; let plaintext = b"decrypted media content"; let record = build_acquisition_record("https://example.com", &key, Some(plaintext), None); - assert!(record.success, "success should be true when plaintext is provided"); - assert!(record.file_hash_result.is_some(), "file_hash_result should be set"); + assert!( + record.success, + "success should be true when plaintext is provided" + ); + assert!( + record.file_hash_result.is_some(), + "file_hash_result should be set" + ); assert_eq!(record.file_size_bytes, Some(plaintext.len() as u64)); } @@ -182,7 +203,8 @@ mod tests { #[test] fn test_build_acquisition_record_examiner_preserved() { let key = vec![0u8; 32]; - let record = build_acquisition_record("https://example.com", &key, None, Some("examiner_alice")); + let record = + build_acquisition_record("https://example.com", &key, None, Some("examiner_alice")); assert_eq!(record.examiner.as_deref(), Some("examiner_alice")); } @@ -190,7 +212,10 @@ mod tests { fn test_build_acquisition_record_timestamp_not_empty() { let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); - assert!(!record.timestamp_utc.is_empty(), "timestamp_utc must be set"); + assert!( + !record.timestamp_utc.is_empty(), + "timestamp_utc must be set" + ); } // ── append_to_log tests ─────────────────────────────────────────────────── @@ -199,7 +224,10 @@ mod tests { fn test_append_to_log_creates_file() { let dir = tempdir().unwrap(); let log_path = dir.path().join("cdn_acquisition.jsonl"); - assert!(!log_path.exists(), "log file should not exist before append"); + assert!( + !log_path.exists(), + "log file should not exist before append" + ); let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); @@ -222,7 +250,11 @@ mod tests { let content = std::fs::read_to_string(&log_path).unwrap(); let lines: Vec<&str> = content.lines().filter(|l| !l.is_empty()).collect(); - assert_eq!(lines.len(), 2, "should have exactly 2 lines after 2 appends"); + assert_eq!( + lines.len(), + 2, + "should have exactly 2 lines after 2 appends" + ); } #[test] @@ -231,7 +263,8 @@ mod tests { let log_path = dir.path().join("cdn_acquisition.jsonl"); let key = vec![0u8; 32]; - let r1 = build_acquisition_record("https://example.com/a", &key, Some(b"content"), Some("ex")); + let r1 = + build_acquisition_record("https://example.com/a", &key, Some(b"content"), Some("ex")); let r2 = build_acquisition_record("https://example.com/b", &key, None, None); append_to_log(&log_path, &r1).unwrap(); @@ -239,8 +272,8 @@ mod tests { let content = std::fs::read_to_string(&log_path).unwrap(); for line in content.lines().filter(|l| !l.is_empty()) { - let parsed: serde_json::Value = serde_json::from_str(line) - .expect("each line must be valid JSON"); + let parsed: serde_json::Value = + serde_json::from_str(line).expect("each line must be valid JSON"); assert!(parsed.is_object(), "each line must be a JSON object"); } } @@ -252,8 +285,10 @@ mod tests { let short_key = vec![0u8; 10]; // too short — need 32+ let blob = vec![0u8; 64]; let result = decrypt_whatsapp_media(&short_key, &blob); - assert!(matches!(result, Err(CdnError::KeyTooShort(_))), - "should return KeyTooShort error for short key"); + assert!( + matches!(result, Err(CdnError::KeyTooShort(_))), + "should return KeyTooShort error for short key" + ); } #[test] @@ -261,22 +296,24 @@ mod tests { let key = vec![0u8; 32]; let short_blob = vec![0u8; 5]; // too short — need at least 17 bytes (IV + 1 block) let result = decrypt_whatsapp_media(&key, &short_blob); - assert!(matches!(result, Err(CdnError::BlobTooShort(_, _))), - "should return BlobTooShort error for short blob"); + assert!( + matches!(result, Err(CdnError::BlobTooShort(_, _))), + "should return BlobTooShort error for short blob" + ); } #[test] fn test_decrypt_aes_cbc_basic() { // Construct a known AES-256-CBC encrypted payload using the aes+cbc crates // to test round-trip decryption. - use aes::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; + use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; type Aes256CbcEnc = cbc::Encryptor; // media_key: first 16 bytes = IV, bytes 16..48 = AES key (in simplified MVP mode) let iv_bytes = [0x01u8; 16]; let aes_key_bytes = [0x02u8; 32]; let mut media_key = Vec::new(); - media_key.extend_from_slice(&iv_bytes); // bytes 0..16 = IV + media_key.extend_from_slice(&iv_bytes); // bytes 0..16 = IV media_key.extend_from_slice(&aes_key_bytes); // bytes 16..48 = AES key let plaintext = b"Hello, WhatsApp media!"; @@ -287,8 +324,16 @@ mod tests { .unwrap(); let result = decrypt_whatsapp_media(&media_key, ciphertext); - assert!(result.is_ok(), "decrypt should succeed for valid AES-256-CBC payload: {:?}", result); - assert_eq!(result.unwrap(), plaintext, "decrypted bytes should match original plaintext"); + assert!( + result.is_ok(), + "decrypt should succeed for valid AES-256-CBC payload: {:?}", + result + ); + assert_eq!( + result.unwrap(), + plaintext, + "decrypted bytes should match original plaintext" + ); } // ── audit: no URL or key in serialized record ───────────────────────────── @@ -300,17 +345,27 @@ mod tests { let record = build_acquisition_record(url, key, None, None); let json = serde_json::to_string(&record).unwrap(); - assert!(!json.contains(url), "serialized JSON must NOT contain the original URL"); + assert!( + !json.contains(url), + "serialized JSON must NOT contain the original URL" + ); // Also verify the raw key bytes don't appear (hex encoded) let key_hex = hex::encode(key); - assert!(!json.contains(&key_hex), "serialized JSON must NOT contain the raw key hex"); + assert!( + !json.contains(&key_hex), + "serialized JSON must NOT contain the raw key hex" + ); } #[test] fn test_acquisition_record_examiner_preserved_in_json() { let key = vec![0u8; 32]; - let record = build_acquisition_record("https://example.com", &key, None, Some("forensic_lab_01")); + let record = + build_acquisition_record("https://example.com", &key, None, Some("forensic_lab_01")); let json = serde_json::to_string(&record).unwrap(); - assert!(json.contains("forensic_lab_01"), "examiner should appear in JSON"); + assert!( + json.contains("forensic_lab_01"), + "examiner should appear in JSON" + ); } } diff --git a/crates/plugins/chat4n6-whatsapp/src/columns.rs b/crates/plugins/chat4n6-whatsapp/src/columns.rs new file mode 100644 index 0000000..3a5d028 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/src/columns.rs @@ -0,0 +1,409 @@ +//! Name-based column resolution for msgstore.db tables. +//! +//! WhatsApp reorders, inserts and removes `message`/`chat`/`jid`/`call_log` +//! columns between app releases, so a fixed ordinal is only ever right for the +//! one schema it was written against. Reading the wrong column produces a +//! structurally complete report full of wrong values — the failure class that +//! is indistinguishable from a correct one. +//! +//! Every ordinal used by the extractor is therefore derived at run time from +//! the database's own `CREATE TABLE` SQL (read out of `sqlite_master`), and a +//! column that the schema does not declare resolves to `None` rather than to a +//! neighbouring column's data. +//! +//! # Record layout invariant +//! +//! SQLite stores an `INTEGER PRIMARY KEY` column as a serial-type-0 NULL **at +//! its declared position** in the record body; the real value lives in the +//! cell's rowid varint. The btree walker decodes serial types in order, so +//! `values[i]` is the column declared at DDL position `i` — including when the +//! rowid alias is not the first declared column. + +use std::collections::HashMap; + +/// Zero-based column positions for one table, keyed by lowercased column name. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TableColumns { + by_name: HashMap, +} + +impl TableColumns { + /// Parse a `CREATE TABLE` statement into a name → position map. + /// + /// Table-level constraints (`PRIMARY KEY (…)`, `UNIQUE (…)`, `CHECK (…)`, + /// `FOREIGN KEY (…)`, `CONSTRAINT …`) do not occupy a column position and + /// are skipped. Malformed input yields an empty map rather than a panic. + pub fn from_ddl(_ddl: &str) -> Self { + // RED stub — resolution is implemented in the GREEN commit. + Self::default() + } + + /// Zero-based position of `name`, or `None` when the table has no such column. + pub fn get(&self, _name: &str) -> Option { + None + } + + /// Position of the first of `names` that the table declares. + /// + /// Used where one logical field is spelled differently across schema + /// generations; the order of `names` is the preference order. + pub fn first_of(&self, _names: &[&str]) -> Option { + None + } + + /// Number of declared columns. + pub fn len(&self) -> usize { + self.by_name.len() + } + + pub fn is_empty(&self) -> bool { + self.by_name.is_empty() + } +} + +/// Column maps for every table in a database, built from `ForensicEngine::table_ddl`. +#[derive(Debug, Clone, Default)] +pub struct SchemaColumns { + tables: HashMap, + empty: TableColumns, +} + +impl SchemaColumns { + /// Build from a `table name → CREATE TABLE SQL` map. + pub fn from_ddl_map(_ddl: &HashMap) -> Self { + // RED stub — resolution is implemented in the GREEN commit. + Self::default() + } + + /// Columns of `table`; an empty map when the database has no such table. + /// + /// Returning an empty map (rather than `Option`) keeps every downstream + /// lookup on the same degrade-to-`None` path. + pub fn table(&self, name: &str) -> &TableColumns { + self.tables + .get(&name.to_ascii_lowercase()) + .unwrap_or(&self.empty) + } + + /// Whether any table DDL was resolved at all. + /// + /// A database whose schema could not be read is a bootstrap failure, not an + /// empty database — callers must fail loudly rather than report no results. + pub fn is_empty(&self) -> bool { + self.tables.is_empty() + } +} + +/// Resolved `message` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MessageColumns { + pub chat_row_id: Option, + pub sender_jid_row_id: Option, + pub from_me: Option, + pub timestamp: Option, + pub text_data: Option, + pub message_type: Option, + pub media_mime_type: Option, + pub media_name: Option, + pub starred: Option, + pub edit_version: Option, + pub key_id: Option, +} + +impl MessageColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + chat_row_id: cols.get("chat_row_id"), + sender_jid_row_id: cols.get("sender_jid_row_id"), + from_me: cols.get("from_me"), + timestamp: cols.get("timestamp"), + text_data: cols.get("text_data"), + message_type: cols.get("message_type"), + media_mime_type: cols.get("media_mime_type"), + media_name: cols.get("media_name"), + starred: cols.get("starred"), + edit_version: cols.get("edit_version"), + key_id: cols.get("key_id"), + } + } +} + +/// Resolved `jid` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct JidColumns { + pub raw_string: Option, +} + +impl JidColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + raw_string: cols.get("raw_string"), + } + } +} + +/// Resolved `chat` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ChatColumns { + pub jid_row_id: Option, + pub subject: Option, + pub archived: Option, +} + +impl ChatColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + jid_row_id: cols.get("jid_row_id"), + subject: cols.get("subject"), + archived: cols.get("archived"), + } + } +} + +/// Resolved `call_log` column positions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CallLogColumns { + pub jid_row_id: Option, + pub from_me: Option, + pub video_call: Option, + pub duration: Option, + pub timestamp: Option, + pub call_result: Option, + pub call_row_id: Option, + pub call_creator_device_jid_row_id: Option, +} + +impl CallLogColumns { + pub fn resolve(cols: &TableColumns) -> Self { + Self { + jid_row_id: cols.get("jid_row_id"), + from_me: cols.get("from_me"), + video_call: cols.get("video_call"), + duration: cols.get("duration"), + timestamp: cols.get("timestamp"), + call_result: cols.get("call_result"), + call_row_id: cols.get("call_row_id"), + call_creator_device_jid_row_id: cols.get("call_creator_device_jid_row_id"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The `message` DDL read out of a real 2023-era Android device. + const REAL_MESSAGE_DDL: &str = "CREATE TABLE message (\ + _id INTEGER PRIMARY KEY AUTOINCREMENT, chat_row_id INTEGER NOT NULL, \ + from_me INTEGER NOT NULL, key_id TEXT NOT NULL, sender_jid_row_id INTEGER, \ + status INTEGER, broadcast INTEGER, recipient_count INTEGER, \ + participant_hash TEXT, origination_flags INTEGER, origin INTEGER, \ + timestamp INTEGER, received_timestamp INTEGER, receipt_server_timestamp INTEGER, \ + message_type INTEGER, text_data TEXT, starred INTEGER, lookup_tables INTEGER, \ + sort_id INTEGER, message_add_on_flags INTEGER, view_mode INTEGER)"; + + const REAL_JID_DDL: &str = "CREATE TABLE jid (_id INTEGER PRIMARY KEY AUTOINCREMENT, \ + user TEXT NOT NULL, server TEXT NOT NULL, agent INTEGER, type INTEGER, \ + raw_string TEXT, device INTEGER)"; + + const REAL_CHAT_DDL: &str = "CREATE TABLE chat (_id INTEGER PRIMARY KEY AUTOINCREMENT, \ + jid_row_id INTEGER UNIQUE, hidden INTEGER, subject TEXT, created_timestamp INTEGER, \ + display_message_row_id INTEGER, last_message_row_id INTEGER, \ + last_read_message_row_id INTEGER, last_read_receipt_sent_message_row_id INTEGER, \ + last_important_message_row_id INTEGER, archived INTEGER, sort_timestamp INTEGER)"; + + const REAL_CALL_LOG_DDL: &str = "CREATE TABLE call_log (\ + _id INTEGER PRIMARY KEY AUTOINCREMENT, jid_row_id INTEGER, from_me INTEGER, \ + call_id TEXT, transaction_id INTEGER, timestamp INTEGER, video_call INTEGER, \ + duration INTEGER, call_result INTEGER, is_dnd_mode_on INTEGER)"; + + #[test] + fn resolves_declared_column_positions() { + let c = + TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, b TEXT, c INTEGER)"); + assert_eq!(c.get("_id"), Some(0)); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("c"), Some(2)); + assert_eq!(c.len(), 3); + } + + #[test] + fn lookup_is_case_insensitive() { + let c = TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, Text_Data TEXT)"); + assert_eq!(c.get("text_data"), Some(1)); + assert_eq!(c.get("TEXT_DATA"), Some(1)); + } + + #[test] + fn absent_column_resolves_to_none() { + let c = TableColumns::from_ddl("CREATE TABLE t (_id INTEGER PRIMARY KEY, b TEXT)"); + assert_eq!( + c.get("media_mime_type"), + None, + "a column the schema does not declare must not resolve to a neighbour" + ); + } + + #[test] + fn table_level_constraints_do_not_occupy_positions() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER, b INTEGER, c TEXT, \ + PRIMARY KEY (a, b), UNIQUE (c), \ + FOREIGN KEY (b) REFERENCES other (id), \ + CHECK (a > 0), CONSTRAINT uq1 UNIQUE (a, c))", + ); + assert_eq!(c.get("a"), Some(0)); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("c"), Some(2)); + assert_eq!( + c.len(), + 3, + "table constraints must not be counted as columns" + ); + } + + #[test] + fn commas_inside_parentheses_do_not_split_columns() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER PRIMARY KEY, b NUMERIC(10, 2), d TEXT)", + ); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("d"), Some(2)); + } + + #[test] + fn commas_inside_string_defaults_do_not_split_columns() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (a INTEGER PRIMARY KEY, b TEXT DEFAULT 'x,y', d TEXT)", + ); + assert_eq!(c.get("b"), Some(1)); + assert_eq!(c.get("d"), Some(2)); + } + + #[test] + fn quoted_identifiers_are_unwrapped() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (`_id` INTEGER PRIMARY KEY, \"text_data\" TEXT, [starred] INTEGER)", + ); + assert_eq!(c.get("_id"), Some(0)); + assert_eq!(c.get("text_data"), Some(1)); + assert_eq!(c.get("starred"), Some(2)); + } + + #[test] + fn quoted_table_name_and_if_not_exists_are_tolerated() { + let c = TableColumns::from_ddl( + "CREATE TABLE IF NOT EXISTS \"message\" (_id INTEGER PRIMARY KEY, timestamp INTEGER)", + ); + assert_eq!(c.get("timestamp"), Some(1)); + } + + #[test] + fn malformed_ddl_yields_empty_map_without_panicking() { + for ddl in ["", "CREATE TABLE t", "not sql at all", "CREATE TABLE t ("] { + let c = TableColumns::from_ddl(ddl); + assert!( + c.is_empty(), + "malformed DDL {ddl:?} must resolve to no columns" + ); + } + } + + #[test] + fn first_of_returns_the_first_declared_alias() { + let c = TableColumns::from_ddl( + "CREATE TABLE t (_id INTEGER PRIMARY KEY, parent_message_row_id INTEGER)", + ); + assert_eq!( + c.first_of(&["message_row_id", "parent_message_row_id"]), + Some(1) + ); + assert_eq!(c.first_of(&["nope", "also_nope"]), None); + } + + // ── Real-device DDL: the positions the extractor must resolve ──────────── + + #[test] + fn real_message_ddl_resolves_confirmed_positions() { + let c = TableColumns::from_ddl(REAL_MESSAGE_DDL); + assert_eq!(c.get("chat_row_id"), Some(1)); + assert_eq!(c.get("from_me"), Some(2)); + assert_eq!(c.get("key_id"), Some(3)); + assert_eq!(c.get("sender_jid_row_id"), Some(4)); + assert_eq!(c.get("timestamp"), Some(11)); + assert_eq!(c.get("message_type"), Some(14)); + assert_eq!(c.get("text_data"), Some(15)); + assert_eq!(c.get("starred"), Some(16)); + } + + #[test] + fn real_message_ddl_has_no_media_columns() { + let c = TableColumns::from_ddl(REAL_MESSAGE_DDL); + let m = MessageColumns::resolve(&c); + assert_eq!( + m.media_mime_type, None, + "modern schema moved media to message_media" + ); + assert_eq!(m.media_name, None); + assert_eq!(m.edit_version, None); + assert_eq!(m.timestamp, Some(11)); + assert_eq!(m.text_data, Some(15)); + } + + #[test] + fn real_jid_ddl_resolves_raw_string_not_user() { + let c = TableColumns::from_ddl(REAL_JID_DDL); + let j = JidColumns::resolve(&c); + assert_eq!( + j.raw_string, + Some(5), + "raw_string is at 5; `user` at 1 is the bare number" + ); + } + + #[test] + fn real_chat_ddl_resolves_subject_and_archived() { + let c = TableColumns::from_ddl(REAL_CHAT_DDL); + let ch = ChatColumns::resolve(&c); + assert_eq!(ch.jid_row_id, Some(1)); + assert_eq!(ch.subject, Some(3), "subject is at 3; `hidden` sits at 2"); + assert_eq!(ch.archived, Some(10)); + } + + #[test] + fn real_call_log_ddl_resolves_duration_and_video_call() { + let c = TableColumns::from_ddl(REAL_CALL_LOG_DDL); + let cl = CallLogColumns::resolve(&c); + assert_eq!(cl.jid_row_id, Some(1)); + assert_eq!(cl.from_me, Some(2)); + assert_eq!(cl.timestamp, Some(5)); + assert_eq!(cl.video_call, Some(6)); + assert_eq!(cl.duration, Some(7)); + assert_eq!(cl.call_result, Some(8)); + assert_eq!( + cl.call_row_id, None, + "no call_row_id column: calls must not be grouped by a neighbouring value" + ); + } + + // ── SchemaColumns ──────────────────────────────────────────────────────── + + #[test] + fn schema_columns_resolves_per_table() { + let mut ddl = HashMap::new(); + ddl.insert("message".to_string(), REAL_MESSAGE_DDL.to_string()); + ddl.insert("jid".to_string(), REAL_JID_DDL.to_string()); + let sc = SchemaColumns::from_ddl_map(&ddl); + assert!(!sc.is_empty()); + assert_eq!(sc.table("message").get("timestamp"), Some(11)); + assert_eq!(sc.table("jid").get("raw_string"), Some(5)); + } + + #[test] + fn schema_columns_unknown_table_is_empty_not_a_panic() { + let sc = SchemaColumns::from_ddl_map(&HashMap::new()); + assert!(sc.is_empty()); + assert!(sc.table("message").is_empty()); + assert_eq!(sc.table("message").get("timestamp"), None); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/src/contact_report.rs b/crates/plugins/chat4n6-whatsapp/src/contact_report.rs index d2aefc4..9265904 100644 --- a/crates/plugins/chat4n6-whatsapp/src/contact_report.rs +++ b/crates/plugins/chat4n6-whatsapp/src/contact_report.rs @@ -1,11 +1,10 @@ +use chrono::{DateTime, Datelike, FixedOffset, Timelike}; +use serde::{Deserialize, Serialize}; /// Per-contact HTML forensic dossier. /// /// Builds activity statistics for a single contact and renders /// a self-contained, no-external-deps HTML report. - use std::collections::HashMap; -use chrono::{DateTime, Datelike, FixedOffset, Timelike}; -use serde::{Deserialize, Serialize}; /// Activity heatmap: hour (0-23) → message count pub type HourlyHeatmap = [u32; 24]; @@ -71,7 +70,13 @@ pub(crate) fn html_escape(s: &str) -> String { fn extract_domains(text: &str) -> Vec { let mut domains = Vec::new(); for part in text.split("://").skip(1) { - let host = part.split('/').next().unwrap_or("").split('?').next().unwrap_or(""); + let host = part + .split('/') + .next() + .unwrap_or("") + .split('?') + .next() + .unwrap_or(""); let host = host.split('#').next().unwrap_or("").trim(); if !host.is_empty() { domains.push(host.to_lowercase()); @@ -143,7 +148,9 @@ pub fn build_contact_stats( if reaction.reactor_jid == contact_jid { *reactions_given.entry(reaction.emoji.clone()).or_insert(0) += 1; } else { - *reactions_received.entry(reaction.emoji.clone()).or_insert(0) += 1; + *reactions_received + .entry(reaction.emoji.clone()) + .or_insert(0) += 1; } } } @@ -300,12 +307,7 @@ mod tests { } } - fn make_media_msg( - id: i64, - sender_jid: Option<&str>, - from_me: bool, - ts_ms: i64, - ) -> Message { + fn make_media_msg(id: i64, sender_jid: Option<&str>, from_me: bool, ts_ms: i64) -> Message { Message { id, chat_id: 1, @@ -405,8 +407,20 @@ mod tests { #[test] fn test_stats_top_link_domains_sorted() { - let m1 = make_text_msg(1, Some(CONTACT_JID), false, 1000, "check https://example.com/foo"); - let m2 = make_text_msg(2, Some(CONTACT_JID), false, 2000, "also https://example.com/bar and https://other.com/baz"); + let m1 = make_text_msg( + 1, + Some(CONTACT_JID), + false, + 1000, + "check https://example.com/foo", + ); + let m2 = make_text_msg( + 2, + Some(CONTACT_JID), + false, + 2000, + "also https://example.com/bar and https://other.com/baz", + ); let msgs: Vec<&Message> = vec![&m1, &m2]; let stats = build_contact_stats(CONTACT_JID, None, &msgs, 0); assert!(!stats.top_link_domains.is_empty()); diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index 7d98713..c07ae33 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -1,4 +1,6 @@ -use crate::anti_forensics::{detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans}; +use crate::anti_forensics::{ + detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans, +}; use crate::schema::{cols, SchemaVersion}; pub use crate::schema::{default_mime_for_type, is_media_type, msg_type_label}; use anyhow::{Context, Result}; @@ -51,10 +53,7 @@ pub fn extract_from_msgstore( let jid_map = build_jid_map(tbl(&by_table, "jid")); // Build chat map: chat_id → Chat (populated with messages below) - let mut chats = build_chats( - tbl(&by_table, "chat"), - &jid_map, - ); + let mut chats = build_chats(tbl(&by_table, "chat"), &jid_map); // Map messages into chats. If the chat record was deleted/unrecovered, // create a stub so forensically-recovered messages are never silently dropped. @@ -100,7 +99,10 @@ pub fn extract_from_msgstore( } // Ghost recovery: upgrade tombstone messages whose original text // was preserved in message_quoted. - if matches!(&msg.content, MessageContent::Unknown(15) | MessageContent::Deleted) { + if matches!( + &msg.content, + MessageContent::Unknown(15) | MessageContent::Deleted + ) { if let Some(ghost_text) = ghost_map.get(&msg.id) { msg.content = MessageContent::GhostRecovered(ghost_text.clone()); } @@ -135,15 +137,12 @@ pub fn extract_from_msgstore( Some(SqlValue::Int(n)) => jid_map.get(n).cloned().unwrap_or_default(), _ => String::new(), }; - reactions_map - .entry(msg_row_id) - .or_default() - .push(Reaction { - emoji, - reactor_jid, - timestamp: ForensicTimestamp::from_millis(ts_ms, tz_offset_secs), - source: r.source.clone(), - }); + reactions_map.entry(msg_row_id).or_default().push(Reaction { + emoji, + reactor_jid, + timestamp: ForensicTimestamp::from_millis(ts_ms, tz_offset_secs), + source: r.source.clone(), + }); } } @@ -280,17 +279,15 @@ pub fn extract_from_msgstore( // ── §2.6 Anti-forensics detectors ──────────────────────────────────────── // Collect live message IDs for thumbnail orphan detection. - let live_message_ids: HashSet = chats.values() + let live_message_ids: HashSet = chats + .values() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); let total_messages = live_message_ids.len() as u32; // Collect message_thumbnails row IDs. let thumbnail_records = tbl(&by_table, "message_thumbnails"); - let thumbnail_row_ids: Vec = thumbnail_records - .iter() - .filter_map(|r| r.row_id) - .collect(); + let thumbnail_row_ids: Vec = thumbnail_records.iter().filter_map(|r| r.row_id).collect(); let chats_vec: Vec<_> = chats.into_values().collect(); @@ -362,7 +359,10 @@ pub fn extract_parallel( /// Look up a table name in a `partition_by_table` map and return its records as a slice. /// Returns an empty slice when the table is absent. -fn tbl<'a>(by: &'a HashMap>, name: &str) -> &'a [&'a RecoveredRecord] { +fn tbl<'a>( + by: &'a HashMap>, + name: &str, +) -> &'a [&'a RecoveredRecord] { by.get(name).map(|v| v.as_slice()).unwrap_or_default() } @@ -542,7 +542,12 @@ fn record_to_message( } else if msg_type == 53 || msg_type == 54 { // View-once image (53) or video (54) — media key/CDN URL may survive device deletion let mime = media_mime.unwrap_or_else(|| { - if msg_type == 54 { "video/mp4" } else { "image/jpeg" }.to_string() + if msg_type == 54 { + "video/mp4" + } else { + "image/jpeg" + } + .to_string() }); MessageContent::ViewOnce(MediaRef { file_path: media_name.unwrap_or_default(), @@ -841,8 +846,7 @@ fn build_ghost_map(records: &[&RecoveredRecord]) -> HashMap { /// Extract contacts from wa.db bytes using the forensic B-tree walker. /// wa_contacts table: [0]=Null(_id), [1]=jid, [2]=display_name, [3]=status, [4]=number pub fn extract_contacts(wa_db_bytes: &[u8]) -> Result> { - let engine = ForensicEngine::new(wa_db_bytes, None) - .context("failed to open wa.db")?; + let engine = ForensicEngine::new(wa_db_bytes, None).context("failed to open wa.db")?; let records = engine.recover_layer1().context("wa.db layer 1 recovery")?; let by_table = partition_by_table(&records); let contact_records = tbl(&by_table, "wa_contacts"); @@ -962,7 +966,10 @@ mod tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); assert_eq!(result.calls.len(), 1); // Fixture inserts call_result=1 (Connected) - assert_eq!(result.calls[0].call_result, chat4n6_plugin_api::CallResult::Connected); + assert_eq!( + result.calls[0].call_result, + chat4n6_plugin_api::CallResult::Connected + ); } // ── E5: quoted message tests ───────────────────────────────────────── @@ -1021,7 +1028,10 @@ mod tests { fn test_extract_contacts_from_wa_db() { let wa_db = make_wa_db(); let contacts = extract_contacts(&wa_db).unwrap(); - assert!(contacts.len() >= 2, "should have at least 2 contacts with names"); + assert!( + contacts.len() >= 2, + "should have at least 2 contacts with names" + ); let alice = contacts .iter() .find(|c| c.jid == "4155550100@s.whatsapp.net") @@ -1108,9 +1118,16 @@ mod tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let chat1 = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); // Message 6 has message_type=15 (tombstone, no message_quoted entry) → Deleted - let msg6 = chat1.messages.iter().find(|m| m.id == 6).expect("msg 6 (tombstone)"); + let msg6 = chat1 + .messages + .iter() + .find(|m| m.id == 6) + .expect("msg 6 (tombstone)"); assert!( - matches!(&msg6.content, MessageContent::Deleted | MessageContent::GhostRecovered(_)), + matches!( + &msg6.content, + MessageContent::Deleted | MessageContent::GhostRecovered(_) + ), "msg_type=15 tombstone should produce Deleted or GhostRecovered, got {:?}", msg6.content ); @@ -1133,14 +1150,18 @@ mod tests { #[test] fn schema_version_is_read_from_pragma_user_version() { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch("PRAGMA user_version = 215;").unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - assert_eq!(result.schema_version, 215, - "schema_version must be read from PRAGMA user_version, not hardcoded"); + assert_eq!( + result.schema_version, 215, + "schema_version must be read from PRAGMA user_version, not hardcoded" + ); } // ── C8: ghost message recovery from message_quoted ─────────────────── @@ -1148,30 +1169,39 @@ mod tests { #[test] fn ghost_message_recovered_from_message_quoted() { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch( "INSERT INTO message_quoted VALUES (99, 6, 1, NULL, 1, 1710513600000, 'Secret deleted message', 0, NULL, NULL);", ).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); let db = std::fs::read(tmp.path()).unwrap(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let ghost = result.chats.iter() + let ghost = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .find(|m| matches!(&m.content, MessageContent::GhostRecovered(_))); - assert!(ghost.is_some(), "msg_type=15 with message_quoted entry must produce GhostRecovered"); + assert!( + ghost.is_some(), + "msg_type=15 with message_quoted entry must produce GhostRecovered" + ); if let Some(MessageContent::GhostRecovered(text)) = ghost.map(|m| &m.content) { - assert!(text.contains("Secret deleted message"), - "GhostRecovered text must contain the quoted text_data"); + assert!( + text.contains("Secret deleted message"), + "GhostRecovered text must contain the quoted text_data" + ); } } #[test] fn test_is_media_type_helper() { - assert!(is_media_type(1)); // image - assert!(is_media_type(2)); // audio - assert!(is_media_type(3)); // video - assert!(is_media_type(8)); // document + assert!(is_media_type(1)); // image + assert!(is_media_type(2)); // audio + assert!(is_media_type(3)); // video + assert!(is_media_type(8)); // document assert!(is_media_type(13)); // gif assert!(is_media_type(20)); // sticker assert!(!is_media_type(0)); // text @@ -1216,7 +1246,8 @@ mod tests { INSERT INTO call_log VALUES (1, 1, 0, 0, 90, 1710513500000, 1, NULL, 2); "#).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1273,7 +1304,8 @@ mod tests { INSERT INTO call_log VALUES (3, 1, 1, 0, 60, 1710513500000, 1, NULL, NULL); "#).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1282,7 +1314,11 @@ mod tests { let db = make_msgstore_with_group_call(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let group_calls: Vec<_> = result.calls.iter().filter(|c| c.group_call).collect(); - assert_eq!(group_calls.len(), 1, "two rows with same call_row_id → 1 merged record"); + assert_eq!( + group_calls.len(), + 1, + "two rows with same call_row_id → 1 merged record" + ); assert_eq!(group_calls[0].participants.len(), 2); } @@ -1290,8 +1326,14 @@ mod tests { fn test_group_call_participants_contain_both_jids() { let db = make_msgstore_with_group_call(); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let gc = result.calls.iter().find(|c| c.group_call).expect("no group call"); - assert!(gc.participants.contains(&"alice@s.whatsapp.net".to_string())); + let gc = result + .calls + .iter() + .find(|c| c.group_call) + .expect("no group call"); + assert!(gc + .participants + .contains(&"alice@s.whatsapp.net".to_string())); assert!(gc.participants.contains(&"bob@s.whatsapp.net".to_string())); } @@ -1487,7 +1529,10 @@ mod features_i1_i8_tests { let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let chat1 = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); let msg2 = chat1.messages.iter().find(|m| m.id == 2).expect("msg 2"); - assert!(msg2.starred, "msg 2 starred=1 in DB must produce Message.starred=true"); + assert!( + msg2.starred, + "msg 2 starred=1 in DB must produce Message.starred=true" + ); let msg1 = chat1.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!(!msg1.starred, "msg 1 starred=0 must be false"); } @@ -1519,7 +1564,10 @@ mod features_i1_i8_tests { Some(8), "msg 5 forward_score should be Some(8)" ); - assert!(msg5.is_forwarded, "msg 5 with forward_score=8 must be is_forwarded=true"); + assert!( + msg5.is_forwarded, + "msg 5 with forward_score=8 must be is_forwarded=true" + ); } } @@ -1530,10 +1578,12 @@ mod proptest_redo_tests { fn make_db_with_sql(extra_sql: &str) -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/modern_schema.sql")) + .unwrap(); conn.execute_batch(extra_sql).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1545,11 +1595,16 @@ mod proptest_redo_tests { VALUES (1, 1, 0, 1710514000000, NULL, 53, 'image/jpeg', 'Media/WhatsApp View Once/VIEW-001.jpg');", ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let view_once_count = result.chats.iter() + let view_once_count = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .filter(|m| matches!(&m.content, MessageContent::ViewOnce(_))) .count(); - assert_eq!(view_once_count, 1, "msg_type 53 must produce ViewOnce, not Unknown"); + assert_eq!( + view_once_count, 1, + "msg_type 53 must produce ViewOnce, not Unknown" + ); } #[test] @@ -1560,7 +1615,9 @@ mod proptest_redo_tests { VALUES (1, 1, 0, 1710514100000, NULL, 54, 'video/mp4', 'Media/WhatsApp View Once/VIEW-002.mp4');", ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let view_once_count = result.chats.iter() + let view_once_count = result + .chats + .iter() .flat_map(|c| c.messages.iter()) .filter(|m| matches!(&m.content, MessageContent::ViewOnce(_))) .count(); @@ -1576,7 +1633,10 @@ mod proptest_redo_tests { ); let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); let archived_chats: Vec<_> = result.chats.iter().filter(|c| c.archived).collect(); - assert!(!archived_chats.is_empty(), "archived=1 in DB must produce Chat.archived=true"); + assert!( + !archived_chats.is_empty(), + "archived=1 in DB must produce Chat.archived=true" + ); } #[test] @@ -1691,7 +1751,8 @@ mod media_type_extended_tests { INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, NULL, {}, NULL, NULL, 0, 0, NULL); "#, msg_type_val)).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1703,7 +1764,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(mr) if mr.mime_type.contains("geo")), - "msg_type=5 (location) should produce Media with geo mime, got: {:?}", msg.content + "msg_type=5 (location) should produce Media with geo mime, got: {:?}", + msg.content ); } @@ -1715,7 +1777,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(_)), - "msg_type=42 (live location) should produce Media, got: {:?}", msg.content + "msg_type=42 (live location) should produce Media, got: {:?}", + msg.content ); } @@ -1727,7 +1790,8 @@ mod media_type_extended_tests { let msg = chat.messages.iter().find(|m| m.id == 1).expect("msg 1"); assert!( matches!(&msg.content, MessageContent::Media(mr) if mr.mime_type == "text/vcard"), - "msg_type=64 (contact card) should produce Media with text/vcard, got: {:?}", msg.content + "msg_type=64 (contact card) should produce Media with text/vcard, got: {:?}", + msg.content ); } @@ -1754,7 +1818,8 @@ mod streaming_extraction_tests { fn make_multi_message_db(msg_count: usize) -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 200; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); @@ -1774,15 +1839,23 @@ mod streaming_extraction_tests { ); INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); INSERT INTO chat VALUES (1, 1, NULL); - "#).unwrap(); + "#, + ) + .unwrap(); for i in 1..=msg_count { conn.execute( "INSERT INTO message VALUES (?, 1, NULL, 0, ?, ?, 0, NULL, NULL, 0, 0, NULL)", - rusqlite::params![i as i64, (1710000000000i64 + i as i64 * 1000), format!("msg {}", i)], - ).unwrap(); + rusqlite::params![ + i as i64, + (1710000000000i64 + i as i64 * 1000), + format!("msg {}", i) + ], + ) + .unwrap(); } let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1793,7 +1866,8 @@ mod streaming_extraction_tests { let count_clone = Arc::clone(&count); extract_streaming(&db, 0, SchemaVersion::Modern, |_msg| { count_clone.fetch_add(1, Ordering::SeqCst); - }).expect("extract_streaming should succeed"); + }) + .expect("extract_streaming should succeed"); assert_eq!( count.load(Ordering::SeqCst), 10, @@ -1807,7 +1881,9 @@ mod streaming_extraction_tests { // Batch extraction let batch = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let mut batch_ids: Vec = batch.chats.iter() + let mut batch_ids: Vec = batch + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); batch_ids.sort(); @@ -1816,7 +1892,8 @@ mod streaming_extraction_tests { let mut streaming_ids = Vec::new(); extract_streaming(&db, 0, SchemaVersion::Modern, |msg| { streaming_ids.push(msg.id); - }).expect("extract_streaming should succeed"); + }) + .expect("extract_streaming should succeed"); streaming_ids.sort(); assert_eq!( @@ -1830,13 +1907,17 @@ mod streaming_extraction_tests { let db = make_multi_message_db(20); let serial = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); - let mut serial_ids: Vec = serial.chats.iter() + let mut serial_ids: Vec = serial + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); serial_ids.sort(); let parallel = extract_parallel(&db, 0, SchemaVersion::Modern).unwrap(); - let mut parallel_ids: Vec = parallel.chats.iter() + let mut parallel_ids: Vec = parallel + .chats + .iter() .flat_map(|c| c.messages.iter().map(|m| m.id)) .collect(); parallel_ids.sort(); @@ -1854,9 +1935,11 @@ mod fts5_tests { fn make_fts5_db() -> Vec { let conn = rusqlite::Connection::open_in_memory().unwrap(); - conn.execute_batch(include_str!("../tests/fixtures/fts5_schema.sql")).unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/fts5_schema.sql")) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -1865,10 +1948,14 @@ mod fts5_tests { let db = make_fts5_db(); let fragments = extract_fts5_content(&db).unwrap(); // Should find at least one _content table with text fragments - let content_tables: Vec<_> = fragments.keys() + let content_tables: Vec<_> = fragments + .keys() .filter(|k| k.ends_with("_content")) .collect(); - assert!(!content_tables.is_empty(), "must find at least one FTS5 _content table"); + assert!( + !content_tables.is_empty(), + "must find at least one FTS5 _content table" + ); let all_texts: Vec<_> = fragments.values().flatten().collect(); assert!( all_texts.iter().any(|t| t.contains("forensics")), @@ -1961,7 +2048,8 @@ mod edit_version_tests { INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, 'original text', 0, NULL, NULL, 0, {}); "#, edit_version_val, edit_version_val)).unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -2043,7 +2131,11 @@ mod tbl_helper_tests { let mut map: HashMap> = HashMap::new(); map.insert("message".to_string(), vec![&record]); let result = tbl(&map, "message"); - assert_eq!(result.len(), 1, "known key must return the inserted records"); + assert_eq!( + result.len(), + 1, + "known key must return the inserted records" + ); } } @@ -2061,7 +2153,8 @@ mod truncated_record_tests { let conn = rusqlite::Connection::open_in_memory().unwrap(); // Deliberately omit media_mime_type, media_name, starred, edit_version // so that column indices 7, 8, 9, 10 are absent from the physical row. - conn.execute_batch(r#" + conn.execute_batch( + r#" PRAGMA user_version = 100; CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); @@ -2077,9 +2170,12 @@ mod truncated_record_tests { INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); INSERT INTO chat VALUES (1, 1, NULL); INSERT INTO message VALUES (1, 1, NULL, 1, 1710513500000, 'hello short', 0); - "#).unwrap(); + "#, + ) + .unwrap(); let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -2105,3 +2201,208 @@ mod truncated_record_tests { ); } } + +// ── T1: extraction against the real 2023-era device schema ─────────────────── + +/// These exercise the same code path as the simplified fixtures, but against +/// the `CREATE TABLE` shapes read out of a real device. Every assertion here +/// is one the hardcoded-ordinal reader gets wrong: it lands on +/// `sender_jid_row_id` instead of `timestamp`, `user` instead of `raw_string`, +/// `hidden` instead of `subject`, and `duration` instead of `call_row_id`. +#[cfg(test)] +mod real_schema_tests { + use super::*; + use chrono::Datelike; + + fn make_real_msgstore() -> Vec { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/real_modern_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + std::fs::read(tmp.path()).unwrap() + } + + fn all_messages(result: &ExtractionResult) -> Vec<&Message> { + result + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .collect() + } + + #[test] + fn message_timestamps_are_2022_not_epoch_zero() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + assert_eq!(msgs.len(), 3, "all three messages must be extracted"); + for m in &msgs { + assert_eq!( + m.timestamp.utc.year(), + 2022, + "message {} timestamp resolved to {} — the reader is on the wrong column", + m.id, + m.timestamp.utc + ); + } + } + + #[test] + fn message_text_comes_from_text_data() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let texts: Vec<&str> = all_messages(&result) + .iter() + .filter_map(|m| match &m.content { + MessageContent::Text(s) => Some(s.as_str()), + _ => None, + }) + .collect(); + assert!( + texts.contains(&"Incoming real-schema message"), + "text_data must be read by name, got: {texts:?}" + ); + } + + #[test] + fn sender_resolves_via_jid_raw_string_not_bare_user() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let received = all_messages(&result) + .into_iter() + .find(|m| m.id == 2) + .expect("message 2"); + assert_eq!( + received.sender_jid.as_deref(), + Some("4155550100@s.whatsapp.net"), + "sender must come from jid.raw_string, not the bare `user` column" + ); + } + + #[test] + fn from_me_is_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + let outgoing = msgs.iter().find(|m| m.id == 1).expect("message 1"); + let incoming = msgs.iter().find(|m| m.id == 2).expect("message 2"); + assert!(outgoing.from_me, "message 1 has from_me = 1"); + assert!(!incoming.from_me, "message 2 has from_me = 0"); + } + + #[test] + fn starred_flag_is_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let msgs = all_messages(&result); + assert!( + msgs.iter().find(|m| m.id == 2).expect("message 2").starred, + "message 2 has starred = 1 at DDL position 16" + ); + assert!( + !msgs.iter().find(|m| m.id == 1).expect("message 1").starred, + "message 1 has starred = 0" + ); + } + + #[test] + fn chat_subject_and_archived_are_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let group = result.chats.iter().find(|c| c.id == 2).expect("chat 2"); + assert_eq!( + group.name.as_deref(), + Some("Real Schema Group"), + "subject sits at position 3; position 2 is `hidden`" + ); + assert!(group.archived, "chat 2 has archived = 1 at position 10"); + let direct = result.chats.iter().find(|c| c.id == 1).expect("chat 1"); + assert!(!direct.archived, "chat 1 has archived = 0"); + } + + #[test] + fn chat_jid_resolves_and_group_is_flagged() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let group = result.chats.iter().find(|c| c.id == 2).expect("chat 2"); + assert_eq!(group.jid, "120363001234567890@g.us"); + assert!(group.is_group); + } + + #[test] + fn calls_are_not_collapsed_by_a_mistaken_grouping_key() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + assert_eq!( + result.calls.len(), + 3, + "three call_log rows must yield three records; the real schema has no \ + call_row_id column, so nothing may be merged" + ); + assert!( + result.calls.iter().all(|c| !c.group_call), + "no call may be flagged as a group call" + ); + } + + #[test] + fn call_duration_and_video_flag_are_read_by_name() { + let db = make_real_msgstore(); + let result = extract_from_msgstore(&db, 0, SchemaVersion::Modern).unwrap(); + let video = result + .calls + .iter() + .find(|c| c.call_id == 2) + .expect("call 2"); + assert!(video.video, "call 2 has video_call = 1 at position 6"); + assert_eq!(video.duration_secs, 42, "duration sits at position 7"); + let voice = result + .calls + .iter() + .find(|c| c.call_id == 1) + .expect("call 1"); + assert!(!voice.video); + assert_eq!(voice.duration_secs, 137); + } + + /// The invariant every resolved ordinal rests on: SQLite writes the + /// `INTEGER PRIMARY KEY` column as a NULL **at its declared position**, so + /// `values[i]` is the column at DDL position `i` — even when the rowid + /// alias is not declared first. + #[test] + fn rowid_alias_is_null_at_its_declared_position() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t (a TEXT, _id INTEGER PRIMARY KEY, c INTEGER); + INSERT INTO t VALUES ('first', 7, 42);", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let engine = ForensicEngine::new(&db, None).unwrap(); + let records = engine.recover_layer1().unwrap(); + let r = records.iter().find(|r| r.table == "t").expect("table t"); + + assert_eq!(r.row_id, Some(7), "rowid comes from the cell header"); + assert!( + matches!(r.values.first(), Some(SqlValue::Text(s)) if s == "first"), + "position 0 is the declared first column, got {:?}", + r.values.first() + ); + assert!( + matches!(r.values.get(1), Some(SqlValue::Null)), + "the rowid alias reads as Null at its own declared position, got {:?}", + r.values.get(1) + ); + assert!( + matches!(r.values.get(2), Some(SqlValue::Int(42))), + "position 2 is the declared third column, got {:?}", + r.values.get(2) + ); + } +} diff --git a/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs b/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs index 5ac85f4..ddad0a0 100644 --- a/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs +++ b/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs @@ -2,15 +2,35 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum GroupChangeKind { - SubjectChanged { old: Option, new: String }, - IconChanged { old_jpeg_b64: Option, new_jpeg_b64: Option }, - DescriptionChanged { old: Option, new: String }, - AdminOnlyEditChanged { admins_only: bool }, - AdminOnlySendChanged { admins_only: bool }, - DisappearingTimerChanged { old_secs: Option, new_secs: Option }, + SubjectChanged { + old: Option, + new: String, + }, + IconChanged { + old_jpeg_b64: Option, + new_jpeg_b64: Option, + }, + DescriptionChanged { + old: Option, + new: String, + }, + AdminOnlyEditChanged { + admins_only: bool, + }, + AdminOnlySendChanged { + admins_only: bool, + }, + DisappearingTimerChanged { + old_secs: Option, + new_secs: Option, + }, InviteLinkReset, - ApprovalModeChanged { requires_approval: bool }, - MembershipApprovalChanged { requires_approval: bool }, + ApprovalModeChanged { + requires_approval: bool, + }, + MembershipApprovalChanged { + requires_approval: bool, + }, Unknown(i32), } @@ -58,8 +78,12 @@ pub fn parse_group_change( new_secs: new_value.and_then(|s| s.parse::().ok()), }, 83 => GroupChangeKind::InviteLinkReset, - 84 => GroupChangeKind::ApprovalModeChanged { requires_approval: true }, - 85 => GroupChangeKind::ApprovalModeChanged { requires_approval: false }, + 84 => GroupChangeKind::ApprovalModeChanged { + requires_approval: true, + }, + 85 => GroupChangeKind::ApprovalModeChanged { + requires_approval: false, + }, other => GroupChangeKind::Unknown(other), }; @@ -104,7 +128,11 @@ mod tests { fn test_icon_changed_jpeg_b64_preserved() { let fake_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ"; let c = parse(6, Some(fake_b64), Some(fake_b64)); - if let GroupChangeKind::IconChanged { old_jpeg_b64, new_jpeg_b64 } = c { + if let GroupChangeKind::IconChanged { + old_jpeg_b64, + new_jpeg_b64, + } = c + { assert_eq!(old_jpeg_b64, Some(fake_b64.to_string())); assert_eq!(new_jpeg_b64, Some(fake_b64.to_string())); } else { @@ -121,25 +149,37 @@ mod tests { #[test] fn test_admin_only_edit_on() { let c = parse(29, None, None); - assert_eq!(c, GroupChangeKind::AdminOnlyEditChanged { admins_only: true }); + assert_eq!( + c, + GroupChangeKind::AdminOnlyEditChanged { admins_only: true } + ); } #[test] fn test_admin_only_edit_off() { let c = parse(30, None, None); - assert_eq!(c, GroupChangeKind::AdminOnlyEditChanged { admins_only: false }); + assert_eq!( + c, + GroupChangeKind::AdminOnlyEditChanged { admins_only: false } + ); } #[test] fn test_admin_only_send_on() { let c = parse(31, None, None); - assert_eq!(c, GroupChangeKind::AdminOnlySendChanged { admins_only: true }); + assert_eq!( + c, + GroupChangeKind::AdminOnlySendChanged { admins_only: true } + ); } #[test] fn test_admin_only_send_off() { let c = parse(32, None, None); - assert_eq!(c, GroupChangeKind::AdminOnlySendChanged { admins_only: false }); + assert_eq!( + c, + GroupChangeKind::AdminOnlySendChanged { admins_only: false } + ); } #[test] @@ -162,7 +202,12 @@ mod tests { #[test] fn test_approval_on() { let c = parse(84, None, None); - assert_eq!(c, GroupChangeKind::ApprovalModeChanged { requires_approval: true }); + assert_eq!( + c, + GroupChangeKind::ApprovalModeChanged { + requires_approval: true + } + ); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/lib.rs b/crates/plugins/chat4n6-whatsapp/src/lib.rs index 600607b..605d608 100644 --- a/crates/plugins/chat4n6-whatsapp/src/lib.rs +++ b/crates/plugins/chat4n6-whatsapp/src/lib.rs @@ -2,25 +2,26 @@ pub mod anti_forensics; +pub mod album; pub mod cdn; +pub mod columns; pub mod contact_report; pub mod decrypt; pub mod extractor; -pub mod orphaned_media; -pub mod platform; -pub mod system_event; -pub mod schema; -pub mod timezone; -pub mod album; -pub mod poll; pub mod group_metadata; +pub mod link; pub mod location; pub mod mention; +pub mod orphaned_media; pub mod pin; +pub mod platform; +pub mod poll; +pub mod schema; pub mod status; -pub mod link; +pub mod system_event; +pub mod timezone; -use crate::extractor::{extract_contacts, extract_from_msgstore, build_contact_names}; +use crate::extractor::{build_contact_names, extract_contacts, extract_from_msgstore}; use crate::schema::detect_schema_version; use anyhow::{Context, Result}; use chat4n6_plugin_api::{ExtractionResult, ForensicFs, ForensicPlugin}; @@ -156,10 +157,9 @@ fn detect_timezone(fs: &dyn ForensicFs) -> i32 { impl WhatsAppPlugin { fn try_decrypt(&self, bytes: &[u8]) -> Result> { - let key = self - .key - .as_ref() - .ok_or_else(|| anyhow::anyhow!("database is not SQLite and no decryption key provided"))?; + let key = self.key.as_ref().ok_or_else(|| { + anyhow::anyhow!("database is not SQLite and no decryption key provided") + })?; // Try crypt15 first, then crypt14 if let Ok(plain) = decrypt::decrypt_db(bytes, key, decrypt::CryptVersion::Crypt15) { return Ok(plain); @@ -299,7 +299,8 @@ mod tests { fn db_to_bytes(conn: &rusqlite::Connection) -> Vec { let tmp = tempfile::NamedTempFile::new().unwrap(); - conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None).unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); std::fs::read(tmp.path()).unwrap() } @@ -332,7 +333,8 @@ mod tests { jid TEXT, display_name TEXT, status TEXT, number TEXT); INSERT INTO wa_contacts VALUES (1, '{jid}', '{display_name}', NULL, NULL); " - )).unwrap(); + )) + .unwrap(); db_to_bytes(&conn) } @@ -340,9 +342,7 @@ mod tests { fn wa_db_contact_names_applied_to_chats() { let msgstore = minimal_msgstore("alice@s.whatsapp.net"); let wa_db = minimal_wa_db("alice@s.whatsapp.net", "Alice"); - let fs = MockFs::new() - .add(DB_PATH, msgstore) - .add(WA_DB_PATH, wa_db); + let fs = MockFs::new().add(DB_PATH, msgstore).add(WA_DB_PATH, wa_db); let result = WhatsAppPlugin::new().extract(&fs, None).unwrap(); let chat = result.chats.first().expect("expected at least one chat"); assert_eq!( @@ -355,9 +355,10 @@ mod tests { #[test] fn timezone_autodetect_from_property_file() { let msgstore = minimal_msgstore("test@s.whatsapp.net"); - let fs = MockFs::new() - .add(DB_PATH, msgstore) - .add("data/property/persist.sys.timezone", b"Asia/Manila\n".to_vec()); + let fs = MockFs::new().add(DB_PATH, msgstore).add( + "data/property/persist.sys.timezone", + b"Asia/Manila\n".to_vec(), + ); let result = WhatsAppPlugin::new().extract(&fs, None).unwrap(); assert_eq!( result.timezone_offset_seconds, diff --git a/crates/plugins/chat4n6-whatsapp/src/link.rs b/crates/plugins/chat4n6-whatsapp/src/link.rs index 41bb6f7..8a811e8 100644 --- a/crates/plugins/chat4n6-whatsapp/src/link.rs +++ b/crates/plugins/chat4n6-whatsapp/src/link.rs @@ -38,7 +38,10 @@ pub fn parse_url_components(url: &str) -> Option { let caps = re.captures(url)?; let scheme = caps.get(1)?.as_str().to_lowercase(); let domain = caps.get(2)?.as_str().to_string(); - let path = caps.get(3).map(|m| m.as_str().to_string()).unwrap_or_default(); + let path = caps + .get(3) + .map(|m| m.as_str().to_string()) + .unwrap_or_default(); Some(ExtractedLink { url: caps.get(0)?.as_str().to_string(), scheme, @@ -80,7 +83,11 @@ mod tests { fn test_url_with_path_and_query() { let urls = extract_urls("https://example.com/path/to/page?q=hello&lang=en"); assert_eq!(urls.len(), 1); - assert!(urls[0].path.contains("path/to/page"), "path: {}", urls[0].path); + assert!( + urls[0].path.contains("path/to/page"), + "path: {}", + urls[0].path + ); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/location.rs b/crates/plugins/chat4n6-whatsapp/src/location.rs index bc7f5c0..2930ca5 100644 --- a/crates/plugins/chat4n6-whatsapp/src/location.rs +++ b/crates/plugins/chat4n6-whatsapp/src/location.rs @@ -141,9 +141,15 @@ mod tests { #[test] fn test_osm_url_format() { let url = osm_url(51.5074, -0.1278); - assert!(url.starts_with("https://www.openstreetmap.org"), "url: {url}"); + assert!( + url.starts_with("https://www.openstreetmap.org"), + "url: {url}" + ); assert!(url.contains("51.5074"), "url: {url}"); - assert!(url.contains("-0.1278") || url.contains("0.1278"), "url: {url}"); + assert!( + url.contains("-0.1278") || url.contains("0.1278"), + "url: {url}" + ); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs b/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs index c55c950..e3d272a 100644 --- a/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs +++ b/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs @@ -7,14 +7,14 @@ pub struct OrphanedMedia { pub file_path: PathBuf, pub file_size: u64, pub extension: String, - pub file_hash: Option, // SHA-256 hex, computed lazily - pub linked_media_path: Option, // set after rescue pass + pub file_hash: Option, // SHA-256 hex, computed lazily + pub linked_media_path: Option, // set after rescue pass } /// Recognized media file extensions (lowercase). const RECOGNIZED_EXTENSIONS: &[&str] = &[ - "jpg", "jpeg", "png", "gif", "mp4", "mov", "avi", "opus", "ogg", "mp3", "aac", - "pdf", "doc", "docx", "webp", + "jpg", "jpeg", "png", "gif", "mp4", "mov", "avi", "opus", "ogg", "mp3", "aac", "pdf", "doc", + "docx", "webp", ]; fn is_recognized_extension(ext: &str) -> bool { @@ -24,10 +24,7 @@ fn is_recognized_extension(ext: &str) -> bool { /// Walk `media_dir` and find files with recognized extensions that are NOT /// in the `known_paths` set. Return them as OrphanedMedia records (file_hash=None initially). -pub fn scan_orphaned_media( - media_dir: &Path, - known_paths: &HashSet, -) -> Vec { +pub fn scan_orphaned_media(media_dir: &Path, known_paths: &HashSet) -> Vec { let mut orphans = Vec::new(); let read_dir = match std::fs::read_dir(media_dir) { Ok(rd) => rd, @@ -149,7 +146,11 @@ mod tests { let mut known = HashSet::new(); known.insert(path.to_string_lossy().to_string()); let orphans = scan_orphaned_media(dir.path(), &known); - assert_eq!(orphans.len(), 1, "should find only the orphan, not the known file"); + assert_eq!( + orphans.len(), + 1, + "should find only the orphan, not the known file" + ); assert!(orphans[0].file_path.file_name().unwrap() == "orphan.png"); } @@ -185,7 +186,10 @@ mod tests { let known: HashSet = HashSet::new(); let orphans = scan_orphaned_media(dir.path(), &known); assert_eq!(orphans.len(), 1); - assert!(orphans[0].file_hash.is_none(), "file_hash should be None before hashing"); + assert!( + orphans[0].file_hash.is_none(), + "file_hash should be None before hashing" + ); } // ── hash_orphans tests ──────────────────────────────────────────────────── @@ -220,7 +224,10 @@ mod tests { hash_orphans(&mut orphans); for o in &orphans { - assert!(o.file_hash.is_some(), "all orphans should have hashes after hash_orphans"); + assert!( + o.file_hash.is_some(), + "all orphans should have hashes after hash_orphans" + ); } } @@ -240,17 +247,21 @@ mod tests { let hash_hex = hex::encode(Sha256::digest(content)); // Convert hex to base64 for the missing_media format use base64::Engine; - let hash_b64 = base64::engine::general_purpose::STANDARD.encode( - &hex::decode(&hash_hex).unwrap() - ); + let hash_b64 = + base64::engine::general_purpose::STANDARD.encode(&hex::decode(&hash_hex).unwrap()); - let missing = vec![ - ("expected/path/media.jpg".to_string(), content.len() as u64, Some(hash_b64)), - ]; + let missing = vec![( + "expected/path/media.jpg".to_string(), + content.len() as u64, + Some(hash_b64), + )]; let matched = rescue_orphans(&mut orphans, &missing); assert_eq!(matched, 1, "should match 1 orphan"); - assert!(orphans[0].linked_media_path.is_some(), "linked_media_path should be set after rescue"); + assert!( + orphans[0].linked_media_path.is_some(), + "linked_media_path should be set after rescue" + ); } #[test] @@ -265,12 +276,17 @@ mod tests { use base64::Engine; let wrong_hash_b64 = base64::engine::general_purpose::STANDARD.encode(b"wrong hash bytes"); - let missing = vec![ - ("expected/path/media.jpg".to_string(), b"actual content".len() as u64, Some(wrong_hash_b64)), - ]; + let missing = vec![( + "expected/path/media.jpg".to_string(), + b"actual content".len() as u64, + Some(wrong_hash_b64), + )]; let matched = rescue_orphans(&mut orphans, &missing); - assert_eq!(matched, 0, "should not match when hash differs even if size matches"); + assert_eq!( + matched, 0, + "should not match when hash differs even if size matches" + ); } #[test] @@ -282,9 +298,7 @@ mod tests { let mut orphans = scan_orphaned_media(dir.path(), &known); hash_orphans(&mut orphans); - let missing = vec![ - ("expected/path/media.jpg".to_string(), 99999u64, None), - ]; + let missing = vec![("expected/path/media.jpg".to_string(), 99999u64, None)]; let matched = rescue_orphans(&mut orphans, &missing); assert_eq!(matched, 0, "should not match when size differs"); @@ -302,10 +316,12 @@ mod tests { let mut orphans = scan_orphaned_media(dir.path(), &known); hash_orphans(&mut orphans); - use sha2::{Digest, Sha256}; use base64::Engine; - let h1 = base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content1).as_slice()); - let h2 = base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content2).as_slice()); + use sha2::{Digest, Sha256}; + let h1 = + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content1).as_slice()); + let h2 = + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content2).as_slice()); let missing = vec![ ("path/a.jpg".to_string(), content1.len() as u64, Some(h1)), diff --git a/crates/plugins/chat4n6-whatsapp/src/platform.rs b/crates/plugins/chat4n6-whatsapp/src/platform.rs index 094756e..2eb5c52 100644 --- a/crates/plugins/chat4n6-whatsapp/src/platform.rs +++ b/crates/plugins/chat4n6-whatsapp/src/platform.rs @@ -4,11 +4,11 @@ use serde::{Deserialize, Serialize}; pub enum SenderPlatform { Android, IPhone, - Companion, // Web/Desktop linked device - AndroidLinked, // secondary Android device - IPhoneLinked, // secondary iPhone device - BusinessApi, // WhatsApp Business Cloud API bot - OldAndroid, // numeric key_id ≤10 chars + Companion, // Web/Desktop linked device + AndroidLinked, // secondary Android device + IPhoneLinked, // secondary iPhone device + BusinessApi, // WhatsApp Business Cloud API bot + OldAndroid, // numeric key_id ≤10 chars Unknown, } @@ -77,9 +77,7 @@ pub fn classify_key_id( platform: SenderPlatform::IPhone, confidence: 0.95, } - } else if upper.starts_with("3F") - || upper.starts_with("3E") - || upper.starts_with("3B") + } else if upper.starts_with("3F") || upper.starts_with("3E") || upper.starts_with("3B") { PlatformClassification { platform: SenderPlatform::Companion, @@ -164,14 +162,20 @@ mod tests { fn test_len8_from_me_true_android_085() { let (p, c) = classify("ABCD1234", true, None); assert_eq!(p, SenderPlatform::Android); - assert!((c - 0.85).abs() < 0.001, "confidence should be 0.85, got {c}"); + assert!( + (c - 0.85).abs() < 0.001, + "confidence should be 0.85, got {c}" + ); } #[test] fn test_len8_from_me_false_android_075() { let (p, c) = classify("ABCD1234", false, None); assert_eq!(p, SenderPlatform::Android); - assert!((c - 0.75).abs() < 0.001, "confidence should be 0.75, got {c}"); + assert!( + (c - 0.75).abs() < 0.001, + "confidence should be 0.75, got {c}" + ); } // ── len=16 tests ───────────────────────────────────────────────────────── @@ -187,7 +191,10 @@ mod tests { fn test_len16_prefix_ac_lowercase_android_097() { let (p, c) = classify("ac1234567890abcd", false, None); assert_eq!(p, SenderPlatform::Android); - assert!((c - 0.97).abs() < 0.001, "case-insensitive prefix match should give 0.97"); + assert!( + (c - 0.97).abs() < 0.001, + "case-insensitive prefix match should give 0.97" + ); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/poll.rs b/crates/plugins/chat4n6-whatsapp/src/poll.rs index 73c9389..5e4d596 100644 --- a/crates/plugins/chat4n6-whatsapp/src/poll.rs +++ b/crates/plugins/chat4n6-whatsapp/src/poll.rs @@ -39,14 +39,22 @@ pub fn build_poll( // Deduplicate voter JIDs for this option let unique_jids: Vec = { let mut seen = HashSet::new(); - voter_jids.into_iter().filter(|j| seen.insert(j.clone())).collect() + voter_jids + .into_iter() + .filter(|j| seen.insert(j.clone())) + .collect() }; for jid in &unique_jids { all_voters.insert(jid.clone()); } let voter_names: Vec = unique_jids .iter() - .map(|jid| voter_name_map.get(jid).cloned().unwrap_or_else(|| jid.clone())) + .map(|jid| { + voter_name_map + .get(jid) + .cloned() + .unwrap_or_else(|| jid.clone()) + }) .collect(); PollOption { option_id: idx as i64, diff --git a/crates/plugins/chat4n6-whatsapp/src/schema.rs b/crates/plugins/chat4n6-whatsapp/src/schema.rs index 35d3042..6da7f01 100644 --- a/crates/plugins/chat4n6-whatsapp/src/schema.rs +++ b/crates/plugins/chat4n6-whatsapp/src/schema.rs @@ -26,16 +26,16 @@ pub fn detect_schema_version(user_version: u32, tables: &[&str]) -> SchemaVersio /// 15 = Deleted (deleted-for-all tombstone placeholder; NOT "ProductSingle") pub fn msg_type_label(n: i32) -> &'static str { match n { - 0 => "Text", - 1 => "Image", - 2 => "Audio", - 3 => "Video", - 4 => "Contact", - 5 => "Location", - 6 => "MediaOmitted", - 7 => "StatusUpdate", - 8 => "VoiceNote", - 9 => "Document", + 0 => "Text", + 1 => "Image", + 2 => "Audio", + 3 => "Video", + 4 => "Contact", + 5 => "Location", + 6 => "MediaOmitted", + 7 => "StatusUpdate", + 8 => "VoiceNote", + 9 => "Document", 10 => "MissedVoiceCall", 11 => "MissedVideoCall", 12 => "MediaCiphertextUnknown", @@ -44,7 +44,7 @@ pub fn msg_type_label(n: i32) -> &'static str { 15 => "Deleted", 16 => "LiveLocation", 20 => "Sticker", - _ => "Unknown", + _ => "Unknown", } } @@ -124,14 +124,27 @@ mod tests { ); } + /// A real 2023-era device reports `user_version = 1`, so the version number + /// carries no schema-generation signal. Table presence is the reliable one. #[test] - fn test_schema_modern_by_user_version() { + fn real_device_user_version_1_with_modern_tables_is_modern() { assert_eq!( - detect_schema_version(200, &["messages"]), + detect_schema_version(1, &["message", "jid", "chat", "call_log"]), SchemaVersion::Modern ); } + /// `user_version` alone must not promote a legacy-table database to Modern: + /// the number is app-defined and unrelated to the schema generation. + #[test] + fn high_user_version_alone_does_not_imply_modern() { + assert_eq!( + detect_schema_version(200, &["messages", "wa_contacts"]), + SchemaVersion::Legacy, + "classification must follow the tables that are actually present" + ); + } + #[test] fn test_schema_modern_by_tables() { assert_eq!( diff --git a/crates/plugins/chat4n6-whatsapp/src/status.rs b/crates/plugins/chat4n6-whatsapp/src/status.rs index b462685..4b4d004 100644 --- a/crates/plugins/chat4n6-whatsapp/src/status.rs +++ b/crates/plugins/chat4n6-whatsapp/src/status.rs @@ -25,11 +25,11 @@ pub struct StatusRecord { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum StatusType { - Image, // type 1 - Text, // type 2 - Video, // type 3 - Gif, // type 43 - Audio, // type 44 + Image, // type 1 + Text, // type 2 + Video, // type 3 + Gif, // type 43 + Audio, // type 44 Unknown(i32), } @@ -126,7 +126,14 @@ mod tests { #[test] fn test_view_count_preserved() { let mut r = make_record(3); - enrich_with_stats(&mut r, Some(StatusStats { view_count: 100, reaction_count: 0, reactions: vec![] })); + enrich_with_stats( + &mut r, + Some(StatusStats { + view_count: 100, + reaction_count: 0, + reactions: vec![], + }), + ); assert_eq!(r.stats.unwrap().view_count, 100); } @@ -134,10 +141,25 @@ mod tests { fn test_reactions_list_preserved() { let mut r = make_record(1); let reactions = vec![ - StatusReaction { reactor_jid: "a@s.whatsapp.net".to_string(), emoji: "👍".to_string(), timestamp_ms: None }, - StatusReaction { reactor_jid: "b@s.whatsapp.net".to_string(), emoji: "🔥".to_string(), timestamp_ms: Some(5000) }, + StatusReaction { + reactor_jid: "a@s.whatsapp.net".to_string(), + emoji: "👍".to_string(), + timestamp_ms: None, + }, + StatusReaction { + reactor_jid: "b@s.whatsapp.net".to_string(), + emoji: "🔥".to_string(), + timestamp_ms: Some(5000), + }, ]; - enrich_with_stats(&mut r, Some(StatusStats { view_count: 0, reaction_count: 2, reactions })); + enrich_with_stats( + &mut r, + Some(StatusStats { + view_count: 0, + reaction_count: 2, + reactions, + }), + ); assert_eq!(r.stats.unwrap().reactions.len(), 2); } } diff --git a/crates/plugins/chat4n6-whatsapp/src/system_event.rs b/crates/plugins/chat4n6-whatsapp/src/system_event.rs index e19216f..9eac0cc 100644 --- a/crates/plugins/chat4n6-whatsapp/src/system_event.rs +++ b/crates/plugins/chat4n6-whatsapp/src/system_event.rs @@ -3,48 +3,48 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum SystemEventType { // Group admin events - GroupSubjectChanged, // 1 - GroupIconChanged, // 6 - GroupDescriptionChanged, // 19 - GroupInviteLinkReset, // 83 (same int — use context) + GroupSubjectChanged, // 1 + GroupIconChanged, // 6 + GroupDescriptionChanged, // 19 + GroupInviteLinkReset, // 83 (same int — use context) // Participant events - ParticipantAdded, // 12 - ParticipantLeft, // 5 - ParticipantRemoved, // 14 - ParticipantJoinedViaLink, // 20 - ApprovalRequest, // 83 + ParticipantAdded, // 12 + ParticipantLeft, // 5 + ParticipantRemoved, // 14 + ParticipantJoinedViaLink, // 20 + ApprovalRequest, // 83 // Security/E2E - SecurityCodeChanged, // 18 - E2EEncryptedNotification, // 67 + SecurityCodeChanged, // 18 + E2EEncryptedNotification, // 67 // Admin role changes - ParticipantPromotedToAdmin, // 84 - ParticipantDemotedFromAdmin, // 84 (need context) + ParticipantPromotedToAdmin, // 84 + ParticipantDemotedFromAdmin, // 84 (need context) // Number change - NumberChanged, // 46 + NumberChanged, // 46 // Disappearing messages - DisappearingTimerChanged, // 56 + DisappearingTimerChanged, // 56 // Message pinned/unpinned - MessagePinned, // 79 - MessageUnpinned, // 79 (need context) + MessagePinned, // 79 + MessageUnpinned, // 79 (need context) // Community events - CommunityCreated, // 97 - CommunityJoined, // 98 - CommunitySubgroupAdded, // 99 - CommunitySubgroupRemoved, // 100 - CommunitySubgroupUnlinked, // 101 - CommunityOwnerChanged, // 102 + CommunityCreated, // 97 + CommunityJoined, // 98 + CommunitySubgroupAdded, // 99 + CommunitySubgroupRemoved, // 100 + CommunitySubgroupUnlinked, // 101 + CommunityOwnerChanged, // 102 // Channel events - ChannelCreated, // 134 - ChannelDeleted, // 135 - ChannelPrivacyNotice, // 136 + ChannelCreated, // 134 + ChannelDeleted, // 135 + ChannelPrivacyNotice, // 136 // Permission changes PermissionAddMemberChanged, // 77 @@ -54,8 +54,8 @@ pub enum SystemEventType { PermissionJoinChanged, // 106 // Business - MetaAiDisclaimer, // 117 - BusinessMetaManaged, // 118 + MetaAiDisclaimer, // 117 + BusinessMetaManaged, // 118 // Unknown (preserves the raw integer) Unknown(i32), @@ -64,7 +64,7 @@ pub enum SystemEventType { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SystemEvent { pub event_type: SystemEventType, - pub label: String, // human-readable display string + pub label: String, // human-readable display string pub actor_jid: Option, pub target_jid: Option, } @@ -89,52 +89,135 @@ pub fn parse_system_event( fn decode_event(msg_type: i32, text_data: Option<&str>) -> (SystemEventType, String) { match msg_type { - 1 => (SystemEventType::GroupSubjectChanged, "Group subject changed".to_string()), - 5 => (SystemEventType::ParticipantLeft, "Participant left group".to_string()), - 6 => (SystemEventType::GroupIconChanged, "Group icon changed".to_string()), - 12 => (SystemEventType::ParticipantAdded, "Participant added to group".to_string()), - 14 => (SystemEventType::ParticipantRemoved, "Participant removed from group".to_string()), - 18 => (SystemEventType::SecurityCodeChanged, "Security code changed".to_string()), - 19 => (SystemEventType::GroupDescriptionChanged, "Group description changed".to_string()), - 20 => (SystemEventType::ParticipantJoinedViaLink, "Participant joined via link".to_string()), + 1 => ( + SystemEventType::GroupSubjectChanged, + "Group subject changed".to_string(), + ), + 5 => ( + SystemEventType::ParticipantLeft, + "Participant left group".to_string(), + ), + 6 => ( + SystemEventType::GroupIconChanged, + "Group icon changed".to_string(), + ), + 12 => ( + SystemEventType::ParticipantAdded, + "Participant added to group".to_string(), + ), + 14 => ( + SystemEventType::ParticipantRemoved, + "Participant removed from group".to_string(), + ), + 18 => ( + SystemEventType::SecurityCodeChanged, + "Security code changed".to_string(), + ), + 19 => ( + SystemEventType::GroupDescriptionChanged, + "Group description changed".to_string(), + ), + 20 => ( + SystemEventType::ParticipantJoinedViaLink, + "Participant joined via link".to_string(), + ), 46 => { let label = decode_number_change(text_data); (SystemEventType::NumberChanged, label) } - 56 => (SystemEventType::DisappearingTimerChanged, "Disappearing timer changed".to_string()), - 67 => (SystemEventType::E2EEncryptedNotification, "End-to-end encryption enabled".to_string()), - 77 => (SystemEventType::PermissionAddMemberChanged, "Permission to add members changed".to_string()), - 78 => (SystemEventType::PermissionEditChanged, "Permission to edit group changed".to_string()), + 56 => ( + SystemEventType::DisappearingTimerChanged, + "Disappearing timer changed".to_string(), + ), + 67 => ( + SystemEventType::E2EEncryptedNotification, + "End-to-end encryption enabled".to_string(), + ), + 77 => ( + SystemEventType::PermissionAddMemberChanged, + "Permission to add members changed".to_string(), + ), + 78 => ( + SystemEventType::PermissionEditChanged, + "Permission to edit group changed".to_string(), + ), 79 => (SystemEventType::MessagePinned, "Message pinned".to_string()), - 83 => (SystemEventType::GroupInviteLinkReset, "Group invite link reset".to_string()), - 84 => (SystemEventType::ParticipantPromotedToAdmin, "Participant promoted to admin".to_string()), - 97 => (SystemEventType::CommunityCreated, "Community created".to_string()), - 98 => (SystemEventType::CommunityJoined, "Joined community".to_string()), - 99 => (SystemEventType::CommunitySubgroupAdded, "Community subgroup added".to_string()), - 100 => (SystemEventType::CommunitySubgroupRemoved, "Community subgroup removed".to_string()), - 101 => (SystemEventType::CommunitySubgroupUnlinked, "Community subgroup unlinked".to_string()), - 102 => (SystemEventType::CommunityOwnerChanged, "Community owner changed".to_string()), - 104 => (SystemEventType::PermissionSendMessageChanged, "Permission to send messages changed".to_string()), - 105 => (SystemEventType::PermissionInviteChanged, "Permission to invite members changed".to_string()), - 106 => (SystemEventType::PermissionJoinChanged, "Permission to join changed".to_string()), - 117 => (SystemEventType::MetaAiDisclaimer, "Meta AI disclaimer".to_string()), - 118 => (SystemEventType::BusinessMetaManaged, "Business managed by Meta".to_string()), - 134 => (SystemEventType::ChannelCreated, "Channel created".to_string()), - 135 => (SystemEventType::ChannelDeleted, "Channel deleted".to_string()), - 136 => (SystemEventType::ChannelPrivacyNotice, "Channel privacy notice".to_string()), - other => (SystemEventType::Unknown(other), format!("Unknown system event (type={other})")), + 83 => ( + SystemEventType::GroupInviteLinkReset, + "Group invite link reset".to_string(), + ), + 84 => ( + SystemEventType::ParticipantPromotedToAdmin, + "Participant promoted to admin".to_string(), + ), + 97 => ( + SystemEventType::CommunityCreated, + "Community created".to_string(), + ), + 98 => ( + SystemEventType::CommunityJoined, + "Joined community".to_string(), + ), + 99 => ( + SystemEventType::CommunitySubgroupAdded, + "Community subgroup added".to_string(), + ), + 100 => ( + SystemEventType::CommunitySubgroupRemoved, + "Community subgroup removed".to_string(), + ), + 101 => ( + SystemEventType::CommunitySubgroupUnlinked, + "Community subgroup unlinked".to_string(), + ), + 102 => ( + SystemEventType::CommunityOwnerChanged, + "Community owner changed".to_string(), + ), + 104 => ( + SystemEventType::PermissionSendMessageChanged, + "Permission to send messages changed".to_string(), + ), + 105 => ( + SystemEventType::PermissionInviteChanged, + "Permission to invite members changed".to_string(), + ), + 106 => ( + SystemEventType::PermissionJoinChanged, + "Permission to join changed".to_string(), + ), + 117 => ( + SystemEventType::MetaAiDisclaimer, + "Meta AI disclaimer".to_string(), + ), + 118 => ( + SystemEventType::BusinessMetaManaged, + "Business managed by Meta".to_string(), + ), + 134 => ( + SystemEventType::ChannelCreated, + "Channel created".to_string(), + ), + 135 => ( + SystemEventType::ChannelDeleted, + "Channel deleted".to_string(), + ), + 136 => ( + SystemEventType::ChannelPrivacyNotice, + "Channel privacy notice".to_string(), + ), + other => ( + SystemEventType::Unknown(other), + format!("Unknown system event (type={other})"), + ), } } fn decode_number_change(text_data: Option<&str>) -> String { if let Some(text) = text_data { if let Ok(v) = serde_json::from_str::(text) { - let old = v.get("nc_old_phone") - .and_then(|p| p.as_str()) - .unwrap_or(""); - let new = v.get("nc_new_phone") - .and_then(|p| p.as_str()) - .unwrap_or(""); + let old = v.get("nc_old_phone").and_then(|p| p.as_str()).unwrap_or(""); + let new = v.get("nc_new_phone").and_then(|p| p.as_str()).unwrap_or(""); if !old.is_empty() || !new.is_empty() { return format!("Phone number changed: {} → {}", old, new); } @@ -248,8 +331,11 @@ mod tests { let e = parse_with_text(46, json); assert_eq!(e.event_type, SystemEventType::NumberChanged); // Label should include phone numbers when JSON is parseable - assert!(e.label.contains("15551234567") || e.label.contains("number"), - "label should reference the number change, got: {}", e.label); + assert!( + e.label.contains("15551234567") || e.label.contains("number"), + "label should reference the number change, got: {}", + e.label + ); } #[test] @@ -365,7 +451,12 @@ mod tests { #[test] fn test_actor_target_jid_preserved() { - let e = parse_system_event(12, None, Some("actor@s.whatsapp.net"), Some("target@s.whatsapp.net")); + let e = parse_system_event( + 12, + None, + Some("actor@s.whatsapp.net"), + Some("target@s.whatsapp.net"), + ); assert_eq!(e.actor_jid.as_deref(), Some("actor@s.whatsapp.net")); assert_eq!(e.target_jid.as_deref(), Some("target@s.whatsapp.net")); } diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql new file mode 100644 index 0000000..37db26e --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/real_modern_schema.sql @@ -0,0 +1,140 @@ +-- Real-device WhatsApp Android msgstore.db schema (2023-era handset). +-- +-- PROVENANCE +-- The CREATE TABLE shapes below reproduce the DDL read out of a real +-- `msgstore.db` acquired in a live forensic case. DDL is not case data — +-- no identifiers, names, numbers, paths or message content from that device +-- appear here. Every INSERT below is synthetic. +-- +-- Column positions confirmed against that device: +-- message : chat_row_id(1) from_me(2) key_id(3) sender_jid_row_id(4) +-- timestamp(11) message_type(14) text_data(15) starred(16) +-- jid : raw_string(5) +-- chat : jid_row_id(1) hidden(2) subject(3) created_timestamp(4) archived(10) +-- call_log: jid_row_id(1) from_me(2) call_id(3) transaction_id(4) +-- timestamp(5) video_call(6) duration(7) call_result(8) +-- Columns at the intervening positions carry the names published in the +-- WhatsApp-schema forensic literature; they are position-preserving and are +-- never read by the extractor. Correctness of this fixture depends only on +-- the confirmed names above sitting at the confirmed positions. +-- +-- WHY THIS FIXTURE EXISTS +-- `modern_schema.sql` was authored to the same simplified layout the +-- extractor's hardcoded ordinals assumed, so it could never falsify them. +-- This fixture can: the values a positional reader picks up here are the +-- wrong columns, exactly as on the real device. +-- +-- PRAGMA user_version is 1 on the real device — the modern schema is NOT +-- identified by a high user_version. + +PRAGMA user_version = 1; + +CREATE TABLE jid ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + user TEXT NOT NULL, + server TEXT NOT NULL, + agent INTEGER, + type INTEGER, + raw_string TEXT, + device INTEGER +); + +CREATE TABLE chat ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid_row_id INTEGER UNIQUE, + hidden INTEGER, + subject TEXT, + created_timestamp INTEGER, + display_message_row_id INTEGER, + last_message_row_id INTEGER, + last_read_message_row_id INTEGER, + last_read_receipt_sent_message_row_id INTEGER, + last_important_message_row_id INTEGER, + archived INTEGER, + sort_timestamp INTEGER, + mod_tag INTEGER, + gen TEXT, + spam_detection INTEGER, + unseen_earliest_message_received_time INTEGER, + unseen_message_count INTEGER, + unseen_missed_calls_count INTEGER, + unseen_row_count INTEGER, + plaintext_disabled INTEGER, + vcard_ui_dismissed INTEGER, + change_number_notified_message_row_id INTEGER, + show_group_description INTEGER, + ephemeral_expiration INTEGER, + last_read_ephemeral_message_row_id INTEGER, + ephemeral_setting_timestamp INTEGER, + unseen_important_message_count INTEGER +); + +CREATE TABLE message ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_row_id INTEGER NOT NULL, + from_me INTEGER NOT NULL, + key_id TEXT NOT NULL, + sender_jid_row_id INTEGER, + status INTEGER, + broadcast INTEGER, + recipient_count INTEGER, + participant_hash TEXT, + origination_flags INTEGER, + origin INTEGER, + timestamp INTEGER, + received_timestamp INTEGER, + receipt_server_timestamp INTEGER, + message_type INTEGER, + text_data TEXT, + starred INTEGER, + lookup_tables INTEGER, + sort_id INTEGER, + message_add_on_flags INTEGER, + view_mode INTEGER +); + +CREATE TABLE call_log ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid_row_id INTEGER, + from_me INTEGER, + call_id TEXT, + transaction_id INTEGER, + timestamp INTEGER, + video_call INTEGER, + duration INTEGER, + call_result INTEGER, + is_dnd_mode_on INTEGER, + bytes_transferred INTEGER, + group_jid_row_id INTEGER, + is_joinable_group_call INTEGER, + call_creator_device_jid_row_id INTEGER, + call_random_id TEXT +); + +-- ── Synthetic rows ─────────────────────────────────────────────────────────── +-- All timestamps are on 2022-09-15 UTC (1663243200000 = 2022-09-15T12:00:00Z). + +INSERT INTO jid (_id, user, server, agent, type, raw_string, device) VALUES + (1, '4155550100', 's.whatsapp.net', 0, 0, '4155550100@s.whatsapp.net', 0), + (2, '120363001234567890', 'g.us', 0, 1, '120363001234567890@g.us', 0); + +INSERT INTO chat (_id, jid_row_id, hidden, subject, created_timestamp, archived) VALUES + (1, 1, 0, NULL, 1663200000000, 0), + (2, 2, 0, 'Real Schema Group', 1663200000000, 1); + +INSERT INTO message + (_id, chat_row_id, from_me, key_id, sender_jid_row_id, timestamp, message_type, text_data, starred) +VALUES + (1, 1, 1, 'AAAA1111', NULL, 1663243200000, 0, 'Outgoing real-schema message', 0), + (2, 1, 0, 'BBBB2222', 1, 1663243260000, 0, 'Incoming real-schema message', 1), + (3, 2, 0, 'CCCC3333', 1, 1663243320000, 0, 'Group real-schema message', 0); + +-- Three distinct calls. Calls 1 and 3 share a duration (137 s) but are +-- unrelated: a reader that mistakes `duration` for a group-call grouping key +-- collapses them into one record. +INSERT INTO call_log + (_id, jid_row_id, from_me, call_id, transaction_id, timestamp, video_call, duration, call_result) +VALUES + (1, 1, 1, 'CALL-AAAA', 11, 1663243400000, 0, 137, 1), + (2, 1, 0, 'CALL-BBBB', 12, 1663243500000, 1, 42, 1), + (3, 1, 1, 'CALL-CCCC', 13, 1663243600000, 0, 137, 1); From b9e1bfd4387a4d175534db5fa503da82b49290e6 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:18:40 +0800 Subject: [PATCH 02/11] fix(green): resolve msgstore.db column positions by name from the DDL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces every hardcoded column ordinal in the WhatsApp extractor with a position resolved from the database's own CREATE TABLE SQL. - columns.rs: depth- and quote-aware DDL parser building a name -> position map per table. Commas inside NUMERIC(10, 2) and DEFAULT 'x,y' do not split a column; PRIMARY KEY/UNIQUE/CHECK/FOREIGN/CONSTRAINT table constraints occupy no position; quoted identifiers are unwrapped. Malformed DDL yields an empty map, never a panic. - A column the schema does not declare resolves to None and the field is omitted, so no field is ever read off a neighbouring column. The modern schema has no call_row_id, so calls are no longer merged on `duration` — that alone was collapsing unrelated calls into single group-call records. - Every table the extractor reads goes through the resolver: message, jid, chat, call_log, message_quoted, message_add_on, message_edit_info, receipt_user, message_forwarded, group_participant_user, and wa_contacts. - Bootstrap gate: a database whose sqlite_master yields no CREATE TABLE at all is an error naming the input size, not an empty result set that reads like a clean database. - detect_schema_version now classifies on table presence only. user_version is app-defined and reads 1 on a real 2023-era device, so the `>= 100` arm was noise; the plugin now passes the real table list instead of an empty slice. - Drops schema::cols and the bespoke key_id DDL scanner it fed. 314 tests pass (286 before + 28 new). Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat4n6-whatsapp/src/columns.rs | 132 ++++- .../plugins/chat4n6-whatsapp/src/extractor.rs | 516 ++++++++---------- crates/plugins/chat4n6-whatsapp/src/lib.rs | 11 +- crates/plugins/chat4n6-whatsapp/src/schema.rs | 57 +- 4 files changed, 363 insertions(+), 353 deletions(-) diff --git a/crates/plugins/chat4n6-whatsapp/src/columns.rs b/crates/plugins/chat4n6-whatsapp/src/columns.rs index 3a5d028..184af05 100644 --- a/crates/plugins/chat4n6-whatsapp/src/columns.rs +++ b/crates/plugins/chat4n6-whatsapp/src/columns.rs @@ -33,22 +33,33 @@ impl TableColumns { /// Table-level constraints (`PRIMARY KEY (…)`, `UNIQUE (…)`, `CHECK (…)`, /// `FOREIGN KEY (…)`, `CONSTRAINT …`) do not occupy a column position and /// are skipped. Malformed input yields an empty map rather than a panic. - pub fn from_ddl(_ddl: &str) -> Self { - // RED stub — resolution is implemented in the GREEN commit. - Self::default() + pub fn from_ddl(ddl: &str) -> Self { + let Some(body) = column_body(ddl) else { + return Self::default(); + }; + let mut by_name = HashMap::new(); + let mut position = 0usize; + for part in split_top_level(body) { + let Some(name) = column_name(part) else { + continue; // table-level constraint: occupies no column position + }; + by_name.entry(name).or_insert(position); + position += 1; + } + Self { by_name } } /// Zero-based position of `name`, or `None` when the table has no such column. - pub fn get(&self, _name: &str) -> Option { - None + pub fn get(&self, name: &str) -> Option { + self.by_name.get(&name.to_ascii_lowercase()).copied() } /// Position of the first of `names` that the table declares. /// /// Used where one logical field is spelled differently across schema /// generations; the order of `names` is the preference order. - pub fn first_of(&self, _names: &[&str]) -> Option { - None + pub fn first_of(&self, names: &[&str]) -> Option { + names.iter().find_map(|n| self.get(n)) } /// Number of declared columns. @@ -70,9 +81,14 @@ pub struct SchemaColumns { impl SchemaColumns { /// Build from a `table name → CREATE TABLE SQL` map. - pub fn from_ddl_map(_ddl: &HashMap) -> Self { - // RED stub — resolution is implemented in the GREEN commit. - Self::default() + pub fn from_ddl_map(ddl: &HashMap) -> Self { + Self { + tables: ddl + .iter() + .map(|(name, sql)| (name.to_ascii_lowercase(), TableColumns::from_ddl(sql))) + .collect(), + empty: TableColumns::default(), + } } /// Columns of `table`; an empty map when the database has no such table. @@ -94,6 +110,102 @@ impl SchemaColumns { } } +// ── DDL parsing ────────────────────────────────────────────────────────────── + +/// Closing delimiter for an opening quote/bracket, or `None` if `b` doesn't open one. +fn closing_delimiter(b: u8) -> Option { + match b { + b'\'' | b'"' | b'`' => Some(b), + b'[' => Some(b']'), + _ => None, + } +} + +/// The text between a `CREATE TABLE`'s outermost parentheses. +/// +/// Returns `None` when the statement has no balanced parenthesised body. +fn column_body(ddl: &str) -> Option<&str> { + let start = ddl.find('(')?; + let mut depth = 0usize; + let mut closing: Option = None; + for (i, &b) in ddl.as_bytes().iter().enumerate().skip(start) { + if let Some(c) = closing { + if b == c { + closing = None; + } + continue; + } + if let Some(c) = closing_delimiter(b) { + closing = Some(c); + } else if b == b'(' { + depth += 1; + } else if b == b')' { + depth = depth.saturating_sub(1); + if depth == 0 { + // Both delimiters are ASCII, so these are char boundaries. + return Some(&ddl[start + 1..i]); + } + } + } + None +} + +/// Split a column body on commas that are neither nested nor quoted. +fn split_top_level(body: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut depth = 0usize; + let mut closing: Option = None; + let mut start = 0usize; + for (i, &b) in body.as_bytes().iter().enumerate() { + if let Some(c) = closing { + if b == c { + closing = None; + } + continue; + } + if let Some(c) = closing_delimiter(b) { + closing = Some(c); + } else if b == b'(' { + depth += 1; + } else if b == b')' { + depth = depth.saturating_sub(1); + } else if b == b',' && depth == 0 { + parts.push(&body[start..i]); + start = i + 1; + } + } + parts.push(&body[start..]); + parts +} + +/// The lowercased column name a body part declares, or `None` when the part is +/// a table-level constraint. +/// +/// SQLite reserves `PRIMARY`, `UNIQUE`, `CHECK`, `FOREIGN` and `CONSTRAINT`, so +/// a column that really carries one of those names has to be quoted — and a +/// quoted name skips the keyword test. +fn column_name(part: &str) -> Option { + const CONSTRAINT_KEYWORDS: [&str; 5] = ["primary", "unique", "check", "foreign", "constraint"]; + + let s = part.trim(); + let first = s.chars().next()?; + if let Some(close) = closing_delimiter(first as u8) { + let rest = s.get(first.len_utf8()..)?; + let end = rest.find(close as char)?; + let name = rest.get(..end)?; + return (!name.is_empty()).then(|| name.to_ascii_lowercase()); + } + + let end = s + .find(|c: char| c.is_whitespace() || c == '(' || c == ',') + .unwrap_or(s.len()); + let name = s.get(..end)?.to_ascii_lowercase(); + if name.is_empty() || CONSTRAINT_KEYWORDS.contains(&name.as_str()) { + return None; + } + Some(name) +} + /// Resolved `message` column positions. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct MessageColumns { diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index c07ae33..7f270cf 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -1,9 +1,12 @@ use crate::anti_forensics::{ detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans, }; -use crate::schema::{cols, SchemaVersion}; +use crate::columns::{ + CallLogColumns, ChatColumns, JidColumns, MessageColumns, SchemaColumns, TableColumns, +}; +use crate::schema::SchemaVersion; pub use crate::schema::{default_mime_for_type, is_media_type, msg_type_label}; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use chat4n6_plugin_api::{ CallRecord, CallResult, Chat, Contact, EditHistoryEntry, ExtractionResult, ForensicTimestamp, GroupParticipantEvent, MediaRef, Message, MessageContent, MessageReceipt, ParticipantAction, @@ -21,18 +24,10 @@ use std::collections::{HashMap, HashSet}; /// /// `tz_offset_secs` is seconds east of UTC for local time display. /// -/// NOTE on column indices: the btree walker stores INTEGER PRIMARY KEY as -/// SqlValue::Null at values[0]. Real column data starts at values[1]. -/// Schema layout (zero-based values[] index, after the leading Null): -/// -/// jid: [0]=Null(_id), [1]=raw_string -/// chat: [0]=Null(_id), [1]=jid_row_id, [2]=subject -/// message: [0]=Null(_id), [1]=chat_row_id, [2]=sender_jid_row_id, -/// [3]=from_me, [4]=timestamp, [5]=text_data, [6]=message_type, -/// [7]=media_mime_type, [8]=media_name, [9]=starred, -/// [10]=edit_version (newer schemas; 5=deleted-for-me, 7=deleted-for-all) -/// call_log: [0]=Null(_id), [1]=jid_row_id, [2]=from_me, -/// [3]=video_call,[4]=duration, [5]=timestamp +/// Every column position is resolved by name from the database's own +/// `CREATE TABLE` SQL (see [`crate::columns`]). WhatsApp reorders these tables +/// between releases, so a fixed ordinal is only ever right for one schema — +/// reading the wrong one yields a complete report full of wrong values. pub fn extract_from_msgstore( db_bytes: &[u8], tz_offset_secs: i32, @@ -41,8 +36,22 @@ pub fn extract_from_msgstore( let engine = ForensicEngine::new(db_bytes, Some(tz_offset_secs)) .context("failed to open msgstore.db")?; - // Read DDL map for schema-aware column index resolution (e.g. key_id position). + // Resolve every column position from the database's own schema. A database + // whose sqlite_master cannot be read is a bootstrap failure, not an empty + // database: report it rather than returning a plausible-looking empty result. let ddl_map = engine.table_ddl(); + if ddl_map.is_empty() { + bail!( + "no CREATE TABLE statement could be read from sqlite_master (page 1 of \ + {} bytes); refusing to extract with unresolved column positions", + db_bytes.len() + ); + } + let schema_cols = SchemaColumns::from_ddl_map(&ddl_map); + let msg_cols = MessageColumns::resolve(schema_cols.table("message")); + let jid_cols = JidColumns::resolve(schema_cols.table("jid")); + let chat_cols = ChatColumns::resolve(schema_cols.table("chat")); + let call_cols = CallLogColumns::resolve(schema_cols.table("call_log")); let records = engine.recover_layer1().context("Layer 1 recovery failed")?; @@ -50,16 +59,16 @@ pub fn extract_from_msgstore( let by_table = partition_by_table(&records); // Build JID lookup: id → raw_string - let jid_map = build_jid_map(tbl(&by_table, "jid")); + let jid_map = build_jid_map(tbl(&by_table, "jid"), &jid_cols); // Build chat map: chat_id → Chat (populated with messages below) - let mut chats = build_chats(tbl(&by_table, "chat"), &jid_map); + let mut chats = build_chats(tbl(&by_table, "chat"), &jid_map, &chat_cols); // Map messages into chats. If the chat record was deleted/unrecovered, // create a stub so forensically-recovered messages are never silently dropped. let msg_records = tbl(&by_table, "message"); for rec in msg_records { - if let Some(msg) = record_to_message(rec, &jid_map, tz_offset_secs) { + if let Some(msg) = record_to_message(rec, &jid_map, tz_offset_secs, &msg_cols) { chats .entry(msg.chat_id) .or_insert_with(|| Chat { @@ -76,20 +85,20 @@ pub fn extract_from_msgstore( } // Build key_id → Vec map for DuplicateStanzaId detection. - // Uses DDL-parsed column index to be robust to varying schema column order. - let key_id_map = build_key_id_map(msg_records, &ddl_map); + let key_id_map = build_key_id_map(msg_records, msg_cols.key_id); // ── Quoted messages ────────────────────────────────────────────────── // Build a map of message_row_id → (text, sender_jid, from_me, timestamp) // from the message_quoted table, then attach to parent messages. + let quoted_cols = schema_cols.table("message_quoted"); let quoted_records = tbl(&by_table, "message_quoted"); - let quoted_map = build_quoted_map(quoted_records, &jid_map, tz_offset_secs); + let quoted_map = build_quoted_map(quoted_records, &jid_map, tz_offset_secs, quoted_cols); // Build ghost map: message_row_id → text_data for ghost recovery. // When a deleted/tombstone message (msg_type=15) is later quoted, the // message_quoted table preserves its original text. We index that here // so we can upgrade Deleted/Unknown(15) messages to GhostRecovered. - let ghost_map = build_ghost_map(quoted_records); + let ghost_map = build_ghost_map(quoted_records, quoted_cols); for chat in chats.values_mut() { for msg in &mut chat.messages { @@ -113,30 +122,29 @@ pub fn extract_from_msgstore( // ── Reactions (message_add_on type=56) ────────────────────────────────── // Build map: message_row_id → Vec let mut reactions_map: HashMap> = HashMap::new(); + let add_on_cols = schema_cols.table("message_add_on"); + let add_on_msg_row_id = add_on_cols.get("message_row_id"); + let add_on_sender = add_on_cols.get("sender_jid_row_id"); + let add_on_ts = add_on_cols.get("timestamp"); + let add_on_type_col = add_on_cols.get("type"); + let add_on_text = add_on_cols.get("text_data"); let add_on_records = tbl(&by_table, "message_add_on"); for r in add_on_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, add_on_msg_row_id) else { + continue; }; - let add_on_type = match r.values.get(5) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(add_on_type) = int_at(r, add_on_type_col) else { + continue; }; if add_on_type == 56 { // Reaction: emoji in text_data - let emoji = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => s.clone(), - _ => continue, - }; - let ts_ms = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, - }; - let reactor_jid = match r.values.get(3) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned().unwrap_or_default(), - _ => String::new(), + let Some(emoji) = text_at(r, add_on_text) else { + continue; }; + let ts_ms = int_at(r, add_on_ts).unwrap_or(0); + let reactor_jid = int_at(r, add_on_sender) + .and_then(|n| jid_map.get(&n).cloned()) + .unwrap_or_default(); reactions_map.entry(msg_row_id).or_default().push(Reaction { emoji, reactor_jid, @@ -149,19 +157,18 @@ pub fn extract_from_msgstore( // ── Edit history (message_edit_info) ──────────────────────────────────── // Build map: message_row_id → Vec let mut edits_map: HashMap> = HashMap::new(); + let edit_cols = schema_cols.table("message_edit_info"); + let edit_msg_row_id = edit_cols.get("message_row_id"); + let edit_ts = edit_cols.get("edited_timestamp"); + let edit_text = edit_cols.get("original_text"); let edit_records = tbl(&by_table, "message_edit_info"); for r in edit_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let edited_ts = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(msg_row_id) = int_at(r, edit_msg_row_id) else { + continue; }; - let original_text = match r.values.get(3) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, + let edited_ts = int_at(r, edit_ts).unwrap_or(0); + let Some(original_text) = text_at(r, edit_text) else { + continue; }; edits_map .entry(msg_row_id) @@ -176,24 +183,23 @@ pub fn extract_from_msgstore( // ── Receipts (receipt_user) ───────────────────────────────────────────── // Build map: message_row_id → Vec let mut receipts_map: HashMap> = HashMap::new(); + let receipt_cols = schema_cols.table("receipt_user"); + let receipt_msg_row_id = receipt_cols.get("message_row_id"); + let receipt_jid_row_id = receipt_cols.get("receipt_user_jid_row_id"); + let receipt_status = receipt_cols.get("status"); + let receipt_ts = receipt_cols.get("timestamp"); let receipt_records = tbl(&by_table, "receipt_user"); for r in receipt_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let jid_row_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, receipt_msg_row_id) else { + continue; }; - let status = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, receipt_jid_row_id) else { + continue; }; - let ts_ms = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(status) = int_at(r, receipt_status) else { + continue; }; + let ts_ms = int_at(r, receipt_ts).unwrap_or(0); let receipt_type = match status { 5 => ReceiptType::Delivered, 13 => ReceiptType::Read, @@ -215,17 +221,18 @@ pub fn extract_from_msgstore( // ── Forwarded messages (message_forwarded) ─────────────────────────────── // Build map: message_row_id → forward_score let mut forwarded_map: HashMap = HashMap::new(); + let fwd_cols = schema_cols.table("message_forwarded"); + let fwd_msg_row_id = fwd_cols.get("message_row_id"); + let fwd_score = fwd_cols.get("forward_score"); let fwd_records = tbl(&by_table, "message_forwarded"); for r in fwd_records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, fwd_msg_row_id) else { + continue; }; - let score = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n as u32, - _ => continue, + let Some(score) = int_at(r, fwd_score) else { + continue; }; - forwarded_map.insert(msg_row_id, score); + forwarded_map.insert(msg_row_id, score as u32); } // Attach reactions, edit history, receipts, and forwarding to messages @@ -256,14 +263,18 @@ pub fn extract_from_msgstore( let call_records = tbl(&by_table, "call_log"); let raw_calls: Vec<(CallRecord, Option)> = call_records .iter() - .filter_map(|r| record_to_call(r, &jid_map, tz_offset_secs)) + .filter_map(|r| record_to_call(r, &jid_map, tz_offset_secs, &call_cols)) .collect(); let calls = merge_group_calls(raw_calls); // ── Group participant events (group_participant_user) ─────────────────── let gpe_records = tbl(&by_table, "group_participant_user"); - let group_participant_events = - build_group_participant_events(gpe_records, &jid_map, tz_offset_secs); + let group_participant_events = build_group_participant_events( + gpe_records, + &jid_map, + tz_offset_secs, + schema_cols.table("group_participant_user"), + ); // WAL deltas (placeholder — WAL integration in CLI layer) let wal_deltas: Vec = Vec::new(); @@ -366,175 +377,127 @@ fn tbl<'a>( by.get(name).map(|v| v.as_slice()).unwrap_or_default() } +/// Read the integer at a resolved column position. +/// +/// `None` when the schema does not declare the column, the record is shorter +/// than that position, or the stored value is not an integer — never a +/// neighbouring column's value. +fn int_at(r: &RecoveredRecord, idx: Option) -> Option { + match r.values.get(idx?) { + Some(SqlValue::Int(n)) => Some(*n), + _ => None, + } +} + +/// Read the non-empty text at a resolved column position. +fn text_at(r: &RecoveredRecord, idx: Option) -> Option { + match r.values.get(idx?) { + Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), + _ => None, + } +} + +/// Read a resolved column as a boolean flag (any non-zero integer is true). +fn flag_at(r: &RecoveredRecord, idx: Option) -> bool { + int_at(r, idx).is_some_and(|n| n != 0) +} + /// Build a map of key_id (XMPP stanza ID) → list of message row_ids. /// -/// Parses the CREATE TABLE DDL for the `message` table to determine the -/// zero-based values[] index of the `key_id` column (accounting for the -/// leading Null at index 0 for the INTEGER PRIMARY KEY). Returns an empty -/// map if the column doesn't exist in this schema. +/// Empty when the schema declares no `key_id` column. fn build_key_id_map( msg_records: &[&RecoveredRecord], - ddl_map: &HashMap, + key_id_idx: Option, ) -> HashMap> { - // Find the values[] index for key_id by parsing the DDL column list. - // The btree walker puts INTEGER PRIMARY KEY at values[0] as Null, - // so real column n (1-based in DDL order after _id) → values[n]. - let key_id_idx = match ddl_map.get("message") { - Some(ddl) => key_id_column_index(ddl), - None => return HashMap::new(), - }; - let Some(idx) = key_id_idx else { - return HashMap::new(); - }; - let mut map: HashMap> = HashMap::new(); + if key_id_idx.is_none() { + return map; + } for rec in msg_records { - let row_id = match rec.row_id { - Some(id) => id, - None => continue, + let Some(row_id) = rec.row_id else { + continue; }; - if let Some(SqlValue::Text(kid)) = rec.values.get(idx) { - if !kid.is_empty() { - map.entry(kid.clone()).or_default().push(row_id); - } + if let Some(kid) = text_at(rec, key_id_idx) { + map.entry(kid).or_default().push(row_id); } } map } -/// Parse a CREATE TABLE DDL string and return the 1-based values[] index -/// for the `key_id` column (i.e. column position counting from 1, since -/// index 0 is the INTEGER PRIMARY KEY alias Null). +/// jid table: row_id is `_id`; the JID string lives in `raw_string`. /// -/// Returns `None` if the column is not found. -fn key_id_column_index(ddl: &str) -> Option { - // Strip everything up to the first '(' and after the last ')'. - let start = ddl.find('(')?; - let end = ddl.rfind(')')?; - let cols_str = &ddl[start + 1..end]; - - // Split on commas (naive but sufficient for well-formed SQLite DDL). - // Column 0 in values[] is the INTEGER PRIMARY KEY (always first column). - // Real columns start at index 1 in values[]. - let mut idx = 0usize; - for col_def in cols_str.split(',') { - let col_def = col_def.trim(); - // Extract first token as column name (may be quoted with `backticks` or plain). - let col_name = col_def - .split_whitespace() - .next() - .unwrap_or("") - .trim_matches('`') - .trim_matches('"'); - if col_name.eq_ignore_ascii_case("key_id") { - return Some(idx); - } - idx += 1; - } - None -} - -/// jid table: row_id=_id, values[0]=Null(_id alias), values[1]=raw_string -fn build_jid_map(records: &[&RecoveredRecord]) -> HashMap { +/// Reading the neighbouring `user` column instead yields a bare phone number, +/// which looks plausible enough to hide the mistake. +fn build_jid_map(records: &[&RecoveredRecord], cols: &JidColumns) -> HashMap { let mut map = HashMap::new(); for r in records { - let id = match r.row_id { - Some(id) => id, - None => continue, + let Some(id) = r.row_id else { + continue; }; - let raw = match r.values.get(1) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, + let Some(raw) = text_at(r, cols.raw_string) else { + continue; }; map.insert(id, raw); } map } -/// chat table: row_id=_id, values[0]=Null, [1]=jid_row_id, [2]=subject -fn build_chats(records: &[&RecoveredRecord], jid_map: &HashMap) -> HashMap { +/// chat table: row_id is `_id`; `jid_row_id` links to `jid`. +fn build_chats( + records: &[&RecoveredRecord], + jid_map: &HashMap, + cols: &ChatColumns, +) -> HashMap { let mut map = HashMap::new(); for r in records { - let id = match r.row_id { - Some(id) => id, - None => continue, + let Some(id) = r.row_id else { + continue; }; - let jid_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, cols.jid_row_id) else { + continue; }; let jid = jid_map.get(&jid_row_id).cloned().unwrap_or_default(); - let subject = match r.values.get(2) { - Some(SqlValue::Text(s)) => Some(s.clone()), - _ => None, - }; let is_group = jid.ends_with("@g.us"); map.insert( id, Chat { id, jid, - name: subject, + name: text_at(r, cols.subject), is_group, messages: Vec::new(), - archived: matches!(r.values.get(3), Some(SqlValue::Int(1))), + archived: flag_at(r, cols.archived), }, ); } map } -/// message table: row_id=_id, values[0]=Null, [1]=chat_row_id, -/// [2]=sender_jid_row_id, [3]=from_me, [4]=timestamp, [5]=text_data, [6]=message_type, -/// [7]=media_mime_type, [8]=media_name, [9]=starred, [10]=edit_version (newer schemas) +/// Map one `message` record using resolved column positions. +/// +/// A record without a resolvable `chat_row_id` or `timestamp` is skipped rather +/// than attributed to a guessed column. fn record_to_message( r: &RecoveredRecord, jid_map: &HashMap, tz_offset_secs: i32, + cols: &MessageColumns, ) -> Option { - use cols::message as col; let id = r.row_id?; - let chat_id = match r.values.get(col::CHAT_ROW_ID)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let sender_jid = match r.values.get(col::SENDER_JID_ROW_ID) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; - let from_me = match r.values.get(col::FROM_ME) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let ts_ms = match r.values.get(col::TIMESTAMP)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let msg_type = match r.values.get(col::MESSAGE_TYPE) { - Some(SqlValue::Int(n)) => *n as i32, - _ => 0, - }; - let media_mime = match r.values.get(col::MEDIA_MIME_TYPE) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let media_name = match r.values.get(col::MEDIA_NAME) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let starred = matches!(r.values.get(col::STARRED), Some(SqlValue::Int(n)) if *n != 0); + let chat_id = int_at(r, cols.chat_row_id)?; + let sender_jid = int_at(r, cols.sender_jid_row_id).and_then(|n| jid_map.get(&n).cloned()); + let from_me = flag_at(r, cols.from_me); + let ts_ms = int_at(r, cols.timestamp)?; + let msg_type = int_at(r, cols.message_type).unwrap_or(0) as i32; + let media_mime = text_at(r, cols.media_mime_type); + let media_name = text_at(r, cols.media_name); + let starred = flag_at(r, cols.starred); // edit_version column added in newer schema versions. // 5 = deleted-for-me (local deletion only) // 7 = deleted-for-all (sender deleted from everyone's view) // Both cases override the content to Deleted regardless of msg_type or text_data. - let edit_version = match r.values.get(col::EDIT_VERSION) { - Some(SqlValue::Int(n)) => *n, - _ => 0, - }; - let text_data = match r.values.get(col::TEXT_DATA) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; + let edit_version = int_at(r, cols.edit_version).unwrap_or(0); + let text_data = text_at(r, cols.text_data); let content = if edit_version == 5 || edit_version == 7 { // Message deleted: override any msg_type or text content. // edit_version=5 → deleted-for-me; edit_version=7 → deleted-for-all. @@ -605,50 +568,30 @@ fn record_to_message( }) } -/// call_log: row_id=_id, values[0]=Null, [1]=jid_row_id, [2]=from_me, -/// [3]=video_call, [4]=duration, [5]=timestamp, [6]=call_result, -/// [7]=call_row_id (optional grouping key), [8]=call_creator_device_jid_row_id (optional) +/// Map one `call_log` record using resolved column positions. /// -/// Returns `(CallRecord, call_row_id)` where call_row_id groups group-call participants. +/// Returns `(CallRecord, call_row_id)`, where `call_row_id` groups the +/// participants of one group call. Schemas that declare no such column yield +/// `None`, so unrelated calls are never merged on a neighbouring value — the +/// modern schema has no `call_row_id`, and reading `duration` in its place +/// collapses every pair of calls that happened to run the same length. fn record_to_call( r: &RecoveredRecord, jid_map: &HashMap, tz_offset_secs: i32, + cols: &CallLogColumns, ) -> Option<(CallRecord, Option)> { let id = r.row_id?; - let jid_row_id = match r.values.get(1)? { - SqlValue::Int(n) => *n, - _ => return None, - }; + let jid_row_id = int_at(r, cols.jid_row_id)?; let participant = jid_map.get(&jid_row_id).cloned().unwrap_or_default(); - let from_me = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let video = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, - }; - let duration = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n as u32, - _ => 0, - }; - let ts_ms = match r.values.get(5)? { - SqlValue::Int(n) => *n, - _ => return None, - }; - let call_result = match r.values.get(6) { - Some(SqlValue::Int(n)) => CallResult::from(*n), - _ => CallResult::Unknown, - }; - let call_row_id = match r.values.get(7) { - Some(SqlValue::Int(n)) => Some(*n), - _ => None, - }; - let call_creator_device_jid = match r.values.get(8) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; + let from_me = flag_at(r, cols.from_me); + let video = flag_at(r, cols.video_call); + let duration = int_at(r, cols.duration).unwrap_or(0) as u32; + let ts_ms = int_at(r, cols.timestamp)?; + let call_result = int_at(r, cols.call_result).map_or(CallResult::Unknown, CallResult::from); + let call_row_id = int_at(r, cols.call_row_id); + let call_creator_device_jid = + int_at(r, cols.call_creator_device_jid_row_id).and_then(|n| jid_map.get(&n).cloned()); Some(( CallRecord { call_id: id, @@ -703,48 +646,37 @@ fn merge_group_calls(raw: Vec<(CallRecord, Option)>) -> Vec { // ── Quoted message support ─────────────────────────────────────────────────── -/// message_quoted: [0]=Null(_id), [1]=message_row_id, [2]=chat_row_id, -/// [3]=sender_jid_row_id, [4]=from_me, [5]=timestamp, [6]=text_data, [7]=message_type +/// message_quoted: the snapshot a reply keeps of the message it quotes. fn build_quoted_map( records: &[&RecoveredRecord], jid_map: &HashMap, tz_offset_secs: i32, + cols: &TableColumns, ) -> HashMap { + let msg_row_id_col = cols.get("message_row_id"); + let chat_row_id_col = cols.get("chat_row_id"); + let sender_col = cols.get("sender_jid_row_id"); + let from_me_col = cols.get("from_me"); + let ts_col = cols.get("timestamp"); + let text_col = cols.get("text_data"); + let type_col = cols.get("message_type"); + let mut map = HashMap::new(); for r in records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let chat_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let sender_jid = match r.values.get(3) { - Some(SqlValue::Int(n)) => jid_map.get(n).cloned(), - _ => None, - }; - let from_me = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n != 0, - _ => false, + let Some(msg_row_id) = int_at(r, msg_row_id_col) else { + continue; }; - let ts_ms = match r.values.get(5) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(chat_id) = int_at(r, chat_row_id_col) else { + continue; }; - let msg_type = match r.values.get(7) { - Some(SqlValue::Int(n)) => *n as i32, - _ => 0, - }; - let content = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => MessageContent::Text(s.clone()), - _ => { - if msg_type == 0 { - MessageContent::Deleted - } else { - MessageContent::Unknown(msg_type) - } - } + let sender_jid = int_at(r, sender_col).and_then(|n| jid_map.get(&n).cloned()); + let from_me = flag_at(r, from_me_col); + let ts_ms = int_at(r, ts_col).unwrap_or(0); + let msg_type = int_at(r, type_col).unwrap_or(0) as i32; + let content = match text_at(r, text_col) { + Some(text) => MessageContent::Text(text), + None if msg_type == 0 => MessageContent::Deleted, + None => MessageContent::Unknown(msg_type), }; map.insert( @@ -774,31 +706,30 @@ fn build_quoted_map( // ── Group participant events ───────────────────────────────────────────────── -/// group_participant_user: [0]=Null(_id), [1]=group_jid_row_id, [2]=jid_row_id, -/// [3]=user_action, [4]=action_ts, [5]=actor_jid_row_id +/// group_participant_user: membership changes for group chats. fn build_group_participant_events( records: &[&RecoveredRecord], jid_map: &HashMap, tz_offset_secs: i32, + cols: &TableColumns, ) -> Vec { + let group_col = cols.get("group_jid_row_id"); + let jid_col = cols.get("jid_row_id"); + let action_col = cols.get("user_action"); + let action_ts_col = cols.get("action_ts"); + let mut events = Vec::new(); for r in records { - let group_jid_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, - }; - let jid_row_id = match r.values.get(2) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(group_jid_row_id) = int_at(r, group_col) else { + continue; }; - let user_action = match r.values.get(3) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(jid_row_id) = int_at(r, jid_col) else { + continue; }; - let action_ts = match r.values.get(4) { - Some(SqlValue::Int(n)) => *n, - _ => 0, + let Some(user_action) = int_at(r, action_col) else { + continue; }; + let action_ts = int_at(r, action_ts_col).unwrap_or(0); let action = match user_action { 0 => ParticipantAction::Added, 1 => ParticipantAction::Removed, @@ -825,16 +756,17 @@ fn build_group_participant_events( /// Used to recover the original text of deleted/tombstone messages (msg_type=15) /// that were later quoted by another message. The quoting message preserves /// the original text in message_quoted.text_data. -fn build_ghost_map(records: &[&RecoveredRecord]) -> HashMap { +fn build_ghost_map(records: &[&RecoveredRecord], cols: &TableColumns) -> HashMap { + let msg_row_id_col = cols.get("message_row_id"); + let text_col = cols.get("text_data"); + let mut map = HashMap::new(); for r in records { - let msg_row_id = match r.values.get(1) { - Some(SqlValue::Int(n)) => *n, - _ => continue, + let Some(msg_row_id) = int_at(r, msg_row_id_col) else { + continue; }; - let text = match r.values.get(6) { - Some(SqlValue::Text(s)) if !s.is_empty() => s.clone(), - _ => continue, + let Some(text) = text_at(r, text_col) else { + continue; }; map.insert(msg_row_id, text); } @@ -844,27 +776,27 @@ fn build_ghost_map(records: &[&RecoveredRecord]) -> HashMap { // ── wa.db contact extraction ───────────────────────────────────────────────── /// Extract contacts from wa.db bytes using the forensic B-tree walker. -/// wa_contacts table: [0]=Null(_id), [1]=jid, [2]=display_name, [3]=status, [4]=number +/// +/// `wa_contacts` column positions are resolved by name, as for msgstore.db. pub fn extract_contacts(wa_db_bytes: &[u8]) -> Result> { let engine = ForensicEngine::new(wa_db_bytes, None).context("failed to open wa.db")?; + let schema_cols = SchemaColumns::from_ddl_map(&engine.table_ddl()); + let cols = schema_cols.table("wa_contacts"); + let jid_col = cols.get("jid"); + let display_name_col = cols.get("display_name"); + let number_col = cols.get("number"); + let records = engine.recover_layer1().context("wa.db layer 1 recovery")?; let by_table = partition_by_table(&records); let contact_records = tbl(&by_table, "wa_contacts"); let mut contacts = Vec::new(); for r in contact_records { - let jid = match r.values.get(1) { - Some(SqlValue::Text(s)) => s.clone(), - _ => continue, - }; - let display_name = match r.values.get(2) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, - }; - let phone_number = match r.values.get(4) { - Some(SqlValue::Text(s)) if !s.is_empty() => Some(s.clone()), - _ => None, + let Some(jid) = text_at(r, jid_col) else { + continue; }; + let display_name = text_at(r, display_name_col); + let phone_number = text_at(r, number_col); contacts.push(Contact { jid, display_name, diff --git a/crates/plugins/chat4n6-whatsapp/src/lib.rs b/crates/plugins/chat4n6-whatsapp/src/lib.rs index 605d608..e24afa3 100644 --- a/crates/plugins/chat4n6-whatsapp/src/lib.rs +++ b/crates/plugins/chat4n6-whatsapp/src/lib.rs @@ -111,13 +111,20 @@ impl ForensicPlugin for WhatsAppPlugin { let tz = local_offset_seconds.unwrap_or_else(|| detect_timezone(fs)); - // Detect schema version + // Detect the schema generation from the tables the database actually + // declares. PRAGMA user_version is app-defined (a real 2023-era device + // reports 1), so it cannot stand in for the table list. let user_version = if db_bytes.len() >= 64 { u32::from_be_bytes([db_bytes[60], db_bytes[61], db_bytes[62], db_bytes[63]]) } else { 0 }; - let schema = detect_schema_version(user_version, &[]); + let table_names: Vec = + chat4n6_sqlite_forensics::db::ForensicEngine::new(&db_bytes, None) + .map(|e| e.table_ddl().into_keys().collect()) + .unwrap_or_default(); + let table_refs: Vec<&str> = table_names.iter().map(String::as_str).collect(); + let schema = detect_schema_version(user_version, &table_refs); let mut result = extract_from_msgstore(&db_bytes, tz, schema)?; diff --git a/crates/plugins/chat4n6-whatsapp/src/schema.rs b/crates/plugins/chat4n6-whatsapp/src/schema.rs index 6da7f01..b8aa8b6 100644 --- a/crates/plugins/chat4n6-whatsapp/src/schema.rs +++ b/crates/plugins/chat4n6-whatsapp/src/schema.rs @@ -4,12 +4,15 @@ pub enum SchemaVersion { Modern, } -/// Detect the WhatsApp msgstore.db schema version. -/// - Modern: user_version >= 100 OR has both "message" and "jid" tables +/// Detect the WhatsApp msgstore.db schema version from the tables present. +/// - Modern: has both "message" and "jid" tables /// - Legacy: otherwise ("messages" + "wa_contacts" era) -pub fn detect_schema_version(user_version: u32, tables: &[&str]) -> SchemaVersion { - let has_modern = tables.contains(&"message") && tables.contains(&"jid"); - if has_modern || user_version >= 100 { +/// +/// `user_version` carries no schema-generation signal: a real 2023-era device +/// reports 1, so a `>= 100` test classified legacy databases as modern and +/// modern ones as legacy depending on nothing but the app's own counter. +pub fn detect_schema_version(_user_version: u32, tables: &[&str]) -> SchemaVersion { + if tables.contains(&"message") && tables.contains(&"jid") { SchemaVersion::Modern } else { SchemaVersion::Legacy @@ -68,50 +71,6 @@ pub fn default_mime_for_type(msg_type: i32) -> &'static str { } } -/// Named column index constants for msgstore.db tables. -/// -/// The btree walker stores INTEGER PRIMARY KEY as SqlValue::Null at values[0]. -/// Real column data starts at values[1] (matching the DDL column order after _id). -pub mod cols { - /// message table (modern schema, post-2021) - pub mod message { - pub const CHAT_ROW_ID: usize = 1; - pub const SENDER_JID_ROW_ID: usize = 2; - pub const FROM_ME: usize = 3; - pub const TIMESTAMP: usize = 4; - pub const TEXT_DATA: usize = 5; - pub const MESSAGE_TYPE: usize = 6; - pub const MEDIA_MIME_TYPE: usize = 7; - pub const MEDIA_NAME: usize = 8; - pub const STARRED: usize = 9; - pub const EDIT_VERSION: usize = 10; - } - - /// jid table - pub mod jid { - pub const RAW_STRING: usize = 1; - } - - /// chat table - pub mod chat { - pub const JID_ROW_ID: usize = 1; - pub const SUBJECT: usize = 2; - pub const ARCHIVED: usize = 3; - } - - /// call_log table - pub mod call_log { - pub const JID_ROW_ID: usize = 1; - pub const FROM_ME: usize = 2; - pub const VIDEO_CALL: usize = 3; - pub const DURATION: usize = 4; - pub const TIMESTAMP: usize = 5; - pub const CALL_RESULT: usize = 6; - pub const CALL_ROW_ID: usize = 7; - pub const CALL_CREATOR_DEVICE_JID_ROW_ID: usize = 8; - } -} - #[cfg(test)] mod tests { use super::*; From 1105f6ee1602d610bf0814a9db5a3bbb87ddae24 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:21:04 +0800 Subject: [PATCH 03/11] test(red): fail-loud gate on the resolved message column map (8 tests fail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolving a column by name is only as good as the DDL it came from. If the map is wrong the extractor still emits a structurally complete report — the failure class that cannot be told apart from a correct run. The map must therefore be checked against the data before any of it is interpreted. schema_gate::validate_message_columns is specified to reject: - a message table with no resolvable timestamp or chat_row_id, naming the column - timestamps that are not epoch milliseconds (pre-2009, or past acquisition time plus a day of clock skew), quoting offending values and the resolved ordinal - a column position no record is long enough to carry - non-integer values sitting at the resolved position, shown verbatim and to accept an empty message table and a handful of implausible rows among plausible ones — the real database carries exactly one message whose year does not render, and rejecting good evidence over it would be the worse failure. Ships as a null implementation returning Ok so the tests fail on assertions. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat4n6-whatsapp/src/extractor.rs | 59 +++++ crates/plugins/chat4n6-whatsapp/src/lib.rs | 1 + .../chat4n6-whatsapp/src/schema_gate.rs | 242 ++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 crates/plugins/chat4n6-whatsapp/src/schema_gate.rs diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index 7f270cf..d15a91e 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -2299,6 +2299,65 @@ mod real_schema_tests { assert_eq!(voice.duration_secs, 137); } + /// A database whose `message.timestamp` column does not hold epoch + /// milliseconds is a bootstrap failure: extraction must stop rather than + /// emit a report whose every date is wrong. + #[test] + fn implausible_timestamps_abort_extraction_naming_the_values() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); + CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); + CREATE TABLE message (_id INTEGER PRIMARY KEY, chat_row_id INTEGER NOT NULL, + sender_jid_row_id INTEGER, from_me INTEGER, timestamp INTEGER, + text_data TEXT, message_type INTEGER); + INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); + INSERT INTO chat VALUES (1, 1, NULL); + INSERT INTO message VALUES (1, 1, NULL, 1, 5243, 'a', 0); + INSERT INTO message VALUES (2, 1, NULL, 1, 5244, 'b', 0); + INSERT INTO message VALUES (3, 1, NULL, 1, 5245, 'c', 0);", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("must refuse to report on timestamps that are not epoch millis"); + let msg = format!("{err:#}"); + assert!(msg.contains("timestamp"), "must name the column: {msg}"); + assert!(msg.contains("5243"), "must quote an offending value: {msg}"); + } + + /// A schema that declares no `timestamp` at all must be named, not silently + /// treated as a database with no messages. + #[test] + fn message_table_without_a_timestamp_column_aborts_extraction() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE jid (_id INTEGER PRIMARY KEY, raw_string TEXT NOT NULL); + CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER NOT NULL, subject TEXT); + CREATE TABLE message (_id INTEGER PRIMARY KEY, chat_row_id INTEGER NOT NULL, + text_data TEXT); + INSERT INTO jid VALUES (1, 'alice@s.whatsapp.net'); + INSERT INTO chat VALUES (1, 1, NULL); + INSERT INTO message VALUES (1, 1, 'orphan');", + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("an unrecognisable message schema must be reported"); + assert!( + format!("{err:#}").contains("timestamp"), + "error must name the missing column: {err:#}" + ); + } + /// The invariant every resolved ordinal rests on: SQLite writes the /// `INTEGER PRIMARY KEY` column as a NULL **at its declared position**, so /// `values[i]` is the column at DDL position `i` — even when the rowid diff --git a/crates/plugins/chat4n6-whatsapp/src/lib.rs b/crates/plugins/chat4n6-whatsapp/src/lib.rs index e24afa3..569ea4d 100644 --- a/crates/plugins/chat4n6-whatsapp/src/lib.rs +++ b/crates/plugins/chat4n6-whatsapp/src/lib.rs @@ -17,6 +17,7 @@ pub mod pin; pub mod platform; pub mod poll; pub mod schema; +pub mod schema_gate; pub mod status; pub mod system_event; pub mod timezone; diff --git a/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs new file mode 100644 index 0000000..784cb7b --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs @@ -0,0 +1,242 @@ +//! Bootstrap validation for the resolved `message` column map. +//! +//! Resolving a column by name is only as good as the DDL it was read from. If +//! the map is wrong the extractor still produces a structurally complete +//! report — one that is indistinguishable from a correct one until somebody +//! notices every message is dated 1970. That is the worst failure class a +//! forensic tool has, so the map is checked against the data before any of it +//! is interpreted, and a failed check is an error rather than a report. + +use crate::columns::MessageColumns; +use anyhow::{bail, Result}; +use chat4n6_sqlite_forensics::record::{RecoveredRecord, SqlValue}; + +/// 2009-01-01T00:00:00Z in epoch milliseconds. +/// +/// WhatsApp shipped in 2009, so no genuine message timestamp precedes it. +pub const EARLIEST_PLAUSIBLE_MS: i64 = 1_230_768_000_000; + +/// Grace added to the acquisition time before a timestamp counts as future-dated. +pub const FUTURE_GRACE_MS: i64 = 24 * 60 * 60 * 1000; + +/// Rows examined. Enough to be decisive, small enough to stay free on a +/// quarter-million-row database. +const SAMPLE_SIZE: usize = 512; + +/// Share of sampled timestamps that must be plausible. +/// +/// Deliberately not 1.0: a genuine database can carry a handful of corrupt or +/// out-of-range rows, and rejecting real evidence over one bad row would be a +/// worse failure than the one this gate exists to catch. A misresolved column +/// is wrong for essentially every row, so it fails this bar easily. +const MIN_PLAUSIBLE_RATIO: f64 = 0.90; + +/// Offending values quoted in the error. +const OFFENDERS_SHOWN: usize = 3; + +/// Check that the resolved `message` map actually describes the records. +/// +/// `now_ms` is the acquisition time in epoch milliseconds; it is passed in +/// rather than read from the clock so the bounds are testable. +pub fn validate_message_columns( + _records: &[&RecoveredRecord], + _cols: &MessageColumns, + _now_ms: i64, +) -> Result<()> { + // RED stub — validation is implemented in the GREEN commit. + Ok(()) +} + +/// Render a stored value for a diagnostic, verbatim where it is bounded. +pub fn render_value(value: Option<&SqlValue>) -> String { + match value { + None => "".to_string(), + Some(SqlValue::Null) => "NULL".to_string(), + Some(SqlValue::Int(n)) => n.to_string(), + Some(SqlValue::Real(f)) => f.to_string(), + Some(SqlValue::Text(s)) => format!("TEXT {s:?}"), + Some(SqlValue::Blob(b)) => { + const SHOWN: usize = 16; + let head: String = b.iter().take(SHOWN).map(|x| format!("{x:02x}")).collect(); + if b.len() > SHOWN { + format!("BLOB {} bytes, first {SHOWN}: {head}", b.len()) + } else { + format!("BLOB {} bytes: {head}", b.len()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chat4n6_plugin_api::EvidenceSource; + + /// 2024-03-15T14:32:07Z. + const GOOD_MS: i64 = 1_710_513_127_000; + /// A little after the newest fixture timestamp, standing in for acquisition. + const NOW_MS: i64 = 1_760_000_000_000; + + fn cols_at(timestamp: Option, chat_row_id: Option) -> MessageColumns { + MessageColumns { + chat_row_id, + timestamp, + ..MessageColumns::default() + } + } + + fn record(row_id: i64, values: Vec) -> RecoveredRecord { + RecoveredRecord { + table: "message".to_string(), + row_id: Some(row_id), + values, + source: EvidenceSource::Live, + offset: 0, + confidence: 1.0, + } + } + + /// `n` records whose column 1 holds `ts`. + fn records_with(ts: impl Fn(usize) -> SqlValue, n: usize) -> Vec { + (0..n) + .map(|i| record(i as i64 + 1, vec![SqlValue::Null, ts(i)])) + .collect() + } + + fn refs(v: &[RecoveredRecord]) -> Vec<&RecoveredRecord> { + v.iter().collect() + } + + #[test] + fn plausible_timestamps_pass() { + let recs = records_with(|i| SqlValue::Int(GOOD_MS + i as i64 * 1000), 20); + assert!(validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok()); + } + + #[test] + fn no_message_rows_is_not_an_error() { + assert!( + validate_message_columns(&[], &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "an empty message table is a real state, not a bootstrap failure" + ); + } + + #[test] + fn missing_timestamp_column_is_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 5); + let err = validate_message_columns(&refs(&recs), &cols_at(None, Some(0)), NOW_MS) + .expect_err("a message table with no resolvable timestamp must not be extracted"); + let msg = err.to_string(); + assert!( + msg.contains("timestamp"), + "error must name the column: {msg}" + ); + assert!(msg.contains("message"), "error must name the table: {msg}"); + } + + #[test] + fn missing_chat_row_id_column_is_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 5); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), None), NOW_MS) + .expect_err("messages cannot be attributed without chat_row_id"); + assert!( + err.to_string().contains("chat_row_id"), + "error must name the column: {err}" + ); + } + + #[test] + fn epoch_zero_timestamps_are_rejected_and_the_values_shown() { + // The real failure: sender_jid_row_id read as a timestamp — small ints. + let recs = records_with(|i| SqlValue::Int(5243 + i as i64), 50); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("small integers are not epoch milliseconds"); + let msg = err.to_string(); + assert!(msg.contains("timestamp"), "must name the column: {msg}"); + assert!(msg.contains('1'), "must name the resolved ordinal: {msg}"); + assert!(msg.contains("5243"), "must quote an offending value: {msg}"); + } + + #[test] + fn a_few_implausible_rows_do_not_trip_the_gate() { + // A real database carries the odd corrupt row; rejecting good evidence + // over one of them would be the worse failure. + let mut recs = records_with(|i| SqlValue::Int(GOOD_MS + i as i64 * 1000), 199); + recs.push(record(200, vec![SqlValue::Null, SqlValue::Int(0)])); + assert!( + validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "1 bad row in 200 is an outlier, not a misresolved column" + ); + } + + #[test] + fn future_dated_timestamps_are_rejected() { + let beyond = NOW_MS + FUTURE_GRACE_MS + 1; + let recs = records_with(|i| SqlValue::Int(beyond + i as i64), 20); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("timestamps past the acquisition time cannot be genuine"); + assert!(err.to_string().contains(&beyond.to_string())); + } + + #[test] + fn one_day_of_clock_skew_is_tolerated() { + let recs = records_with(|_| SqlValue::Int(NOW_MS + FUTURE_GRACE_MS - 1), 20); + assert!( + validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS).is_ok(), + "a day of grace absorbs device/host clock skew" + ); + } + + #[test] + fn non_integer_timestamps_are_rejected_and_shown() { + let recs = records_with(|_| SqlValue::Text("AAAA1111".to_string()), 20); + let err = validate_message_columns(&refs(&recs), &cols_at(Some(1), Some(0)), NOW_MS) + .expect_err("a text column is not a timestamp column"); + assert!( + err.to_string().contains("AAAA1111"), + "the offending value must be shown verbatim: {err}" + ); + } + + #[test] + fn records_too_short_for_the_resolved_ordinal_are_rejected() { + let recs = records_with(|_| SqlValue::Int(GOOD_MS), 20); + // Resolve timestamp past the end of every record. + let err = validate_message_columns(&refs(&recs), &cols_at(Some(9), Some(0)), NOW_MS) + .expect_err("no row carries the resolved column"); + assert!( + err.to_string().contains('9'), + "error must name the resolved ordinal: {err}" + ); + } + + // ── render_value ───────────────────────────────────────────────────────── + + #[test] + fn render_value_shows_each_kind() { + assert_eq!(render_value(Some(&SqlValue::Int(-5))), "-5"); + assert_eq!(render_value(Some(&SqlValue::Null)), "NULL"); + assert_eq!( + render_value(Some(&SqlValue::Text("hi".into()))), + "TEXT \"hi\"" + ); + assert_eq!( + render_value(Some(&SqlValue::Blob(vec![0xde, 0xad]))), + "BLOB 2 bytes: dead" + ); + assert!(render_value(None).contains("absent")); + } + + #[test] + fn render_value_labels_a_truncated_blob() { + let rendered = render_value(Some(&SqlValue::Blob(vec![0xab; 40]))); + assert!( + rendered.contains("40 bytes"), + "the full length must be stated: {rendered}" + ); + assert!( + rendered.contains("first 16"), + "the elision must be labelled: {rendered}" + ); + } +} From 5ab9b3618df26084c0905043d17052a8e3583278 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:22:16 +0800 Subject: [PATCH 04/11] feat(green): validate the resolved message column map before extracting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_from_msgstore now samples up to 512 live message rows and requires 90% of them to carry a plausible epoch-millisecond timestamp (2009-01-01 through acquisition time plus a day of clock skew) before interpreting any record. A map the data contradicts aborts extraction instead of producing a report. The error names the table, the column, the ordinal it resolved to, the observed pass rate, the expected bounds, and up to three offending values with their _id — everything needed to identify the schema by hand. Blob values state their full length and label the 16 bytes shown, so nothing is silently elided. An empty message table stays a valid result: degrade-to-empty is only legitimate once the bootstrap is known good. 328 crate tests, 1163 workspace tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat4n6-whatsapp/src/extractor.rs | 10 ++- .../chat4n6-whatsapp/src/schema_gate.rs | 70 +++++++++++++++++-- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index d15a91e..2425a6a 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -6,6 +6,7 @@ use crate::columns::{ }; use crate::schema::SchemaVersion; pub use crate::schema::{default_mime_for_type, is_media_type, msg_type_label}; +use crate::schema_gate::validate_message_columns; use anyhow::{bail, Context, Result}; use chat4n6_plugin_api::{ CallRecord, CallResult, Chat, Contact, EditHistoryEntry, ExtractionResult, ForensicTimestamp, @@ -17,6 +18,7 @@ use chat4n6_sqlite_forensics::{ partition_by_table, record::{RecoveredRecord, SqlValue}, }; +use chrono::Utc; use rayon::prelude::*; use std::collections::{HashMap, HashSet}; @@ -64,9 +66,15 @@ pub fn extract_from_msgstore( // Build chat map: chat_id → Chat (populated with messages below) let mut chats = build_chats(tbl(&by_table, "chat"), &jid_map, &chat_cols); + // Validate the resolved map against the records before interpreting any of + // them. A wrong map still yields a complete report, so this must fail + // loudly rather than degrade. + let msg_records = tbl(&by_table, "message"); + validate_message_columns(msg_records, &msg_cols, Utc::now().timestamp_millis()) + .context("msgstore.db schema validation failed")?; + // Map messages into chats. If the chat record was deleted/unrecovered, // create a stub so forensically-recovered messages are never silently dropped. - let msg_records = tbl(&by_table, "message"); for rec in msg_records { if let Some(msg) = record_to_message(rec, &jid_map, tz_offset_secs, &msg_cols) { chats diff --git a/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs index 784cb7b..ed0aec9 100644 --- a/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs +++ b/crates/plugins/chat4n6-whatsapp/src/schema_gate.rs @@ -39,12 +39,72 @@ const OFFENDERS_SHOWN: usize = 3; /// `now_ms` is the acquisition time in epoch milliseconds; it is passed in /// rather than read from the clock so the bounds are testable. pub fn validate_message_columns( - _records: &[&RecoveredRecord], - _cols: &MessageColumns, - _now_ms: i64, + records: &[&RecoveredRecord], + cols: &MessageColumns, + now_ms: i64, ) -> Result<()> { - // RED stub — validation is implemented in the GREEN commit. - Ok(()) + let Some(ts_idx) = cols.timestamp else { + bail!( + "msgstore `message` table declares no `timestamp` column; the schema is \ + unrecognised and its records will not be interpreted" + ); + }; + if cols.chat_row_id.is_none() { + bail!( + "msgstore `message` table declares no `chat_row_id` column; messages \ + cannot be attributed to a chat and will not be interpreted" + ); + } + if records.is_empty() { + return Ok(()); + } + + let latest_ms = now_ms.saturating_add(FUTURE_GRACE_MS); + let mut sampled = 0usize; + let mut plausible = 0usize; + let mut offenders: Vec = Vec::new(); + let mut short_rows = 0usize; + + for r in records.iter().take(SAMPLE_SIZE) { + let value = r.values.get(ts_idx); + if value.is_none() { + short_rows += 1; + } + sampled += 1; + if matches!(value, Some(SqlValue::Int(ms)) if *ms >= EARLIEST_PLAUSIBLE_MS && *ms <= latest_ms) + { + plausible += 1; + } else if offenders.len() < OFFENDERS_SHOWN { + let row = r + .row_id + .map_or_else(|| "?".to_string(), |id| id.to_string()); + offenders.push(format!("_id={row} -> {}", render_value(value))); + } + } + + #[allow(clippy::cast_precision_loss)] // sample is at most SAMPLE_SIZE rows + let ratio = plausible as f64 / sampled as f64; + if ratio >= MIN_PLAUSIBLE_RATIO { + return Ok(()); + } + + let detail = if short_rows == sampled { + format!("no sampled record is long enough to carry column {ts_idx}") + } else { + format!( + "{plausible} of {sampled} sampled rows carry a plausible epoch-millisecond \ + value (need {:.0}%)", + MIN_PLAUSIBLE_RATIO * 100.0 + ) + }; + + bail!( + "msgstore `message`.`timestamp` resolved to column {ts_idx}, but {detail}. \ + Offending values: [{}]. Expected epoch milliseconds between \ + {EARLIEST_PLAUSIBLE_MS} (2009-01-01) and {latest_ms} (acquisition + 1 day). \ + Refusing to report on a column map the data contradicts.", + offenders.join("; ") + ) } /// Render a stored value for a diagnostic, verbatim where it is bounded. From 6ca3fa015eb428c4e11d31e9d9b528d84a7f451d Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:24:29 +0800 Subject: [PATCH 05/11] test(red): distribution-level timestamp detector (4 tests fail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No forensic warning fired on 245,981 messages all dated 1970-01-01 — the most anomalous timestamp distribution a msgstore can have. detect_timestamp_anomalies compares each message with its neighbour, and a set that is uniformly wrong is in perfect order, so it had nothing to report. The gap is the level of analysis, not the threshold. Specifies detect_timestamp_distribution_anomaly over the whole extraction: fires when more than 5% of messages predate WhatsApp's 2009 release, reporting the count, the share, and the modal implausible instant with its occurrence count. The share is measured across every chat, so ten bad messages in their own chat are 10% of the extraction rather than 100% of one thread. One test asserts the pairwise detector stays silent on the same input, pinning why the set-level check has to exist rather than duplicating an existing one. Adds ForensicWarning::TimestampDistributionAnomaly and its Display arm; the detector ships as a null implementation returning no warnings. Co-Authored-By: Claude Opus 5 (1M context) --- crates/chat4n6-plugin-api/src/lib.rs | 23 +- crates/chat4n6-plugin-api/src/types.rs | 235 ++++++++++++++---- .../chat4n6-whatsapp/src/anti_forensics.rs | 193 ++++++++++++++ 3 files changed, 398 insertions(+), 53 deletions(-) diff --git a/crates/chat4n6-plugin-api/src/lib.rs b/crates/chat4n6-plugin-api/src/lib.rs index 68e2026..1a41eb4 100644 --- a/crates/chat4n6-plugin-api/src/lib.rs +++ b/crates/chat4n6-plugin-api/src/lib.rs @@ -31,7 +31,10 @@ mod tests { assert_eq!(EvidenceSource::WalDeleted.to_string(), "WAL-DELETED"); assert_eq!(EvidenceSource::Journal.to_string(), "JOURNAL"); assert_eq!(EvidenceSource::IndexRecovery.to_string(), "INDEX-RECOVERY"); - assert_eq!(EvidenceSource::CarvedOverflow.to_string(), "CARVED-OVERFLOW"); + assert_eq!( + EvidenceSource::CarvedOverflow.to_string(), + "CARVED-OVERFLOW" + ); assert_eq!( EvidenceSource::CarvedIntraPage { confidence_pct: 75 }.to_string(), "CARVED-INTRA-PAGE 75%" @@ -158,7 +161,10 @@ mod tests { assert_eq!(m, back); assert_eq!(back.file_hash.as_deref(), Some("abc123def456")); assert_eq!(back.encrypted_hash.as_deref(), Some("enc789xyz000")); - assert_eq!(back.cdn_url.as_deref(), Some("https://mmg.whatsapp.net/v/abc")); + assert_eq!( + back.cdn_url.as_deref(), + Some("https://mmg.whatsapp.net/v/abc") + ); assert_eq!(back.media_key_b64.as_deref(), Some("dGVzdGtleQ==")); } @@ -173,11 +179,18 @@ mod tests { "thumbnail_b64": null, "duration_secs": 30 }"#; - let m: MediaRef = serde_json::from_str(old_json).expect("must deserialize old JSON without error"); + let m: MediaRef = + serde_json::from_str(old_json).expect("must deserialize old JSON without error"); assert!(m.file_hash.is_none(), "file_hash should default to None"); - assert!(m.encrypted_hash.is_none(), "encrypted_hash should default to None"); + assert!( + m.encrypted_hash.is_none(), + "encrypted_hash should default to None" + ); assert!(m.cdn_url.is_none(), "cdn_url should default to None"); - assert!(m.media_key_b64.is_none(), "media_key_b64 should default to None"); + assert!( + m.media_key_b64.is_none(), + "media_key_b64 should default to None" + ); assert_eq!(m.mime_type, "video/mp4"); assert_eq!(m.file_size, 4096); assert_eq!(m.duration_secs, Some(30)); diff --git a/crates/chat4n6-plugin-api/src/types.rs b/crates/chat4n6-plugin-api/src/types.rs index 4d5829a..26d58eb 100644 --- a/crates/chat4n6-plugin-api/src/types.rs +++ b/crates/chat4n6-plugin-api/src/types.rs @@ -320,15 +320,38 @@ pub enum ForensicWarning { /// SQLite VACUUM was run — freelist erased, potentially destroying deleted record remnants. DatabaseVacuumed { freelist_page_count: u32 }, /// ROWID gaps concentrate on one JID: possible targeted scrubbing. - SelectiveDeletion { suspect_jid: String, deletion_rate_pct: u8 }, + SelectiveDeletion { + suspect_jid: String, + deletion_rate_pct: u8, + }, /// Timestamp order violated: a later ROWID has an earlier timestamp. - TimestampAnomaly { message_row_id: i64, description: String }, + TimestampAnomaly { + message_row_id: i64, + description: String, + }, + /// The message timestamp distribution is implausible in bulk — a large + /// share of messages predate the app's existence. A property of the whole + /// set, which no per-message check can express. + TimestampDistributionAnomaly { + total_messages: u32, + implausible_count: u32, + ratio_pct: u8, + /// The most frequent implausible instant, and how often it occurs. + modal_utc: DateTime, + modal_occurrences: u32, + }, /// Backup crypt14/15 HMAC does not match payload — file may have been tampered. HmacMismatch, /// PRAGMA user_version inconsistent with claimed app version. - SchemaVersionMismatch { db_version: u32, app_version: String }, + SchemaVersionMismatch { + db_version: u32, + app_version: String, + }, /// SQLite change counter implies writes after acquisition date. - HeaderTampered { change_counter: u32, expected_max: u32 }, + HeaderTampered { + change_counter: u32, + expected_max: u32, + }, /// iOS CoreData primary-key gap indicates deleted records. CoreDataPkGap { entity_name: String, @@ -367,53 +390,131 @@ pub enum ForensicWarning { /// Signal sealed-sender message whose sender identity could not be resolved. SealedSenderUnresolved { thread_id: i64, count: u32 }, /// A forwarded message references a source message ID that is not present in any snapshot. - UnresolvedForwardSource { message_id: i64, forward_from_id: i64 }, + UnresolvedForwardSource { + message_id: i64, + forward_from_id: i64, + }, } impl fmt::Display for ForensicWarning { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::DatabaseVacuumed { freelist_page_count } => { - write!(f, "VACUUM detected (freelist pages remaining: {freelist_page_count})") + Self::DatabaseVacuumed { + freelist_page_count, + } => { + write!( + f, + "VACUUM detected (freelist pages remaining: {freelist_page_count})" + ) } - Self::SelectiveDeletion { suspect_jid, deletion_rate_pct } => { - write!(f, "Selective deletion: {suspect_jid} ({deletion_rate_pct}% gap rate)") + Self::SelectiveDeletion { + suspect_jid, + deletion_rate_pct, + } => { + write!( + f, + "Selective deletion: {suspect_jid} ({deletion_rate_pct}% gap rate)" + ) } - Self::TimestampAnomaly { message_row_id, description } => { - write!(f, "Timestamp anomaly at row {message_row_id}: {description}") + Self::TimestampAnomaly { + message_row_id, + description, + } => { + write!( + f, + "Timestamp anomaly at row {message_row_id}: {description}" + ) } + Self::TimestampDistributionAnomaly { + total_messages, + implausible_count, + ratio_pct, + modal_utc, + modal_occurrences, + } => write!( + f, + "Implausible timestamp distribution: {implausible_count} of {total_messages} \ + messages ({ratio_pct}%) predate 2009-01-01; most frequent is {modal_utc} \ + ({modal_occurrences}×)" + ), Self::HmacMismatch => write!(f, "HMAC mismatch — backup integrity check FAILED"), - Self::SchemaVersionMismatch { db_version, app_version } => { - write!(f, "Schema v{db_version} incompatible with app version {app_version}") + Self::SchemaVersionMismatch { + db_version, + app_version, + } => { + write!( + f, + "Schema v{db_version} incompatible with app version {app_version}" + ) } - Self::HeaderTampered { change_counter, expected_max } => { - write!(f, "Header tamper: change_counter={change_counter} > expected_max={expected_max}") + Self::HeaderTampered { + change_counter, + expected_max, + } => { + write!( + f, + "Header tamper: change_counter={change_counter} > expected_max={expected_max}" + ) } - Self::CoreDataPkGap { entity_name, recovered_count, .. } => { - write!(f, "CoreData PK gap in {entity_name}: {recovered_count} potential deleted rows") + Self::CoreDataPkGap { + entity_name, + recovered_count, + .. + } => { + write!( + f, + "CoreData PK gap in {entity_name}: {recovered_count} potential deleted rows" + ) } - Self::ImpossibleTimestamp { message_row_id, reason, .. } => { - write!(f, "Impossible timestamp at row {message_row_id}: {reason:?}") + Self::ImpossibleTimestamp { + message_row_id, + reason, + .. + } => { + write!( + f, + "Impossible timestamp at row {message_row_id}: {reason:?}" + ) } - Self::DuplicateStanzaId { stanza_id, occurrences } => { + Self::DuplicateStanzaId { + stanza_id, + occurrences, + } => { write!(f, "Duplicate stanza ID '{stanza_id}' seen {occurrences}×") } Self::RowIdReuseDetected { table, rowid, .. } => { write!(f, "ROWID {rowid} reused in table '{table}'") } - Self::ThumbnailOrphanHigh { orphan_thumbnails, ratio_pct, .. } => { - write!(f, "High thumbnail orphan rate: {orphan_thumbnails} orphans ({ratio_pct}%)") + Self::ThumbnailOrphanHigh { + orphan_thumbnails, + ratio_pct, + .. + } => { + write!( + f, + "High thumbnail orphan rate: {orphan_thumbnails} orphans ({ratio_pct}%)" + ) } Self::PerFileHmacMismatch { file_name } => { write!(f, "Per-file HMAC mismatch: {file_name}") } - Self::DisappearingTimerActive { chat_id, timer_seconds, vanished_count } => { + Self::DisappearingTimerActive { + chat_id, + timer_seconds, + vanished_count, + } => { write!(f, "Disappearing timer on chat {chat_id} ({timer_seconds}s): {vanished_count} messages vanished") } Self::SealedSenderUnresolved { thread_id, count } => { - write!(f, "Sealed-sender unresolved: {count} messages in thread {thread_id}") + write!( + f, + "Sealed-sender unresolved: {count} messages in thread {thread_id}" + ) } - Self::UnresolvedForwardSource { message_id, forward_from_id } => { + Self::UnresolvedForwardSource { + message_id, + forward_from_id, + } => { write!(f, "Forward source missing: message {message_id} references absent ID {forward_from_id}") } } @@ -595,9 +696,14 @@ mod new_types_tests { // ── ForensicWarning ─────────────────────────────────────────────────── #[test] fn forensic_warning_vacuum_display() { - let w = ForensicWarning::DatabaseVacuumed { freelist_page_count: 0 }; + let w = ForensicWarning::DatabaseVacuumed { + freelist_page_count: 0, + }; let s = format!("{}", w); - assert!(s.contains("VACUUM") || s.contains("vacuum") || s.contains("Vacuum"), "got: {s}"); + assert!( + s.contains("VACUUM") || s.contains("vacuum") || s.contains("Vacuum"), + "got: {s}" + ); } #[test] @@ -607,14 +713,20 @@ mod new_types_tests { deletion_rate_pct: 87, }; let s = format!("{}", w); - assert!(s.contains("87") || s.contains("selective") || s.contains("Selective"), "got: {s}"); + assert!( + s.contains("87") || s.contains("selective") || s.contains("Selective"), + "got: {s}" + ); } #[test] fn forensic_warning_hmac_mismatch() { let w = ForensicWarning::HmacMismatch; let json = serde_json::to_string(&w).unwrap(); - assert!(json.contains("Hmac") || json.contains("hmac") || json.contains("HMAC"), "got: {json}"); + assert!( + json.contains("Hmac") || json.contains("hmac") || json.contains("HMAC"), + "got: {json}" + ); } #[test] @@ -735,10 +847,14 @@ mod new_types_tests { #[test] fn extraction_result_has_acquisition_timestamps() { let r = ExtractionResult::default(); - assert!(r.extraction_started_at.is_none(), - "extraction_started_at must default to None"); - assert!(r.extraction_finished_at.is_none(), - "extraction_finished_at must default to None"); + assert!( + r.extraction_started_at.is_none(), + "extraction_started_at must default to None" + ); + assert!( + r.extraction_finished_at.is_none(), + "extraction_finished_at must default to None" + ); // Roundtrip with values set let json = r#"{"chats":[],"contacts":[],"calls":[],"wal_deltas":[], "extraction_started_at":"2026-05-06T10:00:00Z", @@ -751,7 +867,10 @@ mod new_types_tests { #[test] fn extraction_result_has_wal_snapshots() { let r = ExtractionResult::default(); - assert!(r.wal_snapshots.is_empty(), "wal_snapshots must default to empty"); + assert!( + r.wal_snapshots.is_empty(), + "wal_snapshots must default to empty" + ); // Build a snapshot and roundtrip let snap = WalSnapshot { frame_number: 3, @@ -778,18 +897,28 @@ mod new_types_tests { original_timestamp: Some(ts), }; let m = Message { - id: 1, chat_id: 1, - sender_jid: None, from_me: false, + id: 1, + chat_id: 1, + sender_jid: None, + from_me: false, timestamp: ForensicTimestamp::from_millis(1_710_513_200_000, 0), content: MessageContent::Text("forwarded".to_string()), - reactions: vec![], quoted_message: None, - source: EvidenceSource::Live, row_offset: 0, - starred: false, forward_score: Some(3), is_forwarded: true, - edit_history: vec![], receipts: vec![], + reactions: vec![], + quoted_message: None, + source: EvidenceSource::Live, + row_offset: 0, + starred: false, + forward_score: Some(3), + is_forwarded: true, + edit_history: vec![], + receipts: vec![], forwarded_from: Some(origin), }; let json = serde_json::to_string(&m).unwrap(); - assert!(json.contains("Channel"), "ForwardOriginKind::Channel must serialise"); + assert!( + json.contains("Channel"), + "ForwardOriginKind::Channel must serialise" + ); let back: Message = serde_json::from_str(&json).unwrap(); let fo = back.forwarded_from.unwrap(); assert_eq!(fo.origin_id, "tg-channel://123456"); @@ -813,7 +942,9 @@ mod new_types_tests { let warnings: Vec = vec![ ForensicWarning::CoreDataPkGap { entity_name: "ZWAMESSAGE".to_string(), - expected_max: 500, observed_max: 450, recovered_count: 10, + expected_max: 500, + observed_max: 450, + recovered_count: 10, }, ForensicWarning::ImpossibleTimestamp { message_row_id: 42, @@ -821,26 +952,34 @@ mod new_types_tests { reason: ImpossibleReason::BeforeUnixEpoch, }, ForensicWarning::DuplicateStanzaId { - stanza_id: "ABC123".to_string(), occurrences: 2, + stanza_id: "ABC123".to_string(), + occurrences: 2, }, ForensicWarning::RowIdReuseDetected { - table: "messages".to_string(), rowid: 99, + table: "messages".to_string(), + rowid: 99, conflicting_timestamps: vec![Utc::now(), Utc::now()], }, ForensicWarning::ThumbnailOrphanHigh { - orphan_thumbnails: 5, total_messages: 10, ratio_pct: 50, + orphan_thumbnails: 5, + total_messages: 10, + ratio_pct: 50, }, ForensicWarning::PerFileHmacMismatch { file_name: "msgstore.db".to_string(), }, ForensicWarning::DisappearingTimerActive { - chat_id: 1, timer_seconds: 86400, vanished_count: 3, + chat_id: 1, + timer_seconds: 86400, + vanished_count: 3, }, ForensicWarning::SealedSenderUnresolved { - thread_id: 7, count: 2, + thread_id: 7, + count: 2, }, ForensicWarning::UnresolvedForwardSource { - message_id: 55, forward_from_id: 99999, + message_id: 55, + forward_from_id: 99999, }, ]; for w in &warnings { diff --git a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs index 5ef9aed..d1f1369 100644 --- a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs +++ b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs @@ -130,6 +130,30 @@ pub fn detect_timestamp_anomalies(result: &ExtractionResult) -> Vec Vec { + // RED stub — detection is implemented in the GREEN commit. + Vec::new() +} + // ── New detectors (§2.6) ───────────────────────────────────────────────────── /// Detect duplicate XMPP stanza IDs (key_id column) in the message table. @@ -608,3 +632,172 @@ mod new_detector_tests { ); } } + +// ── T3: distribution-level timestamp detection ─────────────────────────────── + +#[cfg(test)] +mod timestamp_distribution_tests { + use super::*; + use chat4n6_plugin_api::{EvidenceSource, ForensicTimestamp, Message, MessageContent}; + + /// 2022-09-15T12:00:00Z. + const GOOD_MS: i64 = 1_663_243_200_000; + + fn msg(id: i64, ts_ms: i64) -> Message { + Message { + id, + chat_id: 1, + sender_jid: None, + from_me: false, + timestamp: ForensicTimestamp::from_millis(ts_ms, 0), + content: MessageContent::Text(String::new()), + reactions: vec![], + quoted_message: None, + source: EvidenceSource::Live, + row_offset: 0, + starred: false, + forward_score: None, + is_forwarded: false, + edit_history: vec![], + receipts: vec![], + forwarded_from: None, + } + } + + fn chat_of(messages: Vec) -> Vec { + vec![Chat { + id: 1, + jid: "a@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages, + archived: false, + }] + } + + fn distribution_warning(chats: &[Chat]) -> Option { + detect_timestamp_distribution_anomaly(chats) + .into_iter() + .find(|w| matches!(w, ForensicWarning::TimestampDistributionAnomaly { .. })) + } + + /// The headline failure: every message at epoch zero, in perfect order, so + /// the pairwise detector sees nothing wrong. + #[test] + fn mass_epoch_zero_timestamps_raise_a_warning() { + let chats = chat_of((1..=100).map(|i| msg(i, 0)).collect()); + assert!( + distribution_warning(&chats).is_some(), + "100 messages dated 1970-01-01 must not pass unremarked" + ); + assert!( + detect_timestamp_anomalies(&ExtractionResult { + chats: chats.clone(), + contacts: vec![], + calls: vec![], + wal_deltas: vec![], + timezone_offset_seconds: Some(0), + schema_version: 1, + forensic_warnings: vec![], + group_participant_events: vec![], + extraction_started_at: None, + extraction_finished_at: None, + wal_snapshots: vec![], + }) + .is_empty(), + "the pairwise detector cannot see this — which is why the set-level one exists" + ); + } + + #[test] + fn warning_reports_the_count_share_and_modal_instant() { + let mut messages: Vec = (1..=90).map(|i| msg(i, GOOD_MS + i)).collect(); + messages.extend((91..=100).map(|i| msg(i, 0))); + let warning = distribution_warning(&chat_of(messages)).expect("10% is over the threshold"); + match warning { + ForensicWarning::TimestampDistributionAnomaly { + total_messages, + implausible_count, + ratio_pct, + modal_utc, + modal_occurrences, + } => { + assert_eq!(total_messages, 100); + assert_eq!(implausible_count, 10); + assert_eq!(ratio_pct, 10); + assert_eq!( + modal_utc.timestamp_millis(), + 0, + "epoch zero is the modal instant" + ); + assert_eq!(modal_occurrences, 10); + } + other => panic!("expected TimestampDistributionAnomaly, got {other:?}"), + } + } + + #[test] + fn a_healthy_distribution_raises_nothing() { + let chats = chat_of((1..=100).map(|i| msg(i, GOOD_MS + i * 1000)).collect()); + assert!( + distribution_warning(&chats).is_none(), + "plausible timestamps must not be flagged" + ); + } + + #[test] + fn a_handful_of_bad_rows_stays_below_the_threshold() { + let mut messages: Vec = (1..=98).map(|i| msg(i, GOOD_MS + i * 1000)).collect(); + messages.push(msg(99, 0)); + messages.push(msg(100, 1)); + assert!( + distribution_warning(&chat_of(messages)).is_none(), + "2% of implausible rows is an outlier, not a distribution anomaly" + ); + } + + #[test] + fn an_empty_extraction_raises_nothing() { + assert!(detect_timestamp_distribution_anomaly(&[]).is_empty()); + assert!(distribution_warning(&chat_of(vec![])).is_none()); + } + + #[test] + fn the_share_is_measured_across_all_chats_not_per_chat() { + // 10 implausible messages sitting in their own chat are 10% of the + // extraction, not 100% of one chat. + let good = Chat { + id: 1, + jid: "a@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages: (1..=90).map(|i| msg(i, GOOD_MS + i * 1000)).collect(), + archived: false, + }; + let bad = Chat { + id: 2, + jid: "b@s.whatsapp.net".to_string(), + name: None, + is_group: false, + messages: (91..=100).map(|i| msg(i, 0)).collect(), + archived: false, + }; + let warning = distribution_warning(&[good, bad]).expect("10% overall"); + if let ForensicWarning::TimestampDistributionAnomaly { ratio_pct, .. } = warning { + assert_eq!(ratio_pct, 10); + } + } + + #[test] + fn the_warning_renders_the_numbers_it_carries() { + let chats = chat_of((1..=20).map(|i| msg(i, 0)).collect()); + let rendered = distribution_warning(&chats) + .expect("all implausible") + .to_string(); + assert!(rendered.contains("20"), "must state the counts: {rendered}"); + assert!( + rendered.contains("1970"), + "must state the modal instant: {rendered}" + ); + } +} From 7fb93a3adf038a1579c9b6eed872e9567ffd6f9b Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:26:03 +0800 Subject: [PATCH 06/11] feat(green): raise a warning on an implausible timestamp distribution detect_timestamp_distribution_anomaly counts messages predating 2009-01-01 across the whole extraction and emits TimestampDistributionAnomaly above a 5% share, carrying the total, the count, the percentage, and the modal implausible instant with its occurrence count. Ties on the modal instant break toward the earlier one so repeated runs report the same value. Shares the 2009 bound with the schema gate rather than restating it, and compares in milliseconds so no fallible conversion sits on the hot path. Wired into extract_from_msgstore alongside the existing detectors. Its standing job is genuine tampering: a database whose timestamps were rewritten has the same signature as one read off the wrong column. 335 crate tests, 1170 workspace tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../chat4n6-whatsapp/src/anti_forensics.rs | 41 +++++++++++++++++-- .../plugins/chat4n6-whatsapp/src/extractor.rs | 2 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs index d1f1369..9a75902 100644 --- a/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs +++ b/crates/plugins/chat4n6-whatsapp/src/anti_forensics.rs @@ -3,6 +3,7 @@ //! Detects evidence of tampering, selective deletion, timestamp anomalies, //! and SQLite VACUUM operations that destroy deleted record remnants. +use crate::schema_gate::EARLIEST_PLAUSIBLE_MS; use chat4n6_plugin_api::{Chat, ExtractionResult, ForensicWarning}; use chrono::{DateTime, Utc}; use std::collections::HashMap; @@ -149,9 +150,43 @@ pub const IMPLAUSIBLE_SHARE_PCT: u8 = 5; /// /// Its standing job is genuine tampering — a database whose timestamps were /// rewritten shows the same signature. -pub fn detect_timestamp_distribution_anomaly(_chats: &[Chat]) -> Vec { - // RED stub — detection is implemented in the GREEN commit. - Vec::new() +pub fn detect_timestamp_distribution_anomaly(chats: &[Chat]) -> Vec { + let mut total: u32 = 0; + let mut occurrences: HashMap, u32> = HashMap::new(); + for msg in chats.iter().flat_map(|c| c.messages.iter()) { + total = total.saturating_add(1); + // Compared in milliseconds so the bound needs no fallible conversion. + if msg.timestamp.utc.timestamp_millis() < EARLIEST_PLAUSIBLE_MS { + *occurrences.entry(msg.timestamp.utc).or_insert(0) += 1; + } + } + + let implausible_count: u32 = occurrences.values().sum(); + if total == 0 || implausible_count == 0 { + return vec![]; + } + + // u64 avoids overflowing the ×100 on a large extraction. + let ratio_pct = (u64::from(implausible_count) * 100 / u64::from(total)).min(100) as u8; + if ratio_pct <= IMPLAUSIBLE_SHARE_PCT { + return vec![]; + } + + // Ties broken by the earlier instant so the finding is reproducible. + let Some((&modal_utc, &modal_occurrences)) = occurrences + .iter() + .max_by_key(|(instant, count)| (**count, std::cmp::Reverse(**instant))) + else { + return vec![]; + }; + + vec![ForensicWarning::TimestampDistributionAnomaly { + total_messages: total, + implausible_count, + ratio_pct, + modal_utc, + modal_occurrences, + }] } // ── New detectors (§2.6) ───────────────────────────────────────────────────── diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index 2425a6a..c9ef7ea 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -1,5 +1,6 @@ use crate::anti_forensics::{ detect_duplicate_stanza_ids, detect_rowid_reuse, detect_thumbnail_orphans, + detect_timestamp_distribution_anomaly, }; use crate::columns::{ CallLogColumns, ChatColumns, JidColumns, MessageColumns, SchemaColumns, TableColumns, @@ -318,6 +319,7 @@ pub fn extract_from_msgstore( total_messages, )); forensic_warnings.extend(detect_rowid_reuse(&chats_vec)); + forensic_warnings.extend(detect_timestamp_distribution_anomaly(&chats_vec)); Ok(ExtractionResult { chats: chats_vec, From 52a64f87cc5c1f5096960cc88368212bb5a890d5 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:28:28 +0800 Subject: [PATCH 07/11] test(red): fixtures that can falsify a positional reader (1 test fails) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modern_schema.sql was authored to the same layout the extractor's ordinals assumed, so it agreed with the code while both were wrong about real data. Two fixtures that cannot do that: - shuffled_schema.sql — real_modern_schema.sql with every column order permuted and _id declared mid-table. Paired with the real fixture it makes position-independence a property under test: two databases differing only in declaration order must extract identically. This passes now, which is the point — it is the regression guard that fails the moment an ordinal returns. - legacy_schema.sql — the pre-2018 `messages` + `chat_list` generation, taken from published schema documentation rather than the case device (nothing in the case is of this generation), and labelled as such. The legacy fixture exposes a real gap, and that is the failing test: a legacy database has no `message` table, so extraction reports zero messages — which is indistinguishable from a clean device. It must name the generation it found and refuse. modern_schema.sql keeps its simplified shapes but now says so, and says why it is kept: it is the only fixture exercising media metadata on the `message` table, and its aux-table column names are unverified against any device. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat4n6-whatsapp/src/extractor.rs | 112 ++++++++++++++++++ crates/plugins/chat4n6-whatsapp/src/schema.rs | 18 +++ .../tests/fixtures/legacy_schema.sql | 80 +++++++++++++ .../tests/fixtures/modern_schema.sql | 24 +++- .../tests/fixtures/shuffled_schema.sql | 96 +++++++++++++++ 5 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql create mode 100644 crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index c9ef7ea..9cf4a9f 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -2309,6 +2309,118 @@ mod real_schema_tests { assert_eq!(voice.duration_secs, 137); } + fn make_shuffled_msgstore() -> Vec { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/shuffled_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + std::fs::read(tmp.path()).unwrap() + } + + /// Two databases that differ only in column declaration order must extract + /// identically. Any ordinal creeping back into the extractor breaks this. + #[test] + fn column_order_does_not_change_the_extraction() { + let straight = extract_from_msgstore(&make_real_msgstore(), 0, SchemaVersion::Modern) + .expect("real-order fixture"); + let shuffled = extract_from_msgstore(&make_shuffled_msgstore(), 0, SchemaVersion::Modern) + .expect("permuted-order fixture"); + + let summarise = |r: &ExtractionResult| { + let mut chats: Vec<(i64, String, Option, bool)> = r + .chats + .iter() + .map(|c| (c.id, c.jid.clone(), c.name.clone(), c.archived)) + .collect(); + chats.sort(); + let mut msgs: Vec<(i64, i64, Option, bool, i64, bool)> = r + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .map(|m| { + ( + m.id, + m.chat_id, + m.sender_jid.clone(), + m.from_me, + m.timestamp.utc.timestamp_millis(), + m.starred, + ) + }) + .collect(); + msgs.sort(); + let mut calls: Vec<(i64, bool, u32, i64, bool)> = r + .calls + .iter() + .map(|c| { + ( + c.call_id, + c.video, + c.duration_secs, + c.timestamp.utc.timestamp_millis(), + c.group_call, + ) + }) + .collect(); + calls.sort(); + (chats, msgs, calls) + }; + + assert_eq!( + summarise(&straight), + summarise(&shuffled), + "extraction must depend on column names, not on their declaration order" + ); + // Guard against the comparison passing because both sides are empty. + assert_eq!(summarise(&straight).1.len(), 3); + } + + /// A legacy-generation database has no `message` table at all. Reporting + /// zero messages would be indistinguishable from a clean device. + #[test] + fn a_legacy_schema_database_is_refused_rather_than_reported_empty() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch(include_str!("../tests/fixtures/legacy_schema.sql")) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Legacy) + .expect_err("a legacy database must not yield a silently empty report"); + let msg = format!("{err:#}"); + assert!( + msg.contains("messages"), + "error must name the legacy table it found: {msg}" + ); + assert!( + msg.contains("message") && msg.contains("legacy"), + "error must say which generation it recognised: {msg}" + ); + } + + /// A database with neither table is likewise a refusal, not an empty report. + #[test] + fn a_database_with_no_message_table_is_refused() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE TABLE unrelated (_id INTEGER PRIMARY KEY, x TEXT);") + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + conn.backup(rusqlite::DatabaseName::Main, tmp.path(), None) + .unwrap(); + let db = std::fs::read(tmp.path()).unwrap(); + + let err = extract_from_msgstore(&db, 0, SchemaVersion::Modern) + .expect_err("a database with no message table is not a msgstore.db"); + assert!( + format!("{err:#}").contains("message"), + "error must name what it looked for: {err:#}" + ); + } + /// A database whose `message.timestamp` column does not hold epoch /// milliseconds is a bootstrap failure: extraction must stop rather than /// emit a report whose every date is wrong. diff --git a/crates/plugins/chat4n6-whatsapp/src/schema.rs b/crates/plugins/chat4n6-whatsapp/src/schema.rs index b8aa8b6..fa30dd3 100644 --- a/crates/plugins/chat4n6-whatsapp/src/schema.rs +++ b/crates/plugins/chat4n6-whatsapp/src/schema.rs @@ -75,6 +75,24 @@ pub fn default_mime_for_type(msg_type: i32) -> &'static str { mod tests { use super::*; + /// Classification of the committed legacy fixture, so the legacy branch is + /// pinned to a concrete schema rather than a hand-written table list. + #[test] + fn legacy_fixture_is_classified_legacy() { + let ddl = include_str!("../tests/fixtures/legacy_schema.sql"); + let tables: Vec<&str> = ddl + .lines() + .filter_map(|l| l.trim().strip_prefix("CREATE TABLE ")) + .filter_map(|l| l.split_whitespace().next()) + .filter_map(|t| t.split('(').next()) + .collect(); + assert!( + tables.contains(&"messages"), + "fixture must declare `messages`" + ); + assert_eq!(detect_schema_version(1, &tables), SchemaVersion::Legacy); + } + #[test] fn test_schema_legacy_detection() { assert_eq!( diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql new file mode 100644 index 0000000..abbd4ed --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/legacy_schema.sql @@ -0,0 +1,80 @@ +-- Legacy WhatsApp Android msgstore schema (pre-2018 `messages` + `chat_list` era). +-- +-- PROVENANCE +-- Column names and order follow the legacy schema published in the WhatsApp +-- forensic literature. Unlike real_modern_schema.sql, this was NOT read off +-- the case device — nothing in the case is of this generation. It exists so +-- the legacy branch has something concrete to classify and refuse, rather +-- than being reasoned about in the abstract. +-- +-- A trimmed but faithful subset: the columns below sit in their published +-- relative order. The extractor reads none of them, because it has no +-- legacy path. +-- +-- WHAT THIS FIXTURE IS FOR +-- 1. detect_schema_version must classify it Legacy. +-- 2. Extraction must refuse it loudly. A database of this generation has no +-- `message`, `jid` or `chat` table, so a modern extractor finds nothing — +-- and "nothing" is indistinguishable from a clean device. + +PRAGMA user_version = 1; + +CREATE TABLE messages ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + key_remote_jid TEXT NOT NULL, + key_from_me INTEGER, + key_id TEXT NOT NULL, + status INTEGER, + needs_push INTEGER, + data TEXT, + timestamp INTEGER, + media_url TEXT, + media_mime_type TEXT, + media_wa_type TEXT, + media_size INTEGER, + media_name TEXT, + media_caption TEXT, + media_hash TEXT, + media_duration INTEGER, + origin INTEGER, + latitude REAL, + longitude REAL, + thumb_image TEXT, + remote_resource TEXT, + received_timestamp INTEGER, + send_timestamp INTEGER, + starred INTEGER, + quoted_row_id INTEGER, + edit_version INTEGER, + forwarded INTEGER +); + +CREATE TABLE chat_list ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + key_remote_jid TEXT UNIQUE, + message_table_id INTEGER, + subject TEXT, + creation INTEGER, + archived INTEGER, + sort_timestamp INTEGER +); + +CREATE TABLE wa_contacts ( + _id INTEGER PRIMARY KEY AUTOINCREMENT, + jid TEXT UNIQUE NOT NULL, + display_name TEXT, + status TEXT, + number TEXT +); + +-- Synthetic rows. Timestamps are 2016-06-01T12:00:00Z (1464782400000). +INSERT INTO chat_list (_id, key_remote_jid, message_table_id, subject, creation, archived) +VALUES (1, '4155550100@s.whatsapp.net', 2, NULL, 1464782400000, 0); + +INSERT INTO messages (_id, key_remote_jid, key_from_me, key_id, data, timestamp, media_wa_type) +VALUES + (1, '4155550100@s.whatsapp.net', 1, 'LEGACY0001', 'Outgoing legacy message', 1464782400000, '0'), + (2, '4155550100@s.whatsapp.net', 0, 'LEGACY0002', 'Incoming legacy message', 1464782460000, '0'); + +INSERT INTO wa_contacts (_id, jid, display_name, status, number) +VALUES (1, '4155550100@s.whatsapp.net', 'Legacy Contact', NULL, '+14155550100'); diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql index f24fd80..6daac50 100644 --- a/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/modern_schema.sql @@ -1,5 +1,25 @@ --- Minimal modern WhatsApp Android msgstore schema + sample data --- Column order matches what ForensicEngine/btree walker will emit +-- Simplified WhatsApp Android msgstore schema + sample data. +-- +-- READ THIS BEFORE ADDING A TEST HERE +-- These CREATE TABLE shapes are a simplification, not a device's schema. They +-- were written to the same column layout the extractor's old hardcoded +-- ordinals assumed, so for as long as those ordinals existed this fixture +-- agreed with the code while both were wrong about real data. It is kept for +-- two reasons, neither of which is realism: +-- +-- 1. It carries `media_mime_type` and `media_name` on the `message` table. +-- Those columns are genuine in older modern-generation schemas but the +-- 2023-era device moved media metadata to `message_media`, which the +-- extractor does not yet read. This fixture is what exercises the +-- media-in-message path. +-- 2. Its aux tables (message_quoted, message_add_on, receipt_user, …) have +-- not been checked against a device, so the column names here are the +-- only ones those code paths are known against. +-- +-- For anything about column POSITIONS, use real_modern_schema.sql (real +-- device DDL) or shuffled_schema.sql (same names, permuted order). A test +-- that could pass here purely because the layout happens to match the reader +-- proves nothing. PRAGMA user_version = 200; diff --git a/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql b/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql new file mode 100644 index 0000000..6d7e091 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/fixtures/shuffled_schema.sql @@ -0,0 +1,96 @@ +-- real_modern_schema.sql with every column order permuted. +-- +-- Same table names, same column names, same rows, same values — only the +-- declaration order differs, and `_id` is deliberately not declared first. +-- +-- WHY +-- A fixture built to the layout the code assumes cannot falsify that +-- assumption; that is how the ordinal bug shipped green. This fixture makes +-- position-independence a property under test rather than a claim: paired +-- with real_modern_schema.sql it asserts that two databases differing only in +-- column order extract identically. Any ordinal creeping back into the +-- extractor breaks one of the pair. +-- +-- Declaring `_id` mid-table also exercises the record-layout invariant the +-- resolver rests on: SQLite writes the INTEGER PRIMARY KEY as a NULL at its +-- own declared position, not at position 0. +-- +-- This is a deliberately synthetic permutation. No device ships these orders. + +PRAGMA user_version = 1; + +CREATE TABLE jid ( + raw_string TEXT, + device INTEGER, + _id INTEGER PRIMARY KEY, + server TEXT NOT NULL, + type INTEGER, + user TEXT NOT NULL, + agent INTEGER +); + +CREATE TABLE chat ( + subject TEXT, + sort_timestamp INTEGER, + archived INTEGER, + created_timestamp INTEGER, + hidden INTEGER, + display_message_row_id INTEGER, + _id INTEGER PRIMARY KEY, + jid_row_id INTEGER UNIQUE, + mod_tag INTEGER +); + +CREATE TABLE message ( + text_data TEXT, + timestamp INTEGER, + view_mode INTEGER, + sender_jid_row_id INTEGER, + sort_id INTEGER, + from_me INTEGER NOT NULL, + _id INTEGER PRIMARY KEY, + status INTEGER, + message_type INTEGER, + key_id TEXT NOT NULL, + received_timestamp INTEGER, + starred INTEGER, + chat_row_id INTEGER NOT NULL, + origin INTEGER +); + +CREATE TABLE call_log ( + duration INTEGER, + call_result INTEGER, + _id INTEGER PRIMARY KEY, + video_call INTEGER, + call_id TEXT, + jid_row_id INTEGER, + bytes_transferred INTEGER, + timestamp INTEGER, + transaction_id INTEGER, + from_me INTEGER +); + +-- Identical rows to real_modern_schema.sql, inserted by name. + +INSERT INTO jid (_id, user, server, agent, type, raw_string, device) VALUES + (1, '4155550100', 's.whatsapp.net', 0, 0, '4155550100@s.whatsapp.net', 0), + (2, '120363001234567890', 'g.us', 0, 1, '120363001234567890@g.us', 0); + +INSERT INTO chat (_id, jid_row_id, hidden, subject, created_timestamp, archived) VALUES + (1, 1, 0, NULL, 1663200000000, 0), + (2, 2, 0, 'Real Schema Group', 1663200000000, 1); + +INSERT INTO message + (_id, chat_row_id, from_me, key_id, sender_jid_row_id, timestamp, message_type, text_data, starred) +VALUES + (1, 1, 1, 'AAAA1111', NULL, 1663243200000, 0, 'Outgoing real-schema message', 0), + (2, 1, 0, 'BBBB2222', 1, 1663243260000, 0, 'Incoming real-schema message', 1), + (3, 2, 0, 'CCCC3333', 1, 1663243320000, 0, 'Group real-schema message', 0); + +INSERT INTO call_log + (_id, jid_row_id, from_me, call_id, transaction_id, timestamp, video_call, duration, call_result) +VALUES + (1, 1, 1, 'CALL-AAAA', 11, 1663243400000, 0, 137, 1), + (2, 1, 0, 'CALL-BBBB', 12, 1663243500000, 1, 42, 1), + (3, 1, 1, 'CALL-CCCC', 13, 1663243600000, 0, 137, 1); From 5cb59ae3a53aebd1c09261c7e404dc883437fb91 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:29:18 +0800 Subject: [PATCH 08/11] fix(green): refuse an unreadable msgstore instead of reporting it empty A database with no modern `message` table now aborts extraction rather than producing a report of zero messages, which reads exactly like a clean device. - A legacy `messages` table present and no `message` table: the error names the generation found and states that this extractor reads the modern one only. - Neither table: the error lists the tables that are there, with the full count and an explicit note when names beyond the first 20 are omitted. 339 crate tests, 1174 workspace tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../plugins/chat4n6-whatsapp/src/columns.rs | 7 ++++ .../plugins/chat4n6-whatsapp/src/extractor.rs | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/crates/plugins/chat4n6-whatsapp/src/columns.rs b/crates/plugins/chat4n6-whatsapp/src/columns.rs index 184af05..28f6aaa 100644 --- a/crates/plugins/chat4n6-whatsapp/src/columns.rs +++ b/crates/plugins/chat4n6-whatsapp/src/columns.rs @@ -101,6 +101,13 @@ impl SchemaColumns { .unwrap_or(&self.empty) } + /// Every table name, sorted — for diagnostics that must show what was found. + pub fn table_names(&self) -> Vec<&str> { + let mut names: Vec<&str> = self.tables.keys().map(String::as_str).collect(); + names.sort_unstable(); + names + } + /// Whether any table DDL was resolved at all. /// /// A database whose schema could not be read is a bootstrap failure, not an diff --git a/crates/plugins/chat4n6-whatsapp/src/extractor.rs b/crates/plugins/chat4n6-whatsapp/src/extractor.rs index 9cf4a9f..c52a6a0 100644 --- a/crates/plugins/chat4n6-whatsapp/src/extractor.rs +++ b/crates/plugins/chat4n6-whatsapp/src/extractor.rs @@ -51,6 +51,23 @@ pub fn extract_from_msgstore( ); } let schema_cols = SchemaColumns::from_ddl_map(&ddl_map); + if schema_cols.table("message").is_empty() { + // Reporting zero messages here would be indistinguishable from a clean + // device, so name what was actually found instead. + if !schema_cols.table("messages").is_empty() { + bail!( + "msgstore.db declares the legacy `messages` table and no modern `message` \ + table; this extractor reads the modern generation (message/jid/chat) only. \ + Refusing to report zero messages for a database it cannot read." + ); + } + bail!( + "msgstore.db declares no `message` table, and no legacy `messages` table \ + either. Tables found ({} total): [{}]", + schema_cols.table_names().len(), + summarise_table_names(&schema_cols.table_names()) + ); + } let msg_cols = MessageColumns::resolve(schema_cols.table("message")); let jid_cols = JidColumns::resolve(schema_cols.table("jid")); let chat_cols = ChatColumns::resolve(schema_cols.table("chat")); @@ -387,6 +404,26 @@ fn tbl<'a>( by.get(name).map(|v| v.as_slice()).unwrap_or_default() } +/// Render a table-name list for a diagnostic. +/// +/// A real msgstore declares well over a hundred tables, so whole names beyond +/// the first 20 are omitted — and the omission is stated, with the full count +/// alongside. Every name shown is verbatim. +fn summarise_table_names(names: &[&str]) -> String { + const SHOWN: usize = 20; + let head = names + .iter() + .take(SHOWN) + .copied() + .collect::>() + .join(", "); + if names.len() > SHOWN { + format!("{head}, and {} more", names.len() - SHOWN) + } else { + head + } +} + /// Read the integer at a resolved column position. /// /// `None` when the schema does not declare the column, the record is shorter From 27ca954bca26ff2c4e5f797ba2d55eca70f34f68 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:33:40 +0800 Subject: [PATCH 09/11] test: Tier-1 reconciliation against a real msgstore.db, and docs/validation.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an env-gated integration test that reconciles the extractor against sqlite3 reading the same evidence file — an independent implementation, not a fixture we authored. Expected values are derived from the oracle at run time rather than transcribed as constants, so the test reconciles against whatever database it is pointed at: message/chat/call counts, the per-year timestamp histogram bucket for bucket, no message dated 1970, senders carrying an '@', hourly spread across at least 12 buckets, and no TimestampDistributionAnomaly raised against the extraction's own dates. A count mismatch fails with a breakdown by evidence source and a distinct-rowid tally, so an over-count is diagnosed rather than merely reported. Without CHAT4N6_REAL_MSGSTORE / CHAT4N6_REAL_INPUT_DIR every test skips and says why; without sqlite3 on PATH the reconciling ones skip. A variable that is set but points at no readable file fails loudly — a typo must not look like absent evidence. The hourly-spread check needs 1000 messages before it means anything and skips below that. Nothing prints or asserts on message content, JIDs, subjects, numbers or paths. Verified by pointing the harness at a synthetic stand-in built from real_modern_schema.sql: the oracle plumbing, the year histogram comparison and the skip paths all exercise correctly. docs/validation.md records the method, tiers each check by who confirms it, and lists the gaps that remain open: message_media metadata, unverified aux-table column names, live-layer-only interpretation, the pairwise timestamp detector being unreachable after sorting, and the three unaudited sibling plugins. The Tier-1 run itself needs the case evidence and has not been executed here. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 1 + crates/plugins/chat4n6-whatsapp/Cargo.toml | 1 + .../tests/real_msgstore_validation.rs | 359 ++++++++++++++++++ docs/validation.md | 165 ++++++++ 4 files changed, 526 insertions(+) create mode 100644 crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs create mode 100644 docs/validation.md diff --git a/Cargo.lock b/Cargo.lock index fa9c4a5..95a1227 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,6 +373,7 @@ dependencies = [ "anyhow", "base64", "cbc", + "chat4n6-fs", "chat4n6-plugin-api", "chat4n6-sqlite-forensics", "chrono", diff --git a/crates/plugins/chat4n6-whatsapp/Cargo.toml b/crates/plugins/chat4n6-whatsapp/Cargo.toml index f595c2d..5740d67 100644 --- a/crates/plugins/chat4n6-whatsapp/Cargo.toml +++ b/crates/plugins/chat4n6-whatsapp/Cargo.toml @@ -31,6 +31,7 @@ regex = "1" base64 = { workspace = true } [dev-dependencies] +chat4n6-fs = { path = "../../chat4n6-fs", version = "0.1" } tempfile = "3" rusqlite = { version = "0.31", features = ["backup"] } proptest = "1" diff --git a/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs b/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs new file mode 100644 index 0000000..49c4085 --- /dev/null +++ b/crates/plugins/chat4n6-whatsapp/tests/real_msgstore_validation.rs @@ -0,0 +1,359 @@ +//! Tier-1 validation of the WhatsApp extractor against a real msgstore.db. +//! +//! Ground truth comes from `sqlite3` reading the same file — an independent +//! implementation, not a fixture we authored. Fixtures can only prove the code +//! agrees with itself; this proves it agrees with SQLite about a database +//! neither of them was written for. +//! +//! # Running +//! +//! Both inputs are supplied by the operator and never recorded here: +//! +//! ```text +//! CHAT4N6_REAL_MSGSTORE=/path/to/msgstore.db \ +//! CHAT4N6_REAL_INPUT_DIR=/path/to/extraction-root \ +//! cargo test -p chat4n6-whatsapp --test real_msgstore_validation -- --nocapture +//! ``` +//! +//! Absent the variables every test skips cleanly, so CI stays green without the +//! evidence. `sqlite3` must be on PATH for the oracle comparisons; without it +//! the reconciling tests skip and only the self-contained ones run. +//! +//! # Case hygiene +//! +//! Nothing here prints or asserts on message content, phone numbers, JIDs, +//! subjects or paths. Failures report counts, shapes and years only. Keep it +//! that way: this file is committed, the evidence is not. + +use chat4n6_plugin_api::ExtractionResult; +use chat4n6_whatsapp::extractor::extract_from_msgstore; +use chat4n6_whatsapp::schema::SchemaVersion; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const MSGSTORE_ENV: &str = "CHAT4N6_REAL_MSGSTORE"; +const INPUT_DIR_ENV: &str = "CHAT4N6_REAL_INPUT_DIR"; + +/// The evidence path, or `None` with a skip note. +fn msgstore_path() -> Option { + match std::env::var(MSGSTORE_ENV) { + Ok(p) if !p.is_empty() => { + let path = PathBuf::from(p); + if path.is_file() { + Some(path) + } else { + // A variable pointing at nothing is operator error, not absence + // of evidence — say so rather than skipping quietly. + panic!("{MSGSTORE_ENV} is set but is not a readable file"); + } + } + _ => { + eprintln!("SKIP: {MSGSTORE_ENV} not set — Tier-1 validation not run"); + None + } + } +} + +/// Run one scalar query through the `sqlite3` oracle, read-only and immutable. +fn oracle(db: &Path, sql: &str) -> Option { + let uri = format!("file:{}?mode=ro&immutable=1", db.display()); + let out = Command::new("sqlite3").arg(uri).arg(sql).output().ok()?; + if !out.status.success() { + eprintln!( + "SKIP: sqlite3 oracle failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + return None; + } + Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +fn oracle_count(db: &Path, table: &str) -> Option { + oracle(db, &format!("SELECT COUNT(*) FROM {table};"))? + .parse() + .ok() +} + +/// Extract, or skip. Returns the result and the evidence path. +fn extract() -> Option<(ExtractionResult, PathBuf)> { + let path = msgstore_path()?; + let bytes = std::fs::read(&path).expect("reading the evidence file"); + let result = extract_from_msgstore(&bytes, 0, SchemaVersion::Modern) + .expect("extraction must succeed on a real msgstore.db"); + Some((result, path)) +} + +fn message_count(r: &ExtractionResult) -> usize { + r.chats.iter().map(|c| c.messages.len()).sum() +} + +// ── Reconciliation against the sqlite3 oracle ──────────────────────────────── + +#[test] +fn message_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "message") else { + return; + }; + let actual = message_count(&result) as u64; + + if actual != expected { + // Turn a bare mismatch into a diagnosis: extras are either duplicate + // row_ids or records attributed to a layer other than the live btree. + let mut by_source: BTreeMap = BTreeMap::new(); + let mut ids: Vec = Vec::new(); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + *by_source.entry(format!("{:?}", m.source)).or_insert(0) += 1; + ids.push(m.id); + } + let unique = { + let mut v = ids.clone(); + v.sort_unstable(); + v.dedup(); + v.len() + }; + panic!( + "extracted {actual} messages, sqlite3 reports {expected} in `message` \ + (delta {}). Distinct row_ids: {unique} of {}. By source: {by_source:?}", + actual as i64 - expected as i64, + ids.len() + ); + } +} + +#[test] +fn chat_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "chat") else { + return; + }; + // Chats carrying messages whose own row was not recovered appear as stubs, + // so our count may exceed the table's; it must never fall short. + let stubs = result.chats.iter().filter(|c| c.jid.is_empty()).count(); + assert!( + result.chats.len() as u64 >= expected, + "extracted {} chats, sqlite3 reports {expected} in `chat` ({stubs} stubs)", + result.chats.len() + ); + assert_eq!( + result.chats.len() as u64 - stubs as u64, + expected, + "non-stub chats must reconcile exactly with the `chat` table" + ); +} + +#[test] +fn call_count_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(expected) = oracle_count(&path, "call_log") else { + return; + }; + // Group calls merge several rows into one record; without a call_row_id + // column nothing merges, so the counts are equal. + let participants: usize = result.calls.iter().map(|c| c.participants.len()).sum(); + assert_eq!( + participants as u64, + expected, + "one participant per call_log row: got {participants} across {} records", + result.calls.len() + ); +} + +#[test] +fn year_histogram_matches_the_sqlite3_oracle() { + let Some((result, path)) = extract() else { + return; + }; + let Some(raw) = oracle( + &path, + "SELECT strftime('%Y', timestamp/1000, 'unixepoch'), COUNT(*) \ + FROM message GROUP BY 1 ORDER BY 1;", + ) else { + return; + }; + + let mut expected: BTreeMap = BTreeMap::new(); + for line in raw.lines() { + let (year, count) = line.split_once('|').unwrap_or(("", "")); + if let Ok(n) = count.parse::() { + // A row whose year does not render (NULL or out of range) keys on + // the empty string; it must be carried, not dropped. + expected.insert(year.to_string(), n); + } + } + if expected.is_empty() { + eprintln!("SKIP: oracle returned no year histogram"); + return; + } + + let mut actual: BTreeMap = BTreeMap::new(); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + *actual + .entry(m.timestamp.utc.format("%Y").to_string()) + .or_insert(0) += 1; + } + + // Compare only years both sides can express: SQLite renders an + // unrepresentable timestamp as an empty year, which has no counterpart in a + // parsed DateTime. Report it rather than letting it vanish. + let unrenderable = expected.get("").copied().unwrap_or(0); + if unrenderable > 0 { + eprintln!( + "NOTE: sqlite3 renders no year for {unrenderable} row(s); \ + extractor placed them in: {:?}", + actual + .iter() + .filter(|(y, _)| !expected.contains_key(*y)) + .collect::>() + ); + } + let comparable: BTreeMap<&String, &u64> = + expected.iter().filter(|(y, _)| !y.is_empty()).collect(); + for (year, count) in comparable { + assert_eq!( + actual.get(year).copied().unwrap_or(0), + *count, + "year {year}: extractor and sqlite3 disagree (full histogram: {actual:?})" + ); + } +} + +// ── Self-contained assertions (no oracle needed) ───────────────────────────── + +#[test] +fn no_message_is_dated_1970() { + let Some((result, _)) = extract() else { + return; + }; + let epoch_year: Vec = result + .chats + .iter() + .flat_map(|c| c.messages.iter()) + .filter(|m| m.timestamp.utc.format("%Y").to_string() == "1970") + .map(|m| m.id) + .take(5) + .collect(); + assert!( + epoch_year.is_empty(), + "messages dated 1970 indicate a misread timestamp column; first row_ids: {epoch_year:?}" + ); +} + +#[test] +fn senders_resolve_to_jids_not_bare_numbers() { + let Some((result, _)) = extract() else { + return; + }; + let (mut resolved, mut bare) = (0usize, 0usize); + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + match m.sender_jid.as_deref() { + Some(s) if s.contains('@') => resolved += 1, + Some(_) => bare += 1, + None => {} + } + } + assert!(resolved > 0, "no sender resolved to a JID at all"); + assert_eq!( + bare, 0, + "{bare} senders carry no '@' — that is jid.user, not jid.raw_string \ + ({resolved} resolved correctly)" + ); +} + +#[test] +fn hourly_distribution_is_not_concentrated_in_one_bucket() { + let Some((result, _)) = extract() else { + return; + }; + let mut hours = [0u64; 24]; + let mut total = 0u64; + for m in result.chats.iter().flat_map(|c| c.messages.iter()) { + let h = m + .timestamp + .utc + .format("%H") + .to_string() + .parse::() + .unwrap_or(0); + if let Some(slot) = hours.get_mut(h) { + *slot += 1; + } + total += 1; + } + // A spread is a property of volume: a handful of messages can legitimately + // sit in one hour, so below this the check would report noise as a finding. + const MIN_FOR_SPREAD: u64 = 1000; + if total < MIN_FOR_SPREAD { + eprintln!("SKIP: {total} messages is too few to judge hourly spread"); + return; + } + let occupied = hours.iter().filter(|&&n| n > 0).count(); + let busiest = hours.iter().max().copied().unwrap_or(0); + assert!( + occupied >= 12, + "only {occupied} of 24 hourly buckets occupied — human messaging spreads \ + across the day; a single bucket means every timestamp collapsed to one instant" + ); + assert!( + busiest * 100 / total < 50, + "busiest hour holds {}% of {total} messages", + busiest * 100 / total + ); +} + +#[test] +fn no_implausible_timestamp_distribution_warning_is_raised() { + let Some((result, _)) = extract() else { + return; + }; + let anomalies: Vec = result + .forensic_warnings + .iter() + .filter(|w| { + matches!( + w, + chat4n6_plugin_api::ForensicWarning::TimestampDistributionAnomaly { .. } + ) + }) + .map(std::string::ToString::to_string) + .collect(); + assert!( + anomalies.is_empty(), + "the detector reports the extraction's own dates as implausible: {anomalies:?}" + ); +} + +// ── Pipeline-level run over the extraction tree ────────────────────────────── + +#[test] +fn plugin_detects_and_extracts_from_the_extraction_tree() { + let Ok(root) = std::env::var(INPUT_DIR_ENV) else { + eprintln!("SKIP: {INPUT_DIR_ENV} not set — pipeline-level run not performed"); + return; + }; + if root.is_empty() { + eprintln!("SKIP: {INPUT_DIR_ENV} is empty"); + return; + } + let fs = + chat4n6_fs::PlaintextDirFs::new(Path::new(&root)).expect("opening the extraction tree"); + let plugin = chat4n6_whatsapp::WhatsAppPlugin::new(); + assert!( + chat4n6_plugin_api::ForensicPlugin::detect(&plugin, &fs), + "the WhatsApp plugin must detect its database under {INPUT_DIR_ENV}" + ); + let result = chat4n6_plugin_api::ForensicPlugin::extract(&plugin, &fs, Some(0)) + .expect("pipeline extraction must succeed"); + assert!( + message_count(&result) > 0, + "pipeline extraction produced no messages" + ); +} diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..7efd656 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,165 @@ +# Validation — WhatsApp Android (msgstore.db) + +## Summary + +The WhatsApp extractor resolves every msgstore.db column position by name from +the database's own `CREATE TABLE` SQL, and refuses to report on a database whose +data contradicts the resolved map. This document records how that is checked, +by what, and how far each check can be trusted. + +Three things carry the correctness claim: + +1. **Real-device DDL under test.** A fixture built from the `CREATE TABLE` + shapes of a real 2023-era handset, where the columns a positional reader + would land on are the wrong ones. +2. **Position-independence as a property.** A second fixture with identical + names and data but permuted column order. Both must extract identically. +3. **Reconciliation with `sqlite3`.** An independent implementation reading the + same evidence file, comparing counts and the timestamp year histogram. + +Item 3 is Tier 1 and requires the evidence, which lives only in the operator's +case workspace. **It has not been executed in this repository.** Items 1 and 2 +run in CI on every commit. + +## Evidence tiers + +Tiering is by *who confirms the result*, not by whether the input is synthetic. + +| Check | Tier | Confirmed by | +|---|---|---| +| Counts and year histogram vs. `sqlite3` on the case database | 1 | Independent implementation, real-world artifact | +| Extraction over real-device DDL (`real_modern_schema.sql`) | 2 | Real SQLite engine writes the file; expected values follow from the documented construction | +| Column order permutation (`shuffled_schema.sql`) | 2 | Differential: two constructions must agree, neither one's answer authored by hand | +| DDL parser unit tests (`columns.rs`) | 3 | Rules authored here; correctness is defined by the SQLite grammar they implement | +| Schema-gate behaviour (`schema_gate.rs`) | 3 | Detection rules — the fixture specifies behaviour rather than vouching for a value | + +Tier 3 is legitimate where it sits: those cases define detection rules and +robustness properties, not values an external oracle could adjudicate. The +value-producing paths — timestamps, senders, subjects, call durations — are +covered at Tier 2 by construction and at Tier 1 by `sqlite3`. + +## Method + +### Tier-1 reconciliation + +Ground truth is `sqlite3` reading the evidence read-only and immutable: + +``` +sqlite3 "file:$CHAT4N6_REAL_MSGSTORE?mode=ro&immutable=1" +``` + +The test derives every expected value from that oracle at run time rather than +from transcribed constants, so it reconciles against whatever database it is +pointed at. It asserts: + +- extracted message count equals `SELECT COUNT(*) FROM message` +- non-stub chat count equals `SELECT COUNT(*) FROM chat` +- call participants equal `SELECT COUNT(*) FROM call_log` +- the per-year message histogram matches + `strftime('%Y', timestamp/1000, 'unixepoch')` bucket for bucket +- no message is dated 1970 +- every resolved sender is a JID, not a bare subscriber number +- messages spread across at least 12 hourly buckets, with no single hour + holding half of them +- the extraction raises no `TimestampDistributionAnomaly` against itself + +A count mismatch fails with a breakdown by evidence source and a distinct-rowid +tally, so an over-count is diagnosed rather than merely reported. + +### Running it + +Both paths come from the operator's case workspace and are never recorded in +this repository: + +``` +CHAT4N6_REAL_MSGSTORE=/path/to/msgstore.db \ +CHAT4N6_REAL_INPUT_DIR=/path/to/extraction-root \ + cargo test -p chat4n6-whatsapp --test real_msgstore_validation -- --nocapture +``` + +Without the variables every test skips and prints why; without `sqlite3` on +PATH the reconciling tests skip and the self-contained ones still run. A +variable that is set but points at no readable file fails loudly rather than +skipping, so a typo cannot masquerade as "no evidence available". + +The test file asserts on counts, shapes and years only. No message content, +JID, subject, phone number or path is printed or embedded. + +### Expected values + +Recorded from the oracle on 2026-07-29 against this case's database. Aggregate +counts only. + +| Query | Value | +|---|---| +| `SELECT COUNT(*) FROM message` | 245,227 | +| `SELECT COUNT(*) FROM chat` | 2,309 | +| `SELECT COUNT(*) FROM call_log` | 166 | +| `PRAGMA freelist_count` | 0 | +| Messages dated 1970 | 0 | + +One `message` row renders an empty year under `strftime`, meaning its timestamp +is NULL or outside the representable range. The year-histogram test reports +that row and where the extractor placed it rather than dropping it silently. + +## What is checked in CI + +`cargo test -p chat4n6-whatsapp` covers, without any evidence present: + +- DDL parsing: nested parentheses, commas inside string defaults, quoted and + bracketed identifiers, table-level constraints occupying no column position, + malformed input yielding an empty map rather than a panic +- resolution of the confirmed real-device positions for `message`, `jid`, + `chat` and `call_log` +- extraction over real-device DDL: 2022 timestamps, senders via + `jid.raw_string`, subject via `chat.subject`, `archived`, per-call duration + and video flag, and three call rows staying three records +- identical extraction from the column-permuted fixture +- the record-layout invariant the resolver rests on: SQLite writes an + `INTEGER PRIMARY KEY` as a NULL at its *declared* position, so `values[i]` is + the column at DDL position `i` even when `_id` is declared mid-table +- the schema gate rejecting an unresolvable or contradicted column map, and + accepting an empty message table and a handful of outlier rows +- refusal of a legacy-generation database instead of an empty report + +## Known gaps + +These are real and unaddressed; none is masked by a passing test. + +- **Media metadata on the modern schema.** The 2023-era `message` table has no + `media_mime_type` or `media_name`; that metadata moved to `message_media`, + which the extractor does not read. Media fields resolve to `None` on a modern + database and media messages carry no path or MIME type. The + media-in-message path is still exercised by `modern_schema.sql`, which + represents an older modern-generation schema. +- **Auxiliary table column names are unverified.** `message_quoted`, + `message_add_on`, `message_edit_info`, `receipt_user`, `message_forwarded` + and `group_participant_user` resolve by the names in `modern_schema.sql`, + which were not read off a device. Where a real schema spells a column + differently the field resolves to `None`, so reactions, edits and receipts + may be absent rather than wrong. Auditing those names against the case + database is outstanding. +- **Only the live btree layer is interpreted.** WAL deltas, freelist, journal, + intra-page carve and unallocated carve are not mapped into messages by this + plugin. Unallocated carving additionally needs a raw-image input: the + plaintext-directory filesystem reports no unallocated regions by design. +- **`detect_timestamp_anomalies` cannot fire from this plugin.** Messages are + sorted by timestamp before the detectors run, so the pairwise out-of-order + check has nothing to find. `detect_selective_deletion` is likewise not wired + into extraction. The distribution-level detector is wired and does fire. +- **Other plugins are unaudited.** `chat4n6-ios-whatsapp`, `chat4n6-signal` and + `chat4n6-telegram` still read records by hardcoded ordinals and have not been + checked against real DDL. + +## Fixtures + +| File | What it is | Provenance | +|---|---|---| +| `real_modern_schema.sql` | Real 2023-era device DDL, synthetic rows | Confirmed positions read off the case device; intervening columns from published WhatsApp schema documentation, position-preserving and never read | +| `shuffled_schema.sql` | The same names, data and values with every column order permuted | Synthetic permutation; no device ships these orders | +| `legacy_schema.sql` | Pre-2018 `messages` + `chat_list` generation | Published legacy schema documentation, not read off the case device | +| `modern_schema.sql` | Simplified shapes, retained for the media-in-message path and for the aux tables | Authored here; explicitly not a device schema | + +`real_modern_schema.sql` and `shuffled_schema.sql` exist because a fixture +written to the layout the reader assumes cannot falsify that assumption. Those +two can. From 8118d1f060b130989ec934797fa9882bdc8fec88 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 12:42:31 +0800 Subject: [PATCH 10/11] style: drop incidental reformatting of files this change does not touch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo fmt -p chat4n6-whatsapp` during development reflowed twelve modules this work never edits — album, cdn, contact_report, group_metadata, link, location, orphaned_media, platform, poll, status, system_event and the plugin-api lib root — adding roughly 700 lines of churn that buried the actual change. Restored to their state on main; the ForensicWarning variant in types.rs is re-applied in the file's existing style. The workspace is not rustfmt-clean on main (cli/, chat4n6-fs/ and others drift too). Clearing that belongs in its own sweep, not in a schema-resolution fix. Verified: 1183 workspace tests pass, and the clippy finding set for the chat4n6-whatsapp package tree goes from 32 on main to 15 here, with none introduced. Co-Authored-By: Claude Opus 5 (1M context) --- crates/chat4n6-plugin-api/src/lib.rs | 23 +- crates/chat4n6-plugin-api/src/types.rs | 233 +++++------------- crates/plugins/chat4n6-whatsapp/src/album.rs | 5 +- crates/plugins/chat4n6-whatsapp/src/cdn.rs | 119 +++------ .../chat4n6-whatsapp/src/contact_report.rs | 40 +-- .../chat4n6-whatsapp/src/group_metadata.rs | 77 ++---- crates/plugins/chat4n6-whatsapp/src/link.rs | 11 +- .../plugins/chat4n6-whatsapp/src/location.rs | 10 +- .../chat4n6-whatsapp/src/orphaned_media.rs | 72 +++--- .../plugins/chat4n6-whatsapp/src/platform.rs | 29 +-- crates/plugins/chat4n6-whatsapp/src/poll.rs | 12 +- crates/plugins/chat4n6-whatsapp/src/status.rs | 40 +-- .../chat4n6-whatsapp/src/system_event.rs | 225 +++++------------ 13 files changed, 243 insertions(+), 653 deletions(-) diff --git a/crates/chat4n6-plugin-api/src/lib.rs b/crates/chat4n6-plugin-api/src/lib.rs index 1a41eb4..68e2026 100644 --- a/crates/chat4n6-plugin-api/src/lib.rs +++ b/crates/chat4n6-plugin-api/src/lib.rs @@ -31,10 +31,7 @@ mod tests { assert_eq!(EvidenceSource::WalDeleted.to_string(), "WAL-DELETED"); assert_eq!(EvidenceSource::Journal.to_string(), "JOURNAL"); assert_eq!(EvidenceSource::IndexRecovery.to_string(), "INDEX-RECOVERY"); - assert_eq!( - EvidenceSource::CarvedOverflow.to_string(), - "CARVED-OVERFLOW" - ); + assert_eq!(EvidenceSource::CarvedOverflow.to_string(), "CARVED-OVERFLOW"); assert_eq!( EvidenceSource::CarvedIntraPage { confidence_pct: 75 }.to_string(), "CARVED-INTRA-PAGE 75%" @@ -161,10 +158,7 @@ mod tests { assert_eq!(m, back); assert_eq!(back.file_hash.as_deref(), Some("abc123def456")); assert_eq!(back.encrypted_hash.as_deref(), Some("enc789xyz000")); - assert_eq!( - back.cdn_url.as_deref(), - Some("https://mmg.whatsapp.net/v/abc") - ); + assert_eq!(back.cdn_url.as_deref(), Some("https://mmg.whatsapp.net/v/abc")); assert_eq!(back.media_key_b64.as_deref(), Some("dGVzdGtleQ==")); } @@ -179,18 +173,11 @@ mod tests { "thumbnail_b64": null, "duration_secs": 30 }"#; - let m: MediaRef = - serde_json::from_str(old_json).expect("must deserialize old JSON without error"); + let m: MediaRef = serde_json::from_str(old_json).expect("must deserialize old JSON without error"); assert!(m.file_hash.is_none(), "file_hash should default to None"); - assert!( - m.encrypted_hash.is_none(), - "encrypted_hash should default to None" - ); + assert!(m.encrypted_hash.is_none(), "encrypted_hash should default to None"); assert!(m.cdn_url.is_none(), "cdn_url should default to None"); - assert!( - m.media_key_b64.is_none(), - "media_key_b64 should default to None" - ); + assert!(m.media_key_b64.is_none(), "media_key_b64 should default to None"); assert_eq!(m.mime_type, "video/mp4"); assert_eq!(m.file_size, 4096); assert_eq!(m.duration_secs, Some(30)); diff --git a/crates/chat4n6-plugin-api/src/types.rs b/crates/chat4n6-plugin-api/src/types.rs index 26d58eb..0ce0e57 100644 --- a/crates/chat4n6-plugin-api/src/types.rs +++ b/crates/chat4n6-plugin-api/src/types.rs @@ -320,18 +320,12 @@ pub enum ForensicWarning { /// SQLite VACUUM was run — freelist erased, potentially destroying deleted record remnants. DatabaseVacuumed { freelist_page_count: u32 }, /// ROWID gaps concentrate on one JID: possible targeted scrubbing. - SelectiveDeletion { - suspect_jid: String, - deletion_rate_pct: u8, - }, + SelectiveDeletion { suspect_jid: String, deletion_rate_pct: u8 }, /// Timestamp order violated: a later ROWID has an earlier timestamp. - TimestampAnomaly { - message_row_id: i64, - description: String, - }, - /// The message timestamp distribution is implausible in bulk — a large - /// share of messages predate the app's existence. A property of the whole - /// set, which no per-message check can express. + TimestampAnomaly { message_row_id: i64, description: String }, + /// The message timestamp distribution is implausible in bulk — a large share + /// of messages predate the app's existence. A property of the whole set, + /// which no per-message check can express. TimestampDistributionAnomaly { total_messages: u32, implausible_count: u32, @@ -343,15 +337,9 @@ pub enum ForensicWarning { /// Backup crypt14/15 HMAC does not match payload — file may have been tampered. HmacMismatch, /// PRAGMA user_version inconsistent with claimed app version. - SchemaVersionMismatch { - db_version: u32, - app_version: String, - }, + SchemaVersionMismatch { db_version: u32, app_version: String }, /// SQLite change counter implies writes after acquisition date. - HeaderTampered { - change_counter: u32, - expected_max: u32, - }, + HeaderTampered { change_counter: u32, expected_max: u32 }, /// iOS CoreData primary-key gap indicates deleted records. CoreDataPkGap { entity_name: String, @@ -390,131 +378,58 @@ pub enum ForensicWarning { /// Signal sealed-sender message whose sender identity could not be resolved. SealedSenderUnresolved { thread_id: i64, count: u32 }, /// A forwarded message references a source message ID that is not present in any snapshot. - UnresolvedForwardSource { - message_id: i64, - forward_from_id: i64, - }, + UnresolvedForwardSource { message_id: i64, forward_from_id: i64 }, } impl fmt::Display for ForensicWarning { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::DatabaseVacuumed { - freelist_page_count, - } => { - write!( - f, - "VACUUM detected (freelist pages remaining: {freelist_page_count})" - ) + Self::DatabaseVacuumed { freelist_page_count } => { + write!(f, "VACUUM detected (freelist pages remaining: {freelist_page_count})") } - Self::SelectiveDeletion { - suspect_jid, - deletion_rate_pct, - } => { - write!( - f, - "Selective deletion: {suspect_jid} ({deletion_rate_pct}% gap rate)" - ) + Self::SelectiveDeletion { suspect_jid, deletion_rate_pct } => { + write!(f, "Selective deletion: {suspect_jid} ({deletion_rate_pct}% gap rate)") } - Self::TimestampAnomaly { - message_row_id, - description, - } => { - write!( - f, - "Timestamp anomaly at row {message_row_id}: {description}" - ) + Self::TimestampAnomaly { message_row_id, description } => { + write!(f, "Timestamp anomaly at row {message_row_id}: {description}") } Self::TimestampDistributionAnomaly { - total_messages, - implausible_count, - ratio_pct, - modal_utc, - modal_occurrences, - } => write!( - f, - "Implausible timestamp distribution: {implausible_count} of {total_messages} \ - messages ({ratio_pct}%) predate 2009-01-01; most frequent is {modal_utc} \ - ({modal_occurrences}×)" - ), - Self::HmacMismatch => write!(f, "HMAC mismatch — backup integrity check FAILED"), - Self::SchemaVersionMismatch { - db_version, - app_version, + total_messages, implausible_count, ratio_pct, modal_utc, modal_occurrences } => { - write!( - f, - "Schema v{db_version} incompatible with app version {app_version}" - ) + write!(f, "Implausible timestamp distribution: {implausible_count} of {total_messages} messages ({ratio_pct}%) predate 2009-01-01; most frequent is {modal_utc} ({modal_occurrences}×)") } - Self::HeaderTampered { - change_counter, - expected_max, - } => { - write!( - f, - "Header tamper: change_counter={change_counter} > expected_max={expected_max}" - ) + Self::HmacMismatch => write!(f, "HMAC mismatch — backup integrity check FAILED"), + Self::SchemaVersionMismatch { db_version, app_version } => { + write!(f, "Schema v{db_version} incompatible with app version {app_version}") } - Self::CoreDataPkGap { - entity_name, - recovered_count, - .. - } => { - write!( - f, - "CoreData PK gap in {entity_name}: {recovered_count} potential deleted rows" - ) + Self::HeaderTampered { change_counter, expected_max } => { + write!(f, "Header tamper: change_counter={change_counter} > expected_max={expected_max}") } - Self::ImpossibleTimestamp { - message_row_id, - reason, - .. - } => { - write!( - f, - "Impossible timestamp at row {message_row_id}: {reason:?}" - ) + Self::CoreDataPkGap { entity_name, recovered_count, .. } => { + write!(f, "CoreData PK gap in {entity_name}: {recovered_count} potential deleted rows") } - Self::DuplicateStanzaId { - stanza_id, - occurrences, - } => { + Self::ImpossibleTimestamp { message_row_id, reason, .. } => { + write!(f, "Impossible timestamp at row {message_row_id}: {reason:?}") + } + Self::DuplicateStanzaId { stanza_id, occurrences } => { write!(f, "Duplicate stanza ID '{stanza_id}' seen {occurrences}×") } Self::RowIdReuseDetected { table, rowid, .. } => { write!(f, "ROWID {rowid} reused in table '{table}'") } - Self::ThumbnailOrphanHigh { - orphan_thumbnails, - ratio_pct, - .. - } => { - write!( - f, - "High thumbnail orphan rate: {orphan_thumbnails} orphans ({ratio_pct}%)" - ) + Self::ThumbnailOrphanHigh { orphan_thumbnails, ratio_pct, .. } => { + write!(f, "High thumbnail orphan rate: {orphan_thumbnails} orphans ({ratio_pct}%)") } Self::PerFileHmacMismatch { file_name } => { write!(f, "Per-file HMAC mismatch: {file_name}") } - Self::DisappearingTimerActive { - chat_id, - timer_seconds, - vanished_count, - } => { + Self::DisappearingTimerActive { chat_id, timer_seconds, vanished_count } => { write!(f, "Disappearing timer on chat {chat_id} ({timer_seconds}s): {vanished_count} messages vanished") } Self::SealedSenderUnresolved { thread_id, count } => { - write!( - f, - "Sealed-sender unresolved: {count} messages in thread {thread_id}" - ) + write!(f, "Sealed-sender unresolved: {count} messages in thread {thread_id}") } - Self::UnresolvedForwardSource { - message_id, - forward_from_id, - } => { + Self::UnresolvedForwardSource { message_id, forward_from_id } => { write!(f, "Forward source missing: message {message_id} references absent ID {forward_from_id}") } } @@ -696,14 +611,9 @@ mod new_types_tests { // ── ForensicWarning ─────────────────────────────────────────────────── #[test] fn forensic_warning_vacuum_display() { - let w = ForensicWarning::DatabaseVacuumed { - freelist_page_count: 0, - }; + let w = ForensicWarning::DatabaseVacuumed { freelist_page_count: 0 }; let s = format!("{}", w); - assert!( - s.contains("VACUUM") || s.contains("vacuum") || s.contains("Vacuum"), - "got: {s}" - ); + assert!(s.contains("VACUUM") || s.contains("vacuum") || s.contains("Vacuum"), "got: {s}"); } #[test] @@ -713,20 +623,14 @@ mod new_types_tests { deletion_rate_pct: 87, }; let s = format!("{}", w); - assert!( - s.contains("87") || s.contains("selective") || s.contains("Selective"), - "got: {s}" - ); + assert!(s.contains("87") || s.contains("selective") || s.contains("Selective"), "got: {s}"); } #[test] fn forensic_warning_hmac_mismatch() { let w = ForensicWarning::HmacMismatch; let json = serde_json::to_string(&w).unwrap(); - assert!( - json.contains("Hmac") || json.contains("hmac") || json.contains("HMAC"), - "got: {json}" - ); + assert!(json.contains("Hmac") || json.contains("hmac") || json.contains("HMAC"), "got: {json}"); } #[test] @@ -847,14 +751,10 @@ mod new_types_tests { #[test] fn extraction_result_has_acquisition_timestamps() { let r = ExtractionResult::default(); - assert!( - r.extraction_started_at.is_none(), - "extraction_started_at must default to None" - ); - assert!( - r.extraction_finished_at.is_none(), - "extraction_finished_at must default to None" - ); + assert!(r.extraction_started_at.is_none(), + "extraction_started_at must default to None"); + assert!(r.extraction_finished_at.is_none(), + "extraction_finished_at must default to None"); // Roundtrip with values set let json = r#"{"chats":[],"contacts":[],"calls":[],"wal_deltas":[], "extraction_started_at":"2026-05-06T10:00:00Z", @@ -867,10 +767,7 @@ mod new_types_tests { #[test] fn extraction_result_has_wal_snapshots() { let r = ExtractionResult::default(); - assert!( - r.wal_snapshots.is_empty(), - "wal_snapshots must default to empty" - ); + assert!(r.wal_snapshots.is_empty(), "wal_snapshots must default to empty"); // Build a snapshot and roundtrip let snap = WalSnapshot { frame_number: 3, @@ -897,28 +794,18 @@ mod new_types_tests { original_timestamp: Some(ts), }; let m = Message { - id: 1, - chat_id: 1, - sender_jid: None, - from_me: false, + id: 1, chat_id: 1, + sender_jid: None, from_me: false, timestamp: ForensicTimestamp::from_millis(1_710_513_200_000, 0), content: MessageContent::Text("forwarded".to_string()), - reactions: vec![], - quoted_message: None, - source: EvidenceSource::Live, - row_offset: 0, - starred: false, - forward_score: Some(3), - is_forwarded: true, - edit_history: vec![], - receipts: vec![], + reactions: vec![], quoted_message: None, + source: EvidenceSource::Live, row_offset: 0, + starred: false, forward_score: Some(3), is_forwarded: true, + edit_history: vec![], receipts: vec![], forwarded_from: Some(origin), }; let json = serde_json::to_string(&m).unwrap(); - assert!( - json.contains("Channel"), - "ForwardOriginKind::Channel must serialise" - ); + assert!(json.contains("Channel"), "ForwardOriginKind::Channel must serialise"); let back: Message = serde_json::from_str(&json).unwrap(); let fo = back.forwarded_from.unwrap(); assert_eq!(fo.origin_id, "tg-channel://123456"); @@ -942,9 +829,7 @@ mod new_types_tests { let warnings: Vec = vec![ ForensicWarning::CoreDataPkGap { entity_name: "ZWAMESSAGE".to_string(), - expected_max: 500, - observed_max: 450, - recovered_count: 10, + expected_max: 500, observed_max: 450, recovered_count: 10, }, ForensicWarning::ImpossibleTimestamp { message_row_id: 42, @@ -952,34 +837,26 @@ mod new_types_tests { reason: ImpossibleReason::BeforeUnixEpoch, }, ForensicWarning::DuplicateStanzaId { - stanza_id: "ABC123".to_string(), - occurrences: 2, + stanza_id: "ABC123".to_string(), occurrences: 2, }, ForensicWarning::RowIdReuseDetected { - table: "messages".to_string(), - rowid: 99, + table: "messages".to_string(), rowid: 99, conflicting_timestamps: vec![Utc::now(), Utc::now()], }, ForensicWarning::ThumbnailOrphanHigh { - orphan_thumbnails: 5, - total_messages: 10, - ratio_pct: 50, + orphan_thumbnails: 5, total_messages: 10, ratio_pct: 50, }, ForensicWarning::PerFileHmacMismatch { file_name: "msgstore.db".to_string(), }, ForensicWarning::DisappearingTimerActive { - chat_id: 1, - timer_seconds: 86400, - vanished_count: 3, + chat_id: 1, timer_seconds: 86400, vanished_count: 3, }, ForensicWarning::SealedSenderUnresolved { - thread_id: 7, - count: 2, + thread_id: 7, count: 2, }, ForensicWarning::UnresolvedForwardSource { - message_id: 55, - forward_from_id: 99999, + message_id: 55, forward_from_id: 99999, }, ]; for w in &warnings { diff --git a/crates/plugins/chat4n6-whatsapp/src/album.rs b/crates/plugins/chat4n6-whatsapp/src/album.rs index 7b2e2f0..e7332cf 100644 --- a/crates/plugins/chat4n6-whatsapp/src/album.rs +++ b/crates/plugins/chat4n6-whatsapp/src/album.rs @@ -75,10 +75,7 @@ mod tests { note.contains("WhatsApp expected"), "note format: got {note}" ); - assert!( - note.contains("possible evidence gap"), - "note format: got {note}" - ); + assert!(note.contains("possible evidence gap"), "note format: got {note}"); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/cdn.rs b/crates/plugins/chat4n6-whatsapp/src/cdn.rs index fa3515d..12db901 100644 --- a/crates/plugins/chat4n6-whatsapp/src/cdn.rs +++ b/crates/plugins/chat4n6-whatsapp/src/cdn.rs @@ -18,12 +18,12 @@ pub enum CdnError { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CdnAcquisitionRecord { - pub url_hash: String, // SHA-256 hex of the URL (NOT the URL itself) - pub media_key_hash: String, // SHA-256 hex of the raw media key bytes - pub timestamp_utc: String, // ISO 8601 UTC timestamp of download attempt + pub url_hash: String, // SHA-256 hex of the URL (NOT the URL itself) + pub media_key_hash: String, // SHA-256 hex of the raw media key bytes + pub timestamp_utc: String, // ISO 8601 UTC timestamp of download attempt pub file_hash_result: Option, // SHA-256 hex of plaintext bytes (None if download failed) pub file_size_bytes: Option, - pub examiner: Option, // examiner identifier for chain of custody + pub examiner: Option, // examiner identifier for chain of custody pub success: bool, } @@ -43,7 +43,7 @@ pub fn decrypt_whatsapp_media( media_key_bytes: &[u8], encrypted_bytes: &[u8], ) -> Result, CdnError> { - use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, KeyIvInit}; + use aes::cipher::{BlockDecryptMut, KeyIvInit, block_padding::Pkcs7}; type Aes256CbcDec = cbc::Decryptor; // Minimum: need at least 1 AES block (16 bytes) of ciphertext — check blob first @@ -64,7 +64,7 @@ pub fn decrypt_whatsapp_media( let mut buf = encrypted_bytes.to_vec(); let plaintext_len = Aes256CbcDec::new(aes_key.into(), iv.into()) .decrypt_padded_mut::(&mut buf) - .map_err(|_| CdnError::HmacMismatch)? // unpad failure treated as decrypt error + .map_err(|_| CdnError::HmacMismatch)? // unpad failure treated as decrypt error .len(); buf.truncate(plaintext_len); Ok(buf) @@ -136,10 +136,7 @@ mod tests { assert!(!record.url_hash.is_empty(), "url_hash must not be empty"); // Verify it's the expected SHA-256 hex let expected = hex::encode(Sha256::digest(url.as_bytes())); - assert_eq!( - record.url_hash, expected, - "url_hash should be SHA-256 of the URL" - ); + assert_eq!(record.url_hash, expected, "url_hash should be SHA-256 of the URL"); } #[test] @@ -148,30 +145,18 @@ mod tests { let key = b"0123456789abcdef0123456789abcdef"; // 32 bytes let record = build_acquisition_record(url, key, None, None); let expected = hex::encode(Sha256::digest(key)); - assert_eq!( - record.media_key_hash, expected, - "media_key_hash should be SHA-256 of key bytes" - ); + assert_eq!(record.media_key_hash, expected, "media_key_hash should be SHA-256 of key bytes"); // Also verify it doesn't leak the actual key let key_hex = hex::encode(key); - assert_ne!( - record.media_key_hash, key_hex, - "media_key_hash should be SHA-256, not hex of key" - ); + assert_ne!(record.media_key_hash, key_hex, "media_key_hash should be SHA-256, not hex of key"); } #[test] fn test_build_acquisition_record_success_false_when_no_plaintext() { let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); - assert!( - !record.success, - "success should be false when plaintext is None" - ); - assert!( - record.file_hash_result.is_none(), - "file_hash_result should be None when no plaintext" - ); + assert!(!record.success, "success should be false when plaintext is None"); + assert!(record.file_hash_result.is_none(), "file_hash_result should be None when no plaintext"); assert!(record.file_size_bytes.is_none()); } @@ -180,14 +165,8 @@ mod tests { let key = vec![0u8; 32]; let plaintext = b"decrypted media content"; let record = build_acquisition_record("https://example.com", &key, Some(plaintext), None); - assert!( - record.success, - "success should be true when plaintext is provided" - ); - assert!( - record.file_hash_result.is_some(), - "file_hash_result should be set" - ); + assert!(record.success, "success should be true when plaintext is provided"); + assert!(record.file_hash_result.is_some(), "file_hash_result should be set"); assert_eq!(record.file_size_bytes, Some(plaintext.len() as u64)); } @@ -203,8 +182,7 @@ mod tests { #[test] fn test_build_acquisition_record_examiner_preserved() { let key = vec![0u8; 32]; - let record = - build_acquisition_record("https://example.com", &key, None, Some("examiner_alice")); + let record = build_acquisition_record("https://example.com", &key, None, Some("examiner_alice")); assert_eq!(record.examiner.as_deref(), Some("examiner_alice")); } @@ -212,10 +190,7 @@ mod tests { fn test_build_acquisition_record_timestamp_not_empty() { let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); - assert!( - !record.timestamp_utc.is_empty(), - "timestamp_utc must be set" - ); + assert!(!record.timestamp_utc.is_empty(), "timestamp_utc must be set"); } // ── append_to_log tests ─────────────────────────────────────────────────── @@ -224,10 +199,7 @@ mod tests { fn test_append_to_log_creates_file() { let dir = tempdir().unwrap(); let log_path = dir.path().join("cdn_acquisition.jsonl"); - assert!( - !log_path.exists(), - "log file should not exist before append" - ); + assert!(!log_path.exists(), "log file should not exist before append"); let key = vec![0u8; 32]; let record = build_acquisition_record("https://example.com", &key, None, None); @@ -250,11 +222,7 @@ mod tests { let content = std::fs::read_to_string(&log_path).unwrap(); let lines: Vec<&str> = content.lines().filter(|l| !l.is_empty()).collect(); - assert_eq!( - lines.len(), - 2, - "should have exactly 2 lines after 2 appends" - ); + assert_eq!(lines.len(), 2, "should have exactly 2 lines after 2 appends"); } #[test] @@ -263,8 +231,7 @@ mod tests { let log_path = dir.path().join("cdn_acquisition.jsonl"); let key = vec![0u8; 32]; - let r1 = - build_acquisition_record("https://example.com/a", &key, Some(b"content"), Some("ex")); + let r1 = build_acquisition_record("https://example.com/a", &key, Some(b"content"), Some("ex")); let r2 = build_acquisition_record("https://example.com/b", &key, None, None); append_to_log(&log_path, &r1).unwrap(); @@ -272,8 +239,8 @@ mod tests { let content = std::fs::read_to_string(&log_path).unwrap(); for line in content.lines().filter(|l| !l.is_empty()) { - let parsed: serde_json::Value = - serde_json::from_str(line).expect("each line must be valid JSON"); + let parsed: serde_json::Value = serde_json::from_str(line) + .expect("each line must be valid JSON"); assert!(parsed.is_object(), "each line must be a JSON object"); } } @@ -285,10 +252,8 @@ mod tests { let short_key = vec![0u8; 10]; // too short — need 32+ let blob = vec![0u8; 64]; let result = decrypt_whatsapp_media(&short_key, &blob); - assert!( - matches!(result, Err(CdnError::KeyTooShort(_))), - "should return KeyTooShort error for short key" - ); + assert!(matches!(result, Err(CdnError::KeyTooShort(_))), + "should return KeyTooShort error for short key"); } #[test] @@ -296,24 +261,22 @@ mod tests { let key = vec![0u8; 32]; let short_blob = vec![0u8; 5]; // too short — need at least 17 bytes (IV + 1 block) let result = decrypt_whatsapp_media(&key, &short_blob); - assert!( - matches!(result, Err(CdnError::BlobTooShort(_, _))), - "should return BlobTooShort error for short blob" - ); + assert!(matches!(result, Err(CdnError::BlobTooShort(_, _))), + "should return BlobTooShort error for short blob"); } #[test] fn test_decrypt_aes_cbc_basic() { // Construct a known AES-256-CBC encrypted payload using the aes+cbc crates // to test round-trip decryption. - use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; + use aes::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; type Aes256CbcEnc = cbc::Encryptor; // media_key: first 16 bytes = IV, bytes 16..48 = AES key (in simplified MVP mode) let iv_bytes = [0x01u8; 16]; let aes_key_bytes = [0x02u8; 32]; let mut media_key = Vec::new(); - media_key.extend_from_slice(&iv_bytes); // bytes 0..16 = IV + media_key.extend_from_slice(&iv_bytes); // bytes 0..16 = IV media_key.extend_from_slice(&aes_key_bytes); // bytes 16..48 = AES key let plaintext = b"Hello, WhatsApp media!"; @@ -324,16 +287,8 @@ mod tests { .unwrap(); let result = decrypt_whatsapp_media(&media_key, ciphertext); - assert!( - result.is_ok(), - "decrypt should succeed for valid AES-256-CBC payload: {:?}", - result - ); - assert_eq!( - result.unwrap(), - plaintext, - "decrypted bytes should match original plaintext" - ); + assert!(result.is_ok(), "decrypt should succeed for valid AES-256-CBC payload: {:?}", result); + assert_eq!(result.unwrap(), plaintext, "decrypted bytes should match original plaintext"); } // ── audit: no URL or key in serialized record ───────────────────────────── @@ -345,27 +300,17 @@ mod tests { let record = build_acquisition_record(url, key, None, None); let json = serde_json::to_string(&record).unwrap(); - assert!( - !json.contains(url), - "serialized JSON must NOT contain the original URL" - ); + assert!(!json.contains(url), "serialized JSON must NOT contain the original URL"); // Also verify the raw key bytes don't appear (hex encoded) let key_hex = hex::encode(key); - assert!( - !json.contains(&key_hex), - "serialized JSON must NOT contain the raw key hex" - ); + assert!(!json.contains(&key_hex), "serialized JSON must NOT contain the raw key hex"); } #[test] fn test_acquisition_record_examiner_preserved_in_json() { let key = vec![0u8; 32]; - let record = - build_acquisition_record("https://example.com", &key, None, Some("forensic_lab_01")); + let record = build_acquisition_record("https://example.com", &key, None, Some("forensic_lab_01")); let json = serde_json::to_string(&record).unwrap(); - assert!( - json.contains("forensic_lab_01"), - "examiner should appear in JSON" - ); + assert!(json.contains("forensic_lab_01"), "examiner should appear in JSON"); } } diff --git a/crates/plugins/chat4n6-whatsapp/src/contact_report.rs b/crates/plugins/chat4n6-whatsapp/src/contact_report.rs index 9265904..d2aefc4 100644 --- a/crates/plugins/chat4n6-whatsapp/src/contact_report.rs +++ b/crates/plugins/chat4n6-whatsapp/src/contact_report.rs @@ -1,10 +1,11 @@ -use chrono::{DateTime, Datelike, FixedOffset, Timelike}; -use serde::{Deserialize, Serialize}; /// Per-contact HTML forensic dossier. /// /// Builds activity statistics for a single contact and renders /// a self-contained, no-external-deps HTML report. + use std::collections::HashMap; +use chrono::{DateTime, Datelike, FixedOffset, Timelike}; +use serde::{Deserialize, Serialize}; /// Activity heatmap: hour (0-23) → message count pub type HourlyHeatmap = [u32; 24]; @@ -70,13 +71,7 @@ pub(crate) fn html_escape(s: &str) -> String { fn extract_domains(text: &str) -> Vec { let mut domains = Vec::new(); for part in text.split("://").skip(1) { - let host = part - .split('/') - .next() - .unwrap_or("") - .split('?') - .next() - .unwrap_or(""); + let host = part.split('/').next().unwrap_or("").split('?').next().unwrap_or(""); let host = host.split('#').next().unwrap_or("").trim(); if !host.is_empty() { domains.push(host.to_lowercase()); @@ -148,9 +143,7 @@ pub fn build_contact_stats( if reaction.reactor_jid == contact_jid { *reactions_given.entry(reaction.emoji.clone()).or_insert(0) += 1; } else { - *reactions_received - .entry(reaction.emoji.clone()) - .or_insert(0) += 1; + *reactions_received.entry(reaction.emoji.clone()).or_insert(0) += 1; } } } @@ -307,7 +300,12 @@ mod tests { } } - fn make_media_msg(id: i64, sender_jid: Option<&str>, from_me: bool, ts_ms: i64) -> Message { + fn make_media_msg( + id: i64, + sender_jid: Option<&str>, + from_me: bool, + ts_ms: i64, + ) -> Message { Message { id, chat_id: 1, @@ -407,20 +405,8 @@ mod tests { #[test] fn test_stats_top_link_domains_sorted() { - let m1 = make_text_msg( - 1, - Some(CONTACT_JID), - false, - 1000, - "check https://example.com/foo", - ); - let m2 = make_text_msg( - 2, - Some(CONTACT_JID), - false, - 2000, - "also https://example.com/bar and https://other.com/baz", - ); + let m1 = make_text_msg(1, Some(CONTACT_JID), false, 1000, "check https://example.com/foo"); + let m2 = make_text_msg(2, Some(CONTACT_JID), false, 2000, "also https://example.com/bar and https://other.com/baz"); let msgs: Vec<&Message> = vec![&m1, &m2]; let stats = build_contact_stats(CONTACT_JID, None, &msgs, 0); assert!(!stats.top_link_domains.is_empty()); diff --git a/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs b/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs index ddad0a0..5ac85f4 100644 --- a/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs +++ b/crates/plugins/chat4n6-whatsapp/src/group_metadata.rs @@ -2,35 +2,15 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum GroupChangeKind { - SubjectChanged { - old: Option, - new: String, - }, - IconChanged { - old_jpeg_b64: Option, - new_jpeg_b64: Option, - }, - DescriptionChanged { - old: Option, - new: String, - }, - AdminOnlyEditChanged { - admins_only: bool, - }, - AdminOnlySendChanged { - admins_only: bool, - }, - DisappearingTimerChanged { - old_secs: Option, - new_secs: Option, - }, + SubjectChanged { old: Option, new: String }, + IconChanged { old_jpeg_b64: Option, new_jpeg_b64: Option }, + DescriptionChanged { old: Option, new: String }, + AdminOnlyEditChanged { admins_only: bool }, + AdminOnlySendChanged { admins_only: bool }, + DisappearingTimerChanged { old_secs: Option, new_secs: Option }, InviteLinkReset, - ApprovalModeChanged { - requires_approval: bool, - }, - MembershipApprovalChanged { - requires_approval: bool, - }, + ApprovalModeChanged { requires_approval: bool }, + MembershipApprovalChanged { requires_approval: bool }, Unknown(i32), } @@ -78,12 +58,8 @@ pub fn parse_group_change( new_secs: new_value.and_then(|s| s.parse::().ok()), }, 83 => GroupChangeKind::InviteLinkReset, - 84 => GroupChangeKind::ApprovalModeChanged { - requires_approval: true, - }, - 85 => GroupChangeKind::ApprovalModeChanged { - requires_approval: false, - }, + 84 => GroupChangeKind::ApprovalModeChanged { requires_approval: true }, + 85 => GroupChangeKind::ApprovalModeChanged { requires_approval: false }, other => GroupChangeKind::Unknown(other), }; @@ -128,11 +104,7 @@ mod tests { fn test_icon_changed_jpeg_b64_preserved() { let fake_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ"; let c = parse(6, Some(fake_b64), Some(fake_b64)); - if let GroupChangeKind::IconChanged { - old_jpeg_b64, - new_jpeg_b64, - } = c - { + if let GroupChangeKind::IconChanged { old_jpeg_b64, new_jpeg_b64 } = c { assert_eq!(old_jpeg_b64, Some(fake_b64.to_string())); assert_eq!(new_jpeg_b64, Some(fake_b64.to_string())); } else { @@ -149,37 +121,25 @@ mod tests { #[test] fn test_admin_only_edit_on() { let c = parse(29, None, None); - assert_eq!( - c, - GroupChangeKind::AdminOnlyEditChanged { admins_only: true } - ); + assert_eq!(c, GroupChangeKind::AdminOnlyEditChanged { admins_only: true }); } #[test] fn test_admin_only_edit_off() { let c = parse(30, None, None); - assert_eq!( - c, - GroupChangeKind::AdminOnlyEditChanged { admins_only: false } - ); + assert_eq!(c, GroupChangeKind::AdminOnlyEditChanged { admins_only: false }); } #[test] fn test_admin_only_send_on() { let c = parse(31, None, None); - assert_eq!( - c, - GroupChangeKind::AdminOnlySendChanged { admins_only: true } - ); + assert_eq!(c, GroupChangeKind::AdminOnlySendChanged { admins_only: true }); } #[test] fn test_admin_only_send_off() { let c = parse(32, None, None); - assert_eq!( - c, - GroupChangeKind::AdminOnlySendChanged { admins_only: false } - ); + assert_eq!(c, GroupChangeKind::AdminOnlySendChanged { admins_only: false }); } #[test] @@ -202,12 +162,7 @@ mod tests { #[test] fn test_approval_on() { let c = parse(84, None, None); - assert_eq!( - c, - GroupChangeKind::ApprovalModeChanged { - requires_approval: true - } - ); + assert_eq!(c, GroupChangeKind::ApprovalModeChanged { requires_approval: true }); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/link.rs b/crates/plugins/chat4n6-whatsapp/src/link.rs index 8a811e8..41bb6f7 100644 --- a/crates/plugins/chat4n6-whatsapp/src/link.rs +++ b/crates/plugins/chat4n6-whatsapp/src/link.rs @@ -38,10 +38,7 @@ pub fn parse_url_components(url: &str) -> Option { let caps = re.captures(url)?; let scheme = caps.get(1)?.as_str().to_lowercase(); let domain = caps.get(2)?.as_str().to_string(); - let path = caps - .get(3) - .map(|m| m.as_str().to_string()) - .unwrap_or_default(); + let path = caps.get(3).map(|m| m.as_str().to_string()).unwrap_or_default(); Some(ExtractedLink { url: caps.get(0)?.as_str().to_string(), scheme, @@ -83,11 +80,7 @@ mod tests { fn test_url_with_path_and_query() { let urls = extract_urls("https://example.com/path/to/page?q=hello&lang=en"); assert_eq!(urls.len(), 1); - assert!( - urls[0].path.contains("path/to/page"), - "path: {}", - urls[0].path - ); + assert!(urls[0].path.contains("path/to/page"), "path: {}", urls[0].path); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/location.rs b/crates/plugins/chat4n6-whatsapp/src/location.rs index 2930ca5..bc7f5c0 100644 --- a/crates/plugins/chat4n6-whatsapp/src/location.rs +++ b/crates/plugins/chat4n6-whatsapp/src/location.rs @@ -141,15 +141,9 @@ mod tests { #[test] fn test_osm_url_format() { let url = osm_url(51.5074, -0.1278); - assert!( - url.starts_with("https://www.openstreetmap.org"), - "url: {url}" - ); + assert!(url.starts_with("https://www.openstreetmap.org"), "url: {url}"); assert!(url.contains("51.5074"), "url: {url}"); - assert!( - url.contains("-0.1278") || url.contains("0.1278"), - "url: {url}" - ); + assert!(url.contains("-0.1278") || url.contains("0.1278"), "url: {url}"); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs b/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs index e3d272a..c55c950 100644 --- a/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs +++ b/crates/plugins/chat4n6-whatsapp/src/orphaned_media.rs @@ -7,14 +7,14 @@ pub struct OrphanedMedia { pub file_path: PathBuf, pub file_size: u64, pub extension: String, - pub file_hash: Option, // SHA-256 hex, computed lazily - pub linked_media_path: Option, // set after rescue pass + pub file_hash: Option, // SHA-256 hex, computed lazily + pub linked_media_path: Option, // set after rescue pass } /// Recognized media file extensions (lowercase). const RECOGNIZED_EXTENSIONS: &[&str] = &[ - "jpg", "jpeg", "png", "gif", "mp4", "mov", "avi", "opus", "ogg", "mp3", "aac", "pdf", "doc", - "docx", "webp", + "jpg", "jpeg", "png", "gif", "mp4", "mov", "avi", "opus", "ogg", "mp3", "aac", + "pdf", "doc", "docx", "webp", ]; fn is_recognized_extension(ext: &str) -> bool { @@ -24,7 +24,10 @@ fn is_recognized_extension(ext: &str) -> bool { /// Walk `media_dir` and find files with recognized extensions that are NOT /// in the `known_paths` set. Return them as OrphanedMedia records (file_hash=None initially). -pub fn scan_orphaned_media(media_dir: &Path, known_paths: &HashSet) -> Vec { +pub fn scan_orphaned_media( + media_dir: &Path, + known_paths: &HashSet, +) -> Vec { let mut orphans = Vec::new(); let read_dir = match std::fs::read_dir(media_dir) { Ok(rd) => rd, @@ -146,11 +149,7 @@ mod tests { let mut known = HashSet::new(); known.insert(path.to_string_lossy().to_string()); let orphans = scan_orphaned_media(dir.path(), &known); - assert_eq!( - orphans.len(), - 1, - "should find only the orphan, not the known file" - ); + assert_eq!(orphans.len(), 1, "should find only the orphan, not the known file"); assert!(orphans[0].file_path.file_name().unwrap() == "orphan.png"); } @@ -186,10 +185,7 @@ mod tests { let known: HashSet = HashSet::new(); let orphans = scan_orphaned_media(dir.path(), &known); assert_eq!(orphans.len(), 1); - assert!( - orphans[0].file_hash.is_none(), - "file_hash should be None before hashing" - ); + assert!(orphans[0].file_hash.is_none(), "file_hash should be None before hashing"); } // ── hash_orphans tests ──────────────────────────────────────────────────── @@ -224,10 +220,7 @@ mod tests { hash_orphans(&mut orphans); for o in &orphans { - assert!( - o.file_hash.is_some(), - "all orphans should have hashes after hash_orphans" - ); + assert!(o.file_hash.is_some(), "all orphans should have hashes after hash_orphans"); } } @@ -247,21 +240,17 @@ mod tests { let hash_hex = hex::encode(Sha256::digest(content)); // Convert hex to base64 for the missing_media format use base64::Engine; - let hash_b64 = - base64::engine::general_purpose::STANDARD.encode(&hex::decode(&hash_hex).unwrap()); + let hash_b64 = base64::engine::general_purpose::STANDARD.encode( + &hex::decode(&hash_hex).unwrap() + ); - let missing = vec![( - "expected/path/media.jpg".to_string(), - content.len() as u64, - Some(hash_b64), - )]; + let missing = vec![ + ("expected/path/media.jpg".to_string(), content.len() as u64, Some(hash_b64)), + ]; let matched = rescue_orphans(&mut orphans, &missing); assert_eq!(matched, 1, "should match 1 orphan"); - assert!( - orphans[0].linked_media_path.is_some(), - "linked_media_path should be set after rescue" - ); + assert!(orphans[0].linked_media_path.is_some(), "linked_media_path should be set after rescue"); } #[test] @@ -276,17 +265,12 @@ mod tests { use base64::Engine; let wrong_hash_b64 = base64::engine::general_purpose::STANDARD.encode(b"wrong hash bytes"); - let missing = vec![( - "expected/path/media.jpg".to_string(), - b"actual content".len() as u64, - Some(wrong_hash_b64), - )]; + let missing = vec![ + ("expected/path/media.jpg".to_string(), b"actual content".len() as u64, Some(wrong_hash_b64)), + ]; let matched = rescue_orphans(&mut orphans, &missing); - assert_eq!( - matched, 0, - "should not match when hash differs even if size matches" - ); + assert_eq!(matched, 0, "should not match when hash differs even if size matches"); } #[test] @@ -298,7 +282,9 @@ mod tests { let mut orphans = scan_orphaned_media(dir.path(), &known); hash_orphans(&mut orphans); - let missing = vec![("expected/path/media.jpg".to_string(), 99999u64, None)]; + let missing = vec![ + ("expected/path/media.jpg".to_string(), 99999u64, None), + ]; let matched = rescue_orphans(&mut orphans, &missing); assert_eq!(matched, 0, "should not match when size differs"); @@ -316,12 +302,10 @@ mod tests { let mut orphans = scan_orphaned_media(dir.path(), &known); hash_orphans(&mut orphans); - use base64::Engine; use sha2::{Digest, Sha256}; - let h1 = - base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content1).as_slice()); - let h2 = - base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content2).as_slice()); + use base64::Engine; + let h1 = base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content1).as_slice()); + let h2 = base64::engine::general_purpose::STANDARD.encode(Sha256::digest(content2).as_slice()); let missing = vec![ ("path/a.jpg".to_string(), content1.len() as u64, Some(h1)), diff --git a/crates/plugins/chat4n6-whatsapp/src/platform.rs b/crates/plugins/chat4n6-whatsapp/src/platform.rs index 2eb5c52..094756e 100644 --- a/crates/plugins/chat4n6-whatsapp/src/platform.rs +++ b/crates/plugins/chat4n6-whatsapp/src/platform.rs @@ -4,11 +4,11 @@ use serde::{Deserialize, Serialize}; pub enum SenderPlatform { Android, IPhone, - Companion, // Web/Desktop linked device - AndroidLinked, // secondary Android device - IPhoneLinked, // secondary iPhone device - BusinessApi, // WhatsApp Business Cloud API bot - OldAndroid, // numeric key_id ≤10 chars + Companion, // Web/Desktop linked device + AndroidLinked, // secondary Android device + IPhoneLinked, // secondary iPhone device + BusinessApi, // WhatsApp Business Cloud API bot + OldAndroid, // numeric key_id ≤10 chars Unknown, } @@ -77,7 +77,9 @@ pub fn classify_key_id( platform: SenderPlatform::IPhone, confidence: 0.95, } - } else if upper.starts_with("3F") || upper.starts_with("3E") || upper.starts_with("3B") + } else if upper.starts_with("3F") + || upper.starts_with("3E") + || upper.starts_with("3B") { PlatformClassification { platform: SenderPlatform::Companion, @@ -162,20 +164,14 @@ mod tests { fn test_len8_from_me_true_android_085() { let (p, c) = classify("ABCD1234", true, None); assert_eq!(p, SenderPlatform::Android); - assert!( - (c - 0.85).abs() < 0.001, - "confidence should be 0.85, got {c}" - ); + assert!((c - 0.85).abs() < 0.001, "confidence should be 0.85, got {c}"); } #[test] fn test_len8_from_me_false_android_075() { let (p, c) = classify("ABCD1234", false, None); assert_eq!(p, SenderPlatform::Android); - assert!( - (c - 0.75).abs() < 0.001, - "confidence should be 0.75, got {c}" - ); + assert!((c - 0.75).abs() < 0.001, "confidence should be 0.75, got {c}"); } // ── len=16 tests ───────────────────────────────────────────────────────── @@ -191,10 +187,7 @@ mod tests { fn test_len16_prefix_ac_lowercase_android_097() { let (p, c) = classify("ac1234567890abcd", false, None); assert_eq!(p, SenderPlatform::Android); - assert!( - (c - 0.97).abs() < 0.001, - "case-insensitive prefix match should give 0.97" - ); + assert!((c - 0.97).abs() < 0.001, "case-insensitive prefix match should give 0.97"); } #[test] diff --git a/crates/plugins/chat4n6-whatsapp/src/poll.rs b/crates/plugins/chat4n6-whatsapp/src/poll.rs index 5e4d596..73c9389 100644 --- a/crates/plugins/chat4n6-whatsapp/src/poll.rs +++ b/crates/plugins/chat4n6-whatsapp/src/poll.rs @@ -39,22 +39,14 @@ pub fn build_poll( // Deduplicate voter JIDs for this option let unique_jids: Vec = { let mut seen = HashSet::new(); - voter_jids - .into_iter() - .filter(|j| seen.insert(j.clone())) - .collect() + voter_jids.into_iter().filter(|j| seen.insert(j.clone())).collect() }; for jid in &unique_jids { all_voters.insert(jid.clone()); } let voter_names: Vec = unique_jids .iter() - .map(|jid| { - voter_name_map - .get(jid) - .cloned() - .unwrap_or_else(|| jid.clone()) - }) + .map(|jid| voter_name_map.get(jid).cloned().unwrap_or_else(|| jid.clone())) .collect(); PollOption { option_id: idx as i64, diff --git a/crates/plugins/chat4n6-whatsapp/src/status.rs b/crates/plugins/chat4n6-whatsapp/src/status.rs index 4b4d004..b462685 100644 --- a/crates/plugins/chat4n6-whatsapp/src/status.rs +++ b/crates/plugins/chat4n6-whatsapp/src/status.rs @@ -25,11 +25,11 @@ pub struct StatusRecord { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum StatusType { - Image, // type 1 - Text, // type 2 - Video, // type 3 - Gif, // type 43 - Audio, // type 44 + Image, // type 1 + Text, // type 2 + Video, // type 3 + Gif, // type 43 + Audio, // type 44 Unknown(i32), } @@ -126,14 +126,7 @@ mod tests { #[test] fn test_view_count_preserved() { let mut r = make_record(3); - enrich_with_stats( - &mut r, - Some(StatusStats { - view_count: 100, - reaction_count: 0, - reactions: vec![], - }), - ); + enrich_with_stats(&mut r, Some(StatusStats { view_count: 100, reaction_count: 0, reactions: vec![] })); assert_eq!(r.stats.unwrap().view_count, 100); } @@ -141,25 +134,10 @@ mod tests { fn test_reactions_list_preserved() { let mut r = make_record(1); let reactions = vec![ - StatusReaction { - reactor_jid: "a@s.whatsapp.net".to_string(), - emoji: "👍".to_string(), - timestamp_ms: None, - }, - StatusReaction { - reactor_jid: "b@s.whatsapp.net".to_string(), - emoji: "🔥".to_string(), - timestamp_ms: Some(5000), - }, + StatusReaction { reactor_jid: "a@s.whatsapp.net".to_string(), emoji: "👍".to_string(), timestamp_ms: None }, + StatusReaction { reactor_jid: "b@s.whatsapp.net".to_string(), emoji: "🔥".to_string(), timestamp_ms: Some(5000) }, ]; - enrich_with_stats( - &mut r, - Some(StatusStats { - view_count: 0, - reaction_count: 2, - reactions, - }), - ); + enrich_with_stats(&mut r, Some(StatusStats { view_count: 0, reaction_count: 2, reactions })); assert_eq!(r.stats.unwrap().reactions.len(), 2); } } diff --git a/crates/plugins/chat4n6-whatsapp/src/system_event.rs b/crates/plugins/chat4n6-whatsapp/src/system_event.rs index 9eac0cc..e19216f 100644 --- a/crates/plugins/chat4n6-whatsapp/src/system_event.rs +++ b/crates/plugins/chat4n6-whatsapp/src/system_event.rs @@ -3,48 +3,48 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum SystemEventType { // Group admin events - GroupSubjectChanged, // 1 - GroupIconChanged, // 6 - GroupDescriptionChanged, // 19 - GroupInviteLinkReset, // 83 (same int — use context) + GroupSubjectChanged, // 1 + GroupIconChanged, // 6 + GroupDescriptionChanged, // 19 + GroupInviteLinkReset, // 83 (same int — use context) // Participant events - ParticipantAdded, // 12 - ParticipantLeft, // 5 - ParticipantRemoved, // 14 - ParticipantJoinedViaLink, // 20 - ApprovalRequest, // 83 + ParticipantAdded, // 12 + ParticipantLeft, // 5 + ParticipantRemoved, // 14 + ParticipantJoinedViaLink, // 20 + ApprovalRequest, // 83 // Security/E2E - SecurityCodeChanged, // 18 - E2EEncryptedNotification, // 67 + SecurityCodeChanged, // 18 + E2EEncryptedNotification, // 67 // Admin role changes - ParticipantPromotedToAdmin, // 84 - ParticipantDemotedFromAdmin, // 84 (need context) + ParticipantPromotedToAdmin, // 84 + ParticipantDemotedFromAdmin, // 84 (need context) // Number change - NumberChanged, // 46 + NumberChanged, // 46 // Disappearing messages - DisappearingTimerChanged, // 56 + DisappearingTimerChanged, // 56 // Message pinned/unpinned - MessagePinned, // 79 - MessageUnpinned, // 79 (need context) + MessagePinned, // 79 + MessageUnpinned, // 79 (need context) // Community events - CommunityCreated, // 97 - CommunityJoined, // 98 - CommunitySubgroupAdded, // 99 - CommunitySubgroupRemoved, // 100 - CommunitySubgroupUnlinked, // 101 - CommunityOwnerChanged, // 102 + CommunityCreated, // 97 + CommunityJoined, // 98 + CommunitySubgroupAdded, // 99 + CommunitySubgroupRemoved, // 100 + CommunitySubgroupUnlinked, // 101 + CommunityOwnerChanged, // 102 // Channel events - ChannelCreated, // 134 - ChannelDeleted, // 135 - ChannelPrivacyNotice, // 136 + ChannelCreated, // 134 + ChannelDeleted, // 135 + ChannelPrivacyNotice, // 136 // Permission changes PermissionAddMemberChanged, // 77 @@ -54,8 +54,8 @@ pub enum SystemEventType { PermissionJoinChanged, // 106 // Business - MetaAiDisclaimer, // 117 - BusinessMetaManaged, // 118 + MetaAiDisclaimer, // 117 + BusinessMetaManaged, // 118 // Unknown (preserves the raw integer) Unknown(i32), @@ -64,7 +64,7 @@ pub enum SystemEventType { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SystemEvent { pub event_type: SystemEventType, - pub label: String, // human-readable display string + pub label: String, // human-readable display string pub actor_jid: Option, pub target_jid: Option, } @@ -89,135 +89,52 @@ pub fn parse_system_event( fn decode_event(msg_type: i32, text_data: Option<&str>) -> (SystemEventType, String) { match msg_type { - 1 => ( - SystemEventType::GroupSubjectChanged, - "Group subject changed".to_string(), - ), - 5 => ( - SystemEventType::ParticipantLeft, - "Participant left group".to_string(), - ), - 6 => ( - SystemEventType::GroupIconChanged, - "Group icon changed".to_string(), - ), - 12 => ( - SystemEventType::ParticipantAdded, - "Participant added to group".to_string(), - ), - 14 => ( - SystemEventType::ParticipantRemoved, - "Participant removed from group".to_string(), - ), - 18 => ( - SystemEventType::SecurityCodeChanged, - "Security code changed".to_string(), - ), - 19 => ( - SystemEventType::GroupDescriptionChanged, - "Group description changed".to_string(), - ), - 20 => ( - SystemEventType::ParticipantJoinedViaLink, - "Participant joined via link".to_string(), - ), + 1 => (SystemEventType::GroupSubjectChanged, "Group subject changed".to_string()), + 5 => (SystemEventType::ParticipantLeft, "Participant left group".to_string()), + 6 => (SystemEventType::GroupIconChanged, "Group icon changed".to_string()), + 12 => (SystemEventType::ParticipantAdded, "Participant added to group".to_string()), + 14 => (SystemEventType::ParticipantRemoved, "Participant removed from group".to_string()), + 18 => (SystemEventType::SecurityCodeChanged, "Security code changed".to_string()), + 19 => (SystemEventType::GroupDescriptionChanged, "Group description changed".to_string()), + 20 => (SystemEventType::ParticipantJoinedViaLink, "Participant joined via link".to_string()), 46 => { let label = decode_number_change(text_data); (SystemEventType::NumberChanged, label) } - 56 => ( - SystemEventType::DisappearingTimerChanged, - "Disappearing timer changed".to_string(), - ), - 67 => ( - SystemEventType::E2EEncryptedNotification, - "End-to-end encryption enabled".to_string(), - ), - 77 => ( - SystemEventType::PermissionAddMemberChanged, - "Permission to add members changed".to_string(), - ), - 78 => ( - SystemEventType::PermissionEditChanged, - "Permission to edit group changed".to_string(), - ), + 56 => (SystemEventType::DisappearingTimerChanged, "Disappearing timer changed".to_string()), + 67 => (SystemEventType::E2EEncryptedNotification, "End-to-end encryption enabled".to_string()), + 77 => (SystemEventType::PermissionAddMemberChanged, "Permission to add members changed".to_string()), + 78 => (SystemEventType::PermissionEditChanged, "Permission to edit group changed".to_string()), 79 => (SystemEventType::MessagePinned, "Message pinned".to_string()), - 83 => ( - SystemEventType::GroupInviteLinkReset, - "Group invite link reset".to_string(), - ), - 84 => ( - SystemEventType::ParticipantPromotedToAdmin, - "Participant promoted to admin".to_string(), - ), - 97 => ( - SystemEventType::CommunityCreated, - "Community created".to_string(), - ), - 98 => ( - SystemEventType::CommunityJoined, - "Joined community".to_string(), - ), - 99 => ( - SystemEventType::CommunitySubgroupAdded, - "Community subgroup added".to_string(), - ), - 100 => ( - SystemEventType::CommunitySubgroupRemoved, - "Community subgroup removed".to_string(), - ), - 101 => ( - SystemEventType::CommunitySubgroupUnlinked, - "Community subgroup unlinked".to_string(), - ), - 102 => ( - SystemEventType::CommunityOwnerChanged, - "Community owner changed".to_string(), - ), - 104 => ( - SystemEventType::PermissionSendMessageChanged, - "Permission to send messages changed".to_string(), - ), - 105 => ( - SystemEventType::PermissionInviteChanged, - "Permission to invite members changed".to_string(), - ), - 106 => ( - SystemEventType::PermissionJoinChanged, - "Permission to join changed".to_string(), - ), - 117 => ( - SystemEventType::MetaAiDisclaimer, - "Meta AI disclaimer".to_string(), - ), - 118 => ( - SystemEventType::BusinessMetaManaged, - "Business managed by Meta".to_string(), - ), - 134 => ( - SystemEventType::ChannelCreated, - "Channel created".to_string(), - ), - 135 => ( - SystemEventType::ChannelDeleted, - "Channel deleted".to_string(), - ), - 136 => ( - SystemEventType::ChannelPrivacyNotice, - "Channel privacy notice".to_string(), - ), - other => ( - SystemEventType::Unknown(other), - format!("Unknown system event (type={other})"), - ), + 83 => (SystemEventType::GroupInviteLinkReset, "Group invite link reset".to_string()), + 84 => (SystemEventType::ParticipantPromotedToAdmin, "Participant promoted to admin".to_string()), + 97 => (SystemEventType::CommunityCreated, "Community created".to_string()), + 98 => (SystemEventType::CommunityJoined, "Joined community".to_string()), + 99 => (SystemEventType::CommunitySubgroupAdded, "Community subgroup added".to_string()), + 100 => (SystemEventType::CommunitySubgroupRemoved, "Community subgroup removed".to_string()), + 101 => (SystemEventType::CommunitySubgroupUnlinked, "Community subgroup unlinked".to_string()), + 102 => (SystemEventType::CommunityOwnerChanged, "Community owner changed".to_string()), + 104 => (SystemEventType::PermissionSendMessageChanged, "Permission to send messages changed".to_string()), + 105 => (SystemEventType::PermissionInviteChanged, "Permission to invite members changed".to_string()), + 106 => (SystemEventType::PermissionJoinChanged, "Permission to join changed".to_string()), + 117 => (SystemEventType::MetaAiDisclaimer, "Meta AI disclaimer".to_string()), + 118 => (SystemEventType::BusinessMetaManaged, "Business managed by Meta".to_string()), + 134 => (SystemEventType::ChannelCreated, "Channel created".to_string()), + 135 => (SystemEventType::ChannelDeleted, "Channel deleted".to_string()), + 136 => (SystemEventType::ChannelPrivacyNotice, "Channel privacy notice".to_string()), + other => (SystemEventType::Unknown(other), format!("Unknown system event (type={other})")), } } fn decode_number_change(text_data: Option<&str>) -> String { if let Some(text) = text_data { if let Ok(v) = serde_json::from_str::(text) { - let old = v.get("nc_old_phone").and_then(|p| p.as_str()).unwrap_or(""); - let new = v.get("nc_new_phone").and_then(|p| p.as_str()).unwrap_or(""); + let old = v.get("nc_old_phone") + .and_then(|p| p.as_str()) + .unwrap_or(""); + let new = v.get("nc_new_phone") + .and_then(|p| p.as_str()) + .unwrap_or(""); if !old.is_empty() || !new.is_empty() { return format!("Phone number changed: {} → {}", old, new); } @@ -331,11 +248,8 @@ mod tests { let e = parse_with_text(46, json); assert_eq!(e.event_type, SystemEventType::NumberChanged); // Label should include phone numbers when JSON is parseable - assert!( - e.label.contains("15551234567") || e.label.contains("number"), - "label should reference the number change, got: {}", - e.label - ); + assert!(e.label.contains("15551234567") || e.label.contains("number"), + "label should reference the number change, got: {}", e.label); } #[test] @@ -451,12 +365,7 @@ mod tests { #[test] fn test_actor_target_jid_preserved() { - let e = parse_system_event( - 12, - None, - Some("actor@s.whatsapp.net"), - Some("target@s.whatsapp.net"), - ); + let e = parse_system_event(12, None, Some("actor@s.whatsapp.net"), Some("target@s.whatsapp.net")); assert_eq!(e.actor_jid.as_deref(), Some("actor@s.whatsapp.net")); assert_eq!(e.target_jid.as_deref(), Some("target@s.whatsapp.net")); } From ef7299de3f8fd930c2ac25d5d013db6d88d7c8d8 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 30 Jul 2026 13:04:54 +0800 Subject: [PATCH 11/11] docs: file the deferred schema-resolution work as user stories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handoff deferred three items out of the column-resolution fix and asked that they be routed somewhere durable rather than left in a working doc. Filed as stories in the repo's existing format, all passes:false: - whatsapp/message-media-metadata — read modern-schema media metadata from message_media, which replaced the media columns on the message table, plus confirming the auxiliary table column names against real device DDL. Both are named gaps in docs/validation.md today. - platforms/schema-resolution-audit — chat4n6-ios-whatsapp, chat4n6-signal and chat4n6-telegram still read records at fixed values[] indices and have not been checked against real DDL. Notes that the iOS timestamp base is seconds since 2001-01-01, so the gate's plausibility bounds need converting rather than copying. - sqlite-engine/raw-image-unallocated-input — accept a raw decrypted partition image so unallocated space can be carved at all (PlaintextDirFs reports no unallocated regions by design), and map the recovery layers beyond the live btree into messages through the same resolved column maps. Each story names the docs/validation.md gap it closes, so the doc and the backlog cannot drift apart. Steps describe the evidence class generically; no case identifiers, paths or content. Co-Authored-By: Claude Opus 5 (1M context) --- .../platforms/schema-resolution-audit.json | 17 ++++++++++ .../raw-image-unallocated-input.json | 32 +++++++++++++++++++ .../whatsapp/message-media-metadata.json | 29 +++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 docs/user-stories/platforms/schema-resolution-audit.json create mode 100644 docs/user-stories/sqlite-engine/raw-image-unallocated-input.json create mode 100644 docs/user-stories/whatsapp/message-media-metadata.json diff --git a/docs/user-stories/platforms/schema-resolution-audit.json b/docs/user-stories/platforms/schema-resolution-audit.json new file mode 100644 index 0000000..49c5090 --- /dev/null +++ b/docs/user-stories/platforms/schema-resolution-audit.json @@ -0,0 +1,17 @@ +[ + { + "description": "iOS WhatsApp, Signal and Telegram resolve column positions by name instead of hardcoded ordinals", + "steps": [ + "Audit chat4n6-ios-whatsapp, chat4n6-signal and chat4n6-telegram for the pattern the Android WhatsApp plugin carried: record fields read at a fixed values[] index (helpers::cols::* in Signal, cols::* in Telegram, CoreData attribute positions in iOS)", + "For each plugin, record the real CREATE TABLE DDL from a genuine database of that platform (DDL only, no case content) and check it against the ordinals the code assumes", + "Add a real-DDL fixture per plugin whose column layout differs from the simplified fixture, so a positional reader fails by construction", + "Add a column-order-permuted fixture per plugin, as tests/fixtures/shuffled_schema.sql does for Android WhatsApp, and assert both extract identically", + "Reuse chat4n6-whatsapp's columns module for name-to-ordinal resolution rather than reimplementing it per plugin; promote it to a shared crate if three plugins need it", + "Apply the equivalent of schema_gate::validate_message_columns per platform, so a contradicted column map aborts instead of producing a complete but wrong report", + "Note the iOS timestamp base differs (seconds since 2001-01-01), so the plausibility bounds need converting rather than copying", + "Update the 'other plugins are unaudited' gap in docs/validation.md", + "cargo test --workspace passes" + ], + "passes": false + } +] diff --git a/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json b/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json new file mode 100644 index 0000000..52b1df9 --- /dev/null +++ b/docs/user-stories/sqlite-engine/raw-image-unallocated-input.json @@ -0,0 +1,32 @@ +[ + { + "description": "A raw decrypted partition image is an accepted input, so unallocated space can be carved", + "steps": [ + "Accepted inputs today are a plaintext directory, a .dar archive and an iOS backup; a raw decrypted userdata partition image cannot be supplied at all", + "Add a raw-image input type to chat4n6-fs that opens an ext4 partition image and exposes it through the ForensicFs trait", + "Wire it into the CLI input auto-detection alongside the existing three", + "Implement unallocated_regions() for it from the ext4 block bitmap: PlaintextDirFs returns an empty vec by design, which is why layer 8 has nothing to carve on this evidence class", + "Build a small synthetic ext4 image as a committed fixture so the path is exercised without a multi-gigabyte artifact, keeping the coverage gate satisfiable from committed bytes alone", + "Assert files under data/data/com.whatsapp/databases are readable through the raw-image filesystem", + "Assert unallocated_regions() returns non-empty regions for an image with freed blocks, and that layer 8 carving runs over them", + "Validate against an independent oracle on a real image: reconcile the carved region list against what The Sleuth Kit reports as unallocated", + "Update the 'unallocated carving needs a raw-image input' gap in docs/validation.md", + "cargo test -p chat4n6-fs passes" + ], + "passes": false + }, + { + "description": "Recovery layers beyond the live btree are mapped into messages by the WhatsApp plugin", + "steps": [ + "extract_from_msgstore consumes recover_layer1 only, so WAL deltas, freelist, journal, intra-page carve and unallocated carve are recovered by the engine but never interpreted as messages", + "Feed recover_all() records through the same resolved column maps the live layer uses, so no layer reinvents a column position", + "Confirm carvers that pattern-match record shapes against the message schema use the resolved shape rather than a fixture shape", + "Assert a message present only in a WAL frame appears in the extraction with source WalHistoric", + "Assert a message present only in a freelist page appears with source Freelist", + "Assert records recovered from a non-live layer still pass through the schema gate, so a wrong map cannot enter through a side door", + "Re-verify WAL parsing against a real database after the column-resolution fix, since the WAL path shared the ordinals that were wrong", + "cargo test -p chat4n6-whatsapp passes" + ], + "passes": false + } +] diff --git a/docs/user-stories/whatsapp/message-media-metadata.json b/docs/user-stories/whatsapp/message-media-metadata.json new file mode 100644 index 0000000..88d6de7 --- /dev/null +++ b/docs/user-stories/whatsapp/message-media-metadata.json @@ -0,0 +1,29 @@ +[ + { + "description": "Media metadata for the modern schema is read from message_media, which replaced the media columns on the message table", + "steps": [ + "Extend tests/fixtures/real_modern_schema.sql with a message_media table (message_row_id, mime_type, file_path, media_name, file_length, media_duration, file_hash, enc_file_hash, media_key, direct_path) and a row joined to a media message", + "Resolve every message_media column by name via columns::TableColumns, as message/jid/chat/call_log already are", + "Call extract_from_msgstore on that fixture", + "Assert the media message produces MessageContent::Media with the mime_type and file_path from message_media, not None", + "Assert file_size, duration_secs, file_hash and encrypted_hash are populated from the joined row", + "Assert a modern-schema database with no message_media row for a media message still extracts the message, with media fields absent rather than wrong", + "Assert modern_schema.sql (media columns on message) keeps working, so both schema generations are covered", + "cargo test -p chat4n6-whatsapp -- message_media passes" + ], + "passes": false + }, + { + "description": "Auxiliary msgstore table column names are confirmed against real device DDL", + "steps": [ + "Record the CREATE TABLE DDL for message_quoted, message_add_on, message_add_on_reaction, message_edit_info, receipt_user, message_forwarded and group_participant_user from a real device (DDL only, no case content)", + "Compare each against the names the extractor resolves today, which come from the simplified modern_schema.sql and are unverified", + "Where a real schema spells a field differently (for example parent_message_row_id for message_row_id, or message_add_on_type for type), resolve both spellings via TableColumns::first_of", + "Add a real-DDL fixture for those tables alongside real_modern_schema.sql", + "Assert reactions, edits and receipts extract from the real-DDL fixture rather than resolving to None", + "Update the 'auxiliary table column names are unverified' gap in docs/validation.md", + "cargo test -p chat4n6-whatsapp -- real_schema passes" + ], + "passes": false + } +]