From 6831ac41698b2c3104a5ac73d15a088e6654d52c Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Tue, 23 Jun 2026 16:56:19 +0200 Subject: [PATCH 1/2] Stream acquisitions to zip archives --- README.md | 4 +- acquisition/acquisition.go | 196 ++++-------------- acquisition/acquisition_test.go | 119 +++++++---- acquisition/secure.go | 132 ------------ .../{encrypted_stream.go => streaming_zip.go} | 153 ++++++++------ ...d_stream_test.go => streaming_zip_test.go} | 11 +- log/logger.go | 2 +- main.go | 36 +--- modules/backup.go | 28 +-- modules/bugreport.go | 28 +-- modules/dumpsys.go | 9 +- modules/env.go | 9 +- modules/files.go | 9 +- modules/getprop.go | 9 +- modules/intrusion_logs.go | 62 ++---- modules/logcat.go | 9 +- modules/logs.go | 86 ++------ modules/modules.go | 62 +----- modules/mounts.go | 9 +- modules/packages.go | 150 +++----------- modules/paths.go | 42 ---- modules/paths_test.go | 71 ------- modules/processes.go | 9 +- modules/root_binaries.go | 9 +- modules/selinux.go | 9 +- modules/services.go | 9 +- modules/settings.go | 9 +- modules/temp.go | 73 ++----- 28 files changed, 321 insertions(+), 1033 deletions(-) delete mode 100644 acquisition/secure.go rename acquisition/{encrypted_stream.go => streaming_zip.go} (59%) rename acquisition/{encrypted_stream_test.go => streaming_zip_test.go} (93%) diff --git a/README.md b/README.md index 4e8a9d4..fdea4f7 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Before launching androidqf you need to have the target Android device connected Once USB debugging is enabled, you can proceed launching androidqf. It will first attempt to connect to the device over the USB bridge, which should result in the Android phone to prompt you to manually authorize the host keys. Make sure to authorize them, ideally permanently so that the prompt wouldn't appear again. -Now androidqf should be executing and creating an acquisition folder in your current working directory. At some point in the execution, androidqf will prompt you some choices: these prompts will pause the acquisition until you provide a selection, so pay attention. +Now androidqf should be executing and creating an acquisition zip archive in your current working directory, or in the directory provided with `-output`. At some point in the execution, androidqf will prompt you some choices: these prompts will pause the acquisition until you provide a selection, so pay attention. The following data can be extracted: @@ -186,7 +186,7 @@ Ideally you should have the drive fully encrypted, but that might not always be Alternatively, androidqf allows to encrypt each acquisition with a provided [age](https://age-encryption.org) public key. Preferably, this public key belongs to a keypair for which the end-user does not possess, or at least carry, the private key. In this way, the end-user would not be able to decrypt the acquired data even under duress. -If you place a file called `key.txt` in the current working directory, androidqf will automatically attempt to compress and encrypt each acquisition and delete the original unencrypted copies. androidqf also checks for `key.txt` in the same folder as the executable; if both files exist, the current working directory takes precedence. +androidqf streams each acquisition into a zip archive. If you place a file called `key.txt` in the current working directory, androidqf will encrypt the zip stream with age and write `.zip.age`; otherwise, it writes an unencrypted `.zip`. androidqf also checks for `key.txt` in the same folder as the executable; if both files exist, the current working directory takes precedence. Once you have retrieved an encrypted acquisition file, you can decrypt it with age like so: diff --git a/acquisition/acquisition.go b/acquisition/acquisition.go index f68c8f1..01739da 100644 --- a/acquisition/acquisition.go +++ b/acquisition/acquisition.go @@ -7,16 +7,12 @@ package acquisition import ( "bytes" - "encoding/csv" "encoding/json" "fmt" "io" - "os" - "path/filepath" "strings" "time" - "github.com/botherder/go-savetime/hashes" "github.com/google/uuid" "github.com/mvt-project/androidqf/adb" "github.com/mvt-project/androidqf/assets" @@ -36,7 +32,7 @@ type Acquisition struct { SdCard string `json:"sdcard"` Cpu string `json:"cpu"` closeLog func() `json:"-"` - EncryptedWriter *EncryptedZipWriter `json:"-"` + ZipWriter *StreamingZipWriter `json:"-"` StreamingMode bool `json:"streaming_mode"` StreamingPuller *StreamingPuller `json:"-"` logBuffer *bytes.Buffer `json:"-"` @@ -48,28 +44,11 @@ func New(path string) (*Acquisition, error) { UUID: uuid.New().String(), Started: time.Now().UTC(), AndroidQFVersion: utils.Version, - } - - if path == "" { - acq.StoragePath = acq.UUID - } else { - acq.StoragePath = path - } - // Check if the path exist - stat, err := os.Stat(acq.StoragePath) - if os.IsNotExist(err) { - err := os.Mkdir(acq.StoragePath, 0o755) - if err != nil { - return nil, fmt.Errorf("failed to create acquisition folder: %v", err) - } - } else { - if !stat.IsDir() { - return nil, fmt.Errorf("path exist and is not a folder") - } + StreamingMode: true, } // Get system information first to get tmp folder - err = acq.GetSystemInformation() + err := acq.GetSystemInformation() if err != nil { return nil, err } @@ -81,39 +60,24 @@ func New(path string) (*Acquisition, error) { } acq.Collector = coll - // Try to initialize encrypted streaming mode - encWriter, err := NewEncryptedZipWriter(acq.UUID) + zipWriter, err := NewStreamingZipWriter(acq.UUID, path) if err != nil { - // No key file or encryption setup failed, use normal mode - log.Debug("Encrypted streaming not available, using normal mode") - acq.StreamingMode = false - - // Init logging file for normal mode - logPath := filepath.Join(acq.StoragePath, "command.log") - closeLog, err := log.EnableFileLog(log.DEBUG, logPath) - if err != nil { - return nil, fmt.Errorf("failed to enable file logging: %v", err) - } - acq.closeLog = closeLog - } else { - // Encrypted streaming mode enabled - log.Info("Using encrypted streaming mode - data will be written directly to encrypted archive") - acq.StreamingMode = true - acq.EncryptedWriter = encWriter + return nil, err + } + acq.ZipWriter = zipWriter + acq.StoragePath = zipWriter.GetOutputPath() - // Initialize streaming puller for direct operations - acq.StreamingPuller = NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, 100) // 100MB max memory + // Initialize streaming puller for direct operations. + acq.StreamingPuller = NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, 100) - // Create buffer for command.log (will be written to archive at completion) - acq.logBuffer = new(bytes.Buffer) + // Create buffer for command.log (will be written to archive at completion). + acq.logBuffer = new(bytes.Buffer) - // Init logging to write to buffer - closeLog, err := log.EnableWriterLog(log.DEBUG, acq.logBuffer) - if err != nil { - return nil, fmt.Errorf("failed to enable writer logging: %v", err) - } - acq.closeLog = closeLog + closeLog, err := log.EnableWriterLog(log.DEBUG, acq.logBuffer) + if err != nil { + return nil, fmt.Errorf("failed to enable writer logging: %v", err) } + acq.closeLog = closeLog return &acq, nil } @@ -123,16 +87,15 @@ func (a *Acquisition) Complete() { a.Completed = time.Now().UTC() } - // Handle streaming mode completion - if a.StreamingMode && a.EncryptedWriter != nil { - // Store acquisition info in the encrypted zip + if a.ZipWriter != nil { + // Store acquisition info in the zip info, err := json.MarshalIndent(a, "", " ") if err != nil { - log.Error("Failed to marshal acquisition info for encrypted archive") + log.Error("Failed to marshal acquisition info for archive") } else { - err = a.EncryptedWriter.CreateFileFromBytes("acquisition.json", info) + err = a.ZipWriter.CreateFileFromBytes("acquisition.json", info) if err != nil { - log.ErrorExc("Failed to store acquisition info in encrypted archive", err) + log.ErrorExc("Failed to store acquisition info in archive", err) } } @@ -142,33 +105,22 @@ func (a *Acquisition) Complete() { a.closeLog() } - // Write buffered command.log to encrypted archive + // Write buffered command.log to archive if a.logBuffer != nil && a.logBuffer.Len() > 0 { - err = a.EncryptedWriter.CreateFileFromBytes("command.log", a.logBuffer.Bytes()) + err = a.ZipWriter.CreateFileFromBytes("command.log", a.logBuffer.Bytes()) if err != nil { - log.ErrorExc("Failed to add command.log to encrypted archive", err) + log.ErrorExc("Failed to add command.log to archive", err) } } - err = a.EncryptedWriter.CreateHashList() + err = a.ZipWriter.CreateHashList() if err != nil { - log.ErrorExc("Failed to add hashes.csv to encrypted archive", err) + log.ErrorExc("Failed to add hashes.csv to archive", err) } - // Close the encrypted writer - err = a.EncryptedWriter.Close() + err = a.ZipWriter.Close() if err != nil { - log.ErrorExc("Failed to close encrypted archive", err) - } - - // Remove the temporary storage directory if it was created and used - if a.StoragePath != "" { - if _, err := os.Stat(a.StoragePath); err == nil { - err = os.RemoveAll(a.StoragePath) - if err != nil { - log.ErrorExc("Failed to clean up temporary storage directory", err) - } - } + log.ErrorExc("Failed to close archive", err) } } else { // Ensure log file is closed before cleanup operations @@ -225,80 +177,7 @@ func (a *Acquisition) GetSystemInformation() error { return nil } -func (a *Acquisition) HashFiles() error { - // In streaming mode, files are directly encrypted and no local files exist to hash - if a.StreamingMode { - log.Debug("Skipping hash generation in streaming mode (data is encrypted)") - return nil - } - - log.Info("Generating list of files hashes...") - - csvFile, err := os.Create(filepath.Join(a.StoragePath, "hashes.csv")) - if err != nil { - return err - } - defer csvFile.Close() - - csvWriter := csv.NewWriter(csvFile) - - walkErr := filepath.Walk(a.StoragePath, func(filePath string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - - if fileInfo.IsDir() { - return nil - } - // Makes files read only - os.Chmod(filePath, 0o400) - - sha256, err := hashes.FileSHA256(filePath) - if err != nil { - return err - } - - return csvWriter.Write([]string{filePath, sha256}) - }) - - csvWriter.Flush() - if err := csvWriter.Error(); err != nil { - return err - } - - return walkErr -} - -func (a *Acquisition) StoreInfo() error { - // In streaming mode, info is stored during Complete() - if a.StreamingMode { - return nil - } - - if a.Completed.IsZero() { - a.Completed = time.Now().UTC() - } - - log.Info("Saving details about acquisition and device...") - - info, err := json.MarshalIndent(a, "", " ") - if err != nil { - return fmt.Errorf("failed to json marshal the acquisition details: %v", - err) - } - - infoPath := filepath.Join(a.StoragePath, "acquisition.json") - - err = os.WriteFile(infoPath, info, 0o644) - if err != nil { - return fmt.Errorf("failed to write acquisition details to file: %v", - err) - } - - return nil -} - -// StreamAPKToZip streams an APK file directly to encrypted zip with certificate processing +// StreamAPKToZip streams an APK file directly to the zip with certificate processing func (a *Acquisition) StreamAPKToZip(remotePath, zipPath string, processFunc func(io.Reader) error) error { if err := a.validateStreamingMode(); err != nil { return err @@ -325,16 +204,15 @@ func (a *Acquisition) StreamAPKToZip(remotePath, zipPath string, processFunc fun } } - // Stream to encrypted zip - err = a.EncryptedWriter.CreateFileFromReader(zipPath, buffer.Reader()) + err = a.ZipWriter.CreateFileFromReader(zipPath, buffer.Reader()) if err != nil { - return fmt.Errorf("failed to add APK %q to encrypted zip: %v", remotePath, err) + return fmt.Errorf("failed to add APK %q to zip: %v", remotePath, err) } return nil } -// StreamBackupToZip streams a backup directly to encrypted zip +// StreamBackupToZip streams a backup directly to the zip func (a *Acquisition) StreamBackupToZip(arg, zipPath string) error { if err := a.validateStreamingMode(); err != nil { return err @@ -348,7 +226,7 @@ func (a *Acquisition) StreamBackupToZip(arg, zipPath string) error { } // Create zip entry writer - writer, err := a.EncryptedWriter.CreateFile(zipPath) + writer, err := a.ZipWriter.CreateFile(zipPath) if err != nil { return fmt.Errorf("failed to create zip entry for backup: %v", err) } @@ -362,7 +240,7 @@ func (a *Acquisition) StreamBackupToZip(arg, zipPath string) error { return nil } -// StreamBugreportToZip streams a bugreport directly to encrypted zip +// StreamBugreportToZip streams a bugreport directly to the zip func (a *Acquisition) StreamBugreportToZip(zipPath string) error { if err := a.validateStreamingMode(); err != nil { return err @@ -373,7 +251,7 @@ func (a *Acquisition) StreamBugreportToZip(zipPath string) error { } // Create zip entry writer - writer, err := a.EncryptedWriter.CreateFile(zipPath) + writer, err := a.ZipWriter.CreateFile(zipPath) if err != nil { return fmt.Errorf("failed to create zip entry for bugreport: %v", err) } @@ -392,8 +270,8 @@ func (a *Acquisition) validateStreamingMode() error { if !a.StreamingMode { return fmt.Errorf("streaming mode not enabled") } - if a.EncryptedWriter == nil { - return fmt.Errorf("encrypted writer not initialized") + if a.ZipWriter == nil { + return fmt.Errorf("zip writer not initialized") } if a.StreamingPuller == nil { return fmt.Errorf("streaming puller not initialized") diff --git a/acquisition/acquisition_test.go b/acquisition/acquisition_test.go index 64515ec..12e3865 100644 --- a/acquisition/acquisition_test.go +++ b/acquisition/acquisition_test.go @@ -1,33 +1,51 @@ package acquisition import ( + "archive/zip" + "bytes" "encoding/json" + "io" "os" "path/filepath" "testing" + "time" ) -func TestStoreInfoSetsCompletedTimestamp(t *testing.T) { - acq := &Acquisition{ - UUID: "test-acquisition", - StoragePath: t.TempDir(), +func TestCompleteWritesMetadataToStreamingZip(t *testing.T) { + outputDir := t.TempDir() + t.Chdir(outputDir) + + zipWriter, err := NewStreamingZipWriter("test-acquisition", outputDir) + if err != nil { + t.Fatalf("NewStreamingZipWriter() error = %v", err) } - if err := acq.StoreInfo(); err != nil { - t.Fatalf("StoreInfo() error = %v", err) + started := time.Now().UTC() + acq := &Acquisition{ + UUID: "test-acquisition", + StoragePath: zipWriter.GetOutputPath(), + Started: started, + ZipWriter: zipWriter, + StreamingMode: true, + logBuffer: bytes.NewBufferString("logged command\n"), } + acq.Complete() + if acq.Completed.IsZero() { - t.Fatal("StoreInfo() left Completed unset") + t.Fatal("Complete() left Completed unset") } - info, err := os.ReadFile(filepath.Join(acq.StoragePath, "acquisition.json")) - if err != nil { - t.Fatalf("ReadFile(acquisition.json) error = %v", err) + files := readZipFiles(t, filepath.Join(outputDir, "test-acquisition.zip")) + if files["command.log"] != "logged command\n" { + t.Fatalf("command.log = %q", files["command.log"]) + } + if _, ok := files["hashes.csv"]; !ok { + t.Fatal("hashes.csv missing from archive") } var stored Acquisition - if err := json.Unmarshal(info, &stored); err != nil { + if err := json.Unmarshal([]byte(files["acquisition.json"]), &stored); err != nil { t.Fatalf("json.Unmarshal(acquisition.json) error = %v", err) } if stored.Completed.IsZero() { @@ -36,15 +54,23 @@ func TestStoreInfoSetsCompletedTimestamp(t *testing.T) { } func TestCompleteDoesNotOverwriteExistingCompletedTimestamp(t *testing.T) { - acq := &Acquisition{ - UUID: "test-acquisition", - StoragePath: t.TempDir(), + outputDir := t.TempDir() + t.Chdir(outputDir) + + zipWriter, err := NewStreamingZipWriter("test-acquisition", outputDir) + if err != nil { + t.Fatalf("NewStreamingZipWriter() error = %v", err) } - if err := acq.StoreInfo(); err != nil { - t.Fatalf("StoreInfo() error = %v", err) + completed := time.Now().UTC().Add(-time.Hour) + acq := &Acquisition{ + UUID: "test-acquisition", + StoragePath: zipWriter.GetOutputPath(), + Started: completed.Add(-time.Hour), + Completed: completed, + ZipWriter: zipWriter, + StreamingMode: true, } - completed := acq.Completed acq.Complete() @@ -53,33 +79,54 @@ func TestCompleteDoesNotOverwriteExistingCompletedTimestamp(t *testing.T) { } } -func TestStoreSecurelyUsesCurrentWorkingDirectory(t *testing.T) { - cwd := t.TempDir() - t.Chdir(cwd) - writeTestAgeKey(t, cwd) +func readZipFiles(t *testing.T, archivePath string) map[string]string { + t.Helper() - storagePath := filepath.Join(cwd, "test-acquisition") - if err := os.Mkdir(storagePath, 0o755); err != nil { - t.Fatalf("Mkdir(storagePath) error = %v", err) + reader, err := zip.OpenReader(archivePath) + if err != nil { + t.Fatalf("zip.OpenReader(%q) error = %v", archivePath, err) } - if err := os.WriteFile(filepath.Join(storagePath, "data.txt"), []byte("evidence"), 0o600); err != nil { - t.Fatalf("WriteFile(data.txt) error = %v", err) + defer reader.Close() + + files := make(map[string]string) + for _, file := range reader.File { + readCloser, err := file.Open() + if err != nil { + t.Fatalf("Open(%q) error = %v", file.Name, err) + } + content, err := io.ReadAll(readCloser) + readCloser.Close() + if err != nil { + t.Fatalf("ReadAll(%q) error = %v", file.Name, err) + } + files[file.Name] = string(content) } - acq := &Acquisition{ - UUID: "test-acquisition", - StoragePath: storagePath, + return files +} + +func TestNewStreamingZipWriterWithoutKeyCreatesPlainZip(t *testing.T) { + cwd := t.TempDir() + t.Chdir(cwd) + + ezw, err := NewStreamingZipWriter("test-acquisition", cwd) + if err != nil { + t.Fatalf("NewStreamingZipWriter() error = %v", err) } + defer os.Remove(ezw.GetOutputPath()) - if err := acq.StoreSecurely(); err != nil { - t.Fatalf("StoreSecurely() error = %v", err) + if ezw.IsEncrypted() { + t.Fatal("writer is encrypted without key.txt") + } + if err := ezw.Close(); err != nil { + t.Fatalf("Close() error = %v", err) } - wantPath := filepath.Join(cwd, "test-acquisition.zip.age") - if _, err := os.Stat(wantPath); err != nil { - t.Fatalf("Stat(encrypted output) error = %v", err) + wantPath := filepath.Join(cwd, "test-acquisition.zip") + if ezw.GetOutputPath() != wantPath { + t.Fatalf("output path = %q, want %q", ezw.GetOutputPath(), wantPath) } - if _, err := os.Stat(storagePath); !os.IsNotExist(err) { - t.Fatalf("storage path still exists or returned unexpected error: %v", err) + if _, err := os.Stat(wantPath); err != nil { + t.Fatalf("Stat(output) error = %v", err) } } diff --git a/acquisition/secure.go b/acquisition/secure.go deleted file mode 100644 index 065ddec..0000000 --- a/acquisition/secure.go +++ /dev/null @@ -1,132 +0,0 @@ -// androidqf - Android Quick Forensics -// Copyright (c) 2021-2022 Claudio Guarnieri. -// Use of this software is governed by the MVT License 1.1 that can be found at -// https://license.mvt.re/1.1/ - -package acquisition - -import ( - "archive/zip" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "filippo.io/age" - "github.com/mvt-project/androidqf/log" -) - -func createZipFile(sourceDir, zipPath string) error { - zipFile, err := os.Create(zipPath) - if err != nil { - return fmt.Errorf("failed to create ZIP file: %v", err) - } - defer zipFile.Close() - - zipWriter := zip.NewWriter(zipFile) - defer zipWriter.Close() - - // Use AddFS to add the entire directory - fsys := os.DirFS(sourceDir) - err = zipWriter.AddFS(fsys) - if err != nil { - return fmt.Errorf("failed to add directory to ZIP: %v", err) - } - - return nil -} - -func (a *Acquisition) StoreSecurely() error { - // In streaming mode, data is already encrypted during collection - if a.StreamingMode { - return nil - } - - cwd, err := os.Getwd() - if err != nil { - return err - } - - keyFilePath, ok, err := findAgeKeyFile() - if err != nil { - return err - } - if !ok { - return nil - } - - log.Info("You provided an age public key, storing the acquisition securely.") - - zipFileName := fmt.Sprintf("%s.zip", a.UUID) - zipFilePath := filepath.Join(cwd, zipFileName) - - log.Info("Compressing the acquisition folder. This might take a while...") - - err = createZipFile(a.StoragePath, zipFilePath) - if err != nil { - return err - } - - log.Info("Encrypting the compressed archive. This might take a while...") - - publicKey, err := os.ReadFile(keyFilePath) - if err != nil { - return err - } - publicKeyStr := strings.TrimSpace(string(publicKey)) - - recipient, err := age.ParseX25519Recipient(publicKeyStr) - if err != nil { - return fmt.Errorf("failed to parse public key %q: %v", publicKeyStr, err) - } - - zipFile, err := os.Open(zipFilePath) - if err != nil { - return err - } - defer zipFile.Close() - - encFileName := fmt.Sprintf("%s.age", zipFileName) - encFilePath := filepath.Join(cwd, encFileName) - encFile, err := os.OpenFile(encFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) - if err != nil { - return fmt.Errorf("unable to create encrypted file: %v", err) - } - defer encFile.Close() - - w, err := age.Encrypt(encFile, recipient) - if err != nil { - return fmt.Errorf("failed to create encrypted file: %v", err) - } - - _, err = io.Copy(w, zipFile) - if err != nil { - return fmt.Errorf("failed to write to encrypted file: %v", err) - } - - if err := w.Close(); err != nil { - return fmt.Errorf("failed to close encrypted file: %v", err) - } - - log.Infof("Acquisition successfully encrypted at %s", encFilePath) - - // TODO: we should securely wipe the files. - zipFile.Close() - err = os.Remove(zipFilePath) - if err != nil { - return fmt.Errorf("failed to delete the unencrypted compressed archive: %v", err) - } - - // Ensure log file is closed before removing the acquisition directory - if a.closeLog != nil { - defer a.closeLog() - } - - err = os.RemoveAll(a.StoragePath) - if err != nil { - return fmt.Errorf("failed to delete the original unencrypted acquisition folder: %v", err) - } - - return nil -} diff --git a/acquisition/encrypted_stream.go b/acquisition/streaming_zip.go similarity index 59% rename from acquisition/encrypted_stream.go rename to acquisition/streaming_zip.go index 059e49f..40ceecd 100644 --- a/acquisition/encrypted_stream.go +++ b/acquisition/streaming_zip.go @@ -24,12 +24,14 @@ import ( "github.com/mvt-project/androidqf/log" ) -// EncryptedZipWriter provides streaming encrypted zip functionality -type EncryptedZipWriter struct { +// StreamingZipWriter provides streaming zip functionality, optionally wrapped +// in age encryption when a public key is available. +type StreamingZipWriter struct { file *os.File encWriter io.WriteCloser zipWriter *zip.Writer outputPath string + encrypted bool closed bool hashes []*zipHash } @@ -52,73 +54,95 @@ func (hw *hashingWriter) Write(p []byte) (int, error) { return n, err } -// NewEncryptedZipWriter creates a new encrypted zip writer if key.txt exists -func NewEncryptedZipWriter(uuid string) (*EncryptedZipWriter, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, err +// NewStreamingZipWriter creates a streaming zip writer in outputDir. If key.txt +// exists, the zip stream is age-encrypted and written as .zip.age. +func NewStreamingZipWriter(uuid, outputDir string) (*StreamingZipWriter, error) { + if outputDir == "" { + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + outputDir = cwd } - // Check if key file exists - keyFilePath, ok, err := findAgeKeyFile() - if err != nil { - return nil, err - } - if !ok { - return nil, fmt.Errorf("key.txt not found, encrypted streaming not available") + stat, err := os.Stat(outputDir) + if os.IsNotExist(err) { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return nil, fmt.Errorf("failed to create output folder: %v", err) + } + } else if err != nil { + return nil, fmt.Errorf("failed to stat output folder: %v", err) + } else if !stat.IsDir() { + return nil, fmt.Errorf("output path exists and is not a folder") } - log.Info("Found age public key, using encrypted streaming mode.") - - // Read and parse public key - publicKey, err := os.ReadFile(keyFilePath) + keyFilePath, ok, err := findAgeKeyFile() if err != nil { - return nil, fmt.Errorf("failed to read public key: %v", err) + return nil, err } - publicKeyStr := strings.TrimSpace(string(publicKey)) - recipient, err := age.ParseX25519Recipient(publicKeyStr) - if err != nil { - return nil, fmt.Errorf("failed to parse public key %q: %v", publicKeyStr, err) + fileName := fmt.Sprintf("%s.zip", uuid) + if ok { + fileName += ".age" } - - // Create output file - encFileName := fmt.Sprintf("%s.zip.age", uuid) - outputPath := filepath.Join(cwd, encFileName) + outputPath := filepath.Join(outputDir, fileName) file, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return nil, fmt.Errorf("failed to create output file: %v", err) } - // Create encryption writer - encWriter, err := age.Encrypt(file, recipient) - if err != nil { - file.Close() - os.Remove(outputPath) // Clean up the created file - return nil, fmt.Errorf("failed to create encrypted writer: %v", err) + var encWriter io.WriteCloser + var zipSink io.Writer = file + if ok { + log.Info("Found age public key, streaming to encrypted zip archive.") + + publicKey, err := os.ReadFile(keyFilePath) + if err != nil { + file.Close() + os.Remove(outputPath) + return nil, fmt.Errorf("failed to read public key: %v", err) + } + publicKeyStr := strings.TrimSpace(string(publicKey)) + + recipient, err := age.ParseX25519Recipient(publicKeyStr) + if err != nil { + file.Close() + os.Remove(outputPath) + return nil, fmt.Errorf("failed to parse public key %q: %v", publicKeyStr, err) + } + + encWriter, err = age.Encrypt(file, recipient) + if err != nil { + file.Close() + os.Remove(outputPath) + return nil, fmt.Errorf("failed to create encrypted writer: %v", err) + } + zipSink = encWriter + } else { + log.Info("No age public key found, streaming to unencrypted zip archive.") } - // Create zip writer - zipWriter := zip.NewWriter(encWriter) + zipWriter := zip.NewWriter(zipSink) - log.Infof("Started encrypted streaming to %s", outputPath) + log.Infof("Started streaming to %s", outputPath) - return &EncryptedZipWriter{ + return &StreamingZipWriter{ file: file, encWriter: encWriter, zipWriter: zipWriter, outputPath: outputPath, + encrypted: ok, closed: false, }, nil } -// CreateFile creates a new file in the encrypted zip and returns a writer -func (ezw *EncryptedZipWriter) CreateFile(name string) (io.Writer, error) { +// CreateFile creates a new file in the zip and returns a writer +func (ezw *StreamingZipWriter) CreateFile(name string) (io.Writer, error) { return ezw.createFile(name, true) } -func (ezw *EncryptedZipWriter) createFile(name string, trackHash bool) (io.Writer, error) { +func (ezw *StreamingZipWriter) createFile(name string, trackHash bool) (io.Writer, error) { if err := ezw.checkClosed(); err != nil { return nil, err } @@ -182,8 +206,8 @@ func validateZipEntryName(name string) error { return nil } -// CreateFileFromReader copies data from a reader to a file in the encrypted zip -func (ezw *EncryptedZipWriter) CreateFileFromReader(name string, src io.Reader) error { +// CreateFileFromReader copies data from a reader to a file in the zip +func (ezw *StreamingZipWriter) CreateFileFromReader(name string, src io.Reader) error { if src == nil { return fmt.Errorf("source reader cannot be nil") } @@ -201,9 +225,9 @@ func (ezw *EncryptedZipWriter) CreateFileFromReader(name string, src io.Reader) return nil } -// CreateHashList adds hashes.csv to the encrypted zip with SHA-256 hashes of +// CreateHashList adds hashes.csv to the zip with SHA-256 hashes of // the plaintext zip entries written so far. -func (ezw *EncryptedZipWriter) CreateHashList() error { +func (ezw *StreamingZipWriter) CreateHashList() error { if err := ezw.checkClosed(); err != nil { return err } @@ -234,18 +258,18 @@ func (ezw *EncryptedZipWriter) CreateHashList() error { return nil } -// CreateFileFromString creates a file with string content in the encrypted zip -func (ezw *EncryptedZipWriter) CreateFileFromString(name, content string) error { +// CreateFileFromString creates a file with string content in the zip +func (ezw *StreamingZipWriter) CreateFileFromString(name, content string) error { return ezw.CreateFileFromReader(name, strings.NewReader(content)) } -// CreateFileFromBytes creates a file with byte content in the encrypted zip -func (ezw *EncryptedZipWriter) CreateFileFromBytes(name string, content []byte) error { +// CreateFileFromBytes creates a file with byte content in the zip +func (ezw *StreamingZipWriter) CreateFileFromBytes(name string, content []byte) error { return ezw.CreateFileFromReader(name, bytes.NewReader(content)) } -// CreateFileFromPath reads a file from disk and adds it to the encrypted zip -func (ezw *EncryptedZipWriter) CreateFileFromPath(name, filePath string) error { +// CreateFileFromPath reads a file from disk and adds it to the zip +func (ezw *StreamingZipWriter) CreateFileFromPath(name, filePath string) error { file, err := os.Open(filePath) if err != nil { return fmt.Errorf("failed to open source file %q: %v", filePath, err) @@ -255,8 +279,8 @@ func (ezw *EncryptedZipWriter) CreateFileFromPath(name, filePath string) error { return ezw.CreateFileFromReader(name, file) } -// Close finalizes and closes the encrypted zip -func (ezw *EncryptedZipWriter) Close() error { +// Close finalizes and closes the zip +func (ezw *StreamingZipWriter) Close() error { if ezw.closed { return nil } @@ -269,10 +293,11 @@ func (ezw *EncryptedZipWriter) Close() error { lastErr = fmt.Errorf("failed to close zip writer: %v", err) } - // Close encryption writer - if err := ezw.encWriter.Close(); err != nil { - if lastErr == nil { - lastErr = fmt.Errorf("failed to close encryption writer: %v", err) + if ezw.encWriter != nil { + if err := ezw.encWriter.Close(); err != nil { + if lastErr == nil { + lastErr = fmt.Errorf("failed to close encryption writer: %v", err) + } } } @@ -284,25 +309,29 @@ func (ezw *EncryptedZipWriter) Close() error { } if lastErr == nil { - log.Infof("Encrypted archive created successfully at %s", ezw.outputPath) + log.Infof("Archive created successfully at %s", ezw.outputPath) } return lastErr } -// GetOutputPath returns the path to the encrypted zip file -func (ezw *EncryptedZipWriter) GetOutputPath() string { +// GetOutputPath returns the path to the zip file +func (ezw *StreamingZipWriter) GetOutputPath() string { return ezw.outputPath } +func (ezw *StreamingZipWriter) IsEncrypted() bool { + return ezw.encrypted +} + // IsClosed returns whether the writer has been closed -func (ezw *EncryptedZipWriter) IsClosed() bool { +func (ezw *StreamingZipWriter) IsClosed() bool { return ezw.closed } // checkClosed is a helper method to check if the writer is closed -func (ezw *EncryptedZipWriter) checkClosed() error { +func (ezw *StreamingZipWriter) checkClosed() error { if ezw.closed { - return fmt.Errorf("encrypted zip writer is closed") + return fmt.Errorf("streaming zip writer is closed") } return nil } diff --git a/acquisition/encrypted_stream_test.go b/acquisition/streaming_zip_test.go similarity index 93% rename from acquisition/encrypted_stream_test.go rename to acquisition/streaming_zip_test.go index 0e258dc..9937fac 100644 --- a/acquisition/encrypted_stream_test.go +++ b/acquisition/streaming_zip_test.go @@ -17,7 +17,7 @@ import ( func TestCreateHashListTracksPlaintextZipEntries(t *testing.T) { var archive bytes.Buffer - ezw := &EncryptedZipWriter{ + ezw := &StreamingZipWriter{ zipWriter: zip.NewWriter(&archive), } @@ -111,17 +111,20 @@ func writeTestAgeKey(t *testing.T, dir string) { } } -func TestNewEncryptedZipWriterUsesCurrentWorkingDirectory(t *testing.T) { +func TestNewStreamingZipWriterUsesCurrentWorkingDirectory(t *testing.T) { cwd := t.TempDir() t.Chdir(cwd) writeTestAgeKey(t, cwd) - ezw, err := NewEncryptedZipWriter("test-acquisition") + ezw, err := NewStreamingZipWriter("test-acquisition", cwd) if err != nil { - t.Fatalf("NewEncryptedZipWriter() error = %v", err) + t.Fatalf("NewStreamingZipWriter() error = %v", err) } defer os.Remove(ezw.GetOutputPath()) + if !ezw.IsEncrypted() { + t.Fatal("writer is not encrypted with key.txt present") + } if err := ezw.Close(); err != nil { t.Fatalf("Close() error = %v", err) } diff --git a/log/logger.go b/log/logger.go index 21909c0..0b96776 100644 --- a/log/logger.go +++ b/log/logger.go @@ -111,7 +111,7 @@ func (log *Logger) out(level LEVEL, format string, v ...any) { } } - // Print to writer if active (for streaming to encrypted archive) + // Print to writer if active (for streaming to the acquisition archive) if log.writerActive && log.writer != nil { var msg string if level >= log.FileLogLevel { diff --git a/main.go b/main.go index 213745d..83312e2 100644 --- a/main.go +++ b/main.go @@ -128,22 +128,13 @@ func main() { } // Start acquisitions - log.Info(fmt.Sprintf("Started new acquisition in %s", acq.StoragePath)) + log.Info(fmt.Sprintf("Started new acquisition archive in %s", acq.StoragePath)) mods := modules.List() for _, mod := range mods { if (module != "") && (module != mod.Name()) { continue } - err = mod.InitStorage(acq.StoragePath) - if err != nil { - log.Infof( - "ERROR: failed to initialize storage for module %s: %v", - mod.Name(), - err, - ) - continue - } err = mod.Run(acq, fast) if err != nil { @@ -151,30 +142,7 @@ func main() { } } - if acq.StreamingMode { - // In streaming mode, all data is already encrypted in the zip stream - log.Info("Finalizing encrypted acquisition...") - } else { - // Traditional mode: hash files, then encrypt if key exists - err = acq.HashFiles() - if err != nil { - log.ErrorExc("Failed to generate list of file hashes", err) - return - } - - err = acq.StoreInfo() - if err != nil { - log.ErrorExc("Failed to store acquisition info", err) - return - } - - err = acq.StoreSecurely() - if err != nil { - log.ErrorExc("Something failed while encrypting the acquisition", err) - log.Warning("WARNING: The secure storage of the acquisition folder failed! The data is unencrypted!") - } - } - + log.Info("Finalizing acquisition archive...") acq.Complete() log.Info("Acquisition completed.") diff --git a/modules/backup.go b/modules/backup.go index 56706ae..707ae7d 100644 --- a/modules/backup.go +++ b/modules/backup.go @@ -7,11 +7,9 @@ package modules import ( "fmt" - "path/filepath" "github.com/manifoldco/promptui" "github.com/mvt-project/androidqf/acquisition" - "github.com/mvt-project/androidqf/adb" "github.com/mvt-project/androidqf/log" ) @@ -21,9 +19,7 @@ const ( backupNothing = "No backup" ) -type Backup struct { - StoragePath string -} +type Backup struct{} func NewBackup() *Backup { return &Backup{} @@ -33,11 +29,6 @@ func (b *Backup) Name() string { return "backup" } -func (b *Backup) InitStorage(storagePath string) error { - b.StoragePath = storagePath - return nil -} - func (b *Backup) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Would you like to take a backup of the device?") promptBackup := promptui.Select{ @@ -64,20 +55,9 @@ func (b *Backup) Run(acq *acquisition.Acquisition, fast bool) error { arg, ) - if acq.StreamingMode && acq.EncryptedWriter != nil { - // Streaming mode: stream backup directly to encrypted zip without temp files - err = acq.StreamBackupToZip(arg, "backup.ab") - if err != nil { - return fmt.Errorf("failed to stream backup to encrypted archive: %v", err) - } - } else { - // Traditional mode: write backup directly into acquisition directory - backupPath := filepath.Join(b.StoragePath, "backup.ab") - err = adb.Client.Backup(backupPath, arg) - if err != nil { - log.Debugf("Impossible to get backup: %v", err) - return err - } + err = acq.StreamBackupToZip(arg, "backup.ab") + if err != nil { + return fmt.Errorf("failed to stream backup to archive: %v", err) } log.Info("Backup completed!") diff --git a/modules/bugreport.go b/modules/bugreport.go index 9a8f5df..b11e34a 100644 --- a/modules/bugreport.go +++ b/modules/bugreport.go @@ -7,16 +7,12 @@ package modules import ( "fmt" - "path/filepath" "github.com/mvt-project/androidqf/acquisition" - "github.com/mvt-project/androidqf/adb" "github.com/mvt-project/androidqf/log" ) -type Bugreport struct { - StoragePath string -} +type Bugreport struct{} func NewBugreport() *Bugreport { return &Bugreport{} @@ -26,30 +22,14 @@ func (b *Bugreport) Name() string { return "bugreport" } -func (b *Bugreport) InitStorage(storagePath string) error { - b.StoragePath = storagePath - return nil -} - func (b *Bugreport) Run(acq *acquisition.Acquisition, fast bool) error { log.Info( "Generating a bugreport for the device...", ) - if acq.StreamingMode && acq.EncryptedWriter != nil { - // Streaming mode: stream bugreport directly to encrypted zip without temp files - err := acq.StreamBugreportToZip("bugreport.zip") - if err != nil { - return fmt.Errorf("failed to stream bugreport to encrypted archive: %v", err) - } - } else { - // Traditional mode: write directly into acquisition dir. - bugreportPath := filepath.Join(b.StoragePath, "bugreport.zip") - err := adb.Client.Bugreport(bugreportPath) - if err != nil { - log.Debugf("Impossible to generate bugreport: %v", err) - return err - } + err := acq.StreamBugreportToZip("bugreport.zip") + if err != nil { + return fmt.Errorf("failed to stream bugreport to archive: %v", err) } log.Debug("Bugreport completed!") diff --git a/modules/dumpsys.go b/modules/dumpsys.go index e1ed78e..b52510a 100644 --- a/modules/dumpsys.go +++ b/modules/dumpsys.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Dumpsys struct { - StoragePath string -} +type Dumpsys struct{} func NewDumpsys() *Dumpsys { return &Dumpsys{} @@ -24,11 +22,6 @@ func (d *Dumpsys) Name() string { return "dumpsys" } -func (d *Dumpsys) InitStorage(storagePath string) error { - d.StoragePath = storagePath - return nil -} - func (d *Dumpsys) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting device diagnostic information. This might take a while...") diff --git a/modules/env.go b/modules/env.go index 44623aa..a56bea6 100644 --- a/modules/env.go +++ b/modules/env.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Environment struct { - StoragePath string -} +type Environment struct{} func NewEnvironment() *Environment { return &Environment{} @@ -24,11 +22,6 @@ func (e *Environment) Name() string { return "environment" } -func (e *Environment) InitStorage(storagePath string) error { - e.StoragePath = storagePath - return nil -} - func (e *Environment) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting environment...") diff --git a/modules/files.go b/modules/files.go index 1a15b25..cc9c024 100644 --- a/modules/files.go +++ b/modules/files.go @@ -11,9 +11,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Files struct { - StoragePath string -} +type Files struct{} func NewFiles() *Files { return &Files{} @@ -23,11 +21,6 @@ func (f *Files) Name() string { return "files" } -func (f *Files) InitStorage(storagePath string) error { - f.StoragePath = storagePath - return nil -} - func (f *Files) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting list of files... This might take a while...") var fileFounds []string diff --git a/modules/getprop.go b/modules/getprop.go index a393391..aa15cb0 100644 --- a/modules/getprop.go +++ b/modules/getprop.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type GetProp struct { - StoragePath string -} +type GetProp struct{} func NewGetProp() *GetProp { return &GetProp{} @@ -24,11 +22,6 @@ func (g *GetProp) Name() string { return "getprop" } -func (g *GetProp) InitStorage(storagePath string) error { - g.StoragePath = storagePath - return nil -} - func (g *GetProp) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting device properties...") diff --git a/modules/intrusion_logs.go b/modules/intrusion_logs.go index a6dca5f..391c7a7 100644 --- a/modules/intrusion_logs.go +++ b/modules/intrusion_logs.go @@ -11,7 +11,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "strings" "time" @@ -27,8 +26,6 @@ const ( ) type IL struct { - StoragePath string - ILPath string DirOnDevice string } @@ -42,21 +39,6 @@ func (m *IL) Name() string { return "intrusion_logs" } -func (m *IL) InitStorage(storagePath string) error { - m.StoragePath = storagePath - m.ILPath = filepath.Join(storagePath, "intrusion_logs") - - // Only create directory in traditional mode - if storagePath != "" { - err := os.Mkdir(m.ILPath, 0o755) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create Intrusion Logging folder: %v", err) - } - } - - return nil -} - func (m *IL) Run(acq *acquisition.Acquisition, fast bool) error { // Check whether the device supports AAPM. compatible, err := m.isAAPMCompatibleDevice() @@ -250,19 +232,6 @@ func (m *IL) waitForNewFiles( } func (m *IL) pullAll(acq *acquisition.Acquisition, deviceFiles []string) error { - streaming := acq.StreamingMode && acq.EncryptedWriter != nil - var localRoot *os.Root - var puller *acquisition.StreamingPuller - if !streaming { - var err error - localRoot, err = os.OpenRoot(m.ILPath) - if err != nil { - return fmt.Errorf("failed to open intrusion logs output root: %v", err) - } - defer localRoot.Close() - puller = acquisition.NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, 100) - } - for _, file := range deviceFiles { if file == m.DirOnDevice { continue @@ -274,28 +243,21 @@ func (m *IL) pullAll(acq *acquisition.Acquisition, deviceFiles []string) error { continue } - if streaming { - zipPath := path.Join("intrusion_logs", rel) - - writer, err := acq.EncryptedWriter.CreateFile(zipPath) - if err != nil { - log.Errorf("Failed to create zip entry for IL file %s: %v\n", file, err) - continue - } + zipPath := path.Join("intrusion_logs", rel) - err = acq.StreamingPuller.PullToWriter(file, writer) - if err != nil { - log.Errorf("Failed to stream IL file %s: %v\n", file, err) - continue - } + writer, err := acq.ZipWriter.CreateFile(zipPath) + if err != nil { + log.Errorf("Failed to create zip entry for IL file %s: %v\n", file, err) + continue + } - log.Debugf("Streamed IL file %s directly to encrypted archive as %s", file, zipPath) - } else { - if err := streamDeviceChildToRoot(localRoot, puller, rel, file); err != nil { - log.Errorf("Failed to pull IL file %s: %v\n", file, err) - continue - } + err = acq.StreamingPuller.PullToWriter(file, writer) + if err != nil { + log.Errorf("Failed to stream IL file %s: %v\n", file, err) + continue } + + log.Debugf("Streamed IL file %s directly to archive as %s", file, zipPath) } return nil diff --git a/modules/logcat.go b/modules/logcat.go index c90ee52..ffae6c0 100644 --- a/modules/logcat.go +++ b/modules/logcat.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Logcat struct { - StoragePath string -} +type Logcat struct{} func NewLogcat() *Logcat { return &Logcat{} @@ -24,11 +22,6 @@ func (l *Logcat) Name() string { return "logcat" } -func (l *Logcat) InitStorage(storagePath string) error { - l.StoragePath = storagePath - return nil -} - func (l *Logcat) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting logcat...") diff --git a/modules/logs.go b/modules/logs.go index d0ebdba..abb385f 100644 --- a/modules/logs.go +++ b/modules/logs.go @@ -6,9 +6,6 @@ package modules import ( "fmt" - "os" - "path/filepath" - "strings" "github.com/botherder/go-savetime/text" "github.com/mvt-project/androidqf/acquisition" @@ -16,10 +13,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Logs struct { - StoragePath string - LogsPath string -} +type Logs struct{} func NewLogs() *Logs { return &Logs{} @@ -29,21 +23,6 @@ func (l *Logs) Name() string { return "logs" } -func (l *Logs) InitStorage(storagePath string) error { - l.StoragePath = storagePath - l.LogsPath = filepath.Join(storagePath, "logs") - - // Only create directory in traditional mode - if storagePath != "" { - err := os.Mkdir(l.LogsPath, 0o755) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create logs folder: %v", err) - } - } - - return nil -} - func (l *Logs) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting system logs...") @@ -70,59 +49,26 @@ func (l *Logs) Run(acq *acquisition.Acquisition, fast bool) error { } for _, logFile := range logFiles { - // logFile is device controlled; validate it stays within LogsPath. - rel, err := filepath.Rel(l.LogsPath, filepath.Join(l.LogsPath, logFile)) - if err != nil || !filepath.IsLocal(rel) { - log.Errorf("Skipping log file with path traversal: %s", logFile) - continue - } - - if acq.StreamingMode && acq.EncryptedWriter != nil { - // Streaming mode: stream directly from ADB to encrypted zip without temp files - log.Debugf("From: %s", logFile) - log.Debugf("To encrypted archive as: logs%s", logFile) + log.Debugf("From: %s", logFile) - // Create zip path with logs/ prefix - zipPath := fmt.Sprintf("logs%s", logFile) + zipPath := fmt.Sprintf("logs%s", logFile) + log.Debugf("To archive as: %s", zipPath) - // Create zip entry writer - writer, err := acq.EncryptedWriter.CreateFile(zipPath) - if err != nil { - log.Errorf("Failed to create zip entry for log %s: %v\n", logFile, err) - continue - } - - // Stream log file directly to encrypted zip using acquisition's streaming puller - err = acq.StreamingPuller.PullToWriter(logFile, writer) - if err != nil { - if !text.ContainsNoCase(err.Error(), "Permission denied") { - log.Errorf("Failed to stream log file %s: %v\n", logFile, err) - } - continue - } - - log.Debugf("Streamed log file %s directly to encrypted archive", logFile) - } else { - // Traditional mode: create local directory structure and pull files - localPath := filepath.Join(l.LogsPath, logFile) - localDir, _ := filepath.Split(localPath) - log.Debugf("From: %s", logFile) - log.Debugf("To: %s", localPath) - - err := os.MkdirAll(localDir, 0o755) - if err != nil { - log.Errorf("Failed to create folders for logs %s: %v\n", localDir, err) - continue - } + writer, err := acq.ZipWriter.CreateFile(zipPath) + if err != nil { + log.Errorf("Failed to create zip entry for log %s: %v\n", logFile, err) + continue + } - out, err := adb.Client.Pull(logFile, localPath) - if err != nil { - if !text.ContainsNoCase(out, "Permission denied") { - log.Errorf("Failed to pull log file %s: %s\n", logFile, strings.TrimSpace(out)) - } - continue + err = acq.StreamingPuller.PullToWriter(logFile, writer) + if err != nil { + if !text.ContainsNoCase(err.Error(), "Permission denied") { + log.Errorf("Failed to stream log file %s: %v\n", logFile, err) } + continue } + + log.Debugf("Streamed log file %s directly to archive", logFile) } return nil diff --git a/modules/modules.go b/modules/modules.go index 4af36db..0a9b307 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -7,15 +7,12 @@ package modules import ( "encoding/json" "fmt" - "os" - "path/filepath" "github.com/mvt-project/androidqf/acquisition" ) type Module interface { Name() string - InitStorage(storagePath string) error Run(acq *acquisition.Acquisition, fast bool) error } @@ -41,7 +38,7 @@ func List() []Module { } } -// saveDataToAcquisition saves data to either encrypted stream or file based on acquisition mode +// saveDataToAcquisition saves JSON data to the acquisition archive. func saveDataToAcquisition(acq *acquisition.Acquisition, filename string, data any) error { if filename == "" { return fmt.Errorf("filename cannot be empty") @@ -50,34 +47,27 @@ func saveDataToAcquisition(acq *acquisition.Acquisition, filename string, data a return fmt.Errorf("data cannot be nil") } - if acq.StreamingMode && acq.EncryptedWriter != nil { - return saveDataToStream(acq.EncryptedWriter, filename, data) + if acq.ZipWriter == nil { + return fmt.Errorf("zip writer cannot be nil") } - - // Fall back to traditional file saving - filePath := filepath.Join(acq.StoragePath, filename) - return saveDataToFile(filePath, data) + return saveDataToStream(acq.ZipWriter, filename, data) } -// saveStringToAcquisition saves string content to either encrypted stream or file +// saveStringToAcquisition saves string content to the acquisition archive. func saveStringToAcquisition(acq *acquisition.Acquisition, filename, content string) error { if filename == "" { return fmt.Errorf("filename cannot be empty") } - - if acq.StreamingMode && acq.EncryptedWriter != nil { - return acq.EncryptedWriter.CreateFileFromString(filename, content) + if acq.ZipWriter == nil { + return fmt.Errorf("zip writer cannot be nil") } - - // Fall back to traditional file saving - filePath := filepath.Join(acq.StoragePath, filename) - return saveStringToFile(filePath, content) + return acq.ZipWriter.CreateFileFromString(filename, content) } -// saveDataToStream saves JSON data to encrypted zip stream -func saveDataToStream(writer *acquisition.EncryptedZipWriter, filename string, data any) error { +// saveDataToStream saves JSON data to a zip stream. +func saveDataToStream(writer *acquisition.StreamingZipWriter, filename string, data any) error { if writer == nil { - return fmt.Errorf("encrypted writer cannot be nil") + return fmt.Errorf("zip writer cannot be nil") } jsonData, err := json.MarshalIndent(&data, "", " ") @@ -86,33 +76,3 @@ func saveDataToStream(writer *acquisition.EncryptedZipWriter, filename string, d } return writer.CreateFileFromString(filename, string(jsonData)) } - -// saveDataToFile saves JSON data to a file (traditional mode) -func saveDataToFile(filePath string, data any) error { - jsonData, err := json.MarshalIndent(&data, "", " ") - if err != nil { - return fmt.Errorf("failed to convert data to JSON: %v", err) - } - return saveStringToFile(filePath, string(jsonData)) -} - -// saveStringToFile saves string content to a file (traditional mode) -func saveStringToFile(filePath, content string) error { - file, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("failed to create file %q: %v", filePath, err) - } - defer file.Close() - - _, err = file.WriteString(content) - if err != nil { - return fmt.Errorf("failed to write content to file %q: %v", filePath, err) - } - - err = file.Sync() - if err != nil { - return fmt.Errorf("failed to sync file %q: %v", filePath, err) - } - - return nil -} diff --git a/modules/mounts.go b/modules/mounts.go index 818940e..9863ba8 100644 --- a/modules/mounts.go +++ b/modules/mounts.go @@ -14,9 +14,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Mounts struct { - StoragePath string -} +type Mounts struct{} func NewMounts() *Mounts { return &Mounts{} @@ -26,11 +24,6 @@ func (m *Mounts) Name() string { return "mounts" } -func (m *Mounts) InitStorage(storagePath string) error { - m.StoragePath = storagePath - return nil -} - func (m *Mounts) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting mount information") diff --git a/modules/packages.go b/modules/packages.go index bdf4068..eafa995 100644 --- a/modules/packages.go +++ b/modules/packages.go @@ -6,7 +6,6 @@ package modules import ( "fmt" - "os" "path/filepath" "strings" @@ -25,10 +24,7 @@ const ( apkKeepAll = "No" ) -type Packages struct { - StoragePath string - ApksPath string -} +type Packages struct{} func NewPackages() *Packages { return &Packages{} @@ -38,46 +34,6 @@ func (p *Packages) Name() string { return "packages" } -func (p *Packages) InitStorage(storagePath string) error { - p.StoragePath = storagePath - p.ApksPath = filepath.Join(storagePath, "apks") - - // Only create directory in traditional mode - if storagePath != "" { - err := os.Mkdir(p.ApksPath, 0o755) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create apks folder: %v", err) - } - } - - return nil -} - -func (p *Packages) getPathToLocalCopy(packageName, filePath string) (string, error) { - suffix, err := p.extractFileName(filePath) - if err != nil { - return "", err - } - base := fmt.Sprintf("%s%s.apk", packageName, suffix) - if !filepath.IsLocal(base) { - return "", fmt.Errorf("non-local APK basename: %q", base) - } - localPath := filepath.Join(p.ApksPath, base) - - counter := 0 - for { - if _, err := os.Stat(localPath); os.IsNotExist(err) { - break - } - counter++ - localPath = filepath.Join( - p.ApksPath, - fmt.Sprintf("%s%s_%d.apk", packageName, suffix, counter), - ) - } - return localPath, nil -} - func (p *Packages) extractFileName(filePath string) (string, error) { if !strings.Contains(filePath, "==/") { return "", nil @@ -137,13 +93,10 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { var keepOption string - // Only ask about certificate removal for unencrypted output - if acq.StreamingMode && acq.EncryptedWriter != nil { - // For encrypted output, always keep all APKs (skip certificate checking) + if acq.ZipWriter != nil && acq.ZipWriter.IsEncrypted() { keepOption = apkKeepAll } else { - // Ask if the user want to remove trusted packages for unencrypted output - log.Info("Would you like to remove copies of apps signed with a trusted certificate to limit the size of the output folder?") + log.Info("Would you like to remove copies of apps signed with a trusted certificate to limit the size of the output archive?") promptAll := promptui.Select{ Label: "Remove", Items: []string{apkRemoveTrusted, apkKeepAll}, @@ -167,56 +120,9 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { for ipf := 0; ipf < len(packages[ip].Files); ipf++ { packageFile := &packages[ip].Files[ipf] - if acq.StreamingMode && acq.EncryptedWriter != nil { - // Streaming mode: stream directly to encrypted zip without temp files - if err := p.processAPKStreaming(packages[ip].Name, packageFile, keepOption, acq); err != nil { - log.Debugf("ERROR: failed to process APK %s: %v", packageFile.Path, err) - continue - } - } else { - // Traditional mode: download to local storage - localPath, err := p.getPathToLocalCopy(packages[ip].Name, packageFile.Path) - if err != nil { - log.Errorf("Skipping APK with unsafe path %q: %v", packageFile.Path, err) - packageFile.Error = err.Error() - continue - } - - out, err := adb.Client.Pull(packageFile.Path, localPath) - if err != nil { - packageFile.Error = out - log.Debugf("ERROR: failed to download %s: %s", packageFile.Path, out) - continue - } - - log.Debugf("Downloaded %s to %s", packageFile.Path, localPath) - - // Check the certificate - verified, cert, err := utils.VerifyCertificate(localPath) - if cert == nil { - // Couldn't extract certificate - log.Debugf("Couldn't parse certificate for app %s", localPath) - packageFile.CertificateError = err.Error() - packageFile.VerifiedCertificate = false - } else { - packageFile.Certificate = *cert - packageFile.VerifiedCertificate = false - if err != nil { - // Extracted certificate but couldn't verify it - packageFile.CertificateError = err.Error() - } else { - packageFile.CertificateError = "" - packageFile.VerifiedCertificate = verified - if utils.IsTrusted(*cert) { - packageFile.TrustedCertificate = true - if keepOption == apkRemoveTrusted { - log.Debugf("Trusted APK removed: %s - %s", - localPath, packageFile.SHA256) - os.Remove(localPath) - } - } - } - } + if err := p.processAPKStreaming(packages[ip].Name, packageFile, keepOption, acq); err != nil { + log.Debugf("ERROR: failed to process APK %s: %v", packageFile.Path, err) + continue } } } @@ -225,7 +131,6 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { return saveDataToAcquisition(acq, "packages.json", &packages) } -// processAPKStreaming handles APK processing in streaming mode func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.PackageFile, keepOption string, acq *acquisition.Acquisition) error { zipPath, err := p.generateZipPath(packageName, packageFile.Path) if err != nil { @@ -234,43 +139,42 @@ func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.Pack return nil } - // For encrypted output, skip certificate processing entirely - if acq.EncryptedWriter != nil { - log.Debugf("Skipping certificate check for encrypted output: %s", packageFile.Path) + if acq.ZipWriter != nil && acq.ZipWriter.IsEncrypted() { + log.Debugf("Skipping certificate check for encrypted archive: %s", packageFile.Path) + err = acq.StreamAPKToZip(packageFile.Path, zipPath, nil) + if err != nil { + packageFile.Error = fmt.Sprintf("Failed to stream to archive: %v", err) + return err + } } else { - // Process certificate and determine if APK should be skipped (unencrypted output only) - shouldSkip, err := p.processCertificate(packageFile, keepOption, acq) + buffer, err := acq.StreamingPuller.PullToBuffer(packageFile.Path) if err != nil { - packageFile.Error = fmt.Sprintf("Certificate processing failed: %v", err) + packageFile.Error = fmt.Sprintf("Failed to pull APK: %v", err) return err } + shouldSkip, err := p.processCertificate(packageFile, keepOption, buffer) + if err != nil { + packageFile.Error = fmt.Sprintf("Certificate processing failed: %v", err) + return err + } if shouldSkip { log.Debugf("Trusted APK skipped for streaming: %s", packageFile.Path) return nil } + err = acq.ZipWriter.CreateFileFromReader(zipPath, buffer.Reader()) + if err != nil { + packageFile.Error = fmt.Sprintf("Failed to stream to archive: %v", err) + return err + } } - // Stream APK directly to encrypted zip - err = acq.StreamAPKToZip(packageFile.Path, zipPath, nil) - if err != nil { - packageFile.Error = fmt.Sprintf("Failed to stream to encrypted archive: %v", err) - return err - } - - log.Debugf("Streamed %s directly to encrypted archive as %s", packageFile.Path, zipPath) + log.Debugf("Streamed %s directly to archive as %s", packageFile.Path, zipPath) return nil } // processCertificate handles certificate verification and returns whether APK should be skipped -func (p *Packages) processCertificate(packageFile *adb.PackageFile, keepOption string, acq *acquisition.Acquisition) (bool, error) { - // Pull APK to buffer for certificate verification - buffer, err := acq.StreamingPuller.PullToBuffer(packageFile.Path) - if err != nil { - return false, fmt.Errorf("failed to pull APK for certificate verification: %v", err) - } - - // Verify certificate from buffer using in-memory verification +func (p *Packages) processCertificate(packageFile *adb.PackageFile, keepOption string, buffer *acquisition.StreamingBuffer) (bool, error) { verified, cert, err := utils.VerifyCertificateFromReader(buffer.Reader()) if cert == nil { packageFile.CertificateError = "No certificate found" diff --git a/modules/paths.go b/modules/paths.go index e32cf81..bd81cfe 100644 --- a/modules/paths.go +++ b/modules/paths.go @@ -6,17 +6,11 @@ package modules import ( "fmt" - "io" - "os" "path" "path/filepath" "strings" ) -type pullToWriter interface { - PullToWriter(remotePath string, writer io.Writer) error -} - func relativeDeviceChild(deviceRoot, devicePath string) (string, error) { if deviceRoot == "" { return "", fmt.Errorf("device root cannot be empty") @@ -47,39 +41,3 @@ func relativeDeviceChild(deviceRoot, devicePath string) (string, error) { return rel, nil } - -func createRootFile(root *os.Root, rel string) (*os.File, error) { - localRel := filepath.FromSlash(rel) - if !filepath.IsLocal(localRel) { - return nil, fmt.Errorf("unsafe local path %q", rel) - } - - if err := root.MkdirAll(filepath.Dir(localRel), 0o755); err != nil { - return nil, fmt.Errorf("failed to create destination folders for %q: %v", rel, err) - } - - file, err := root.OpenFile(localRel, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - if err != nil { - return nil, fmt.Errorf("failed to create destination file %q: %v", rel, err) - } - - return file, nil -} - -func streamDeviceChildToRoot(root *os.Root, puller pullToWriter, rel, devicePath string) error { - file, err := createRootFile(root, rel) - if err != nil { - return err - } - defer file.Close() - - if err := puller.PullToWriter(devicePath, file); err != nil { - return err - } - - if err := file.Sync(); err != nil { - return fmt.Errorf("failed to sync destination file %q: %v", rel, err) - } - - return nil -} diff --git a/modules/paths_test.go b/modules/paths_test.go index ad5a83c..38e7ada 100644 --- a/modules/paths_test.go +++ b/modules/paths_test.go @@ -5,10 +5,6 @@ package modules import ( - "errors" - "os" - "path/filepath" - "runtime" "testing" ) @@ -82,70 +78,3 @@ func TestRelativeDeviceChild(t *testing.T) { }) } } - -func TestCreateRootFile(t *testing.T) { - rootDir := t.TempDir() - root, err := os.OpenRoot(rootDir) - if err != nil { - t.Fatalf("OpenRoot() error = %v", err) - } - defer root.Close() - - file, err := createRootFile(root, "nested/file.txt") - if err != nil { - t.Fatalf("createRootFile() error = %v", err) - } - if _, err := file.WriteString("ok"); err != nil { - t.Fatalf("WriteString() error = %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("Close() error = %v", err) - } - - got, err := os.ReadFile(filepath.Join(rootDir, "nested", "file.txt")) - if err != nil { - t.Fatalf("ReadFile() error = %v", err) - } - if string(got) != "ok" { - t.Fatalf("created file content = %q, want %q", got, "ok") - } - - file, err = createRootFile(root, "file.txt") - if err != nil { - t.Fatalf("createRootFile() root file error = %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("Close() root file error = %v", err) - } - - if file, err := createRootFile(root, "../escape"); err == nil { - file.Close() - t.Fatal("createRootFile() error = nil, want lexical traversal rejection") - } -} - -func TestCreateRootFileRejectsSymlinkEscape(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("symlink creation requires extra privileges on Windows") - } - - rootDir := t.TempDir() - outsideDir := t.TempDir() - if err := os.Symlink(outsideDir, filepath.Join(rootDir, "escape")); err != nil { - if errors.Is(err, os.ErrPermission) { - t.Skipf("symlink creation not permitted: %v", err) - } - t.Fatalf("Symlink() error = %v", err) - } - - root, err := os.OpenRoot(rootDir) - if err != nil { - t.Fatalf("OpenRoot() error = %v", err) - } - defer root.Close() - - if file, err := createRootFile(root, "escape/file.txt"); err == nil { - file.Close() - t.Fatal("createRootFile() error = nil, want symlink escape rejection") - } -} diff --git a/modules/processes.go b/modules/processes.go index 86f0208..06c9b84 100644 --- a/modules/processes.go +++ b/modules/processes.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Processes struct { - StoragePath string -} +type Processes struct{} func NewProcesses() *Processes { return &Processes{} @@ -24,11 +22,6 @@ func (p *Processes) Name() string { return "processes" } -func (p *Processes) InitStorage(storagePath string) error { - p.StoragePath = storagePath - return nil -} - func (p *Processes) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting list of running processes...") diff --git a/modules/root_binaries.go b/modules/root_binaries.go index 906689b..b2a0378 100644 --- a/modules/root_binaries.go +++ b/modules/root_binaries.go @@ -13,9 +13,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type RootBinaries struct { - StoragePath string -} +type RootBinaries struct{} func NewRootBinaries() *RootBinaries { return &RootBinaries{} @@ -25,11 +23,6 @@ func (r *RootBinaries) Name() string { return "root_binaries" } -func (r *RootBinaries) InitStorage(storagePath string) error { - r.StoragePath = storagePath - return nil -} - func (r *RootBinaries) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Checking for traces of rooting") root_binaries := []string{ diff --git a/modules/selinux.go b/modules/selinux.go index 7949132..b0fd9f1 100644 --- a/modules/selinux.go +++ b/modules/selinux.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type SELinux struct { - StoragePath string -} +type SELinux struct{} func NewSELinux() *SELinux { return &SELinux{} @@ -24,11 +22,6 @@ func (s *SELinux) Name() string { return "selinux" } -func (s *SELinux) InitStorage(storagePath string) error { - s.StoragePath = storagePath - return nil -} - func (s *SELinux) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting SELinux status...") diff --git a/modules/services.go b/modules/services.go index 029f138..bd2b995 100644 --- a/modules/services.go +++ b/modules/services.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Services struct { - StoragePath string -} +type Services struct{} func NewServices() *Services { return &Services{} @@ -24,11 +22,6 @@ func (s *Services) Name() string { return "services" } -func (s *Services) InitStorage(storagePath string) error { - s.StoragePath = storagePath - return nil -} - func (s *Services) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting list of services...") diff --git a/modules/settings.go b/modules/settings.go index cbcaee0..133c6cb 100644 --- a/modules/settings.go +++ b/modules/settings.go @@ -12,9 +12,7 @@ import ( "github.com/mvt-project/androidqf/log" ) -type Settings struct { - StoragePath string -} +type Settings struct{} func NewSettings() *Settings { return &Settings{} @@ -24,11 +22,6 @@ func (s *Settings) Name() string { return "settings" } -func (s *Settings) InitStorage(storagePath string) error { - s.StoragePath = storagePath - return nil -} - func (s *Settings) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting device settings...") diff --git a/modules/temp.go b/modules/temp.go index 9de493a..101326d 100644 --- a/modules/temp.go +++ b/modules/temp.go @@ -6,20 +6,14 @@ package modules import ( "fmt" - "os" "path" - "path/filepath" - "github.com/botherder/go-savetime/text" "github.com/mvt-project/androidqf/acquisition" "github.com/mvt-project/androidqf/adb" "github.com/mvt-project/androidqf/log" ) -type Temp struct { - StoragePath string - TempPath string -} +type Temp struct{} func NewTemp() *Temp { return &Temp{} @@ -29,37 +23,9 @@ func (t *Temp) Name() string { return "temp" } -func (t *Temp) InitStorage(storagePath string) error { - t.StoragePath = storagePath - t.TempPath = filepath.Join(storagePath, "tmp") - - // Only create directory in traditional mode - if storagePath != "" { - err := os.Mkdir(t.TempPath, 0o755) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create tmp folder: %v", err) - } - } - - return nil -} - func (t *Temp) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting files in tmp folder...") - streaming := acq.StreamingMode && acq.EncryptedWriter != nil - var localRoot *os.Root - var puller *acquisition.StreamingPuller - if !streaming { - var err error - localRoot, err = os.OpenRoot(t.TempPath) - if err != nil { - return fmt.Errorf("failed to open tmp output root: %v", err) - } - defer localRoot.Close() - puller = acquisition.NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, 100) - } - // TODO: Also check default tmp folders tmpFiles, err := adb.Client.ListFiles(acq.TmpDir, true) if err != nil { @@ -77,34 +43,21 @@ func (t *Temp) Run(acq *acquisition.Acquisition, fast bool) error { continue } - if streaming { - // Streaming mode: stream directly from ADB to encrypted zip without temp files - zipPath := path.Join("tmp", rel) - - // Create zip entry writer - writer, err := acq.EncryptedWriter.CreateFile(zipPath) - if err != nil { - log.Errorf("Failed to create zip entry for temp file %s: %v\n", file, err) - continue - } + zipPath := path.Join("tmp", rel) - // Stream temp file directly to encrypted zip using acquisition's streaming puller - err = acq.StreamingPuller.PullToWriter(file, writer) - if err != nil { - log.Errorf("Failed to stream temp file %s: %v\n", file, err) - continue - } + writer, err := acq.ZipWriter.CreateFile(zipPath) + if err != nil { + log.Errorf("Failed to create zip entry for temp file %s: %v\n", file, err) + continue + } - log.Debugf("Streamed temp file %s directly to encrypted archive as %s", file, zipPath) - } else { - // Traditional mode: stream into a file opened relative to t.TempPath. - if err := streamDeviceChildToRoot(localRoot, puller, rel, file); err != nil { - if !text.ContainsNoCase(err.Error(), "Permission denied") { - log.Errorf("Failed to pull temp file %s: %v\n", file, err) - } - continue - } + err = acq.StreamingPuller.PullToWriter(file, writer) + if err != nil { + log.Errorf("Failed to stream temp file %s: %v\n", file, err) + continue } + + log.Debugf("Streamed temp file %s directly to archive as %s", file, zipPath) } return nil } From 0554271ce56d5717b6f4b58e667e2078fbeb124f Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Tue, 23 Jun 2026 17:29:28 +0200 Subject: [PATCH 2/2] Fix APK archive collection regressions --- acquisition/acquisition.go | 16 ++- acquisition/streaming_buffer.go | 7 +- acquisition/streaming_buffer_test.go | 40 ++++++ modules/packages.go | 196 +++++++++++++++++++++------ modules/packages_test.go | 50 +++++++ 5 files changed, 266 insertions(+), 43 deletions(-) create mode 100644 acquisition/streaming_buffer_test.go create mode 100644 modules/packages_test.go diff --git a/acquisition/acquisition.go b/acquisition/acquisition.go index 01739da..c3d5326 100644 --- a/acquisition/acquisition.go +++ b/acquisition/acquisition.go @@ -8,6 +8,7 @@ package acquisition import ( "bytes" "encoding/json" + "errors" "fmt" "io" "strings" @@ -20,6 +21,8 @@ import ( "github.com/mvt-project/androidqf/utils" ) +const streamingPullerMemoryLimitMB = 500 + // Acquisition is the main object containing all phone information type Acquisition struct { UUID string `json:"uuid"` @@ -68,7 +71,7 @@ func New(path string) (*Acquisition, error) { acq.StoragePath = zipWriter.GetOutputPath() // Initialize streaming puller for direct operations. - acq.StreamingPuller = NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, 100) + acq.StreamingPuller = NewStreamingPuller(adb.Client.ExePath, adb.Client.Serial, streamingPullerMemoryLimitMB) // Create buffer for command.log (will be written to archive at completion). acq.logBuffer = new(bytes.Buffer) @@ -193,6 +196,17 @@ func (a *Acquisition) StreamAPKToZip(remotePath, zipPath string, processFunc fun // Pull APK data to memory buffer buffer, err := a.StreamingPuller.PullToBuffer(remotePath) if err != nil { + if errors.Is(err, ErrStreamingBufferMemoryLimit) && processFunc == nil { + log.Debugf("APK %s exceeded streaming buffer limit; streaming directly to archive", remotePath) + writer, err := a.ZipWriter.CreateFile(zipPath) + if err != nil { + return fmt.Errorf("failed to create zip entry for APK %q: %v", remotePath, err) + } + if err := a.StreamingPuller.PullToWriter(remotePath, writer); err != nil { + return fmt.Errorf("failed to stream APK %q to zip: %v", remotePath, err) + } + return nil + } return fmt.Errorf("failed to pull APK %q: %v", remotePath, err) } diff --git a/acquisition/streaming_buffer.go b/acquisition/streaming_buffer.go index 5e81d21..9dd437b 100644 --- a/acquisition/streaming_buffer.go +++ b/acquisition/streaming_buffer.go @@ -7,12 +7,15 @@ package acquisition import ( "bytes" + "errors" "fmt" "io" "os/exec" "strings" ) +var ErrStreamingBufferMemoryLimit = errors.New("streaming buffer memory limit exceeded") + // StreamingBuffer manages in-memory buffering for direct streaming operations type StreamingBuffer struct { buffer *bytes.Buffer @@ -32,7 +35,7 @@ func NewStreamingBuffer(maxMemoryMB int) *StreamingBuffer { // Write implements io.Writer interface with memory limit enforcement func (sb *StreamingBuffer) Write(p []byte) (int, error) { if sb.size+int64(len(p)) > sb.maxMem { - return 0, fmt.Errorf("write would exceed memory limit of %d bytes", sb.maxMem) + return 0, fmt.Errorf("%w: write would exceed memory limit of %d bytes", ErrStreamingBufferMemoryLimit, sb.maxMem) } n, err := sb.buffer.Write(p) @@ -98,7 +101,7 @@ func (sp *StreamingPuller) PullToBuffer(remotePath string) (*StreamingBuffer, er err := cmd.Run() if err != nil { - return nil, fmt.Errorf("failed to pull %q to buffer: %v", remotePath, err) + return nil, fmt.Errorf("failed to pull %q to buffer: %w", remotePath, err) } return buffer, nil diff --git a/acquisition/streaming_buffer_test.go b/acquisition/streaming_buffer_test.go new file mode 100644 index 0000000..cf4d8db --- /dev/null +++ b/acquisition/streaming_buffer_test.go @@ -0,0 +1,40 @@ +package acquisition + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestStreamingBufferMemoryLimitError(t *testing.T) { + buffer := NewStreamingBuffer(1) + + if _, err := buffer.Write(make([]byte, 1024*1024)); err != nil { + t.Fatalf("Write() at memory limit returned error: %v", err) + } + + _, err := buffer.Write([]byte("x")) + if !errors.Is(err, ErrStreamingBufferMemoryLimit) { + t.Fatalf("Write() error = %v, want ErrStreamingBufferMemoryLimit", err) + } +} + +func TestDefaultStreamingPullerMemoryLimit(t *testing.T) { + if streamingPullerMemoryLimitMB != 500 { + t.Fatalf("streamingPullerMemoryLimitMB = %d, want 500", streamingPullerMemoryLimitMB) + } +} + +func TestPullToBufferPreservesMemoryLimitError(t *testing.T) { + fakeADB := filepath.Join(t.TempDir(), "adb") + if err := os.WriteFile(fakeADB, []byte("#!/bin/sh\nhead -c 1048577 /dev/zero\n"), 0o700); err != nil { + t.Fatalf("WriteFile(fake adb) error = %v", err) + } + + puller := NewStreamingPuller(fakeADB, "", 1) + _, err := puller.PullToBuffer("/data/app/large.apk") + if !errors.Is(err, ErrStreamingBufferMemoryLimit) { + t.Fatalf("PullToBuffer() error = %v, want ErrStreamingBufferMemoryLimit", err) + } +} diff --git a/modules/packages.go b/modules/packages.go index eafa995..c216f2a 100644 --- a/modules/packages.go +++ b/modules/packages.go @@ -5,7 +5,9 @@ package modules import ( + "errors" "fmt" + "os" "path/filepath" "strings" @@ -64,6 +66,26 @@ func (p *Packages) generateZipPath(packageName, filePath string) (string, error) return "apks/" + base, nil } +func reserveUniqueZipPath(zipPath string, used map[string]struct{}) string { + if used == nil { + return zipPath + } + if _, ok := used[zipPath]; !ok { + used[zipPath] = struct{}{} + return zipPath + } + + ext := filepath.Ext(zipPath) + base := strings.TrimSuffix(zipPath, ext) + for counter := 1; ; counter++ { + candidate := fmt.Sprintf("%s_%d%s", base, counter, ext) + if _, ok := used[candidate]; !ok { + used[candidate] = struct{}{} + return candidate + } + } +} + func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { log.Info("Collecting information on installed apps. This might take a while...") @@ -93,21 +115,18 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { var keepOption string - if acq.ZipWriter != nil && acq.ZipWriter.IsEncrypted() { - keepOption = apkKeepAll - } else { - log.Info("Would you like to remove copies of apps signed with a trusted certificate to limit the size of the output archive?") - promptAll := promptui.Select{ - Label: "Remove", - Items: []string{apkRemoveTrusted, apkKeepAll}, - } - _, keepOption, err = promptAll.Run() - if err != nil { - return fmt.Errorf("failed to make selection for download option: %v", - err) - } + log.Info("Would you like to remove copies of apps signed with a trusted certificate to limit the size of the output archive?") + promptAll := promptui.Select{ + Label: "Remove", + Items: []string{apkRemoveTrusted, apkKeepAll}, + } + _, keepOption, err = promptAll.Run() + if err != nil { + return fmt.Errorf("failed to make selection for download option: %v", + err) } + usedZipPaths := make(map[string]struct{}) for ip := 0; ip < len(packages); ip++ { // If we the user did not request to download all packages and if // the package is marked as system, we skip it. @@ -120,7 +139,7 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { for ipf := 0; ipf < len(packages[ip].Files); ipf++ { packageFile := &packages[ip].Files[ipf] - if err := p.processAPKStreaming(packages[ip].Name, packageFile, keepOption, acq); err != nil { + if err := p.processAPKStreaming(packages[ip].Name, packageFile, keepOption, acq, usedZipPaths); err != nil { log.Debugf("ERROR: failed to process APK %s: %v", packageFile.Path, err) continue } @@ -131,7 +150,7 @@ func (p *Packages) Run(acq *acquisition.Acquisition, fast bool) error { return saveDataToAcquisition(acq, "packages.json", &packages) } -func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.PackageFile, keepOption string, acq *acquisition.Acquisition) error { +func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.PackageFile, keepOption string, acq *acquisition.Acquisition, usedZipPaths map[string]struct{}) error { zipPath, err := p.generateZipPath(packageName, packageFile.Path) if err != nil { log.Errorf("Skipping APK with unsafe path %q: %v", packageFile.Path, err) @@ -139,40 +158,137 @@ func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.Pack return nil } - if acq.ZipWriter != nil && acq.ZipWriter.IsEncrypted() { - log.Debugf("Skipping certificate check for encrypted archive: %s", packageFile.Path) - err = acq.StreamAPKToZip(packageFile.Path, zipPath, nil) - if err != nil { - packageFile.Error = fmt.Sprintf("Failed to stream to archive: %v", err) - return err - } - } else { - buffer, err := acq.StreamingPuller.PullToBuffer(packageFile.Path) - if err != nil { - packageFile.Error = fmt.Sprintf("Failed to pull APK: %v", err) - return err - } + buffer, err := acq.StreamingPuller.PullToBuffer(packageFile.Path) + if err != nil { + if errors.Is(err, acquisition.ErrStreamingBufferMemoryLimit) { + if acq.ZipWriter != nil && acq.ZipWriter.IsEncrypted() { + zipPath, err := p.processLargeEncryptedAPK(packageFile, zipPath, acq, usedZipPaths) + if err != nil { + packageFile.Error = err.Error() + return err + } + log.Debugf("Streamed %s directly to archive as %s", packageFile.Path, zipPath) + return nil + } - shouldSkip, err := p.processCertificate(packageFile, keepOption, buffer) - if err != nil { - packageFile.Error = fmt.Sprintf("Certificate processing failed: %v", err) - return err - } - if shouldSkip { - log.Debugf("Trusted APK skipped for streaming: %s", packageFile.Path) + zipPath, skipped, err := p.processLargeAPKFromTemp(packageFile, keepOption, zipPath, acq, usedZipPaths) + if err != nil { + packageFile.Error = err.Error() + return err + } + if skipped { + log.Debugf("Trusted APK skipped for streaming: %s", packageFile.Path) + return nil + } + log.Debugf("Streamed %s directly to archive as %s", packageFile.Path, zipPath) return nil } - err = acq.ZipWriter.CreateFileFromReader(zipPath, buffer.Reader()) - if err != nil { - packageFile.Error = fmt.Sprintf("Failed to stream to archive: %v", err) - return err - } + packageFile.Error = fmt.Sprintf("Failed to pull APK: %v", err) + return err + } + + shouldSkip, err := p.processCertificate(packageFile, keepOption, buffer) + if err != nil { + packageFile.Error = fmt.Sprintf("Certificate processing failed: %v", err) + return err + } + if shouldSkip { + log.Debugf("Trusted APK skipped for streaming: %s", packageFile.Path) + return nil + } + zipPath = reserveUniqueZipPath(zipPath, usedZipPaths) + err = acq.ZipWriter.CreateFileFromReader(zipPath, buffer.Reader()) + if err != nil { + packageFile.Error = fmt.Sprintf("Failed to stream to archive: %v", err) + return err } log.Debugf("Streamed %s directly to archive as %s", packageFile.Path, zipPath) return nil } +func (p *Packages) processLargeEncryptedAPK(packageFile *adb.PackageFile, zipPath string, acq *acquisition.Acquisition, usedZipPaths map[string]struct{}) (string, error) { + log.Debugf("APK %s exceeded streaming buffer limit; streaming directly to encrypted archive without certificate check", packageFile.Path) + + packageFile.CertificateError = "Skipped certificate check: APK exceeds streaming buffer limit" + packageFile.VerifiedCertificate = false + + zipPath = reserveUniqueZipPath(zipPath, usedZipPaths) + writer, err := acq.ZipWriter.CreateFile(zipPath) + if err != nil { + return "", fmt.Errorf("failed to create zip entry for APK: %v", err) + } + if err := acq.StreamingPuller.PullToWriter(packageFile.Path, writer); err != nil { + return "", fmt.Errorf("failed to stream APK to archive: %v", err) + } + return zipPath, nil +} + +func (p *Packages) processLargeAPKFromTemp(packageFile *adb.PackageFile, keepOption, zipPath string, acq *acquisition.Acquisition, usedZipPaths map[string]struct{}) (string, bool, error) { + log.Debugf("APK %s exceeded streaming buffer limit; using temporary file for certificate check", packageFile.Path) + + tempFile, err := os.CreateTemp("", "androidqf-apk-*.apk") + if err != nil { + return "", false, fmt.Errorf("failed to create temporary APK file: %v", err) + } + tempPath := tempFile.Name() + defer os.Remove(tempPath) + + if err := acq.StreamingPuller.PullToWriter(packageFile.Path, tempFile); err != nil { + tempFile.Close() + return "", false, fmt.Errorf("failed to pull APK to temporary file: %v", err) + } + if err := tempFile.Close(); err != nil { + return "", false, fmt.Errorf("failed to close temporary APK file: %v", err) + } + + shouldSkip, err := p.processCertificateFromPath(packageFile, keepOption, tempPath) + if err != nil { + return "", false, fmt.Errorf("certificate processing failed: %v", err) + } + if shouldSkip { + return "", true, nil + } + + zipPath = reserveUniqueZipPath(zipPath, usedZipPaths) + if err := acq.ZipWriter.CreateFileFromPath(zipPath, tempPath); err != nil { + return "", false, fmt.Errorf("failed to stream to archive: %v", err) + } + return zipPath, false, nil +} + +func (p *Packages) processCertificateFromPath(packageFile *adb.PackageFile, keepOption, path string) (bool, error) { + verified, cert, err := utils.VerifyCertificate(path) + if cert == nil { + packageFile.CertificateError = "No certificate found" + if err != nil { + packageFile.CertificateError = err.Error() + } + packageFile.VerifiedCertificate = false + return false, nil + } + + // Set certificate information + packageFile.Certificate = *cert + packageFile.VerifiedCertificate = verified + + if err != nil { + packageFile.CertificateError = err.Error() + } else { + packageFile.CertificateError = "" + } + + // Check if certificate is trusted and should be removed + if utils.IsTrusted(*cert) { + packageFile.TrustedCertificate = true + if keepOption == apkRemoveTrusted { + return true, nil // Skip this APK + } + } + + return false, nil +} + // processCertificate handles certificate verification and returns whether APK should be skipped func (p *Packages) processCertificate(packageFile *adb.PackageFile, keepOption string, buffer *acquisition.StreamingBuffer) (bool, error) { verified, cert, err := utils.VerifyCertificateFromReader(buffer.Reader()) diff --git a/modules/packages_test.go b/modules/packages_test.go new file mode 100644 index 0000000..d38005c --- /dev/null +++ b/modules/packages_test.go @@ -0,0 +1,50 @@ +package modules + +import "testing" + +func TestReserveUniqueZipPathAddsCounterForDuplicateAPKs(t *testing.T) { + used := make(map[string]struct{}) + paths := []string{ + "apks/com.example.apk", + "apks/com.example.apk", + "apks/com.example.apk", + } + want := []string{ + "apks/com.example.apk", + "apks/com.example_1.apk", + "apks/com.example_2.apk", + } + + for i, path := range paths { + if got := reserveUniqueZipPath(path, used); got != want[i] { + t.Fatalf("reserveUniqueZipPath(%q) = %q, want %q", path, got, want[i]) + } + } +} + +func TestGenerateZipPathWithReservationKeepsSplitAPKNamesUnique(t *testing.T) { + packages := NewPackages() + used := make(map[string]struct{}) + files := []string{ + "/data/app/com.example-1/base.apk", + "/data/app/com.example-1/split_config.en.apk", + "/data/app/~~abc==/base.apk", + "/data/app/~~abc==/split_config.en.apk", + } + want := []string{ + "apks/com.example.apk", + "apks/com.example_1.apk", + "apks/com.example_base.apk", + "apks/com.example_split_config.en.apk", + } + + for i, file := range files { + zipPath, err := packages.generateZipPath("com.example", file) + if err != nil { + t.Fatalf("generateZipPath(%q) error = %v", file, err) + } + if got := reserveUniqueZipPath(zipPath, used); got != want[i] { + t.Fatalf("reserved zip path for %q = %q, want %q", file, got, want[i]) + } + } +}