Add pet clustering with feedback, naming, and search

mobile/apps/photos/ios/Podfile.lock

@@ -18,6 +18,8 @@ PODS:
     - Flutter
   - ente_photos_rust (0.0.1):
     - Flutter
+  - ente_qr (0.0.1):
+    - Flutter
   - ente_rust (0.0.1):
     - Flutter
   - ffmpeg_kit_custom (6.0.3)
@@ -271,6 +273,7 @@ DEPENDENCIES:
   - dart_ui_isolate (from `.symlinks/plugins/dart_ui_isolate/ios`)
   - device_info_plus (from `.symlinks/plugins/device_info_plus/ios`)
   - ente_photos_rust (from `.symlinks/plugins/ente_photos_rust/ios`)
+  - ente_qr (from `.symlinks/plugins/ente_qr/ios`)
   - ente_rust (from `.symlinks/plugins/ente_rust/ios`)
   - ffmpeg_kit_flutter (from `.symlinks/plugins/ffmpeg_kit_flutter/ios`)
   - file_saver (from `.symlinks/plugins/file_saver/ios`)
@@ -369,6 +372,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/device_info_plus/ios"
   ente_photos_rust:
     :path: ".symlinks/plugins/ente_photos_rust/ios"
+  ente_qr:
+    :path: ".symlinks/plugins/ente_qr/ios"
   ente_rust:
     :path: ".symlinks/plugins/ente_rust/ios"
   ffmpeg_kit_flutter:
@@ -490,6 +495,7 @@ SPEC CHECKSUMS:
   dart_ui_isolate: 46f6714abe6891313267153ef6f9748d8ecfcab1
   device_info_plus: 21fcca2080fbcd348be798aa36c3e5ed849eefbe
   ente_photos_rust: 79732f3ba67df7b0fb22089627afb80d692c2b2d
+  ente_qr: 2a30f91f9e938c229ebcbc98a6f8f018d1ac665d
   ente_rust: 9c976e3d083f0ce90f31693deb99629b28d8fe4c
   ffmpeg_kit_custom: 682b4f2f1ff1f8abae5a92f6c3540f2441d5be99
   ffmpeg_kit_flutter: 915b345acc97d4142e8a9a8549d177ff10f043f5

mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj

@@ -637,6 +637,7 @@
 				"${BUILT_PRODUCTS_DIR}/dart_ui_isolate/dart_ui_isolate.framework",
 				"${BUILT_PRODUCTS_DIR}/device_info_plus/device_info_plus.framework",
 				"${BUILT_PRODUCTS_DIR}/ente_photos_rust/ente_photos_rust.framework",
+				"${BUILT_PRODUCTS_DIR}/ente_qr/ente_qr.framework",
 				"${BUILT_PRODUCTS_DIR}/ente_rust/ente_rust.framework",
 				"${BUILT_PRODUCTS_DIR}/file_saver/file_saver.framework",
 				"${BUILT_PRODUCTS_DIR}/flutter_email_sender/flutter_email_sender.framework",
@@ -739,6 +740,7 @@
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/dart_ui_isolate.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/device_info_plus.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_photos_rust.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_qr.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_rust.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/file_saver.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_email_sender.framework",

mobile/apps/photos/lib/db/ml/db.dart

@@ -106,6 +106,19 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
     createPetBodiesTable,
     createPetFaceVectorIdMappingTable,
     createPetBodyVectorIdMappingTable,
+    createPetIndexedFilesTable,
+    dropPetIndexedFilesTable,
+    dropPetFaceClustersTable,
+    dropPetClusterSummaryTable,
+    dropPetFacesTable, // 23: recreate with UNIQUE faceVectorId
+    createPetFacesTable,
+    dropPetBodiesTable, // 25: recreate with UNIQUE bodyVectorId
+    createPetBodiesTable,
+    createPetFaceClustersTable, // 26: recreate (dropped in step 20)
+    petFcClusterIDIndex, // 27
+    createPetClusterSummaryTable, // 28: recreate (dropped in step 21)
+    createPetNamesTable, // 29: pet name persistence
+    createNotPetFeedbackTable, // 30: not-pet feedback for manual reassignment
   ];
   static const List<String> _offlineMigrationScripts = [
     ..._defaultMigrationScripts,
@@ -2460,21 +2473,28 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
       }
     }
 
-    // Delete mapping table entries and detection rows atomically.
+    // Delete mapping table entries, cluster assignments, and detection rows
+    // atomically. Also clean up cluster summaries that become empty.
     await db.writeTransaction((tx) async {
       if (faceIdsToRemove.isNotEmpty) {
-        final placeholders = List.filled(faceIdsToRemove.length, '?').join(',');
+        final fpH = List.filled(faceIdsToRemove.length, '?').join(',');
         await tx.execute(
           'DELETE FROM $petFaceVectorIdMappingTable '
-          'WHERE $petFaceIDColumn IN ($placeholders)',
+          'WHERE $petFaceIDColumn IN ($fpH)',
+          faceIdsToRemove,
+        );
+        // Remove cluster assignments for deleted faces
+        await tx.execute(
+          'DELETE FROM $petFaceClustersTable '
+          'WHERE $petFaceIDColumn IN ($fpH)',
           faceIdsToRemove,
         );
       }
       if (bodyIdsToRemove.isNotEmpty) {
-        final placeholders = List.filled(bodyIdsToRemove.length, '?').join(',');
+        final bpH = List.filled(bodyIdsToRemove.length, '?').join(',');
         await tx.execute(
           'DELETE FROM $petBodyVectorIdMappingTable '
-          'WHERE $petBodyIDColumn IN ($placeholders)',
+          'WHERE $petBodyIDColumn IN ($bpH)',
           bodyIdsToRemove,
         );
       }
@@ -2486,6 +2506,12 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
         'DELETE FROM $petBodiesTable WHERE $fileIDColumn IN ($placeholders)',
         fileIDs,
       );
+      // Clean up cluster summaries that no longer have any face assignments
+      await tx.execute(
+        'DELETE FROM $petClusterSummaryTable '
+        'WHERE $clusterIDColumn NOT IN '
+        '(SELECT DISTINCT $clusterIDColumn FROM $petFaceClustersTable)',
+      );
     });
   }
 

mobile/apps/photos/lib/db/ml/schema.dart

@@ -212,6 +212,7 @@ const createPetFacesTable = '''CREATE TABLE IF NOT EXISTS $petFacesTable (
 ''';
 
 const deletePetFacesTable = 'DELETE FROM $petFacesTable';
+const dropPetFacesTable = 'DROP TABLE IF EXISTS $petFacesTable';
 
 // ── Pet Bodies Table ──
 
@@ -235,6 +236,7 @@ const createPetBodiesTable = '''CREATE TABLE IF NOT EXISTS $petBodiesTable (
 ''';
 
 const deletePetBodiesTable = 'DELETE FROM $petBodiesTable';
+const dropPetBodiesTable = 'DROP TABLE IF EXISTS $petBodiesTable';
 
 // ── Vector ID Mapping Tables ──
 
@@ -265,3 +267,73 @@ CREATE TABLE IF NOT EXISTS $petBodyVectorIdMappingTable (
 
 const deletePetBodyVectorIdMappingTable =
     'DELETE FROM $petBodyVectorIdMappingTable';
+
+// ── Pet Face Clusters Table ──
+
+const petFaceClustersTable = 'pet_face_clusters';
+
+const createPetFaceClustersTable = '''
+CREATE TABLE IF NOT EXISTS $petFaceClustersTable (
+  $petFaceIDColumn TEXT NOT NULL PRIMARY KEY,
+  $clusterIDColumn TEXT NOT NULL
+);
+''';
+
+const dropPetFaceClustersTable = 'DROP TABLE IF EXISTS $petFaceClustersTable';
+
+const petFcClusterIDIndex =
+    'CREATE INDEX IF NOT EXISTS idx_pet_fc_cluster ON $petFaceClustersTable ($clusterIDColumn)';
+
+// ── Pet Cluster Summary Table ──
+
+const petClusterSummaryTable = 'pet_cluster_summary';
+
+const createPetClusterSummaryTable = '''
+CREATE TABLE IF NOT EXISTS $petClusterSummaryTable (
+  $clusterIDColumn TEXT NOT NULL PRIMARY KEY,
+  $avgColumn BLOB NOT NULL,
+  $countColumn INTEGER NOT NULL DEFAULT 0,
+  $speciesColumn INTEGER NOT NULL DEFAULT -1
+);
+''';
+
+const dropPetClusterSummaryTable =
+    'DROP TABLE IF EXISTS $petClusterSummaryTable';
+
+// ── Pet Names Table ──
+
+const petNamesTable = 'pet_names';
+const petNameColumn = 'name';
+
+const createPetNamesTable = '''
+CREATE TABLE IF NOT EXISTS $petNamesTable (
+  $clusterIDColumn TEXT NOT NULL PRIMARY KEY,
+  $petNameColumn TEXT NOT NULL DEFAULT '',
+  $speciesColumn INTEGER NOT NULL DEFAULT -1
+);
+''';
+
+// ── Not-Pet Feedback Table ──
+
+const notPetFeedbackTable = 'not_pet_feedback';
+
+const createNotPetFeedbackTable = '''
+CREATE TABLE IF NOT EXISTS $notPetFeedbackTable (
+  $clusterIDColumn TEXT NOT NULL,
+  $petFaceIDColumn TEXT NOT NULL,
+  PRIMARY KEY($clusterIDColumn, $petFaceIDColumn)
+);
+''';
+
+// ── Pet Indexing Tracking Table ──
+
+const petIndexedFilesTable = 'pet_indexed_files';
+
+const createPetIndexedFilesTable = '''
+CREATE TABLE IF NOT EXISTS $petIndexedFilesTable (
+  $fileIDColumn INTEGER NOT NULL PRIMARY KEY,
+  $mlVersionColumn INTEGER NOT NULL DEFAULT -1
+);
+''';
+
+const dropPetIndexedFilesTable = 'DROP TABLE IF EXISTS $petIndexedFilesTable';

mobile/apps/photos/lib/events/pets_changed_event.dart

@@ -0,0 +1,10 @@
+import "package:photos/events/event.dart";
+
+class PetsChangedEvent extends Event {
+  final String source;
+
+  PetsChangedEvent({this.source = ""});
+
+  @override
+  String get reason => '$runtimeType{"via": $source}';
+}

mobile/apps/photos/lib/l10n/intl_en.arb

@@ -1508,6 +1508,7 @@
   },
   "faces": "Faces",
   "people": "People",
+  "peopleAndPets": "People & Pets",
   "contents": "Contents",
   "addNew": "Add new",
   "@addNew": {
@@ -2904,5 +2905,8 @@
   "offlineNameFaceBannerSubtitle": "Sign up to tag them and easily find their photos later.",
   "signUp": "Sign up",
   "dog": "Dog",
-  "cat": "Cat"
+  "cat": "Cat",
+  "pet": "Pet",
+  "moveTo": "Move to",
+  "notThisPet": "Not this pet"
 }

mobile/apps/photos/lib/models/search/search_constants.dart

@@ -3,5 +3,8 @@ const kPersonWidgetKey = 'person_widget_key';
 const kPersonPinned = 'person_pinned';
 const kClusterParamId = 'cluster_id';
 const kFileID = 'file_id';
+const kPetClusterParamId = 'pet_cluster_id';
+const kPetSpecies = 'pet_species';
+const kPetHasCustomName = 'pet_has_custom_name';
 const kContactEmail = 'contact_email';
 const kContactCollections = 'contact_collections';

mobile/apps/photos/lib/models/search/search_types.dart

@@ -9,13 +9,13 @@ import "package:photos/events/event.dart";
 import "package:photos/events/location_tag_updated_event.dart";
 import "package:photos/events/magic_cache_updated_event.dart";
 import "package:photos/events/people_changed_event.dart";
+import "package:photos/events/pets_changed_event.dart";
 import "package:photos/generated/l10n.dart";
 import "package:photos/models/collection/collection.dart";
 import "package:photos/models/collection/collection_items.dart";
 import "package:photos/models/search/search_result.dart";
 import "package:photos/models/typedefs.dart";
 import "package:photos/services/collections_service.dart";
-import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
 import "package:photos/services/search_service.dart";
 import "package:photos/ui/viewer/gallery/collection_page.dart";
 import "package:photos/ui/viewer/location/add_location_sheet.dart";
@@ -252,12 +252,7 @@ extension SectionTypeExtensions on SectionType {
   }) {
     switch (this) {
       case SectionType.face:
-        return SearchService.instance.getAllFace(
-          limit,
-          minClusterSize: limit == null
-              ? kMinimumClusterSizeAllFaces
-              : kMinimumClusterSizeSearchResult,
-        );
+        return SearchService.instance.getAllPeopleAndPets(limit);
       case SectionType.magic:
         return SearchService.instance.getMagicSectionResults(context!);
       case SectionType.wrapped:
@@ -290,7 +285,10 @@ extension SectionTypeExtensions on SectionType {
       case SectionType.album:
         return [Bus.instance.on<CollectionUpdatedEvent>()];
       case SectionType.face:
-        return [Bus.instance.on<PeopleChangedEvent>()];
+        return [
+          Bus.instance.on<PeopleChangedEvent>(),
+          Bus.instance.on<PetsChangedEvent>(),
+        ];
       case SectionType.contacts:
         return [Bus.instance.on<PeopleChangedEvent>()];
       default:
@@ -302,6 +300,8 @@ extension SectionTypeExtensions on SectionType {
   ///events listened to in AllSectionsExampleState.
   List<Stream<Event>> sectionUpdateEvents() {
     switch (this) {
+      case SectionType.face:
+        return [Bus.instance.on<PetsChangedEvent>()];
       case SectionType.location:
         return [Bus.instance.on<LocationTagUpdatedEvent>()];
       case SectionType.magic:

mobile/apps/photos/lib/services/machine_learning/ml_service.dart

@@ -13,6 +13,7 @@ import "package:photos/db/ml/db_pet_model_mappers.dart";
 import "package:photos/db/offline_files_db.dart";
 import "package:photos/events/compute_control_event.dart";
 import "package:photos/events/people_changed_event.dart";
+import "package:photos/events/pets_changed_event.dart";
 import "package:photos/models/ml/clip.dart";
 import "package:photos/models/ml/face/face.dart";
 import "package:photos/models/ml/ml_versions.dart";
@@ -24,6 +25,7 @@ import "package:photos/services/machine_learning/face_ml/face_detection/detectio
 import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
 import "package:photos/services/machine_learning/ml_indexing_isolate.dart";
 import 'package:photos/services/machine_learning/ml_result.dart';
+import "package:photos/services/machine_learning/pet_ml/pet_clustering_service.dart";
 import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";
 import "package:photos/services/search_service.dart";
 import "package:photos/services/video_preview_service.dart";
@@ -215,6 +217,14 @@ class MLService {
       if ((await mlDataDB.getUnclusteredFaceCount()) > 0) {
         await clusterAllImages();
       }
+      if (_hasModeChanged(mode)) {
+        _logger.info("App mode changed during ML run, stopping");
+        return;
+      }
+      // Pet clustering (internal users only)
+      if (flagService.petEnabled && localSettings.petRecognitionEnabled) {
+        await _clusterPets(mlDataDB, mode);
+      }
       if (_mlControllerStatus == true) {
         if (_hasModeChanged(mode)) {
           _logger.info("App mode changed during ML run, stopping");
@@ -533,6 +543,19 @@ class MLService {
     }
   }
 
+  Future<void> _clusterPets(MLDataDB mlDataDB, MLMode mode) async {
+    if (_shouldPauseIndexingAndClustering) return;
+    try {
+      await PetClusteringService.instance.clusterPets(
+        mlDataDB: mlDataDB,
+        isOffline: mode == MLMode.offline,
+      );
+      Bus.instance.fire(PetsChangedEvent(source: "clustering"));
+    } catch (e, s) {
+      _logger.severe("Pet clustering failed", e, s);
+    }
+  }
+
   Future<bool> processImage(FileMLInstruction instruction) async {
     bool actuallyRanML = false;
 
@@ -692,6 +715,9 @@ class MLService {
           );
         }
       }
+      if (instruction.shouldRunPets) {
+        await mlDataDB.markPetIndexed(result.fileId, petMlVersion);
+      }
       _logger.info("ML result for fileID ${result.fileId} stored remote+local");
       return actuallyRanML;
     } catch (e, s) {
@@ -730,6 +756,9 @@ class MLService {
             [DBPetFace.empty(instruction.fileKey, error: true)],
           );
         }
+        if (instruction.shouldRunPets) {
+          await mlDataDB.markPetIndexed(instruction.fileKey, petMlVersion);
+        }
         return true;
       }
       _logger.severe(