From be218c7170ce3acde19540148e944062ff835289 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Mar 2026 12:15:24 +0530 Subject: [PATCH 01/18] Improve progressive face thumbnails and remove redundant quality decode --- .../face_thumbnail_generator.dart | 50 ++- .../ui/viewer/people/person_face_widget.dart | 338 ++++++++++++++---- .../lib/utils/face/face_thumbnail_cache.dart | 75 +++- .../utils/face/face_thumbnail_quality.dart | 176 +++++++++ .../apps/photos/lib/utils/image_ml_util.dart | 40 ++- .../lib/utils/isolate/isolate_operations.dart | 47 +++ .../ente_feature_flag/lib/src/service.dart | 2 + .../rust/src/api/image_processing_api.rs | 32 +- 8 files changed, 676 insertions(+), 84 deletions(-) create mode 100644 mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart diff --git a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart index c7de0fab49e..f6310d75797 100644 --- a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart +++ b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart @@ -8,6 +8,18 @@ import "package:photos/utils/image_ml_util.dart"; import "package:photos/utils/isolate/isolate_operations.dart"; import "package:photos/utils/isolate/super_isolate.dart"; +class FaceThumbnailGenerationResult { + final List thumbnails; + final int sourceWidth; + final int sourceHeight; + + const FaceThumbnailGenerationResult({ + required this.thumbnails, + required this.sourceWidth, + required this.sourceHeight, + }); +} + @pragma('vm:entry-point') class FaceThumbnailGenerator extends SuperIsolate { @override @@ -35,6 +47,18 @@ class FaceThumbnailGenerator extends SuperIsolate { Future> generateFaceThumbnails( String imagePath, List faceBoxes, + ) async { + final result = await generateFaceThumbnailsWithSourceDimensions( + imagePath, + faceBoxes, + ); + return result.thumbnails; + } + + Future + generateFaceThumbnailsWithSourceDimensions( + String imagePath, + List faceBoxes, ) async { try { final useRustForFaceThumbnails = flagService.useRustForFaceThumbnails; @@ -43,25 +67,39 @@ class FaceThumbnailGenerator extends SuperIsolate { ); final List> faceBoxesJson = faceBoxes.map((box) => box.toJson()).toList(); - final List faces = await runInIsolate( - IsolateOperation.generateFaceThumbnails, + final Map rawResult = await runInIsolate( + IsolateOperation.generateFaceThumbnailsWithSourceDimensions, { 'imagePath': imagePath, 'faceBoxesList': faceBoxesJson, 'useRustForFaceThumbnails': useRustForFaceThumbnails, }, - ).then((value) => value.cast()); - _logger.info("Generated face thumbnails"); + ).then((value) => Map.from(value)); + final List faces = + (rawResult['thumbnails'] as List).cast(); + final int sourceWidth = rawResult['sourceWidth'] as int? ?? 0; + final int sourceHeight = rawResult['sourceHeight'] as int? ?? 0; + _logger.info( + "Generated face thumbnails with source dimensions ${sourceWidth}x$sourceHeight", + ); if (useRustForFaceThumbnails) { // Rust path already emits compressed JPEG bytes. - return faces; + return FaceThumbnailGenerationResult( + thumbnails: faces, + sourceWidth: sourceWidth, + sourceHeight: sourceHeight, + ); } final compressedFaces = await compressFaceThumbnails({'listPngBytes': faces}); _logger.fine( "Compressed face thumbnails from sizes ${faces.map((e) => e.length / 1024).toList()} to ${compressedFaces.map((e) => e.length / 1024).toList()} kilobytes", ); - return compressedFaces; + return FaceThumbnailGenerationResult( + thumbnails: compressedFaces, + sourceWidth: sourceWidth, + sourceHeight: sourceHeight, + ); } catch (e, s) { _logger.severe("Failed to generate face thumbnails", e, s); diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 70fa06b0625..4c8456a4a1a 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -1,23 +1,27 @@ -import "dart:typed_data"; - -import "package:flutter/foundation.dart" show kDebugMode; -import "package:flutter/material.dart"; -import "package:logging/logging.dart"; -import "package:photos/db/files_db.dart"; -import "package:photos/db/ml/db.dart"; -import "package:photos/db/offline_files_db.dart"; +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart' show kDebugMode; +import 'package:flutter/material.dart'; +import 'package:logging/logging.dart'; +import 'package:photos/core/cache/lru_map.dart'; +import 'package:photos/db/files_db.dart'; +import 'package:photos/db/ml/db.dart'; +import 'package:photos/db/offline_files_db.dart'; import 'package:photos/models/file/file.dart'; -import "package:photos/models/ml/face/face.dart"; -import "package:photos/models/ml/face/person.dart"; -import "package:photos/service_locator.dart" show isOfflineMode; -import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; -import "package:photos/services/machine_learning/ml_result.dart"; -import "package:photos/services/search_service.dart"; -import "package:photos/theme/ente_theme.dart"; -import "package:photos/ui/common/loading_widget.dart"; -import "package:photos/utils/face/face_thumbnail_cache.dart"; - -final _logger = Logger("PersonFaceWidget"); +import 'package:photos/models/ml/face/face.dart'; +import 'package:photos/models/ml/face/person.dart'; +import 'package:photos/service_locator.dart' show flagService, isOfflineMode; +import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import 'package:photos/services/machine_learning/face_ml/person/person_service.dart'; +import 'package:photos/services/machine_learning/ml_result.dart'; +import 'package:photos/services/search_service.dart'; +import 'package:photos/theme/ente_theme.dart'; +import 'package:photos/ui/common/loading_widget.dart'; +import 'package:photos/utils/face/face_thumbnail_cache.dart'; +import 'package:photos/utils/face/face_thumbnail_quality.dart'; + +final _logger = Logger('PersonFaceWidget'); class PersonFaceWidget extends StatefulWidget { final String? personId; @@ -48,7 +52,7 @@ class PersonFaceWidget extends StatefulWidget { super.key, }) : assert( personId != null || clusterID != null, - "PersonFaceWidget requires either personId or clusterID to be non-null", + 'PersonFaceWidget requires either personId or clusterID to be non-null', ); @override @@ -59,13 +63,24 @@ class _PersonFaceWidgetState extends State with AutomaticKeepAliveClientMixin { Future? faceCropFuture; EnteFile? fileForFaceCrop; + Face? _faceForFaceCrop; int? _faceCropFileId; String? _personName; bool _showingFallback = false; bool _fallbackEverUsed = false; + bool _disposed = false; + int _upgradeGeneration = 0; + + static final LRUMap _clusterToFileCountCache = LRUMap(1000); bool get isPerson => widget.personId != null; + bool get _shouldUseProgressiveStrategy { + return widget.useFullFile && + !isOfflineMode && + flagService.progressivePersonFaceThumbnailsEnabled; + } + @override bool get wantKeepAlive => widget.keepAlive; @@ -77,12 +92,16 @@ class _PersonFaceWidgetState extends State @override void dispose() { + _disposed = true; + _upgradeGeneration += 1; if (_faceCropFileId != null) { - checkStopTryingToGenerateFaceThumbnails( - _faceCropFileId!, - useFullFile: widget.useFullFile, - ); - if (_fallbackEverUsed) { + if (widget.useFullFile) { + checkStopTryingToGenerateFaceThumbnails( + _faceCropFileId!, + useFullFile: true, + ); + } + if (_fallbackEverUsed || _shouldUseProgressiveStrategy) { checkStopTryingToGenerateFaceThumbnails( _faceCropFileId!, useFullFile: false, @@ -134,7 +153,7 @@ class _PersonFaceWidgetState extends State borderRadius: BorderRadius.circular(4), ), child: const Text( - "(T)", + '(T)', style: TextStyle( color: Colors.white, fontSize: 10, @@ -154,13 +173,13 @@ class _PersonFaceWidgetState extends State } if (snapshot.hasError) { _logger.severe( - "Error getting cover face for person", + 'Error getting cover face for person', snapshot.error, snapshot.stackTrace, ); } else { _logger.severe( - "faceCropFuture is null, no cover face found for person or cluster.", + 'faceCropFuture is null, no cover face found for person or cluster.', ); } return _EmptyPersonThumbnail( @@ -181,8 +200,15 @@ class _PersonFaceWidgetState extends State return thumbnailCrop; } - final Uint8List? fullCrop = - await _getFaceCrop(useFullFile: widget.useFullFile); + if (!_shouldUseProgressiveStrategy) { + return _loadFaceCropLegacy(); + } + + return _loadFaceCropProgressive(); + } + + Future _loadFaceCropLegacy() async { + final Uint8List? fullCrop = await _getFaceCrop(useFullFile: true); if (fullCrop != null) { _showingFallback = false; return fullCrop; @@ -190,7 +216,7 @@ class _PersonFaceWidgetState extends State final String personOrClusterId = widget.personId ?? widget.clusterID!; _logger.warning( - "Full face crop unavailable for $personOrClusterId, attempting thumbnail fallback.", + 'Full face crop unavailable for $personOrClusterId, attempting thumbnail fallback.', ); final Uint8List? fallbackCrop = await _getFaceCrop(useFullFile: false); @@ -200,18 +226,191 @@ class _PersonFaceWidgetState extends State return fallbackCrop; } - _logger.warning( - "Thumbnail fallback also unavailable for $personOrClusterId.", - ); + _logger + .warning('Thumbnail fallback also unavailable for $personOrClusterId.'); return null; } - Future _getFaceCrop({required bool useFullFile}) async { + Future _loadFaceCropProgressive() async { + final String personOrClusterId = widget.personId ?? widget.clusterID!; + final Uint8List? thumbnailCrop = await _getFaceCrop(useFullFile: false); + if (thumbnailCrop == null) { + _logger.warning( + 'Thumbnail face crop unavailable for $personOrClusterId, attempting full-file generation.', + ); + final Uint8List? fullCrop = await _getFaceCrop(useFullFile: true); + if (fullCrop != null) { + _showingFallback = false; + return fullCrop; + } + _logger.warning( + 'Full face crop also unavailable for $personOrClusterId after thumbnail miss.', + ); + return null; + } + + _showingFallback = true; + _fallbackEverUsed = true; + final generation = ++_upgradeGeneration; + unawaited(_attemptFullQualityUpgrade(generation)); + return thumbnailCrop; + } + + Future _attemptFullQualityUpgrade(int generation) async { + if (_shouldAbortUpgrade(generation)) { + return; + } + + final Face? face = _faceForFaceCrop; + final EnteFile? sourceFile = fileForFaceCrop; + if (face == null || sourceFile == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=missing_face_or_file person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + if (!isPerson && await _isSmallUnnamedCluster()) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=small_unnamed_cluster cluster=${widget.clusterID}', + ); + return; + } + + if (await hasPersistedFullFaceCrop(face.faceID)) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=full_crop_cached face=${face.faceID}', + ); + if (_showingFallback && mounted && !_shouldAbortUpgrade(generation)) { + setState(() { + _showingFallback = false; + }); + } else { + _showingFallback = false; + } + return; + } + + if (sourceFile.width <= 0 || sourceFile.height <= 0) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=missing_full_dimensions file=${sourceFile.uploadedFileID}', + ); + return; + } + + final cachedThumbnailDimensions = _faceCropFileId != null + ? getCachedThumbnailSourceDimensionsForFileId(_faceCropFileId!) + : null; + final thumbnailDimensions = cachedThumbnailDimensions ?? + estimateThumbnailDimensionsFromFullDimensions( + fullWidth: sourceFile.width, + fullHeight: sourceFile.height, + ); + if (thumbnailDimensions == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=missing_thumbnail_dimensions file=${sourceFile.uploadedFileID}', + ); + return; + } + if (cachedThumbnailDimensions == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_thumbnail_dimensions_source=estimated ' + 'file=${sourceFile.uploadedFileID} ' + 'estimated=${thumbnailDimensions.width}x${thumbnailDimensions.height}', + ); + } + + final decision = shouldUpgradeFromThumbnail( + faceBox: face.detection.box, + thumbnailWidth: thumbnailDimensions.width, + thumbnailHeight: thumbnailDimensions.height, + fullWidth: sourceFile.width, + fullHeight: sourceFile.height, + ); + if (!decision.shouldUpgrade) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=${decision.reason} ' + 'thumbUpscale=${decision.thumbnailUpscaleFactor.toStringAsFixed(2)} ' + 'fullUpscale=${decision.fullUpscaleFactor.toStringAsFixed(2)} ' + 'improvement=${decision.improvementRatio.toStringAsFixed(2)} ' + 'person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + _logger.fine( + 'person_face_thumbnail_upgrade_waiting_for_idle ' + 'person=${widget.personId} cluster=${widget.clusterID} ' + 'thumbUpscale=${decision.thumbnailUpscaleFactor.toStringAsFixed(2)}', + ); + await waitForThumbnailFaceGenerationIdle( + shouldStopWaiting: () => _shouldAbortUpgrade(generation), + ); + if (_shouldAbortUpgrade(generation)) { + return; + } + + _logger.info( + 'person_face_thumbnail_upgrade_started person=${widget.personId} cluster=${widget.clusterID}', + ); + final Uint8List? fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation)) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=full_crop_unavailable person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + if (!mounted) { + return; + } + setState(() { + _showingFallback = false; + faceCropFuture = Future.value(fullCrop); + }); + + _logger.info( + 'person_face_thumbnail_upgrade_applied person=${widget.personId} cluster=${widget.clusterID}', + ); + } + + bool _shouldAbortUpgrade(int generation) { + return _disposed || generation != _upgradeGeneration; + } + + Future _isSmallUnnamedCluster() async { + final clusterID = widget.clusterID; + if (clusterID == null || isPerson) { + return false; + } + final cachedFileCount = _clusterToFileCountCache.get(clusterID); + if (cachedFileCount != null) { + return cachedFileCount < kMinimumClusterSizeSearchResult; + } + + final mlDataDB = + isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; + final fileCount = await mlDataDB + .getFileIDsOfClusterID(clusterID) + .then((fileIDs) => fileIDs.length); + _clusterToFileCountCache.put(clusterID, fileCount); + return fileCount < kMinimumClusterSizeSearchResult; + } + + Future _getFaceCrop({ + required bool useFullFile, + bool notifyOnError = true, + }) async { try { final String personOrClusterId = widget.personId ?? widget.clusterID!; final tryInMemoryCachedCrop = checkInMemoryCachedCropForPersonOrClusterID(personOrClusterId); - if (tryInMemoryCachedCrop != null) return tryInMemoryCachedCrop; + if (tryInMemoryCachedCrop != null) { + return tryInMemoryCachedCrop; + } String? fixedFaceID; PersonEntity? personEntity; final mlDataDB = @@ -220,7 +419,7 @@ class _PersonFaceWidgetState extends State personEntity = await PersonService.instance.getPerson(widget.personId!); if (personEntity == null) { _logger.severe( - "Person with ID ${widget.personId} not found, cannot get cover face.", + 'Person with ID ${widget.personId} not found, cannot get cover face.', ); return null; } @@ -230,7 +429,7 @@ class _PersonFaceWidgetState extends State fixedFaceID ??= await checkUsedFaceIDForPersonOrClusterId(personOrClusterId); - EnteFile? fileForFaceCrop; + EnteFile? selectedFileForFaceCrop; if (isOfflineMode) { final allFiles = await SearchService.instance.getAllFilesForSearch(); final localIdToFile = {}; @@ -249,15 +448,15 @@ class _PersonFaceWidgetState extends State personOrClusterId, ); } else { - fileForFaceCrop = localIdToFile[localId]; - if (fileForFaceCrop == null) { + selectedFileForFaceCrop = localIdToFile[localId]; + if (selectedFileForFaceCrop == null) { await checkRemoveCachedFaceIDForPersonOrClusterId( personOrClusterId, ); } } } - if (fileForFaceCrop == null) { + if (selectedFileForFaceCrop == null) { final List allFaces = isPerson ? await mlDataDB .getFaceIDsForPersonOrderedByScore(widget.personId!) @@ -273,14 +472,14 @@ class _PersonFaceWidgetState extends State final localId = localIdMap[localIntId]; final candidate = localId != null ? localIdToFile[localId] : null; if (candidate != null) { - fileForFaceCrop = candidate; + selectedFileForFaceCrop = candidate; fixedFaceID = faceID; break; } } - if (fileForFaceCrop == null) { + if (selectedFileForFaceCrop == null) { _logger.severe( - "No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } @@ -294,23 +493,23 @@ class _PersonFaceWidgetState extends State final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); if (fileInDB == null) { _logger.severe( - "File with ID $fileID not found in DB, cannot get cover face.", + 'File with ID $fileID not found in DB, cannot get cover face.', ); await checkRemoveCachedFaceIDForPersonOrClusterId( personOrClusterId, ); } else if (hiddenFileIDs.contains(fileInDB.uploadedFileID)) { _logger.info( - "File with ID $fileID is hidden, skipping it for face crop.", + 'File with ID $fileID is hidden, skipping it for face crop.', ); await checkRemoveCachedFaceIDForPersonOrClusterId( personOrClusterId, ); } else { - fileForFaceCrop = fileInDB; + selectedFileForFaceCrop = fileInDB; } } - if (fileForFaceCrop == null) { + if (selectedFileForFaceCrop == null) { final List allFaces = isPerson ? await mlDataDB .getFaceIDsForPersonOrderedByScore(widget.personId!) @@ -320,47 +519,50 @@ class _PersonFaceWidgetState extends State final fileID = getFileIdFromFaceId(faceID); if (hiddenFileIDs.contains(fileID)) { _logger.info( - "File with ID $fileID is hidden, skipping it for face crop.", + 'File with ID $fileID is hidden, skipping it for face crop.', ); continue; } - fileForFaceCrop = await FilesDB.instance.getAnyUploadedFile(fileID); - if (fileForFaceCrop != null) { + selectedFileForFaceCrop = + await FilesDB.instance.getAnyUploadedFile(fileID); + if (selectedFileForFaceCrop != null) { _logger.info( - "Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); fixedFaceID = faceID; break; } } - if (fileForFaceCrop == null) { + if (selectedFileForFaceCrop == null) { _logger.severe( - "No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } } } + int? recentFileID; if (isOfflineMode) { - final localId = fileForFaceCrop.localID; + final localId = selectedFileForFaceCrop.localID; if (localId == null || localId.isEmpty) { _logger.severe( - "Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } recentFileID = await OfflineFilesDB.instance.getOrCreateLocalIntId(localId); } else { - recentFileID = fileForFaceCrop.uploadedFileID; + recentFileID = selectedFileForFaceCrop.uploadedFileID; } if (recentFileID == null) { _logger.severe( - "Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } + final Face? face = await mlDataDB.getCoverFaceForPerson( recentFileID: recentFileID, avatarFaceId: fixedFaceID, @@ -369,36 +571,38 @@ class _PersonFaceWidgetState extends State ); if (face == null) { _logger.severe( - "No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID", - ); - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, + 'No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', ); + await checkRemoveCachedFaceIDForPersonOrClusterId(personOrClusterId); return null; } + final cropMap = await getCachedFaceCrops( - fileForFaceCrop, + selectedFileForFaceCrop, [face], useFullFile: useFullFile, personOrClusterID: personOrClusterId, useTempCache: false, ); - this.fileForFaceCrop = fileForFaceCrop; + fileForFaceCrop = selectedFileForFaceCrop; + _faceForFaceCrop = face; _faceCropFileId = recentFileID; final result = cropMap?[face.faceID]; if (result == null) { _logger.severe( - "Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID", + 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', ); } return result; } catch (e, s) { _logger.severe( - "Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}", + 'Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}', e, s, ); - widget.onErrorCallback?.call(); + if (notifyOnError) { + widget.onErrorCallback?.call(); + } return null; } } diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index aafe96a1519..e0a07637916 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -23,6 +23,8 @@ final _logger = Logger("FaceCropUtils"); const int _retryLimit = 3; final LRUMap _faceCropCache = LRUMap(100); final LRUMap _faceCropThumbnailCache = LRUMap(100); +final LRUMap + _thumbnailSourceDimensionsByFileId = LRUMap(2000); final LRUMap _personOrClusterIdToCachedFaceID = LRUMap(2000); @@ -47,6 +49,12 @@ Uint8List? checkInMemoryCachedCropForPersonOrClusterID( return cachedCover; } +({int width, int height})? getCachedThumbnailSourceDimensionsForFileId( + int fileId, +) { + return _thumbnailSourceDimensionsByFileId.get(fileId); +} + Uint8List? _checkInMemoryCachedCropForFaceID(String faceID) { final Uint8List? cachedCover = _faceCropCache.get(faceID); return cachedCover; @@ -299,6 +307,31 @@ void checkStopTryingToGenerateFaceThumbnails( } } +bool areThumbnailFaceGenerationQueuesIdle() { + return _queueThumbnailFaceGenerations.pendingTasksCount == 0 && + _queueThumbnailFaceGenerations.runningTasksCount == 0; +} + +Future waitForThumbnailFaceGenerationIdle({ + Duration pollInterval = const Duration(milliseconds: 120), + bool Function()? shouldStopWaiting, +}) async { + while (true) { + if (shouldStopWaiting?.call() ?? false) { + return; + } + if (areThumbnailFaceGenerationQueuesIdle()) { + return; + } + await Future.delayed(pollInterval); + } +} + +Future hasPersistedFullFaceCrop(String faceID) async { + final faceCropCacheFile = cachedFaceCropPath(faceID, false); + return faceCropCacheFile.exists(); +} + Future?> _getFaceCropsUsingHeapPriorityQueue( EnteFile file, Map faceBoxeMap, { @@ -349,6 +382,35 @@ Future _faceCropTaskId( return "$baseId${useFullFile ? "-full" : "-thumbnail"}"; } +Future _faceCropFileIdForDimensions(EnteFile file) async { + if (isOfflineMode) { + final localId = file.localID; + if (localId == null || localId.isEmpty) { + return null; + } + return OfflineFilesDB.instance.getOrCreateLocalIntId(localId); + } + return file.uploadedFileID; +} + +Future _cacheThumbnailSourceDimensionsForFile( + EnteFile file, { + required int width, + required int height, +}) async { + if (width <= 0 || height <= 0) { + return; + } + final fileId = await _faceCropFileIdForDimensions(file); + if (fileId == null) { + return; + } + _thumbnailSourceDimensionsByFileId.put( + fileId, + (width: width, height: height), + ); +} + Future?> _getFaceCrops( EnteFile file, Map faceBoxeMap, { @@ -376,12 +438,19 @@ Future?> _getFaceCrops( faceIds.add(e.key); faceBoxes.add(e.value); } - final List faceCrop = - await FaceThumbnailGenerator.instance.generateFaceThumbnails( - // await generateJpgFaceThumbnails( + final generationResult = await FaceThumbnailGenerator.instance + .generateFaceThumbnailsWithSourceDimensions( imagePath, faceBoxes, ); + if (!useFullFile) { + await _cacheThumbnailSourceDimensionsForFile( + file, + width: generationResult.sourceWidth, + height: generationResult.sourceHeight, + ); + } + final List faceCrop = generationResult.thumbnails; final Map result = {}; for (int i = 0; i < faceCrop.length; i++) { result[faceIds[i]] = faceCrop[i]; diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart new file mode 100644 index 00000000000..2ab7bc039c3 --- /dev/null +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart @@ -0,0 +1,176 @@ +import 'package:photos/core/constants.dart' show thumbnailLargeSize; +import 'package:photos/models/ml/face/box.dart'; + +const int kFaceThumbnailTargetShortSide = 512; +const double kFaceThumbnailRegularPadding = 0.4; +const double kFaceThumbnailMinimumPadding = 0.1; + +class FaceThumbnailUpgradeDecision { + final bool shouldUpgrade; + final String reason; + final double thumbnailUpscaleFactor; + final double fullUpscaleFactor; + final double improvementRatio; + + const FaceThumbnailUpgradeDecision({ + required this.shouldUpgrade, + required this.reason, + required this.thumbnailUpscaleFactor, + required this.fullUpscaleFactor, + required this.improvementRatio, + }); +} + +typedef ImageDimensions = ({int width, int height}); + +ImageDimensions? estimateThumbnailDimensionsFromFullDimensions({ + required int fullWidth, + required int fullHeight, + int thumbnailMaxSide = thumbnailLargeSize, +}) { + if (fullWidth <= 0 || fullHeight <= 0 || thumbnailMaxSide <= 0) { + return null; + } + + if (fullWidth <= thumbnailMaxSide && fullHeight <= thumbnailMaxSide) { + return (width: fullWidth, height: fullHeight); + } + + if (fullWidth >= fullHeight) { + return ( + width: thumbnailMaxSide, + height: _max(1, (fullHeight * thumbnailMaxSide / fullWidth).round()), + ); + } + + return ( + width: _max(1, (fullWidth * thumbnailMaxSide / fullHeight).round()), + height: thumbnailMaxSide, + ); +} + +FaceThumbnailUpgradeDecision shouldUpgradeFromThumbnail({ + required FaceBox faceBox, + required int thumbnailWidth, + required int thumbnailHeight, + required int fullWidth, + required int fullHeight, + double upscaleThreshold = 1.6, + double minImprovementRatio = 1.4, +}) { + final thumbnailCropShortSide = _computeCropShortSide( + faceBox, + imageWidth: thumbnailWidth, + imageHeight: thumbnailHeight, + ); + if (thumbnailCropShortSide == null || thumbnailCropShortSide <= 0) { + return const FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'invalid_thumbnail_crop', + thumbnailUpscaleFactor: 0, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final fullCropShortSide = _computeCropShortSide( + faceBox, + imageWidth: fullWidth, + imageHeight: fullHeight, + ); + if (fullCropShortSide == null || fullCropShortSide <= 0) { + return const FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'invalid_full_crop', + thumbnailUpscaleFactor: 0, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final thumbnailUpscaleFactor = + kFaceThumbnailTargetShortSide / thumbnailCropShortSide; + if (thumbnailUpscaleFactor <= upscaleThreshold) { + return FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'below_upscale_threshold', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final fullUpscaleFactor = kFaceThumbnailTargetShortSide / fullCropShortSide; + final improvementRatio = thumbnailUpscaleFactor / fullUpscaleFactor; + if (improvementRatio < minImprovementRatio) { + return FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'insufficient_improvement', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: fullUpscaleFactor, + improvementRatio: improvementRatio, + ); + } + + return FaceThumbnailUpgradeDecision( + shouldUpgrade: true, + reason: 'upgrade_needed', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: fullUpscaleFactor, + improvementRatio: improvementRatio, + ); +} + +double? _computeCropShortSide( + FaceBox faceBox, { + required int imageWidth, + required int imageHeight, +}) { + if (imageWidth <= 0 || imageHeight <= 0) { + return null; + } + + final width = imageWidth.toDouble(); + final height = imageHeight.toDouble(); + + final xMinAbs = faceBox.x * width; + final yMinAbs = faceBox.y * height; + final widthAbs = faceBox.width * width; + final heightAbs = faceBox.height * height; + + if (widthAbs <= 0 || heightAbs <= 0) { + return null; + } + + final xCrop = xMinAbs - widthAbs * kFaceThumbnailRegularPadding; + final xOvershoot = (xCrop < 0 ? -xCrop : 0) / widthAbs; + final widthCrop = widthAbs * (1 + 2 * kFaceThumbnailRegularPadding) - + 2 * + _min( + xOvershoot, + kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, + ) * + widthAbs; + + final yCrop = yMinAbs - heightAbs * kFaceThumbnailRegularPadding; + final yOvershoot = (yCrop < 0 ? -yCrop : 0) / heightAbs; + final heightCrop = heightAbs * (1 + 2 * kFaceThumbnailRegularPadding) - + 2 * + _min( + yOvershoot, + kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, + ) * + heightAbs; + + final xCropSafe = xCrop.clamp(0, width).toDouble(); + final yCropSafe = yCrop.clamp(0, height).toDouble(); + final widthCropSafe = widthCrop.clamp(0, width - xCropSafe).toDouble(); + final heightCropSafe = heightCrop.clamp(0, height - yCropSafe).toDouble(); + + final shortSide = _min(widthCropSafe, heightCropSafe); + return shortSide > 0 ? shortSide : null; +} + +double _min(double a, double b) => a <= b ? a : b; + +int _max(int a, int b) => a >= b ? a : b; diff --git a/mobile/apps/photos/lib/utils/image_ml_util.dart b/mobile/apps/photos/lib/utils/image_ml_util.dart index 519593b41a6..7bf4299ba7f 100644 --- a/mobile/apps/photos/lib/utils/image_ml_util.dart +++ b/mobile/apps/photos/lib/utils/image_ml_util.dart @@ -183,6 +183,24 @@ Future _getByteDataFromImage( Future> generateFaceThumbnailsUsingCanvas( String imagePath, List faceBoxes, +) async { + final result = await generateFaceThumbnailsUsingCanvasWithSourceDimensions( + imagePath, + faceBoxes, + ); + return result.thumbnails; +} + +typedef FaceThumbnailCanvasGenerationResult = ({ + List thumbnails, + int sourceWidth, + int sourceHeight +}); + +Future + generateFaceThumbnailsUsingCanvasWithSourceDimensions( + String imagePath, + List faceBoxes, ) async { int i = 0; // Index of the faceBox, initialized here for logging purposes try { @@ -195,7 +213,11 @@ Future> generateFaceThumbnailsUsingCanvas( final Image? img = decodedImage.image; if (img == null) { _logger.severe('Image is null, cannot generate face thumbnails'); - return []; + return ( + thumbnails: [], + sourceWidth: dimensions.width, + sourceHeight: dimensions.height, + ); } final futureFaceThumbnails = >[]; for (final faceBox in faceBoxes) { @@ -236,14 +258,24 @@ Future> generateFaceThumbnailsUsingCanvas( } final List faceThumbnails = await Future.wait(futureFaceThumbnails); - return faceThumbnails; + return ( + thumbnails: faceThumbnails, + sourceWidth: dimensions.width, + sourceHeight: dimensions.height, + ); } catch (e, s) { + final problematicFaceBox = + faceBoxes.isNotEmpty && i < faceBoxes.length ? faceBoxes[i] : null; _logger.severe( - 'Error generating face thumbnails. cropImage problematic input argument: ${i}th facebox: ${faceBoxes[i].toString()}', + 'Error generating face thumbnails. cropImage problematic input argument: ${i}th facebox: $problematicFaceBox', e, s, ); - return []; + return ( + thumbnails: [], + sourceWidth: 0, + sourceHeight: 0, + ); } } diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart index c2c7a63b297..901d74e4ed2 100644 --- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart +++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart @@ -42,6 +42,9 @@ enum IsolateOperation { /// [MLComputer] generateFaceThumbnails, + /// [MLComputer] + generateFaceThumbnailsWithSourceDimensions, + /// [MLComputer] loadModel, @@ -186,6 +189,50 @@ Future isolateFunction( ); return List.from(results); + /// MLComputer + case IsolateOperation.generateFaceThumbnailsWithSourceDimensions: + final imagePath = args['imagePath'] as String; + final useRustForFaceThumbnails = + args['useRustForFaceThumbnails'] as bool? ?? false; + final faceBoxesJson = args['faceBoxesList'] as List>; + final List faceBoxes = + faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); + + if (useRustForFaceThumbnails) { + await _ensureRustLoaded(); + final rustFaceBoxes = faceBoxes + .map( + (box) => rust_image_processing.RustFaceBox( + x: box.x, + y: box.y, + width: box.width, + height: box.height, + ), + ) + .toList(growable: false); + final batch = + await rust_image_processing.generateFaceThumbnailsWithMetadata( + imagePath: imagePath, + faceBoxes: rustFaceBoxes, + ); + return { + 'thumbnails': List.from(batch.thumbnails), + 'sourceWidth': batch.sourceWidth, + 'sourceHeight': batch.sourceHeight, + }; + } + + final results = + await generateFaceThumbnailsUsingCanvasWithSourceDimensions( + imagePath, + faceBoxes, + ); + return { + 'thumbnails': List.from(results.thumbnails), + 'sourceWidth': results.sourceWidth, + 'sourceHeight': results.sourceHeight, + }; + /// MLComputer case IsolateOperation.loadModel: final modelName = args['modelName'] as String; diff --git a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart index 2309a58c2e0..ca8bf58288f 100644 --- a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart +++ b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart @@ -106,6 +106,8 @@ class FlagService { bool get useRustForFaceThumbnails => internalUser; + bool get progressivePersonFaceThumbnailsEnabled => internalUser; + Future tryRefreshFlags() async { try { await _fetch(); diff --git a/mobile/apps/photos/rust/src/api/image_processing_api.rs b/mobile/apps/photos/rust/src/api/image_processing_api.rs index 4a7e1157955..044ebaadea1 100644 --- a/mobile/apps/photos/rust/src/api/image_processing_api.rs +++ b/mobile/apps/photos/rust/src/api/image_processing_api.rs @@ -11,21 +11,45 @@ pub struct RustFaceBox { pub height: f64, } +#[derive(Clone, Debug)] +pub struct RustFaceThumbnailBatch { + pub thumbnails: Vec>, + pub source_width: u32, + pub source_height: u32, +} + pub fn generate_face_thumbnails( image_path: String, face_boxes: Vec, ) -> Result>, String> { + generate_face_thumbnails_with_metadata(image_path, face_boxes).map(|batch| batch.thumbnails) +} + +pub fn generate_face_thumbnails_with_metadata( + image_path: String, + face_boxes: Vec, +) -> Result { let decoded = decode_image_from_path(&image_path).map_err(|e| e.to_string())?; - let face_boxes = face_boxes + let face_boxes = parse_face_boxes(face_boxes)?; + let thumbnails = + generate_face_thumbnails_impl(&decoded, &face_boxes).map_err(|e| e.to_string())?; + + Ok(RustFaceThumbnailBatch { + thumbnails, + source_width: decoded.dimensions.width, + source_height: decoded.dimensions.height, + }) +} + +fn parse_face_boxes(face_boxes: Vec) -> Result, String> { + face_boxes .into_iter() .enumerate() .map(|(index, face_box)| { FaceBox::try_from(face_box) .map_err(|e| format!("invalid face box at index {index}: {e}")) }) - .collect::, _>>()?; - - generate_face_thumbnails_impl(&decoded, &face_boxes).map_err(|e| e.to_string()) + .collect::, _>>() } impl TryFrom for FaceBox { From 56fef7e10d900ef9a5ab02e67281c2b6024be4cd Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Mar 2026 12:16:28 +0530 Subject: [PATCH 02/18] Lower progressive face upgrade cluster threshold to 5 --- .../photos/lib/ui/viewer/people/person_face_widget.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 4c8456a4a1a..a3589757461 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -12,7 +12,6 @@ import 'package:photos/models/file/file.dart'; import 'package:photos/models/ml/face/face.dart'; import 'package:photos/models/ml/face/person.dart'; import 'package:photos/service_locator.dart' show flagService, isOfflineMode; -import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:photos/services/machine_learning/face_ml/person/person_service.dart'; import 'package:photos/services/machine_learning/ml_result.dart'; import 'package:photos/services/search_service.dart'; @@ -22,6 +21,7 @@ import 'package:photos/utils/face/face_thumbnail_cache.dart'; import 'package:photos/utils/face/face_thumbnail_quality.dart'; final _logger = Logger('PersonFaceWidget'); +const _kMinUnnamedClusterSizeForProgressiveUpgrade = 5; class PersonFaceWidget extends StatefulWidget { final String? personId; @@ -388,7 +388,7 @@ class _PersonFaceWidgetState extends State } final cachedFileCount = _clusterToFileCountCache.get(clusterID); if (cachedFileCount != null) { - return cachedFileCount < kMinimumClusterSizeSearchResult; + return cachedFileCount < _kMinUnnamedClusterSizeForProgressiveUpgrade; } final mlDataDB = @@ -397,7 +397,7 @@ class _PersonFaceWidgetState extends State .getFileIDsOfClusterID(clusterID) .then((fileIDs) => fileIDs.length); _clusterToFileCountCache.put(clusterID, fileCount); - return fileCount < kMinimumClusterSizeSearchResult; + return fileCount < _kMinUnnamedClusterSizeForProgressiveUpgrade; } Future _getFaceCrop({ From 18c1718152e39890af197670715356ee41766636 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Mar 2026 12:37:59 +0530 Subject: [PATCH 03/18] Skip progressive full-upgrade for video face crops --- .../photos/lib/ui/viewer/people/person_face_widget.dart | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index a3589757461..476e0484860 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -9,6 +9,7 @@ import 'package:photos/db/files_db.dart'; import 'package:photos/db/ml/db.dart'; import 'package:photos/db/offline_files_db.dart'; import 'package:photos/models/file/file.dart'; +import 'package:photos/models/file/file_type.dart'; import 'package:photos/models/ml/face/face.dart'; import 'package:photos/models/ml/face/person.dart'; import 'package:photos/service_locator.dart' show flagService, isOfflineMode; @@ -269,6 +270,12 @@ class _PersonFaceWidgetState extends State ); return; } + if (sourceFile.fileType == FileType.video) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=video_source person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } if (!isPerson && await _isSmallUnnamedCluster()) { _logger.fine( From 62e6ae53d5013c7867b4cc5c67764387f59f9722 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 3 Mar 2026 13:00:05 +0530 Subject: [PATCH 04/18] [mob][photos] Apply cached full face crop in upgrade path --- .../lib/ui/viewer/people/person_face_widget.dart | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 476e0484860..e97dc45afde 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -288,13 +288,17 @@ class _PersonFaceWidgetState extends State _logger.fine( 'person_face_thumbnail_upgrade_skipped reason=full_crop_cached face=${face.faceID}', ); - if (_showingFallback && mounted && !_shouldAbortUpgrade(generation)) { - setState(() { - _showingFallback = false; - }); - } else { - _showingFallback = false; + final Uint8List? fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation) || !mounted) { + return; } + setState(() { + _showingFallback = false; + faceCropFuture = Future.value(fullCrop); + }); return; } From 8049abf0b4524d90264d37910b41cdce331828b0 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Thu, 5 Mar 2026 10:49:00 +0530 Subject: [PATCH 05/18] Gate progressive face thumbnail generation by flag --- .../face_thumbnail_generator.dart | 44 ++++++++++++++++--- .../lib/utils/face/face_thumbnail_cache.dart | 32 +++++++++----- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart index f6310d75797..9184cb1b02b 100644 --- a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart +++ b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart @@ -48,11 +48,36 @@ class FaceThumbnailGenerator extends SuperIsolate { String imagePath, List faceBoxes, ) async { - final result = await generateFaceThumbnailsWithSourceDimensions( - imagePath, - faceBoxes, - ); - return result.thumbnails; + try { + final useRustForFaceThumbnails = flagService.useRustForFaceThumbnails; + _logger.info( + "Generating face thumbnails for ${faceBoxes.length} face boxes in $imagePath", + ); + final List> faceBoxesJson = + faceBoxes.map((box) => box.toJson()).toList(); + final List faces = await runInIsolate( + IsolateOperation.generateFaceThumbnails, + { + 'imagePath': imagePath, + 'faceBoxesList': faceBoxesJson, + 'useRustForFaceThumbnails': useRustForFaceThumbnails, + }, + ).then((value) => value.cast()); + _logger.info("Generated face thumbnails"); + if (useRustForFaceThumbnails) { + // Rust path already emits compressed JPEG bytes. + return faces; + } + final compressedFaces = + await compressFaceThumbnails({'listPngBytes': faces}); + _logger.fine( + "Compressed face thumbnails from sizes ${faces.map((e) => e.length / 1024).toList()} to ${compressedFaces.map((e) => e.length / 1024).toList()} kilobytes", + ); + return compressedFaces; + } catch (e, s) { + _logger.severe("Failed to generate face thumbnails", e, s); + rethrow; + } } Future @@ -60,6 +85,15 @@ class FaceThumbnailGenerator extends SuperIsolate { String imagePath, List faceBoxes, ) async { + if (!flagService.progressivePersonFaceThumbnailsEnabled) { + final thumbnails = await generateFaceThumbnails(imagePath, faceBoxes); + return FaceThumbnailGenerationResult( + thumbnails: thumbnails, + sourceWidth: 0, + sourceHeight: 0, + ); + } + try { final useRustForFaceThumbnails = flagService.useRustForFaceThumbnails; _logger.info( diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index e0a07637916..ec17618528a 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -13,7 +13,7 @@ import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/face/face.dart"; -import "package:photos/service_locator.dart" show isOfflineMode; +import "package:photos/service_locator.dart" show flagService, isOfflineMode; import "package:photos/services/machine_learning/face_thumbnail_generator.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/thumbnail_util.dart"; @@ -438,19 +438,27 @@ Future?> _getFaceCrops( faceIds.add(e.key); faceBoxes.add(e.value); } - final generationResult = await FaceThumbnailGenerator.instance - .generateFaceThumbnailsWithSourceDimensions( - imagePath, - faceBoxes, - ); - if (!useFullFile) { - await _cacheThumbnailSourceDimensionsForFile( - file, - width: generationResult.sourceWidth, - height: generationResult.sourceHeight, + late final List faceCrop; + if (flagService.progressivePersonFaceThumbnailsEnabled) { + final generationResult = await FaceThumbnailGenerator.instance + .generateFaceThumbnailsWithSourceDimensions( + imagePath, + faceBoxes, + ); + if (!useFullFile) { + await _cacheThumbnailSourceDimensionsForFile( + file, + width: generationResult.sourceWidth, + height: generationResult.sourceHeight, + ); + } + faceCrop = generationResult.thumbnails; + } else { + faceCrop = await FaceThumbnailGenerator.instance.generateFaceThumbnails( + imagePath, + faceBoxes, ); } - final List faceCrop = generationResult.thumbnails; final Map result = {}; for (int i = 0; i < faceCrop.length; i++) { result[faceIds[i]] = faceCrop[i]; From 5cc811fd72e41d87b89fa773f624da59c2e10971 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 17 Mar 2026 13:26:12 +0530 Subject: [PATCH 06/18] Minor improvement --- .../ui/viewer/people/person_face_widget.dart | 38 ++++++++++++++++--- .../lib/utils/face/face_thumbnail_cache.dart | 30 +++++++++++++-- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index e97dc45afde..7b604db5211 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -23,6 +23,9 @@ import 'package:photos/utils/face/face_thumbnail_quality.dart'; final _logger = Logger('PersonFaceWidget'); const _kMinUnnamedClusterSizeForProgressiveUpgrade = 5; +const _kProgressiveUpgradeUpscaleThreshold = 1.35; +const _kProgressiveUpgradeMinImprovementRatio = 1.2; +const _kProgressiveUpgradeIdleWaitBudget = Duration(seconds: 2); class PersonFaceWidget extends StatefulWidget { final String? personId; @@ -102,7 +105,7 @@ class _PersonFaceWidgetState extends State useFullFile: true, ); } - if (_fallbackEverUsed || _shouldUseProgressiveStrategy) { + if (_fallbackEverUsed) { checkStopTryingToGenerateFaceThumbnails( _faceCropFileId!, useFullFile: false, @@ -337,6 +340,8 @@ class _PersonFaceWidgetState extends State thumbnailHeight: thumbnailDimensions.height, fullWidth: sourceFile.width, fullHeight: sourceFile.height, + upscaleThreshold: _kProgressiveUpgradeUpscaleThreshold, + minImprovementRatio: _kProgressiveUpgradeMinImprovementRatio, ); if (!decision.shouldUpgrade) { _logger.fine( @@ -354,12 +359,18 @@ class _PersonFaceWidgetState extends State 'person=${widget.personId} cluster=${widget.clusterID} ' 'thumbUpscale=${decision.thumbnailUpscaleFactor.toStringAsFixed(2)}', ); - await waitForThumbnailFaceGenerationIdle( + final didReachThumbnailIdle = await waitForThumbnailFaceGenerationIdle( shouldStopWaiting: () => _shouldAbortUpgrade(generation), + maxWait: _kProgressiveUpgradeIdleWaitBudget, ); if (_shouldAbortUpgrade(generation)) { return; } + if (!didReachThumbnailIdle) { + _logger.fine( + 'person_face_thumbnail_upgrade_wait_idle_timeout person=${widget.personId} cluster=${widget.clusterID}', + ); + } _logger.info( 'person_face_thumbnail_upgrade_started person=${widget.personId} cluster=${widget.clusterID}', @@ -422,6 +433,15 @@ class _PersonFaceWidgetState extends State if (tryInMemoryCachedCrop != null) { return tryInMemoryCachedCrop; } + if (!useFullFile) { + final tryInMemoryCachedThumbnailCrop = + checkInMemoryCachedThumbnailCropForPersonOrClusterID( + personOrClusterId, + ); + if (tryInMemoryCachedThumbnailCrop != null) { + return tryInMemoryCachedThumbnailCrop; + } + } String? fixedFaceID; PersonEntity? personEntity; final mlDataDB = @@ -496,9 +516,13 @@ class _PersonFaceWidgetState extends State } } } else { - final hiddenFileIDs = await SearchService.instance - .getHiddenFiles() - .then((onValue) => onValue.map((e) => e.uploadedFileID)); + final hiddenFileIDs = + await SearchService.instance.getHiddenFiles().then( + (files) => files + .map((file) => file.uploadedFileID) + .whereType() + .toSet(), + ); if (fixedFaceID != null) { final fileID = getFileIdFromFaceId(fixedFaceID); final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); @@ -587,6 +611,10 @@ class _PersonFaceWidgetState extends State await checkRemoveCachedFaceIDForPersonOrClusterId(personOrClusterId); return null; } + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + face.faceID, + ); final cropMap = await getCachedFaceCrops( selectedFileForFaceCrop, diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index ec17618528a..a72ddb0a63c 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -49,6 +49,15 @@ Uint8List? checkInMemoryCachedCropForPersonOrClusterID( return cachedCover; } +Uint8List? checkInMemoryCachedThumbnailCropForPersonOrClusterID( + String personOrClusterID, +) { + final String? faceID = + _personOrClusterIdToCachedFaceID.get(personOrClusterID); + if (faceID == null) return null; + return _faceCropThumbnailCache.get(faceID); +} + ({int width, int height})? getCachedThumbnailSourceDimensionsForFileId( int fileId, ) { @@ -87,6 +96,16 @@ Future putFaceIdCachedForPersonOrCluster( _personOrClusterIdToCachedFaceID.put(personOrClusterID, faceID); } +Future cacheFaceIdForPersonOrClusterIfNeeded( + String personOrClusterID, + String faceID, +) async { + if (_personOrClusterIdToCachedFaceID.get(personOrClusterID) == faceID) { + return; + } + await putFaceIdCachedForPersonOrCluster(personOrClusterID, faceID); +} + Future _putCachedCropForFaceID( String faceID, Uint8List data, [ @@ -312,16 +331,21 @@ bool areThumbnailFaceGenerationQueuesIdle() { _queueThumbnailFaceGenerations.runningTasksCount == 0; } -Future waitForThumbnailFaceGenerationIdle({ +Future waitForThumbnailFaceGenerationIdle({ Duration pollInterval = const Duration(milliseconds: 120), + Duration? maxWait, bool Function()? shouldStopWaiting, }) async { + final startedAt = DateTime.now(); while (true) { if (shouldStopWaiting?.call() ?? false) { - return; + return false; } if (areThumbnailFaceGenerationQueuesIdle()) { - return; + return true; + } + if (maxWait != null && DateTime.now().difference(startedAt) >= maxWait) { + return false; } await Future.delayed(pollInterval); } From bf3aa5c0e7f9083169d8260857707e788de2df82 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 17 Mar 2026 15:32:08 +0530 Subject: [PATCH 07/18] Speed up progressive people face thumbnails --- .../models/ml/face/person_face_source.dart | 16 + .../lib/models/search/search_constants.dart | 2 + .../photos/lib/services/search_service.dart | 74 ++ .../ui/viewer/people/person_face_widget.dart | 868 ++++++++++++------ .../result/people_section_all_page.dart | 7 + .../result/search_thumbnail_widget.dart | 9 + .../ui/viewer/search_tab/people_section.dart | 7 + .../lib/utils/face/face_thumbnail_cache.dart | 28 + .../utils/face/face_thumbnail_quality.dart | 80 +- 9 files changed, 791 insertions(+), 300 deletions(-) create mode 100644 mobile/apps/photos/lib/models/ml/face/person_face_source.dart diff --git a/mobile/apps/photos/lib/models/ml/face/person_face_source.dart b/mobile/apps/photos/lib/models/ml/face/person_face_source.dart new file mode 100644 index 00000000000..ea56fa3f216 --- /dev/null +++ b/mobile/apps/photos/lib/models/ml/face/person_face_source.dart @@ -0,0 +1,16 @@ +import 'package:photos/models/file/file.dart'; +import 'package:photos/models/ml/face/face.dart'; + +class PersonFaceSource { + final EnteFile file; + final Face face; + final int resolvedFileId; + final String? personName; + + const PersonFaceSource({ + required this.file, + required this.face, + required this.resolvedFileId, + this.personName, + }); +} diff --git a/mobile/apps/photos/lib/models/search/search_constants.dart b/mobile/apps/photos/lib/models/search/search_constants.dart index c5d64a2645f..669cce71704 100644 --- a/mobile/apps/photos/lib/models/search/search_constants.dart +++ b/mobile/apps/photos/lib/models/search/search_constants.dart @@ -1,6 +1,8 @@ const kPersonParamID = 'person_id'; const kPersonWidgetKey = 'person_widget_key'; const kPersonPinned = 'person_pinned'; +const kPersonAvatarFaceID = 'person_avatar_face_id'; +const kPersonFaceSource = 'person_face_source'; const kClusterParamId = 'cluster_id'; const kFileID = 'file_id'; const kContactEmail = 'contact_email'; diff --git a/mobile/apps/photos/lib/services/search_service.dart b/mobile/apps/photos/lib/services/search_service.dart index ec2f6c73ad9..7194035b4e9 100644 --- a/mobile/apps/photos/lib/services/search_service.dart +++ b/mobile/apps/photos/lib/services/search_service.dart @@ -32,6 +32,7 @@ import "package:photos/models/memories/memories_cache.dart"; import "package:photos/models/memories/memory.dart"; import "package:photos/models/memories/smart_memory.dart"; import "package:photos/models/ml/face/person.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import 'package:photos/models/search/album_search_result.dart'; import 'package:photos/models/search/device_album_search_result.dart'; import 'package:photos/models/search/generic_search_result.dart'; @@ -66,6 +67,7 @@ import "package:photos/utils/file_util.dart"; import "package:photos/utils/people_sort_util.dart"; class SearchService { + static const int _initialFaceSourcePrefetchCount = 24; Future>? _cachedFilesFuture; Future>? _cachedFilesForSearch; Future>? _cachedFilesForHierarchicalSearch; @@ -197,6 +199,75 @@ class SearchService { unawaited(memoriesCacheService.clearMemoriesCache()); } + Future _enrichInitialFaceSources( + List results, + ) async { + final resultsToPrefetch = results.take(_initialFaceSourcePrefetchCount); + await Future.wait( + resultsToPrefetch.map((result) async { + try { + final source = await _tryBuildInitialPersonFaceSource(result); + if (source != null) { + result.params[kPersonFaceSource] = source; + } + } catch (e, s) { + _logger.fine( + "Failed to prefetch initial face source for ${result.params[kPersonParamID]}", + e, + s, + ); + } + }), + ); + } + + Future _tryBuildInitialPersonFaceSource( + GenericSearchResult result, + ) async { + final personID = result.params[kPersonParamID] as String?; + final avatarFaceID = result.params[kPersonAvatarFaceID] as String?; + final previewFile = result.previewThumbnail(); + final previewFileID = previewFile?.uploadedFileID; + if (personID == null || + avatarFaceID == null || + avatarFaceID.isEmpty || + previewFile == null || + previewFileID == null) { + return null; + } + + final face = await mlDataDB.getCoverFaceForPerson( + recentFileID: previewFileID, + personID: personID, + avatarFaceId: avatarFaceID, + ); + if (face == null) { + return null; + } + + EnteFile sourceFile = previewFile; + if (face.fileID != previewFileID) { + EnteFile? faceFile; + for (final file in result.resultFiles()) { + if (file.uploadedFileID == face.fileID) { + faceFile = file; + break; + } + } + if (faceFile == null) { + return null; + } + sourceFile = faceFile; + } + + return PersonFaceSource( + file: sourceFile, + face: face, + resolvedFileId: face.fileID, + personName: result.name(), + ); + } + // getFilteredCollectionsWithThumbnail removes deleted or archived or // collections which don't have a file from search result Future> getCollectionSearchResults( @@ -1083,6 +1154,7 @@ class SearchService { params: { kPersonWidgetKey: p.data.avatarFaceID ?? p.hashCode.toString(), kPersonParamID: personID, + kPersonAvatarFaceID: p.data.avatarFaceID, kFileID: files.first.uploadedFileID, kPersonPinned: p.data.isPinned, }, @@ -1100,6 +1172,7 @@ class SearchService { kPersonWidgetKey: p.data.avatarFaceID ?? p.hashCode.toString(), kPersonParamID: personID, + kPersonAvatarFaceID: p.data.avatarFaceID, kPersonPinned: p.data.isPinned, kFileID: files.first.uploadedFileID, }, @@ -1206,6 +1279,7 @@ class SearchService { photosSortAscending: localSettings.peoplePhotosSortAscending, ), ); + await _enrichInitialFaceSources(facesResult); if (limit != null) { return facesResult.sublist(0, min(limit, facesResult.length)); } else { diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 7b604db5211..e405b572d5c 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -5,13 +5,15 @@ import 'package:flutter/foundation.dart' show kDebugMode; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; import 'package:photos/core/cache/lru_map.dart'; +import 'package:photos/core/cache/thumbnail_in_memory_cache.dart'; import 'package:photos/db/files_db.dart'; import 'package:photos/db/ml/db.dart'; import 'package:photos/db/offline_files_db.dart'; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; +import 'package:photos/models/ml/face/box.dart'; import 'package:photos/models/ml/face/face.dart'; -import 'package:photos/models/ml/face/person.dart'; +import 'package:photos/models/ml/face/person_face_source.dart'; import 'package:photos/service_locator.dart' show flagService, isOfflineMode; import 'package:photos/services/machine_learning/face_ml/person/person_service.dart'; import 'package:photos/services/machine_learning/ml_result.dart'; @@ -20,6 +22,7 @@ import 'package:photos/theme/ente_theme.dart'; import 'package:photos/ui/common/loading_widget.dart'; import 'package:photos/utils/face/face_thumbnail_cache.dart'; import 'package:photos/utils/face/face_thumbnail_quality.dart'; +import 'package:photos/utils/thumbnail_util.dart'; final _logger = Logger('PersonFaceWidget'); const _kMinUnnamedClusterSizeForProgressiveUpgrade = 5; @@ -27,12 +30,48 @@ const _kProgressiveUpgradeUpscaleThreshold = 1.35; const _kProgressiveUpgradeMinImprovementRatio = 1.2; const _kProgressiveUpgradeIdleWaitBudget = Duration(seconds: 2); +class _PersonFaceLoadResult { + final Uint8List? faceCropBytes; + final Uint8List? thumbnailBytes; + final PersonFaceSource? faceSource; + final String? personName; + + const _PersonFaceLoadResult._({ + this.faceCropBytes, + this.thumbnailBytes, + this.faceSource, + this.personName, + }); + + const _PersonFaceLoadResult.faceCrop({ + required Uint8List faceCropBytes, + String? personName, + }) : this._( + faceCropBytes: faceCropBytes, + personName: personName, + ); + + const _PersonFaceLoadResult.thumbnailPreview({ + required Uint8List thumbnailBytes, + required PersonFaceSource faceSource, + String? personName, + }) : this._( + thumbnailBytes: thumbnailBytes, + faceSource: faceSource, + personName: personName, + ); +} + class PersonFaceWidget extends StatefulWidget { final String? personId; final String? clusterID; final bool useFullFile; final VoidCallback? onErrorCallback; final bool keepAlive; + final PersonFaceSource? initialFaceSource; + final String? initialPersonName; + final String? initialAvatarFaceId; + final EnteFile? initialPreviewFile; /// Physical pixel width for image decoding optimization. /// @@ -44,14 +83,16 @@ class PersonFaceWidget extends StatefulWidget { /// If null or <= 0, the image is decoded at full resolution. final int? cachedPixelWidth; - // PersonFaceWidget constructor checks that both personId and clusterID are not null - // and that the file is not null const PersonFaceWidget({ this.personId, this.clusterID, this.useFullFile = true, this.onErrorCallback, this.keepAlive = false, + this.initialFaceSource, + this.initialPersonName, + this.initialAvatarFaceId, + this.initialPreviewFile, this.cachedPixelWidth, super.key, }) : assert( @@ -65,15 +106,17 @@ class PersonFaceWidget extends StatefulWidget { class _PersonFaceWidgetState extends State with AutomaticKeepAliveClientMixin { - Future? faceCropFuture; + Future<_PersonFaceLoadResult?>? _faceLoadFuture; EnteFile? fileForFaceCrop; Face? _faceForFaceCrop; int? _faceCropFileId; + PersonFaceSource? _resolvedFaceSource; String? _personName; bool _showingFallback = false; bool _fallbackEverUsed = false; bool _disposed = false; int _upgradeGeneration = 0; + EnteFile? _requestedThumbnailPreviewFile; static final LRUMap _clusterToFileCountCache = LRUMap(1000); @@ -91,7 +134,8 @@ class _PersonFaceWidgetState extends State @override void initState() { super.initState(); - faceCropFuture = _loadFaceCrop(); + _personName = widget.initialPersonName; + _faceLoadFuture = _loadFace(); } @override @@ -105,70 +149,47 @@ class _PersonFaceWidgetState extends State useFullFile: true, ); } - if (_fallbackEverUsed) { + if (_fallbackEverUsed || _shouldUseProgressiveStrategy) { checkStopTryingToGenerateFaceThumbnails( _faceCropFileId!, useFullFile: false, ); } } + if (_requestedThumbnailPreviewFile != null) { + removePendingGetThumbnailRequestIfAny(_requestedThumbnailPreviewFile!); + } super.dispose(); } @override Widget build(BuildContext context) { - super.build( - context, - ); // Calling super.build for AutomaticKeepAliveClientMixin + super.build(context); - return FutureBuilder( - future: faceCropFuture, + return FutureBuilder<_PersonFaceLoadResult?>( + future: _faceLoadFuture, builder: (context, snapshot) { - if (snapshot.hasData && snapshot.data != null) { - // Only cacheWidth (not cacheHeight) to preserve aspect ratio. - // Face crops are typically portrait, so constraining width ensures - // sufficient height for BoxFit.cover without upscaling. - final shouldOptimize = - widget.cachedPixelWidth != null && widget.cachedPixelWidth! > 0; - final ImageProvider imageProvider = shouldOptimize - ? Image.memory( - snapshot.data!, - cacheWidth: widget.cachedPixelWidth, - ).image - : MemoryImage(snapshot.data!); - return Stack( - fit: StackFit.expand, - children: [ - Image( - image: imageProvider, - fit: BoxFit.cover, - ), - if (kDebugMode && _showingFallback) - Positioned( - top: 4, - right: 4, - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: 4, - vertical: 2, - ), - decoration: BoxDecoration( - color: Colors.black.withValues(alpha: 0.6), - borderRadius: BorderRadius.circular(4), - ), - child: const Text( - '(T)', - style: TextStyle( - color: Colors.white, - fontSize: 10, - fontWeight: FontWeight.bold, - ), - ), - ), - ), - ], + final loadResult = snapshot.data; + if (loadResult?.faceCropBytes != null) { + return _buildFaceImage( + _buildImageFromBytes(loadResult!.faceCropBytes!), ); } + if (loadResult?.thumbnailBytes != null && + loadResult?.faceSource != null) { + final previewDimensions = _previewImageDimensionsForSource( + loadResult!.faceSource!, + ); + if (previewDimensions != null) { + return _buildFaceImage( + _DirectThumbnailFacePreview( + thumbnailBytes: loadResult.thumbnailBytes!, + faceBox: loadResult.faceSource!.face.detection.box, + imageDimensions: previewDimensions, + ), + ); + } + } if (snapshot.connectionState == ConnectionState.waiting || snapshot.connectionState == ConnectionState.active) { return EnteLoadingWidget( @@ -183,39 +204,92 @@ class _PersonFaceWidgetState extends State ); } else { _logger.severe( - 'faceCropFuture is null, no cover face found for person or cluster.', + 'No cover face found for person or cluster.', ); } return _EmptyPersonThumbnail( - initial: isPerson ? _personName : null, + initial: isPerson ? (loadResult?.personName ?? _personName) : null, ); }, ); } - Future _loadFaceCrop() async { + Widget _buildImageFromBytes(Uint8List imageBytes) { + final shouldOptimize = + widget.cachedPixelWidth != null && widget.cachedPixelWidth! > 0; + final ImageProvider imageProvider = shouldOptimize + ? Image.memory( + imageBytes, + cacheWidth: widget.cachedPixelWidth, + ).image + : MemoryImage(imageBytes); + return Image( + image: imageProvider, + fit: BoxFit.cover, + ); + } + + Widget _buildFaceImage(Widget child) { + return Stack( + fit: StackFit.expand, + children: [ + child, + if (kDebugMode && _showingFallback) + Positioned( + top: 4, + right: 4, + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: 4, + vertical: 2, + ), + decoration: BoxDecoration( + color: Colors.black.withValues(alpha: 0.6), + borderRadius: BorderRadius.circular(4), + ), + child: const Text( + '(T)', + style: TextStyle( + color: Colors.white, + fontSize: 10, + fontWeight: FontWeight.bold, + ), + ), + ), + ), + ], + ); + } + + Future<_PersonFaceLoadResult?> _loadFace() async { if (!widget.useFullFile) { - final Uint8List? thumbnailCrop = - await _getFaceCrop(useFullFile: widget.useFullFile); - if (thumbnailCrop != null) { - _fallbackEverUsed = true; + final thumbnailCrop = await _getFaceCrop(useFullFile: false); + if (thumbnailCrop == null) { + return null; } + _fallbackEverUsed = true; _showingFallback = false; - return thumbnailCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: thumbnailCrop, + personName: _personName, + ); } if (!_shouldUseProgressiveStrategy) { - return _loadFaceCropLegacy(); + return _loadFaceLegacy(); } - return _loadFaceCropProgressive(); + return _loadFaceProgressive(); } - Future _loadFaceCropLegacy() async { - final Uint8List? fullCrop = await _getFaceCrop(useFullFile: true); + Future<_PersonFaceLoadResult?> _loadFaceLegacy() async { + final fullCrop = await _getFaceCrop(useFullFile: true); if (fullCrop != null) { _showingFallback = false; - return fullCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ); } final String personOrClusterId = widget.personId ?? widget.clusterID!; @@ -223,11 +297,14 @@ class _PersonFaceWidgetState extends State 'Full face crop unavailable for $personOrClusterId, attempting thumbnail fallback.', ); - final Uint8List? fallbackCrop = await _getFaceCrop(useFullFile: false); + final fallbackCrop = await _getFaceCrop(useFullFile: false); if (fallbackCrop != null) { _showingFallback = true; _fallbackEverUsed = true; - return fallbackCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fallbackCrop, + personName: _personName, + ); } _logger @@ -235,29 +312,377 @@ class _PersonFaceWidgetState extends State return null; } - Future _loadFaceCropProgressive() async { + Future<_PersonFaceLoadResult?> _loadFaceProgressive() async { final String personOrClusterId = widget.personId ?? widget.clusterID!; - final Uint8List? thumbnailCrop = await _getFaceCrop(useFullFile: false); - if (thumbnailCrop == null) { - _logger.warning( - 'Thumbnail face crop unavailable for $personOrClusterId, attempting full-file generation.', + final faceSource = await _resolveFaceSource(); + if (faceSource == null) { + return null; + } + + if (_previewImageDimensionsForSource(faceSource) != null) { + final thumbnailBytes = await _loadThumbnailPreviewBytes(faceSource.file); + if (thumbnailBytes != null) { + _showingFallback = true; + _fallbackEverUsed = true; + final generation = ++_upgradeGeneration; + unawaited(_attemptFullQualityUpgrade(generation)); + return _PersonFaceLoadResult.thumbnailPreview( + thumbnailBytes: thumbnailBytes, + faceSource: faceSource, + personName: _personName, + ); + } + } + + final thumbnailCrop = await _getFaceCrop( + useFullFile: false, + resolvedSource: faceSource, + ); + if (thumbnailCrop != null) { + _showingFallback = true; + _fallbackEverUsed = true; + final generation = ++_upgradeGeneration; + unawaited(_attemptFullQualityUpgrade(generation)); + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: thumbnailCrop, + personName: _personName, ); - final Uint8List? fullCrop = await _getFaceCrop(useFullFile: true); - if (fullCrop != null) { - _showingFallback = false; - return fullCrop; + } + + _logger.warning( + 'Thumbnail face crop unavailable for $personOrClusterId, attempting full-file generation.', + ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + resolvedSource: faceSource, + ); + if (fullCrop != null) { + _showingFallback = false; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ); + } + _logger.warning( + 'Full face crop also unavailable for $personOrClusterId after thumbnail miss.', + ); + return null; + } + + ImageDimensions? _previewImageDimensionsForSource( + PersonFaceSource faceSource, + ) { + final cachedThumbnailDimensions = + getCachedThumbnailSourceDimensionsForFileId(faceSource.resolvedFileId); + if (cachedThumbnailDimensions != null) { + return cachedThumbnailDimensions; + } + if (faceSource.file.width > 0 && faceSource.file.height > 0) { + return ( + width: faceSource.file.width, + height: faceSource.file.height, + ); + } + return null; + } + + Future _loadThumbnailPreviewBytes(EnteFile file) async { + final cachedThumbnailBytes = ThumbnailInMemoryLruCache.get(file); + if (cachedThumbnailBytes != null) { + return cachedThumbnailBytes; + } + + _requestedThumbnailPreviewFile = file; + try { + return await getThumbnail(file); + } finally { + if (_requestedThumbnailPreviewFile == file) { + _requestedThumbnailPreviewFile = null; } - _logger.warning( - 'Full face crop also unavailable for $personOrClusterId after thumbnail miss.', + } + } + + void _applyResolvedFaceSource(PersonFaceSource faceSource) { + _resolvedFaceSource = faceSource; + fileForFaceCrop = faceSource.file; + _faceForFaceCrop = faceSource.face; + _faceCropFileId = faceSource.resolvedFileId; + _personName = faceSource.personName ?? _personName; + cacheFaceSourceForPersonOrClusterID( + widget.personId ?? widget.clusterID!, + faceSource, + ); + } + + bool _canUseResolvedFaceSource( + PersonFaceSource faceSource, + String personOrClusterId, + ) { + final currentFaceID = + checkInMemoryCachedFaceIDForPersonOrClusterID(personOrClusterId); + return currentFaceID == null || currentFaceID == faceSource.face.faceID; + } + + Future _resolveFaceSource({ + bool notifyOnError = true, + }) async { + final personOrClusterId = widget.personId ?? widget.clusterID!; + + if (widget.initialFaceSource != null && + _canUseResolvedFaceSource( + widget.initialFaceSource!, + personOrClusterId, + )) { + _applyResolvedFaceSource(widget.initialFaceSource!); + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + widget.initialFaceSource!.face.faceID, ); - return null; + return widget.initialFaceSource; + } + + if (_resolvedFaceSource != null) { + return _resolvedFaceSource; + } + + final cachedFaceSource = + checkCachedFaceSourceForPersonOrClusterID(personOrClusterId); + if (cachedFaceSource != null && + _canUseResolvedFaceSource(cachedFaceSource, personOrClusterId)) { + _applyResolvedFaceSource(cachedFaceSource); + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + cachedFaceSource.face.faceID, + ); + return cachedFaceSource; } - _showingFallback = true; - _fallbackEverUsed = true; - final generation = ++_upgradeGeneration; - unawaited(_attemptFullQualityUpgrade(generation)); - return thumbnailCrop; + try { + String? fixedFaceID = widget.initialAvatarFaceId; + final mlDataDB = + isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; + if (isPerson && !isOfflineMode && fixedFaceID == null) { + final personEntity = + await PersonService.instance.getPerson(widget.personId!); + if (personEntity == null) { + _logger.severe( + 'Person with ID ${widget.personId} not found, cannot get cover face.', + ); + return null; + } + _personName = personEntity.data.name; + fixedFaceID = personEntity.data.avatarFaceID; + } + fixedFaceID ??= await checkUsedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + + EnteFile? selectedFileForFaceCrop; + if (isOfflineMode) { + final allFiles = await SearchService.instance.getAllFilesForSearch(); + final localIdToFile = {}; + for (final file in allFiles) { + final localId = file.localID; + if (localId != null && localId.isNotEmpty) { + localIdToFile[localId] = file; + } + } + + if (fixedFaceID != null) { + final localIntId = getFileIdFromFaceId(fixedFaceID); + final localId = + await OfflineFilesDB.instance.getLocalIdForIntId(localIntId); + if (localId == null) { + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } else { + selectedFileForFaceCrop = localIdToFile[localId]; + if (selectedFileForFaceCrop == null) { + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } + } + } + + if (selectedFileForFaceCrop == null) { + final List allFaces = isPerson + ? await mlDataDB + .getFaceIDsForPersonOrderedByScore(widget.personId!) + : await mlDataDB + .getFaceIDsForClusterOrderedByScore(widget.clusterID!); + final localIntIds = allFaces + .map((faceID) => getFileIdFromFaceId(faceID)) + .toSet(); + final localIdMap = + await OfflineFilesDB.instance.getLocalIdsForIntIds(localIntIds); + for (final faceID in allFaces) { + final localIntId = getFileIdFromFaceId(faceID); + final localId = localIdMap[localIntId]; + final candidate = localId != null ? localIdToFile[localId] : null; + if (candidate != null) { + selectedFileForFaceCrop = candidate; + fixedFaceID = faceID; + break; + } + } + } + + if (selectedFileForFaceCrop == null) { + final initialPreviewFile = widget.initialPreviewFile; + final localId = initialPreviewFile?.localID; + if (localId != null && localId.isNotEmpty) { + selectedFileForFaceCrop = localIdToFile[localId]; + } + } + + if (selectedFileForFaceCrop == null) { + _logger.severe( + 'No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; + } + } else { + final hiddenFileIDs = + await SearchService.instance.getHiddenFiles().then( + (files) => files + .map((file) => file.uploadedFileID) + .whereType() + .toSet(), + ); + + final initialPreviewFile = widget.initialPreviewFile; + if (fixedFaceID != null) { + final fileID = getFileIdFromFaceId(fixedFaceID); + if (initialPreviewFile?.uploadedFileID == fileID && + !hiddenFileIDs.contains(fileID)) { + selectedFileForFaceCrop = initialPreviewFile; + } else { + final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); + if (fileInDB == null) { + _logger.severe( + 'File with ID $fileID not found in DB, cannot get cover face.', + ); + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } else if (hiddenFileIDs.contains(fileInDB.uploadedFileID)) { + _logger.info( + 'File with ID $fileID is hidden, skipping it for face crop.', + ); + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } else { + selectedFileForFaceCrop = fileInDB; + } + } + } + + if (selectedFileForFaceCrop == null) { + final List allFaces = isPerson + ? await mlDataDB + .getFaceIDsForPersonOrderedByScore(widget.personId!) + : await mlDataDB + .getFaceIDsForClusterOrderedByScore(widget.clusterID!); + for (final faceID in allFaces) { + final fileID = getFileIdFromFaceId(faceID); + if (hiddenFileIDs.contains(fileID)) { + _logger.info( + 'File with ID $fileID is hidden, skipping it for face crop.', + ); + continue; + } + selectedFileForFaceCrop = + await FilesDB.instance.getAnyUploadedFile(fileID); + if (selectedFileForFaceCrop != null) { + _logger.info( + 'Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + fixedFaceID = faceID; + break; + } + } + } + + if (selectedFileForFaceCrop == null && + initialPreviewFile != null && + initialPreviewFile.uploadedFileID != null && + !hiddenFileIDs.contains(initialPreviewFile.uploadedFileID)) { + selectedFileForFaceCrop = initialPreviewFile; + } + + if (selectedFileForFaceCrop == null) { + _logger.severe( + 'No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; + } + } + + final int? recentFileID; + if (isOfflineMode) { + final localId = selectedFileForFaceCrop.localID; + if (localId == null || localId.isEmpty) { + _logger.severe( + 'Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; + } + recentFileID = + await OfflineFilesDB.instance.getOrCreateLocalIntId(localId); + } else { + recentFileID = selectedFileForFaceCrop.uploadedFileID; + } + if (recentFileID == null) { + _logger.severe( + 'Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; + } + + final face = await mlDataDB.getCoverFaceForPerson( + recentFileID: recentFileID, + avatarFaceId: fixedFaceID, + personID: widget.personId, + clusterID: widget.clusterID, + ); + if (face == null) { + _logger.severe( + 'No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', + ); + await checkRemoveCachedFaceIDForPersonOrClusterId(personOrClusterId); + return null; + } + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + face.faceID, + ); + + final faceSource = PersonFaceSource( + file: selectedFileForFaceCrop, + face: face, + resolvedFileId: recentFileID, + personName: _personName ?? widget.initialPersonName, + ); + _applyResolvedFaceSource(faceSource); + return faceSource; + } catch (e, s) { + _logger.severe( + 'Error resolving face source for person: ${widget.personId} or cluster ${widget.clusterID}', + e, + s, + ); + if (notifyOnError) { + widget.onErrorCallback?.call(); + } + return null; + } } Future _attemptFullQualityUpgrade(int generation) async { @@ -265,8 +690,8 @@ class _PersonFaceWidgetState extends State return; } - final Face? face = _faceForFaceCrop; - final EnteFile? sourceFile = fileForFaceCrop; + final face = _faceForFaceCrop; + final sourceFile = fileForFaceCrop; if (face == null || sourceFile == null) { _logger.fine( 'person_face_thumbnail_upgrade_skipped reason=missing_face_or_file person=${widget.personId} cluster=${widget.clusterID}', @@ -291,7 +716,7 @@ class _PersonFaceWidgetState extends State _logger.fine( 'person_face_thumbnail_upgrade_skipped reason=full_crop_cached face=${face.faceID}', ); - final Uint8List? fullCrop = await _getFaceCrop( + final fullCrop = await _getFaceCrop( useFullFile: true, notifyOnError: false, ); @@ -300,7 +725,12 @@ class _PersonFaceWidgetState extends State } setState(() { _showingFallback = false; - faceCropFuture = Future.value(fullCrop); + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); }); return; } @@ -375,7 +805,7 @@ class _PersonFaceWidgetState extends State _logger.info( 'person_face_thumbnail_upgrade_started person=${widget.personId} cluster=${widget.clusterID}', ); - final Uint8List? fullCrop = await _getFaceCrop( + final fullCrop = await _getFaceCrop( useFullFile: true, notifyOnError: false, ); @@ -391,7 +821,12 @@ class _PersonFaceWidgetState extends State } setState(() { _showingFallback = false; - faceCropFuture = Future.value(fullCrop); + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); }); _logger.info( @@ -424,6 +859,7 @@ class _PersonFaceWidgetState extends State Future _getFaceCrop({ required bool useFullFile, + PersonFaceSource? resolvedSource, bool notifyOnError = true, }) async { try { @@ -442,194 +878,25 @@ class _PersonFaceWidgetState extends State return tryInMemoryCachedThumbnailCrop; } } - String? fixedFaceID; - PersonEntity? personEntity; - final mlDataDB = - isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; - if (isPerson && !isOfflineMode) { - personEntity = await PersonService.instance.getPerson(widget.personId!); - if (personEntity == null) { - _logger.severe( - 'Person with ID ${widget.personId} not found, cannot get cover face.', - ); - return null; - } - _personName = personEntity.data.name; - fixedFaceID = personEntity.data.avatarFaceID; - } - fixedFaceID ??= - await checkUsedFaceIDForPersonOrClusterId(personOrClusterId); - - EnteFile? selectedFileForFaceCrop; - if (isOfflineMode) { - final allFiles = await SearchService.instance.getAllFilesForSearch(); - final localIdToFile = {}; - for (final file in allFiles) { - final localId = file.localID; - if (localId != null && localId.isNotEmpty) { - localIdToFile[localId] = file; - } - } - if (fixedFaceID != null) { - final localIntId = getFileIdFromFaceId(fixedFaceID); - final localId = - await OfflineFilesDB.instance.getLocalIdForIntId(localIntId); - if (localId == null) { - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); - } else { - selectedFileForFaceCrop = localIdToFile[localId]; - if (selectedFileForFaceCrop == null) { - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); - } - } - } - if (selectedFileForFaceCrop == null) { - final List allFaces = isPerson - ? await mlDataDB - .getFaceIDsForPersonOrderedByScore(widget.personId!) - : await mlDataDB - .getFaceIDsForClusterOrderedByScore(widget.clusterID!); - final localIntIds = allFaces - .map((faceID) => getFileIdFromFaceId(faceID)) - .toSet(); - final localIdMap = - await OfflineFilesDB.instance.getLocalIdsForIntIds(localIntIds); - for (final faceID in allFaces) { - final localIntId = getFileIdFromFaceId(faceID); - final localId = localIdMap[localIntId]; - final candidate = localId != null ? localIdToFile[localId] : null; - if (candidate != null) { - selectedFileForFaceCrop = candidate; - fixedFaceID = faceID; - break; - } - } - if (selectedFileForFaceCrop == null) { - _logger.severe( - 'No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', - ); - return null; - } - } - } else { - final hiddenFileIDs = - await SearchService.instance.getHiddenFiles().then( - (files) => files - .map((file) => file.uploadedFileID) - .whereType() - .toSet(), - ); - if (fixedFaceID != null) { - final fileID = getFileIdFromFaceId(fixedFaceID); - final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); - if (fileInDB == null) { - _logger.severe( - 'File with ID $fileID not found in DB, cannot get cover face.', - ); - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); - } else if (hiddenFileIDs.contains(fileInDB.uploadedFileID)) { - _logger.info( - 'File with ID $fileID is hidden, skipping it for face crop.', - ); - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); - } else { - selectedFileForFaceCrop = fileInDB; - } - } - if (selectedFileForFaceCrop == null) { - final List allFaces = isPerson - ? await mlDataDB - .getFaceIDsForPersonOrderedByScore(widget.personId!) - : await mlDataDB - .getFaceIDsForClusterOrderedByScore(widget.clusterID!); - for (final faceID in allFaces) { - final fileID = getFileIdFromFaceId(faceID); - if (hiddenFileIDs.contains(fileID)) { - _logger.info( - 'File with ID $fileID is hidden, skipping it for face crop.', - ); - continue; - } - selectedFileForFaceCrop = - await FilesDB.instance.getAnyUploadedFile(fileID); - if (selectedFileForFaceCrop != null) { - _logger.info( - 'Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', - ); - fixedFaceID = faceID; - break; - } - } - if (selectedFileForFaceCrop == null) { - _logger.severe( - 'No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', - ); - return null; - } - } - } - - int? recentFileID; - if (isOfflineMode) { - final localId = selectedFileForFaceCrop.localID; - if (localId == null || localId.isEmpty) { - _logger.severe( - 'Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', - ); - return null; - } - recentFileID = - await OfflineFilesDB.instance.getOrCreateLocalIntId(localId); - } else { - recentFileID = selectedFileForFaceCrop.uploadedFileID; - } - if (recentFileID == null) { - _logger.severe( - 'Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', - ); - return null; - } - final Face? face = await mlDataDB.getCoverFaceForPerson( - recentFileID: recentFileID, - avatarFaceId: fixedFaceID, - personID: widget.personId, - clusterID: widget.clusterID, - ); - if (face == null) { - _logger.severe( - 'No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', - ); - await checkRemoveCachedFaceIDForPersonOrClusterId(personOrClusterId); + final faceSource = resolvedSource ?? + await _resolveFaceSource(notifyOnError: notifyOnError); + if (faceSource == null) { return null; } - await cacheFaceIdForPersonOrClusterIfNeeded( - personOrClusterId, - face.faceID, - ); final cropMap = await getCachedFaceCrops( - selectedFileForFaceCrop, - [face], + faceSource.file, + [faceSource.face], useFullFile: useFullFile, personOrClusterID: personOrClusterId, useTempCache: false, ); - fileForFaceCrop = selectedFileForFaceCrop; - _faceForFaceCrop = face; - _faceCropFileId = recentFileID; - final result = cropMap?[face.faceID]; + _applyResolvedFaceSource(faceSource); + final result = cropMap?[faceSource.face.faceID]; if (result == null) { _logger.severe( - 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', + 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${faceSource.resolvedFileId}', ); } return result; @@ -647,6 +914,69 @@ class _PersonFaceWidgetState extends State } } +class _DirectThumbnailFacePreview extends StatelessWidget { + final Uint8List thumbnailBytes; + final FaceBox faceBox; + final ImageDimensions imageDimensions; + + const _DirectThumbnailFacePreview({ + required this.thumbnailBytes, + required this.faceBox, + required this.imageDimensions, + }); + + @override + Widget build(BuildContext context) { + final crop = computeNormalizedFaceCrop(faceBox); + if (crop == null) { + return const SizedBox.shrink(); + } + + const imageHeight = 1000.0; + final imageWidth = + imageHeight * imageDimensions.width / imageDimensions.height; + final cropX = crop.x * imageWidth; + final cropY = crop.y * imageHeight; + final cropWidth = crop.width * imageWidth; + final cropHeight = crop.height * imageHeight; + + if (cropWidth <= 0 || cropHeight <= 0) { + return const SizedBox.shrink(); + } + + return ClipRect( + child: SizedBox.expand( + child: FittedBox( + fit: BoxFit.cover, + child: SizedBox( + width: cropWidth, + height: cropHeight, + child: ClipRect( + child: Stack( + fit: StackFit.expand, + children: [ + Positioned( + left: -cropX, + top: -cropY, + width: imageWidth, + height: imageHeight, + child: Image.memory( + thumbnailBytes, + fit: BoxFit.fill, + gaplessPlayback: true, + filterQuality: FilterQuality.low, + ), + ), + ], + ), + ), + ), + ), + ), + ); + } +} + class _EmptyPersonThumbnail extends StatelessWidget { final String? initial; diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart index eb6c3373f37..f89cd53f021 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart @@ -8,6 +8,7 @@ import "package:photos/events/event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/events/people_sort_order_change_event.dart"; import "package:photos/generated/intl/app_localizations.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/recent_searches.dart"; import "package:photos/models/search/search_constants.dart"; @@ -302,10 +303,16 @@ class FaceSearchResult extends StatelessWidget { final params = (searchResult as GenericSearchResult).params; final int cachedPixelWidth = (displaySize * MediaQuery.devicePixelRatioOf(context)).toInt(); + final previewFile = searchResult.previewThumbnail(); + final personName = searchResult.name(); return PersonFaceWidget( personId: params[kPersonParamID], clusterID: params[kClusterParamId], cachedPixelWidth: cachedPixelWidth, + initialFaceSource: params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: personName.isNotEmpty ? personName : null, + initialAvatarFaceId: params[kPersonAvatarFaceID] as String?, + initialPreviewFile: previewFile, key: params.containsKey(kPersonWidgetKey) ? ValueKey(params[kPersonWidgetKey]) : ValueKey(params[kPersonParamID] ?? params[kClusterParamId]), diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart b/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart index 2a0cc624ebb..20ab3eaea0b 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart @@ -2,6 +2,7 @@ import "package:flutter/material.dart"; import "package:logging/logging.dart"; import "package:photos/models/api/collection/user.dart"; import 'package:photos/models/file/file.dart'; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/search_constants.dart"; import "package:photos/models/search/search_result.dart"; @@ -41,6 +42,14 @@ class SearchThumbnailWidget extends StatelessWidget { .params[kPersonParamID], clusterID: (searchResult as GenericSearchResult) .params[kClusterParamId], + initialFaceSource: (searchResult as GenericSearchResult) + .params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: searchResult!.name().isNotEmpty + ? searchResult!.name() + : null, + initialAvatarFaceId: (searchResult as GenericSearchResult) + .params[kPersonAvatarFaceID] as String?, + initialPreviewFile: searchResult!.previewThumbnail(), ) : ThumbnailWidget( file!, diff --git a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart index 942fb477133..71683004a5e 100644 --- a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart @@ -7,6 +7,7 @@ import "package:photos/events/event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/ml/face/person.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/recent_searches.dart"; import "package:photos/models/search/search_constants.dart"; @@ -389,10 +390,16 @@ class FaceSearchResult extends StatelessWidget { final params = (searchResult as GenericSearchResult).params; final int cachedPixelWidth = (displaySize * MediaQuery.devicePixelRatioOf(context)).toInt(); + final previewFile = searchResult.previewThumbnail(); + final personName = searchResult.name(); return PersonFaceWidget( personId: params[kPersonParamID], clusterID: params[kClusterParamId], cachedPixelWidth: cachedPixelWidth, + initialFaceSource: params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: personName.isNotEmpty ? personName : null, + initialAvatarFaceId: params[kPersonAvatarFaceID] as String?, + initialPreviewFile: previewFile, key: params.containsKey(kPersonWidgetKey) ? ValueKey(params[kPersonWidgetKey]) : ValueKey(params[kPersonParamID] ?? params[kClusterParamId]), diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index a72ddb0a63c..c57f19ddca9 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -13,6 +13,7 @@ import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/face/face.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/service_locator.dart" show flagService, isOfflineMode; import "package:photos/services/machine_learning/face_thumbnail_generator.dart"; import "package:photos/utils/file_util.dart"; @@ -27,6 +28,8 @@ final LRUMap _thumbnailSourceDimensionsByFileId = LRUMap(2000); final LRUMap _personOrClusterIdToCachedFaceID = LRUMap(2000); +final LRUMap _personOrClusterIdToFaceSourceCache = + LRUMap(1000); TaskQueue _queueFullFileFaceGenerations = TaskQueue( maxConcurrentTasks: 5, @@ -49,6 +52,12 @@ Uint8List? checkInMemoryCachedCropForPersonOrClusterID( return cachedCover; } +String? checkInMemoryCachedFaceIDForPersonOrClusterID( + String personOrClusterID, +) { + return _personOrClusterIdToCachedFaceID.get(personOrClusterID); +} + Uint8List? checkInMemoryCachedThumbnailCropForPersonOrClusterID( String personOrClusterID, ) { @@ -64,6 +73,19 @@ Uint8List? checkInMemoryCachedThumbnailCropForPersonOrClusterID( return _thumbnailSourceDimensionsByFileId.get(fileId); } +PersonFaceSource? checkCachedFaceSourceForPersonOrClusterID( + String personOrClusterID, +) { + return _personOrClusterIdToFaceSourceCache.get(personOrClusterID); +} + +void cacheFaceSourceForPersonOrClusterID( + String personOrClusterID, + PersonFaceSource faceSource, +) { + _personOrClusterIdToFaceSourceCache.put(personOrClusterID, faceSource); +} + Uint8List? _checkInMemoryCachedCropForFaceID(String faceID) { final Uint8List? cachedCover = _faceCropCache.get(faceID); return cachedCover; @@ -93,6 +115,11 @@ Future putFaceIdCachedForPersonOrCluster( personOrClusterID, faceID, ); + final cachedFaceSource = + _personOrClusterIdToFaceSourceCache.get(personOrClusterID); + if (cachedFaceSource != null && cachedFaceSource.face.faceID != faceID) { + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); + } _personOrClusterIdToCachedFaceID.put(personOrClusterID, faceID); } @@ -125,6 +152,7 @@ Future checkRemoveCachedFaceIDForPersonOrClusterId( await mlDataDB.getFaceIdUsedForPersonOrCluster(personOrClusterID); if (cachedFaceID != null) { _personOrClusterIdToCachedFaceID.remove(personOrClusterID); + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); await mlDataDB.removeFaceIdCachedForPersonOrCluster(personOrClusterID); } } diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart index 2ab7bc039c3..2500c8dbc6f 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart @@ -22,6 +22,12 @@ class FaceThumbnailUpgradeDecision { } typedef ImageDimensions = ({int width, int height}); +typedef NormalizedFaceCrop = ({ + double x, + double y, + double width, + double height +}); ImageDimensions? estimateThumbnailDimensionsFromFullDimensions({ required int fullWidth, @@ -121,53 +127,65 @@ FaceThumbnailUpgradeDecision shouldUpgradeFromThumbnail({ ); } -double? _computeCropShortSide( - FaceBox faceBox, { - required int imageWidth, - required int imageHeight, -}) { - if (imageWidth <= 0 || imageHeight <= 0) { - return null; - } - - final width = imageWidth.toDouble(); - final height = imageHeight.toDouble(); - - final xMinAbs = faceBox.x * width; - final yMinAbs = faceBox.y * height; - final widthAbs = faceBox.width * width; - final heightAbs = faceBox.height * height; - - if (widthAbs <= 0 || heightAbs <= 0) { +NormalizedFaceCrop? computeNormalizedFaceCrop(FaceBox faceBox) { + final widthNorm = faceBox.width; + final heightNorm = faceBox.height; + if (widthNorm <= 0 || heightNorm <= 0) { return null; } - final xCrop = xMinAbs - widthAbs * kFaceThumbnailRegularPadding; - final xOvershoot = (xCrop < 0 ? -xCrop : 0) / widthAbs; - final widthCrop = widthAbs * (1 + 2 * kFaceThumbnailRegularPadding) - + final xCrop = faceBox.x - widthNorm * kFaceThumbnailRegularPadding; + final xOvershoot = (xCrop < 0 ? -xCrop : 0) / widthNorm; + final widthCrop = widthNorm * (1 + 2 * kFaceThumbnailRegularPadding) - 2 * _min( xOvershoot, kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, ) * - widthAbs; + widthNorm; - final yCrop = yMinAbs - heightAbs * kFaceThumbnailRegularPadding; - final yOvershoot = (yCrop < 0 ? -yCrop : 0) / heightAbs; - final heightCrop = heightAbs * (1 + 2 * kFaceThumbnailRegularPadding) - + final yCrop = faceBox.y - heightNorm * kFaceThumbnailRegularPadding; + final yOvershoot = (yCrop < 0 ? -yCrop : 0) / heightNorm; + final heightCrop = heightNorm * (1 + 2 * kFaceThumbnailRegularPadding) - 2 * _min( yOvershoot, kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, ) * - heightAbs; + heightNorm; + + final xCropSafe = xCrop.clamp(0, 1).toDouble(); + final yCropSafe = yCrop.clamp(0, 1).toDouble(); + final widthCropSafe = widthCrop.clamp(0, 1 - xCropSafe).toDouble(); + final heightCropSafe = heightCrop.clamp(0, 1 - yCropSafe).toDouble(); + if (widthCropSafe <= 0 || heightCropSafe <= 0) { + return null; + } - final xCropSafe = xCrop.clamp(0, width).toDouble(); - final yCropSafe = yCrop.clamp(0, height).toDouble(); - final widthCropSafe = widthCrop.clamp(0, width - xCropSafe).toDouble(); - final heightCropSafe = heightCrop.clamp(0, height - yCropSafe).toDouble(); + return ( + x: xCropSafe, + y: yCropSafe, + width: widthCropSafe, + height: heightCropSafe, + ); +} - final shortSide = _min(widthCropSafe, heightCropSafe); +double? _computeCropShortSide( + FaceBox faceBox, { + required int imageWidth, + required int imageHeight, +}) { + if (imageWidth <= 0 || imageHeight <= 0) { + return null; + } + final crop = computeNormalizedFaceCrop(faceBox); + if (crop == null) { + return null; + } + final shortSide = _min( + crop.width * imageWidth, + crop.height * imageHeight, + ); return shortSide > 0 ? shortSide : null; } From ba3af2ea08ad723601e94f9237a43edc352472a3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 17 Mar 2026 15:56:30 +0530 Subject: [PATCH 08/18] Prioritize visible face thumbnail work --- .../photos/lib/services/search_service.dart | 76 ++++++++-- .../ui/viewer/people/person_face_widget.dart | 138 ++++++++++++------ .../people/visible_face_source_prefetch.dart | 81 ++++++++++ .../result/people_section_all_page.dart | 93 +++++++----- .../ui/viewer/search_tab/people_section.dart | 15 +- .../lib/utils/face/face_thumbnail_cache.dart | 17 ++- .../ente_pure_utils/lib/src/task_queue.dart | 21 ++- 7 files changed, 342 insertions(+), 99 deletions(-) create mode 100644 mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart diff --git a/mobile/apps/photos/lib/services/search_service.dart b/mobile/apps/photos/lib/services/search_service.dart index 7194035b4e9..c2045c8d230 100644 --- a/mobile/apps/photos/lib/services/search_service.dart +++ b/mobile/apps/photos/lib/services/search_service.dart @@ -73,6 +73,7 @@ class SearchService { Future>? _cachedFilesForHierarchicalSearch; Future>? _cachedFilesForGenericGallery; Future>? _cachedHiddenFilesFuture; + final Set _faceSourcePrefetchInFlight = {}; final _logger = Logger((SearchService).toString()); final _collectionService = CollectionsService.instance; static const _maximumResultsLimit = 20; @@ -202,25 +203,70 @@ class SearchService { Future _enrichInitialFaceSources( List results, ) async { - final resultsToPrefetch = results.take(_initialFaceSourcePrefetchCount); + await prefetchFaceSourcesInWindow( + results, + startIndex: 0, + count: _initialFaceSourcePrefetchCount, + ); + } + + Future prefetchFaceSourcesInWindow( + List results, { + required int startIndex, + int count = _initialFaceSourcePrefetchCount, + int leadingBuffer = 0, + }) async { + if (results.isEmpty || count <= 0) { + return; + } + final safeStart = max(0, startIndex - leadingBuffer); + final safeEnd = min(results.length, safeStart + count); + if (safeStart >= safeEnd) { + return; + } await Future.wait( - resultsToPrefetch.map((result) async { - try { - final source = await _tryBuildInitialPersonFaceSource(result); - if (source != null) { - result.params[kPersonFaceSource] = source; - } - } catch (e, s) { - _logger.fine( - "Failed to prefetch initial face source for ${result.params[kPersonParamID]}", - e, - s, - ); - } - }), + results.sublist(safeStart, safeEnd).map(_prefetchFaceSourceIfNeeded), ); } + Future _prefetchFaceSourceIfNeeded( + GenericSearchResult result, + ) async { + if (result.params[kPersonFaceSource] != null) { + return; + } + final prefetchKey = _faceSourcePrefetchKey(result); + if (prefetchKey == null || !_faceSourcePrefetchInFlight.add(prefetchKey)) { + return; + } + try { + final source = await _tryBuildInitialPersonFaceSource(result); + if (source != null) { + result.params[kPersonFaceSource] = source; + } + } catch (e, s) { + _logger.fine( + "Failed to prefetch initial face source for ${result.params[kPersonParamID] ?? result.params[kClusterParamId]}", + e, + s, + ); + } finally { + _faceSourcePrefetchInFlight.remove(prefetchKey); + } + } + + String? _faceSourcePrefetchKey(GenericSearchResult result) { + final personID = result.params[kPersonParamID] as String?; + if (personID != null && personID.isNotEmpty) { + return "person:$personID"; + } + final clusterID = result.params[kClusterParamId] as String?; + if (clusterID != null && clusterID.isNotEmpty) { + return "cluster:$clusterID"; + } + return null; + } + Future _tryBuildInitialPersonFaceSource( GenericSearchResult result, ) async { diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index e405b572d5c..b3c41fb260e 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -23,12 +23,14 @@ import 'package:photos/ui/common/loading_widget.dart'; import 'package:photos/utils/face/face_thumbnail_cache.dart'; import 'package:photos/utils/face/face_thumbnail_quality.dart'; import 'package:photos/utils/thumbnail_util.dart'; +import 'package:visibility_detector/visibility_detector.dart'; final _logger = Logger('PersonFaceWidget'); const _kMinUnnamedClusterSizeForProgressiveUpgrade = 5; const _kProgressiveUpgradeUpscaleThreshold = 1.35; const _kProgressiveUpgradeMinImprovementRatio = 1.2; const _kProgressiveUpgradeIdleWaitBudget = Duration(seconds: 2); +const _kVisibleTaskTouchInterval = Duration(seconds: 20); class _PersonFaceLoadResult { final Uint8List? faceCropBytes; @@ -117,6 +119,8 @@ class _PersonFaceWidgetState extends State bool _disposed = false; int _upgradeGeneration = 0; EnteFile? _requestedThumbnailPreviewFile; + bool _isVisible = false; + Timer? _visibleTaskTouchTimer; static final LRUMap _clusterToFileCountCache = LRUMap(1000); @@ -142,6 +146,7 @@ class _PersonFaceWidgetState extends State void dispose() { _disposed = true; _upgradeGeneration += 1; + _cancelVisibleTaskTouchTimer(); if (_faceCropFileId != null) { if (widget.useFullFile) { checkStopTryingToGenerateFaceThumbnails( @@ -166,51 +171,57 @@ class _PersonFaceWidgetState extends State Widget build(BuildContext context) { super.build(context); - return FutureBuilder<_PersonFaceLoadResult?>( - future: _faceLoadFuture, - builder: (context, snapshot) { - final loadResult = snapshot.data; - if (loadResult?.faceCropBytes != null) { - return _buildFaceImage( - _buildImageFromBytes(loadResult!.faceCropBytes!), - ); - } - if (loadResult?.thumbnailBytes != null && - loadResult?.faceSource != null) { - final previewDimensions = _previewImageDimensionsForSource( - loadResult!.faceSource!, - ); - if (previewDimensions != null) { + return VisibilityDetector( + key: ValueKey( + 'person_face_visibility_${_visibilityKeySuffix()}_${identityHashCode(this)}', + ), + onVisibilityChanged: _handleVisibilityChanged, + child: FutureBuilder<_PersonFaceLoadResult?>( + future: _faceLoadFuture, + builder: (context, snapshot) { + final loadResult = snapshot.data; + if (loadResult?.faceCropBytes != null) { return _buildFaceImage( - _DirectThumbnailFacePreview( - thumbnailBytes: loadResult.thumbnailBytes!, - faceBox: loadResult.faceSource!.face.detection.box, - imageDimensions: previewDimensions, - ), + _buildImageFromBytes(loadResult!.faceCropBytes!), ); } - } - if (snapshot.connectionState == ConnectionState.waiting || - snapshot.connectionState == ConnectionState.active) { - return EnteLoadingWidget( - color: getEnteColorScheme(context).fillMuted, - ); - } - if (snapshot.hasError) { - _logger.severe( - 'Error getting cover face for person', - snapshot.error, - snapshot.stackTrace, - ); - } else { - _logger.severe( - 'No cover face found for person or cluster.', + if (loadResult?.thumbnailBytes != null && + loadResult?.faceSource != null) { + final previewDimensions = _previewImageDimensionsForSource( + loadResult!.faceSource!, + ); + if (previewDimensions != null) { + return _buildFaceImage( + _DirectThumbnailFacePreview( + thumbnailBytes: loadResult.thumbnailBytes!, + faceBox: loadResult.faceSource!.face.detection.box, + imageDimensions: previewDimensions, + ), + ); + } + } + if (snapshot.connectionState == ConnectionState.waiting || + snapshot.connectionState == ConnectionState.active) { + return EnteLoadingWidget( + color: getEnteColorScheme(context).fillMuted, + ); + } + if (snapshot.hasError) { + _logger.severe( + 'Error getting cover face for person', + snapshot.error, + snapshot.stackTrace, + ); + } else { + _logger.severe( + 'No cover face found for person or cluster.', + ); + } + return _EmptyPersonThumbnail( + initial: isPerson ? (loadResult?.personName ?? _personName) : null, ); - } - return _EmptyPersonThumbnail( - initial: isPerson ? (loadResult?.personName ?? _personName) : null, - ); - }, + }, + ), ); } @@ -412,6 +423,51 @@ class _PersonFaceWidgetState extends State widget.personId ?? widget.clusterID!, faceSource, ); + _touchVisibleFaceTasks(); + } + + void _handleVisibilityChanged(VisibilityInfo info) { + final nowVisible = info.visibleFraction >= 0.01; + if (nowVisible == _isVisible) { + return; + } + _isVisible = nowVisible; + if (_isVisible) { + _touchVisibleFaceTasks(); + _visibleTaskTouchTimer = Timer.periodic( + _kVisibleTaskTouchInterval, + (_) => _touchVisibleFaceTasks(), + ); + } else { + _cancelVisibleTaskTouchTimer(); + } + } + + void _touchVisibleFaceTasks() { + if (!_isVisible) { + return; + } + final fileId = _faceCropFileId; + if (fileId == null) { + return; + } + touchPendingFaceThumbnailGeneration(fileId, useFullFile: false); + if (widget.useFullFile) { + touchPendingFaceThumbnailGeneration(fileId, useFullFile: true); + } + } + + void _cancelVisibleTaskTouchTimer() { + _visibleTaskTouchTimer?.cancel(); + _visibleTaskTouchTimer = null; + } + + String _visibilityKeySuffix() { + final widgetKey = widget.key; + if (widgetKey is ValueKey) { + return widgetKey.value.toString(); + } + return widget.personId ?? widget.clusterID ?? widgetKey.toString(); } bool _canUseResolvedFaceSource( diff --git a/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart new file mode 100644 index 00000000000..3a3c55592f1 --- /dev/null +++ b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart @@ -0,0 +1,81 @@ +import "dart:async"; + +import "package:flutter/widgets.dart"; +import "package:photos/models/search/generic_search_result.dart"; +import "package:photos/models/search/search_constants.dart"; +import "package:photos/services/search_service.dart"; +import "package:visibility_detector/visibility_detector.dart"; + +class VisibleFaceSourcePrefetch extends StatefulWidget { + final List results; + final int index; + final int prefetchCount; + final int leadingBuffer; + final Widget child; + + const VisibleFaceSourcePrefetch({ + required this.results, + required this.index, + required this.child, + this.prefetchCount = 24, + this.leadingBuffer = 0, + super.key, + }); + + @override + State createState() => + _VisibleFaceSourcePrefetchState(); +} + +class _VisibleFaceSourcePrefetchState extends State { + bool _hasPrefetchedForCurrentItem = false; + + @override + void didUpdateWidget(covariant VisibleFaceSourcePrefetch oldWidget) { + super.didUpdateWidget(oldWidget); + if (oldWidget.index != widget.index || + !identical(oldWidget.results, widget.results)) { + _hasPrefetchedForCurrentItem = false; + } + } + + @override + Widget build(BuildContext context) { + return VisibilityDetector( + key: ValueKey( + "face_source_prefetch_${_visibilityTargetKey()}_${widget.index}", + ), + onVisibilityChanged: (info) { + if (info.visibleFraction >= 0.01) { + _maybePrefetchWindow(); + } + }, + child: widget.child, + ); + } + + void _maybePrefetchWindow() { + if (_hasPrefetchedForCurrentItem) { + return; + } + _hasPrefetchedForCurrentItem = true; + unawaited( + SearchService.instance.prefetchFaceSourcesInWindow( + widget.results, + startIndex: widget.index, + count: widget.prefetchCount, + leadingBuffer: widget.leadingBuffer, + ), + ); + } + + String _visibilityTargetKey() { + if (widget.index < 0 || widget.index >= widget.results.length) { + return widget.index.toString(); + } + final result = widget.results[widget.index]; + final personID = result.params[kPersonParamID] as String?; + final clusterID = result.params[kClusterParamId] as String?; + return personID ?? clusterID ?? widget.index.toString(); + } +} diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart index f89cd53f021..5c4991ac288 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart @@ -32,6 +32,7 @@ import "package:photos/ui/viewer/people/face_thumbnail_squircle.dart"; import "package:photos/ui/viewer/people/person_face_widget.dart"; import "package:photos/ui/viewer/people/person_gallery_suggestion.dart"; import "package:photos/ui/viewer/people/pinned_person_badge.dart"; +import "package:photos/ui/viewer/people/visible_face_source_prefetch.dart"; import "package:photos/ui/viewer/search/result/search_result_page.dart"; import "package:photos/ui/viewer/search_tab/people_section.dart"; import "package:photos/utils/local_settings.dart"; @@ -687,6 +688,8 @@ class _PeopleSectionAllWidgetState extends State { } final screenWidth = MediaQuery.of(context).size.width; final crossAxisCount = (screenWidth / 100).floor(); + final visibleFacePrefetchCount = crossAxisCount * 6; + final visibleFaceLeadingBuffer = crossAxisCount; final itemSize = (screenWidth - ((horizontalEdgePadding * 2) + @@ -719,18 +722,24 @@ class _PeopleSectionAllWidgetState extends State { childCount: searchResults.length, (context, index) { final result = searchResults[index]; - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: result, - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: defaultFaces.contains(result), - ) - : PersonSearchExample( - searchResult: result, - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: searchResults, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: result, + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: defaultFaces.contains(result), + ) + : PersonSearchExample( + searchResult: result, + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), @@ -795,18 +804,24 @@ class _PeopleSectionAllWidgetState extends State { delegate: SliverChildBuilderDelegate( childCount: filteredNormalFaces.length, (context, index) { - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: filteredNormalFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: true, - ) - : PersonSearchExample( - searchResult: filteredNormalFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: filteredNormalFaces, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: filteredNormalFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: true, + ) + : PersonSearchExample( + searchResult: filteredNormalFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), @@ -837,18 +852,24 @@ class _PeopleSectionAllWidgetState extends State { delegate: SliverChildBuilderDelegate( childCount: filteredExtraFaces.length, (context, index) { - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: filteredExtraFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: false, - ) - : PersonSearchExample( - searchResult: filteredExtraFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: filteredExtraFaces, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: filteredExtraFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: false, + ) + : PersonSearchExample( + searchResult: filteredExtraFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), diff --git a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart index 71683004a5e..330c0f59347 100644 --- a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart @@ -23,6 +23,7 @@ import "package:photos/ui/viewer/people/add_person_action_sheet.dart"; import "package:photos/ui/viewer/people/face_thumbnail_squircle.dart"; import "package:photos/ui/viewer/people/people_page.dart"; import 'package:photos/ui/viewer/people/person_face_widget.dart'; +import "package:photos/ui/viewer/people/visible_face_source_prefetch.dart"; import "package:photos/ui/viewer/search/result/people_section_all_page.dart"; import "package:photos/ui/viewer/search/result/search_result_page.dart"; import "package:photos/ui/viewer/search/search_section_cta.dart"; @@ -172,6 +173,8 @@ class SearchExampleRow extends StatelessWidget { @override Widget build(BuildContext context) { + const visibleFacePrefetchCount = 12; + const visibleFaceLeadingBuffer = 2; return SizedBox( height: 128, child: ListView.separated( @@ -180,9 +183,15 @@ class SearchExampleRow extends StatelessWidget { scrollDirection: Axis.horizontal, itemCount: examples.length, itemBuilder: (context, index) { - return PersonSearchExample( - searchResult: examples[index], - selectedPeople: null, + return VisibleFaceSourcePrefetch( + results: examples, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: PersonSearchExample( + searchResult: examples[index], + selectedPeople: null, + ), ); }, separatorBuilder: (context, index) => const SizedBox(width: 3), diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index c57f19ddca9..c31e5a7ea7a 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -22,6 +22,7 @@ import "package:photos/utils/thumbnail_util.dart"; final _logger = Logger("FaceCropUtils"); const int _retryLimit = 3; +const _kFaceGenerationTaskTimeout = Duration(minutes: 5); final LRUMap _faceCropCache = LRUMap(100); final LRUMap _faceCropThumbnailCache = LRUMap(100); final LRUMap @@ -33,12 +34,12 @@ final LRUMap _personOrClusterIdToFaceSourceCache = TaskQueue _queueFullFileFaceGenerations = TaskQueue( maxConcurrentTasks: 5, - taskTimeout: const Duration(minutes: 1), + taskTimeout: _kFaceGenerationTaskTimeout, maxQueueSize: 100, ); TaskQueue _queueThumbnailFaceGenerations = TaskQueue( maxConcurrentTasks: 5, - taskTimeout: const Duration(minutes: 1), + taskTimeout: _kFaceGenerationTaskTimeout, maxQueueSize: 100, ); @@ -354,6 +355,18 @@ void checkStopTryingToGenerateFaceThumbnails( } } +bool touchPendingFaceThumbnailGeneration( + int fileID, { + bool useFullFile = true, +}) { + final taskId = [fileID, useFullFile ? "-full" : "-thumbnail"].join(); + if (useFullFile) { + return _queueFullFileFaceGenerations.touchTask(taskId); + } else { + return _queueThumbnailFaceGenerations.touchTask(taskId); + } +} + bool areThumbnailFaceGenerationQueuesIdle() { return _queueThumbnailFaceGenerations.pendingTasksCount == 0 && _queueThumbnailFaceGenerations.runningTasksCount == 0; diff --git a/mobile/packages/ente_pure_utils/lib/src/task_queue.dart b/mobile/packages/ente_pure_utils/lib/src/task_queue.dart index d82f58a9eba..9c344da71ad 100644 --- a/mobile/packages/ente_pure_utils/lib/src/task_queue.dart +++ b/mobile/packages/ente_pure_utils/lib/src/task_queue.dart @@ -16,9 +16,11 @@ class _QueueItem { counter = 1, completer = Completer(); - void updateTimestamp() { + void updateTimestamp({bool incrementCounter = true}) { lastUpdated = DateTime.now().millisecondsSinceEpoch; - counter++; + if (incrementCounter) { + counter++; + } } bool isTimedOut(Duration timeout) { @@ -117,6 +119,21 @@ class TaskQueue { } } + /// Refresh a pending task so it stays near the front of the queue. + /// + /// Returns true only when the task is still pending. Running or missing + /// tasks cannot be reprioritized here. + bool touchTask(T id) { + final item = _taskMap[id]; + if (item == null) { + return false; + } + _priorityQueue.remove(item); + item.updateTimestamp(incrementCounter: false); + _priorityQueue.add(item); + return true; + } + /// Enforce the maximum queue size by discarding older tasks void _enforceQueueSizeLimit() { // If we're under the limit, no action needed From 028430608b0bf505f158adbbc15ac1ddf6d3216d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 17 Mar 2026 16:09:42 +0530 Subject: [PATCH 09/18] Keep first-paint face thumbnails ahead of upgrades --- .../apps/photos/lib/ui/viewer/people/person_face_widget.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index b3c41fb260e..3086ea7ef61 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -452,7 +452,9 @@ class _PersonFaceWidgetState extends State return; } touchPendingFaceThumbnailGeneration(fileId, useFullFile: false); - if (widget.useFullFile) { + // Keep first-paint work ahead of progressive upgrades. Only reprioritize + // full-file generation when the tile still has no fallback thumbnail. + if (widget.useFullFile && !_fallbackEverUsed) { touchPendingFaceThumbnailGeneration(fileId, useFullFile: true); } } From 87fbcf864771049511e3cbc649759ff1d599eab4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Tue, 17 Mar 2026 18:23:40 +0530 Subject: [PATCH 10/18] Fix hidden-face source reuse and shared queue cancellation --- .../ui/viewer/people/person_face_widget.dart | 76 +++++++++++++------ .../lib/utils/face/face_thumbnail_cache.dart | 4 + 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 3086ea7ef61..1c29d48ac1b 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -147,20 +147,6 @@ class _PersonFaceWidgetState extends State _disposed = true; _upgradeGeneration += 1; _cancelVisibleTaskTouchTimer(); - if (_faceCropFileId != null) { - if (widget.useFullFile) { - checkStopTryingToGenerateFaceThumbnails( - _faceCropFileId!, - useFullFile: true, - ); - } - if (_fallbackEverUsed || _shouldUseProgressiveStrategy) { - checkStopTryingToGenerateFaceThumbnails( - _faceCropFileId!, - useFullFile: false, - ); - } - } if (_requestedThumbnailPreviewFile != null) { removePendingGetThumbnailRequestIfAny(_requestedThumbnailPreviewFile!); } @@ -481,13 +467,47 @@ class _PersonFaceWidgetState extends State return currentFaceID == null || currentFaceID == faceSource.face.faceID; } + Future _canReuseResolvedFaceSource( + PersonFaceSource faceSource, + String personOrClusterId, { + bool clearSharedCache = false, + }) async { + if (!_canUseResolvedFaceSource(faceSource, personOrClusterId)) { + return false; + } + if (isOfflineMode) { + return true; + } + + final fileId = faceSource.file.uploadedFileID ?? faceSource.resolvedFileId; + + final hiddenFileIDs = await SearchService.instance.getHiddenFiles().then( + (files) => + files.map((file) => file.uploadedFileID).whereType().toSet(), + ); + if (!hiddenFileIDs.contains(fileId)) { + return true; + } + + _logger.info( + 'Skipping cached face source for hidden file ID $fileId for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + return false; + } + Future _resolveFaceSource({ bool notifyOnError = true, }) async { final personOrClusterId = widget.personId ?? widget.clusterID!; if (widget.initialFaceSource != null && - _canUseResolvedFaceSource( + await _canReuseResolvedFaceSource( widget.initialFaceSource!, personOrClusterId, )) { @@ -500,19 +520,29 @@ class _PersonFaceWidgetState extends State } if (_resolvedFaceSource != null) { - return _resolvedFaceSource; + if (await _canReuseResolvedFaceSource( + _resolvedFaceSource!, + personOrClusterId, + )) { + return _resolvedFaceSource; + } } final cachedFaceSource = checkCachedFaceSourceForPersonOrClusterID(personOrClusterId); - if (cachedFaceSource != null && - _canUseResolvedFaceSource(cachedFaceSource, personOrClusterId)) { - _applyResolvedFaceSource(cachedFaceSource); - await cacheFaceIdForPersonOrClusterIfNeeded( + if (cachedFaceSource != null) { + if (await _canReuseResolvedFaceSource( + cachedFaceSource, personOrClusterId, - cachedFaceSource.face.faceID, - ); - return cachedFaceSource; + clearSharedCache: true, + )) { + _applyResolvedFaceSource(cachedFaceSource); + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + cachedFaceSource.face.faceID, + ); + return cachedFaceSource; + } } try { diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index c31e5a7ea7a..875dd49316a 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -80,6 +80,10 @@ PersonFaceSource? checkCachedFaceSourceForPersonOrClusterID( return _personOrClusterIdToFaceSourceCache.get(personOrClusterID); } +void removeCachedFaceSourceForPersonOrClusterID(String personOrClusterID) { + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); +} + void cacheFaceSourceForPersonOrClusterID( String personOrClusterID, PersonFaceSource faceSource, From a1db87ed10d5d4d9de8023955a2a69c80e5883d1 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 11:08:07 +0530 Subject: [PATCH 11/18] Fix progressive face crop cache and queue cleanup --- .../ui/viewer/people/person_face_widget.dart | 65 +++++++++++++++++++ .../lib/utils/face/face_thumbnail_cache.dart | 40 ++++++++++++ 2 files changed, 105 insertions(+) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 1c29d48ac1b..ffd4026ba19 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -119,6 +119,8 @@ class _PersonFaceWidgetState extends State bool _disposed = false; int _upgradeGeneration = 0; EnteFile? _requestedThumbnailPreviewFile; + final Map _fullGenerationTaskClaims = {}; + final Map _thumbnailGenerationTaskClaims = {}; bool _isVisible = false; Timer? _visibleTaskTouchTimer; @@ -147,6 +149,7 @@ class _PersonFaceWidgetState extends State _disposed = true; _upgradeGeneration += 1; _cancelVisibleTaskTouchTimer(); + _releasePendingFaceGenerationClaims(); if (_requestedThumbnailPreviewFile != null) { removePendingGetThumbnailRequestIfAny(_requestedThumbnailPreviewFile!); } @@ -316,6 +319,15 @@ class _PersonFaceWidgetState extends State return null; } + final cachedFullCrop = await _loadCachedFullFaceCropIfAvailable(faceSource); + if (cachedFullCrop != null) { + _showingFallback = false; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: cachedFullCrop, + personName: _personName, + ); + } + if (_previewImageDimensionsForSource(faceSource) != null) { final thumbnailBytes = await _loadThumbnailPreviewBytes(faceSource.file); if (thumbnailBytes != null) { @@ -399,6 +411,21 @@ class _PersonFaceWidgetState extends State } } + Future _loadCachedFullFaceCropIfAvailable( + PersonFaceSource faceSource, + ) async { + final personOrClusterId = widget.personId ?? widget.clusterID!; + final cachedFullCrop = + checkInMemoryCachedCropForPersonOrClusterID(personOrClusterId); + if (cachedFullCrop != null) { + return cachedFullCrop; + } + return getPersistedFullFaceCropIfAvailable( + faceSource.face.faceID, + personOrClusterID: personOrClusterId, + ); + } + void _applyResolvedFaceSource(PersonFaceSource faceSource) { _resolvedFaceSource = faceSource; fileForFaceCrop = faceSource.file; @@ -450,6 +477,33 @@ class _PersonFaceWidgetState extends State _visibleTaskTouchTimer = null; } + void _recordPendingFaceGenerationClaim( + int fileId, { + required bool useFullFile, + }) { + final claims = useFullFile + ? _fullGenerationTaskClaims + : _thumbnailGenerationTaskClaims; + claims.update(fileId, (count) => count + 1, ifAbsent: () => 1); + } + + void _releasePendingFaceGenerationClaims() { + void releaseClaims(Map claims, {required bool useFullFile}) { + for (final entry in claims.entries) { + for (var i = 0; i < entry.value; i++) { + checkStopTryingToGenerateFaceThumbnails( + entry.key, + useFullFile: useFullFile, + ); + } + } + claims.clear(); + } + + releaseClaims(_fullGenerationTaskClaims, useFullFile: true); + releaseClaims(_thumbnailGenerationTaskClaims, useFullFile: false); + } + String _visibilityKeySuffix() { final widgetKey = widget.key; if (widgetKey is ValueKey) { @@ -973,11 +1027,22 @@ class _PersonFaceWidgetState extends State return null; } + var didRecordGenerationClaim = false; final cropMap = await getCachedFaceCrops( faceSource.file, [faceSource.face], useFullFile: useFullFile, personOrClusterID: personOrClusterId, + onGenerationTaskQueued: () { + if (didRecordGenerationClaim) { + return; + } + didRecordGenerationClaim = true; + _recordPendingFaceGenerationClaim( + faceSource.resolvedFileId, + useFullFile: useFullFile, + ); + }, useTempCache: false, ); _applyResolvedFaceSource(faceSource); diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index 875dd49316a..0cf86d63899 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -169,6 +169,7 @@ Future?> getCachedFaceCrops( int fetchAttempt = 1, bool useFullFile = true, String? personOrClusterID, + VoidCallback? onGenerationTaskQueued, required bool useTempCache, }) async { try { @@ -234,6 +235,7 @@ Future?> getCachedFaceCrops( final result = await _getFaceCropsUsingHeapPriorityQueue( enteFile, facesWithoutCrops, + onGenerationTaskQueued: onGenerationTaskQueued, useFullFile: useFullFile, ); if (result == null) { @@ -285,6 +287,7 @@ Future?> getCachedFaceCrops( faces, fetchAttempt: fetchAttempt + 1, useFullFile: useFullFile, + onGenerationTaskQueued: onGenerationTaskQueued, useTempCache: useTempCache, ); } @@ -401,9 +404,45 @@ Future hasPersistedFullFaceCrop(String faceID) async { return faceCropCacheFile.exists(); } +Future getPersistedFullFaceCropIfAvailable( + String faceID, { + String? personOrClusterID, +}) async { + final cachedFace = _checkInMemoryCachedCropForFaceID(faceID); + if (cachedFace != null) { + return cachedFace; + } + + final faceCropCacheFile = cachedFaceCropPath(faceID, false); + if (!(await faceCropCacheFile.exists())) { + return null; + } + + try { + final data = await faceCropCacheFile.readAsBytes(); + if (data.isEmpty) { + _logger.warning( + "Persisted face crop for faceID $faceID is empty, deleting file ${faceCropCacheFile.path}", + ); + await faceCropCacheFile.delete(); + return null; + } + await _putCachedCropForFaceID(faceID, data, personOrClusterID); + return data; + } catch (e, s) { + _logger.warning( + "Error reading persisted face crop for faceID $faceID from file ${faceCropCacheFile.path}", + e, + s, + ); + return null; + } +} + Future?> _getFaceCropsUsingHeapPriorityQueue( EnteFile file, Map faceBoxeMap, { + VoidCallback? onGenerationTaskQueued, bool useFullFile = true, }) async { final completer = Completer?>(); @@ -418,6 +457,7 @@ Future?> _getFaceCropsUsingHeapPriorityQueue( taskId = await _faceCropTaskId(file, useFullFile: false); } + onGenerationTaskQueued?.call(); await relevantTaskQueue.addTask(taskId, () async { final faceCrops = await _getFaceCrops( file, From dccea732634c5ed6fac73b47fc1467e58aa4eccc Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 11:33:34 +0530 Subject: [PATCH 12/18] Fall back when preview thumbnail fetch fails --- .../lib/ui/viewer/people/person_face_widget.dart | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index ffd4026ba19..78a49c14283 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -403,7 +403,16 @@ class _PersonFaceWidgetState extends State _requestedThumbnailPreviewFile = file; try { - return await getThumbnail(file); + try { + return await getThumbnail(file); + } catch (e, s) { + _logger.warning( + 'Failed to load preview thumbnail for progressive face fallback for file ${file.uploadedFileID ?? file.localID}', + e, + s, + ); + return null; + } } finally { if (_requestedThumbnailPreviewFile == file) { _requestedThumbnailPreviewFile = null; From 67fa3b05e0d2f83dc2445345a97a4b960f6e4f7f Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 11:54:33 +0530 Subject: [PATCH 13/18] Refresh person avatar before trusting seeded face IDs --- .../lib/ui/viewer/people/person_face_widget.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 78a49c14283..8db35671821 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -612,7 +612,7 @@ class _PersonFaceWidgetState extends State String? fixedFaceID = widget.initialAvatarFaceId; final mlDataDB = isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; - if (isPerson && !isOfflineMode && fixedFaceID == null) { + if (isPerson && !isOfflineMode) { final personEntity = await PersonService.instance.getPerson(widget.personId!); if (personEntity == null) { @@ -622,7 +622,15 @@ class _PersonFaceWidgetState extends State return null; } _personName = personEntity.data.name; - fixedFaceID = personEntity.data.avatarFaceID; + final currentAvatarFaceId = personEntity.data.avatarFaceID; + if (widget.initialAvatarFaceId != null && + widget.initialAvatarFaceId != currentAvatarFaceId) { + _logger.fine( + 'Ignoring stale seeded avatar face ID for person ${widget.personId}: ' + '${widget.initialAvatarFaceId} -> $currentAvatarFaceId', + ); + } + fixedFaceID = currentAvatarFaceId; } fixedFaceID ??= await checkUsedFaceIDForPersonOrClusterId( personOrClusterId, From ff7be855530e401115cbb043440e9fc4f8c6b1b3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 13:24:11 +0530 Subject: [PATCH 14/18] Revalidate face-source fast paths and clear task claims --- .../ui/viewer/people/person_face_widget.dart | 140 ++++++++++++++---- 1 file changed, 111 insertions(+), 29 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 8db35671821..78d312ff779 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -13,6 +13,7 @@ import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; import 'package:photos/models/ml/face/box.dart'; import 'package:photos/models/ml/face/face.dart'; +import 'package:photos/models/ml/face/person.dart'; import 'package:photos/models/ml/face/person_face_source.dart'; import 'package:photos/service_locator.dart' show flagService, isOfflineMode; import 'package:photos/services/machine_learning/face_ml/person/person_service.dart'; @@ -496,6 +497,24 @@ class _PersonFaceWidgetState extends State claims.update(fileId, (count) => count + 1, ifAbsent: () => 1); } + void _clearPendingFaceGenerationClaim( + int fileId, { + required bool useFullFile, + }) { + final claims = useFullFile + ? _fullGenerationTaskClaims + : _thumbnailGenerationTaskClaims; + final currentCount = claims[fileId]; + if (currentCount == null) { + return; + } + if (currentCount <= 1) { + claims.remove(fileId); + } else { + claims[fileId] = currentCount - 1; + } + } + void _releasePendingFaceGenerationClaims() { void releaseClaims(Map claims, {required bool useFullFile}) { for (final entry in claims.entries) { @@ -568,11 +587,68 @@ class _PersonFaceWidgetState extends State bool notifyOnError = true, }) async { final personOrClusterId = widget.personId ?? widget.clusterID!; + Future? currentPersonEntityFuture; + + Future getCurrentPersonEntity() { + if (!isPerson || isOfflineMode) { + return Future.value(null); + } + return currentPersonEntityFuture ??= + PersonService.instance.getPerson(widget.personId!); + } + + Future canReuseResolvedFaceSource( + PersonFaceSource faceSource, { + bool clearSharedCache = false, + }) async { + if (!await _canReuseResolvedFaceSource( + faceSource, + personOrClusterId, + clearSharedCache: clearSharedCache, + )) { + return false; + } + if (!isPerson || isOfflineMode) { + return true; + } + + final personEntity = await getCurrentPersonEntity(); + if (personEntity == null) { + _logger.severe( + 'Person with ID ${widget.personId} not found, cannot validate face source.', + ); + return false; + } + _personName = personEntity.data.name; + + final currentAvatarFaceId = personEntity.data.avatarFaceID; + final seededAvatarFaceId = widget.initialAvatarFaceId; + final shouldRejectFaceSource = currentAvatarFaceId != null + ? faceSource.face.faceID != currentAvatarFaceId + : seededAvatarFaceId != null && + faceSource.face.faceID == seededAvatarFaceId; + if (!shouldRejectFaceSource) { + return true; + } + + _logger.fine( + 'Ignoring stale prefetched face source for person ${widget.personId}: ' + 'seeded=${widget.initialAvatarFaceId} ' + 'current=$currentAvatarFaceId ' + 'face=${faceSource.face.faceID}', + ); + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + return false; + } if (widget.initialFaceSource != null && - await _canReuseResolvedFaceSource( + await canReuseResolvedFaceSource( widget.initialFaceSource!, - personOrClusterId, )) { _applyResolvedFaceSource(widget.initialFaceSource!); await cacheFaceIdForPersonOrClusterIfNeeded( @@ -583,9 +659,8 @@ class _PersonFaceWidgetState extends State } if (_resolvedFaceSource != null) { - if (await _canReuseResolvedFaceSource( + if (await canReuseResolvedFaceSource( _resolvedFaceSource!, - personOrClusterId, )) { return _resolvedFaceSource; } @@ -594,9 +669,8 @@ class _PersonFaceWidgetState extends State final cachedFaceSource = checkCachedFaceSourceForPersonOrClusterID(personOrClusterId); if (cachedFaceSource != null) { - if (await _canReuseResolvedFaceSource( + if (await canReuseResolvedFaceSource( cachedFaceSource, - personOrClusterId, clearSharedCache: true, )) { _applyResolvedFaceSource(cachedFaceSource); @@ -613,8 +687,7 @@ class _PersonFaceWidgetState extends State final mlDataDB = isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; if (isPerson && !isOfflineMode) { - final personEntity = - await PersonService.instance.getPerson(widget.personId!); + final personEntity = await getCurrentPersonEntity(); if (personEntity == null) { _logger.severe( 'Person with ID ${widget.personId} not found, cannot get cover face.', @@ -1045,31 +1118,40 @@ class _PersonFaceWidgetState extends State } var didRecordGenerationClaim = false; - final cropMap = await getCachedFaceCrops( - faceSource.file, - [faceSource.face], - useFullFile: useFullFile, - personOrClusterID: personOrClusterId, - onGenerationTaskQueued: () { - if (didRecordGenerationClaim) { - return; - } - didRecordGenerationClaim = true; - _recordPendingFaceGenerationClaim( + try { + final cropMap = await getCachedFaceCrops( + faceSource.file, + [faceSource.face], + useFullFile: useFullFile, + personOrClusterID: personOrClusterId, + onGenerationTaskQueued: () { + if (didRecordGenerationClaim) { + return; + } + didRecordGenerationClaim = true; + _recordPendingFaceGenerationClaim( + faceSource.resolvedFileId, + useFullFile: useFullFile, + ); + }, + useTempCache: false, + ); + _applyResolvedFaceSource(faceSource); + final result = cropMap?[faceSource.face.faceID]; + if (result == null) { + _logger.severe( + 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${faceSource.resolvedFileId}', + ); + } + return result; + } finally { + if (didRecordGenerationClaim) { + _clearPendingFaceGenerationClaim( faceSource.resolvedFileId, useFullFile: useFullFile, ); - }, - useTempCache: false, - ); - _applyResolvedFaceSource(faceSource); - final result = cropMap?[faceSource.face.faceID]; - if (result == null) { - _logger.severe( - 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${faceSource.resolvedFileId}', - ); + } } - return result; } catch (e, s) { _logger.severe( 'Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}', From 2dcdac38873c3d3429f60818ab087c40ae62ef97 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 15:20:38 +0530 Subject: [PATCH 15/18] Revalidate cached face sources after people changes --- .../ui/viewer/people/person_face_widget.dart | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 78d312ff779..56789317967 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -597,6 +597,28 @@ class _PersonFaceWidgetState extends State PersonService.instance.getPerson(widget.personId!); } + Future isFaceStillAssignedToCurrentSubject( + PersonFaceSource faceSource, { + String? currentAvatarFaceId, + }) async { + final mlDataDB = + isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; + if (!isPerson) { + final currentClusterId = + await mlDataDB.getClusterIDForFaceID(faceSource.face.faceID); + return currentClusterId == widget.clusterID; + } + + if (currentAvatarFaceId != null) { + return faceSource.face.faceID == currentAvatarFaceId; + } + + final personIdsByFace = await mlDataDB.getFaceIdToPersonIdForFaces([ + faceSource.face.faceID, + ]); + return personIdsByFace[faceSource.face.faceID] == widget.personId; + } + Future canReuseResolvedFaceSource( PersonFaceSource faceSource, { bool clearSharedCache = false, @@ -609,7 +631,22 @@ class _PersonFaceWidgetState extends State return false; } if (!isPerson || isOfflineMode) { - return true; + final isStillAssigned = + await isFaceStillAssignedToCurrentSubject(faceSource); + if (isStillAssigned) { + return true; + } + _logger.fine( + 'Ignoring stale prefetched face source for ${widget.clusterID ?? widget.personId}: ' + 'face=${faceSource.face.faceID}', + ); + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + return false; } final personEntity = await getCurrentPersonEntity(); @@ -622,12 +659,11 @@ class _PersonFaceWidgetState extends State _personName = personEntity.data.name; final currentAvatarFaceId = personEntity.data.avatarFaceID; - final seededAvatarFaceId = widget.initialAvatarFaceId; - final shouldRejectFaceSource = currentAvatarFaceId != null - ? faceSource.face.faceID != currentAvatarFaceId - : seededAvatarFaceId != null && - faceSource.face.faceID == seededAvatarFaceId; - if (!shouldRejectFaceSource) { + final isStillAssigned = await isFaceStillAssignedToCurrentSubject( + faceSource, + currentAvatarFaceId: currentAvatarFaceId, + ); + if (isStillAssigned) { return true; } From ccec3e93682ddce2f007dc766aa4a6c8c00b3b43 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 16:20:43 +0530 Subject: [PATCH 16/18] Fix progressive face upgrade and prefetch edge cases --- .../photos/lib/services/search_service.dart | 21 +++++++++----- .../ui/viewer/people/person_face_widget.dart | 18 +++++++++++- .../people/visible_face_source_prefetch.dart | 29 +++++++++++++++---- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/mobile/apps/photos/lib/services/search_service.dart b/mobile/apps/photos/lib/services/search_service.dart index c2045c8d230..7d2d29b1fae 100644 --- a/mobile/apps/photos/lib/services/search_service.dart +++ b/mobile/apps/photos/lib/services/search_service.dart @@ -271,21 +271,28 @@ class SearchService { GenericSearchResult result, ) async { final personID = result.params[kPersonParamID] as String?; + final clusterID = result.params[kClusterParamId] as String?; final avatarFaceID = result.params[kPersonAvatarFaceID] as String?; final previewFile = result.previewThumbnail(); final previewFileID = previewFile?.uploadedFileID; - if (personID == null || - avatarFaceID == null || - avatarFaceID.isEmpty || - previewFile == null || - previewFileID == null) { + if (previewFile == null || previewFileID == null) { + return null; + } + + final hasPersonSeed = personID != null && + personID.isNotEmpty && + avatarFaceID != null && + avatarFaceID.isNotEmpty; + final hasClusterSeed = clusterID != null && clusterID.isNotEmpty; + if (!hasPersonSeed && !hasClusterSeed) { return null; } final face = await mlDataDB.getCoverFaceForPerson( recentFileID: previewFileID, - personID: personID, - avatarFaceId: avatarFaceID, + personID: hasPersonSeed ? personID : null, + avatarFaceId: hasPersonSeed ? avatarFaceID : null, + clusterID: hasClusterSeed ? clusterID : null, ); if (face == null) { return null; diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 56789317967..9b8e1d833bc 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -1005,8 +1005,24 @@ class _PersonFaceWidgetState extends State if (sourceFile.width <= 0 || sourceFile.height <= 0) { _logger.fine( - 'person_face_thumbnail_upgrade_skipped reason=missing_full_dimensions file=${sourceFile.uploadedFileID}', + 'person_face_thumbnail_upgrade_fallback reason=missing_full_dimensions file=${sourceFile.uploadedFileID}', ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation) || !mounted) { + return; + } + setState(() { + _showingFallback = false; + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); + }); return; } diff --git a/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart index 3a3c55592f1..06acf011a12 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart @@ -33,8 +33,17 @@ class _VisibleFaceSourcePrefetchState extends State { @override void didUpdateWidget(covariant VisibleFaceSourcePrefetch oldWidget) { super.didUpdateWidget(oldWidget); + final oldTargetKey = _visibilityTargetKeyFor( + results: oldWidget.results, + index: oldWidget.index, + ); + final newTargetKey = _visibilityTargetKeyFor( + results: widget.results, + index: widget.index, + ); if (oldWidget.index != widget.index || - !identical(oldWidget.results, widget.results)) { + !identical(oldWidget.results, widget.results) || + oldTargetKey != newTargetKey) { _hasPrefetchedForCurrentItem = false; } } @@ -70,12 +79,22 @@ class _VisibleFaceSourcePrefetchState extends State { } String _visibilityTargetKey() { - if (widget.index < 0 || widget.index >= widget.results.length) { - return widget.index.toString(); + return _visibilityTargetKeyFor( + results: widget.results, + index: widget.index, + ); + } + + String _visibilityTargetKeyFor({ + required List results, + required int index, + }) { + if (index < 0 || index >= results.length) { + return index.toString(); } - final result = widget.results[widget.index]; + final result = results[index]; final personID = result.params[kPersonParamID] as String?; final clusterID = result.params[kClusterParamId] as String?; - return personID ?? clusterID ?? widget.index.toString(); + return personID ?? clusterID ?? index.toString(); } } From ff238b5e0500142cb733c405f4b2e31daff8dfea Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 18 Mar 2026 19:43:10 +0530 Subject: [PATCH 17/18] Keep visible face upgrades alive under queue pressure --- .../lib/ui/viewer/people/person_face_widget.dart | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 9b8e1d833bc..6b586092982 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -116,7 +116,6 @@ class _PersonFaceWidgetState extends State PersonFaceSource? _resolvedFaceSource; String? _personName; bool _showingFallback = false; - bool _fallbackEverUsed = false; bool _disposed = false; int _upgradeGeneration = 0; EnteFile? _requestedThumbnailPreviewFile; @@ -268,7 +267,6 @@ class _PersonFaceWidgetState extends State if (thumbnailCrop == null) { return null; } - _fallbackEverUsed = true; _showingFallback = false; return _PersonFaceLoadResult.faceCrop( faceCropBytes: thumbnailCrop, @@ -301,7 +299,6 @@ class _PersonFaceWidgetState extends State final fallbackCrop = await _getFaceCrop(useFullFile: false); if (fallbackCrop != null) { _showingFallback = true; - _fallbackEverUsed = true; return _PersonFaceLoadResult.faceCrop( faceCropBytes: fallbackCrop, personName: _personName, @@ -333,7 +330,6 @@ class _PersonFaceWidgetState extends State final thumbnailBytes = await _loadThumbnailPreviewBytes(faceSource.file); if (thumbnailBytes != null) { _showingFallback = true; - _fallbackEverUsed = true; final generation = ++_upgradeGeneration; unawaited(_attemptFullQualityUpgrade(generation)); return _PersonFaceLoadResult.thumbnailPreview( @@ -350,7 +346,6 @@ class _PersonFaceWidgetState extends State ); if (thumbnailCrop != null) { _showingFallback = true; - _fallbackEverUsed = true; final generation = ++_upgradeGeneration; unawaited(_attemptFullQualityUpgrade(generation)); return _PersonFaceLoadResult.faceCrop( @@ -475,9 +470,9 @@ class _PersonFaceWidgetState extends State return; } touchPendingFaceThumbnailGeneration(fileId, useFullFile: false); - // Keep first-paint work ahead of progressive upgrades. Only reprioritize - // full-file generation when the tile still has no fallback thumbnail. - if (widget.useFullFile && !_fallbackEverUsed) { + // Keep visible full-quality upgrades from aging out of the bounded queue. + // Thumbnail work is still refreshed first so first-paint requests stay hot. + if (widget.useFullFile) { touchPendingFaceThumbnailGeneration(fileId, useFullFile: true); } } From edc828c5ef8d993d4131df663bcffb41c80c3107 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 20 Mar 2026 15:01:29 +0530 Subject: [PATCH 18/18] Reload seeded face sources without overwriting covers --- .../ui/viewer/people/person_face_widget.dart | 144 +++++++++++++++--- 1 file changed, 124 insertions(+), 20 deletions(-) diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 6b586092982..61b2faa85a8 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -144,6 +144,39 @@ class _PersonFaceWidgetState extends State _faceLoadFuture = _loadFace(); } + @override + void didUpdateWidget(covariant PersonFaceWidget oldWidget) { + super.didUpdateWidget(oldWidget); + final seedInputsChanged = !_sameFaceSourceSeed( + oldWidget.initialFaceSource, + widget.initialFaceSource, + ) || + !_samePreviewFileSeed( + oldWidget.initialPreviewFile, + widget.initialPreviewFile, + ) || + oldWidget.initialAvatarFaceId != widget.initialAvatarFaceId; + if (!seedInputsChanged) { + if (_personName == oldWidget.initialPersonName && + widget.initialPersonName != oldWidget.initialPersonName) { + _personName = widget.initialPersonName; + } + return; + } + + _upgradeGeneration += 1; + _personName = widget.initialPersonName; + _resolvedFaceSource = null; + fileForFaceCrop = null; + _faceForFaceCrop = null; + _faceCropFileId = null; + _showingFallback = false; + _faceLoadFuture = _loadFace(); + if (mounted) { + setState(() {}); + } + } + @override void dispose() { _disposed = true; @@ -527,6 +560,30 @@ class _PersonFaceWidgetState extends State releaseClaims(_thumbnailGenerationTaskClaims, useFullFile: false); } + bool _sameFaceSourceSeed( + PersonFaceSource? a, + PersonFaceSource? b, + ) { + if (identical(a, b)) { + return true; + } + if (a == null || b == null) { + return false; + } + return a.face.faceID == b.face.faceID && + a.resolvedFileId == b.resolvedFileId; + } + + bool _samePreviewFileSeed(EnteFile? a, EnteFile? b) { + if (identical(a, b)) { + return true; + } + if (a == null || b == null) { + return false; + } + return a.uploadedFileID == b.uploadedFileID && a.localID == b.localID; + } + String _visibilityKeySuffix() { final widgetKey = widget.key; if (widgetKey is ValueKey) { @@ -583,6 +640,7 @@ class _PersonFaceWidgetState extends State }) async { final personOrClusterId = widget.personId ?? widget.clusterID!; Future? currentPersonEntityFuture; + Future? persistedFaceIdFuture; Future getCurrentPersonEntity() { if (!isPerson || isOfflineMode) { @@ -592,6 +650,23 @@ class _PersonFaceWidgetState extends State PersonService.instance.getPerson(widget.personId!); } + Future getPersistedFaceId() { + return persistedFaceIdFuture ??= + checkUsedFaceIDForPersonOrClusterId(personOrClusterId); + } + + void invalidateResolvedFaceSource( + PersonFaceSource faceSource, { + required bool clearSharedCache, + }) { + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + } + Future isFaceStillAssignedToCurrentSubject( PersonFaceSource faceSource, { String? currentAvatarFaceId, @@ -628,19 +703,32 @@ class _PersonFaceWidgetState extends State if (!isPerson || isOfflineMode) { final isStillAssigned = await isFaceStillAssignedToCurrentSubject(faceSource); - if (isStillAssigned) { + if (!isStillAssigned) { + _logger.fine( + 'Ignoring stale prefetched face source for ${widget.clusterID ?? widget.personId}: ' + 'face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + final persistedFaceId = await getPersistedFaceId(); + if (persistedFaceId == null || + persistedFaceId == faceSource.face.faceID) { return true; } + _logger.fine( - 'Ignoring stale prefetched face source for ${widget.clusterID ?? widget.personId}: ' - 'face=${faceSource.face.faceID}', + 'Ignoring preview-seeded face source for ${widget.clusterID ?? widget.personId}: ' + 'persisted=$persistedFaceId face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, ); - if (identical(_resolvedFaceSource, faceSource)) { - _resolvedFaceSource = null; - } - if (clearSharedCache) { - removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); - } return false; } @@ -658,22 +746,38 @@ class _PersonFaceWidgetState extends State faceSource, currentAvatarFaceId: currentAvatarFaceId, ); - if (isStillAssigned) { + if (!isStillAssigned) { + _logger.fine( + 'Ignoring stale prefetched face source for person ${widget.personId}: ' + 'seeded=${widget.initialAvatarFaceId} ' + 'current=$currentAvatarFaceId ' + 'face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + if (currentAvatarFaceId != null) { + return true; + } + + final persistedFaceId = await getPersistedFaceId(); + if (persistedFaceId == null || + persistedFaceId == faceSource.face.faceID) { return true; } _logger.fine( - 'Ignoring stale prefetched face source for person ${widget.personId}: ' - 'seeded=${widget.initialAvatarFaceId} ' - 'current=$currentAvatarFaceId ' - 'face=${faceSource.face.faceID}', + 'Ignoring preview-seeded face source for person ${widget.personId}: ' + 'persisted=$persistedFaceId face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, ); - if (identical(_resolvedFaceSource, faceSource)) { - _resolvedFaceSource = null; - } - if (clearSharedCache) { - removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); - } return false; }