diff --git a/mobile/apps/photos/lib/models/ml/face/person_face_source.dart b/mobile/apps/photos/lib/models/ml/face/person_face_source.dart new file mode 100644 index 00000000000..ea56fa3f216 --- /dev/null +++ b/mobile/apps/photos/lib/models/ml/face/person_face_source.dart @@ -0,0 +1,16 @@ +import 'package:photos/models/file/file.dart'; +import 'package:photos/models/ml/face/face.dart'; + +class PersonFaceSource { + final EnteFile file; + final Face face; + final int resolvedFileId; + final String? personName; + + const PersonFaceSource({ + required this.file, + required this.face, + required this.resolvedFileId, + this.personName, + }); +} diff --git a/mobile/apps/photos/lib/models/search/search_constants.dart b/mobile/apps/photos/lib/models/search/search_constants.dart index c5d64a2645f..669cce71704 100644 --- a/mobile/apps/photos/lib/models/search/search_constants.dart +++ b/mobile/apps/photos/lib/models/search/search_constants.dart @@ -1,6 +1,8 @@ const kPersonParamID = 'person_id'; const kPersonWidgetKey = 'person_widget_key'; const kPersonPinned = 'person_pinned'; +const kPersonAvatarFaceID = 'person_avatar_face_id'; +const kPersonFaceSource = 'person_face_source'; const kClusterParamId = 'cluster_id'; const kFileID = 'file_id'; const kContactEmail = 'contact_email'; diff --git a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart index c7de0fab49e..9184cb1b02b 100644 --- a/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart +++ b/mobile/apps/photos/lib/services/machine_learning/face_thumbnail_generator.dart @@ -8,6 +8,18 @@ import "package:photos/utils/image_ml_util.dart"; import "package:photos/utils/isolate/isolate_operations.dart"; import "package:photos/utils/isolate/super_isolate.dart"; +class FaceThumbnailGenerationResult { + final List thumbnails; + final int sourceWidth; + final int sourceHeight; + + const FaceThumbnailGenerationResult({ + required this.thumbnails, + required this.sourceWidth, + required this.sourceHeight, + }); +} + @pragma('vm:entry-point') class FaceThumbnailGenerator extends SuperIsolate { @override @@ -64,6 +76,66 @@ class FaceThumbnailGenerator extends SuperIsolate { return compressedFaces; } catch (e, s) { _logger.severe("Failed to generate face thumbnails", e, s); + rethrow; + } + } + + Future + generateFaceThumbnailsWithSourceDimensions( + String imagePath, + List faceBoxes, + ) async { + if (!flagService.progressivePersonFaceThumbnailsEnabled) { + final thumbnails = await generateFaceThumbnails(imagePath, faceBoxes); + return FaceThumbnailGenerationResult( + thumbnails: thumbnails, + sourceWidth: 0, + sourceHeight: 0, + ); + } + + try { + final useRustForFaceThumbnails = flagService.useRustForFaceThumbnails; + _logger.info( + "Generating face thumbnails for ${faceBoxes.length} face boxes in $imagePath", + ); + final List> faceBoxesJson = + faceBoxes.map((box) => box.toJson()).toList(); + final Map rawResult = await runInIsolate( + IsolateOperation.generateFaceThumbnailsWithSourceDimensions, + { + 'imagePath': imagePath, + 'faceBoxesList': faceBoxesJson, + 'useRustForFaceThumbnails': useRustForFaceThumbnails, + }, + ).then((value) => Map.from(value)); + final List faces = + (rawResult['thumbnails'] as List).cast(); + final int sourceWidth = rawResult['sourceWidth'] as int? ?? 0; + final int sourceHeight = rawResult['sourceHeight'] as int? ?? 0; + _logger.info( + "Generated face thumbnails with source dimensions ${sourceWidth}x$sourceHeight", + ); + if (useRustForFaceThumbnails) { + // Rust path already emits compressed JPEG bytes. + return FaceThumbnailGenerationResult( + thumbnails: faces, + sourceWidth: sourceWidth, + sourceHeight: sourceHeight, + ); + } + final compressedFaces = + await compressFaceThumbnails({'listPngBytes': faces}); + _logger.fine( + "Compressed face thumbnails from sizes ${faces.map((e) => e.length / 1024).toList()} to ${compressedFaces.map((e) => e.length / 1024).toList()} kilobytes", + ); + return FaceThumbnailGenerationResult( + thumbnails: compressedFaces, + sourceWidth: sourceWidth, + sourceHeight: sourceHeight, + ); + } catch (e, s) { + _logger.severe("Failed to generate face thumbnails", e, s); rethrow; } diff --git a/mobile/apps/photos/lib/services/search_service.dart b/mobile/apps/photos/lib/services/search_service.dart index ec2f6c73ad9..7d2d29b1fae 100644 --- a/mobile/apps/photos/lib/services/search_service.dart +++ b/mobile/apps/photos/lib/services/search_service.dart @@ -32,6 +32,7 @@ import "package:photos/models/memories/memories_cache.dart"; import "package:photos/models/memories/memory.dart"; import "package:photos/models/memories/smart_memory.dart"; import "package:photos/models/ml/face/person.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import 'package:photos/models/search/album_search_result.dart'; import 'package:photos/models/search/device_album_search_result.dart'; import 'package:photos/models/search/generic_search_result.dart'; @@ -66,11 +67,13 @@ import "package:photos/utils/file_util.dart"; import "package:photos/utils/people_sort_util.dart"; class SearchService { + static const int _initialFaceSourcePrefetchCount = 24; Future>? _cachedFilesFuture; Future>? _cachedFilesForSearch; Future>? _cachedFilesForHierarchicalSearch; Future>? _cachedFilesForGenericGallery; Future>? _cachedHiddenFilesFuture; + final Set _faceSourcePrefetchInFlight = {}; final _logger = Logger((SearchService).toString()); final _collectionService = CollectionsService.instance; static const _maximumResultsLimit = 20; @@ -197,6 +200,127 @@ class SearchService { unawaited(memoriesCacheService.clearMemoriesCache()); } + Future _enrichInitialFaceSources( + List results, + ) async { + await prefetchFaceSourcesInWindow( + results, + startIndex: 0, + count: _initialFaceSourcePrefetchCount, + ); + } + + Future prefetchFaceSourcesInWindow( + List results, { + required int startIndex, + int count = _initialFaceSourcePrefetchCount, + int leadingBuffer = 0, + }) async { + if (results.isEmpty || count <= 0) { + return; + } + final safeStart = max(0, startIndex - leadingBuffer); + final safeEnd = min(results.length, safeStart + count); + if (safeStart >= safeEnd) { + return; + } + await Future.wait( + results.sublist(safeStart, safeEnd).map(_prefetchFaceSourceIfNeeded), + ); + } + + Future _prefetchFaceSourceIfNeeded( + GenericSearchResult result, + ) async { + if (result.params[kPersonFaceSource] != null) { + return; + } + final prefetchKey = _faceSourcePrefetchKey(result); + if (prefetchKey == null || !_faceSourcePrefetchInFlight.add(prefetchKey)) { + return; + } + try { + final source = await _tryBuildInitialPersonFaceSource(result); + if (source != null) { + result.params[kPersonFaceSource] = source; + } + } catch (e, s) { + _logger.fine( + "Failed to prefetch initial face source for ${result.params[kPersonParamID] ?? result.params[kClusterParamId]}", + e, + s, + ); + } finally { + _faceSourcePrefetchInFlight.remove(prefetchKey); + } + } + + String? _faceSourcePrefetchKey(GenericSearchResult result) { + final personID = result.params[kPersonParamID] as String?; + if (personID != null && personID.isNotEmpty) { + return "person:$personID"; + } + final clusterID = result.params[kClusterParamId] as String?; + if (clusterID != null && clusterID.isNotEmpty) { + return "cluster:$clusterID"; + } + return null; + } + + Future _tryBuildInitialPersonFaceSource( + GenericSearchResult result, + ) async { + final personID = result.params[kPersonParamID] as String?; + final clusterID = result.params[kClusterParamId] as String?; + final avatarFaceID = result.params[kPersonAvatarFaceID] as String?; + final previewFile = result.previewThumbnail(); + final previewFileID = previewFile?.uploadedFileID; + if (previewFile == null || previewFileID == null) { + return null; + } + + final hasPersonSeed = personID != null && + personID.isNotEmpty && + avatarFaceID != null && + avatarFaceID.isNotEmpty; + final hasClusterSeed = clusterID != null && clusterID.isNotEmpty; + if (!hasPersonSeed && !hasClusterSeed) { + return null; + } + + final face = await mlDataDB.getCoverFaceForPerson( + recentFileID: previewFileID, + personID: hasPersonSeed ? personID : null, + avatarFaceId: hasPersonSeed ? avatarFaceID : null, + clusterID: hasClusterSeed ? clusterID : null, + ); + if (face == null) { + return null; + } + + EnteFile sourceFile = previewFile; + if (face.fileID != previewFileID) { + EnteFile? faceFile; + for (final file in result.resultFiles()) { + if (file.uploadedFileID == face.fileID) { + faceFile = file; + break; + } + } + if (faceFile == null) { + return null; + } + sourceFile = faceFile; + } + + return PersonFaceSource( + file: sourceFile, + face: face, + resolvedFileId: face.fileID, + personName: result.name(), + ); + } + // getFilteredCollectionsWithThumbnail removes deleted or archived or // collections which don't have a file from search result Future> getCollectionSearchResults( @@ -1083,6 +1207,7 @@ class SearchService { params: { kPersonWidgetKey: p.data.avatarFaceID ?? p.hashCode.toString(), kPersonParamID: personID, + kPersonAvatarFaceID: p.data.avatarFaceID, kFileID: files.first.uploadedFileID, kPersonPinned: p.data.isPinned, }, @@ -1100,6 +1225,7 @@ class SearchService { kPersonWidgetKey: p.data.avatarFaceID ?? p.hashCode.toString(), kPersonParamID: personID, + kPersonAvatarFaceID: p.data.avatarFaceID, kPersonPinned: p.data.isPinned, kFileID: files.first.uploadedFileID, }, @@ -1206,6 +1332,7 @@ class SearchService { photosSortAscending: localSettings.peoplePhotosSortAscending, ), ); + await _enrichInitialFaceSources(facesResult); if (limit != null) { return facesResult.sublist(0, min(limit, facesResult.length)); } else { diff --git a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart index 70fa06b0625..61b2faa85a8 100644 --- a/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/people/person_face_widget.dart @@ -1,23 +1,69 @@ -import "dart:typed_data"; - -import "package:flutter/foundation.dart" show kDebugMode; -import "package:flutter/material.dart"; -import "package:logging/logging.dart"; -import "package:photos/db/files_db.dart"; -import "package:photos/db/ml/db.dart"; -import "package:photos/db/offline_files_db.dart"; +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart' show kDebugMode; +import 'package:flutter/material.dart'; +import 'package:logging/logging.dart'; +import 'package:photos/core/cache/lru_map.dart'; +import 'package:photos/core/cache/thumbnail_in_memory_cache.dart'; +import 'package:photos/db/files_db.dart'; +import 'package:photos/db/ml/db.dart'; +import 'package:photos/db/offline_files_db.dart'; import 'package:photos/models/file/file.dart'; -import "package:photos/models/ml/face/face.dart"; -import "package:photos/models/ml/face/person.dart"; -import "package:photos/service_locator.dart" show isOfflineMode; -import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; -import "package:photos/services/machine_learning/ml_result.dart"; -import "package:photos/services/search_service.dart"; -import "package:photos/theme/ente_theme.dart"; -import "package:photos/ui/common/loading_widget.dart"; -import "package:photos/utils/face/face_thumbnail_cache.dart"; - -final _logger = Logger("PersonFaceWidget"); +import 'package:photos/models/file/file_type.dart'; +import 'package:photos/models/ml/face/box.dart'; +import 'package:photos/models/ml/face/face.dart'; +import 'package:photos/models/ml/face/person.dart'; +import 'package:photos/models/ml/face/person_face_source.dart'; +import 'package:photos/service_locator.dart' show flagService, isOfflineMode; +import 'package:photos/services/machine_learning/face_ml/person/person_service.dart'; +import 'package:photos/services/machine_learning/ml_result.dart'; +import 'package:photos/services/search_service.dart'; +import 'package:photos/theme/ente_theme.dart'; +import 'package:photos/ui/common/loading_widget.dart'; +import 'package:photos/utils/face/face_thumbnail_cache.dart'; +import 'package:photos/utils/face/face_thumbnail_quality.dart'; +import 'package:photos/utils/thumbnail_util.dart'; +import 'package:visibility_detector/visibility_detector.dart'; + +final _logger = Logger('PersonFaceWidget'); +const _kMinUnnamedClusterSizeForProgressiveUpgrade = 5; +const _kProgressiveUpgradeUpscaleThreshold = 1.35; +const _kProgressiveUpgradeMinImprovementRatio = 1.2; +const _kProgressiveUpgradeIdleWaitBudget = Duration(seconds: 2); +const _kVisibleTaskTouchInterval = Duration(seconds: 20); + +class _PersonFaceLoadResult { + final Uint8List? faceCropBytes; + final Uint8List? thumbnailBytes; + final PersonFaceSource? faceSource; + final String? personName; + + const _PersonFaceLoadResult._({ + this.faceCropBytes, + this.thumbnailBytes, + this.faceSource, + this.personName, + }); + + const _PersonFaceLoadResult.faceCrop({ + required Uint8List faceCropBytes, + String? personName, + }) : this._( + faceCropBytes: faceCropBytes, + personName: personName, + ); + + const _PersonFaceLoadResult.thumbnailPreview({ + required Uint8List thumbnailBytes, + required PersonFaceSource faceSource, + String? personName, + }) : this._( + thumbnailBytes: thumbnailBytes, + faceSource: faceSource, + personName: personName, + ); +} class PersonFaceWidget extends StatefulWidget { final String? personId; @@ -25,6 +71,10 @@ class PersonFaceWidget extends StatefulWidget { final bool useFullFile; final VoidCallback? onErrorCallback; final bool keepAlive; + final PersonFaceSource? initialFaceSource; + final String? initialPersonName; + final String? initialAvatarFaceId; + final EnteFile? initialPreviewFile; /// Physical pixel width for image decoding optimization. /// @@ -36,19 +86,21 @@ class PersonFaceWidget extends StatefulWidget { /// If null or <= 0, the image is decoded at full resolution. final int? cachedPixelWidth; - // PersonFaceWidget constructor checks that both personId and clusterID are not null - // and that the file is not null const PersonFaceWidget({ this.personId, this.clusterID, this.useFullFile = true, this.onErrorCallback, this.keepAlive = false, + this.initialFaceSource, + this.initialPersonName, + this.initialAvatarFaceId, + this.initialPreviewFile, this.cachedPixelWidth, super.key, }) : assert( personId != null || clusterID != null, - "PersonFaceWidget requires either personId or clusterID to be non-null", + 'PersonFaceWidget requires either personId or clusterID to be non-null', ); @override @@ -57,180 +109,742 @@ class PersonFaceWidget extends StatefulWidget { class _PersonFaceWidgetState extends State with AutomaticKeepAliveClientMixin { - Future? faceCropFuture; + Future<_PersonFaceLoadResult?>? _faceLoadFuture; EnteFile? fileForFaceCrop; + Face? _faceForFaceCrop; int? _faceCropFileId; + PersonFaceSource? _resolvedFaceSource; String? _personName; bool _showingFallback = false; - bool _fallbackEverUsed = false; + bool _disposed = false; + int _upgradeGeneration = 0; + EnteFile? _requestedThumbnailPreviewFile; + final Map _fullGenerationTaskClaims = {}; + final Map _thumbnailGenerationTaskClaims = {}; + bool _isVisible = false; + Timer? _visibleTaskTouchTimer; + + static final LRUMap _clusterToFileCountCache = LRUMap(1000); bool get isPerson => widget.personId != null; + bool get _shouldUseProgressiveStrategy { + return widget.useFullFile && + !isOfflineMode && + flagService.progressivePersonFaceThumbnailsEnabled; + } + @override bool get wantKeepAlive => widget.keepAlive; @override void initState() { super.initState(); - faceCropFuture = _loadFaceCrop(); + _personName = widget.initialPersonName; + _faceLoadFuture = _loadFace(); } @override - void dispose() { - if (_faceCropFileId != null) { - checkStopTryingToGenerateFaceThumbnails( - _faceCropFileId!, - useFullFile: widget.useFullFile, - ); - if (_fallbackEverUsed) { - checkStopTryingToGenerateFaceThumbnails( - _faceCropFileId!, - useFullFile: false, - ); + void didUpdateWidget(covariant PersonFaceWidget oldWidget) { + super.didUpdateWidget(oldWidget); + final seedInputsChanged = !_sameFaceSourceSeed( + oldWidget.initialFaceSource, + widget.initialFaceSource, + ) || + !_samePreviewFileSeed( + oldWidget.initialPreviewFile, + widget.initialPreviewFile, + ) || + oldWidget.initialAvatarFaceId != widget.initialAvatarFaceId; + if (!seedInputsChanged) { + if (_personName == oldWidget.initialPersonName && + widget.initialPersonName != oldWidget.initialPersonName) { + _personName = widget.initialPersonName; } + return; + } + + _upgradeGeneration += 1; + _personName = widget.initialPersonName; + _resolvedFaceSource = null; + fileForFaceCrop = null; + _faceForFaceCrop = null; + _faceCropFileId = null; + _showingFallback = false; + _faceLoadFuture = _loadFace(); + if (mounted) { + setState(() {}); + } + } + + @override + void dispose() { + _disposed = true; + _upgradeGeneration += 1; + _cancelVisibleTaskTouchTimer(); + _releasePendingFaceGenerationClaims(); + if (_requestedThumbnailPreviewFile != null) { + removePendingGetThumbnailRequestIfAny(_requestedThumbnailPreviewFile!); } super.dispose(); } @override Widget build(BuildContext context) { - super.build( - context, - ); // Calling super.build for AutomaticKeepAliveClientMixin - - return FutureBuilder( - future: faceCropFuture, - builder: (context, snapshot) { - if (snapshot.hasData && snapshot.data != null) { - // Only cacheWidth (not cacheHeight) to preserve aspect ratio. - // Face crops are typically portrait, so constraining width ensures - // sufficient height for BoxFit.cover without upscaling. - final shouldOptimize = - widget.cachedPixelWidth != null && widget.cachedPixelWidth! > 0; - final ImageProvider imageProvider = shouldOptimize - ? Image.memory( - snapshot.data!, - cacheWidth: widget.cachedPixelWidth, - ).image - : MemoryImage(snapshot.data!); - return Stack( - fit: StackFit.expand, - children: [ - Image( - image: imageProvider, - fit: BoxFit.cover, - ), - if (kDebugMode && _showingFallback) - Positioned( - top: 4, - right: 4, - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: 4, - vertical: 2, - ), - decoration: BoxDecoration( - color: Colors.black.withValues(alpha: 0.6), - borderRadius: BorderRadius.circular(4), - ), - child: const Text( - "(T)", - style: TextStyle( - color: Colors.white, - fontSize: 10, - fontWeight: FontWeight.bold, - ), - ), - ), + super.build(context); + + return VisibilityDetector( + key: ValueKey( + 'person_face_visibility_${_visibilityKeySuffix()}_${identityHashCode(this)}', + ), + onVisibilityChanged: _handleVisibilityChanged, + child: FutureBuilder<_PersonFaceLoadResult?>( + future: _faceLoadFuture, + builder: (context, snapshot) { + final loadResult = snapshot.data; + if (loadResult?.faceCropBytes != null) { + return _buildFaceImage( + _buildImageFromBytes(loadResult!.faceCropBytes!), + ); + } + if (loadResult?.thumbnailBytes != null && + loadResult?.faceSource != null) { + final previewDimensions = _previewImageDimensionsForSource( + loadResult!.faceSource!, + ); + if (previewDimensions != null) { + return _buildFaceImage( + _DirectThumbnailFacePreview( + thumbnailBytes: loadResult.thumbnailBytes!, + faceBox: loadResult.faceSource!.face.detection.box, + imageDimensions: previewDimensions, ), - ], - ); - } - if (snapshot.connectionState == ConnectionState.waiting || - snapshot.connectionState == ConnectionState.active) { - return EnteLoadingWidget( - color: getEnteColorScheme(context).fillMuted, - ); - } - if (snapshot.hasError) { - _logger.severe( - "Error getting cover face for person", - snapshot.error, - snapshot.stackTrace, - ); - } else { - _logger.severe( - "faceCropFuture is null, no cover face found for person or cluster.", + ); + } + } + if (snapshot.connectionState == ConnectionState.waiting || + snapshot.connectionState == ConnectionState.active) { + return EnteLoadingWidget( + color: getEnteColorScheme(context).fillMuted, + ); + } + if (snapshot.hasError) { + _logger.severe( + 'Error getting cover face for person', + snapshot.error, + snapshot.stackTrace, + ); + } else { + _logger.severe( + 'No cover face found for person or cluster.', + ); + } + return _EmptyPersonThumbnail( + initial: isPerson ? (loadResult?.personName ?? _personName) : null, ); - } - return _EmptyPersonThumbnail( - initial: isPerson ? _personName : null, - ); - }, + }, + ), ); } - Future _loadFaceCrop() async { + Widget _buildImageFromBytes(Uint8List imageBytes) { + final shouldOptimize = + widget.cachedPixelWidth != null && widget.cachedPixelWidth! > 0; + final ImageProvider imageProvider = shouldOptimize + ? Image.memory( + imageBytes, + cacheWidth: widget.cachedPixelWidth, + ).image + : MemoryImage(imageBytes); + return Image( + image: imageProvider, + fit: BoxFit.cover, + ); + } + + Widget _buildFaceImage(Widget child) { + return Stack( + fit: StackFit.expand, + children: [ + child, + if (kDebugMode && _showingFallback) + Positioned( + top: 4, + right: 4, + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: 4, + vertical: 2, + ), + decoration: BoxDecoration( + color: Colors.black.withValues(alpha: 0.6), + borderRadius: BorderRadius.circular(4), + ), + child: const Text( + '(T)', + style: TextStyle( + color: Colors.white, + fontSize: 10, + fontWeight: FontWeight.bold, + ), + ), + ), + ), + ], + ); + } + + Future<_PersonFaceLoadResult?> _loadFace() async { if (!widget.useFullFile) { - final Uint8List? thumbnailCrop = - await _getFaceCrop(useFullFile: widget.useFullFile); - if (thumbnailCrop != null) { - _fallbackEverUsed = true; + final thumbnailCrop = await _getFaceCrop(useFullFile: false); + if (thumbnailCrop == null) { + return null; } _showingFallback = false; - return thumbnailCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: thumbnailCrop, + personName: _personName, + ); + } + + if (!_shouldUseProgressiveStrategy) { + return _loadFaceLegacy(); } - final Uint8List? fullCrop = - await _getFaceCrop(useFullFile: widget.useFullFile); + return _loadFaceProgressive(); + } + + Future<_PersonFaceLoadResult?> _loadFaceLegacy() async { + final fullCrop = await _getFaceCrop(useFullFile: true); if (fullCrop != null) { _showingFallback = false; - return fullCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ); } final String personOrClusterId = widget.personId ?? widget.clusterID!; _logger.warning( - "Full face crop unavailable for $personOrClusterId, attempting thumbnail fallback.", + 'Full face crop unavailable for $personOrClusterId, attempting thumbnail fallback.', ); - final Uint8List? fallbackCrop = await _getFaceCrop(useFullFile: false); + final fallbackCrop = await _getFaceCrop(useFullFile: false); if (fallbackCrop != null) { _showingFallback = true; - _fallbackEverUsed = true; - return fallbackCrop; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fallbackCrop, + personName: _personName, + ); } + _logger + .warning('Thumbnail fallback also unavailable for $personOrClusterId.'); + return null; + } + + Future<_PersonFaceLoadResult?> _loadFaceProgressive() async { + final String personOrClusterId = widget.personId ?? widget.clusterID!; + final faceSource = await _resolveFaceSource(); + if (faceSource == null) { + return null; + } + + final cachedFullCrop = await _loadCachedFullFaceCropIfAvailable(faceSource); + if (cachedFullCrop != null) { + _showingFallback = false; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: cachedFullCrop, + personName: _personName, + ); + } + + if (_previewImageDimensionsForSource(faceSource) != null) { + final thumbnailBytes = await _loadThumbnailPreviewBytes(faceSource.file); + if (thumbnailBytes != null) { + _showingFallback = true; + final generation = ++_upgradeGeneration; + unawaited(_attemptFullQualityUpgrade(generation)); + return _PersonFaceLoadResult.thumbnailPreview( + thumbnailBytes: thumbnailBytes, + faceSource: faceSource, + personName: _personName, + ); + } + } + + final thumbnailCrop = await _getFaceCrop( + useFullFile: false, + resolvedSource: faceSource, + ); + if (thumbnailCrop != null) { + _showingFallback = true; + final generation = ++_upgradeGeneration; + unawaited(_attemptFullQualityUpgrade(generation)); + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: thumbnailCrop, + personName: _personName, + ); + } + + _logger.warning( + 'Thumbnail face crop unavailable for $personOrClusterId, attempting full-file generation.', + ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + resolvedSource: faceSource, + ); + if (fullCrop != null) { + _showingFallback = false; + return _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ); + } _logger.warning( - "Thumbnail fallback also unavailable for $personOrClusterId.", + 'Full face crop also unavailable for $personOrClusterId after thumbnail miss.', ); return null; } - Future _getFaceCrop({required bool useFullFile}) async { + ImageDimensions? _previewImageDimensionsForSource( + PersonFaceSource faceSource, + ) { + final cachedThumbnailDimensions = + getCachedThumbnailSourceDimensionsForFileId(faceSource.resolvedFileId); + if (cachedThumbnailDimensions != null) { + return cachedThumbnailDimensions; + } + if (faceSource.file.width > 0 && faceSource.file.height > 0) { + return ( + width: faceSource.file.width, + height: faceSource.file.height, + ); + } + return null; + } + + Future _loadThumbnailPreviewBytes(EnteFile file) async { + final cachedThumbnailBytes = ThumbnailInMemoryLruCache.get(file); + if (cachedThumbnailBytes != null) { + return cachedThumbnailBytes; + } + + _requestedThumbnailPreviewFile = file; try { - final String personOrClusterId = widget.personId ?? widget.clusterID!; - final tryInMemoryCachedCrop = - checkInMemoryCachedCropForPersonOrClusterID(personOrClusterId); - if (tryInMemoryCachedCrop != null) return tryInMemoryCachedCrop; - String? fixedFaceID; - PersonEntity? personEntity; + try { + return await getThumbnail(file); + } catch (e, s) { + _logger.warning( + 'Failed to load preview thumbnail for progressive face fallback for file ${file.uploadedFileID ?? file.localID}', + e, + s, + ); + return null; + } + } finally { + if (_requestedThumbnailPreviewFile == file) { + _requestedThumbnailPreviewFile = null; + } + } + } + + Future _loadCachedFullFaceCropIfAvailable( + PersonFaceSource faceSource, + ) async { + final personOrClusterId = widget.personId ?? widget.clusterID!; + final cachedFullCrop = + checkInMemoryCachedCropForPersonOrClusterID(personOrClusterId); + if (cachedFullCrop != null) { + return cachedFullCrop; + } + return getPersistedFullFaceCropIfAvailable( + faceSource.face.faceID, + personOrClusterID: personOrClusterId, + ); + } + + void _applyResolvedFaceSource(PersonFaceSource faceSource) { + _resolvedFaceSource = faceSource; + fileForFaceCrop = faceSource.file; + _faceForFaceCrop = faceSource.face; + _faceCropFileId = faceSource.resolvedFileId; + _personName = faceSource.personName ?? _personName; + cacheFaceSourceForPersonOrClusterID( + widget.personId ?? widget.clusterID!, + faceSource, + ); + _touchVisibleFaceTasks(); + } + + void _handleVisibilityChanged(VisibilityInfo info) { + final nowVisible = info.visibleFraction >= 0.01; + if (nowVisible == _isVisible) { + return; + } + _isVisible = nowVisible; + if (_isVisible) { + _touchVisibleFaceTasks(); + _visibleTaskTouchTimer = Timer.periodic( + _kVisibleTaskTouchInterval, + (_) => _touchVisibleFaceTasks(), + ); + } else { + _cancelVisibleTaskTouchTimer(); + } + } + + void _touchVisibleFaceTasks() { + if (!_isVisible) { + return; + } + final fileId = _faceCropFileId; + if (fileId == null) { + return; + } + touchPendingFaceThumbnailGeneration(fileId, useFullFile: false); + // Keep visible full-quality upgrades from aging out of the bounded queue. + // Thumbnail work is still refreshed first so first-paint requests stay hot. + if (widget.useFullFile) { + touchPendingFaceThumbnailGeneration(fileId, useFullFile: true); + } + } + + void _cancelVisibleTaskTouchTimer() { + _visibleTaskTouchTimer?.cancel(); + _visibleTaskTouchTimer = null; + } + + void _recordPendingFaceGenerationClaim( + int fileId, { + required bool useFullFile, + }) { + final claims = useFullFile + ? _fullGenerationTaskClaims + : _thumbnailGenerationTaskClaims; + claims.update(fileId, (count) => count + 1, ifAbsent: () => 1); + } + + void _clearPendingFaceGenerationClaim( + int fileId, { + required bool useFullFile, + }) { + final claims = useFullFile + ? _fullGenerationTaskClaims + : _thumbnailGenerationTaskClaims; + final currentCount = claims[fileId]; + if (currentCount == null) { + return; + } + if (currentCount <= 1) { + claims.remove(fileId); + } else { + claims[fileId] = currentCount - 1; + } + } + + void _releasePendingFaceGenerationClaims() { + void releaseClaims(Map claims, {required bool useFullFile}) { + for (final entry in claims.entries) { + for (var i = 0; i < entry.value; i++) { + checkStopTryingToGenerateFaceThumbnails( + entry.key, + useFullFile: useFullFile, + ); + } + } + claims.clear(); + } + + releaseClaims(_fullGenerationTaskClaims, useFullFile: true); + releaseClaims(_thumbnailGenerationTaskClaims, useFullFile: false); + } + + bool _sameFaceSourceSeed( + PersonFaceSource? a, + PersonFaceSource? b, + ) { + if (identical(a, b)) { + return true; + } + if (a == null || b == null) { + return false; + } + return a.face.faceID == b.face.faceID && + a.resolvedFileId == b.resolvedFileId; + } + + bool _samePreviewFileSeed(EnteFile? a, EnteFile? b) { + if (identical(a, b)) { + return true; + } + if (a == null || b == null) { + return false; + } + return a.uploadedFileID == b.uploadedFileID && a.localID == b.localID; + } + + String _visibilityKeySuffix() { + final widgetKey = widget.key; + if (widgetKey is ValueKey) { + return widgetKey.value.toString(); + } + return widget.personId ?? widget.clusterID ?? widgetKey.toString(); + } + + bool _canUseResolvedFaceSource( + PersonFaceSource faceSource, + String personOrClusterId, + ) { + final currentFaceID = + checkInMemoryCachedFaceIDForPersonOrClusterID(personOrClusterId); + return currentFaceID == null || currentFaceID == faceSource.face.faceID; + } + + Future _canReuseResolvedFaceSource( + PersonFaceSource faceSource, + String personOrClusterId, { + bool clearSharedCache = false, + }) async { + if (!_canUseResolvedFaceSource(faceSource, personOrClusterId)) { + return false; + } + if (isOfflineMode) { + return true; + } + + final fileId = faceSource.file.uploadedFileID ?? faceSource.resolvedFileId; + + final hiddenFileIDs = await SearchService.instance.getHiddenFiles().then( + (files) => + files.map((file) => file.uploadedFileID).whereType().toSet(), + ); + if (!hiddenFileIDs.contains(fileId)) { + return true; + } + + _logger.info( + 'Skipping cached face source for hidden file ID $fileId for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + return false; + } + + Future _resolveFaceSource({ + bool notifyOnError = true, + }) async { + final personOrClusterId = widget.personId ?? widget.clusterID!; + Future? currentPersonEntityFuture; + Future? persistedFaceIdFuture; + + Future getCurrentPersonEntity() { + if (!isPerson || isOfflineMode) { + return Future.value(null); + } + return currentPersonEntityFuture ??= + PersonService.instance.getPerson(widget.personId!); + } + + Future getPersistedFaceId() { + return persistedFaceIdFuture ??= + checkUsedFaceIDForPersonOrClusterId(personOrClusterId); + } + + void invalidateResolvedFaceSource( + PersonFaceSource faceSource, { + required bool clearSharedCache, + }) { + if (identical(_resolvedFaceSource, faceSource)) { + _resolvedFaceSource = null; + } + if (clearSharedCache) { + removeCachedFaceSourceForPersonOrClusterID(personOrClusterId); + } + } + + Future isFaceStillAssignedToCurrentSubject( + PersonFaceSource faceSource, { + String? currentAvatarFaceId, + }) async { + final mlDataDB = + isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; + if (!isPerson) { + final currentClusterId = + await mlDataDB.getClusterIDForFaceID(faceSource.face.faceID); + return currentClusterId == widget.clusterID; + } + + if (currentAvatarFaceId != null) { + return faceSource.face.faceID == currentAvatarFaceId; + } + + final personIdsByFace = await mlDataDB.getFaceIdToPersonIdForFaces([ + faceSource.face.faceID, + ]); + return personIdsByFace[faceSource.face.faceID] == widget.personId; + } + + Future canReuseResolvedFaceSource( + PersonFaceSource faceSource, { + bool clearSharedCache = false, + }) async { + if (!await _canReuseResolvedFaceSource( + faceSource, + personOrClusterId, + clearSharedCache: clearSharedCache, + )) { + return false; + } + if (!isPerson || isOfflineMode) { + final isStillAssigned = + await isFaceStillAssignedToCurrentSubject(faceSource); + if (!isStillAssigned) { + _logger.fine( + 'Ignoring stale prefetched face source for ${widget.clusterID ?? widget.personId}: ' + 'face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + final persistedFaceId = await getPersistedFaceId(); + if (persistedFaceId == null || + persistedFaceId == faceSource.face.faceID) { + return true; + } + + _logger.fine( + 'Ignoring preview-seeded face source for ${widget.clusterID ?? widget.personId}: ' + 'persisted=$persistedFaceId face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + final personEntity = await getCurrentPersonEntity(); + if (personEntity == null) { + _logger.severe( + 'Person with ID ${widget.personId} not found, cannot validate face source.', + ); + return false; + } + _personName = personEntity.data.name; + + final currentAvatarFaceId = personEntity.data.avatarFaceID; + final isStillAssigned = await isFaceStillAssignedToCurrentSubject( + faceSource, + currentAvatarFaceId: currentAvatarFaceId, + ); + if (!isStillAssigned) { + _logger.fine( + 'Ignoring stale prefetched face source for person ${widget.personId}: ' + 'seeded=${widget.initialAvatarFaceId} ' + 'current=$currentAvatarFaceId ' + 'face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + if (currentAvatarFaceId != null) { + return true; + } + + final persistedFaceId = await getPersistedFaceId(); + if (persistedFaceId == null || + persistedFaceId == faceSource.face.faceID) { + return true; + } + + _logger.fine( + 'Ignoring preview-seeded face source for person ${widget.personId}: ' + 'persisted=$persistedFaceId face=${faceSource.face.faceID}', + ); + invalidateResolvedFaceSource( + faceSource, + clearSharedCache: clearSharedCache, + ); + return false; + } + + if (widget.initialFaceSource != null && + await canReuseResolvedFaceSource( + widget.initialFaceSource!, + )) { + _applyResolvedFaceSource(widget.initialFaceSource!); + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + widget.initialFaceSource!.face.faceID, + ); + return widget.initialFaceSource; + } + + if (_resolvedFaceSource != null) { + if (await canReuseResolvedFaceSource( + _resolvedFaceSource!, + )) { + return _resolvedFaceSource; + } + } + + final cachedFaceSource = + checkCachedFaceSourceForPersonOrClusterID(personOrClusterId); + if (cachedFaceSource != null) { + if (await canReuseResolvedFaceSource( + cachedFaceSource, + clearSharedCache: true, + )) { + _applyResolvedFaceSource(cachedFaceSource); + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + cachedFaceSource.face.faceID, + ); + return cachedFaceSource; + } + } + + try { + String? fixedFaceID = widget.initialAvatarFaceId; final mlDataDB = isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; if (isPerson && !isOfflineMode) { - personEntity = await PersonService.instance.getPerson(widget.personId!); + final personEntity = await getCurrentPersonEntity(); if (personEntity == null) { _logger.severe( - "Person with ID ${widget.personId} not found, cannot get cover face.", + 'Person with ID ${widget.personId} not found, cannot get cover face.', ); return null; } _personName = personEntity.data.name; - fixedFaceID = personEntity.data.avatarFaceID; + final currentAvatarFaceId = personEntity.data.avatarFaceID; + if (widget.initialAvatarFaceId != null && + widget.initialAvatarFaceId != currentAvatarFaceId) { + _logger.fine( + 'Ignoring stale seeded avatar face ID for person ${widget.personId}: ' + '${widget.initialAvatarFaceId} -> $currentAvatarFaceId', + ); + } + fixedFaceID = currentAvatarFaceId; } - fixedFaceID ??= - await checkUsedFaceIDForPersonOrClusterId(personOrClusterId); + fixedFaceID ??= await checkUsedFaceIDForPersonOrClusterId( + personOrClusterId, + ); - EnteFile? fileForFaceCrop; + EnteFile? selectedFileForFaceCrop; if (isOfflineMode) { final allFiles = await SearchService.instance.getAllFilesForSearch(); final localIdToFile = {}; @@ -240,6 +854,7 @@ class _PersonFaceWidgetState extends State localIdToFile[localId] = file; } } + if (fixedFaceID != null) { final localIntId = getFileIdFromFaceId(fixedFaceID); final localId = @@ -248,16 +863,19 @@ class _PersonFaceWidgetState extends State await checkRemoveCachedFaceIDForPersonOrClusterId( personOrClusterId, ); + fixedFaceID = null; } else { - fileForFaceCrop = localIdToFile[localId]; - if (fileForFaceCrop == null) { + selectedFileForFaceCrop = localIdToFile[localId]; + if (selectedFileForFaceCrop == null) { await checkRemoveCachedFaceIDForPersonOrClusterId( personOrClusterId, ); + fixedFaceID = null; } } } - if (fileForFaceCrop == null) { + + if (selectedFileForFaceCrop == null) { final List allFaces = isPerson ? await mlDataDB .getFaceIDsForPersonOrderedByScore(widget.personId!) @@ -273,44 +891,67 @@ class _PersonFaceWidgetState extends State final localId = localIdMap[localIntId]; final candidate = localId != null ? localIdToFile[localId] : null; if (candidate != null) { - fileForFaceCrop = candidate; + selectedFileForFaceCrop = candidate; fixedFaceID = faceID; break; } } - if (fileForFaceCrop == null) { - _logger.severe( - "No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", - ); - return null; + } + + if (selectedFileForFaceCrop == null) { + final initialPreviewFile = widget.initialPreviewFile; + final localId = initialPreviewFile?.localID; + if (localId != null && localId.isNotEmpty) { + selectedFileForFaceCrop = localIdToFile[localId]; } } + + if (selectedFileForFaceCrop == null) { + _logger.severe( + 'No suitable local file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; + } } else { - final hiddenFileIDs = await SearchService.instance - .getHiddenFiles() - .then((onValue) => onValue.map((e) => e.uploadedFileID)); + final hiddenFileIDs = + await SearchService.instance.getHiddenFiles().then( + (files) => files + .map((file) => file.uploadedFileID) + .whereType() + .toSet(), + ); + + final initialPreviewFile = widget.initialPreviewFile; if (fixedFaceID != null) { final fileID = getFileIdFromFaceId(fixedFaceID); - final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); - if (fileInDB == null) { - _logger.severe( - "File with ID $fileID not found in DB, cannot get cover face.", - ); - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); - } else if (hiddenFileIDs.contains(fileInDB.uploadedFileID)) { - _logger.info( - "File with ID $fileID is hidden, skipping it for face crop.", - ); - await checkRemoveCachedFaceIDForPersonOrClusterId( - personOrClusterId, - ); + if (initialPreviewFile?.uploadedFileID == fileID && + !hiddenFileIDs.contains(fileID)) { + selectedFileForFaceCrop = initialPreviewFile; } else { - fileForFaceCrop = fileInDB; + final fileInDB = await FilesDB.instance.getAnyUploadedFile(fileID); + if (fileInDB == null) { + _logger.severe( + 'File with ID $fileID not found in DB, cannot get cover face.', + ); + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } else if (hiddenFileIDs.contains(fileInDB.uploadedFileID)) { + _logger.info( + 'File with ID $fileID is hidden, skipping it for face crop.', + ); + await checkRemoveCachedFaceIDForPersonOrClusterId( + personOrClusterId, + ); + fixedFaceID = null; + } else { + selectedFileForFaceCrop = fileInDB; + } } } - if (fileForFaceCrop == null) { + + if (selectedFileForFaceCrop == null) { final List allFaces = isPerson ? await mlDataDB .getFaceIDsForPersonOrderedByScore(widget.personId!) @@ -320,48 +961,59 @@ class _PersonFaceWidgetState extends State final fileID = getFileIdFromFaceId(faceID); if (hiddenFileIDs.contains(fileID)) { _logger.info( - "File with ID $fileID is hidden, skipping it for face crop.", + 'File with ID $fileID is hidden, skipping it for face crop.', ); continue; } - fileForFaceCrop = await FilesDB.instance.getAnyUploadedFile(fileID); - if (fileForFaceCrop != null) { + selectedFileForFaceCrop = + await FilesDB.instance.getAnyUploadedFile(fileID); + if (selectedFileForFaceCrop != null) { _logger.info( - "Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Using file ID $fileID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); fixedFaceID = faceID; break; } } - if (fileForFaceCrop == null) { - _logger.severe( - "No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", - ); - return null; - } + } + + if (selectedFileForFaceCrop == null && + initialPreviewFile != null && + initialPreviewFile.uploadedFileID != null && + !hiddenFileIDs.contains(initialPreviewFile.uploadedFileID)) { + selectedFileForFaceCrop = initialPreviewFile; + } + + if (selectedFileForFaceCrop == null) { + _logger.severe( + 'No suitable file found for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', + ); + return null; } } - int? recentFileID; + + final int? recentFileID; if (isOfflineMode) { - final localId = fileForFaceCrop.localID; + final localId = selectedFileForFaceCrop.localID; if (localId == null || localId.isEmpty) { _logger.severe( - "Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Missing local ID for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } recentFileID = await OfflineFilesDB.instance.getOrCreateLocalIntId(localId); } else { - recentFileID = fileForFaceCrop.uploadedFileID; + recentFileID = selectedFileForFaceCrop.uploadedFileID; } if (recentFileID == null) { _logger.severe( - "Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}", + 'Missing file id for face crop for person: ${widget.personId} or cluster: ${widget.clusterID}', ); return null; } - final Face? face = await mlDataDB.getCoverFaceForPerson( + + final face = await mlDataDB.getCoverFaceForPerson( recentFileID: recentFileID, avatarFaceId: fixedFaceID, personID: widget.personId, @@ -369,41 +1021,365 @@ class _PersonFaceWidgetState extends State ); if (face == null) { _logger.severe( - "No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID", + 'No cover face for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID', ); - await checkRemoveCachedFaceIDForPersonOrClusterId( + await checkRemoveCachedFaceIDForPersonOrClusterId(personOrClusterId); + return null; + } + await cacheFaceIdForPersonOrClusterIfNeeded( + personOrClusterId, + face.faceID, + ); + + final faceSource = PersonFaceSource( + file: selectedFileForFaceCrop, + face: face, + resolvedFileId: recentFileID, + personName: _personName ?? widget.initialPersonName, + ); + _applyResolvedFaceSource(faceSource); + return faceSource; + } catch (e, s) { + _logger.severe( + 'Error resolving face source for person: ${widget.personId} or cluster ${widget.clusterID}', + e, + s, + ); + if (notifyOnError) { + widget.onErrorCallback?.call(); + } + return null; + } + } + + Future _attemptFullQualityUpgrade(int generation) async { + if (_shouldAbortUpgrade(generation)) { + return; + } + + final face = _faceForFaceCrop; + final sourceFile = fileForFaceCrop; + if (face == null || sourceFile == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=missing_face_or_file person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + if (sourceFile.fileType == FileType.video) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=video_source person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + if (!isPerson && await _isSmallUnnamedCluster()) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=small_unnamed_cluster cluster=${widget.clusterID}', + ); + return; + } + + if (await hasPersistedFullFaceCrop(face.faceID)) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=full_crop_cached face=${face.faceID}', + ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation) || !mounted) { + return; + } + setState(() { + _showingFallback = false; + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); + }); + return; + } + + if (sourceFile.width <= 0 || sourceFile.height <= 0) { + _logger.fine( + 'person_face_thumbnail_upgrade_fallback reason=missing_full_dimensions file=${sourceFile.uploadedFileID}', + ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation) || !mounted) { + return; + } + setState(() { + _showingFallback = false; + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); + }); + return; + } + + final cachedThumbnailDimensions = _faceCropFileId != null + ? getCachedThumbnailSourceDimensionsForFileId(_faceCropFileId!) + : null; + final thumbnailDimensions = cachedThumbnailDimensions ?? + estimateThumbnailDimensionsFromFullDimensions( + fullWidth: sourceFile.width, + fullHeight: sourceFile.height, + ); + if (thumbnailDimensions == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=missing_thumbnail_dimensions file=${sourceFile.uploadedFileID}', + ); + return; + } + if (cachedThumbnailDimensions == null) { + _logger.fine( + 'person_face_thumbnail_upgrade_thumbnail_dimensions_source=estimated ' + 'file=${sourceFile.uploadedFileID} ' + 'estimated=${thumbnailDimensions.width}x${thumbnailDimensions.height}', + ); + } + + final decision = shouldUpgradeFromThumbnail( + faceBox: face.detection.box, + thumbnailWidth: thumbnailDimensions.width, + thumbnailHeight: thumbnailDimensions.height, + fullWidth: sourceFile.width, + fullHeight: sourceFile.height, + upscaleThreshold: _kProgressiveUpgradeUpscaleThreshold, + minImprovementRatio: _kProgressiveUpgradeMinImprovementRatio, + ); + if (!decision.shouldUpgrade) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=${decision.reason} ' + 'thumbUpscale=${decision.thumbnailUpscaleFactor.toStringAsFixed(2)} ' + 'fullUpscale=${decision.fullUpscaleFactor.toStringAsFixed(2)} ' + 'improvement=${decision.improvementRatio.toStringAsFixed(2)} ' + 'person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + _logger.fine( + 'person_face_thumbnail_upgrade_waiting_for_idle ' + 'person=${widget.personId} cluster=${widget.clusterID} ' + 'thumbUpscale=${decision.thumbnailUpscaleFactor.toStringAsFixed(2)}', + ); + final didReachThumbnailIdle = await waitForThumbnailFaceGenerationIdle( + shouldStopWaiting: () => _shouldAbortUpgrade(generation), + maxWait: _kProgressiveUpgradeIdleWaitBudget, + ); + if (_shouldAbortUpgrade(generation)) { + return; + } + if (!didReachThumbnailIdle) { + _logger.fine( + 'person_face_thumbnail_upgrade_wait_idle_timeout person=${widget.personId} cluster=${widget.clusterID}', + ); + } + + _logger.info( + 'person_face_thumbnail_upgrade_started person=${widget.personId} cluster=${widget.clusterID}', + ); + final fullCrop = await _getFaceCrop( + useFullFile: true, + notifyOnError: false, + ); + if (fullCrop == null || _shouldAbortUpgrade(generation)) { + _logger.fine( + 'person_face_thumbnail_upgrade_skipped reason=full_crop_unavailable person=${widget.personId} cluster=${widget.clusterID}', + ); + return; + } + + if (!mounted) { + return; + } + setState(() { + _showingFallback = false; + _faceLoadFuture = Future.value( + _PersonFaceLoadResult.faceCrop( + faceCropBytes: fullCrop, + personName: _personName, + ), + ); + }); + + _logger.info( + 'person_face_thumbnail_upgrade_applied person=${widget.personId} cluster=${widget.clusterID}', + ); + } + + bool _shouldAbortUpgrade(int generation) { + return _disposed || generation != _upgradeGeneration; + } + + Future _isSmallUnnamedCluster() async { + final clusterID = widget.clusterID; + if (clusterID == null || isPerson) { + return false; + } + final cachedFileCount = _clusterToFileCountCache.get(clusterID); + if (cachedFileCount != null) { + return cachedFileCount < _kMinUnnamedClusterSizeForProgressiveUpgrade; + } + + final mlDataDB = + isOfflineMode ? MLDataDB.offlineInstance : MLDataDB.instance; + final fileCount = await mlDataDB + .getFileIDsOfClusterID(clusterID) + .then((fileIDs) => fileIDs.length); + _clusterToFileCountCache.put(clusterID, fileCount); + return fileCount < _kMinUnnamedClusterSizeForProgressiveUpgrade; + } + + Future _getFaceCrop({ + required bool useFullFile, + PersonFaceSource? resolvedSource, + bool notifyOnError = true, + }) async { + try { + final String personOrClusterId = widget.personId ?? widget.clusterID!; + final tryInMemoryCachedCrop = + checkInMemoryCachedCropForPersonOrClusterID(personOrClusterId); + if (tryInMemoryCachedCrop != null) { + return tryInMemoryCachedCrop; + } + if (!useFullFile) { + final tryInMemoryCachedThumbnailCrop = + checkInMemoryCachedThumbnailCropForPersonOrClusterID( personOrClusterId, ); + if (tryInMemoryCachedThumbnailCrop != null) { + return tryInMemoryCachedThumbnailCrop; + } + } + + final faceSource = resolvedSource ?? + await _resolveFaceSource(notifyOnError: notifyOnError); + if (faceSource == null) { return null; } - final cropMap = await getCachedFaceCrops( - fileForFaceCrop, - [face], - useFullFile: useFullFile, - personOrClusterID: personOrClusterId, - useTempCache: false, - ); - this.fileForFaceCrop = fileForFaceCrop; - _faceCropFileId = recentFileID; - final result = cropMap?[face.faceID]; - if (result == null) { - _logger.severe( - "Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID $recentFileID", + + var didRecordGenerationClaim = false; + try { + final cropMap = await getCachedFaceCrops( + faceSource.file, + [faceSource.face], + useFullFile: useFullFile, + personOrClusterID: personOrClusterId, + onGenerationTaskQueued: () { + if (didRecordGenerationClaim) { + return; + } + didRecordGenerationClaim = true; + _recordPendingFaceGenerationClaim( + faceSource.resolvedFileId, + useFullFile: useFullFile, + ); + }, + useTempCache: false, ); + _applyResolvedFaceSource(faceSource); + final result = cropMap?[faceSource.face.faceID]; + if (result == null) { + _logger.severe( + 'Null cover face crop for person: ${widget.personId} or cluster ${widget.clusterID} and fileID ${faceSource.resolvedFileId}', + ); + } + return result; + } finally { + if (didRecordGenerationClaim) { + _clearPendingFaceGenerationClaim( + faceSource.resolvedFileId, + useFullFile: useFullFile, + ); + } } - return result; } catch (e, s) { _logger.severe( - "Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}", + 'Error getting cover face for person: ${widget.personId} or cluster ${widget.clusterID}', e, s, ); - widget.onErrorCallback?.call(); + if (notifyOnError) { + widget.onErrorCallback?.call(); + } return null; } } } +class _DirectThumbnailFacePreview extends StatelessWidget { + final Uint8List thumbnailBytes; + final FaceBox faceBox; + final ImageDimensions imageDimensions; + + const _DirectThumbnailFacePreview({ + required this.thumbnailBytes, + required this.faceBox, + required this.imageDimensions, + }); + + @override + Widget build(BuildContext context) { + final crop = computeNormalizedFaceCrop(faceBox); + if (crop == null) { + return const SizedBox.shrink(); + } + + const imageHeight = 1000.0; + final imageWidth = + imageHeight * imageDimensions.width / imageDimensions.height; + final cropX = crop.x * imageWidth; + final cropY = crop.y * imageHeight; + final cropWidth = crop.width * imageWidth; + final cropHeight = crop.height * imageHeight; + + if (cropWidth <= 0 || cropHeight <= 0) { + return const SizedBox.shrink(); + } + + return ClipRect( + child: SizedBox.expand( + child: FittedBox( + fit: BoxFit.cover, + child: SizedBox( + width: cropWidth, + height: cropHeight, + child: ClipRect( + child: Stack( + fit: StackFit.expand, + children: [ + Positioned( + left: -cropX, + top: -cropY, + width: imageWidth, + height: imageHeight, + child: Image.memory( + thumbnailBytes, + fit: BoxFit.fill, + gaplessPlayback: true, + filterQuality: FilterQuality.low, + ), + ), + ], + ), + ), + ), + ), + ), + ); + } +} + class _EmptyPersonThumbnail extends StatelessWidget { final String? initial; diff --git a/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart new file mode 100644 index 00000000000..06acf011a12 --- /dev/null +++ b/mobile/apps/photos/lib/ui/viewer/people/visible_face_source_prefetch.dart @@ -0,0 +1,100 @@ +import "dart:async"; + +import "package:flutter/widgets.dart"; +import "package:photos/models/search/generic_search_result.dart"; +import "package:photos/models/search/search_constants.dart"; +import "package:photos/services/search_service.dart"; +import "package:visibility_detector/visibility_detector.dart"; + +class VisibleFaceSourcePrefetch extends StatefulWidget { + final List results; + final int index; + final int prefetchCount; + final int leadingBuffer; + final Widget child; + + const VisibleFaceSourcePrefetch({ + required this.results, + required this.index, + required this.child, + this.prefetchCount = 24, + this.leadingBuffer = 0, + super.key, + }); + + @override + State createState() => + _VisibleFaceSourcePrefetchState(); +} + +class _VisibleFaceSourcePrefetchState extends State { + bool _hasPrefetchedForCurrentItem = false; + + @override + void didUpdateWidget(covariant VisibleFaceSourcePrefetch oldWidget) { + super.didUpdateWidget(oldWidget); + final oldTargetKey = _visibilityTargetKeyFor( + results: oldWidget.results, + index: oldWidget.index, + ); + final newTargetKey = _visibilityTargetKeyFor( + results: widget.results, + index: widget.index, + ); + if (oldWidget.index != widget.index || + !identical(oldWidget.results, widget.results) || + oldTargetKey != newTargetKey) { + _hasPrefetchedForCurrentItem = false; + } + } + + @override + Widget build(BuildContext context) { + return VisibilityDetector( + key: ValueKey( + "face_source_prefetch_${_visibilityTargetKey()}_${widget.index}", + ), + onVisibilityChanged: (info) { + if (info.visibleFraction >= 0.01) { + _maybePrefetchWindow(); + } + }, + child: widget.child, + ); + } + + void _maybePrefetchWindow() { + if (_hasPrefetchedForCurrentItem) { + return; + } + _hasPrefetchedForCurrentItem = true; + unawaited( + SearchService.instance.prefetchFaceSourcesInWindow( + widget.results, + startIndex: widget.index, + count: widget.prefetchCount, + leadingBuffer: widget.leadingBuffer, + ), + ); + } + + String _visibilityTargetKey() { + return _visibilityTargetKeyFor( + results: widget.results, + index: widget.index, + ); + } + + String _visibilityTargetKeyFor({ + required List results, + required int index, + }) { + if (index < 0 || index >= results.length) { + return index.toString(); + } + final result = results[index]; + final personID = result.params[kPersonParamID] as String?; + final clusterID = result.params[kClusterParamId] as String?; + return personID ?? clusterID ?? index.toString(); + } +} diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart index eb6c3373f37..5c4991ac288 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/people_section_all_page.dart @@ -8,6 +8,7 @@ import "package:photos/events/event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/events/people_sort_order_change_event.dart"; import "package:photos/generated/intl/app_localizations.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/recent_searches.dart"; import "package:photos/models/search/search_constants.dart"; @@ -31,6 +32,7 @@ import "package:photos/ui/viewer/people/face_thumbnail_squircle.dart"; import "package:photos/ui/viewer/people/person_face_widget.dart"; import "package:photos/ui/viewer/people/person_gallery_suggestion.dart"; import "package:photos/ui/viewer/people/pinned_person_badge.dart"; +import "package:photos/ui/viewer/people/visible_face_source_prefetch.dart"; import "package:photos/ui/viewer/search/result/search_result_page.dart"; import "package:photos/ui/viewer/search_tab/people_section.dart"; import "package:photos/utils/local_settings.dart"; @@ -302,10 +304,16 @@ class FaceSearchResult extends StatelessWidget { final params = (searchResult as GenericSearchResult).params; final int cachedPixelWidth = (displaySize * MediaQuery.devicePixelRatioOf(context)).toInt(); + final previewFile = searchResult.previewThumbnail(); + final personName = searchResult.name(); return PersonFaceWidget( personId: params[kPersonParamID], clusterID: params[kClusterParamId], cachedPixelWidth: cachedPixelWidth, + initialFaceSource: params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: personName.isNotEmpty ? personName : null, + initialAvatarFaceId: params[kPersonAvatarFaceID] as String?, + initialPreviewFile: previewFile, key: params.containsKey(kPersonWidgetKey) ? ValueKey(params[kPersonWidgetKey]) : ValueKey(params[kPersonParamID] ?? params[kClusterParamId]), @@ -680,6 +688,8 @@ class _PeopleSectionAllWidgetState extends State { } final screenWidth = MediaQuery.of(context).size.width; final crossAxisCount = (screenWidth / 100).floor(); + final visibleFacePrefetchCount = crossAxisCount * 6; + final visibleFaceLeadingBuffer = crossAxisCount; final itemSize = (screenWidth - ((horizontalEdgePadding * 2) + @@ -712,18 +722,24 @@ class _PeopleSectionAllWidgetState extends State { childCount: searchResults.length, (context, index) { final result = searchResults[index]; - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: result, - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: defaultFaces.contains(result), - ) - : PersonSearchExample( - searchResult: result, - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: searchResults, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: result, + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: defaultFaces.contains(result), + ) + : PersonSearchExample( + searchResult: result, + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), @@ -788,18 +804,24 @@ class _PeopleSectionAllWidgetState extends State { delegate: SliverChildBuilderDelegate( childCount: filteredNormalFaces.length, (context, index) { - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: filteredNormalFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: true, - ) - : PersonSearchExample( - searchResult: filteredNormalFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: filteredNormalFaces, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: filteredNormalFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: true, + ) + : PersonSearchExample( + searchResult: filteredNormalFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), @@ -830,18 +852,24 @@ class _PeopleSectionAllWidgetState extends State { delegate: SliverChildBuilderDelegate( childCount: filteredExtraFaces.length, (context, index) { - return !widget.namedOnly - ? SelectablePersonSearchExample( - searchResult: filteredExtraFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - isDefaultFace: false, - ) - : PersonSearchExample( - searchResult: filteredExtraFaces[index], - size: itemSize, - selectedPeople: widget.selectedPeople!, - ); + return VisibleFaceSourcePrefetch( + results: filteredExtraFaces, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: !widget.namedOnly + ? SelectablePersonSearchExample( + searchResult: filteredExtraFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + isDefaultFace: false, + ) + : PersonSearchExample( + searchResult: filteredExtraFaces[index], + size: itemSize, + selectedPeople: widget.selectedPeople!, + ), + ); }, ), ), diff --git a/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart b/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart index 2a0cc624ebb..20ab3eaea0b 100644 --- a/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart +++ b/mobile/apps/photos/lib/ui/viewer/search/result/search_thumbnail_widget.dart @@ -2,6 +2,7 @@ import "package:flutter/material.dart"; import "package:logging/logging.dart"; import "package:photos/models/api/collection/user.dart"; import 'package:photos/models/file/file.dart'; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/search_constants.dart"; import "package:photos/models/search/search_result.dart"; @@ -41,6 +42,14 @@ class SearchThumbnailWidget extends StatelessWidget { .params[kPersonParamID], clusterID: (searchResult as GenericSearchResult) .params[kClusterParamId], + initialFaceSource: (searchResult as GenericSearchResult) + .params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: searchResult!.name().isNotEmpty + ? searchResult!.name() + : null, + initialAvatarFaceId: (searchResult as GenericSearchResult) + .params[kPersonAvatarFaceID] as String?, + initialPreviewFile: searchResult!.previewThumbnail(), ) : ThumbnailWidget( file!, diff --git a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart index 942fb477133..330c0f59347 100644 --- a/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart +++ b/mobile/apps/photos/lib/ui/viewer/search_tab/people_section.dart @@ -7,6 +7,7 @@ import "package:photos/events/event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/ml/face/person.dart"; +import "package:photos/models/ml/face/person_face_source.dart"; import "package:photos/models/search/generic_search_result.dart"; import "package:photos/models/search/recent_searches.dart"; import "package:photos/models/search/search_constants.dart"; @@ -22,6 +23,7 @@ import "package:photos/ui/viewer/people/add_person_action_sheet.dart"; import "package:photos/ui/viewer/people/face_thumbnail_squircle.dart"; import "package:photos/ui/viewer/people/people_page.dart"; import 'package:photos/ui/viewer/people/person_face_widget.dart'; +import "package:photos/ui/viewer/people/visible_face_source_prefetch.dart"; import "package:photos/ui/viewer/search/result/people_section_all_page.dart"; import "package:photos/ui/viewer/search/result/search_result_page.dart"; import "package:photos/ui/viewer/search/search_section_cta.dart"; @@ -171,6 +173,8 @@ class SearchExampleRow extends StatelessWidget { @override Widget build(BuildContext context) { + const visibleFacePrefetchCount = 12; + const visibleFaceLeadingBuffer = 2; return SizedBox( height: 128, child: ListView.separated( @@ -179,9 +183,15 @@ class SearchExampleRow extends StatelessWidget { scrollDirection: Axis.horizontal, itemCount: examples.length, itemBuilder: (context, index) { - return PersonSearchExample( - searchResult: examples[index], - selectedPeople: null, + return VisibleFaceSourcePrefetch( + results: examples, + index: index, + prefetchCount: visibleFacePrefetchCount, + leadingBuffer: visibleFaceLeadingBuffer, + child: PersonSearchExample( + searchResult: examples[index], + selectedPeople: null, + ), ); }, separatorBuilder: (context, index) => const SizedBox(width: 3), @@ -389,10 +399,16 @@ class FaceSearchResult extends StatelessWidget { final params = (searchResult as GenericSearchResult).params; final int cachedPixelWidth = (displaySize * MediaQuery.devicePixelRatioOf(context)).toInt(); + final previewFile = searchResult.previewThumbnail(); + final personName = searchResult.name(); return PersonFaceWidget( personId: params[kPersonParamID], clusterID: params[kClusterParamId], cachedPixelWidth: cachedPixelWidth, + initialFaceSource: params[kPersonFaceSource] as PersonFaceSource?, + initialPersonName: personName.isNotEmpty ? personName : null, + initialAvatarFaceId: params[kPersonAvatarFaceID] as String?, + initialPreviewFile: previewFile, key: params.containsKey(kPersonWidgetKey) ? ValueKey(params[kPersonWidgetKey]) : ValueKey(params[kPersonParamID] ?? params[kClusterParamId]), diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart index aafe96a1519..0cf86d63899 100644 --- a/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_cache.dart @@ -13,7 +13,8 @@ import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; import "package:photos/models/ml/face/box.dart"; import "package:photos/models/ml/face/face.dart"; -import "package:photos/service_locator.dart" show isOfflineMode; +import "package:photos/models/ml/face/person_face_source.dart"; +import "package:photos/service_locator.dart" show flagService, isOfflineMode; import "package:photos/services/machine_learning/face_thumbnail_generator.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/thumbnail_util.dart"; @@ -21,19 +22,24 @@ import "package:photos/utils/thumbnail_util.dart"; final _logger = Logger("FaceCropUtils"); const int _retryLimit = 3; +const _kFaceGenerationTaskTimeout = Duration(minutes: 5); final LRUMap _faceCropCache = LRUMap(100); final LRUMap _faceCropThumbnailCache = LRUMap(100); +final LRUMap + _thumbnailSourceDimensionsByFileId = LRUMap(2000); final LRUMap _personOrClusterIdToCachedFaceID = LRUMap(2000); +final LRUMap _personOrClusterIdToFaceSourceCache = + LRUMap(1000); TaskQueue _queueFullFileFaceGenerations = TaskQueue( maxConcurrentTasks: 5, - taskTimeout: const Duration(minutes: 1), + taskTimeout: _kFaceGenerationTaskTimeout, maxQueueSize: 100, ); TaskQueue _queueThumbnailFaceGenerations = TaskQueue( maxConcurrentTasks: 5, - taskTimeout: const Duration(minutes: 1), + taskTimeout: _kFaceGenerationTaskTimeout, maxQueueSize: 100, ); @@ -47,6 +53,44 @@ Uint8List? checkInMemoryCachedCropForPersonOrClusterID( return cachedCover; } +String? checkInMemoryCachedFaceIDForPersonOrClusterID( + String personOrClusterID, +) { + return _personOrClusterIdToCachedFaceID.get(personOrClusterID); +} + +Uint8List? checkInMemoryCachedThumbnailCropForPersonOrClusterID( + String personOrClusterID, +) { + final String? faceID = + _personOrClusterIdToCachedFaceID.get(personOrClusterID); + if (faceID == null) return null; + return _faceCropThumbnailCache.get(faceID); +} + +({int width, int height})? getCachedThumbnailSourceDimensionsForFileId( + int fileId, +) { + return _thumbnailSourceDimensionsByFileId.get(fileId); +} + +PersonFaceSource? checkCachedFaceSourceForPersonOrClusterID( + String personOrClusterID, +) { + return _personOrClusterIdToFaceSourceCache.get(personOrClusterID); +} + +void removeCachedFaceSourceForPersonOrClusterID(String personOrClusterID) { + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); +} + +void cacheFaceSourceForPersonOrClusterID( + String personOrClusterID, + PersonFaceSource faceSource, +) { + _personOrClusterIdToFaceSourceCache.put(personOrClusterID, faceSource); +} + Uint8List? _checkInMemoryCachedCropForFaceID(String faceID) { final Uint8List? cachedCover = _faceCropCache.get(faceID); return cachedCover; @@ -76,9 +120,24 @@ Future putFaceIdCachedForPersonOrCluster( personOrClusterID, faceID, ); + final cachedFaceSource = + _personOrClusterIdToFaceSourceCache.get(personOrClusterID); + if (cachedFaceSource != null && cachedFaceSource.face.faceID != faceID) { + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); + } _personOrClusterIdToCachedFaceID.put(personOrClusterID, faceID); } +Future cacheFaceIdForPersonOrClusterIfNeeded( + String personOrClusterID, + String faceID, +) async { + if (_personOrClusterIdToCachedFaceID.get(personOrClusterID) == faceID) { + return; + } + await putFaceIdCachedForPersonOrCluster(personOrClusterID, faceID); +} + Future _putCachedCropForFaceID( String faceID, Uint8List data, [ @@ -98,6 +157,7 @@ Future checkRemoveCachedFaceIDForPersonOrClusterId( await mlDataDB.getFaceIdUsedForPersonOrCluster(personOrClusterID); if (cachedFaceID != null) { _personOrClusterIdToCachedFaceID.remove(personOrClusterID); + _personOrClusterIdToFaceSourceCache.remove(personOrClusterID); await mlDataDB.removeFaceIdCachedForPersonOrCluster(personOrClusterID); } } @@ -109,6 +169,7 @@ Future?> getCachedFaceCrops( int fetchAttempt = 1, bool useFullFile = true, String? personOrClusterID, + VoidCallback? onGenerationTaskQueued, required bool useTempCache, }) async { try { @@ -174,6 +235,7 @@ Future?> getCachedFaceCrops( final result = await _getFaceCropsUsingHeapPriorityQueue( enteFile, facesWithoutCrops, + onGenerationTaskQueued: onGenerationTaskQueued, useFullFile: useFullFile, ); if (result == null) { @@ -225,6 +287,7 @@ Future?> getCachedFaceCrops( faces, fetchAttempt: fetchAttempt + 1, useFullFile: useFullFile, + onGenerationTaskQueued: onGenerationTaskQueued, useTempCache: useTempCache, ); } @@ -299,9 +362,87 @@ void checkStopTryingToGenerateFaceThumbnails( } } +bool touchPendingFaceThumbnailGeneration( + int fileID, { + bool useFullFile = true, +}) { + final taskId = [fileID, useFullFile ? "-full" : "-thumbnail"].join(); + if (useFullFile) { + return _queueFullFileFaceGenerations.touchTask(taskId); + } else { + return _queueThumbnailFaceGenerations.touchTask(taskId); + } +} + +bool areThumbnailFaceGenerationQueuesIdle() { + return _queueThumbnailFaceGenerations.pendingTasksCount == 0 && + _queueThumbnailFaceGenerations.runningTasksCount == 0; +} + +Future waitForThumbnailFaceGenerationIdle({ + Duration pollInterval = const Duration(milliseconds: 120), + Duration? maxWait, + bool Function()? shouldStopWaiting, +}) async { + final startedAt = DateTime.now(); + while (true) { + if (shouldStopWaiting?.call() ?? false) { + return false; + } + if (areThumbnailFaceGenerationQueuesIdle()) { + return true; + } + if (maxWait != null && DateTime.now().difference(startedAt) >= maxWait) { + return false; + } + await Future.delayed(pollInterval); + } +} + +Future hasPersistedFullFaceCrop(String faceID) async { + final faceCropCacheFile = cachedFaceCropPath(faceID, false); + return faceCropCacheFile.exists(); +} + +Future getPersistedFullFaceCropIfAvailable( + String faceID, { + String? personOrClusterID, +}) async { + final cachedFace = _checkInMemoryCachedCropForFaceID(faceID); + if (cachedFace != null) { + return cachedFace; + } + + final faceCropCacheFile = cachedFaceCropPath(faceID, false); + if (!(await faceCropCacheFile.exists())) { + return null; + } + + try { + final data = await faceCropCacheFile.readAsBytes(); + if (data.isEmpty) { + _logger.warning( + "Persisted face crop for faceID $faceID is empty, deleting file ${faceCropCacheFile.path}", + ); + await faceCropCacheFile.delete(); + return null; + } + await _putCachedCropForFaceID(faceID, data, personOrClusterID); + return data; + } catch (e, s) { + _logger.warning( + "Error reading persisted face crop for faceID $faceID from file ${faceCropCacheFile.path}", + e, + s, + ); + return null; + } +} + Future?> _getFaceCropsUsingHeapPriorityQueue( EnteFile file, Map faceBoxeMap, { + VoidCallback? onGenerationTaskQueued, bool useFullFile = true, }) async { final completer = Completer?>(); @@ -316,6 +457,7 @@ Future?> _getFaceCropsUsingHeapPriorityQueue( taskId = await _faceCropTaskId(file, useFullFile: false); } + onGenerationTaskQueued?.call(); await relevantTaskQueue.addTask(taskId, () async { final faceCrops = await _getFaceCrops( file, @@ -349,6 +491,35 @@ Future _faceCropTaskId( return "$baseId${useFullFile ? "-full" : "-thumbnail"}"; } +Future _faceCropFileIdForDimensions(EnteFile file) async { + if (isOfflineMode) { + final localId = file.localID; + if (localId == null || localId.isEmpty) { + return null; + } + return OfflineFilesDB.instance.getOrCreateLocalIntId(localId); + } + return file.uploadedFileID; +} + +Future _cacheThumbnailSourceDimensionsForFile( + EnteFile file, { + required int width, + required int height, +}) async { + if (width <= 0 || height <= 0) { + return; + } + final fileId = await _faceCropFileIdForDimensions(file); + if (fileId == null) { + return; + } + _thumbnailSourceDimensionsByFileId.put( + fileId, + (width: width, height: height), + ); +} + Future?> _getFaceCrops( EnteFile file, Map faceBoxeMap, { @@ -376,12 +547,27 @@ Future?> _getFaceCrops( faceIds.add(e.key); faceBoxes.add(e.value); } - final List faceCrop = - await FaceThumbnailGenerator.instance.generateFaceThumbnails( - // await generateJpgFaceThumbnails( - imagePath, - faceBoxes, - ); + late final List faceCrop; + if (flagService.progressivePersonFaceThumbnailsEnabled) { + final generationResult = await FaceThumbnailGenerator.instance + .generateFaceThumbnailsWithSourceDimensions( + imagePath, + faceBoxes, + ); + if (!useFullFile) { + await _cacheThumbnailSourceDimensionsForFile( + file, + width: generationResult.sourceWidth, + height: generationResult.sourceHeight, + ); + } + faceCrop = generationResult.thumbnails; + } else { + faceCrop = await FaceThumbnailGenerator.instance.generateFaceThumbnails( + imagePath, + faceBoxes, + ); + } final Map result = {}; for (int i = 0; i < faceCrop.length; i++) { result[faceIds[i]] = faceCrop[i]; diff --git a/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart new file mode 100644 index 00000000000..2500c8dbc6f --- /dev/null +++ b/mobile/apps/photos/lib/utils/face/face_thumbnail_quality.dart @@ -0,0 +1,194 @@ +import 'package:photos/core/constants.dart' show thumbnailLargeSize; +import 'package:photos/models/ml/face/box.dart'; + +const int kFaceThumbnailTargetShortSide = 512; +const double kFaceThumbnailRegularPadding = 0.4; +const double kFaceThumbnailMinimumPadding = 0.1; + +class FaceThumbnailUpgradeDecision { + final bool shouldUpgrade; + final String reason; + final double thumbnailUpscaleFactor; + final double fullUpscaleFactor; + final double improvementRatio; + + const FaceThumbnailUpgradeDecision({ + required this.shouldUpgrade, + required this.reason, + required this.thumbnailUpscaleFactor, + required this.fullUpscaleFactor, + required this.improvementRatio, + }); +} + +typedef ImageDimensions = ({int width, int height}); +typedef NormalizedFaceCrop = ({ + double x, + double y, + double width, + double height +}); + +ImageDimensions? estimateThumbnailDimensionsFromFullDimensions({ + required int fullWidth, + required int fullHeight, + int thumbnailMaxSide = thumbnailLargeSize, +}) { + if (fullWidth <= 0 || fullHeight <= 0 || thumbnailMaxSide <= 0) { + return null; + } + + if (fullWidth <= thumbnailMaxSide && fullHeight <= thumbnailMaxSide) { + return (width: fullWidth, height: fullHeight); + } + + if (fullWidth >= fullHeight) { + return ( + width: thumbnailMaxSide, + height: _max(1, (fullHeight * thumbnailMaxSide / fullWidth).round()), + ); + } + + return ( + width: _max(1, (fullWidth * thumbnailMaxSide / fullHeight).round()), + height: thumbnailMaxSide, + ); +} + +FaceThumbnailUpgradeDecision shouldUpgradeFromThumbnail({ + required FaceBox faceBox, + required int thumbnailWidth, + required int thumbnailHeight, + required int fullWidth, + required int fullHeight, + double upscaleThreshold = 1.6, + double minImprovementRatio = 1.4, +}) { + final thumbnailCropShortSide = _computeCropShortSide( + faceBox, + imageWidth: thumbnailWidth, + imageHeight: thumbnailHeight, + ); + if (thumbnailCropShortSide == null || thumbnailCropShortSide <= 0) { + return const FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'invalid_thumbnail_crop', + thumbnailUpscaleFactor: 0, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final fullCropShortSide = _computeCropShortSide( + faceBox, + imageWidth: fullWidth, + imageHeight: fullHeight, + ); + if (fullCropShortSide == null || fullCropShortSide <= 0) { + return const FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'invalid_full_crop', + thumbnailUpscaleFactor: 0, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final thumbnailUpscaleFactor = + kFaceThumbnailTargetShortSide / thumbnailCropShortSide; + if (thumbnailUpscaleFactor <= upscaleThreshold) { + return FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'below_upscale_threshold', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: 0, + improvementRatio: 0, + ); + } + + final fullUpscaleFactor = kFaceThumbnailTargetShortSide / fullCropShortSide; + final improvementRatio = thumbnailUpscaleFactor / fullUpscaleFactor; + if (improvementRatio < minImprovementRatio) { + return FaceThumbnailUpgradeDecision( + shouldUpgrade: false, + reason: 'insufficient_improvement', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: fullUpscaleFactor, + improvementRatio: improvementRatio, + ); + } + + return FaceThumbnailUpgradeDecision( + shouldUpgrade: true, + reason: 'upgrade_needed', + thumbnailUpscaleFactor: thumbnailUpscaleFactor, + fullUpscaleFactor: fullUpscaleFactor, + improvementRatio: improvementRatio, + ); +} + +NormalizedFaceCrop? computeNormalizedFaceCrop(FaceBox faceBox) { + final widthNorm = faceBox.width; + final heightNorm = faceBox.height; + if (widthNorm <= 0 || heightNorm <= 0) { + return null; + } + + final xCrop = faceBox.x - widthNorm * kFaceThumbnailRegularPadding; + final xOvershoot = (xCrop < 0 ? -xCrop : 0) / widthNorm; + final widthCrop = widthNorm * (1 + 2 * kFaceThumbnailRegularPadding) - + 2 * + _min( + xOvershoot, + kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, + ) * + widthNorm; + + final yCrop = faceBox.y - heightNorm * kFaceThumbnailRegularPadding; + final yOvershoot = (yCrop < 0 ? -yCrop : 0) / heightNorm; + final heightCrop = heightNorm * (1 + 2 * kFaceThumbnailRegularPadding) - + 2 * + _min( + yOvershoot, + kFaceThumbnailRegularPadding - kFaceThumbnailMinimumPadding, + ) * + heightNorm; + + final xCropSafe = xCrop.clamp(0, 1).toDouble(); + final yCropSafe = yCrop.clamp(0, 1).toDouble(); + final widthCropSafe = widthCrop.clamp(0, 1 - xCropSafe).toDouble(); + final heightCropSafe = heightCrop.clamp(0, 1 - yCropSafe).toDouble(); + if (widthCropSafe <= 0 || heightCropSafe <= 0) { + return null; + } + + return ( + x: xCropSafe, + y: yCropSafe, + width: widthCropSafe, + height: heightCropSafe, + ); +} + +double? _computeCropShortSide( + FaceBox faceBox, { + required int imageWidth, + required int imageHeight, +}) { + if (imageWidth <= 0 || imageHeight <= 0) { + return null; + } + final crop = computeNormalizedFaceCrop(faceBox); + if (crop == null) { + return null; + } + final shortSide = _min( + crop.width * imageWidth, + crop.height * imageHeight, + ); + return shortSide > 0 ? shortSide : null; +} + +double _min(double a, double b) => a <= b ? a : b; + +int _max(int a, int b) => a >= b ? a : b; diff --git a/mobile/apps/photos/lib/utils/image_ml_util.dart b/mobile/apps/photos/lib/utils/image_ml_util.dart index 519593b41a6..7bf4299ba7f 100644 --- a/mobile/apps/photos/lib/utils/image_ml_util.dart +++ b/mobile/apps/photos/lib/utils/image_ml_util.dart @@ -183,6 +183,24 @@ Future _getByteDataFromImage( Future> generateFaceThumbnailsUsingCanvas( String imagePath, List faceBoxes, +) async { + final result = await generateFaceThumbnailsUsingCanvasWithSourceDimensions( + imagePath, + faceBoxes, + ); + return result.thumbnails; +} + +typedef FaceThumbnailCanvasGenerationResult = ({ + List thumbnails, + int sourceWidth, + int sourceHeight +}); + +Future + generateFaceThumbnailsUsingCanvasWithSourceDimensions( + String imagePath, + List faceBoxes, ) async { int i = 0; // Index of the faceBox, initialized here for logging purposes try { @@ -195,7 +213,11 @@ Future> generateFaceThumbnailsUsingCanvas( final Image? img = decodedImage.image; if (img == null) { _logger.severe('Image is null, cannot generate face thumbnails'); - return []; + return ( + thumbnails: [], + sourceWidth: dimensions.width, + sourceHeight: dimensions.height, + ); } final futureFaceThumbnails = >[]; for (final faceBox in faceBoxes) { @@ -236,14 +258,24 @@ Future> generateFaceThumbnailsUsingCanvas( } final List faceThumbnails = await Future.wait(futureFaceThumbnails); - return faceThumbnails; + return ( + thumbnails: faceThumbnails, + sourceWidth: dimensions.width, + sourceHeight: dimensions.height, + ); } catch (e, s) { + final problematicFaceBox = + faceBoxes.isNotEmpty && i < faceBoxes.length ? faceBoxes[i] : null; _logger.severe( - 'Error generating face thumbnails. cropImage problematic input argument: ${i}th facebox: ${faceBoxes[i].toString()}', + 'Error generating face thumbnails. cropImage problematic input argument: ${i}th facebox: $problematicFaceBox', e, s, ); - return []; + return ( + thumbnails: [], + sourceWidth: 0, + sourceHeight: 0, + ); } } diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart index c2c7a63b297..901d74e4ed2 100644 --- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart +++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart @@ -42,6 +42,9 @@ enum IsolateOperation { /// [MLComputer] generateFaceThumbnails, + /// [MLComputer] + generateFaceThumbnailsWithSourceDimensions, + /// [MLComputer] loadModel, @@ -186,6 +189,50 @@ Future isolateFunction( ); return List.from(results); + /// MLComputer + case IsolateOperation.generateFaceThumbnailsWithSourceDimensions: + final imagePath = args['imagePath'] as String; + final useRustForFaceThumbnails = + args['useRustForFaceThumbnails'] as bool? ?? false; + final faceBoxesJson = args['faceBoxesList'] as List>; + final List faceBoxes = + faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); + + if (useRustForFaceThumbnails) { + await _ensureRustLoaded(); + final rustFaceBoxes = faceBoxes + .map( + (box) => rust_image_processing.RustFaceBox( + x: box.x, + y: box.y, + width: box.width, + height: box.height, + ), + ) + .toList(growable: false); + final batch = + await rust_image_processing.generateFaceThumbnailsWithMetadata( + imagePath: imagePath, + faceBoxes: rustFaceBoxes, + ); + return { + 'thumbnails': List.from(batch.thumbnails), + 'sourceWidth': batch.sourceWidth, + 'sourceHeight': batch.sourceHeight, + }; + } + + final results = + await generateFaceThumbnailsUsingCanvasWithSourceDimensions( + imagePath, + faceBoxes, + ); + return { + 'thumbnails': List.from(results.thumbnails), + 'sourceWidth': results.sourceWidth, + 'sourceHeight': results.sourceHeight, + }; + /// MLComputer case IsolateOperation.loadModel: final modelName = args['modelName'] as String; diff --git a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart index 2309a58c2e0..ca8bf58288f 100644 --- a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart +++ b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart @@ -106,6 +106,8 @@ class FlagService { bool get useRustForFaceThumbnails => internalUser; + bool get progressivePersonFaceThumbnailsEnabled => internalUser; + Future tryRefreshFlags() async { try { await _fetch(); diff --git a/mobile/apps/photos/rust/src/api/image_processing_api.rs b/mobile/apps/photos/rust/src/api/image_processing_api.rs index 4a7e1157955..044ebaadea1 100644 --- a/mobile/apps/photos/rust/src/api/image_processing_api.rs +++ b/mobile/apps/photos/rust/src/api/image_processing_api.rs @@ -11,21 +11,45 @@ pub struct RustFaceBox { pub height: f64, } +#[derive(Clone, Debug)] +pub struct RustFaceThumbnailBatch { + pub thumbnails: Vec>, + pub source_width: u32, + pub source_height: u32, +} + pub fn generate_face_thumbnails( image_path: String, face_boxes: Vec, ) -> Result>, String> { + generate_face_thumbnails_with_metadata(image_path, face_boxes).map(|batch| batch.thumbnails) +} + +pub fn generate_face_thumbnails_with_metadata( + image_path: String, + face_boxes: Vec, +) -> Result { let decoded = decode_image_from_path(&image_path).map_err(|e| e.to_string())?; - let face_boxes = face_boxes + let face_boxes = parse_face_boxes(face_boxes)?; + let thumbnails = + generate_face_thumbnails_impl(&decoded, &face_boxes).map_err(|e| e.to_string())?; + + Ok(RustFaceThumbnailBatch { + thumbnails, + source_width: decoded.dimensions.width, + source_height: decoded.dimensions.height, + }) +} + +fn parse_face_boxes(face_boxes: Vec) -> Result, String> { + face_boxes .into_iter() .enumerate() .map(|(index, face_box)| { FaceBox::try_from(face_box) .map_err(|e| format!("invalid face box at index {index}: {e}")) }) - .collect::, _>>()?; - - generate_face_thumbnails_impl(&decoded, &face_boxes).map_err(|e| e.to_string()) + .collect::, _>>() } impl TryFrom for FaceBox { diff --git a/mobile/packages/ente_pure_utils/lib/src/task_queue.dart b/mobile/packages/ente_pure_utils/lib/src/task_queue.dart index d82f58a9eba..9c344da71ad 100644 --- a/mobile/packages/ente_pure_utils/lib/src/task_queue.dart +++ b/mobile/packages/ente_pure_utils/lib/src/task_queue.dart @@ -16,9 +16,11 @@ class _QueueItem { counter = 1, completer = Completer(); - void updateTimestamp() { + void updateTimestamp({bool incrementCounter = true}) { lastUpdated = DateTime.now().millisecondsSinceEpoch; - counter++; + if (incrementCounter) { + counter++; + } } bool isTimedOut(Duration timeout) { @@ -117,6 +119,21 @@ class TaskQueue { } } + /// Refresh a pending task so it stays near the front of the queue. + /// + /// Returns true only when the task is still pending. Running or missing + /// tasks cannot be reprioritized here. + bool touchTask(T id) { + final item = _taskMap[id]; + if (item == null) { + return false; + } + _priorityQueue.remove(item); + item.updateTimestamp(incrementCounter: false); + _priorityQueue.add(item); + return true; + } + /// Enforce the maximum queue size by discarding older tasks void _enforceQueueSizeLimit() { // If we're under the limit, no action needed