From 61d4587a2387af788ffe9f6d51bc87c16d558747 Mon Sep 17 00:00:00 2001 From: Nicolas Ledez <247138+nledez@users.noreply.github.com> Date: Sat, 6 Jun 2026 09:31:13 +0200 Subject: [PATCH 1/4] fix(commands): mark preview file as broken when renormalization fails --- zou/app/utils/commands.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index d6341ef0b..1c697720e 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -1078,6 +1078,14 @@ def renormalize_movie_preview_files( print( f"Renormalization of preview file {preview_file_id} failed: {e}" ) + try: + preview_files_service.set_preview_file_as_broken( + preview_file_id + ) + except Exception as mark_err: + print( + f"Could not mark {preview_file_id} as broken: {mark_err}" + ) continue From b82ae91953849901a398445b5c93c276a6480241 Mon Sep 17 00:00:00 2001 From: Nicolas Ledez <247138+nledez@users.noreply.github.com> Date: Sat, 6 Jun 2026 10:25:56 +0200 Subject: [PATCH 2/4] feat(sync): add --no-include-broken flag to sync-full-files to skip broken preview files --- zou/app/services/sync_service.py | 12 +++++++++++- zou/app/utils/commands.py | 2 ++ zou/cli.py | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/zou/app/services/sync_service.py b/zou/app/services/sync_service.py index 7a4fa5d8a..2a25bf4a3 100644 --- a/zou/app/services/sync_service.py +++ b/zou/app/services/sync_service.py @@ -1113,6 +1113,7 @@ def download_files_from_another_instance( number_workers=30, number_attemps=3, force_resync=False, + include_broken=True, ): """ Download all files from source instance. @@ -1151,6 +1152,7 @@ def download_files_from_another_instance( number_attemps=number_attemps, force=force_resync, dict_errors=dict_errors, + include_broken=include_broken, ) download_preview_background_files_from_another_instance( project=project, @@ -1248,7 +1250,12 @@ def download_thumbnail_from_another_instance( def download_preview_files_from_another_instance( - project=None, pool=None, number_attemps=3, force=False, dict_errors={} + project=None, + pool=None, + number_attemps=3, + force=False, + dict_errors={}, + include_broken=True, ): """ Download all preview files and related (thumbnails and low def included). @@ -1261,6 +1268,9 @@ def download_preview_files_from_another_instance( else: preview_files = PreviewFile.query + if not include_broken: + preview_files = preview_files.filter(PreviewFile.status != "broken") + number_of_preview_files = preview_files.count() logger.info(f"Downloading preview files ({number_of_preview_files})...") for i, preview_file in enumerate(preview_files): diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index 1c697720e..6aeb4dfe7 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -813,6 +813,7 @@ def import_files_from_another_instance( number_workers=30, number_attemps=3, force_resync=False, + include_broken=True, ): """ Retrieve and save all the data related most recent events from another API @@ -828,6 +829,7 @@ def import_files_from_another_instance( number_workers=number_workers, number_attemps=number_attemps, force_resync=force_resync, + include_broken=include_broken, ) diff --git a/zou/cli.py b/zou/cli.py index c5d46cf63..85afecbff 100755 --- a/zou/cli.py +++ b/zou/cli.py @@ -710,6 +710,12 @@ def sync_push_verify(target, project): @click.option("--number-workers", default=30, show_default=True, type=int) @click.option("--number-attemps", default=3, show_default=True, type=int) @click.option("--force-resync", is_flag=True, show_default=True, default=False) +@click.option( + "--include-broken/--no-include-broken", + show_default=True, + default=True, + help="Sync preview files whose status is 'broken'. Enabled by default; use --no-include-broken to skip them.", +) def sync_full_files( source, project, @@ -717,6 +723,7 @@ def sync_full_files( number_workers, number_attemps, force_resync, + include_broken, ): """ Retrieve all files from source instance. It expects that credentials to @@ -737,6 +744,7 @@ def sync_full_files( number_workers=number_workers, number_attemps=number_attemps, force_resync=force_resync, + include_broken=include_broken, ) print("Syncing ended.") if dict_errors: From ac69c4e55e708d00d821ca8cb638cf9668c3919e Mon Sep 17 00:00:00 2001 From: Nicolas Ledez <247138+nledez@users.noreply.github.com> Date: Sat, 6 Jun 2026 11:14:17 +0200 Subject: [PATCH 3/4] feat(previews): add internal missing status with EXPOSE_MISSING flag and dedicated cli filters --- specs/configuration.md | 7 +++ tests/misc/test_commands.py | 39 ++++++++++++++ tests/services/test_preview_files_service.py | 55 ++++++++++++++++++++ zou/app/config.py | 1 + zou/app/models/preview_file.py | 30 ++++++++++- zou/app/services/preview_files_service.py | 21 +++++--- zou/app/services/sync_service.py | 12 ++++- zou/app/utils/commands.py | 52 ++++++++++++++---- zou/cli.py | 23 ++++++-- 9 files changed, 215 insertions(+), 25 deletions(-) diff --git a/specs/configuration.md b/specs/configuration.md index ebf5a2e35..7fd0ab614 100644 --- a/specs/configuration.md +++ b/specs/configuration.md @@ -59,6 +59,13 @@ All configuration is in `zou/app/config.py`, read from environment variables. | `SENTRY_DSN` | | Sentry error tracking | | `PLUGIN_FOLDER` | | Path to plugins directory | +## Preview files + +| Variable | Default | Description | +|----------|---------|-------------| +| `PREVIEW_SAVE_SOURCE_FILE` | false | Keep the uploaded source movie alongside the normalized preview | +| `EXPOSE_MISSING` | false | If true, the API exposes the `missing` preview file status verbatim. If false, `missing` rows are returned as `broken` for backward compatibility. | + ## LDAP / SAML | Variable | Default | Description | diff --git a/tests/misc/test_commands.py b/tests/misc/test_commands.py index 7a43b8d62..b7525ac67 100644 --- a/tests/misc/test_commands.py +++ b/tests/misc/test_commands.py @@ -163,3 +163,42 @@ def test_skips_when_local_copy_missing(self): self.assertIn("Local copy of source movie is missing or", output) mock_prepare.assert_not_called() + + def test_source_missing_marks_preview_as_missing(self): + from zou.app.models.preview_file import PreviewFile + + with patch.object( + file_store, "exists_movie", return_value=False + ), patch.object(preview_files_service, "prepare_and_store_movie"): + self._run_renormalize() + + reloaded = PreviewFile.get(self.preview_file_id) + self.assertEqual(str(reloaded.status), "missing") + + def test_all_missing_filter_selects_only_missing_rows(self): + from zou.app.models.preview_file import PreviewFile + + # Existing self.preview_file has status="broken"; add a missing one. + missing_preview = self.generate_fixture_preview_file( + name="missing_one", revision=2, status="missing" + ) + missing_id = str(missing_preview.id) + seen_ids = [] + + def fake_exists(prefix, pid): + seen_ids.append(pid) + return False + + buf = io.StringIO() + with redirect_stdout(buf), patch.object( + file_store, "exists_movie", side_effect=fake_exists + ), patch.object(preview_files_service, "prepare_and_store_movie"): + commands.renormalize_movie_preview_files(all_missing=True) + + self.assertIn(missing_id, seen_ids) + self.assertNotIn(self.preview_file_id, seen_ids) + # The broken row must still be broken; the missing row stays missing. + self.assertEqual( + str(PreviewFile.get(self.preview_file_id).status), "broken" + ) + self.assertEqual(str(PreviewFile.get(missing_id).status), "missing") diff --git a/tests/services/test_preview_files_service.py b/tests/services/test_preview_files_service.py index f158d37ed..aa20ea876 100644 --- a/tests/services/test_preview_files_service.py +++ b/tests/services/test_preview_files_service.py @@ -441,6 +441,61 @@ def test_extract_skips_metadata_only_previews(self): self.assertIsNone(extract_tile_from_preview_file(preview_file)) +class MissingStatusTestCase(ApiDBTestCase): + def setUp(self): + super().setUp() + self.generate_base_context() + self.generate_fixture_asset() + self.generate_fixture_assigner() + self.generate_fixture_person() + self.generate_fixture_task() + self.preview_file = self.generate_fixture_preview_file( + status="processing" + ) + + def tearDown(self): + super().tearDown() + self.delete_test_folder() + + def _reload_preview(self): + from zou.app.models.preview_file import PreviewFile + + return PreviewFile.get(self.preview_file.id) + + def test_set_preview_file_as_missing_persists_missing(self): + preview_files_service.set_preview_file_as_missing( + str(self.preview_file.id) + ) + reloaded = self._reload_preview() + self.assertEqual(str(reloaded.status), "missing") + + def test_api_views_remap_missing_to_broken_by_default(self): + from zou.app import config + + preview_files_service.set_preview_file_as_missing( + str(self.preview_file.id) + ) + reloaded = self._reload_preview() + + with patch.object(config, "EXPOSE_MISSING", False): + self.assertEqual(reloaded.serialize()["status"], "broken") + self.assertEqual(reloaded.present()["status"], "broken") + self.assertEqual(reloaded.present_minimal()["status"], "broken") + + def test_api_views_expose_missing_when_flag_enabled(self): + from zou.app import config + + preview_files_service.set_preview_file_as_missing( + str(self.preview_file.id) + ) + reloaded = self._reload_preview() + + with patch.object(config, "EXPOSE_MISSING", True): + self.assertEqual(reloaded.serialize()["status"], "missing") + self.assertEqual(reloaded.present()["status"], "missing") + self.assertEqual(reloaded.present_minimal()["status"], "missing") + + class ExtractAnnotationFrameTestCase(ApiDBTestCase): def setUp(self): super().setUp() diff --git a/zou/app/config.py b/zou/app/config.py index 9cde63ae9..cf387b26f 100644 --- a/zou/app/config.py +++ b/zou/app/config.py @@ -80,6 +80,7 @@ os.getenv("THUMBNAIL_FOLDER", os.path.join(os.getcwd(), "previews")), ) PREVIEW_SAVE_SOURCE_FILE = envtobool("PREVIEW_SAVE_SOURCE_FILE", False) +EXPOSE_MISSING = envtobool("EXPOSE_MISSING", False) TMP_DIR = os.getenv("TMP_DIR", os.path.join(tempfile.gettempdir(), "zou")) EVENT_STREAM_HOST = os.getenv("EVENT_STREAM_HOST", "localhost") diff --git a/zou/app/models/preview_file.py b/zou/app/models/preview_file.py index 6b2caa768..e473c8516 100644 --- a/zou/app/models/preview_file.py +++ b/zou/app/models/preview_file.py @@ -12,6 +12,7 @@ ("processing", "Processing"), ("ready", "Ready"), ("broken", "Broken"), + ("missing", "Missing"), ] VALIDATION_STATUSES = [ @@ -75,6 +76,31 @@ class PreviewFile(db.Model, BaseMixin, SerializerMixin): def __repr__(self): return f"" + def _api_status(self): + """ + Return the status value as exposed to API clients. + + The internal `missing` status is folded back to `broken` when + `EXPOSE_MISSING` is disabled (default), so existing clients keep + seeing the legacy two-state model. + """ + from zou.app import config + + raw = str(self.status) + if raw == "missing" and not getattr(config, "EXPOSE_MISSING", False): + return "broken" + return raw + + def serialize(self, *args, **kwargs): + data = super().serialize(*args, **kwargs) + from zou.app import config + + if data.get("status") == "missing" and not getattr( + config, "EXPOSE_MISSING", False + ): + data["status"] = "broken" + return data + @classmethod def create_from_import(cls, data): data.pop("type", None) @@ -98,7 +124,7 @@ def present(self): "revision": self.revision, "position": self.position, "file_size": self.file_size, - "status": self.status, + "status": self._api_status(), "validation_status": self.validation_status, "task_id": self.task_id, "person_id": self.person_id, @@ -116,6 +142,6 @@ def present_minimal(self): "extension": self.extension, "revision": self.revision, "position": self.position, - "status": self.status, + "status": self._api_status(), } ) diff --git a/zou/app/services/preview_files_service.py b/zou/app/services/preview_files_service.py index 7b67fc73e..56121224c 100644 --- a/zou/app/services/preview_files_service.py +++ b/zou/app/services/preview_files_service.py @@ -212,6 +212,15 @@ def set_preview_file_as_broken(preview_file_id): return update_preview_file(preview_file_id, {"status": "broken"}) +def set_preview_file_as_missing(preview_file_id): + """ + Mark a preview file as missing (its source binary is gone from + storage). Exposed to API clients as ``broken`` unless + ``EXPOSE_MISSING`` is enabled. + """ + return update_preview_file(preview_file_id, {"status": "missing"}) + + def set_preview_file_as_ready(preview_file_id): """ Mark given preview file as ready. @@ -718,15 +727,15 @@ def _clear_empty_annotations(annotations): def get_running_preview_files(cursor_preview_file_id=None, limit=None): """ - Return preview files for all productions with status equals to broken - or processing using cursor-based pagination. + Return preview files for all productions with status equals to broken, + missing or processing using cursor-based pagination. """ query = ( PreviewFile.query.join(Task) .join(Project) .join(ProjectStatus, ProjectStatus.id == Project.project_status_id) .filter(ProjectStatus.name.in_(("Active", "open", "Open"))) - .filter(PreviewFile.status.in_(("broken", "processing"))) + .filter(PreviewFile.status.in_(("broken", "missing", "processing"))) .add_columns(Task.project_id, Task.task_type_id, Task.entity_id) .order_by(PreviewFile.created_at.desc()) ) @@ -1156,7 +1165,7 @@ def reset_movie_files_metadata(): .join(Project) .join(ProjectStatus, Project.project_status_id == ProjectStatus.id) .filter(ProjectStatus.name.in_(("Active", "open", "Open"))) - .filter(PreviewFile.status.not_in(("broken", "processing"))) + .filter(PreviewFile.status.not_in(("broken", "missing", "processing"))) .filter(PreviewFile.extension == "mp4") ) for preview_file in preview_files: @@ -1199,7 +1208,7 @@ def reset_picture_files_metadata(): .join(Project) .join(ProjectStatus, Project.project_status_id == ProjectStatus.id) .filter(ProjectStatus.name.in_(("Active", "open", "Open"))) - .filter(PreviewFile.status.not_in(("broken", "processing"))) + .filter(PreviewFile.status.not_in(("broken", "missing", "processing"))) .filter(PreviewFile.extension == "png") ) for preview_file in preview_files: @@ -1253,7 +1262,7 @@ def generate_preview_extra( .join(Project) .join(ProjectStatus, Project.project_status_id == ProjectStatus.id) .filter(ProjectStatus.name.in_(("Active", "open", "Open"))) - .filter(PreviewFile.status.not_in(("broken", "processing"))) + .filter(PreviewFile.status.not_in(("broken", "missing", "processing"))) .filter(PreviewFile.extension.in_(("mp4", "png"))) ) if project is not None: diff --git a/zou/app/services/sync_service.py b/zou/app/services/sync_service.py index 2a25bf4a3..93c58866f 100644 --- a/zou/app/services/sync_service.py +++ b/zou/app/services/sync_service.py @@ -1114,6 +1114,7 @@ def download_files_from_another_instance( number_attemps=3, force_resync=False, include_broken=True, + include_missing=True, ): """ Download all files from source instance. @@ -1153,6 +1154,7 @@ def download_files_from_another_instance( force=force_resync, dict_errors=dict_errors, include_broken=include_broken, + include_missing=include_missing, ) download_preview_background_files_from_another_instance( project=project, @@ -1256,6 +1258,7 @@ def download_preview_files_from_another_instance( force=False, dict_errors={}, include_broken=True, + include_missing=True, ): """ Download all preview files and related (thumbnails and low def included). @@ -1268,8 +1271,15 @@ def download_preview_files_from_another_instance( else: preview_files = PreviewFile.query + excluded_statuses = [] if not include_broken: - preview_files = preview_files.filter(PreviewFile.status != "broken") + excluded_statuses.append("broken") + if not include_missing: + excluded_statuses.append("missing") + if excluded_statuses: + preview_files = preview_files.filter( + PreviewFile.status.notin_(excluded_statuses) + ) number_of_preview_files = preview_files.count() logger.info(f"Downloading preview files ({number_of_preview_files})...") diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index 6aeb4dfe7..25d884336 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -814,6 +814,7 @@ def import_files_from_another_instance( number_attemps=3, force_resync=False, include_broken=True, + include_missing=True, ): """ Retrieve and save all the data related most recent events from another API @@ -830,6 +831,7 @@ def import_files_from_another_instance( number_attemps=number_attemps, force_resync=force_resync, include_broken=include_broken, + include_missing=include_missing, ) @@ -968,19 +970,33 @@ def create_bot( print(bot["access_token"]) +class _SourceMovieMissing(RuntimeError): + """ + Raised when the source movie binary is gone from storage during a + renormalize pass. Used to distinguish 'broken' from 'missing' status. + """ + + def renormalize_movie_preview_files( preview_file_id=None, project_id=None, all_broken=None, all_processing=None, + all_missing=None, days=None, hours=None, minutes=None, ): with app.app_context(): - if preview_file_id is None and not all_broken and not all_processing: + if ( + preview_file_id is None + and not all_broken + and not all_processing + and not all_missing + ): print( - "You must specify at least one flag from --all-broken or --all-processing." + "You must specify at least one flag from --all-broken, " + "--all-missing or --all-processing." ) sys.exit(1) @@ -1004,14 +1020,15 @@ def renormalize_movie_preview_files( PreviewFile.project_id == project_id ) - if all_broken and all_processing: - query = query.filter( - PreviewFile.status.in_(("broken", "processing")) - ) - elif all_broken: - query = query.filter(PreviewFile.status == "broken") - elif all_processing: - query = query.filter(PreviewFile.status == "processing") + selected_statuses = [] + if all_broken: + selected_statuses.append("broken") + if all_missing: + selected_statuses.append("missing") + if all_processing: + selected_statuses.append("processing") + if selected_statuses: + query = query.filter(PreviewFile.status.in_(selected_statuses)) preview_files = query.all() len_preview_files = len(preview_files) @@ -1031,7 +1048,7 @@ def renormalize_movie_preview_files( f"{preview_file_id}.{extension}.tmp", ) if not file_store.exists_movie("source", preview_file_id): - raise RuntimeError( + raise _SourceMovieMissing( f"Source movie missing in storage for preview " f"{preview_file_id}; skipping renormalization." ) @@ -1076,6 +1093,19 @@ def renormalize_movie_preview_files( normalize=True, add_source_to_file_store=False, ) + except _SourceMovieMissing as e: + print( + f"Renormalization of preview file {preview_file_id} failed: {e}" + ) + try: + preview_files_service.set_preview_file_as_missing( + preview_file_id + ) + except Exception as mark_err: + print( + f"Could not mark {preview_file_id} as missing: {mark_err}" + ) + continue except Exception as e: print( f"Renormalization of preview file {preview_file_id} failed: {e}" diff --git a/zou/cli.py b/zou/cli.py index 85afecbff..16a5557e3 100755 --- a/zou/cli.py +++ b/zou/cli.py @@ -711,10 +711,18 @@ def sync_push_verify(target, project): @click.option("--number-attemps", default=3, show_default=True, type=int) @click.option("--force-resync", is_flag=True, show_default=True, default=False) @click.option( - "--include-broken/--no-include-broken", + "--skip-broken", + is_flag=True, show_default=True, - default=True, - help="Sync preview files whose status is 'broken'. Enabled by default; use --no-include-broken to skip them.", + default=False, + help="Skip preview files whose status is 'broken' (synced by default).", +) +@click.option( + "--skip-missing", + is_flag=True, + show_default=True, + default=False, + help="Skip preview files whose status is 'missing' (synced by default).", ) def sync_full_files( source, @@ -723,7 +731,8 @@ def sync_full_files( number_workers, number_attemps, force_resync, - include_broken, + skip_broken, + skip_missing, ): """ Retrieve all files from source instance. It expects that credentials to @@ -744,7 +753,8 @@ def sync_full_files( number_workers=number_workers, number_attemps=number_attemps, force_resync=force_resync, - include_broken=include_broken, + include_broken=not skip_broken, + include_missing=not skip_missing, ) print("Syncing ended.") if dict_errors: @@ -1029,6 +1039,7 @@ def create_bot( show_default=True, ) @click.option("--all-broken", is_flag=True, default=False, show_default=True) +@click.option("--all-missing", is_flag=True, default=False, show_default=True) @click.option( "--all-processing", is_flag=True, default=False, show_default=True ) @@ -1039,6 +1050,7 @@ def renormalize_movie_preview_files( preview_file_id, project_id, all_broken, + all_missing, all_processing, days=None, hours=None, @@ -1054,6 +1066,7 @@ def renormalize_movie_preview_files( project_id, all_broken, all_processing, + all_missing=all_missing, days=days, hours=hours, minutes=minutes, From f51007dba6820c3235391bb96c56c94e063b25b8 Mon Sep 17 00:00:00 2001 From: Nicolas Ledez <247138+nledez@users.noreply.github.com> Date: Sat, 6 Jun 2026 12:05:29 +0200 Subject: [PATCH 4/4] fix(previews): use Choice.code in _api_status to preserve lowercase status casing --- tests/misc/test_commands.py | 6 +++--- tests/services/test_preview_files_service.py | 2 +- zou/app/models/preview_file.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/misc/test_commands.py b/tests/misc/test_commands.py index b7525ac67..a26d37698 100644 --- a/tests/misc/test_commands.py +++ b/tests/misc/test_commands.py @@ -173,7 +173,7 @@ def test_source_missing_marks_preview_as_missing(self): self._run_renormalize() reloaded = PreviewFile.get(self.preview_file_id) - self.assertEqual(str(reloaded.status), "missing") + self.assertEqual(reloaded.status.code, "missing") def test_all_missing_filter_selects_only_missing_rows(self): from zou.app.models.preview_file import PreviewFile @@ -199,6 +199,6 @@ def fake_exists(prefix, pid): self.assertNotIn(self.preview_file_id, seen_ids) # The broken row must still be broken; the missing row stays missing. self.assertEqual( - str(PreviewFile.get(self.preview_file_id).status), "broken" + PreviewFile.get(self.preview_file_id).status.code, "broken" ) - self.assertEqual(str(PreviewFile.get(missing_id).status), "missing") + self.assertEqual(PreviewFile.get(missing_id).status.code, "missing") diff --git a/tests/services/test_preview_files_service.py b/tests/services/test_preview_files_service.py index aa20ea876..872d92260 100644 --- a/tests/services/test_preview_files_service.py +++ b/tests/services/test_preview_files_service.py @@ -467,7 +467,7 @@ def test_set_preview_file_as_missing_persists_missing(self): str(self.preview_file.id) ) reloaded = self._reload_preview() - self.assertEqual(str(reloaded.status), "missing") + self.assertEqual(reloaded.status.code, "missing") def test_api_views_remap_missing_to_broken_by_default(self): from zou.app import config diff --git a/zou/app/models/preview_file.py b/zou/app/models/preview_file.py index e473c8516..7f833d37a 100644 --- a/zou/app/models/preview_file.py +++ b/zou/app/models/preview_file.py @@ -86,7 +86,7 @@ def _api_status(self): """ from zou.app import config - raw = str(self.status) + raw = getattr(self.status, "code", self.status) if raw == "missing" and not getattr(config, "EXPOSE_MISSING", False): return "broken" return raw