Feat/preview missing status

specs/configuration.md

@@ -59,6 +59,13 @@ All configuration is in `zou/app/config.py`, read from environment variables.
 | `SENTRY_DSN` | | Sentry error tracking |
 | `PLUGIN_FOLDER` | | Path to plugins directory |
 
+## Preview files
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `PREVIEW_SAVE_SOURCE_FILE` | false | Keep the uploaded source movie alongside the normalized preview |
+| `EXPOSE_MISSING` | false | If true, the API exposes the `missing` preview file status verbatim. If false, `missing` rows are returned as `broken` for backward compatibility. |
+
 ## LDAP / SAML
 
 | Variable | Default | Description |

tests/misc/test_commands.py

@@ -163,3 +163,42 @@ def test_skips_when_local_copy_missing(self):
 
         self.assertIn("Local copy of source movie is missing or", output)
         mock_prepare.assert_not_called()
+
+    def test_source_missing_marks_preview_as_missing(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        with patch.object(
+            file_store, "exists_movie", return_value=False
+        ), patch.object(preview_files_service, "prepare_and_store_movie"):
+            self._run_renormalize()
+
+        reloaded = PreviewFile.get(self.preview_file_id)
+        self.assertEqual(reloaded.status.code, "missing")
+
+    def test_all_missing_filter_selects_only_missing_rows(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        # Existing self.preview_file has status="broken"; add a missing one.
+        missing_preview = self.generate_fixture_preview_file(
+            name="missing_one", revision=2, status="missing"
+        )
+        missing_id = str(missing_preview.id)
+        seen_ids = []
+
+        def fake_exists(prefix, pid):
+            seen_ids.append(pid)
+            return False
+
+        buf = io.StringIO()
+        with redirect_stdout(buf), patch.object(
+            file_store, "exists_movie", side_effect=fake_exists
+        ), patch.object(preview_files_service, "prepare_and_store_movie"):
+            commands.renormalize_movie_preview_files(all_missing=True)
+
+        self.assertIn(missing_id, seen_ids)
+        self.assertNotIn(self.preview_file_id, seen_ids)
+        # The broken row must still be broken; the missing row stays missing.
+        self.assertEqual(
+            PreviewFile.get(self.preview_file_id).status.code, "broken"
+        )
+        self.assertEqual(PreviewFile.get(missing_id).status.code, "missing")

tests/services/test_preview_files_service.py

@@ -441,6 +441,61 @@ def test_extract_skips_metadata_only_previews(self):
         self.assertIsNone(extract_tile_from_preview_file(preview_file))
 
 
+class MissingStatusTestCase(ApiDBTestCase):
+    def setUp(self):
+        super().setUp()
+        self.generate_base_context()
+        self.generate_fixture_asset()
+        self.generate_fixture_assigner()
+        self.generate_fixture_person()
+        self.generate_fixture_task()
+        self.preview_file = self.generate_fixture_preview_file(
+            status="processing"
+        )
+
+    def tearDown(self):
+        super().tearDown()
+        self.delete_test_folder()
+
+    def _reload_preview(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        return PreviewFile.get(self.preview_file.id)
+
+    def test_set_preview_file_as_missing_persists_missing(self):
+        preview_files_service.set_preview_file_as_missing(
+            str(self.preview_file.id)
+        )
+        reloaded = self._reload_preview()
+        self.assertEqual(reloaded.status.code, "missing")
+
+    def test_api_views_remap_missing_to_broken_by_default(self):
+        from zou.app import config
+
+        preview_files_service.set_preview_file_as_missing(
+            str(self.preview_file.id)
+        )
+        reloaded = self._reload_preview()
+
+        with patch.object(config, "EXPOSE_MISSING", False):
+            self.assertEqual(reloaded.serialize()["status"], "broken")
+            self.assertEqual(reloaded.present()["status"], "broken")
+            self.assertEqual(reloaded.present_minimal()["status"], "broken")
+
+    def test_api_views_expose_missing_when_flag_enabled(self):
+        from zou.app import config
+
+        preview_files_service.set_preview_file_as_missing(
+            str(self.preview_file.id)
+        )
+        reloaded = self._reload_preview()
+
+        with patch.object(config, "EXPOSE_MISSING", True):
+            self.assertEqual(reloaded.serialize()["status"], "missing")
+            self.assertEqual(reloaded.present()["status"], "missing")
+            self.assertEqual(reloaded.present_minimal()["status"], "missing")
+
+
 class ExtractAnnotationFrameTestCase(ApiDBTestCase):
     def setUp(self):
         super().setUp()

zou/app/config.py

@@ -80,6 +80,7 @@
     os.getenv("THUMBNAIL_FOLDER", os.path.join(os.getcwd(), "previews")),
 )
 PREVIEW_SAVE_SOURCE_FILE = envtobool("PREVIEW_SAVE_SOURCE_FILE", False)
+EXPOSE_MISSING = envtobool("EXPOSE_MISSING", False)
 TMP_DIR = os.getenv("TMP_DIR", os.path.join(tempfile.gettempdir(), "zou"))
 
 EVENT_STREAM_HOST = os.getenv("EVENT_STREAM_HOST", "localhost")

zou/app/models/preview_file.py

@@ -12,6 +12,7 @@
     ("processing", "Processing"),
     ("ready", "Ready"),
     ("broken", "Broken"),
+    ("missing", "Missing"),
 ]
 
 VALIDATION_STATUSES = [
@@ -75,6 +76,31 @@ class PreviewFile(db.Model, BaseMixin, SerializerMixin):
     def __repr__(self):
         return f"<PreviewFile {self.id}>"
 
+    def _api_status(self):
+        """
+        Return the status value as exposed to API clients.
+
+        The internal `missing` status is folded back to `broken` when
+        `EXPOSE_MISSING` is disabled (default), so existing clients keep
+        seeing the legacy two-state model.
+        """
+        from zou.app import config
+
+        raw = getattr(self.status, "code", self.status)
+        if raw == "missing" and not getattr(config, "EXPOSE_MISSING", False):
+            return "broken"
+        return raw
+
+    def serialize(self, *args, **kwargs):
+        data = super().serialize(*args, **kwargs)
+        from zou.app import config
+
+        if data.get("status") == "missing" and not getattr(
+            config, "EXPOSE_MISSING", False
+        ):
+            data["status"] = "broken"
+        return data
+
     @classmethod
     def create_from_import(cls, data):
         data.pop("type", None)
@@ -98,7 +124,7 @@ def present(self):
                 "revision": self.revision,
                 "position": self.position,
                 "file_size": self.file_size,
-                "status": self.status,
+                "status": self._api_status(),
                 "validation_status": self.validation_status,
                 "task_id": self.task_id,
                 "person_id": self.person_id,
@@ -116,6 +142,6 @@ def present_minimal(self):
                 "extension": self.extension,
                 "revision": self.revision,
                 "position": self.position,
-                "status": self.status,
+                "status": self._api_status(),
             }
         )

zou/app/services/preview_files_service.py

@@ -212,6 +212,15 @@ def set_preview_file_as_broken(preview_file_id):
     return update_preview_file(preview_file_id, {"status": "broken"})
 
 
+def set_preview_file_as_missing(preview_file_id):
+    """
+    Mark a preview file as missing (its source binary is gone from
+    storage). Exposed to API clients as ``broken`` unless
+    ``EXPOSE_MISSING`` is enabled.
+    """
+    return update_preview_file(preview_file_id, {"status": "missing"})
+
+
 def set_preview_file_as_ready(preview_file_id):
     """
     Mark given preview file as ready.
@@ -718,15 +727,15 @@ def _clear_empty_annotations(annotations):
 
 def get_running_preview_files(cursor_preview_file_id=None, limit=None):
     """
-    Return preview files for all productions with status equals to broken
-    or processing using cursor-based pagination.
+    Return preview files for all productions with status equals to broken,
+    missing or processing using cursor-based pagination.
     """
     query = (
         PreviewFile.query.join(Task)
         .join(Project)
         .join(ProjectStatus, ProjectStatus.id == Project.project_status_id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.in_(("broken", "processing")))
+        .filter(PreviewFile.status.in_(("broken", "missing", "processing")))
         .add_columns(Task.project_id, Task.task_type_id, Task.entity_id)
         .order_by(PreviewFile.created_at.desc())
     )
@@ -1156,7 +1165,7 @@ def reset_movie_files_metadata():
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension == "mp4")
     )
     for preview_file in preview_files:
@@ -1199,7 +1208,7 @@ def reset_picture_files_metadata():
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension == "png")
     )
     for preview_file in preview_files:
@@ -1253,7 +1262,7 @@ def generate_preview_extra(
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension.in_(("mp4", "png")))
     )
     if project is not None:

zou/app/services/sync_service.py

@@ -1113,6 +1113,8 @@ def download_files_from_another_instance(
     number_workers=30,
     number_attemps=3,
     force_resync=False,
+    include_broken=True,
+    include_missing=True,
 ):
     """
     Download all files from source instance.
@@ -1151,6 +1153,8 @@ def download_files_from_another_instance(
         number_attemps=number_attemps,
         force=force_resync,
         dict_errors=dict_errors,
+        include_broken=include_broken,
+        include_missing=include_missing,
     )
     download_preview_background_files_from_another_instance(
         project=project,
@@ -1248,7 +1252,13 @@ def download_thumbnail_from_another_instance(
 
 
 def download_preview_files_from_another_instance(
-    project=None, pool=None, number_attemps=3, force=False, dict_errors={}
+    project=None,
+    pool=None,
+    number_attemps=3,
+    force=False,
+    dict_errors={},
+    include_broken=True,
+    include_missing=True,
 ):
     """
     Download all preview files and related (thumbnails and low def included).
@@ -1261,6 +1271,16 @@ def download_preview_files_from_another_instance(
     else:
         preview_files = PreviewFile.query
 
+    excluded_statuses = []
+    if not include_broken:
+        excluded_statuses.append("broken")
+    if not include_missing:
+        excluded_statuses.append("missing")
+    if excluded_statuses:
+        preview_files = preview_files.filter(
+            PreviewFile.status.notin_(excluded_statuses)
+        )
+
     number_of_preview_files = preview_files.count()
     logger.info(f"Downloading preview files ({number_of_preview_files})...")
     for i, preview_file in enumerate(preview_files):