Feat/preview missing status

specs/configuration.md

@@ -59,6 +59,12 @@ All configuration is in `zou/app/config.py`, read from environment variables.
 | `SENTRY_DSN` | | Sentry error tracking |
 | `PLUGIN_FOLDER` | | Path to plugins directory |
 
+## Preview files
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `PREVIEW_SAVE_SOURCE_FILE` | false | Keep the uploaded source movie alongside the normalized preview |
+
 ## LDAP / SAML
 
 | Variable | Default | Description |

tests/misc/test_commands.py

@@ -163,3 +163,42 @@ def test_skips_when_local_copy_missing(self):
 
         self.assertIn("Local copy of source movie is missing or", output)
         mock_prepare.assert_not_called()
+
+    def test_source_missing_marks_preview_as_missing(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        with patch.object(
+            file_store, "exists_movie", return_value=False
+        ), patch.object(preview_files_service, "prepare_and_store_movie"):
+            self._run_renormalize()
+
+        reloaded = PreviewFile.get(self.preview_file_id)
+        self.assertEqual(reloaded.status.code, "missing")
+
+    def test_all_missing_filter_selects_only_missing_rows(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        # Existing self.preview_file has status="broken"; add a missing one.
+        missing_preview = self.generate_fixture_preview_file(
+            name="missing_one", revision=2, status="missing"
+        )
+        missing_id = str(missing_preview.id)
+        seen_ids = []
+
+        def fake_exists(prefix, pid):
+            seen_ids.append(pid)
+            return False
+
+        buf = io.StringIO()
+        with redirect_stdout(buf), patch.object(
+            file_store, "exists_movie", side_effect=fake_exists
+        ), patch.object(preview_files_service, "prepare_and_store_movie"):
+            commands.renormalize_movie_preview_files(all_missing=True)
+
+        self.assertIn(missing_id, seen_ids)
+        self.assertNotIn(self.preview_file_id, seen_ids)
+        # The broken row must still be broken; the missing row stays missing.
+        self.assertEqual(
+            PreviewFile.get(self.preview_file_id).status.code, "broken"
+        )
+        self.assertEqual(PreviewFile.get(missing_id).status.code, "missing")

tests/services/test_preview_files_service.py

@@ -441,6 +441,70 @@ def test_extract_skips_metadata_only_previews(self):
         self.assertIsNone(extract_tile_from_preview_file(preview_file))
 
 
+class MissingStatusTestCase(ApiDBTestCase):
+    def setUp(self):
+        super().setUp()
+        self.generate_base_context()
+        self.generate_fixture_asset()
+        self.generate_fixture_assigner()
+        self.generate_fixture_person()
+        self.generate_fixture_task()
+        self.preview_file = self.generate_fixture_preview_file(
+            status="processing"
+        )
+
+    def tearDown(self):
+        super().tearDown()
+        self.delete_test_folder()
+
+    def _reload_preview(self):
+        from zou.app.models.preview_file import PreviewFile
+
+        return PreviewFile.get(self.preview_file.id)
+
+    def test_set_preview_file_as_missing_persists_missing(self):
+        preview_files_service.set_preview_file_as_missing(
+            str(self.preview_file.id)
+        )
+        reloaded = self._reload_preview()
+        self.assertEqual(reloaded.status.code, "missing")
+        self.assertEqual(reloaded.serialize()["status"], "missing")
+        self.assertEqual(reloaded.present()["status"], "missing")
+        self.assertEqual(reloaded.present_minimal()["status"], "missing")
+
+    def test_mark_helper_flips_ready_to_missing(self):
+        preview_files_service.set_preview_file_as_ready(
+            str(self.preview_file.id)
+        )
+        preview_files_service.mark_preview_file_as_missing_on_storage_404(
+            str(self.preview_file.id)
+        )
+        self.assertEqual(self._reload_preview().status.code, "missing")
+
+    def test_mark_helper_skips_processing_rows(self):
+        # Preview is still in 'processing' (mid-upload race).
+        preview_files_service.mark_preview_file_as_missing_on_storage_404(
+            str(self.preview_file.id)
+        )
+        self.assertEqual(self._reload_preview().status.code, "processing")
+
+    def test_mark_helper_is_idempotent_on_missing_rows(self):
+        preview_files_service.set_preview_file_as_missing(
+            str(self.preview_file.id)
+        )
+        preview_files_service.mark_preview_file_as_missing_on_storage_404(
+            str(self.preview_file.id)
+        )
+        self.assertEqual(self._reload_preview().status.code, "missing")
+
+    def test_mark_helper_is_safe_on_unknown_id(self):
+        from zou.app.utils import fields
+
+        preview_files_service.mark_preview_file_as_missing_on_storage_404(
+            fields.gen_uuid()
+        )
+
+
 class ExtractAnnotationFrameTestCase(ApiDBTestCase):
     def setUp(self):
         super().setUp()

zou/app/blueprints/previews/resources.py

@@ -748,6 +748,9 @@ def get(self, instance_id):
                 current_app.logger.error(
                     f"Movie file was not found for: {instance_id}"
                 )
+            preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                instance_id
+            )
             raise PreviewFileNotFoundException
 
 
@@ -799,6 +802,9 @@ def get(self, instance_id):
                     current_app.logger.error(
                         f"Movie file was not found for: {instance_id}"
                     )
+                preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                    instance_id
+                )
                 raise PreviewFileNotFoundException
 
 
@@ -846,6 +852,9 @@ def get(self, instance_id):
                 current_app.logger.error(
                     f"Movie file was not found for: {instance_id}"
                 )
+            preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                instance_id
+            )
             raise PreviewFileNotFoundException
 
 
@@ -920,6 +929,9 @@ def get(self, instance_id, extension):
                 current_app.logger.error(
                     f"Non-movie file was not found for: {instance_id}"
                 )
+            preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                instance_id
+            )
             raise PreviewFileNotFoundException
 
 
@@ -993,6 +1005,9 @@ def get(self, instance_id):
                 current_app.logger.error(
                     f"Standard file was not found for: {instance_id}"
                 )
+            preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                instance_id
+            )
             raise PreviewFileNotFoundException
 
 
@@ -1115,6 +1130,9 @@ def get(self, instance_id):
                 current_app.logger.error(
                     f"Picture file was not found for: {instance_id}"
                 )
+            preview_files_service.mark_preview_file_as_missing_on_storage_404(
+                instance_id
+            )
             raise PreviewFileNotFoundException
 
 

zou/app/models/preview_file.py

@@ -12,6 +12,7 @@
     ("processing", "Processing"),
     ("ready", "Ready"),
     ("broken", "Broken"),
+    ("missing", "Missing"),
 ]
 
 VALIDATION_STATUSES = [

zou/app/services/preview_files_service.py

@@ -7,6 +7,7 @@
 import zipfile
 
 import ffmpeg
+from flask_fs.errors import FileNotFound
 from PIL import Image
 
 from sqlalchemy.orm import aliased
@@ -212,6 +213,39 @@ def set_preview_file_as_broken(preview_file_id):
     return update_preview_file(preview_file_id, {"status": "broken"})
 
 
+def set_preview_file_as_missing(preview_file_id):
+    """
+    Mark a preview file as missing: its source binary is gone from
+    storage and renormalization is no longer possible.
+    """
+    return update_preview_file(preview_file_id, {"status": "missing"})
+
+
+def mark_preview_file_as_missing_on_storage_404(preview_file_id):
+    """
+    Flag a preview file as ``missing`` because its binary turned out to
+    be absent from object storage at read time (Swift/S3 404).
+
+    Safe to call from any code path that catches a "file not found"
+    storage error: this is a best-effort, fire-and-forget marker that
+    skips rows still being uploaded (``processing``) or already flagged
+    as ``missing``, and never raises if the database write fails.
+    """
+    try:
+        preview_file = PreviewFile.get(preview_file_id)
+    except Exception:
+        return
+    if preview_file is None:
+        return
+    status_code = getattr(preview_file.status, "code", preview_file.status)
+    if status_code in ("processing", "missing"):
+        return
+    try:
+        set_preview_file_as_missing(preview_file_id)
+    except Exception:
+        pass
+
+
 def set_preview_file_as_ready(preview_file_id):
     """
     Mark given preview file as ready.
@@ -718,15 +752,15 @@ def _clear_empty_annotations(annotations):
 
 def get_running_preview_files(cursor_preview_file_id=None, limit=None):
     """
-    Return preview files for all productions with status equals to broken
-    or processing using cursor-based pagination.
+    Return preview files for all productions with status equals to broken,
+    missing or processing using cursor-based pagination.
     """
     query = (
         PreviewFile.query.join(Task)
         .join(Project)
         .join(ProjectStatus, ProjectStatus.id == Project.project_status_id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.in_(("broken", "processing")))
+        .filter(PreviewFile.status.in_(("broken", "missing", "processing")))
         .add_columns(Task.project_id, Task.task_type_id, Task.entity_id)
         .order_by(PreviewFile.created_at.desc())
     )
@@ -802,14 +836,18 @@ def extract_frame_from_preview_file(preview_file, frame_number):
         raise PreviewFileNotFoundException
 
     if preview_file["extension"] == "mp4":
-        preview_file_path = fs.get_file_path_and_file(
-            config,
-            file_store.get_local_movie_path,
-            file_store.open_movie,
-            "previews",
-            preview_file["id"],
-            "mp4",
-        )
+        try:
+            preview_file_path = fs.get_file_path_and_file(
+                config,
+                file_store.get_local_movie_path,
+                file_store.open_movie,
+                "previews",
+                preview_file["id"],
+                "mp4",
+            )
+        except FileNotFound:
+            mark_preview_file_as_missing_on_storage_404(preview_file["id"])
+            raise
     else:
         raise PreviewFileNotFoundException
 
@@ -915,6 +953,9 @@ def _copy_picture_preview_to_temp_png(preview_file):
             preview_file["id"],
             preview_file["extension"],
         )
+    except FileNotFound:
+        mark_preview_file_as_missing_on_storage_404(preview_file["id"])
+        return None
     except Exception:
         return None
     fd, temp_path = tempfile.mkstemp(suffix=".png")
@@ -1156,7 +1197,7 @@ def reset_movie_files_metadata():
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension == "mp4")
     )
     for preview_file in preview_files:
@@ -1184,6 +1225,11 @@ def reset_movie_files_metadata():
             print(
                 f"Size information stored preview file {preview_file.id}",
             )
+        except FileNotFound:
+            mark_preview_file_as_missing_on_storage_404(str(preview_file.id))
+            print(
+                f"Preview file {preview_file.id} marked as missing (binary gone from storage)"
+            )
         except Exception as e:
             print(
                 f"Failed to store information for preview file {preview_file.id}: {e}"
@@ -1199,7 +1245,7 @@ def reset_picture_files_metadata():
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension == "png")
     )
     for preview_file in preview_files:
@@ -1225,6 +1271,11 @@ def reset_picture_files_metadata():
             print(
                 f"Size information stored for preview file {preview_file.id}",
             )
+        except FileNotFound:
+            mark_preview_file_as_missing_on_storage_404(str(preview_file.id))
+            print(
+                f"Preview file {preview_file.id} marked as missing (binary gone from storage)"
+            )
         except Exception as e:
             print(
                 f"Failed to store information for preview file {preview_file.id}: {e}"
@@ -1253,7 +1304,7 @@ def generate_preview_extra(
         .join(Project)
         .join(ProjectStatus, Project.project_status_id == ProjectStatus.id)
         .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
-        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.status.not_in(("broken", "missing", "processing")))
         .filter(PreviewFile.extension.in_(("mp4", "png")))
     )
     if project is not None:
@@ -1366,6 +1417,12 @@ def _retrieve_preview_file(config, file_store, prefix, preview_file):
             str(preview_file.id),
             preview_file.extension,
         )
+    except FileNotFound:
+        mark_preview_file_as_missing_on_storage_404(str(preview_file.id))
+        print(
+            f"Preview file {preview_file.id} marked as missing (binary gone from storage)."
+        )
+        return None
     except Exception as e:
         print(f"Failed to get preview file {preview_file.id}: {e}.")
         return None