tests: add reproducer for pack retaining deleted files between builds (#710)

charmcraft/parts/lifecycle.py

@@ -96,6 +96,13 @@ def run(self, target_step: Step) -> None:
                         self._lcm.clean(Step.BUILD, part_names=["charm"])
                         self._lcm.reload_state()
 
+            # Workaround for https://github.com/canonical/craft-parts/issues/851:
+            # craft-parts' local source update copies new/modified files but does
+            # not remove files deleted from the source directory. Clean the pull
+            # step for any affected part so stale files are not included in the
+            # repacked charm archive.
+            self._clean_stale_parts()
+
             emit.debug(f"Executing parts lifecycle in {str(self._project_dir)!r}")
             actions = self._lcm.plan(target_step)
             emit.debug(f"Parts actions: {actions}")
@@ -127,6 +134,63 @@ def run(self, target_step: Step) -> None:
         finally:
             os.chdir(previous_dir)
 
+    def _clean_stale_parts(self) -> None:
+        """Clean parts whose local source has had files deleted since the last pull.
+
+        Craft-parts' local source ``update()`` copies new/modified files but does
+        not remove files from the part src directory that have been deleted from
+        the original source. This method detects such deletions and triggers a
+        clean pull for the affected parts so stale files are not included in the
+        repacked charm archive.
+
+        Workaround for https://github.com/canonical/craft-parts/issues/851.
+        """
+        parts_dir = self._lcm.project_info.dirs.parts_dir
+        parts_to_clean: list[str] = []
+
+        for part_name, part_spec in self._all_parts.items():
+            source = part_spec.get("source", "")
+            if not source:
+                continue
+
+            source_path = pathlib.Path(source)
+            if not source_path.is_absolute():
+                source_path = self._project_dir / source_path
+
+            source_subdir = part_spec.get("source-subdir", "")
+            if source_subdir:
+                source_path = source_path / source_subdir
+
+            if not source_path.is_dir():
+                continue
+
+            part_src_dir = parts_dir / part_name / "src"
+            if not part_src_dir.exists():
+                continue
+
+            src_files = {
+                f.relative_to(part_src_dir)
+                for f in part_src_dir.rglob("*")
+                if f.is_file()
+            }
+            source_files = {
+                f.relative_to(source_path)
+                for f in source_path.rglob("*")
+                if f.is_file()
+            }
+
+            if src_files - source_files:
+                emit.debug(
+                    f"Source files deleted from part {part_name!r}; "
+                    "cleaning to remove stale files from stage and prime."
+                )
+                parts_to_clean.append(part_name)
+
+        if parts_to_clean:
+            for part_name in parts_to_clean:
+                self._lcm.clean(Step.PULL, part_names=[part_name])
+            self._lcm.reload_state()
+
 
 def _get_dispatch_entrypoint(dirname: pathlib.Path) -> str:
     """Read the entrypoint from the dispatch file."""

pyproject.toml

@@ -14,7 +14,6 @@ dependencies = [
     "distro>=1.7.0",
     "docker>=7.0.0",
     "humanize>=2.6.0",
-    "jsonschema~=4.0",
     "jinja2",
     "pydantic~=2.0",
     "python-dateutil",

tests/unit/parts/test_lifecycle.py

@@ -200,3 +200,51 @@ def test_parthelpers_get_dispatch_entrypoint(tmp_path):
 def test_parthelpers_get_dispatch_entrypoint_no_file(tmp_path):
     entrypoint = lifecycle._get_dispatch_entrypoint(tmp_path)
     assert entrypoint == ""
+
+
+# Reproducer for https://github.com/canonical/charmcraft/issues/710
+def test_stale_files_removed_from_prime_after_repack(tmp_path: pathlib.Path):
+    """Deleted source files must not appear in the prime dir on a subsequent pack.
+
+    When a file is deleted from the charm source between two consecutive packs,
+    charmcraft detects the change and removes the stale file from the stage and
+    prime directories so it does not end up in the final .charm archive.
+
+    Upstream issue: https://github.com/canonical/craft-parts/issues/851
+    """
+    src_dir = tmp_path / "src"
+    src_dir.mkdir()
+    (src_dir / "charm.py").write_text("# charm entrypoint")
+    alert_rules_dir = src_dir / "prometheus_alert_rules"
+    alert_rules_dir.mkdir()
+    alert_rule = alert_rules_dir / "always_firing.rule"
+    alert_rule.write_text("groups: []")
+
+    all_parts = {"my-charm": {"plugin": "dump", "source": str(src_dir)}}
+
+    def _run_lifecycle(work_dir: pathlib.Path) -> pathlib.Path:
+        lc = lifecycle.PartsLifecycle(
+            all_parts=all_parts,
+            work_dir=work_dir,
+            project_dir=work_dir,
+            project_name="test-charm",
+            ignore_local_sources=[],
+        )
+        lc.run(Step.PRIME)
+        return lc.prime_dir
+
+    # First pack: both files should be present in prime.
+    prime_dir = _run_lifecycle(tmp_path)
+    assert (prime_dir / "charm.py").exists()
+    assert (prime_dir / "prometheus_alert_rules" / "always_firing.rule").exists()
+
+    # Simulate the user deleting the alert rule file between packs.
+    alert_rule.unlink()
+    assert not alert_rule.exists()
+
+    # Second pack: the deleted file must NOT be present in prime.
+    prime_dir = _run_lifecycle(tmp_path)
+    assert (prime_dir / "charm.py").exists()
+    assert not (prime_dir / "prometheus_alert_rules" / "always_firing.rule").exists(), (
+        "Stale file from deleted source is still present in prime directory after repack"
+    )