diff --git a/CHANGES/386.feature b/CHANGES/386.feature new file mode 100644 index 00000000..d2d8bcf4 --- /dev/null +++ b/CHANGES/386.feature @@ -0,0 +1,6 @@ +Implemented ``dependency_solving`` on the advanced copy endpoint. When enabled, the +copy task BFS-walks the Depends/Pre-Depends closure of every initial package, picks +the newest version satisfying each relation from the source repository, honours +Provides aliases, and skips relations already satisfied by the destination base +version. Unsatisfiable relations raise an explicit error. Resolves +:redmine:`386`. diff --git a/pulp_deb/app/serializers/repository_serializers.py b/pulp_deb/app/serializers/repository_serializers.py index eb770fa0..af3f6478 100644 --- a/pulp_deb/app/serializers/repository_serializers.py +++ b/pulp_deb/app/serializers/repository_serializers.py @@ -179,8 +179,9 @@ class CopySerializer(ValidateFieldsMixin, serializers.Serializer): dependency_solving = serializers.BooleanField( help_text=_( - "Also copy dependencies of any packages being copied. NOT YET" - 'IMPLEMENTED! You must keep this at "False"!' + "Also copy the transitive Depends/Pre-Depends closure of every package being copied. " + "Already-satisfied relations (present in dest_base_version) are skipped. " + "Raises an error if any relation cannot be satisfied from the source repository." ), default=False, ) diff --git a/pulp_deb/app/tasks/copy.py b/pulp_deb/app/tasks/copy.py index 576bea44..f5a6851e 100644 --- a/pulp_deb/app/tasks/copy.py +++ b/pulp_deb/app/tasks/copy.py @@ -14,6 +14,7 @@ Release, ReleaseArchitecture, ) +from pulp_deb.app.tasks.dependency_solving import solve_dependencies log = logging.getLogger(__name__) @@ -104,9 +105,6 @@ def process_entry(entry): content_filter, ) - if dependency_solving: - raise NotImplementedError("Advanced copy with dependency solving is not yet implemented.") - for entry in config: ( source_repo_version, @@ -116,6 +114,16 @@ def process_entry(entry): ) = process_entry(entry) content_to_copy = source_repo_version.content.filter(content_filter) + + if dependency_solving: + initial_packages = Package.objects.filter( + pk__in=content_to_copy.filter(pulp_type=Package.get_pulp_type()).only("pk") + ) + extra_pks = solve_dependencies(initial_packages, source_repo_version, dest_base_version) + content_to_copy = source_repo_version.content.filter( + Q(pk__in=content_to_copy.values("pk")) | Q(pk__in=extra_pks) + ) + if structured: content_to_copy = find_structured_publish_content(content_to_copy, source_repo_version) diff --git a/pulp_deb/app/tasks/dependency_solving.py b/pulp_deb/app/tasks/dependency_solving.py new file mode 100644 index 00000000..3e16464d --- /dev/null +++ b/pulp_deb/app/tasks/dependency_solving.py @@ -0,0 +1,160 @@ +from collections import deque +from gettext import gettext as _ + +from debian.deb822 import PkgRelation +from debian.debian_support import Version +from django.db.models import Q + +from pulp_deb.app.models import Package + + +class DependencySolveError(Exception): + """Raised when a transitive dependency cannot be satisfied from the source repository.""" + + +_OP_FUNCS = { + ">=": lambda a, b: a >= b, + ">>": lambda a, b: a > b, + "<=": lambda a, b: a <= b, + "<<": lambda a, b: a < b, + "=": lambda a, b: a == b, +} + + +def _parse_relations(*depends_strings): + combined = ", ".join(s for s in depends_strings if s) + if not combined.strip(): + return [] + try: + return PkgRelation.parse_relations(combined) + except Exception: + return [] + + +def _version_satisfies(pkg_version, constraint): + if not constraint: + return True + op, target = constraint + return _OP_FUNCS.get(op, lambda a, b: False)(Version(pkg_version), Version(target)) + + +def _candidates_for(alt, package_qs, pkg_arch=None): + name = alt["name"] + qs = package_qs.filter(package=name) + alt_arch = alt.get("arch") + if alt_arch: + qs = qs.filter(architecture=alt_arch) + elif pkg_arch: + # Restrict deps to the consumer's arch or arch-independent (`all`). + qs = qs.filter(Q(architecture=pkg_arch) | Q(architecture="all")) + return qs + + +def _find_satisfier(alternative, package_qs, pkg_arch=None): + """Best Package satisfying a single OR-alternative (direct hit or via Provides).""" + constraint = alternative.get("version") + + direct = list(_candidates_for(alternative, package_qs, pkg_arch)) + direct.sort(key=lambda p: Version(p.version), reverse=True) + for pkg in direct: + if _version_satisfies(pkg.version, constraint): + return pkg + + # Provides: a Package P providing the name we need. Versioned Provides + # are rare; we honour them when present, otherwise treat as unversioned. + name = alternative["name"] + provides_qs = package_qs.exclude(provides__isnull=True).exclude(provides="") + if pkg_arch: + provides_qs = provides_qs.filter(Q(architecture=pkg_arch) | Q(architecture="all")) + for pkg in provides_qs: + for or_group in _parse_relations(pkg.provides): + for prov in or_group: + if prov["name"] != name: + continue + prov_version = prov.get("version") + if not constraint: + return pkg + if ( + prov_version + and prov_version[0] == "=" + and _version_satisfies(prov_version[1], constraint) + ): + return pkg + # Unversioned Provides cannot satisfy a versioned dependency per policy. + return None + + +def _relation_satisfied_by_set(relation, package_qs, pkg_arch=None): + return any(_find_satisfier(alt, package_qs, pkg_arch) is not None for alt in relation) + + +def _packages_in(repo_version): + if repo_version is None: + return Package.objects.none() + return Package.objects.filter( + pk__in=repo_version.content.filter(pulp_type=Package.get_pulp_type()).only("pk") + ) + + +def _format_relation(relation): + parts = [] + for alt in relation: + s = alt["name"] + if alt.get("arch"): + s += ":{}".format(alt["arch"]) + if alt.get("version"): + s += " ({} {})".format(*alt["version"]) + parts.append(s) + return " | ".join(parts) + + +def solve_dependencies(initial_packages, source_repo_version, dest_base_version=None): + """BFS over Depends + Pre-Depends. Returns the set of Package pks to copy. + + A relation already satisfied by ``dest_base_version`` is skipped; otherwise + the newest satisfying Package in ``source_repo_version`` is chosen, then + walked transitively. Raises :class:`DependencySolveError` on unsatisfiable + relations. + """ + source_pkgs = _packages_in(source_repo_version) + dest_pkgs = _packages_in(dest_base_version) + + seen = set() + queue = deque(initial_packages) + final = set() + + while queue: + pkg = queue.popleft() + if pkg.pk in seen: + continue + seen.add(pkg.pk) + final.add(pkg.pk) + + for relation in _parse_relations(pkg.depends, pkg.pre_depends): + if dest_base_version is not None and _relation_satisfied_by_set( + relation, dest_pkgs, pkg.architecture + ): + continue + + chosen = None + for alt in relation: + chosen = _find_satisfier(alt, source_pkgs, pkg.architecture) + if chosen: + break + + if chosen is None: + raise DependencySolveError( + _( + "Cannot solve dependency for {pkg}: " + "'{rel}' is not satisfiable in source repository version {srv}" + ).format( + pkg=pkg.name, + rel=_format_relation(relation), + srv=source_repo_version.pk, + ) + ) + + if chosen.pk not in seen: + queue.append(chosen) + + return final diff --git a/pulp_deb/tests/functional/api/test_copy.py b/pulp_deb/tests/functional/api/test_copy.py index c4a8da2d..d667c753 100644 --- a/pulp_deb/tests/functional/api/test_copy.py +++ b/pulp_deb/tests/functional/api/test_copy.py @@ -76,3 +76,32 @@ def test_copy_empty_content( target_repo = deb_get_repository_by_href(target_repo.pulp_href) assert target_repo.latest_version_href.endswith("/versions/0/") + + +@pytest.mark.parallel +def test_copy_with_dependency_solving( + deb_init_and_sync, + deb_repository_factory, + apt_package_api, + deb_copy_content, + deb_get_repository_by_href, + deb_get_content_summary, +): + """Smoke test that ``dependency_solving=True`` reaches the new code path.""" + source_repo, _ = deb_init_and_sync() + target_repo = deb_repository_factory() + package = apt_package_api.list( + package="frigg", + repository_version=source_repo.latest_version_href, + ).results[0] + + deb_copy_content( + source_repo_version=source_repo.latest_version_href, + dest_repo=target_repo.pulp_href, + content=[package.pulp_href], + dependency_solving=True, + ) + + target_repo = deb_get_repository_by_href(target_repo.pulp_href) + added_summary = deb_get_content_summary(target_repo).added + assert DEB_ADVANCED_COPY_FIXTURE_SUMMARY == get_counts_from_content_summary(added_summary) diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index 65aabe97..9490f472 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -467,19 +467,26 @@ def _deb_acs_factory(**kwargs): def deb_copy_content(apt_copy_api, monitor_task): """Fixture that copies deb content from a source repository version to a target repository.""" - def _deb_copy_content(source_repo_version, dest_repo, content=None, structured=True): + def _deb_copy_content( + source_repo_version, + dest_repo, + content=None, + structured=True, + dependency_solving=False, + ): """Copy deb content from a source repository version to a target repository. :param source_repo_version: The repository version href from where the content is copied. :dest_repo: The repository href where the content should be copied to. :content: List of package hrefs that should be copied from the source. Default: None :structured: Whether or not the content should be structured copied. Default: True + :dependency_solving: Also copy the Depends/Pre-Depends closure. Default: False :returns: The task of the copy operation. """ config = {"source_repo_version": source_repo_version, "dest_repo": dest_repo} if content is not None: config["content"] = content - data = Copy(config=[config], structured=structured) + data = Copy(config=[config], structured=structured, dependency_solving=dependency_solving) response = apt_copy_api.copy_content(data) return monitor_task(response.task) diff --git a/pulp_deb/tests/unit/test_dependency_solving.py b/pulp_deb/tests/unit/test_dependency_solving.py new file mode 100644 index 00000000..2307bf39 --- /dev/null +++ b/pulp_deb/tests/unit/test_dependency_solving.py @@ -0,0 +1,116 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from pulp_deb.app.tasks.dependency_solving import ( + DependencySolveError, + _find_satisfier, + _parse_relations, + _version_satisfies, + solve_dependencies, +) + + +@pytest.fixture +def pkg(): + """Build a Package mock with the fields the solver reads.""" + + def _make(name, version, depends=None, provides=None, pk=None, arch="amd64"): + p = MagicMock() + p.package = name + p.version = version + p.architecture = arch + p.depends = depends + p.pre_depends = None + p.provides = provides + p.pk = pk if pk is not None else id(p) + p.name = "{}_{}_{}".format(name, version, arch) + return p + + return _make + + +@pytest.fixture +def qs(): + """Build a Django QuerySet stub supporting filter()/exclude() and iteration.""" + + def _make(items): + q = MagicMock() + q._items = list(items) + + def _filter(*args, **kwargs): + out = list(q._items) + for key, val in kwargs.items(): + field = key.split("__")[0] + out = [p for p in out if getattr(p, field, None) == val] + return _make(out) + + q.filter.side_effect = _filter + q.exclude.side_effect = lambda **kw: _make( + [p for p in q._items if not all(getattr(p, k, None) == v for k, v in kw.items())] + ) + q.__iter__.side_effect = lambda: iter(q._items) + q.__bool__.side_effect = lambda: bool(q._items) + q.none.return_value = _make([]) if items else q + return q + + return _make + + +def test_parses_or_group_with_version(): + rels = _parse_relations("libfoo (>= 1.2.3) | libbar") + assert len(rels) == 1 and len(rels[0]) == 2 + assert rels[0][0]["version"] == (">=", "1.2.3") + assert rels[0][1]["name"] == "libbar" + + +def test_version_satisfies_respects_epoch(): + # Epoch wins over upstream version per Debian policy. + assert _version_satisfies("2:1.0", (">=", "1:9.9")) + assert not _version_satisfies("1.0", (">=", "2.0")) + + +def test_find_satisfier_picks_newest(pkg, qs): + pkgs = [pkg("libssl3", v) for v in ("3.0.7-25", "3.0.7-27", "3.0.7-26")] + assert _find_satisfier({"name": "libssl3"}, qs(pkgs)).version == "3.0.7-27" + + +def test_find_satisfier_via_provides(pkg, qs): + pkgs = [pkg("libssl3", "3.0", provides="libssl1.1")] + assert _find_satisfier({"name": "libssl1.1"}, qs(pkgs)).package == "libssl3" + # Unversioned Provides cannot satisfy a versioned dep. + assert _find_satisfier({"name": "libssl1.1", "version": (">=", "1.1.1")}, qs(pkgs)) is None + + +def test_solver_walks_chain(pkg, qs): + a = pkg("a", "1.0", depends="b", pk=1) + b = pkg("b", "1.0", depends="c", pk=2) + c = pkg("c", "1.0", pk=3) + with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg: + Pkg.objects.filter.return_value = qs([a, b, c]) + Pkg.get_pulp_type.return_value = "deb.package" + srv = MagicMock() + srv.content.filter.return_value.only.return_value = qs([a, b, c]) + assert solve_dependencies([a], srv, dest_base_version=None) == {1, 2, 3} + + +def test_solver_skips_deps_already_in_dest(pkg, qs): + a = pkg("a", "1.0", depends="b (>= 1.0)", pk=10) + b = pkg("b", "1.0", pk=11) + with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg: + Pkg.objects.filter.side_effect = [qs([a, b]), qs([b])] + srv, dest = MagicMock(), MagicMock() + srv.content.filter.return_value.only.return_value = qs([a, b]) + dest.content.filter.return_value.only.return_value = qs([b]) + assert solve_dependencies([a], srv, dest_base_version=dest) == {10} + + +def test_solver_raises_on_unsatisfiable(pkg, qs): + a = pkg("a", "1.0", depends="missing-pkg (>= 1.0)", pk=20) + with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg: + Pkg.objects.filter.return_value = qs([a]) + srv = MagicMock() + srv.pk = 99 + srv.content.filter.return_value.only.return_value = qs([a]) + with pytest.raises(DependencySolveError): + solve_dependencies([a], srv)