From 888fdcbe8380532bb570fa1249e67b907af368f7 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Tue, 28 Oct 2025 11:18:42 +0100 Subject: [PATCH 1/3] feat(updating)!: introduce new update algorithm based on `git merge` BREAKING CHANGE: The `-o,--conflict` flag is removed, as `git merge` only supports inline conflicts. Omit the flag and corresponding API parameter as of now. BREAKING CHANGE: The `-c,--context-lines` flag is removed, as `git merge` does not support configurable context sizes. Omit the flag and corresponding API parameter as of now. BREAKING CHANGE: Inline conflict marker labels are different, as `git merge` does not support their customization. BREAKING CHANGE: `gitattributes` settings affect Copier's internal `git merge` call. --- copier/_cli.py | 20 -- copier/_main.py | 528 +++++++++++++------------------------ copier/_tools.py | 88 ------- docs/configuring.md | 37 --- docs/creating.md | 2 - docs/updating.md | 104 +++----- tests/helpers.py | 25 ++ tests/test_cli.py | 6 - tests/test_dirty_local.py | 2 +- tests/test_subdirectory.py | 40 +-- tests/test_tools.py | 4 +- tests/test_updatediff.py | 295 ++++++--------------- 12 files changed, 346 insertions(+), 805 deletions(-) diff --git a/copier/_cli.py b/copier/_cli.py index 0365f9364..5ad0bfecc 100644 --- a/copier/_cli.py +++ b/copier/_cli.py @@ -62,7 +62,6 @@ from collections.abc import Callable, Iterable from pathlib import Path from textwrap import dedent -from typing import Literal, cast import yaml from plumbum import LocalPath, cli, colors @@ -395,23 +394,6 @@ class CopierUpdateSubApp(_Subcommand): """ ) - conflict = cli.SwitchAttr( - ["-o", "--conflict"], - cli.Set("rej", "inline"), - default="inline", - help=( - "Behavior on conflict: Create .rej files, or add inline conflict markers." - ), - ) - context_lines = cli.SwitchAttr( - ["-c", "--context-lines"], - int, - default=3, - help=( - "Lines of context to use for detecting conflicts. Increase for " - "accuracy, decrease for resilience." - ), - ) defaults = cli.Flag( ["-l", "-f", "--defaults"], help="Use default answers to questions, which might be null if not specified.", @@ -456,8 +438,6 @@ def inner() -> None: overwrite=True, pretend=self.pretend, quiet=self.quiet, - conflict=cast(Literal["rej", "inline"], self.conflict), - context_lines=self.context_lines, unsafe=self.unsafe, skip_answered=self.skip_answered, skip_tasks=self.skip_tasks, diff --git a/copier/_main.py b/copier/_main.py index a584dba48..d9874a69e 100644 --- a/copier/_main.py +++ b/copier/_main.py @@ -9,10 +9,9 @@ import sys import warnings from collections.abc import Callable, Iterable, Mapping, Sequence -from contextlib import suppress +from contextlib import ExitStack, suppress from contextvars import ContextVar from dataclasses import field, replace -from filecmp import dircmp from fnmatch import fnmatchcase from functools import cached_property, partial, wraps from itertools import chain @@ -39,7 +38,7 @@ from pathspec import PathSpec, __version__ as pathspec_version from plumbum import ProcessExecutionError, colors from plumbum.machines import local -from pydantic import ConfigDict, PositiveInt +from pydantic import ConfigDict from pydantic.dataclasses import dataclass from pydantic_core import to_jsonable_python from questionary import confirm, unsafe_prompt @@ -53,11 +52,8 @@ OS, Style, cast_to_bool, - escape_git_path, - normalize_git_path, printf, scantree, - set_git_alternates, ) from ._types import ( MISSING, @@ -207,18 +203,6 @@ class Worker: See [quiet][]. - conflict: - One of "inline" (default), "rej". - - context_lines: - Lines of context to consider when solving conflicts in updates. - - With more lines, context resolution is more accurate, but it will - also produce more conflicts if your subproject has evolved. - - With less lines, context resolution is less accurate, but it will - respect better the evolution of your subproject. - unsafe: When `True`, allow usage of unsafe templates. @@ -253,8 +237,6 @@ class Worker: overwrite: bool = False pretend: bool = False quiet: bool = False - conflict: Literal["inline", "rej"] = "inline" - context_lines: PositiveInt = 3 unsafe: bool = False skip_answered: bool = False skip_tasks: bool = False @@ -452,8 +434,6 @@ def _render_context(self) -> AnyByStrMutableMapping: "overwrite": lambda: self.overwrite, "pretend": lambda: self.pretend, "quiet": lambda: self.quiet, - "conflict": lambda: self.conflict, - "context_lines": lambda: self.context_lines, "unsafe": lambda: self.unsafe, "skip_answered": lambda: self.skip_answered, "skip_tasks": lambda: self.skip_tasks, @@ -945,6 +925,7 @@ def _sync_git_index_executable_bit(self, dst_relpath: Path, src_mode: int) -> No """ subproject_root = self.subproject.local_abspath git = get_git(context_dir=subproject_root) + desired_executable = bool(src_mode & 0o111) try: # ``--type=bool`` normalizes truthy/falsy spellings to # ``true``/``false``. Exits 1 if the key is unset. @@ -968,13 +949,23 @@ def _sync_git_index_executable_bit(self, dst_relpath: Path, src_mode: int) -> No # git can't read the index — fall back to a silent no-op. return if not result: + if _operation.get() == "update" and desired_executable: + # HACK: If we're in update mode, we're generating fresh copies + # from old and new template versions into empty Git repos. Thus, + # the file is not yet tracked, but we still want to stage it as + # executable if the new template version has it set. + try: + git("add", "-f", "--chmod=+x", "--", str(dst_relpath)) + except (OSError, ProcessExecutionError): + # git not installed, or some other unrelated git failure + # — silently fall back so we never break the render path. + pass # File is not tracked yet; nothing to update. return # Format: " \t" meta = result.split("\t", 1)[0].split() current_index_mode = int(meta[0], 8) current_index_sha = meta[1] - desired_executable = bool(src_mode & 0o111) current_executable = bool(current_index_mode & 0o111) if desired_executable == current_executable: return @@ -1370,6 +1361,21 @@ def _apply_update(self) -> None: # noqa: C901 ).strip() ) subproject_subdir = self.subproject.local_abspath.relative_to(subproject_top) + subproject_gitpath = git["-C", subproject_top, "rev-parse", "--git-path"] + + # Collect all paths in the subproject except those in the `.git` directory. + with local.cwd(self.subproject.local_abspath): + subproject_paths = set( + p + for entry in scantree(".", follow_symlinks=False) + if (p := Path(entry.path)).parts[0] != ".git" + ) + # Filter all paths in the subproject that match a skip-if-exists pattern for + # further processing below. Convert them to POSIX-style absolute paths to make + # them Gitignore patterns matching exact paths anchored at the subproject root. + subproject_skip_paths = set( + f"/{p.as_posix()}" for p in subproject_paths if self.match_skip(p) + ) with ( TemporaryDirectory( @@ -1379,7 +1385,11 @@ def _apply_update(self) -> None: # noqa: C901 prefix=f"{__name__}.new_copy.", ) as new_copy, ): - # Copy old template into a temporary destination + git("-C", old_copy, "init") + git("-C", new_copy, "init") + + # Create a fresh copy based on the old template into a temporary + # destination. with replace( self, dst_path=old_copy / subproject_subdir, @@ -1391,311 +1401,186 @@ def _apply_update(self) -> None: # noqa: C901 # Exclude also paths listed in the new template version, so they # won't be included in the diff as deleted paths to prevent deletion. # https://github.com/orgs/copier-org/discussions/2345 - exclude=[*self.template.exclude, *self.exclude], + exclude=[ + *map(str, subproject_skip_paths), + *self.template.exclude, + *self.exclude, + ], ask=(), ) as old_worker: old_worker.run_copy() - # Run pre-migration tasks + + # Run pre-migration tasks. with Phase.use(Phase.MIGRATE): self._execute_tasks( self.template.migration_tasks("before", self.subproject.template) # type: ignore[arg-type] ) - # Create a Git tree object from the current (possibly dirty) index - # and keep the object reference. - with local.cwd(subproject_top): - subproject_head = git("write-tree").strip() - with local.cwd(old_copy): - self._git_initialize_repo() - # Configure borrowing Git objects from the real destination. - set_git_alternates(subproject_top) - # Save a list of files that were intentionally removed in the generated - # project to avoid recreating them during the update. - # Files listed in `skip_if_exists` should only be skipped if they exist. - # They should even be recreated if deleted intentionally. - files_removed = git( - "diff-tree", - "-r", - "--diff-filter=D", - "--name-only", - "HEAD", - subproject_head, - ).splitlines() - exclude_plus_removed = list( - set(self.exclude).union( - f"/{escape_git_path(path)}" - for path in map(normalize_git_path, files_removed) - if not (subproject_top / path).exists() - and not self.match_skip(Path(path)) - ) - ) - # Clear last answers cache to load possible answers migration, if - # skip_answered flag is not set + + # Clear last answers cache to load possible answers migration if the + # `skip_answered` flag is not set. if self.skip_answered is False: self.answers = AnswersMap(external=self._external_data()) with suppress(AttributeError): del self.subproject.last_answers - # Do a normal update in final destination - with replace( - self, - # Don't regenerate intentionally deleted paths - exclude=exclude_plus_removed, - # Files can change due to the historical diff, and those - # changes are not detected in this process, so it's better to - # say nothing than lie. - # TODO - quiet=True, - ) as current_worker: - current_worker.run_copy() - self.answers = current_worker.answers - self.answers.external = self._external_data() - # Render with the same answers in an empty dir to avoid pollution + + # Copy new template into a temporary destination. with replace( self, dst_path=new_copy / subproject_subdir, - data={ - k: v - for k, v in self.answers.combined.items() - if not k.startswith("_") - and k not in self.answers.hidden - and isinstance(k, JSONSerializable) - and isinstance(v, JSONSerializable) - }, - defaults=True, - quiet=True, src_path=self.subproject.template.url, # type: ignore[union-attr] - exclude=exclude_plus_removed, vcs_ref=self.resolved_vcs_ref, - ask=(), + exclude=[*map(str, subproject_skip_paths), *self.exclude], ) as new_worker: + # HACK: Use the recorded answers from the subproject's answers file. + new_worker.subproject.last_answers = self.subproject.last_answers new_worker.run_copy() - with local.cwd(new_copy): - self._git_initialize_repo() - new_copy_head = git("rev-parse", "HEAD").strip() - # Extract diff between temporary destination and real destination - # with some special handling of newly added files in both the project - # and the template. - with local.cwd(old_copy): - # Configure borrowing Git objects from the real destination and - # temporary destination of the new template. - set_git_alternates(subproject_top, Path(new_copy)) - # Create an empty file in the temporary destination when the - # same file was added in *both* the project and the temporary - # destination of the new template. With this minor change, the - # diff between the temporary destination and the real - # destination for such files will use the "update file mode" - # instead of the "new file mode" which avoids deleting the file - # content previously added in the project. - diff_added_cmd = git[ - "diff-tree", "-r", "--diff-filter=A", "--name-only" - ] - for filename in ( - set(diff_added_cmd("HEAD", subproject_head).splitlines()) - ) & set(diff_added_cmd("HEAD", new_copy_head).splitlines()): - f = Path(filename) - f.parent.mkdir(parents=True, exist_ok=True) - f.touch((subproject_top / filename).stat().st_mode) - git("add", "--force", filename) - self._git_commit("add new empty files") - # Extract diff between temporary destination and real - # destination - diff_cmd = git[ - "diff-tree", - f"--unified={self.context_lines}", - "HEAD", - subproject_head, - ] - # Get the list of modified files in the subproject directory - # that match the skip-if-exists patterns. These are relative - # paths anchored at the Git repo root because they will be used - # with `git apply --exclude` later, which expects paths relative - # to the repo root. Importantly, the skip-if-exists patterns - # are anchored at the subproject root, which may be a - # subdirectory of the Git repo, so we need to relativize the - # paths accordingly for pattern matching. - skip_if_exists_files = [ - escape_git_path(f) - for f in map( - normalize_git_path, - diff_cmd( - "-r", "--no-commit-id", "--name-only", subproject_subdir - ).splitlines(), - ) - if self.match_skip(Path(f).relative_to(subproject_subdir)) - ] - try: - diff = diff_cmd("--inter-hunk-context=-1") - except ProcessExecutionError: - print( - colors.warn - | "Make sure Git >= 2.24 is installed to improve updates.", - file=sys.stderr, + # HACK: Assign the new answers to the current worker. + self.answers = new_worker.answers + + # Construct the Git tree of the fresh copy based on the new template. + with ( + local.cwd(new_copy), + # Reuse the Git object directory of the subproject's Git repository, + # so the blob and tree objects stored there. + local.env( + GIT_OBJECT_DIRECTORY=str( + subproject_top / subproject_gitpath("objects").strip() ) - diff = diff_cmd("--inter-hunk-context=0") - compared = dircmp(old_copy, new_copy) - # Try to apply cached diff into final destination - with local.cwd(subproject_top): - apply_cmd = git[ - "apply", - "--reject", - "--exclude", - subproject_subdir / self.answers_relpath, - ] - # Exclude modified files that match the skip-if-exists patterns - # to exclude them from the patch application. - for filename in skip_if_exists_files: - apply_cmd = apply_cmd["--exclude", filename] - ignored_files = git["status", "--ignored", "--porcelain"]() - # returns "!! file1\n !! file2\n" - # adds `--exclude file1 --exclude file2` to `git apply` command - for filename in ignored_files.splitlines(): - if filename.startswith("!! "): - filepath = filename[3:] - # Don't exclude template-generated files that happen to - # be gitignored — they should still be updated. - # Fixes #2729, regression of #1162. - if (Path(new_copy) / normalize_git_path(filepath)).exists(): - continue - apply_cmd = apply_cmd["--exclude", filepath] - (apply_cmd << diff)(retcode=None) - if self.conflict == "inline": - conflicted = [] - old_path = Path(old_copy) - new_path = Path(new_copy) - # `--ignored` so we still find .rej files when the - # destination has a `*.rej` ignore rule. - status = ( - git("status", "--porcelain", "--ignored").strip().splitlines() + ), + ): + # Collect all files of the fresh copy based on the new template for + # later use. + new_copy_files = set( + path + for entry in scantree(".", follow_symlinks=False) + if (path := Path(entry.path)).parts[0] != ".git" + ) + + # Stage all files including Git-ignored ones. + git("add", "-f", ".") + # Make a commit to run Git hooks if applicable. + self._git_commit() + + # Get the Git tree ID. + tree_id_new = git("rev-parse", "--verify", "HEAD^{tree}").strip() + + # Construct the Git tree of the fresh copy based on the old template. + with ( + local.cwd(old_copy), + # Reuse the Git object directory of the subproject's Git repository, + # so the blob and tree objects stored there. + local.env( + GIT_OBJECT_DIRECTORY=str( + subproject_top / subproject_gitpath("objects").strip() ) - for line in status: - # Filter merge rejections (part 1/2) - if not line.startswith(("?? ", "!! ")): - continue - # Remove prefix - fname = line[3:] - # Normalize name - fname = normalize_git_path(fname) - # Filter merge rejections (part 2/2) - if not fname.endswith(".rej"): - continue - # Remove ".rej" suffix - fname = fname[:-4] - # Undo possible non-rejected chunks - git( + ), + ): + # Remove all files present in the new copy but missing in the current + # subproject if they match a skip-if-exists pattern and are present in + # the old copy. This way, `git merge` will re-create files that were + # deleted in the subproject but match a skip-if-exists pattern. + old_copy_files = set( + path + for entry in scantree(".", follow_symlinks=False) + if (path := Path(entry.path)).parts[0] != ".git" + ) + for path in new_copy_files - subproject_paths: + if self.match_skip(path) and path in old_copy_files: + path.unlink() + + # Stage all files including Git-ignored ones. + git("add", "-f", ".") + # Make a commit to run Git hooks if applicable. + self._git_commit() + + # Get the Git tree ID. + tree_id_old = git("rev-parse", "--verify", "HEAD^{tree}").strip() + + if not self.pretend: + with local.cwd(subproject_top): + # Create the Git tree of the subproject including changes from + # pre-migration tasks. + git("add", "-u", ".") + tree_id_head = git("write-tree").strip() + + git_commit_tree = git["commit-tree", "--no-gpg-sign"] + # Create a synthetic commit graph that reflects the relationship + # between the fresh copy based on the old template ... + commit_id_old = git_commit_tree( + "-m", "old copy", tree_id_old + ).strip() + # ... and the fresh copy based on the new template ... + commit_id_new = git_commit_tree( + "-m", "new copy", "-p", commit_id_old, tree_id_new + ).strip() + # ... and the current subproject. + commit_id_head = git_commit_tree( + "-m", "current subproject", "-p", commit_id_old, tree_id_head + ).strip() + + with ExitStack() as stack: + current_branch = git( + "rev-parse", "--abbrev-ref", "HEAD" + ).strip() + + git_checkout = git[ # Ignore hooks to avoid errors from them or # issues when .pre-commit-config.yaml is changed - "-c", - f"core.hooksPath={os.devnull}", - "checkout", - "--", - fname, + "-c", f"core.hooksPath={os.devnull}", "checkout" + ] + + # Check out the `HEAD` commit as detached `HEAD`. + git_checkout(commit_id_head) + # Defer reattaching `HEAD` to the previously checked out branch. + # This way, `HEAD` will point to the previous branch ref again + # while the working tree and index remain exactly as-is, + # retaining possible merge conflicts and mid-merge state. + stack.callback( + lambda: Path( + git("rev-parse", "--absolute-git-dir").strip(), "HEAD" + ).write_text(f"ref: refs/heads/{current_branch}\n", "utf-8") ) - # 3-way-merge the file directly - git( - "merge-file", - "-L", - "before updating", - "-L", - "last update", - "-L", - "after updating", - fname, - old_path / fname, - new_path / fname, - retcode=None, + + ref = "copier/after-updating" + # Create a namespaced ref that points to the synthetic commit of + # the fresh copy based on the new template. When using this ref + # in the `git merge` command below, it is used as the label of + # the conflict marker for incoming changes. + git("update-ref", f"refs/{ref}", commit_id_new) + # Defer deleting the namespaced ref. + stack.callback(lambda: git("update-ref", "-d", f"refs/{ref}")) + + # Perform a 3-way squash-merge without committing the changes. + git("merge", "--no-commit", "--squash", ref, retcode=None) + + # Always use the answers file from the fresh copy based on the + # new template. + git_checkout( + "-f", + tree_id_new, + "--", + subproject_subdir / self.answers_relpath, ) - # Remove rejection witness - Path(f"{fname}.rej").unlink() - # The 3-way merge might have resolved conflicts automatically, - # so we need to check if the file contains conflict markers - # before storing the file name for marking it as unmerged after - # the loop. - with Path(fname).open("rb") as conflicts_candidate: - if any( - line.rstrip() - in { - b"<<<<<<< before updating", - b">>>>>>> after updating", - } - for line in conflicts_candidate - ): - conflicted.append(fname) - # We ran `git merge-file` outside of a regular merge operation, - # which means no merge conflict is recorded in the index. - # Only the usual stage 0 is recorded, with the hash of the current - # version. - # We therefore update the index with the missing stages: - # 1 = current (before updating) - # 2 = base (last update) - # 3 = other (after updating) - # See this SO post: https://stackoverflow.com/questions/79309642/ - # and Git docs: https://git-scm.com/docs/git-update-index#_using_index_info. - if conflicted: - input_lines = [] - for line in ( - git("ls-files", "--stage", *conflicted).strip().splitlines() - ): - perms_sha_mode, path = line.split("\t") - perms, sha, _ = perms_sha_mode.split() - input_lines.append(f"0 {'0' * 40}\t{path}") - input_lines.append(f"{perms} {sha} 2\t{path}") - with suppress(ProcessExecutionError): - # The following command will fail - # if the file did not exist in the previous version. - old_sha = git( - "hash-object", - "-w", - old_path / normalize_git_path(path), - ).strip() - input_lines.append(f"{perms} {old_sha} 1\t{path}") - with suppress(ProcessExecutionError): - # The following command will fail - # if the file was deleted in the latest version. - new_sha = git( - "hash-object", - "-w", - new_path / normalize_git_path(path), - ).strip() - input_lines.append(f"{perms} {new_sha} 3\t{path}") - ( - git["update-index", "--index-info"] - << "\n".join(input_lines) - )() - # Trigger recursive removal of deleted files in last template version - _remove_old_files(subproject_top, compared) - - # Run post-migration tasks + + if not self.skip_tasks: + with Phase.use(Phase.TASKS): + self._execute_tasks(self.template.tasks) + + # Run post-migration tasks. with Phase.use(Phase.MIGRATE): self._execute_tasks( self.template.migration_tasks("after", self.subproject.template) # type: ignore[arg-type] ) - def _git_initialize_repo(self) -> None: - """Initialize a git repository in the current directory.""" - git = get_git() - git("init", retcode=None) - git("add", ".") - self._git_commit() - def _git_commit(self, message: str = "dumb commit") -> None: git = get_git() # 1st commit could fail if any pre-commit hook reformats code # 2nd commit uses --no-verify to disable pre-commit-like checks - git( - "commit", - "--allow-empty", - "-am", - f"{message} 1", - "--no-gpg-sign", - retcode=None, - ) - git( - "commit", - "--allow-empty", - "-am", - f"{message} 2", - "--no-gpg-sign", - "--no-verify", - ) + cmd = git["commit", "--allow-empty", "-am", message, "--no-gpg-sign"] + try: + cmd() + except ProcessExecutionError: + cmd("--no-verify") def run_copy( @@ -1822,8 +1707,6 @@ def run_update( overwrite: bool = False, pretend: bool = False, quiet: bool = False, - conflict: Literal["inline", "rej"] = "inline", - context_lines: PositiveInt = 3, unsafe: bool = False, skip_answered: bool = False, skip_tasks: bool = False, @@ -1853,8 +1736,6 @@ def run_update( overwrite=overwrite, pretend=pretend, quiet=quiet, - conflict=conflict, - context_lines=context_lines, unsafe=unsafe, skip_answered=skip_answered, skip_tasks=skip_tasks, @@ -1907,50 +1788,3 @@ def get_update_data( latest_version = str(worker.template.version) return (update_available, current_version, latest_version) - - -def _remove_old_files(prefix: Path, cmp: dircmp[str], rm_common: bool = False) -> None: - """Remove files and directories only found in "old" template. - - This is an internal helper method used to process a comparison of 2 - directories, where the left one is considered the "old" one, and the - right one is the "new" one. - - Then, it will recursively try to remove anything that is only in the old - directory. - - Args: - prefix: - Where we start removing. It can be different from the directories - being compared. - cmp: - The comparison result. - rm_common: - Should we remove common files and directories? - """ - # Gather files and dirs to remove - to_rm = [] - subdirs = {} - with suppress(NotADirectoryError, FileNotFoundError): - to_rm = cmp.left_only - if rm_common: - to_rm += cmp.common_files + cmp.common_dirs - subdirs = cmp.subdirs - # Remove files found only in old template copy - for name in to_rm: - target = prefix / name - if target.is_file(): - target.unlink() - else: - # Recurse in dirs totally removed in latest template - _remove_old_files(target, dircmp(Path(cmp.left, name), target), True) - # Remove subdir if it ends empty - with suppress(OSError): - target.rmdir() # Raises if dir not empty - # Recurse - for key, value in subdirs.items(): - subdir = prefix / key - _remove_old_files(subdir, value) - # Remove subdir if it ends empty - with suppress(OSError): - subdir.rmdir() # Raises if dir not empty diff --git a/copier/_tools.py b/copier/_tools.py index c4f25c2c5..f1afaf6d6 100644 --- a/copier/_tools.py +++ b/copier/_tools.py @@ -5,7 +5,6 @@ import errno import os import platform -import re import stat import sys from collections.abc import Callable, Iterator @@ -19,7 +18,6 @@ import colorama from packaging.version import Version -from pathspec.patterns.gitwildmatch import GitWildMatchPattern from pydantic import StrictBool colorama.just_fix_windows_console() @@ -178,92 +176,6 @@ def handle_remove_readonly( raise -_re_whitespace = re.compile(r"^\s+|\s+$") - - -def normalize_git_path(path: str) -> str: - r"""Convert weird characters returned by Git to normal UTF-8 path strings. - - A filename like âñ will be reported by Git as "\\303\\242\\303\\261" (octal - notation). - Similarly, a filename like "foo\bar" will be reported as "\tfoo\\b\nar". - This can be disabled with `git config core.quotepath off`. - - Args: - path: The Git path to normalize. - - Returns: - str: The normalized Git path. - """ - # Remove surrounding quotes - if path[0] == path[-1] == '"': - path = path[1:-1] - # Repair double-quotes - path = path.replace('\\"', '"') - # Unescape escape characters - path = path.encode("latin-1", "backslashreplace").decode("unicode-escape") - # Convert octal to utf8 - return path.encode("latin-1", "backslashreplace").decode("utf-8") - - -def escape_git_path(path: str) -> str: - """Escape paths that will be used as literal gitwildmatch patterns. - - If the path was returned by a Git command, it should be unescaped completely. - ``normalize_git_path`` can be used for this purpose. - - Args: - path: The Git path to escape. - - Returns: - str: The escaped Git path. - """ - # Prior to PathSpec v1.1.0, `GitWildMatchPattern.escape` does not escape backslashes - # or trailing whitespace. - # TODO: Remove this workaround when support for PathSpec prior to v1.1.0 is dropped. - if GitWildMatchPattern.escape("\\") == "\\": - path = path.replace("\\", "\\\\") - path = GitWildMatchPattern.escape(path) - return _re_whitespace.sub( - lambda match: "".join(f"\\{whitespace}" for whitespace in match.group()), - path, - ) - - -def get_git_objects_dir(path: Path) -> Path: - """Get the absolute path of a Git repository's objects directory.""" - # FIXME: A lazy import is currently necessary to avoid circular imports with - # `errors.py`. - from ._vcs import get_git - - git = get_git() - return path.joinpath( - git( - "-C", - path, - "rev-parse", - "--git-path", - "objects", - ).strip() - ).absolute() - - -def set_git_alternates(*repos: Path, path: Path = Path()) -> None: - """Set Git alternates to borrow Git objects from other repositories. - - Alternates are paths of other repositories' object directories written to - `$GIT_DIR/objects/info/alternates` and delimited by the newline character. - - Args: - *repos: The paths of repositories from which to borrow Git objects. - path: The path of the repository where to set Git alternates. Defaults - to the current working directory. - """ - alternates_file = get_git_objects_dir(path) / "info" / "alternates" - alternates_file.parent.mkdir(parents=True, exist_ok=True) - alternates_file.write_bytes(b"\n".join(map(bytes, map(get_git_objects_dir, repos)))) - - def scantree(path: str, follow_symlinks: bool) -> Iterator[os.DirEntry[str]]: """A recursive extension of `os.scandir`.""" for entry in os.scandir(path): diff --git a/docs/configuring.md b/docs/configuring.md index a32f40123..a05f84d61 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -835,43 +835,6 @@ will delete that folder. Copier will never delete the folder if it didn't create it. For this reason, when running `copier update`, this setting has no effect. -!!! info - - Not supported in `copier.yml`. - -### `conflict` - -- Format: `Literal["rej", "inline"]` -- CLI flags: `-o`, `--conflict` (only available in `copier update` subcommand) -- Default value: `inline` - -When updating a project, sometimes Copier doesn't know what to do with a diff code hunk. -This option controls the output format if this happens. Using `rej`, creates `*.rej` -files that contain the unresolved diffs. The `inline` option (default) includes the diff -code hunk in the file itself, similar to the behavior of `git merge`. - -!!! info - - Not supported in `copier.yml`. - -### `context_lines` - -- Format: `Int` -- CLI flags: `-c`, `--context-lines` (only available in `copier update` subcommand) -- Default value: `1` - -During a project update, Copier needs to compare the template evolution with the -subproject evolution. This way, it can detect what changed, where and how to merge those -changes. [Refer here for more details on this process](updating.md). - -The more lines you use, the more accurate Copier will be when detecting conflicts. But -you will also have more conflicts to solve by yourself. FWIW, Git uses 3 lines by -default. - -The less lines you use, the less conflicts you will have. However, Copier will not be so -accurate and could even move lines around if the file it's comparing has several similar -code chunks. - !!! info Not supported in `copier.yml`. diff --git a/docs/creating.md b/docs/creating.md index 5ffd33169..75b27fa9a 100644 --- a/docs/creating.md +++ b/docs/creating.md @@ -108,8 +108,6 @@ Attributes: | ------------------ | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `answers_file` | `PurePath` | The path for [the answers file](configuring.md#the-copier-answersyml-file) relative to `dst_path`.
See the [`answers_file`](configuring.md#answers_file) setting for related information. | | `cleanup_on_error` | `bool` | When `True`, delete `dst_path` if there's an error.
See the [`cleanup_on_error`](configuring.md#cleanup_on_error) setting for related information. | -| `conflict` | `Literal["inline", "rej"]` | The output format of a diff code hunk when [updating][updating-a-project] a file yields conflicts.
See the [`conflict`](configuring.md#conflict) setting for related information. | -| `context_lines` | `PositiveInt` | Lines of context to consider when solving conflicts in updates.
See the [`context_lines`](configuring.md#context_lines) setting for related information. | | `data` | `dict[str, Any]` | Answers to the questionnaire, defined in the template, provided via CLI (`-d,--data`) or API (`data`).
See the [`data`](configuring.md#data) setting for related information.
⚠️ May contain secret answers. | | `defaults` | `bool` | When `True`, use default answers to questions.
See the [`defaults`](configuring.md#defaults) setting for related information. | | `dst_path` | `PurePath` | Destination path where to render the subproject.
⚠️ When [updating a project](updating.md), it may be a temporary directory, as Copier's update algorithm generates fresh copies using the old and new template versions in temporary locations. | diff --git a/docs/updating.md b/docs/updating.md index c4613b57d..423d126cf 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -38,15 +38,11 @@ other Git ref you want. When updating, Copier will do its best to respect your project evolution by using the answers you provided when copied last time. However, sometimes it's impossible for -Copier to know what to do with a diff code hunk. In those cases, copier handles the -conflict in one of two ways, controlled with the `--conflict` option: - -- `--conflict rej`: Creates a separate `.rej` file for each file with conflicts. These - files contain the unresolved diffs. -- `--conflict inline` (default): Updates the file with conflict markers. This is quite - similar to the conflict markers created when a `git merge` command encounters a - conflict. For more information, see the "Checking Out Conflicts" section of the - [`git` documentation](https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging). +Copier to know how to merge the changes from the evolved template into the evolved +project. In those cases, Copier updates a conflicting file with conflict markers in the +same ways as a `git merge` command encounters conflicts; in fact, Copier uses +`git merge` internally. For more information, see the "Checking Out Conflicts" section +of the [`git` documentation](https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging). If the update results in conflicts, _you should review those manually_ before committing. @@ -56,13 +52,11 @@ Git history, but if you aren't careful, it's easy to make mistakes. That's why the recommended way to prevent these mistakes is to add a [pre-commit](https://pre-commit.com/) (or equivalent) hook that forbids committing -conflict files or markers. The recommended hook configuration depends on the `conflict` -setting you use. +conflict markers. ## Preventing Commit of Merge Conflicts -If you use `--conflict inline` (the default) then you need to check for conflicts -markers in your files: +You need to check for conflict markers in your files: ```yaml title=".pre-commit-config.yaml" repos: @@ -74,29 +68,6 @@ repos: args: [--assume-in-merge] ``` -If you use `--conflict rej` then you need to review and remove all generated `.rej` -files: - -```yaml title=".pre-commit-config.yaml" -repos: - - repo: local - hooks: - # Prevent committing .rej files - - id: forbidden-files - name: forbidden files - entry: - found Copier update rejection files; review and remove them before - merging. - language: fail - files: "\\.rej$" -``` - -!!! note - - For projects that use both `rej` and `inline` depending on each user's preference, - you can add both hooks to your `pre-commit-config.yaml` file, making sure that no - unresolved merge conflicts are committed. - ## Never change the answers file manually !!! important @@ -151,53 +122,55 @@ graph TD %% nodes ---------------------------------------------------------- template_repo("template repository") -template_current("/tmp/template
(current tag)") -template_latest("/tmp/template
(latest tag)") +template_current("/tmp/template-old
(current tag)") +template_latest("/tmp/template-new
(latest tag)") -project_regen("/tmp/project
(fresh, current version)") +project_regen_current("/tmp/project-old
(fresh, current version)") +project_regen_latest("/tmp/project-new
(fresh, latest version)") project_current("current project") project_half("half migrated
project") project_updated("updated project") -project_applied("updated project
(diff applied)") project_full("fully updated
and migrated project") -update["update current
project in-place
(prompting)
+ run tasks again"] -compare["compare to get diff"] -apply["apply diff"] - -diff("diff") +update["3-way merge
& run tasks again"] +regen_current["generate and run tasks"] +regen_latest["generate and run tasks"] %% edges ---------------------------------------------------------- template_repo --> |git clone| template_current template_repo --> |git clone| template_latest - template_current --> |generate and run tasks| project_regen - project_current --> compare + template_current --> regen_current + project_current .-> |use answers| regen_current + regen_current --> project_regen_current + template_latest --> regen_latest + regen_latest --> project_regen_latest project_current --> |apply pre-migrations| project_half - project_regen --> compare + project_half .-> |use answers| regen_latest project_half --> update - template_latest --> update +project_regen_current --> update + project_regen_latest --> update update --> project_updated - compare --> diff - diff --> apply - project_updated --> apply - apply --> project_applied - project_applied --> |apply post-migrations| project_full + project_updated --> |apply post-migrations| project_full %% style ---------------------------------------------------------- classDef blackborder stroke:#000; -class compare,update,apply blackborder; +class regen_current,regen_latest,update blackborder; ``` As you can see here, `copier` does several things: -- It regenerates a fresh project from the current template version. -- Then, it compares both version to get the diff from "fresh project" to "current - project". -- Now, it applies pre-migrations to your project, and updates the current project with - the latest template changes (asking for confirmation). -- Finally, it re-applies the previously obtained diff and then runs the - post-migrations. +- Regenerate the project fresh from the **current** template version, using the + project's existing answers – this becomes the merge-base. +- Regenerate the project fresh from the **latest** template version, using the same + answers (with pre-migrations applied to the project beforehand). +- Build a synthetic Git commit graph from these three states: the current-version + regeneration (common ancestor), the latest-version regeneration, and the actual + current project. +- Perform a Git 3-way merge (using `git merge`) of the latest-version regeneration + into the current project, using the current-version regeneration as their common + ancestor – conflicts are marked like any normal `git merge` conflict. +- Run post-migrations on the merged result to produce the fully updated project. ### Handling of deleted paths @@ -242,10 +215,15 @@ branch. The following strategies won't work: - `git checkout ` – _error: you need to resolve your current index first_ - `git checkout .` – _error: path '<filename>' is unmerged_ -- `git merge --abort` – _fatal: There is no merge to abort (MERGE_HEAD missing)_ Here is what you can do using Git in the terminal to throw away all changes: +```shell +git merge --abort +``` + +or + ```shell git reset # throw away merge conflict information git checkout . # restore modified files diff --git a/tests/helpers.py b/tests/helpers.py index 1a6535ed8..0c9ffc4a3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -193,3 +193,28 @@ def git_init(message: str = "hello world") -> None: git("init") git("add", ".") git("commit", "-m", message) + + +def normalize_git_path(path: str) -> str: + r"""Convert weird characters returned by Git to normal UTF-8 path strings. + + A filename like âñ will be reported by Git as "\\303\\242\\303\\261" (octal + notation). + Similarly, a filename like "foo\bar" will be reported as "\tfoo\\b\nar". + This can be disabled with `git config core.quotepath off`. + + Args: + path: The Git path to normalize. + + Returns: + str: The normalized Git path. + """ + # Remove surrounding quotes + if path[0] == path[-1] == '"': + path = path[1:-1] + # Repair double-quotes + path = path.replace('\\"', '"') + # Unescape escape characters + path = path.encode("latin-1", "backslashreplace").decode("unicode-escape") + # Convert octal to utf8 + return path.encode("latin-1", "backslashreplace").decode("utf-8") diff --git a/tests/test_cli.py b/tests/test_cli.py index bd3566765..25d1afc7a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -481,9 +481,6 @@ def test_update_help(capsys: pytest.CaptureFixture[str]) -> None: --ask VALUE:str Ask the questions matching the given glob- pattern, even if they would be skipped by other options; may be given multiple times - -c, --context-lines VALUE:int Lines of context to use for detecting - conflicts. Increase for accuracy, decrease - for resilience.; the default is 3 -d, --data VARIABLE=VALUE:str Make VARIABLE available as VALUE when rendering the template; may be given multiple times @@ -493,9 +490,6 @@ def test_update_help(capsys: pytest.CaptureFixture[str]) -> None: -l, -f, --defaults Use default answers to questions, which might be null if not specified. -n, --pretend Run but do not make any changes - -o, --conflict VALUE:{rej, inline} Behavior on conflict: Create .rej files, or - add inline conflict markers.; the default is - inline -q, --quiet Suppress status output -r, --vcs-ref VALUE:str Git reference to checkout in `template_src`. If you do not specify it, it will try to diff --git a/tests/test_dirty_local.py b/tests/test_dirty_local.py index a5f443104..c2488c9bc 100644 --- a/tests/test_dirty_local.py +++ b/tests/test_dirty_local.py @@ -305,7 +305,7 @@ def test_parallel_projects_in_subdirs( assert (dst / "file.txt").read_text() == f"Updated {dst.name}" # Verify subdirectories are both now dirty - expected = "M project1/.copier-answers.yml\n M project1/file.txt\n M project2/.copier-answers.yml\n M project2/file.txt" + expected = "M project1/.copier-answers.yml\nM project1/file.txt\nM project2/.copier-answers.yml\nM project2/file.txt" with local.cwd(parent): assert git("status", "--porcelain").strip() == expected diff --git a/tests/test_subdirectory.py b/tests/test_subdirectory.py index 3c59f015d..8a44acf4b 100644 --- a/tests/test_subdirectory.py +++ b/tests/test_subdirectory.py @@ -1,6 +1,5 @@ from pathlib import Path from textwrap import dedent -from typing import Literal import pytest from plumbum import local @@ -178,34 +177,8 @@ def test_update_subdirectory_from_root_path( assert (dst / "subfolder" / "file1").read_text() == "version 2\nhello\na1\nbye\n" -@pytest.mark.parametrize( - "conflict, readme, expect_reject", - [ - ( - "rej", - "upstream version 2\n", - True, - ), - ( - "inline", - dedent( - """\ - <<<<<<< before updating - downstream version 1 - ======= - upstream version 2 - >>>>>>> after updating - """ - ), - False, - ), - ], -) def test_new_version_uses_subdirectory( tmp_path_factory: pytest.TempPathFactory, - conflict: Literal["rej", "inline"], - readme: str, - expect_reject: bool, ) -> None: # Template in v1 doesn't have a _subdirectory; # in v2 it moves all things into a subdir and adds that key to copier.yml. @@ -260,15 +233,22 @@ def test_new_version_uses_subdirectory( git("tag", "v2") # Finally, update the generated project - copier.run_update(dst_path=dst, defaults=True, overwrite=True, conflict=conflict) + copier.run_update(dst_path=dst, defaults=True, overwrite=True) assert load_answersfile_data(dst).get("_commit") == "v2" # Assert that the README still exists, and the conflicts were handled # correctly. assert (dst / "README.md").exists() - assert (dst / "README.md").read_text().splitlines() == readme.splitlines() - assert (dst / "README.md.rej").exists() == expect_reject + assert (dst / "README.md").read_text() == dedent( + """\ + <<<<<<< HEAD + downstream version 1 + ======= + upstream version 2 + >>>>>>> copier/after-updating + """ + ) # Also assert the subdirectory itself was not rendered assert not (dst / subdir).exists() diff --git a/tests/test_tools.py b/tests/test_tools.py index 01587f71e..f2dfce3fb 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -5,9 +5,9 @@ import pytest from poethepoet.app import PoeThePoet -from copier._tools import cast_to_bool, normalize_git_path +from copier._tools import cast_to_bool -from .helpers import git +from .helpers import git, normalize_git_path def test_types() -> None: diff --git a/tests/test_updatediff.py b/tests/test_updatediff.py index 08f888489..06b53a59a 100644 --- a/tests/test_updatediff.py +++ b/tests/test_updatediff.py @@ -5,7 +5,6 @@ from pathlib import Path from shutil import rmtree from textwrap import dedent -from typing import Literal import pexpect import pytest @@ -15,7 +14,6 @@ from copier._cli import CopierApp from copier._main import Worker, run_copy, run_update -from copier._tools import normalize_git_path from copier._types import VcsRef from copier._user_data import load_answersfile_data from copier._vcs import get_git_version @@ -31,6 +29,7 @@ git, git_init, git_save, + normalize_git_path, ) @@ -353,13 +352,6 @@ def test_commit_hooks_respected(tmp_path_factory: pytest.TempPathFactory) -> Non rev: v3.12.0 hooks: - id: commitizen - - repo: local - hooks: - - id: forbidden-files - name: forbidden files - entry: found forbidden files; remove them - language: fail - files: "\\.rej$" """ ), "life.yml.tmpl": ( @@ -416,14 +408,7 @@ def test_commit_hooks_respected(tmp_path_factory: pytest.TempPathFactory) -> Non git("commit", "-m", "feat: commit 2") git("tag", "v2") # Update subproject to v2 - run_update( - dst_path=dst1, - defaults=True, - overwrite=True, - conflict="rej", - context_lines=1, - unsafe=True, - ) + run_update(dst_path=dst1, defaults=True, overwrite=True, unsafe=True) with local.cwd(dst1): git("commit", "-am", "feat: copied v2") assert life.read_text() == dedent( @@ -436,59 +421,6 @@ def test_commit_hooks_respected(tmp_path_factory: pytest.TempPathFactory) -> Non Line 5: bye bye world """ ) - # No .rej files created (update diff was smart) - assert not git("status", "--porcelain") - # Subproject evolves - life.write_text( - dedent( - """\ - Line 1: hello world - Line 2: grow up - Line 2.5: make friends - Line 3: grog - Line 4: grow old - Line 4.5: no more work - Line 5: bye bye world - """ - ) - ) - git("commit", "-am", "chore: subproject is evolved") - # A new subproject appears, which is a shallow clone of the 1st one. - # Using file:// prefix to allow local shallow clones. - git("clone", "--depth=1", f"file://{dst1}", dst2) - with local.cwd(dst2): - # Subproject re-updates just to change some values - run_update( - data={"what": "study"}, - defaults=True, - overwrite=True, - conflict="rej", - context_lines=1, - unsafe=True, - ) - git("commit", "-am", "chore: re-updated to change values after evolving") - # Subproject evolution was respected up to sane possibilities. - # In an ideal world, this file would be exactly the same as what's written - # a few lines above, just changing "grog" for "study". However, that's nearly - # impossible to achieve, because each change hunk needs at least 1 line of - # context to let git apply that patch smartly, and that context couldn't be - # found because we changed data when updating, so the sanest thing we can - # do is to provide a .rej file to notify those - # unresolvable diffs. OTOH, some other changes are be applied. - # If some day you are able to produce that ideal result, you should be - # happy to modify these asserts. - assert life.read_text() == dedent( - """\ - Line 1: hello world - Line 2: grow up - Line 3: study - Line 4: grow old - Line 4.5: no more work - Line 5: bye bye world - """ - ) - # This time a .rej file is unavoidable - assert Path(f"{life}.rej").is_file() def test_post_checkout_hook_ignored(tmp_path_factory: pytest.TempPathFactory) -> None: @@ -731,8 +663,22 @@ def test_skip_update_deleted( ), "file with whitespace", " leading_whitespace", - "trailing_whitespace ", - " multi_whitespace ", + pytest.param( + "trailing_whitespace ", + # https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/file-folder-name-whitespace-characters#summary + marks=pytest.mark.skipif( + platform.system() == "Windows", + reason="OS filesystem strips trailing whitespaces", + ), + ), + pytest.param( + " multi_whitespace ", + # https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/file-folder-name-whitespace-characters#summary + marks=pytest.mark.skipif( + platform.system() == "Windows", + reason="OS filesystem strips trailing whitespaces", + ), + ), pytest.param( "\tother_whitespace\t\\t", marks=pytest.mark.skipif( @@ -819,7 +765,11 @@ def test_update_deleted_path( run_update(dst, overwrite=True) assert dont_wildmatch.exists() assert dont_wildmatch.read_text() == "baz" - assert not updated_file.exists() + assert updated_file.exists() + assert updated_file.read_text() == "bar" + assert git( + "-C", dst, "ls-files", "-z", "--unmerged", "--format=%(stage) %(path)" + ).strip() == (f"1 {file_name}\x003 {file_name}\x00") def test_update_deleted_path_not_used_as_pattern( @@ -959,8 +909,8 @@ def test_file_removed(tmp_path_factory: pytest.TempPathFactory) -> None: with pytest.raises( UserMessageError, match="Enable overwrite to update a subproject." ): - run_update(conflict="rej") - run_update(conflict="rej", overwrite=True) + run_update() + run_update(overwrite=True) # Check what must still exist assert (dst / ".copier-answers.yml").is_file() assert (dst / "I.txt").is_file() @@ -1050,7 +1000,7 @@ def test_update_inline_changed_answers_and_questions( git("commit", "-am2") # Update from template, inline, with answer changes if interactive: - tui = spawn(COPIER_PATH + ("update", "--conflict=inline")) + tui = spawn(COPIER_PATH + ("update",)) tui.expect_exact("b (bool)") tui.expect_exact("(Y/n)") tui.sendline() @@ -1059,27 +1009,22 @@ def test_update_inline_changed_answers_and_questions( tui.send("y") tui.expect_exact(pexpect.EOF) else: - run_update( - data={"c": True}, defaults=True, overwrite=True, conflict="inline" - ) + run_update(data={"c": True}, defaults=True, overwrite=True) assert Path("content").read_text() == dedent( """\ aaa bbb - <<<<<<< before updating + <<<<<<< HEAD jjj ======= ccc - >>>>>>> after updating + >>>>>>> copier/after-updating zzz """ ) -@pytest.mark.parametrize("conflict", ["rej", "inline"]) -def test_update_in_repo_subdirectory( - tmp_path_factory: pytest.TempPathFactory, conflict: Literal["rej", "inline"] -) -> None: +def test_update_in_repo_subdirectory(tmp_path_factory: pytest.TempPathFactory) -> None: src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) subdir = Path("subdir") @@ -1117,50 +1062,24 @@ def test_update_in_repo_subdirectory( git("commit", "-m2") git("tag", "v2") - run_update(dst / subdir, overwrite=True, conflict=conflict) + run_update(dst / subdir, overwrite=True) assert (dst / subdir / ".copier-answers.yml").is_file() assert (dst / subdir / "version.txt").is_file() - if conflict == "rej": - assert (dst / subdir / "version.txt").read_text() == "v2" - assert (dst / subdir / "version.txt.rej").is_file() - else: - assert (dst / subdir / "version.txt").read_text() == dedent( - """\ - <<<<<<< before updating - v1 edited - ======= - v2 - >>>>>>> after updating - """ - ) + assert (dst / subdir / "version.txt").read_text() == dedent( + """\ + <<<<<<< HEAD + v1 edited + ======= + v2 + >>>>>>> copier/after-updating + """ + ) -@pytest.mark.parametrize( - "context_lines", - [ - pytest.param( - 1, - marks=pytest.mark.xfail( - raises=AssertionError, - reason="Not enough context lines to resolve the conflict.", - strict=True, - ), - ), - pytest.param( - 2, - marks=pytest.mark.xfail( - raises=AssertionError, - reason="Not enough context lines to resolve the conflict.", - strict=True, - ), - ), - 3, - ], -) @pytest.mark.parametrize("api", [True, False]) -def test_update_needs_more_context( - tmp_path_factory: pytest.TempPathFactory, context_lines: int, api: bool +def test_update_has_enough_context( + tmp_path_factory: pytest.TempPathFactory, api: bool ) -> None: src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) # Create a template where some code blocks are similar @@ -1257,14 +1176,9 @@ def function_two(): git("tag", "v2") # Update the project if api: - run_update(dst, overwrite=True, conflict="inline", context_lines=context_lines) + run_update(dst, overwrite=True) else: - COPIER_CMD( - "update", - str(dst), - "--conflict=inline", - f"--context-lines={context_lines}", - ) + COPIER_CMD("update", str(dst)) # Check the update result assert (dst / "sample.py").read_text() == dedent( """\ @@ -1362,7 +1276,7 @@ def test_conflicted_files_are_marked_unmerged( git("tag", "v2") # Finally, update the generated project - run_update(dst_path=dst, defaults=True, overwrite=True, conflict="inline") + run_update(dst_path=dst, defaults=True, overwrite=True) assert load_answersfile_data(dst).get("_commit") == "v2" # Assert that the file still exists, has inline conflict markers, @@ -1371,11 +1285,11 @@ def test_conflicted_files_are_marked_unmerged( expected_contents = dedent( """\ - <<<<<<< before updating + <<<<<<< HEAD upstream version 1 + downstream ======= upstream version 2 - >>>>>>> after updating + >>>>>>> copier/after-updating """ ) assert (dst / filename).read_text().splitlines() == expected_contents.splitlines() @@ -1436,7 +1350,7 @@ def test_3way_merged_files_without_conflicts_are_not_marked_unmerged( git("tag", "v2") # Finally, update the generated project - run_update(dst_path=dst, defaults=True, overwrite=True, conflict="inline") + run_update(dst_path=dst, defaults=True, overwrite=True) assert load_answersfile_data(dst).get("_commit") == "v2" # Assert that the file still exists, does not have inline conflict markers, @@ -1509,14 +1423,14 @@ def test_update_with_new_file_in_template_and_project( git("commit", "-m", "v2") git("tag", "v2") - run_update(dst_path=dst, defaults=True, overwrite=True, conflict="inline") + run_update(dst_path=dst, defaults=True, overwrite=True) assert load_answersfile_data(dst).get("_commit") == "v2" assert (dst / ".gitlab-ci.yml").read_text() == dedent( """\ tests: stage: test script: - <<<<<<< before updating + <<<<<<< HEAD - ./test.sh pages: @@ -1525,7 +1439,7 @@ def test_update_with_new_file_in_template_and_project( - ./deploy.sh ======= - ./test.sh --slow - >>>>>>> after updating + >>>>>>> copier/after-updating """ ) @@ -1631,13 +1545,11 @@ def test_update_with_new_file_in_template_and_project_via_migration( git("commit", "-m", "v2") git("tag", "v2") - run_update( - dst_path=dst, defaults=True, overwrite=True, conflict="inline", unsafe=True - ) + run_update(dst_path=dst, defaults=True, overwrite=True, unsafe=True) assert load_answersfile_data(dst).get("_commit") == "v2" assert (dst / ".gitlab-ci.yml").read_text() == dedent( """\ - <<<<<<< before updating + <<<<<<< HEAD tests: stage: test script: @@ -1650,7 +1562,7 @@ def test_update_with_new_file_in_template_and_project_via_migration( ======= include: - local: .gitlab/ci/main.yml - >>>>>>> after updating + >>>>>>> copier/after-updating """ ) assert (dst / ".gitlab" / "ci" / "main.yml").read_text() == dedent( @@ -1658,7 +1570,7 @@ def test_update_with_new_file_in_template_and_project_via_migration( tests: stage: test script: - <<<<<<< before updating + <<<<<<< HEAD - ./test.sh pages: @@ -1667,7 +1579,7 @@ def test_update_with_new_file_in_template_and_project_via_migration( - ./deploy.sh ======= - ./test.sh --slow - >>>>>>> after updating + >>>>>>> copier/after-updating """ ) @@ -2086,16 +1998,16 @@ def test_conflict_on_update_with_unicode_in_content( ) git("commit", "-am2") # Update from template, inline, with answer changes - run_update(data={"c": True}, defaults=True, overwrite=True, conflict="inline") + run_update(data={"c": True}, defaults=True, overwrite=True) assert Path("content").read_text(encoding="utf-8") == dedent( """\ aaa🐍 bbb🐍 - <<<<<<< before updating + <<<<<<< HEAD jjj🐍 ======= ccc🐍 - >>>>>>> after updating + >>>>>>> copier/after-updating zzz🐍 """ ) @@ -2338,35 +2250,15 @@ def test_update_propagates_executable_bit_addition( # On disk, the bit must be set after the update. assert (dst / "launcher.sh").stat().st_mode & 0o111 != 0 - with local.cwd(dst): - # TODO: simplify with ``--format %(objectmode)`` once the minimum - # git version is raised to 2.38+ (--format cannot be combined with - # --stage; see git-ls-files(1)). - mode = git("ls-files", "--stage", "--", "launcher.sh").strip().split()[0] - # Preserve the leading space — porcelain columns are index (XY) and - # worktree, and a leading space distinguishes " M" (unstaged) from - # "M " (staged). ``.strip()`` would swallow the space. - status = git("status", "--porcelain", "--", "launcher.sh").rstrip("\n") - if file_mode: - # With ``core.fileMode=true``, copier must NOT stage the index mode - # change — git picks up the on-disk ``chmod`` as an unstaged - # modification, matching copier's normal behavior of leaving - # rendered changes unstaged for user review. Guards against - # auto-staging regression. - assert mode == "100644" - assert status.startswith(" M"), f"expected unstaged change, got {status!r}" - # A subsequent ``git add`` records 100755 via git's normal flow. - with local.cwd(dst): - git("add", "--", "launcher.sh") - mode_after_add = ( - git("ls-files", "--stage", "--", "launcher.sh").strip().split()[0] - ) - assert mode_after_add == "100755" - else: - # With ``core.fileMode=false``, git would ignore the on-disk chmod, - # so copier must explicitly call ``git update-index --chmod=+x`` - # to record the new mode in the index. - assert mode == "100755" + # TODO: simplify with ``--format %(objectmode)`` once the minimum + # git version is raised to 2.38+ (--format cannot be combined with + # --stage; see git-ls-files(1)). + mode = ( + git("-C", str(dst), "ls-files", "--stage", "--", "launcher.sh") + .strip() + .split()[0] + ) + assert mode == "100755" @pytest.mark.skipif( @@ -2426,29 +2318,15 @@ def test_update_propagates_executable_bit_removal( run_update(str(dst), defaults=True, overwrite=True) assert (dst / "launcher.sh").stat().st_mode & 0o111 == 0 - with local.cwd(dst): - # TODO: simplify with ``--format %(objectmode)`` once the minimum - # git version is raised to 2.38+ (--format cannot be combined with - # --stage; see git-ls-files(1)). - mode = git("ls-files", "--stage", "--", "launcher.sh").strip().split()[0] - # Preserve the leading space — porcelain columns are index (XY) and - # worktree, and a leading space distinguishes " M" (unstaged) from - # "M " (staged). ``.strip()`` would swallow the space. - status = git("status", "--porcelain", "--", "launcher.sh").rstrip("\n") - if file_mode: - # Same rationale as the addition test: with ``core.fileMode=true`` - # the chmod should be visible to git as unstaged, and copier must - # not pre-stage it via ``git update-index --chmod``. - assert mode == "100755" - assert status.startswith(" M"), f"expected unstaged change, got {status!r}" - with local.cwd(dst): - git("add", "--", "launcher.sh") - mode_after_add = ( - git("ls-files", "--stage", "--", "launcher.sh").strip().split()[0] - ) - assert mode_after_add == "100644" - else: - assert mode == "100644" + # TODO: simplify with ``--format %(objectmode)`` once the minimum + # git version is raised to 2.38+ (--format cannot be combined with + # --stage; see git-ls-files(1)). + mode = ( + git("-C", str(dst), "ls-files", "--stage", "--", "launcher.sh") + .strip() + .split()[0] + ) + assert mode == "100644" @pytest.mark.skipif( @@ -2576,14 +2454,14 @@ def test_update_with_exec_bit_change_and_merge_conflict( git("commit", "-m", "make executable and update content") git("tag", "v2") - run_update(dst_path=dst, defaults=True, overwrite=True, conflict="inline") + run_update(dst_path=dst, defaults=True, overwrite=True) # The conflict must still be registered. If our ``update-index`` # index manipulation were called after stages 1/2/3 were registered, # it would collapse them into stage 0 and the conflict would # silently disappear. Quick human-readable sanity check that # conflict markers were written to the working tree: - assert "<<<<<<< before updating" in (dst / filename).read_text() + assert "<<<<<<< HEAD" in (dst / filename).read_text() # Snapshot the full human-readable ``git status`` — a natural user # view of the post-update state. If ``_sync_git_index_executable_bit`` # ever regresses into stomping conflict stages, this snapshot will @@ -2601,30 +2479,29 @@ def test_update_with_exec_bit_change_and_merge_conflict( # editorconfig-checker-disable assert git("status") == snapshot("""\ On branch main +Changes to be committed: + (use "git restore --staged ..." to unstage) + modified: .copier-answers.yml + Unmerged paths: (use "git restore --staged ..." to unstage) (use "git add ..." to mark resolution) both modified: launcher.sh -Changes not staged for commit: - (use "git add ..." to update what will be committed) - (use "git restore ..." to discard changes in working directory) - modified: .copier-answers.yml - -no changes added to commit (use "git add" and/or "git commit -a") """) assert git("diff", "--", filename) == snapshot("""\ diff --cc launcher.sh index f163f4b,d20125d..0000000 +mode 100644,100755..100755 --- a/launcher.sh +++ b/launcher.sh @@@ -1,1 -1,1 +1,5 @@@ - upstream version 1 + downstream -upstream version 2 -++<<<<<<< before updating +++<<<<<<< HEAD ++upstream version 1 + downstream ++======= ++upstream version 2 -++>>>>>>> after updating +++>>>>>>> copier/after-updating """) # editorconfig-checker-enable From cdc97651c3a308d45e0714b6de5122272521298f Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Wed, 15 Jul 2026 15:31:23 +0200 Subject: [PATCH 2/3] fix(updating)!: do not run tasks after project update Running tasks after an update mixes concerns with post-copy tasks and post-update migration tasks. BREAKING CHANGE: Copier no longer runs tasks after updating a project. Use post-update migration tasks instead. --- copier/_main.py | 4 ---- docs/updating.md | 2 +- tests/test_legacy_migration.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/copier/_main.py b/copier/_main.py index d9874a69e..b1c6b927d 100644 --- a/copier/_main.py +++ b/copier/_main.py @@ -1562,10 +1562,6 @@ def _apply_update(self) -> None: # noqa: C901 subproject_subdir / self.answers_relpath, ) - if not self.skip_tasks: - with Phase.use(Phase.TASKS): - self._execute_tasks(self.template.tasks) - # Run post-migration tasks. with Phase.use(Phase.MIGRATE): self._execute_tasks( diff --git a/docs/updating.md b/docs/updating.md index 423d126cf..c3096b4c0 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -132,7 +132,7 @@ project_half("half migrated
project") project_updated("updated project") project_full("fully updated
and migrated project") -update["3-way merge
& run tasks again"] +update["3-way merge"] regen_current["generate and run tasks"] regen_latest["generate and run tasks"] diff --git a/tests/test_legacy_migration.py b/tests/test_legacy_migration.py index 82cffe0d0..9dbaf0714 100644 --- a/tests/test_legacy_migration.py +++ b/tests/test_legacy_migration.py @@ -79,7 +79,7 @@ def test_migrations_and_tasks(tmp_path: Path, skip_tasks: bool) -> None: assert not (dst / "created-with-tasks.txt").exists() assert (dst / "delete-in-tasks.txt").exists() else: - assert (dst / "created-with-tasks.txt").read_text() == "task 1\ntask 2\n" * 2 + assert (dst / "created-with-tasks.txt").read_text() == "task 1\ntask 2\n" assert not (dst / "delete-in-tasks.txt").exists() assert not (dst / "delete-in-migration-v2.txt").exists() assert not (dst / "migrations.py").exists() From 777a369dffcd9954b7f0733d539ca84fcb81ad62 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Wed, 15 Jul 2026 17:53:51 +0200 Subject: [PATCH 3/3] feat(updating)!: apply pre-update migrations on fresh project from current template version BREAKING CHANGE: Pre-update migrations are now run on the fresh project generated from the current template version. In most cases, this should only improve update quality by reducing merge conflicts, but it is possible that this change will break template updates in rare cases. Thus, this change is marked as breaking out of an abundance of caution. --- copier/_main.py | 20 +++++++--- docs/updating.md | 5 ++- tests/test_migrations.py | 79 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/copier/_main.py b/copier/_main.py index b1c6b927d..18deb2696 100644 --- a/copier/_main.py +++ b/copier/_main.py @@ -368,11 +368,15 @@ def _answers_to_remember(self) -> Mapping[str, Any]: ) return answers - def _execute_tasks(self, tasks: Sequence[Task]) -> None: + def _execute_tasks( + self, tasks: Sequence[Task], *, directory: Path | None = None + ) -> None: """Run the given tasks. Arguments: tasks: The list of tasks to run. + directory: The working directory to run the tasks in. Defaults to the + subproject's path. """ operation = _operation.get() for i, task in enumerate(tasks): @@ -404,7 +408,7 @@ def _execute_tasks(self, tasks: Sequence[Task]) -> None: working_directory = ( # We can't use _render_path here, as that function has special handling # for files in the template - self.subproject.local_abspath + (directory or self.subproject.local_abspath) / Path(self._render_string(str(task.working_directory), extra_context)) ).absolute() @@ -1410,11 +1414,13 @@ def _apply_update(self) -> None: # noqa: C901 ) as old_worker: old_worker.run_copy() - # Run pre-migration tasks. with Phase.use(Phase.MIGRATE): - self._execute_tasks( - self.template.migration_tasks("before", self.subproject.template) # type: ignore[arg-type] + pre_migration_tasks = self.template.migration_tasks( + "before", + self.subproject.template, # type: ignore[arg-type] ) + # Run pre-migration tasks on the current project. + self._execute_tasks(pre_migration_tasks) # Clear last answers cache to load possible answers migration if the # `skip_answered` flag is not set. @@ -1488,6 +1494,10 @@ def _apply_update(self) -> None: # noqa: C901 if self.match_skip(path) and path in old_copy_files: path.unlink() + # Run pre-migration tasks on old copy. + self._execute_tasks( + pre_migration_tasks, directory=old_copy / subproject_subdir + ) # Stage all files including Git-ignored ones. git("add", "-f", ".") # Make a commit to run Git hooks if applicable. diff --git a/docs/updating.md b/docs/updating.md index c3096b4c0..58eec49a1 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -133,7 +133,7 @@ project_updated("updated project") project_full("fully updated
and migrated project") update["3-way merge"] -regen_current["generate and run tasks"] +regen_current["generate and run tasks
& apply pre-migrations"] regen_latest["generate and run tasks"] %% edges ---------------------------------------------------------- @@ -161,7 +161,8 @@ class regen_current,regen_latest,update blackborder; As you can see here, `copier` does several things: - Regenerate the project fresh from the **current** template version, using the - project's existing answers – this becomes the merge-base. + project's existing answers (with pre-migrations applied afterwards) – this becomes + the merge-base. - Regenerate the project fresh from the **latest** template version, using the same answers (with pre-migrations applied to the project beforehand). - Build a synthetic Git commit graph from these three states: the current-version diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 73863556e..69bfef575 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -4,6 +4,7 @@ from typing import Any import pytest +from inline_snapshot import snapshot from plumbum import local from copier import run_copy, run_update @@ -513,6 +514,84 @@ def test_migration_env_variables( assert (f"{variable}={value}" in env) == with_version +def test_pre_migration_runs_on_old_copy( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) + + build_file_tree( + { + src / "copier.yml": "_subdirectory: template/", + src / "template" / "{{ _copier_conf.answers_file }}.jinja": ( + "{{ _copier_answers|to_yaml }}" + ), + src / "template" / "pyproject.toml.jinja": ( + """\ + [tool.poetry.group.dev.dependencies] + pytest = "*" + """ + ), + } + ) + git_save(src, tag="v1") + + build_file_tree( + { + src / "copier.yml": ( + """\ + _subdirectory: template/ + + _migrations: + - version: v2 + when: "{{ _stage == 'before' }}" + command: "{{ _copier_python }} {{ _copier_conf.src_path / 'migrate_pep735.py' }}" + """ + ), + src / "template" / "pyproject.toml.jinja": ( + """\ + [dependency-groups] + dev = ["pytest"] + """ + ), + src / "migrate_pep735.py": ( + """\ + from pathlib import Path + import tomlkit + + pyproject = Path("pyproject.toml") + doc = tomlkit.parse(pyproject.read_bytes()) + print(pyproject.read_bytes()) + + deps = sorted(doc["tool"]["poetry"]["group"]["dev"]["dependencies"]) + del doc["tool"]["poetry"]["group"]["dev"] + + dep_groups = tomlkit.table() + dep_groups.add("dev", deps) + doc.add("dependency-groups", dep_groups) + + pyproject.write_text(tomlkit.dumps(doc).lstrip()) + """ + ), + } + ) + git_save(src, tag="v2") + + run_copy(str(src), dst, vcs_ref="v1") + + git_save(dst, "init") + pyproject = dst / "pyproject.toml" + pyproject.write_text(pyproject.read_text() + 'mypy = "*"\n') + git_save(dst, "add mypy in poetry group") + + run_update(dst, overwrite=True, unsafe=True) + assert pyproject.read_text() == snapshot( + """\ +[dependency-groups] +dev = ["mypy", "pytest"] +""" + ) + + @pytest.mark.parametrize("with_version", [True, False]) def test_migration_jinja_variables( tmp_path_factory: pytest.TempPathFactory, with_version: bool