diff --git a/.gitignore b/.gitignore index a6090d76b1..b94b0db950 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Prek automatic checks +.prek/local.toml +.prek/.state.toml +.prek/.enabled # Claude Code .worktrees diff --git a/.prek/README.md b/.prek/README.md new file mode 100644 index 0000000000..c0fd764648 --- /dev/null +++ b/.prek/README.md @@ -0,0 +1,153 @@ +# Pre-push local verification + +> **Not a general prek/pre-commit rollout.** prek is used only to install a **pre-push** hook; gate logic lives in `tools/prek.py`. This is not incremental formatting or per-file lint on staged changes — it runs full `make fl` / `make test-common-p` when the hook decides they are needed. + +Optional pre-push checks: run `make fl` and/or `make test-common-p` on push when the current tree’s +fingerprint is not already in local pass history (up to 50 recorded passes per check in +`.prek/.state.toml`). + +**Manual commands always run.** `make fl` and `make test-common-p` execute in full whenever you +invoke them. The cache applies only to pre-push (`git push`), `make prek`, and `make prek-dry`. +Successful manual runs still record passes (after hook install) so the next push can skip. + +The hook is **opt-in**. Without `.prek/local.toml`, the pre-push hook does nothing. + +## Quick start + +1. Copy `local.example.toml` to `local.toml` and set each check `mode` (`off`, `auto`, or `confirm`). +2. From the repo root: `make install-prepush-hooks` +3. Push as usual. Bypass once: `git push --no-verify` +4. Preview without pushing: `make prek-dry`. Run the gate manually: `make prek` +5. Remove the hook: `make uninstall-prepush-hooks` + +## Prerequisites + +- Root dev env: `make dev` (for `make fl` and `make test-common-p`) +- Docs Python env: `cd docs && make dev` (once). `make fl` also runs `npm install` in `docs/website` for Biome. +- Optional `[gate] only_when_pr_open = true`: requires [GitHub CLI](https://cli.github.com/) (`gh auth login`) + +## Configuration (`local.toml`) + +Copy from `local.example.toml`. Gitignored — per-developer only. + +| Section | Keys | Meaning | +|---------|------|---------| +| `[gate]` | `only_when_pr_open` | If `true`, skip all checks unless the current branch has an open PR (`gh pr view`) | +| `[lint]` | `mode` | How to handle a stale lint fingerprint | +| `[test_common_p]` | `mode` | How to handle a stale common-test fingerprint | + +### Modes + +| Mode | When the fingerprint is stale | +|------|---------------------| +| `off` | Never run this check | +| `auto` | Run the make target | +| `confirm` | Ask on the terminal; declining aborts the push | + +**Confirm prompts** look like: `Run make fl before push? [Y/n] ` + +- Enter or `y` / `yes` → run the check +- `n` / `no` → abort push +- Non-interactive stdin (no TTY) → treated as declined (push blocked) + +## What runs + +Checks run in order; a failed lint blocks tests. + +| Check | Make target | Command recorded in state | +|-------|-------------|---------------------------| +| `lint` | `fl` | `make fl` | +| `test_common_p` | `test-common-p` | `make test-common-p` | + +`make fl` runs format (root, docs, website deps) in parallel, then root and docs lint in parallel. + +A check runs only when its **fingerprint** (hash of tracked files listed for that check) is not among the +last 50 successful passes stored in `.prek/.state.toml` (also gitignored). Each pass records +`fingerprint`, `passed_at`, and `command`. That history lets branch switches and reverts reuse +a prior pass when the tree matches again. Passing prepends a record and trims the list +to 50 entries per check. + +Example: + +```toml +[[lint.passes]] +fingerprint = "abc123..." +passed_at = "2026-05-29T12:00:00+00:00" +command = "make fl" +``` + +After `make install-prepush-hooks`, successful `make fl` and `make test-common-p` also update +state (no extra commands). Plain `make lint` does not update prek state. + +## Unstaged changes on push + +prek may stash unstaged edits to `~/.cache/prek/patches/` while the hook runs, then restore them. +Built-in prek behavior (from pre-commit), not configurable. Keeps lint/tests from failing on WIP you +are not pushing. + +## Fingerprint inputs (`pyproject.toml`) + +Defines which tracked files feed each check fingerprint (`[tool.dlt.prepush.fingerprints.lint]` and +`[tool.dlt.prepush.fingerprints.test_common_p]`). Edit when adding new trees that should trigger +re-lint or re-test. + +**Lint** — `dlt`, `tests`, `tools`, `docs` (`.py`, `.md`, `.ipynb`), plus root/docs config and +embedded-snippet lint setup files. + +**Common tests** — `dlt` and selected `tests/*` suites (see `[tool.dlt.prepush.fingerprints]` in +`pyproject.toml`), plus `pyproject.toml`, +`uv.lock`, `tests/conftest.py`, `tests/load/test_dummy_client.py`. + +Inspect a fingerprint: + +```bash +uv run python -m tools.prek fingerprint lint +uv run python -m tools.prek fingerprint test_common_p +``` + +## Makefile targets + +| Target | Purpose | +|--------|---------| +| `make install-prepush-hooks` | Install prek pre-push hook and enable state recording (fails if another pre-push hook exists) | +| `make uninstall-prepush-hooks` | Remove the prek pre-push hook (no-op if none; fails if hook is not from prek) | +| `make prek` | Run the gate now (same logic as on push) | +| `make prek-dry` | Show what would run; no make, no state update | +| `make fl` | Format root + docs (parallel), then lint root + docs (parallel) | + +prek is installed with `uv tool install`, not as a repo dependency. + +## Troubleshooting + +**Existing pre-push hook** — `make install-prepush-hooks` refuses to install if `.git/hooks/pre-push` +already exists and is not from prek. prek cannot share the hook file with another tool. Remove or +relocate your hook first, or skip prek setup for now. + +**Uninstall with a foreign hook** — `make uninstall-prepush-hooks` only removes a prek-managed hook. +If `.git/hooks/pre-push` exists but was not installed via `make install-prepush-hooks`, uninstall +refuses to run so your hook is not deleted. + +**Hook never runs checks** — Ensure `.prek/local.toml` exists and at least one check has `mode` not +`off`. Run `make prek-dry` to see whether the gate is active and which checks are stale. + +**Gate skipped** — With `only_when_pr_open = true`, there must be an open PR on the current branch. + +**Docs lint fails** — Run `cd docs && make dev`, then `make fl` (or `cd docs && make format && make lint`). + +**Want to re-run after a pass** — Delete `.prek/.state.toml`, remove all `[[lint.passes]]` or +`[[test_common_p.passes]]` entries for that check, or change a tracked file in that check’s fingerprint inputs. + +**Stale fingerprint / wrong cache** — Same as above. State keeps up to 50 pass records per check +for branch hopping. + +## Files in this directory + +| File | Role | +|------|------| +| `README.md` | This guide | +| `local.example.toml` | Config template | +| `prek.toml` | prek hook definition (`uv run python -m tools.prek`) | + +Implementation and tests: `tools/prek.py` (run via `python -m tools.prek`). + +Gitignored: `local.toml`, `.state.toml`, `.enabled` diff --git a/.prek/local.example.toml b/.prek/local.example.toml new file mode 100644 index 0000000000..54b35d63d6 --- /dev/null +++ b/.prek/local.example.toml @@ -0,0 +1,8 @@ +[gate] +only_when_pr_open = false # if true, run checks only when the current branch has an open PR + +[lint] +mode = "auto" # off | auto | confirm + +[test_common_p] +mode = "off" # off | auto | confirm diff --git a/.prek/prek.toml b/.prek/prek.toml new file mode 100644 index 0000000000..bdf1dfdb32 --- /dev/null +++ b/.prek/prek.toml @@ -0,0 +1,11 @@ +[[repos]] +repo = "local" + +[[repos.hooks]] +id = "pre-push-gate" +name = "dlt pre-push gate" +language = "system" +entry = "uv run python -m tools.prek" +pass_filenames = false +always_run = true +stages = ["pre-push"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec96ea3561..bf4b5e304b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -152,6 +152,11 @@ Our goal is to maintain stability and compatibility across all environments. Ple `dlt` uses `mypy` and `flake8` (with several plugins) for linting. You can run the linter locally with `make lint`. We also run a code formatter with `black` which you can run with `make format`. The lint step will also ensure that the code is formatted correctly. It is good practice to run `make format && make lint` before every commit. +### Pre-push hooks (optional) + +You can run `make fl` and/or `make test-common-p` automatically before each push when tracked +files in scope change. Setup and configuration: [`.prek/README.md`](.prek/README.md) (`make install-prepush-hooks`). + ## Testing `dlt` uses `pytest` for testing. @@ -180,6 +185,8 @@ If, for any reason, you need to access the `pytest-xdist` worker id, do it with You can view our GitHub Actions setup in `.github/workflows` to see which tests are run with which dependencies / extras installed, and which platforms and python versions are used for linting and testing. The main entry point is `.github/workflows/main.yml` which orchestrates all other workflows. Certain dependencies exist, for example no tests will be run if the linter reports problems. Some workflows use test matrixes to test several destinations or run tests on various operating systems and with various python versions or dependency resolution strategies. To reduce CI execution time and improve feedback cycles, parallel test execution via `pytest-xdist` has been enabled in CI. Try to run any test suite that is involved in your development work in parallel if possible, since that is how it will be run in CI. Some CI tests have been restricted the number of workers due to destination performance reasons. +PR label `test-destinations-early`: jobs that normally wait for `test_common` (destination, sources, dbt runner, etc.) start in parallel with lint instead. Lint and common still run serially. Use this label only if you run lint and common locally (see [`.prek/README.md`](.prek/README.md)). + ### Common Components To test components that don’t require external resources, run: diff --git a/Makefile b/Makefile index f77ecd4203..0052495585 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := help -.PHONY: install-uv has-uv dev lint test test-common test-common-p reset-test-storage recreate-compiled-deps build-library-prerelease build-library publish-library test-load-local test-load-local-p test-load-local-postgres test-load-local-postgres-p install-snowflake-extras test-remote-snowflake test-remote-snowflake-p install-common-core test-common-core install-common-core-source test-common-core-source install-common-source install-pipeline-min test-pipeline-min install-pipeline-arrow test-pipeline-arrow install-pipeline-min-arrow test-pipeline-min-arrow install-workspace test-workspace test-workspace-dashboard install-hub-minimal test-hub-minimal test-hub install-pipeline-full test-pipeline-full install-pipeline-full-sql test-pipeline-full-sql install-sqlalchemy2 test-with-sqlalchemy-2 test-dest-load test-dest-remote-essential test-dest-remote-nonessential test-dbt-no-venv test-dbt-runner-venv test-sources-load test-sources-sql-database +.PHONY: install-uv has-uv dev lint lint-parallel lint-full lint-docs format format-docs docs-website-deps fl test test-common test-common-p reset-test-storage recreate-compiled-deps build-library-prerelease build-library publish-library test-load-local test-load-local-p test-load-local-postgres test-load-local-postgres-p install-snowflake-extras test-remote-snowflake test-remote-snowflake-p install-common-core test-common-core install-common-core-source test-common-core-source install-common-source install-pipeline-min test-pipeline-min install-pipeline-arrow test-pipeline-arrow install-pipeline-min-arrow test-pipeline-min-arrow install-workspace test-workspace test-workspace-dashboard install-hub-minimal test-hub-minimal test-hub install-pipeline-full test-pipeline-full install-pipeline-full-sql test-pipeline-full-sql install-sqlalchemy2 test-with-sqlalchemy-2 test-dest-load test-dest-remote-essential test-dest-remote-nonessential test-dbt-no-venv test-dbt-runner-venv test-sources-load test-sources-sql-database install-prepush-hooks uninstall-prepush-hooks prek prek-dry PYV=$(shell python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") .SILENT:has-uv @@ -37,7 +37,30 @@ dev-airflow: has-uv ## Prepares development environment with airflow support dev-hub: has-uv ## Prepares development environment with hub support uv sync --all-extras --group workspace-deps --group dev --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group adbc --group dashboard-tests -lint: lint-core lint-security lint-docstrings lint-lock lint-deps ## Runs all linters (mypy, ruff, flake8, bandit, docstrings, lockfile, deps) +LINT_TARGETS := lint-core lint-security lint-docstrings lint-lock lint-deps + +lint: $(LINT_TARGETS) ## Runs all linters (mypy, ruff, flake8, bandit, docstrings, lockfile, deps) + @: + +lint-parallel: ## Runs all linters in parallel (used by make fl) + $(MAKE) -j $(words $(LINT_TARGETS)) lint + +lint-full: lint lint-docs ## Root + docs lint (sequential) + +lint-docs: ## Runs docs linting (embedded snippets, notebooks, docs tooling) + cd docs && $(MAKE) lint + +format-docs: ## Formats docs tooling, website, examples, and notebooks + cd docs && $(MAKE) format + +docs-website-deps: ## Install docs website node deps (biome; used by make fl) + cd docs/website && npm install + +fl: ## Format then lint root and docs in parallel (prek pre-push gate) + set -e; \ + $(MAKE) format & $(MAKE) format-docs & $(MAKE) docs-website-deps & wait; \ + $(MAKE) lint-parallel & $(MAKE) -C docs lint-parallel & wait + @if [ -f .prek/.enabled ]; then uv run python -m tools.prek --record lint; fi lint-lock: ## Checks uv lockfile is in sync uv lock --check @@ -151,11 +174,13 @@ TEST_COMMON_PATHS = \ tests/libs \ tests/destinations +test-common: PYTEST_MARKERS = not rfam test-common: ## Tests common components without external resources $(call RUN_XDIST_SAFE_SPLIT,$(TEST_COMMON_PATHS)) test-common-p: ## Tests common components in parallel $(MAKE) test-common PYTEST_XDIST_N=auto + @if [ -f .prek/.enabled ]; then uv run python -m tools.prek --record test_common_p; fi # ---------------------------------------------------------------------- # Local load tests @@ -432,3 +457,34 @@ test-e2e-dashboard-headed: ## Runs dashboard e2e tests with visible browser create-test-pipelines: ## Creates test pipelines for manual dashboard testing uv run python tests/workspace/helpers/dashboard/example_pipelines.py + +PREK_VERSION ?= 0.4.2 + +prek: ## Run pre-push gate now (same as the git hook) + uv run python -m tools.prek + +prek-dry: ## Show what the pre-push gate would run (no make, no state update) + uv run python -m tools.prek --dry-run + +install-prepush-hooks: ## Install prek pre-push hook (fails if another pre-push hook exists) + @if [ -f .git/hooks/pre-push ] && ! grep -Fq 'File generated by prek' .git/hooks/pre-push 2>/dev/null; then \ + echo "Error: .git/hooks/pre-push already exists."; \ + echo "prek is not compatible with an existing pre-push hook."; \ + echo "Remove or relocate your hook, then run make install-prepush-hooks again."; \ + exit 1; \ + fi + uv tool install prek@$(PREK_VERSION) + prek install --hook-type pre-push --config .prek/prek.toml -f + @touch .prek/.enabled + +uninstall-prepush-hooks: ## Remove prek pre-push hook (no-op if none; fails if hook is not from prek) + @if [ ! -f .git/hooks/pre-push ]; then \ + echo "No pre-push hook to remove."; \ + elif ! grep -Fq 'File generated by prek' .git/hooks/pre-push 2>/dev/null; then \ + echo "Error: .git/hooks/pre-push exists but was not installed by make install-prepush-hooks."; \ + echo "make uninstall-prepush-hooks will not remove it."; \ + exit 1; \ + else \ + prek uninstall --hook-type pre-push --config .prek/prek.toml; \ + fi + @rm -f .prek/.enabled diff --git a/docs/Makefile b/docs/Makefile index 69a5f82277..6dda788026 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,5 @@ .DEFAULT_GOAL := help +.PHONY: lint lint-parallel lint-core lint-embedded-snippets lint-notebooks lint-website # Add " ## description" after any target name to include it in `make help` output. # Example: my-target: ## Does something useful @@ -8,7 +9,13 @@ help: ## Shows this help message dev: ## Prepares development environment for docs tooling uv sync -lint: lint-core lint-embedded-snippets lint-notebooks lint-website ## Runs all linters +LINT_TARGETS := lint-core lint-embedded-snippets lint-notebooks lint-website + +lint: $(LINT_TARGETS) ## Runs all linters + @: + +lint-parallel: ## Runs all linters in parallel (used by make fl) + $(MAKE) -j $(words $(LINT_TARGETS)) lint lint-website: ## Lints the docusaurus website JS/TS sources with Biome cd website && npm run lint diff --git a/docs/uv.lock b/docs/uv.lock index 2520d7ee4a..00a591c540 100644 --- a/docs/uv.lock +++ b/docs/uv.lock @@ -1096,6 +1096,7 @@ dev = [ { name = "requests-mock", specifier = ">=1.10.0,<2" }, { name = "ruff", specifier = ">=0.3.2,<0.4" }, { name = "sqlfluff", specifier = ">=2.3.2,<3" }, + { name = "tomli", specifier = ">=2.0.1,<3" }, { name = "types-cachetools", specifier = ">=4.2.9" }, { name = "types-click", specifier = ">=7.1.8,<8" }, { name = "types-croniter", specifier = ">=6.0.0" }, @@ -1768,6 +1769,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/ed/6bfa4109fcb23a58819600392564fea69cdc6551ffd5e69ccf1d52a40cbc/greenlet-3.2.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8c68325b0d0acf8d91dde4e6f930967dd52a5302cd4062932a6b2e7c2969f47c", size = 271061, upload-time = "2025-08-07T13:17:15.373Z" }, { url = "https://files.pythonhosted.org/packages/2a/fc/102ec1a2fc015b3a7652abab7acf3541d58c04d3d17a8d3d6a44adae1eb1/greenlet-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:94385f101946790ae13da500603491f04a76b6e4c059dab271b3ce2e283b2590", size = 629475, upload-time = "2025-08-07T13:42:54.009Z" }, { url = "https://files.pythonhosted.org/packages/c5/26/80383131d55a4ac0fb08d71660fd77e7660b9db6bdb4e8884f46d9f2cc04/greenlet-3.2.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f10fd42b5ee276335863712fa3da6608e93f70629c631bf77145021600abc23c", size = 640802, upload-time = "2025-08-07T13:45:25.52Z" }, + { url = "https://files.pythonhosted.org/packages/9f/7c/e7833dbcd8f376f3326bd728c845d31dcde4c84268d3921afcae77d90d08/greenlet-3.2.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c8c9e331e58180d0d83c5b7999255721b725913ff6bc6cf39fa2a45841a4fd4b", size = 636703, upload-time = "2025-08-07T13:53:12.622Z" }, { url = "https://files.pythonhosted.org/packages/e9/49/547b93b7c0428ede7b3f309bc965986874759f7d89e4e04aeddbc9699acb/greenlet-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58b97143c9cc7b86fc458f215bd0932f1757ce649e05b640fea2e79b54cedb31", size = 635417, upload-time = "2025-08-07T13:18:25.189Z" }, { url = "https://files.pythonhosted.org/packages/7f/91/ae2eb6b7979e2f9b035a9f612cf70f1bf54aad4e1d125129bef1eae96f19/greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d", size = 584358, upload-time = "2025-08-07T13:18:23.708Z" }, { url = "https://files.pythonhosted.org/packages/f7/85/433de0c9c0252b22b16d413c9407e6cb3b41df7389afc366ca204dbc1393/greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5", size = 1113550, upload-time = "2025-08-07T13:42:37.467Z" }, @@ -1778,6 +1780,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, @@ -1788,6 +1791,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, diff --git a/pyproject.toml b/pyproject.toml index 993bb6eeae..9e9421c81c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,7 @@ dev = [ "pytest-order>=1.0.0", "pytest-cases>=3.8.6", "pytest-forked>=1.3.0", + "tomli>=2.0.1,<3", "types-PyYAML>=6.0.7", "types-cachetools>=4.2.9", "types-protobuf>=3.19.8", @@ -392,6 +393,7 @@ exclude = [ "docs/website/.dlt-repo", "tests/reflection/module_cases/syntax_error.py", "docs/website/docs/general-usage/transformations/transformation-snippets.py", + "docs/education/**/*nbqa_ipynb.py", # nbqa temp files while lint-notebooks runs in parallel ] [tool.ruff.lint] @@ -422,6 +424,39 @@ ignore = [ line-length = 100 preview = true +[tool.dlt.prepush.fingerprints.lint] +paths = ["dlt", "tests", "tools", "docs"] +globs = ["*.py", "*.md", "*.ipynb"] +files = [ + "pyproject.toml", + "uv.lock", + "docs/pyproject.toml", + "docs/uv.lock", + "docs/docs_tools/snippets/lint_setup/template.py", + "docs/docs_tools/snippets/lint_setup/mypy.ini", +] + +[tool.dlt.prepush.fingerprints.test_common_p] +paths = [ + "dlt", + "tests/common", + "tests/normalize", + "tests/extract", + "tests/pipeline", + "tests/reflection", + "tests/sources", + "tests/workspace", + "tests/libs", + "tests/destinations", +] +globs = ["*.py"] +files = [ + "pyproject.toml", + "uv.lock", + "tests/conftest.py", + "tests/load/test_dummy_client.py", +] + [tool.isort] color_output = true line_length = 100 diff --git a/tests/load/ducklake/test_ducklake_client.py b/tests/load/ducklake/test_ducklake_client.py index 8c942755ae..319ae51461 100644 --- a/tests/load/ducklake/test_ducklake_client.py +++ b/tests/load/ducklake/test_ducklake_client.py @@ -438,4 +438,4 @@ def test_ducklake_factory_instantiation() -> None: credentials = DuckLakeCredentials( "lake_catalog", catalog=catalog_credentials, - ) \ No newline at end of file + ) diff --git a/tools/prek.py b/tools/prek.py new file mode 100644 index 0000000000..841bcbd5ff --- /dev/null +++ b/tools/prek.py @@ -0,0 +1,601 @@ +"""Pre-push gate: run lint/tests when tracked fingerprint inputs change.""" + +# ruff: noqa: T201 +# flake8: noqa: T201 + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import os +import subprocess +import sys +import tomli as tomllib +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, NamedTuple, TypedDict, cast + +import pendulum +from pendulum.datetime import DateTime + +Mode = Literal["off", "auto", "confirm"] +VALID_MODES = frozenset({"off", "auto", "confirm"}) +MAX_PASSED_FINGERPRINTS = 50 + + +class PassRecord(TypedDict): + fingerprint: str + passed_at: str + command: str + + +class CheckState(TypedDict): + passes: list[PassRecord] + + +State = dict[str, CheckState] + + +class Check(NamedTuple): + name: str + make_target: str + make_command: str + + +CHECKS: tuple[Check, ...] = ( + Check("lint", "fl", "make fl"), + Check("test_common_p", "test-common-p", "make test-common-p"), +) +CHECK_NAMES = frozenset(check.name for check in CHECKS) + + +class FingerprintDef(NamedTuple): + files: tuple[str, ...] + paths: tuple[str, ...] + globs: tuple[str, ...] + + +class PlannedCheck(NamedTuple): + check: Check + mode: Mode + fingerprint: str + cached_fingerprint: str + stale: bool + + +class GateOutcome(NamedTuple): + exit_code: int + new_state: State + stderr_lines: tuple[str, ...] + + +class ConfigError(Exception): + """Invalid prek local configuration.""" + + +class UnknownCheckError(Exception): + """Check name is missing from [tool.dlt.prepush.fingerprints] in pyproject.toml.""" + + +def repo_root() -> Path: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return Path(result.stdout.strip()) + return Path.cwd() + + +def default_prek_dir() -> Path: + return repo_root() / ".prek" + + +def parse_bool(value: Any, *, key: str) -> bool: + if isinstance(value, bool): + return value + raise ConfigError(f"Invalid boolean {value!r} for {key}") + + +def parse_only_when_pr_open(local_config: dict[str, Any]) -> bool: + gate = local_config.get("gate", {}) + if not isinstance(gate, dict): + raise ConfigError("Invalid [gate] section") + return parse_bool(gate.get("only_when_pr_open", False), key="gate.only_when_pr_open") + + +def parse_mode(local_config: dict[str, Any], check_name: str) -> Mode: + section = local_config.get(check_name, {}) + if not isinstance(section, dict): + raise ConfigError(f"Invalid [{check_name}] section") + mode = section.get("mode", "off") + if mode not in VALID_MODES: + raise ConfigError(f"Invalid mode {mode!r} for check {check_name!r}") + return cast(Mode, mode) + + +def make_command_for(check_name: str) -> str: + for check in CHECKS: + if check.name == check_name: + return check.make_command + valid = ", ".join(sorted(CHECK_NAMES)) + raise ConfigError(f"Unknown check {check_name!r}; expected one of: {valid}") + + +def load_toml(path: Path) -> dict[str, Any]: + with open(path, "rb") as file: + return tomllib.load(file) + + +def load_local_config(path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + return load_toml(path) + + +def load_state(path: Path) -> State: + if not path.is_file(): + return {} + data = load_toml(path) + return { + name: normalize_check_state(section) + for name, section in data.items() + if isinstance(section, dict) + } + + +def normalize_pass_record(raw: dict[str, Any]) -> PassRecord | None: + fingerprint = raw.get("fingerprint") + if not isinstance(fingerprint, str) or not fingerprint: + return None + passed_at = raw.get("passed_at") + command = raw.get("command") + return PassRecord( + fingerprint=fingerprint, + passed_at=passed_at if isinstance(passed_at, str) else "", + command=command if isinstance(command, str) else "", + ) + + +def normalize_check_state(section: dict[str, Any]) -> CheckState: + passes: list[PassRecord] = [] + raw_passes = section.get("passes") + if isinstance(raw_passes, list): + for item in raw_passes: + if isinstance(item, dict): + record = normalize_pass_record(item) + if record is not None: + passes.append(record) + return CheckState(passes=passes[:MAX_PASSED_FINGERPRINTS]) + + +def passed_fingerprints(check_state: CheckState | None) -> list[str]: + if not check_state: + return [] + return [record["fingerprint"] for record in check_state["passes"]] + + +def fingerprint_is_known(check_state: CheckState | None, fingerprint: str) -> bool: + return fingerprint in passed_fingerprints(check_state) + + +def write_state(path: Path, state: State) -> None: + lines: list[str] = [] + for check_name, data in state.items(): + for record in data["passes"]: + lines.append(f"[[{check_name}.passes]]") + lines.append(f'fingerprint = "{record["fingerprint"]}"') + lines.append(f'passed_at = "{record["passed_at"]}"') + lines.append(f'command = "{record["command"]}"') + lines.append("") + path.write_text("\n".join(lines).rstrip() + ("\n" if lines else ""), encoding="utf-8") + + +def fingerprint_def_from_dict(raw: dict[str, list[str]]) -> FingerprintDef: + return FingerprintDef( + files=tuple(raw.get("files", [])), + paths=tuple(raw.get("paths", [])), + globs=tuple(raw.get("globs", [])), + ) + + +def matches_globs(path: str, globs: list[str]) -> bool: + name = os.path.basename(path) + return any(fnmatch.fnmatch(name, pattern) for pattern in globs) + + +def resolve_fingerprint_files( + fingerprint_def: FingerprintDef, + *, + list_tracked: Callable[[list[str]], list[str]], + root: Path, +) -> list[str]: + files: set[str] = set(fingerprint_def.files) + for path_prefix in fingerprint_def.paths: + candidates = list_tracked([path_prefix]) + if fingerprint_def.globs: + files.update( + path for path in candidates if matches_globs(path, list(fingerprint_def.globs)) + ) + else: + files.update(candidates) + return sorted(path for path in files if (root / path).is_file()) + + +def fingerprint_files(paths: list[str], read_bytes: Callable[[str], bytes]) -> str: + aggregate = hashlib.sha256() + for path in paths: + aggregate.update(path.encode()) + aggregate.update(b"\0") + aggregate.update(read_bytes(path)) + return aggregate.hexdigest() + + +def load_fingerprint_defs(config_path: Path) -> dict[str, FingerprintDef]: + raw = load_toml(config_path) + tool = raw.get("tool", {}) + dlt = tool.get("dlt", {}) if isinstance(tool, dict) else {} + prepush = dlt.get("prepush", {}) if isinstance(dlt, dict) else {} + fingerprints = prepush.get("fingerprints", {}) + if not isinstance(fingerprints, dict): + raise ValueError("Invalid [tool.dlt.prepush.fingerprints] section in pyproject.toml") + return { + name: fingerprint_def_from_dict(section) + for name, section in fingerprints.items() + if isinstance(section, dict) + } + + +def git_ls_files(root: Path, pathspecs: list[str]) -> list[str]: + if not pathspecs: + return [] + result = subprocess.run( + ["git", "ls-files", "--", *pathspecs], + cwd=root, + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def make_fingerprint_fn(root: Path, config_path: Path) -> Callable[[str], str]: + fingerprint_defs = load_fingerprint_defs(config_path) + + def fingerprint(check_name: str) -> str: + try: + fingerprint_def = fingerprint_defs[check_name] + except KeyError as exc: + raise UnknownCheckError(f"Unknown check: {check_name}") from exc + paths = resolve_fingerprint_files( + fingerprint_def, + list_tracked=lambda pathspecs: git_ls_files(root, pathspecs), + root=root, + ) + return fingerprint_files(paths, lambda path: (root / path).read_bytes()) + + return fingerprint + + +def gate_active(*, only_when_pr_open: bool, has_open_pr: bool) -> tuple[bool, str]: + if not only_when_pr_open: + return True, "only_when_pr_open=false" + if has_open_pr: + return True, "open PR on current branch" + return False, "only_when_pr_open=true and no open PR for current branch" + + +def plan_checks( + checks: Sequence[Check], + local_config: dict[str, Any], + state: State, + fingerprint: Callable[[str], str], +) -> list[PlannedCheck]: + planned: list[PlannedCheck] = [] + for check in checks: + mode = parse_mode(local_config, check.name) + if mode == "off": + continue + current = fingerprint(check.name) + history = passed_fingerprints(state.get(check.name)) + cached = history[0] if history else "" + planned.append( + PlannedCheck( + check, + mode, + current, + cached, + not fingerprint_is_known(state.get(check.name), current), + ) + ) + return planned + + +def with_passed_check( + state: State, + check_name: str, + *, + fingerprint: str, + command: str, + passed_at: DateTime, +) -> State: + updated: State = {name: CheckState(passes=list(data["passes"])) for name, data in state.items()} + passes = list(updated.get(check_name, CheckState(passes=[]))["passes"]) + passes = [record for record in passes if record["fingerprint"] != fingerprint] + passes.insert( + 0, + PassRecord( + fingerprint=fingerprint, + passed_at=passed_at.replace(microsecond=0).isoformat(), + command=command, + ), + ) + updated[check_name] = CheckState(passes=passes[:MAX_PASSED_FINGERPRINTS]) + return updated + + +def dry_run_no_config_line() -> str: + return "prek gate (dry-run): no .prek/local.toml — hook would no-op on push" + + +def dry_run_lines( + planned: list[PlannedCheck], + *, + local_config_path: Path, + active: bool, + reason: str, + is_tty: bool, +) -> list[str]: + lines = [ + f"prek gate (dry-run): {local_config_path}", + f"gate active: {active} ({reason})", + ] + if not active: + return lines + if not planned: + lines.append("no checks enabled (all off or empty config)") + return lines + + for item in planned: + check_name, make_command = item.check.name, item.check.make_command + if not item.stale: + lines.append(f"[{check_name}] mode={item.mode} up to date ({item.fingerprint[:12]}…)") + continue + action = f"would run {make_command}" + if item.mode == "confirm": + action = ( + f"would prompt, then run {make_command}" + if is_tty + else f"would block push ({make_command}, non-interactive)" + ) + cached_label = item.cached_fingerprint[:12] if item.cached_fingerprint else "none" + lines.append( + f"[{check_name}] mode={item.mode} stale ({item.fingerprint[:12]}…, " + f"was {cached_label}…) → {action}" + ) + return lines + + +@dataclass(frozen=True) +class GateDeps: + run_make: Callable[[str], int] + has_open_pr: Callable[[], bool] + confirm: Callable[[str], bool] + fingerprint: Callable[[str], str] + now: Callable[[], DateTime] + is_tty: Callable[[], bool] + + @classmethod + def from_repo(cls, root: Path) -> GateDeps: + prek_dir = root / ".prek" + return cls( + run_make=lambda target: _run_make_target(root, target), + has_open_pr=lambda: _has_open_pr(root), + confirm=_confirm_run, + fingerprint=make_fingerprint_fn(root, root / "pyproject.toml"), + now=lambda: pendulum.now("UTC"), + is_tty=sys.stdin.isatty, + ) + + +def _run_make_target(root: Path, target: str) -> int: + print(f"Running make {target}...", flush=True) + return subprocess.run(["make", target], cwd=root).returncode + + +def _has_open_pr(root: Path) -> bool: + result = subprocess.run( + ["gh", "pr", "view", "--json", "state", "-q", ".state"], + cwd=root, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return False + return result.stdout.strip().upper() == "OPEN" + + +def _confirm_run(make_command: str) -> bool: + if not sys.stdin.isatty(): + return False + reply = input(f"Run {make_command} before push? [Y/n] ").strip().lower() + return not reply or reply in {"y", "yes"} + + +def run_gate( + *, + local_config: dict[str, Any], + state: State, + deps: GateDeps, + checks: Sequence[Check] = CHECKS, +) -> GateOutcome: + active, reason = gate_active( + only_when_pr_open=parse_only_when_pr_open(local_config), + has_open_pr=deps.has_open_pr(), + ) + if not active: + return GateOutcome(0, state, (f"prek gate: skipped ({reason})",)) + + new_state: State = { + name: CheckState(passes=list(data["passes"])) for name, data in state.items() + } + for check in checks: + mode = parse_mode(local_config, check.name) + if mode == "off": + continue + + fingerprint = deps.fingerprint(check.name) + if fingerprint_is_known(new_state.get(check.name), fingerprint): + continue + + if mode == "confirm" and not deps.confirm(check.make_command): + return GateOutcome(1, state, (f"Declined {check.make_command}. Push aborted.",)) + + if deps.run_make(check.make_target) != 0: + return GateOutcome(1, state, (f"{check.make_command} failed. Push aborted.",)) + + new_state = with_passed_check( + new_state, + check.name, + fingerprint=fingerprint, + command=check.make_command, + passed_at=deps.now(), + ) + + return GateOutcome(0, new_state, ()) + + +def record_passed_check( + *, + check_name: str, + state: State, + hooks_enabled: bool, + deps: GateDeps, +) -> tuple[State, str | None]: + try: + command = make_command_for(check_name) + except ConfigError as exc: + return state, str(exc) + + if not hooks_enabled: + return state, None + + return ( + with_passed_check( + state, + check_name, + fingerprint=deps.fingerprint(check_name), + command=command, + passed_at=deps.now(), + ), + None, + ) + + +def main(*, prek_dir: Path | None = None, argv: list[str] | None = None) -> int: + prek_dir = prek_dir or default_prek_dir() + parser = argparse.ArgumentParser(description="Pre-push gate for lint and common tests") + parser.add_argument( + "--dry-run", action="store_true", help="Show planned checks without running them" + ) + parser.add_argument( + "--record", + metavar="CHECK", + help="Record a successful check (lint, test_common_p)", + ) + args = parser.parse_args(argv) + + root = prek_dir.parent + deps = GateDeps.from_repo(root) + local_path = prek_dir / "local.toml" + state_path = prek_dir / ".state.toml" + enabled_path = prek_dir / ".enabled" + + try: + if args.record: + return _run_record(args.record, deps, state_path, enabled_path) + if args.dry_run: + return _run_dry_run(deps, local_path, state_path) + return _run_gate(deps, local_path, state_path) + except ConfigError as exc: + print(str(exc), file=sys.stderr) + return 1 + + +def main_fingerprint(*, prek_dir: Path | None = None, argv: list[str] | None = None) -> int: + prek_dir = prek_dir or default_prek_dir() + if argv is None: + argv = sys.argv[1:] + if len(argv) != 1: + print("Usage: python -m tools.prek fingerprint ", file=sys.stderr) + return 1 + + root = prek_dir.parent + try: + print(make_fingerprint_fn(root, root / "pyproject.toml")(argv[0])) + except UnknownCheckError as exc: + print(str(exc), file=sys.stderr) + return 1 + return 0 + + +def _run_record(check_name: str, deps: GateDeps, state_path: Path, enabled_path: Path) -> int: + state = load_state(state_path) + new_state, error = record_passed_check( + check_name=check_name, + state=state, + hooks_enabled=enabled_path.is_file(), + deps=deps, + ) + if error: + print(error, file=sys.stderr) + return 1 + if new_state != state: + write_state(state_path, new_state) + return 0 + + +def _run_dry_run(deps: GateDeps, local_config_path: Path, state_path: Path) -> int: + local_config = load_local_config(local_config_path) + if local_config is None: + print(dry_run_no_config_line()) + return 0 + + active, reason = gate_active( + only_when_pr_open=parse_only_when_pr_open(local_config), + has_open_pr=deps.has_open_pr(), + ) + planned = plan_checks(CHECKS, local_config, load_state(state_path), deps.fingerprint) + for line in dry_run_lines( + planned, + local_config_path=local_config_path, + active=active, + reason=reason, + is_tty=deps.is_tty(), + ): + print(line) + return 0 + + +def _run_gate(deps: GateDeps, local_config_path: Path, state_path: Path) -> int: + local_config = load_local_config(local_config_path) + if local_config is None: + return 0 + + state = load_state(state_path) + outcome = run_gate(local_config=local_config, state=state, deps=deps) + for line in outcome.stderr_lines: + print(line, file=sys.stderr) + if outcome.new_state != state: + write_state(state_path, outcome.new_state) + return outcome.exit_code + + +if __name__ == "__main__": + cli_argv = sys.argv[1:] + if cli_argv and cli_argv[0] == "fingerprint": + raise SystemExit(main_fingerprint(argv=cli_argv[1:])) + raise SystemExit(main(argv=cli_argv)) diff --git a/tools/tests/test_prek.py b/tools/tests/test_prek.py new file mode 100644 index 0000000000..d3e042979d --- /dev/null +++ b/tools/tests/test_prek.py @@ -0,0 +1,260 @@ +"""Tests for tools/prek.py.""" + +from collections.abc import Callable +from pathlib import Path +from typing import Optional + +import pendulum +import pytest + +from tools.prek import ( + CHECKS, + ConfigError, + CheckState, + GateDeps, + MAX_PASSED_FINGERPRINTS, + PassRecord, + FingerprintDef, + State, + dry_run_lines, + fingerprint_files, + gate_active, + load_state, + make_command_for, + matches_globs, + parse_bool, + parse_mode, + passed_fingerprints, + plan_checks, + record_passed_check, + repo_root, + resolve_fingerprint_files, + run_gate, + with_passed_check, + write_state, +) + +FIXED_NOW = pendulum.datetime(2026, 5, 29, 12, 0, 0, tz=pendulum.UTC) + + +def make_deps( + *, + run_make: Optional[Callable[[str], int]] = None, + has_open_pr: Callable[[], bool] = lambda: True, + confirm: Callable[[str], bool] = lambda _: True, + fingerprint: Callable[[str], str] = lambda name: f"fp-{name}", + is_tty: Callable[[], bool] = lambda: True, +) -> GateDeps: + return GateDeps( + run_make=run_make or (lambda _target: 0), + has_open_pr=has_open_pr, + confirm=confirm, + fingerprint=fingerprint, + now=lambda: FIXED_NOW, + is_tty=is_tty, + ) + + +def test_repo_root_uses_git_toplevel(monkeypatch: pytest.MonkeyPatch) -> None: + class Result: + returncode = 0 + stdout = "/repo\n" + + monkeypatch.setattr("tools.prek.subprocess.run", lambda *args, **kwargs: Result()) + assert repo_root() == Path("/repo") + + +@pytest.mark.parametrize( + ("value", "expected"), + [(True, True), (False, False)], + ids=["true", "false"], +) +def test_parse_bool(value: bool, expected: bool) -> None: + assert parse_bool(value, key="test.key") is expected + + +def test_parse_mode_and_make_command() -> None: + assert parse_mode({"lint": {"mode": "auto"}}, "lint") == "auto" + assert make_command_for("lint") == "make fl" + with pytest.raises(ConfigError, match="Unknown check"): + make_command_for("missing") + + +@pytest.mark.parametrize( + ("path", "globs", "expected"), + [ + ("tests/common/test_utils.py", ["*.py"], True), + ("tests/common/readme.md", ["*.py"], False), + ], + ids=["match", "no-match"], +) +def test_matches_globs(path: str, globs: list[str], expected: bool) -> None: + assert matches_globs(path, globs) is expected + + +def test_resolve_fingerprint_files_and_fingerprint(tmp_path: Path) -> None: + (tmp_path / "root.toml").write_text("c", encoding="utf-8") + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "keep.py").write_text("a", encoding="utf-8") + + fingerprint_def = FingerprintDef(files=("root.toml",), paths=("pkg",), globs=("*.py",)) + paths = resolve_fingerprint_files( + fingerprint_def, + list_tracked=lambda _: ["pkg/keep.py", "pkg/skip.txt"], + root=tmp_path, + ) + assert paths == ["pkg/keep.py", "root.toml"] + + files = {"a.py": b"one", "b.py": b"two"} + assert fingerprint_files(["a.py", "b.py"], files.__getitem__) == fingerprint_files( + ["a.py", "b.py"], files.__getitem__ + ) + + +@pytest.mark.parametrize( + ("only_when_pr_open", "has_open_pr", "active"), + [(False, False, True), (True, False, False), (True, True, True)], + ids=["always", "no-pr", "open-pr"], +) +def test_gate_active(only_when_pr_open: bool, has_open_pr: bool, active: bool) -> None: + result, _reason = gate_active(only_when_pr_open=only_when_pr_open, has_open_pr=has_open_pr) + assert result is active + + +def test_plan_checks_and_dry_run() -> None: + planned = plan_checks( + CHECKS, + {"lint": {"mode": "confirm"}, "test_common_p": {"mode": "off"}}, + {}, + lambda _name: "fingerprint-value", + ) + lines = dry_run_lines( + planned, + local_config_path=Path(".prek/local.toml"), + active=True, + reason="only_when_pr_open=false", + is_tty=False, + ) + assert any("would block push" in line for line in lines) + + +def test_run_gate_flow() -> None: + calls: list[str] = [] + + def record_make(target: str) -> int: + calls.append(target) + return 0 + + deps = make_deps(run_make=record_make) + + skipped = run_gate( + local_config={"gate": {"only_when_pr_open": True}, "lint": {"mode": "auto"}}, + state={}, + deps=make_deps(has_open_pr=lambda: False), + ) + assert skipped.exit_code == 0 + assert "skipped" in skipped.stderr_lines[0] + + passed = run_gate(local_config={"lint": {"mode": "auto"}}, state={}, deps=deps) + assert passed.exit_code == 0 + assert calls == ["fl"] + assert passed.new_state["lint"]["passes"][0]["fingerprint"] == "fp-lint" + assert passed.new_state["lint"]["passes"][0]["command"] == "make fl" + + failed = run_gate( + local_config={"lint": {"mode": "auto"}}, + state={ + "lint": CheckState( + passes=[ + PassRecord(fingerprint="other", passed_at="t0", command="make fl"), + ] + ) + }, + deps=make_deps(run_make=lambda _target: 1), + ) + assert failed.exit_code == 1 + assert failed.new_state["lint"]["passes"][0]["fingerprint"] == "other" + + declined = run_gate( + local_config={"lint": {"mode": "confirm"}}, + state={}, + deps=make_deps(confirm=lambda _: False), + ) + assert declined.exit_code == 1 + + +def test_record_passed_check() -> None: + deps = make_deps() + unchanged, error = record_passed_check( + check_name="lint", state={}, hooks_enabled=False, deps=deps + ) + assert error is None and unchanged == {} + + updated, error = record_passed_check(check_name="lint", state={}, hooks_enabled=True, deps=deps) + assert error is None + assert updated["lint"]["passes"][0]["command"] == "make fl" + + _, error = record_passed_check(check_name="missing", state={}, hooks_enabled=True, deps=deps) + assert error is not None + + +def test_with_passed_check_and_state_io(tmp_path: Path) -> None: + state: State = { + "lint": CheckState( + passes=[ + PassRecord(fingerprint="old", passed_at="t0", command="make fl"), + ] + ) + } + updated = with_passed_check( + state, "lint", fingerprint="new", command="make fl", passed_at=FIXED_NOW + ) + assert state["lint"]["passes"][0]["fingerprint"] == "old" + assert updated["lint"]["passes"][0]["fingerprint"] == "new" + assert updated["lint"]["passes"][1]["fingerprint"] == "old" + + state_path = tmp_path / ".state.toml" + write_state(state_path, updated) + assert load_state(state_path) == updated + + +def test_fingerprint_history_skips_known_tree() -> None: + deps = make_deps(fingerprint=lambda name: {"lint": "fp-a", "test_common_p": "fp-b"}[name]) + state: State = { + "lint": CheckState( + passes=[ + PassRecord(fingerprint="fp-a", passed_at="t1", command="make fl"), + PassRecord(fingerprint="fp-other", passed_at="t0", command="make fl"), + ] + ) + } + + outcome = run_gate(local_config={"lint": {"mode": "auto"}}, state=state, deps=deps) + assert outcome.exit_code == 0 + assert outcome.new_state == state + + planned = plan_checks( + CHECKS, + {"lint": {"mode": "auto"}}, + state, + deps.fingerprint, + ) + assert planned[0].stale is False + + +def test_fingerprint_history_caps_at_max() -> None: + history = [ + PassRecord(fingerprint=f"fp-{index}", passed_at=f"t{index}", command="make fl") + for index in range(MAX_PASSED_FINGERPRINTS + 5) + ] + state: State = {"lint": CheckState(passes=history)} + updated = with_passed_check( + state, + "lint", + fingerprint="fp-new", + command="make fl", + passed_at=FIXED_NOW, + ) + assert len(updated["lint"]["passes"]) == MAX_PASSED_FINGERPRINTS + assert updated["lint"]["passes"][0]["fingerprint"] == "fp-new" + assert updated["lint"]["passes"][-1]["fingerprint"] == f"fp-{MAX_PASSED_FINGERPRINTS - 2}" diff --git a/uv.lock b/uv.lock index 95c2348184..c2e6743e4e 100644 --- a/uv.lock +++ b/uv.lock @@ -2727,6 +2727,7 @@ dev = [ { name = "requests-mock" }, { name = "ruff" }, { name = "sqlfluff" }, + { name = "tomli" }, { name = "types-cachetools" }, { name = "types-click" }, { name = "types-croniter" }, @@ -2965,6 +2966,7 @@ dev = [ { name = "requests-mock", specifier = ">=1.10.0,<2" }, { name = "ruff", specifier = ">=0.3.2,<0.4" }, { name = "sqlfluff", specifier = ">=2.3.2,<3" }, + { name = "tomli", specifier = ">=2.0.1,<3" }, { name = "types-cachetools", specifier = ">=4.2.9" }, { name = "types-click", specifier = ">=7.1.8,<8" }, { name = "types-croniter", specifier = ">=6.0.0" },