From b6f8215c94388d6669bd81caa0a84b8af506281d Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Sun, 19 Jul 2026 10:35:37 +0900 Subject: [PATCH 1/5] feat: feature-aware iALS via an optional `features:` recipe block Add support for irspack 0.5.0's feature-aware iALS. An optional `features:` block declares item- and/or user-side attribute tables; recotem encodes them into numeric matrices, feeds them to IALSRecommender through Optuna search and the final refit, persists the encoder state in the signed artifact, and uses it at serve time to recommend for cold-start users and for seed items absent from training. Highlights: - Feature tables come from the existing datasource registry (csv / parquet / bigquery / sql / plugins); the block's presence enables feature-aware training with no separate flag. - `features.*.source` reuses the same path-scheme allow-list, chained-scheme rejection, embedded-credential rejection, sha256-for-network rule, and plugin no_expand_fields as the top-level `source`. - Cold start reaches irspack's feature API on `:recommend` and `:recommend-related` (features only; features plus ad-hoc history; and a seed item absent from training). - The encoder state is persisted as plain Python containers only, so the artifact FQCN allow-list is unchanged; a new `features` header descriptor gates the payload shape and fails closed on unknown / newer / malformed versions. - Axis alignment between the search phase and the final refit is enforced structurally (encode requires an explicit row order); partial feature injection is refused with an unconditional raise. - RECOTEM_MAX_FEATURE_DIM caps the encoded dimension per side before the cubic solve. Robustness and observability: - Refuse an all-dead feature block (including an all-dead-numerical block, which a plain n_features check missed) so an artifact never advertises features while serving bias-only. - Count a finite-but-huge cold-start numerical value instead of injecting float32 inf. - Cap cold-start feature request value length (422 on violation). - Map an unrepresentable numerical column to a training-domain error (exit 4) and reject a complex-valued column explicitly. - Reject at recipe load a features id_column that also names a feature column. Includes docs, an examples/feature-aware/ walkthrough, and unit / integration / fuzz tests. --- CHANGELOG.md | 125 ++ CLAUDE.md | 9 + docs/api-reference.md | 209 ++- docs/operations.md | 141 +- docs/plugin-authoring.md | 28 +- docs/recipe-reference.md | 210 ++- docs/security.md | 182 ++ examples/feature-aware/README.md | 184 ++ examples/feature-aware/interactions.csv | 145 ++ examples/feature-aware/items.csv | 16 + examples/feature-aware/recipe.yaml | 64 + src/recotem/_features.py | 824 ++++++++ src/recotem/_idmap.py | 470 ++++- src/recotem/cli.py | 74 +- src/recotem/config.py | 28 + src/recotem/log_redaction.py | 16 +- src/recotem/recipe/loader.py | 404 ++-- src/recotem/recipe/models.py | 324 +++- src/recotem/serving/app.py | 16 + src/recotem/serving/metrics.py | 148 +- src/recotem/serving/routes.py | 940 +++++++--- src/recotem/serving/schemas.py | 106 +- src/recotem/serving/watcher.py | 15 + src/recotem/training/algorithms.py | 21 + src/recotem/training/features.py | 464 +++++ src/recotem/training/pipeline.py | 145 +- src/recotem/training/search.py | 141 +- src/recotem/training/split.py | 78 +- tests/integration/test_serve_predict_e2e.py | 522 ++++++ tests/unit/test_cli.py | 236 +++ tests/unit/test_config_feature_dim.py | 33 + tests/unit/test_features.py | 1656 +++++++++++++++++ tests/unit/test_features_compat.py | 276 +++ tests/unit/test_idmap.py | 665 +++++++ tests/unit/test_log_redaction.py | 31 + tests/unit/test_recipe_loader.py | 713 +++++++ tests/unit/test_recipe_models.py | 203 ++ tests/unit/test_serving_cold_start.py | 1640 ++++++++++++++++ tests/unit/test_serving_metrics.py | 115 ++ tests/unit/test_serving_schemas.py | 76 + tests/unit/test_training_algorithms.py | 23 + tests/unit/test_training_features.py | 674 +++++++ tests/unit/test_training_pipeline.py | 856 +++++++++ .../test_training_pipeline_train_error.py | 12 +- tests/unit/test_training_search.py | 426 ++++- tests/unit/test_training_split.py | 124 +- tests/unit/test_v1_batch_recommend_related.py | 115 ++ tests/unit/test_v1_error_handling.py | 4 + tests/unit/test_v1_metrics_cardinality.py | 2 + tests/unit/test_v1_recommend.py | 2 +- tests/unit/test_v1_recommend_related.py | 7 +- tests/unit/test_v1_status_labels.py | 102 + 52 files changed, 13439 insertions(+), 601 deletions(-) create mode 100644 examples/feature-aware/README.md create mode 100644 examples/feature-aware/interactions.csv create mode 100644 examples/feature-aware/items.csv create mode 100644 examples/feature-aware/recipe.yaml create mode 100644 src/recotem/_features.py create mode 100644 src/recotem/training/features.py create mode 100644 tests/unit/test_config_feature_dim.py create mode 100644 tests/unit/test_features.py create mode 100644 tests/unit/test_features_compat.py create mode 100644 tests/unit/test_serving_cold_start.py create mode 100644 tests/unit/test_training_features.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e51860d0..353f7bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,100 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `RECOTEM_ALLOW_IRSPACK_VERSION_SKEW` — truthy downgrades the skew check to a warning, for operators who know their artifact's algorithm is unaffected. - `recotem_artifact_load_failures_total` gained a `version_skew` reason label. +- **Feature-aware iALS.** A new optional `features:` recipe block (sibling to + `source:` / `item_metadata:`) declares item- and/or user-side attribute + tables — `categorical` (one-hot), `numerical` (standardized), and + `multi_label` (multi-hot) encodings, plus an implicit bias column — that + are encoded and fed to `IALSRecommender` during Optuna search and the + final refit. The mere presence of `features:` turns this on; there is no + separate flag. `lambda_item_feature` / `lambda_user_feature` are tuned by + Optuna over a new recotem-owned range (`5e-2`–`1e6`, log-scale) rather than + irspack's own `default_suggest_parameter`, because irspack ships no default + range for them and their `0.0` constructor default is a hard error whenever + the matching feature matrix is non-empty. See + `docs/recipe-reference.md#features`. +- **Cold-start serving from side features.** `POST + /v1/recipes/{name}:recommend` accepts `user_features` to score an unknown + user from their profile alone; `POST /v1/recipes/{name}:recommend-related` + accepts `user_features` (profile prior added to an ad-hoc seed history) and + `item_features` (keyed by seed id, for seed items absent from training). + Both single and batch verbs support this. A known `user_id`'s supplied + `user_features` are deliberately **ignored**, not rejected — the learned + embedding from real interactions strictly dominates a profile prior. A + request that supplies feature values against a model with no matching + feature state gets a new `400 FEATURES_NOT_SUPPORTED` rather than a guess. + Separately, a supplied `numerical` value whose *standardized* magnitude + (raw value standardized against the column's training mean/std — not the + raw value itself) is large enough to make irspack's per-request cold-start + solve itself fail gets a new `400 FEATURE_VALUE_UNUSABLE` rather than an + unhandled `500` — distinct from `FEATURES_NOT_SUPPORTED` because the model + and feature side both support cold start here; only this particular value + does not. The `detail` message describes the standardized value as + numerically unusable, not the client's raw one, because a column with a + small enough training std can make an entirely ordinary raw value (e.g. + `10000`) standardize to an unusable magnitude just as easily as an + actually-extreme raw value against a normal-sized std. A `numerical` value + large enough to be meaningless but not large enough to break the solver is + **not** caught by this and degrades silently as `200` instead — clamping + that range was a deliberate, deferred modelling decision, not an + oversight. A non-finite supplied value (`Infinity`/`-Infinity`, or a + string like `"nan"`) increments `recotem_v1_feature_unknown_value_total` + rather than degrading invisibly; a missing or otherwise unparseable value + still degrades silently with no signal, unchanged. See + `docs/api-reference.md#feature-aware-cold-start`. +- `RECOTEM_MAX_FEATURE_DIM` (default 5000, clamped [16, 100000]) — caps the + encoded feature dimension per side. The vocabulary is built from the whole + fetched feature table (so cold-start entities are representable), which + means encoded dimension scales with **catalog size, not interaction + count**; `min_frequency` on high-cardinality columns is the only + recipe-level lever. Cost is cubic in this number and multiplies with + `training.parallelism`. See `docs/operations.md#feature-aware-ials-sizing`. +- Artifact headers for feature-aware models gain a `features` block + (`{"version": 1, "item": {...}, "user": {...}}`), inspectable via `recotem + inspect`. Serve checks this version before deserializing the payload: + absent → loads (old artifact or non-feature model); present but + unrecognized → refused (`ArtifactError`, reason `feature_version`) rather + than risk silently mis-encoding a request's features into the wrong vector + space. +- New metrics: `recotem_v1_feature_unknown_value_total` (a request's + categorical/multi_label value was absent from the training vocabulary, or + a numerical value was non-finite — degrades to an all-zero segment / + contributes nothing rather than failing the request) and + `recotem_v1_cold_start_requests_total` (cold-start traffic by case). +- New example: `examples/feature-aware/` — a small interactions CSV, an item + feature table exercising all three encodings, and a README walking + train → serve → cold-start `:recommend-related`. ### Changed +- **A numerical `features:` column with a tiny-but-nonzero training std is + now treated as zero-variance, like an exactly-constant column.** Previously + only an exact `std == 0.0` was floored; a column whose values differ only + by floating-point rounding noise (e.g. `std ≈ 1e-15`) passed that check but + still divided serve-time standardization by a near-zero denominator, + turning an ordinary request value into an astronomically large + standardized one and a false `400 FEATURE_VALUE_UNUSABLE`. + `build_encoder_state` now floors any std no larger than `1e-8 × + max(abs(mean), 1.0)` (relative to the column's own scale) to zero. A + column caught by this floor degrades exactly like a missing value + (`feature_zero_variance_column` warning, unchanged) instead of ever + reaching the standardization divide. This changes training-time encoding + for any feature table containing such a column; retrain to pick it up. See + `docs/api-reference.md#feature-aware-cold-start`. +- **Every recipe's `recipe_hash` changes on upgrade, features or not.** The + hash is computed by JSON-dumping the whole recipe with no `exclude_none`, + so adding the new optional `features` field emits `{"features": null}` for + every existing recipe and changes its hash — the same effect + `item_metadata` already has when absent. Nothing in Recotem compares or + gates on `recipe_hash` today; it is carried through to the artifact header + (`recotem inspect`), the `train_done` log event, and the + `GET /v1/recipes/{name}` response purely for operators' own SIEM/audit + rules. The inference verbs do not echo it: `:recommend` returns + `request_id` / `recipe` / `model_version` / `items` only. If you pin or + diff `recipe_hash` in external tooling, expect every recipe to show a + changed hash on this upgrade even though nothing about the recipe's + behavior changed. + - **irspack upgraded from 0.4.2 to 0.5.0.** irspack 0.5.0 adds feature-aware iALS, cache/Eigen performance work, and a reworked tuning API. Recotem drives Optuna itself and does not call `BaseRecommender.tune`, so none of irspack's @@ -47,6 +138,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 bit-exact, pin sklearn exactly or build train and serve from the same lock file. +### Fixed + +- **Feature-aware iALS: an all-dead-numerical `features:` block is now + refused.** The whole-block-dead guard keyed on `n_features == 1`, which a + block whose only column is a zero-variance (or all-null) `numerical` column + escaped — a numerical column always reserves width 1, so `n_features` stayed + 2 even though it emits nothing. Such a block would sign an artifact + advertising `features` while serving bias-only (== plain iALS). The guard now + refuses a block when no column can emit a non-bias feature, matching the + existing all-categorical-dead and zero-id-overlap refusals. +- **Feature-aware iALS: a finite-but-huge cold-start `numerical` value no + longer injects `inf`.** The non-finite check tested the raw parsed value, but + the matrix stores the value standardized and cast to `float32`; a value + finite in float64 whose standardized magnitude exceeds float32's max became + `±inf` in the matrix and was not counted. It is now counted as an unknown + value (`recotem_v1_feature_unknown_value_total`) and contributes nothing, + like any other unusable value. +- **Cold-start feature request values are now length-capped.** Each string + value in `user_features` / `item_features` is capped at 8192 characters + (`422` on violation, like every other request-schema cap). Previously only + the key count was capped, leaving a single string value unbounded — a + memory-amplification vector via + `multi_label` tokenization, reachable with one API key and multiplied by + batch/related fan-out. The cap covers the batch verbs too. +- **Feature-aware iALS training: an unrepresentable `numerical` column fails + with a training-domain error, not exit 1.** A `numerical` column carrying a + Python int too large for float64 (`>= 309` digits) raised an unmapped + `OverflowError` (exit 1) from the fit's own parser; it now raises a + `TrainingError` (exit 4) naming the column. A complex-valued column, which + previously trained silently on its real part, is now rejected explicitly. +- **Recipe load rejects a `features..id_column` that also names a feature + column.** The collision is guaranteed to fail at train time (the id column is + consumed as the index); it is now caught at recipe load with a clear message. + ### Migrating to irspack 0.5.0 irspack 0.5.0 changed `IALSModelConfig`'s pickled state from a 7-tuple to a diff --git a/CLAUDE.md b/CLAUDE.md index 43ccb8e8..3e5c68a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,10 +38,12 @@ src/recotem/ ├── recipe/ pydantic v2 Recipe + YAML loader + env expansion ├── datasource/ DataSource Protocol + entry_points discovery (csv / parquet / bigquery / sql) ├── training/ Optuna search + irspack train; per-recipe file lock +│ └── features.py fetch feature tables, build encoder state, encode per phase (search vs. final refit) ├── artifact/ HMAC-signed binary container with FQCN allow-list ├── metadata/ item metadata loader (CSV/Parquet via fsspec) ├── serving/ FastAPI app, ModelRegistry, ArtifactWatcher ├── _idmap.py Neutral home for IDMappedRecommender (canonical FQCN) +├── _features.py Neutral home for side-feature encoder state + pure encode logic (shared by training/ and serving/) ├── _irspack_compat.py Verified-compatible allow-list guarding irspack pickle skew ├── _http_fetch.py SSRF-guarded HTTP/HTTPS fetcher with sha256 verify ├── _size_cap.py Shared download-size cap helper (used by csv source + metadata loader) @@ -120,6 +122,12 @@ See `docs/recipe-reference.md` for the full schema. Highlights: - Cleansing block: `drop_null_ids`, `dedup` policy, `min_rows / min_users / min_items` data preconditions. - Multi-algorithm Optuna search with optional per-algorithm trial budgets. +- Optional `features:` block (sibling to `source:` / `item_metadata:`) turns + on feature-aware iALS — no separate flag. `features.item` / `features.user` + each declare a `source` (same datasource registry as the top-level + `source`), an `id_column`, and a `columns` list of `{name, encoding, + delimiter?, min_frequency?}` (`categorical` | `numerical` | `multi_label`). + See `docs/recipe-reference.md#features`. ## Artifact format @@ -221,6 +229,7 @@ uv run ruff format --check src tests | `RECOTEM_STARTUP_PARALLELISM` | (empty = auto) | Number of parallel threads used to load artifacts at `recotem serve` startup. Leave unset (default) for auto-sizing (`min(len(recipes), 8)`). Setting to `0` is NOT a sentinel — it clamps to 1 and emits an `env_var_clamped` warning. Clamped [1, 32]. Set to `1` to force sequential loading for debugging. | | `RECOTEM_MAX_SQL_ROWS` | 50_000_000 | Hard cap on rows returned by the SQL data source. Clamped [1_000, 500_000_000]. Caps **row count**, not DataFrame resident memory — see `docs/data-sources/sql.md` for the memory-bound caveat. | | `RECOTEM_SQL_ALLOW_PRIVATE` | (empty) | Truthy opts the SQL source into private/loopback DSN hosts (default deny, for SSRF). Covers every driver-routing form — netloc, `?host=`, `?hostaddr=`, `?service=`, `?unix_socket=`, absolute-path host, and network DSNs with no host info — all default-deny without this flag. Also disables the DNS-rebinding re-check before each probe/fetch — opting in means trusting the host end-to-end. | +| `RECOTEM_MAX_FEATURE_DIM` | 5000 | Cap on the encoded feature dimension per side (item and user checked independently) for feature-aware iALS. Clamped [16, 100000]. Vocabulary is built from the whole fetched feature table (not just interaction-covered rows), so dimension scales with **catalog size, not interaction count**; `min_frequency` on high-cardinality `categorical`/`multi_label` columns is the only recipe-level lever. Cost is cubic in this number (dense `Fᵀ F` Cholesky) and multiplies with `training.parallelism`. See `docs/operations.md#feature-aware-ials-sizing`. | ## CI diff --git a/docs/api-reference.md b/docs/api-reference.md index ed188718..60a84ceb 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -22,10 +22,11 @@ recipe-name constraint enforced by the recipe loader). | `user_id` | string | yes | – | 1-256 chars | | `limit` | int | no | 10 | 1..1000 | | `exclude_items` | string[] \| null | no | null | ≤1000 items | +| `user_features` | object \| null | no | null | Raw feature values, keyed by the recipe's `features.user` column names. See [Feature-aware cold start](#feature-aware-cold-start) below. ≤64 keys. | **Response body:** see `RecommendResponse` in `src/recotem/serving/schemas.py`. -**Status codes:** 200, 401, 404 (`UNKNOWN_USER` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_USER` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). ### `POST /v1/recipes/{name}:recommend-related` Seed-item → items. @@ -37,14 +38,19 @@ Seed-item → items. | `seed_items` | string[] | yes | – | 1-100 items | | `limit` | int | no | 10 | 1..1000 | | `exclude_items` | string[] \| null | no | null | | +| `user_features` | object \| null | no | null | Raw feature values, keyed by the recipe's `features.user` column names. Adds a profile prior to the seed-history solve. See [Feature-aware cold start](#feature-aware-cold-start). ≤64 keys. | +| `item_features` | object[string, object] \| null | no | null | Raw feature values for seed items absent from training, keyed by seed item id. ≤100 keys; each value ≤64 keys. See [Feature-aware cold start](#feature-aware-cold-start). | -**Status codes:** 200, 401, 404 (`UNKNOWN_SEED_ITEMS` | `NO_CANDIDATES` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_SEED_ITEMS` | `NO_CANDIDATES` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). `UNKNOWN_SEED_ITEMS` means none of the supplied `seed_items` were known to the model id-map (typically a client-side data issue). `NO_CANDIDATES` means at least one seed was known but the ranker did not produce any survivors after its internal filtering — typically a data -distribution issue rather than a client mistake. +distribution issue rather than a client mistake. `NO_CANDIDATES` is only +possible on the pre-existing "all seeds known, no features supplied" path; +see [Feature-aware cold start](#feature-aware-cold-start) for why the two +feature-aware branches never raise it. ### `POST /v1/recipes/{name}:batch-recommend` Multi-user batch. Body: `{ "requests": RecommendRequest[], "include_metadata": bool }` (1..256). @@ -52,6 +58,12 @@ Response: `BatchRecommendResponse`. Per-element `status` ∈ {ok, error}. HTTP 200 on partial failure; HTTP 503 only when the recipe itself is unavailable. +Each element accepts `user_features` exactly as the single `:recommend` +endpoint does (see [Feature-aware cold start](#feature-aware-cold-start)); +an element whose model has no matching feature state surfaces as +`status=error, code=FEATURES_NOT_SUPPORTED` rather than failing the whole +batch. + `include_metadata` (default `false`): when `true`, each `ok` result includes per-item metadata fields (same join as the single-recommend endpoint). Default `false` preserves the performance-first default for @@ -78,6 +90,31 @@ Multi-seed batch. Body: `{ "requests": RecommendRelatedRequest[], "include_meta Same aggregate-limit, per-element validation rules, and `include_metadata` semantics as `:batch-recommend`. +**Cold-seed solve cap.** This verb carries a *second* aggregate cap that +`:batch-recommend` does not need. Case C runs one solve per cold seed, so +the aggregate count of cold seeds — `sum` over elements of the seeds named +in that element's `item_features` — must not exceed **512**. An element +that would push the running total over the cap surfaces as `status=error, +code=VALIDATION_ERROR`, exactly like the aggregate-`limit` cap, and later +elements continue to be processed. + +The two caps guard different dimensions and neither subsumes the other: +`sum(limit)` bounds response volume, while this bounds solver work. A batch +of `limit: 1` elements sits at 2% of the aggregate-`limit` cap while +demanding 25,600 solves. The count is taken from the request alone — a seed +named in `item_features` counts even if it turns out to be a known item +whose learned embedding is used instead — so the same body is always +accepted or rejected identically, regardless of which model is loaded. + +A single `:recommend-related` call cannot reach this cap: `seed_items` is +capped at 100, so a maximal single request is 100 solves. + +Each element accepts `user_features` / `item_features` exactly as the +single `:recommend-related` endpoint does, including the case A/B/C +precedence rules and the `200 {"items": []}` vs `NO_CANDIDATES` asymmetry +described in [Feature-aware cold start](#feature-aware-cold-start) — both +apply per-element here. + **Status codes:** 200, 401, 404 (`RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR` — only for whole-request shape), 503 (`RECIPE_UNAVAILABLE`). ### `GET /v1/recipes` @@ -108,6 +145,168 @@ Prometheus exposition. Excluded from OpenAPI. Requires **Requires `X-API-Key`** — configure your Prometheus scraper with an `authorization` block or `http_headers` accordingly. +## Feature-aware cold start + +`user_features` and `item_features` are only meaningful against a model +trained with a [`features:`](recipe-reference.md#features) block. They are +accepted (and validated) on every model, but a model with no matching +feature state (or whose search winner is not feature-capable — see +`docs/recipe-reference.md#features`) responds `400 FEATURES_NOT_SUPPORTED` +rather than silently ignoring the field or guessing. + +Three cold-start cases, spread across two verbs: + +| Case | Verb | Trigger | What it does | +|---|---|---|---| +| A — unknown user, features only | `:recommend` | `user_id` unknown, `user_features` present | Scores every known item against the profile alone (no interaction history exists yet for this user). | +| B — unknown user, features + ad-hoc history | `:recommend-related` | `user_features` present | Runs the same seed-history solve as the pre-existing path, with the profile added as a joint prior. This is a genuine joint solve, not either/or: it correlates with neither a features-only nor a history-only score alone. | +| C — unknown seed item(s) | `:recommend-related` | one or more `seed_items` absent from training, and a matching entry in `item_features` | Computes each cold seed's embedding from its features, averages it with the known seeds' learned embeddings, and scores as item-item similarity. | + +If a request supplies both a cold seed's `item_features` **and** +`user_features` on `:recommend-related`, case C wins: a cold seed has no row +in the seed-history matrix that case B's solve uses, so running case B alone +would silently drop that seed's contribution. Case C is the only path that +can actually use a cold seed's features. + +**A known `user_id` with `user_features` supplied is not an error.** The +learned embedding from that user's real interaction history strictly +dominates a profile prior, so the server always prefers it and simply +**ignores** the supplied `user_features` — it does not reject the request. +This lets a client always send the user's profile on every request without +needing to know in advance whether the user is new or returning. + +**Unknown feature values degrade, they do not fail the request.** What +"degrade" means, and whether `recotem_v1_feature_unknown_value_total` (see +[operations.md](operations.md#feature-aware-ials-sizing)) actually catches +it, differs by encoding: + +- `categorical` — a value absent from the training vocabulary encodes to an + all-zero segment for that column, and the counter increments. +- `multi_label` — each token is looked up independently: known tokens are + retained (each contributing exactly one `1.0` to its dimension, even if + the token is repeated in the input — see the multi-hot note below), + unknown tokens are dropped. The counter increments whenever **any** + supplied token misses the vocabulary, even if other tokens in the same + value are known. A mixed value such as `"Action|Thrller"` sets the bit + for the known token, drops `Thrller`, and still increments the counter — + a partial typo is caught, not silently absorbed. +- `numerical` — a **missing** value (absent, `null`, or `NaN`) or a value + that fails to parse as a number at all contributes nothing to the row, + equivalent to encoding the standardized mean (`0`), and does **not** + increment the counter. A value that DOES parse as a number but is + **non-finite** (`Infinity` / `-Infinity` — valid in JSON per Python's + parser extension — or a `NaN` reached via a string like `"nan"`) also + contributes nothing to the row, but this case **does** increment the + counter: it is a real, present value the server could not use, not an + absent one. + +Do not rely on this counter as a general typo detector for `numerical` +columns: a **missing or unparseable** value still degrades the +recommendation with no signal at all — only the non-finite case above is +covered. `categorical` and `multi_label` are both reliably covered. + +`multi_label` is multi-**hot**, not a count vector: `"rock|pop|rock"` +contributes `1.0` to the `rock` dimension, not `2.0` — duplicate tokens in +one value are deduplicated before encoding, both at training time and for a +cold-start request's `item_features` / `user_features`. + +**A large `numerical` value degrades silently across a wide range; only the +extreme tail is a hard 400.** Unlike the missing/unparseable case above, a +`numerical` value is standardized at serve time by dividing the raw request +value by the column's *training* mean/std — a fit the request's own value +was never part of (see the "Training is unaffected" note below for why that +matters). Nothing clamps how large the resulting magnitude may get, so +behavior is NOT a clean two-way split ("normal" vs. "hard 400"). An actual +sweep against a column with training std ≈ 0.425 found: + +| value | result | +|---|---| +| `0.3` | `200`, small, normal-looking score | +| `100` | `200`, but the score is already visibly degenerate (order alone, no longer proportional to the profile) | +| `1e6` – `1e18` | `200`, score grows without bound (into the hundreds of millions and beyond) as the value grows | +| ~`1e19`+ | `400 FEATURE_VALUE_UNUSABLE` — only here does irspack's per-request cold-start solver itself give up | + +So roughly `1e2` through `1e18` in this measurement is a **silent degrade**: +`200`, an unbounded and effectively meaningless score, a fixed/degenerate +ranking — and none of these finite values touch +`recotem_v1_feature_unknown_value_total` (per the counter note above, that +counter fires for a `numerical` value only when it is non-finite), so +nothing server-side signals that this happened either. The 400 only fires +once the standardized magnitude is large enough to make the underlying +conjugate-gradient solve singular; **the exact crossover is not a fixed +constant** — it depends on the column's training std and the BLAS +implementation solving the system, so do not hard-code a boundary value +(e.g. `1e22`) as a contract. + +**The 400's `detail` message describes the standardized value, not the +client's raw one — because the raw value need not be extreme.** A column +whose training std is small enough (see the near-constant-column note +below) can make an ordinary raw value like `10000` standardize to a +magnitude that breaks the solver, exactly like `1e22` does against a +normal-sized std. The `detail` string therefore never claims the supplied +value itself was extreme; it says the resulting *standardized* value was +numerically unusable for this model's cold-start scoring, which is true +regardless of which side (raw magnitude vs. tiny std) produced it. + +**A near-constant column is a special case of a small std, not a separate +bug — and training floors the most common cause of it.** A column whose +values are "the same number" up to floating-point rounding noise (e.g. +`std ≈ 1.36e-15`, not exactly `0.0`) would otherwise divide serve-time +standardization by a near-zero denominator, turning a routine value like +`10000` into an astronomically large standardized one — an ordinary client +value producing a 400 for a reason the client cannot see. `build_encoder_state` +(`_features.py`) floors a numerical column's training-time std to zero +whenever it is no larger than a relative tolerance of the column's own +scale (`1e-8 × max(abs(mean), 1.0)`) — tight enough to preserve real, +intentional small variance while absorbing realistic floating-point +rounding noise. A column caught by this floor never reaches the +standardization divide at all: it degrades exactly like a missing value +(logged once as `feature_zero_variance_column`), never a 400. This is a +**training-time behavior change**: a column that previously stood a chance +of triggering `FEATURE_VALUE_UNUSABLE` for a near-constant reason now never +does. It does not eliminate the phenomenon in general — a column with +genuine (not rounding-noise) small variance just above the floor still +standardizes an ordinary value to an unusable magnitude by the same +mechanism as the sweep above, which is exactly why the `detail` message +above is worded the way it is rather than promising the raw value was at +fault. + +**Clamping the standardized magnitude before it reaches the solver — which +would close the silent-degrade band above — was deliberately deferred, not +overlooked.** Picking a clamp bound (how many training standard deviations +is "too many") is a modelling decision that changes what every downstream +consumer of the same encoding sees, including training, not a bugfix to the +400 path added here; it was intentionally scoped out of this fix. This +deferral was previously disclosed nowhere — this paragraph is that +disclosure. + +Training is unaffected either way: the same value flowing through +training-time encoding is untouched by this guard, which only wraps the +serve-time cold-start solve. (Training has its own, much stronger bound: a +numerical column's training-time mean/std are computed from the same values +being standardized, so an outlier inflates the very std it is divided by — +this caps the worst-case training-time standardized magnitude at roughly +`(n_rows - 1) / sqrt(n_rows)` no matter how extreme the raw value is, which +is nowhere near the magnitude needed to break the solver. Serve-time has no +such self-bound, because the request's value is standardized against a +std fit without it.) + +**A pre-existing API asymmetry, documented rather than fixed.** +`:recommend-related`'s original all-seeds-known branch (no `user_features`, +no cold seeds) returns `404 NO_CANDIDATES` when the ranker produces zero +survivors after its own filtering. The two feature-aware branches (B and C) +never raise `NO_CANDIDATES` — an empty result from either comes back as +`200 {"items": []}`. `:recommend` never had a `NO_CANDIDATES` code at all and +returns `200`/`[]` in every case, so it is internally consistent already; +`:recommend-related` is the one verb where the behavior differs by branch. +This was a deliberate call, not an oversight: for a cold-start profile or a +cold seed item, producing nothing is a property of an unproven input, not +evidence that the ranker itself failed — so `200 {"items": []}` was judged +the more defensible response. If your client treats an empty +`:recommend-related` result as actionable (e.g. falling back to +popularity-based recommendations), branch on `items == []` rather than on +HTTP status for this verb. + ## Headers - `X-Request-ID` — accepted (regex `^[A-Za-z0-9_-]{1,128}$`) or generated; @@ -166,8 +365,10 @@ in every error case is one of the three forms above. | `RECIPE_NOT_FOUND` | 404 | no such recipe in registry | | `UNKNOWN_USER` | 404 | user not in idmap | | `UNKNOWN_SEED_ITEMS` | 404 | none of seed_items known to model | -| `NO_CANDIDATES` | 404 | seeds known, but ranker produced no survivors | +| `NO_CANDIDATES` | 404 | seeds known, but ranker produced no survivors (only reachable on `:recommend-related`'s non-feature-aware path — see [Feature-aware cold start](#feature-aware-cold-start)) | | `VALIDATION_ERROR` | 422 | Pydantic schema rejected the request (also used per-element inside batch responses) | +| `FEATURES_NOT_SUPPORTED` | 400 | `user_features` / `item_features` supplied but the model has no matching feature state, or its search winner is not feature-capable (also used per-element inside batch responses) | +| `FEATURE_VALUE_UNUSABLE` | 400 | a supplied `numerical` feature value, once standardized against the column's training mean/std, is large enough to make irspack's cold-start solver itself fail (the exact threshold is std/BLAS-dependent, not a fixed constant, and depends on the column's std as much as the raw value — see [Feature-aware cold start](#feature-aware-cold-start)) — the model and feature side both support cold start, but this particular value does not. Values large enough to be meaningless but not large enough to break the solver degrade silently as `200` instead (also used per-element inside batch responses) | | `MISSING_API_KEY` | 401 | `X-API-Key` header missing | | `INVALID_API_KEY` | 401 | `X-API-Key` header present but did not match any configured digest (also covers short-key / oversize-key rejections so callers cannot fingerprint the guard) | | `INTERNAL_ERROR` | 500 / batch | unhandled server-side exception, or unexpected recommender internal layout (`recommender_layout_unexpected`) — status=500 on single endpoints; per-element `status=error` inside batch responses | diff --git a/docs/operations.md b/docs/operations.md index 6a22c4c7..9dfca4d2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -222,7 +222,7 @@ logic instead of grepping stderr. | 1 | Unknown error | Bug, environment issue, schema generation failure | | 2 | RecipeError | YAML syntax, schema violation, invalid `--env-var`, `--dev-allow-unsigned` without companion confirmation flag, `--dev-allow-unsigned` outside `RECOTEM_ENV=development` | | 3 | DataSourceError | Source-layer failure NOT during HTTP fetch — CSV/Parquet format error, required column missing, local-FS path not found, BigQuery schema mismatch | -| 4 | TrainingError | Includes subcodes `signing_key_missing`, `min_data_violation`, `time_column_parse_error`, `final_training_error`, `no_completed_trials`, `zero_score`, `excessive_per_trial_timeouts` | +| 4 | TrainingError | Includes subcodes `signing_key_missing`, `min_data_violation`, `time_column_parse_error`, `final_training_error`, `no_completed_trials`, `zero_score`, `excessive_per_trial_timeouts`, `feature_table_error`, `feature_axis_error`, `feature_cholesky_error` | | 5 | ArtifactError | Magic mismatch, kid unknown, HMAC mismatch, payload over cap, disallowed FQCN, header JSON over cap | | 6 | LockContestedError | Recipe lock held by another process when `--fail-on-busy` is set | | 7 | HttpFetchError | Any failure during HTTP/HTTPS source fetch — SSRF guard refused the destination, connect/read timeout, HTTP 4xx/5xx, body cap exceeded, redirect cap, scheme-changing redirect, sha256 mismatch on a network-fetched source | @@ -249,8 +249,10 @@ as the basis for SLO and alerting rules. | `training_started` | start | `recipe`, `run_id` | | `fetching_data` | datasource | — | | `data_fetched` | datasource | `n_rows` | +| `feature_table_loaded` | features | `side`, `n_rows`, `n_features`, `columns` (names only — feature values are user PII and are never logged). Only with a `features:` block; emitted before cleansing, since the feature tables are fetched up front. | | `data_cleansed` | cleansing | `n_rows`, `drop_count` | | `splitting_data` / `split_done` | split | `val_offset` | +| `feature_axis_coverage` | features | `side`, `matched`, `total` — how many ids of the axis being encoded the feature table covers. Emitted per side per phase (once for search, once for the final refit). Zero coverage does not emit this event; it aborts with `feature_axis_error` instead. | | `search_started` | tuning | `algorithms`, `n_trials` | | `search_done` | tuning | `best_class`, `best_score`, `n_completed` | | `training_final_model` / `final_model_trained` | refit | `recommender` | @@ -276,7 +278,7 @@ Additional events emitted by the watcher, recipe loader, and size-cap helper tha | `auth_anonymous_bypass_first_seen` | INFO | `serving/auth.py` | First anonymous request from a given `client_host` (per process). The LRU cache tracking first-seen IPs is bounded to 1024 entries to prevent unbounded memory growth. | | `kid_extraction_failed` | WARN | `serving/watcher.py` | An artifact's kid bytes could not be parsed from the raw bytes (too short, out-of-range length, decode error). The kid shown in subsequent log fields is `\x00` — intentionally not collidable with any real kid. | | `artifact_stat_timeout` | WARN | `serving/watcher.py` | A stat() future did not complete within the per-future timeout (`min(watch_interval, 30)` seconds). Hung object-store stats no longer block tick progress or delay SIGTERM handling. | -| `recommender_layout_unexpected` | WARN | `serving/routes.py` | `_any_seed_known` encountered an `AttributeError` on `recommender._mapper.item_id_to_index`. The request is treated as `INTERNAL_ERROR`. Increment counter: `recotem_recommender_layout_unexpected_total`. | +| `recommender_layout_unexpected` | WARN | `serving/routes.py` | `_resolve_recommend` / `_resolve_recommend_related` encountered an `AttributeError` on `recommender._mapper.user_id_to_index` / `item_id_to_index`. The request is treated as `INTERNAL_ERROR`. Increment counter: `recotem_recommender_layout_unexpected_total`. | | `set_load_error_no_entry` | WARN | `serving/watcher.py` | The watcher tried to mark a load error on a recipe with no registry entry. Counter: `recotem_watcher_state_divergence_total`. | | `sidecar_disappeared` | WARN | `serving/watcher.py` | A `.sha256` sidecar file was present on the previous poll but raised ENOENT on the current read — emitted once per disappearance transition. | | `metadata_index_row_error` | WARN | `metadata/loader.py` | A per-row exception occurred during `build_metadata_index`. The row is skipped. Counted by `recotem_metadata_index_build_errors_total{recipe}`. | @@ -382,6 +384,103 @@ For large models (IALS with many components, large item sets), use `recotem insp --- +## Feature-aware iALS sizing + +A recipe's [`features:`](recipe-reference.md#features) block adds costs that +scale differently from the rest of a recotem recipe. All four points below +apply only when `features:` is present. + +### Vocabulary scales with catalog size, not interaction count + +The most surprising operational property of this feature: the encoded +dimension is built from the **whole fetched feature table**, not from the +subset of items/users that actually appear in the interaction data — this is +what lets a cold-start item or user be scored at serve time even though it +never appears in training. The consequence is that a 1M-item catalog whose +interactions cover only 1,000 of those items still pays the full encoded +dimension — and the full per-trial training cost below — for the other +999,000 items, even though their columns are only ever useful for cold-start +requests that may never arrive. + +`RECOTEM_MAX_FEATURE_DIM` (default 5000, clamped [16, 100000]) caps the +encoded dimension per side (item and user are checked independently); +exceeding it raises `TrainingError` (exit 4) at the point the encoder state is +built. `min_frequency` (recipe-level, per column; see +[recipe-reference.md](recipe-reference.md#features)) is the operator's +**only** lever against this cap — raise it on high-cardinality `categorical` / +`multi_label` columns to shrink the vocabulary. There is no way to restrict +the vocabulary to interaction-covered rows from the recipe. + +Be precise about what that lever moves: `min_frequency` bounds the resulting +**dimension**, not the memory spent discovering it. `_vocabulary` counts every +token of the fetched column into a dict and only then prunes, and the +`multi_label` branch first flattens every row's tokens into a single list, so +a high-cardinality column pays its full transient counting cost no matter how +aggressive `min_frequency` is — a column with hundreds of thousands of +distinct values costs tens of MB to count even when the pruned vocabulary +comes back empty. The `RECOTEM_MAX_FEATURE_DIM` check runs **after** every +column's vocabulary is built, so that transient is paid in full even on the +run the cap then rejects. `min_frequency` protects the trials; it does not +protect the encoder-state build. + +### Per-trial time is cubic, memory is quadratic, and both multiply with `training.parallelism` + +irspack forms a dense `Fᵀ F` Gram matrix per side and solves it by Cholesky +decomposition. The two costs scale differently and are worth keeping apart +when sizing a host: **time** grows **cubically** with the encoded dimension +(the decomposition itself), while **memory** grows only **quadratically** — +the Gram matrix is `dim² × 8` bytes at float64, which is exactly what the +Memory column below reports. irspack never errors from either — it only +degrades. Measured per trial: + +| Encoded dimension | Time | Memory | +|---|---|---| +| 5,000 | ~0.6 s | ~200 MB | +| 10,000 | ~4.2 s | ~771 MB | +| 20,000 | ~43 s | ~3 GB | + +`training.parallelism` is Optuna `n_jobs` — **in-process threads**, not +processes — so each concurrently-running trial builds and solves its own +dense Gram matrix independently. At `parallelism=4, dim=10k` that is roughly +4 × 771 MB ≈ 3 GB of Gram matrices alone, on top of everything else the +search holds in memory. Size training hosts (or set `parallelism` and +`RECOTEM_MAX_FEATURE_DIM`) with this multiplication in mind. + +### Payload and serve-side RSS grow with catalog size, not just dimension + +irspack retains `self.item_features` (and `self.user_features`) on the +trained recommender and defines no `__getstate__`, so the encoded feature +matrix is pickled into the artifact payload verbatim. Size scales with +`n_items × nnz_per_row`, not with the encoded dimension alone: projected, +1M items × 500 encoded dimensions × 5 non-zero entries/row ≈ 42 MiB; 1M items +× 5,000 dimensions × 10 non-zero entries/row ≈ 80 MiB — material against the +512 MiB `RECOTEM_MAX_PAYLOAD_BYTES` default but not by itself fatal. +`RECOTEM_MAX_FEATURE_DIM` caps **columns**; nothing caps `n_items × +nnz_per_row`, so a very large catalog with dense per-row encodings (many +`multi_label` tags, low `min_frequency`) can still produce a large payload +even with a modest encoded dimension. The identical bytes also count against +serve-side resident memory (see +[Sizing `recotem serve` memory](#sizing-recotem-serve-memory) above) once the +artifact is loaded. + +### Cold-start latency, and `n_threads` + +Cold-start scoring is an iterative CG solve, not a matrix lookup. Measured +latency (1,000 items, 64 components): a single cold-start request takes +300–500 µs median; batching amortizes this to **8–12 µs/user** — a 30–40× +per-user improvement, which is why the batch verbs +(`:batch-recommend` / `:batch-recommend-related`) are the recommended path +for any bulk cold-start workload. + +The recommender's default `n_threads=16` measurably hurts **single-request** +latency: median 734–857 µs and p95 2.0–2.2 ms at the default, versus faster +at `n_threads` 1–4. `n_threads` is baked into the pickled model at training +time, and there is currently no serve-time override — if single-request +cold-start latency matters for your workload, this is a training-time +decision, not a serving-time one. + +--- + ## Environment variable reference Full list of environment variables recognised by Recotem. Variables marked `serve` apply only to `recotem serve`; those marked `train` apply only to `recotem train`; those with no marking apply to both. @@ -410,6 +509,7 @@ Full list of environment variables recognised by Recotem. Variables marked `serv | `RECOTEM_STARTUP_PARALLELISM` | (auto) | serve | Threads used to load artifacts at startup (clamped [1, 32]). Default: `min(len(recipes), 8)`. Setting to `0` clamps to 1 with a warning. | | `RECOTEM_BQ_REQUIRE_STORAGE_API` | (unset) | train | Truthy raises `DataSourceError` instead of falling back to the REST path when the BigQuery Storage Read API fails. | | `RECOTEM_ALLOW_IRSPACK_VERSION_SKEW` | (unset) | serve | Truthy downgrades the irspack version-skew refusal to a warning and lets the payload reach the deserializer. Does not make an incompatible payload loadable. See [irspack version skew](#irspack-version-skew). | +| `RECOTEM_MAX_FEATURE_DIM` | 5000 | train | Cap on the encoded feature dimension per side (item and user are checked independently), clamped [16, 100000]. See [Feature-aware iALS sizing](#feature-aware-ials-sizing). | | `RECOTEM_RECIPE_*` | — | train | Allow-listed prefix for `${...}` recipe env-var expansion. See [recipe-reference.md](recipe-reference.md#environment-variable-expansion). | > **Note on `signing_key_status` in logs.** The `security.posture` log line emitted at every `recotem serve` startup includes a `signing_key_status` field: `configured` (keys present), `dev_allow_unsigned` (no keys, dev-unsigned mode), or `missing` (keys absent; startup will fail). Use this in SIEM rules to alert on misconfigured deployments. @@ -459,14 +559,17 @@ Available metrics: | Metric | Type | Labels | Purpose | |--------|------|--------|---------| -| `recotem_v1_requests_total` | Counter | `recipe`, `verb`, `status` | v1 request volume; `status` ∈ {`ok`, `unknown_user`, `unknown_seed_items`, `no_candidates`, `recipe_not_found`, `unavailable`, `validation_error`, `error`} | +| `recotem_v1_requests_total` | Counter | `recipe`, `verb`, `status` | v1 request volume; `status` ∈ {`ok`, `unknown_user`, `unknown_seed_items`, `no_candidates`, `recipe_not_found`, `unavailable`, `validation_error`, `features_not_supported`, `feature_value_unusable`, `error`}. Every value except `error` is client-caused and expected in normal operation; `error` is reserved for genuine server faults (HTTP 500) — see [Monitoring SLIs](#monitoring-slis) | | `recotem_v1_request_latency_seconds` | Histogram | `recipe`, `verb` | per-verb end-to-end latency | | `recotem_v1_batch_size` | Histogram | `recipe`, `verb` | observed batch fan-out (only for `batch-recommend` / `batch-recommend-related`) | -| `recotem_v1_batch_element_errors_total` | Counter | `recipe`, `verb`, `code` | per-element errors inside batch HTTP-200 responses; `code` ∈ {`UNKNOWN_USER`, `UNKNOWN_SEED_ITEMS`, `NO_CANDIDATES`, `VALIDATION_ERROR`, `INTERNAL_ERROR`} | +| `recotem_v1_batch_element_errors_total` | Counter | `recipe`, `verb`, `code` | per-element errors inside batch HTTP-200 responses; `code` ∈ {`UNKNOWN_USER`, `UNKNOWN_SEED_ITEMS`, `NO_CANDIDATES`, `VALIDATION_ERROR`, `FEATURES_NOT_SUPPORTED`, `FEATURE_VALUE_UNUSABLE`, `INTERNAL_ERROR`} | | `recotem_v1_metadata_degraded_items_total` | Counter | `recipe`, `verb`, `kind` | items served with degraded metadata; `kind` ∈ {`fallback` (item_id/score only), `dropped` (omitted entirely)} | | `recotem_v1_validation_errors_outside_verb_total` | Counter | — | 422 errors on non-inference paths (e.g. `/v1/recipes` list with bad query) | +| `recotem_v1_feature_unknown_value_total` | Counter | `recipe`, `side`, `column` | `side` ∈ {`item`, `user`}. Fires on a `categorical` value absent from the training vocabulary, a `multi_label` value where any supplied token misses, or a non-finite `numerical` value (`+inf`/`-inf`, or a `NaN` reached via a string); a `numerical` value that is missing or fails to parse as a number at all still degrades the recommendation silently and is **not** counted — see [Feature-aware cold start](api-reference.md#feature-aware-cold-start) for the per-encoding breakdown | +| `recotem_v1_feature_unknown_column_total` | Counter | `recipe`, `side` | cold-start requests carrying at least one feature key the recipe does not declare (e.g. a typo). The encoder never reads such a key, so the request degrades toward a bias-only profile and still returns 200 — this counter is the only signal. Counted **once per request per side**, not per key. Deliberately **not** labelled by column name: unlike `..._unknown_value_total`'s `column` (bounded by your recipe), an undeclared name is unbounded request input and would be a cardinality DoS. To find the offending key, diff the client payload against the recipe's `features:` block | +| `recotem_v1_cold_start_requests_total` | Counter | `recipe`, `case` | cold-start requests served from side features; `case` ∈ {`features_only` (A), `features_and_history` (B), `cold_seeds` (C)} | | `recotem_model_loaded` | Gauge | `recipe` | 1 if the recipe is currently loaded | -| `recotem_artifact_load_failures_total` | Counter | `recipe`, `reason` | artifact-load failures since process start; `reason` ∈ {`read`, `parse`, `hmac`, `header_json`, `deserialize`, `metadata`, `yaml`, `unexpected`, `dir_scan`, `timeout`, `version_skew`} | +| `recotem_artifact_load_failures_total` | Counter | `recipe`, `reason` | artifact-load failures since process start; `reason` ∈ {`read`, `parse`, `hmac`, `header_json`, `deserialize`, `metadata`, `yaml`, `unexpected`, `dir_scan`, `timeout`, `version_skew`, `feature_version`} | | `recotem_active_recipes` | Gauge | — | total recipes in the registry | | `recotem_swap_total` | Counter | `recipe`, `result` | hot-swap attempts (`ok` / `error`) | | `recotem_artifact_stat_failures_total` | Counter | `recipe` | watcher stat() failures | @@ -476,7 +579,7 @@ Available metrics: | `recotem_recipe_rescan_errors_total` | Counter | `recipe` | recipe rescan failures | | `recotem_bigquery_storage_fallback_total` | Counter | `reason` | BQ Storage Read API fell back to REST | | `recotem_recipes_dir_scan_failures_total` | Counter | `error_class` | recipes-dir scan failures | -| `recotem_recommender_layout_unexpected_total` | Counter | `recipe` | `AttributeError` on `recommender._mapper.item_id_to_index` — indicates irspack API incompatibility | +| `recotem_recommender_layout_unexpected_total` | Counter | `recipe` | `AttributeError` on `recommender._mapper.user_id_to_index` (user axis) or `recommender._mapper.item_id_to_index` (item axis) — indicates irspack API incompatibility. Both axes increment the same counter and it carries no axis label, so it cannot tell you which one fired; the accompanying `recommender_layout_unexpected` log event names the `verb` | | `recotem_watcher_state_divergence_total` | Counter | — | watcher tried to mark an error on a non-existent registry entry (ordering bug) | --- @@ -529,6 +632,7 @@ The startup-only event variants are: | `initial_artifact_parse_failed` | Magic / version / header structural error | | `initial_artifact_hmac_failed` | HMAC mismatch or unknown kid | | `initial_artifact_version_skew` | WARNING. The artifact's `(best_class, irspack transition)` is not verified compatible with the running irspack. Reason label `version_skew`; see [irspack version skew](#irspack-version-skew). The guard emits its own `irspack_version_skew` WARNING carrying both versions; this event adds the `kid`. Skew is operational, so neither is ERROR — alert on the `version_skew` metric, not on log level. | +| `initial_artifact_feature_version_refused` | The artifact header has a `features` object, but its `version` sub-field is missing, non-integer, or does not equal this build's known `FEATURE_STATE_VERSION`. Reason label `feature_version`. Fails closed — an unrecognized encoder-state shape would otherwise be silently mis-encoded rather than refused, producing wrong (not missing) recommendations. A header with **no `features` key at all** fails **open** (old artifact, or a model trained without a `features:` block) — it has no state to mis-encode. See [Feature-aware iALS sizing](#feature-aware-ials-sizing). | | `initial_artifact_deserialize_failed` | FQCN allow-list rejection or payload decode error | | `initial_artifact_hmac_skipped_dev` | `--dev-allow-unsigned` | @@ -564,7 +668,8 @@ The high-signal metrics for production alerting: | Batch per-element error rate | `rate(recotem_v1_batch_element_errors_total[5m]) / rate(recotem_v1_requests_total{verb=~"batch-.*"}[5m])` | warn at sustained > 1% per recipe | | Artifact stat failures (watcher poll) | `recotem_artifact_stat_failures_total{recipe=...}` increase | warn | | Watcher unhandled errors | `recotem_watcher_unhandled_errors_total` increase | warn | -| Recommend error rate | `rate(recotem_v1_requests_total{status="error"}[5m]) / rate(recotem_v1_requests_total[5m])` | warn at 1%, page at 10% | +| Recommend error rate | `rate(recotem_v1_requests_total{status="error"}[5m]) / rate(recotem_v1_requests_total[5m])` | warn at 1%, page at 10%. `status="error"` is **only** genuine server faults (HTTP 500) — filter on it exactly, never on `status!="ok"`. Client-caused outcomes (`unknown_user`, `features_not_supported`, `feature_value_unusable`, `validation_error`, ...) carry their own labels precisely so a malformed client cannot page on-call | +| Cold-start client errors | `rate(recotem_v1_requests_total{status=~"features_not_supported\|feature_value_unusable"}[5m])` | warn only, never page — a sustained rate means a client is sending `user_features`/`item_features` to a recipe without a matching `features:` block, or values that cannot be standardized. The remedy is on the caller's side; the model is healthy | | Recommend latency | `histogram_quantile(0.99, sum by (le, recipe, verb) (rate(recotem_v1_request_latency_seconds_bucket[5m])))` | per-recipe, per-verb SLO | | Batch fan-out | `histogram_quantile(0.95, sum by (le, recipe, verb) (rate(recotem_v1_batch_size_bucket[5m])))` | watch for clients approaching the 256-element cap | | Active recipes | `recotem_active_recipes` drop > 0 since last scrape | warn (recipe removed or all stub) | @@ -728,6 +833,28 @@ All Optuna trials scored 0.0. Common causes: - The split produced an empty test set (too few users or interactions). Try `split.scheme: random` or lower `split.heldout_ratio`. - The data after cleansing has too few items for the cutoff. Lower `training.cutoff`. +### `recotem train` exits 4 with `feature_axis_error` + +A [`features:`](recipe-reference.md#features) side's feature table has **zero** id overlap with the interaction data — not one id matched. This aborts a run that previously succeeded if the id column's type changed at the source, so it is worth recognising on sight. The message samples ids from both sides, which usually names the cause by itself: + +``` +features.item: none of the 1200 item ids in the interaction data were found in +the feature table's 'item_id' column, so every item would encode to the bias +column alone ... feature-table ids look like ['1.0', '2.0', '3.0']; interaction +ids look like ['1', '2', '3']. +``` + +It is fatal rather than a warning because the failure is otherwise **silent**: every entity would encode to the bias column alone, so training would run to completion and sign an artifact whose header advertises `features` for what is really plain iALS. The model would serve, and score worse, with nothing in the logs to say why. + +Two causes account for essentially all occurrences: + +- **Id dtype mismatch** — what the sample above shows. A single blank cell in an otherwise-integer id column makes pandas infer `float64`, so `1` reads back as `1.0` while the interaction axis carries `"1"`. Pin the type at the source rather than cleaning the data: on a `csv` feature table add `dtype: {item_id: str}`. `dtype` is csv-only — on `bigquery` / `sql` cast in the query (`CAST(item_id AS STRING)`), and on `parquet` fix the type in the file's schema. +- **A wrong-but-existing `id_column`** — a column that exists but does not hold the entity id passes the presence check at fetch time and fails only here. Check that `features..id_column` names the same id space as `schema.item_column` / `schema.user_column`. + +recotem deliberately does not coerce the id column for you. By the time the frame is fetched, pandas has already inferred `float64` and the original text is unrecoverable — a column reading `1.0` is indistinguishable from one whose ids are literally `"1.0"` — so reformatting integral floats back to ints would silently rewrite ids on a catalog that legitimately uses that form, trading a detectable failure for a quiet corruption. It would also not catch the wrong-`id_column` case at all. + +Only **zero** overlap aborts. Partial coverage is legitimate and expected: an id absent from the feature table encodes to bias-only and degrades to plain iALS for that one entity, which is the same mechanism that makes cold-start scoring possible. There is deliberately no low-coverage warning threshold — a dtype or `id_column` mistake is a property of the whole column and always lands at exactly 0%, so any threshold above zero would fire on correct configurations. Alert on the `feature_axis_coverage` event (`side`, `matched`, `total`) yourself if you want to track coverage. + ### 401 on `/v1/recipes/{name}:recommend` - Trailing or leading whitespace in the `X-API-Key` header is treated as part of the key and will not match. Trim client-side. diff --git a/docs/plugin-authoring.md b/docs/plugin-authoring.md index 2a18306d..6261f478 100644 --- a/docs/plugin-authoring.md +++ b/docs/plugin-authoring.md @@ -119,7 +119,17 @@ class EchoSource: - For most plugins, declare `no_expand_fields: ClassVar[frozenset[str]] = frozenset()` — the global baseline (`query`, `query_parameters`) is already guarded unconditionally by the recipe loader. - For plugins with SQL or parameterised-query fields, list them explicitly: `no_expand_fields: ClassVar[frozenset[str]] = frozenset({"sql", "bind_params"})`. This provides defence-in-depth and documents the security intent for future maintainers. -5. **`fetch(ctx)`** must return a `pandas.DataFrame`. The DataFrame must contain at least the columns referenced in `recipe.schema` (`user_column`, `item_column`, and optionally `time_column`). The training pipeline accesses those columns by name immediately after fetch — a missing column surfaces as a `KeyError` and exits the train run. +5. **`fetch(ctx)`** must return a `pandas.DataFrame`. **This rule applies to + the interaction `source` only.** For that source, the DataFrame must + contain at least the columns referenced in `recipe.schema` + (`user_column`, `item_column`, and optionally `time_column`) — the + training pipeline accesses those columns by name immediately after + fetch, and a missing column surfaces as a `KeyError` and exits the train + run. A plugin used as a `features.item.source` / `features.user.source` + feature table satisfies none of this: it is read against `id_column` and + the declared `columns` list on the feature side config instead, and has + no `recipe.schema` columns to satisfy at all (see + [recipe-reference.md](recipe-reference.md#features)). 6. **`fetch()` must raise `DataSourceError`** for any external or transient failure (auth errors, network errors, query errors, empty results). `DataSourceError` is mapped to exit code 3. Any other exception surfaces as exit code 1. Wrap third-party exceptions explicitly: @@ -309,7 +319,10 @@ its `type_name`. `recotem validate recipes/my_recipe.yaml` instantiates the source class (which exercises the `__init__` deferred-import / extras check) but does -**not** call `fetch()`. If the source defines an optional `probe()` method, +**not** call `fetch()`. This runs for the top-level `source` **and** for +`features.item.source` / `features.user.source` when the recipe declares a +`features:` block — every configured source is probed, not just the +interaction source. If the source defines an optional `probe()` method, `recotem validate` calls it for a lightweight connectivity / auth check: ```python @@ -324,9 +337,14 @@ def probe(self) -> dict: ``` When `probe()` is defined, `recotem validate` reports `DataSource: probe OK -()`; when it is not, it reports `DataSource: extras OK -(, no probe defined)`. The builtin `CSVSource` / `ParquetSource` -use `fsspec` `exists()`, and `BigQuerySource` uses a dry-run query job. +() []`; when it is not, it reports `DataSource: extras OK +(, no probe defined) []`. `` names which source the +line describes — `source`, `features.item.source`, or +`features.user.source` — so a failure or a missing-probe notice is +unambiguous even when a recipe configures more than one source. A probe +failure is reported the same way: `DataSource probe failed []: +`. The builtin `CSVSource` / `ParquetSource` use `fsspec` `exists()`, +and `BigQuerySource` uses a dry-run query job. ## Testing diff --git a/docs/recipe-reference.md b/docs/recipe-reference.md index e89740e3..b68549c0 100644 --- a/docs/recipe-reference.md +++ b/docs/recipe-reference.md @@ -11,6 +11,7 @@ A recipe is a YAML file that defines what data to fetch, how to train, and where | `schema` | object | yes | Column mapping. | | `cleansing` | object | no | Data quality gates. | | `item_metadata` | object | no | Metadata joined into predict responses. | +| `features` | object | no | Item/user side features for feature-aware iALS training and cold-start. | | `training` | object | yes | Algorithm and tuning settings. | | `output` | object | yes | Artifact path and versioning. | @@ -174,6 +175,213 @@ Server-side field suppression is also available via `RECOTEM_METADATA_FIELD_DENY --- +## `features` + +```yaml +features: + item: + source: # datasource discriminated union — same registry as `source` + type: bigquery + query: SELECT item_id, genres, release_year, country FROM items + id_column: item_id + columns: + - {name: genres, encoding: multi_label, delimiter: "|"} + - {name: release_year, encoding: numerical} + - {name: country, encoding: categorical, min_frequency: 5} + user: + source: {type: csv, path: ./users.csv} + id_column: user_id + columns: + - {name: age_band, encoding: categorical} +``` + +The mere presence of this block enables feature-aware iALS training — there +is no separate flag. Item and user side features are declared, encoded, fed +to `IALSRecommender` during Optuna search and the final refit, and persisted +so that `:recommend` / `:recommend-related` can score unknown users and +unknown seed items from their attributes alone. See +[api-reference.md](api-reference.md) for the serving-side cold-start +contract. + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `features.item` | object | conditional | Item-side feature table. At least one of `features.item` / `features.user` must be present. | +| `features.user` | object | conditional | User-side feature table. | + +Each side (`FeatureSideConfig`) has: + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `source` | object | yes | Same datasource discriminated union as top-level `source` (`csv`, `parquet`, `bigquery`, `sql`, or any plugin). Reuses the datasource registry — `FetchContext` carries no interaction-specific fields, so any registered source can serve as a feature table. | +| `id_column` | string | yes | Column in the fetched table that holds the entity id (item id for `features.item`, user id for `features.user`). Non-empty, non-whitespace. Must **not** also appear in `columns` — the id column is consumed as the index and cannot also be a feature. | +| `columns` | list | yes, non-empty | One entry per source column to encode. Column names must be unique within a side. | + +**Null and duplicate ids are dropped before the vocabulary is built.** A row +whose `id_column` is null or empty is dropped and logged as +`feature_table_null_ids_dropped` (`side`, `drop_count`). A row whose +`id_column` repeats an id already seen is also dropped — the **first** +occurrence wins (`keep="first"`) — and logged as +`feature_table_duplicate_ids_dropped` (`side`, `drop_count`). Both log lines +carry only a count, never the offending ids or column values, which are +treated as user PII. + +Each entry in `columns` (`FeatureColumn`): + +| Field | Type | Required | Default | Notes | +|-------|------|----------|---------|-------| +| `name` | string | yes | — | Column name in the fetched feature table. | +| `encoding` | string | yes | — | One of `categorical`, `numerical`, `multi_label`. | +| `delimiter` | string | conditional | `"\|"` | Only valid when `encoding: multi_label`; rejected on any other encoding. Must not be empty. | +| `min_frequency` | int | no | `1` | Must be `>= 1` — `min_frequency: 0` is rejected at schema-validation time; there is no upper bound. Only valid for `categorical` / `multi_label` (vocabulary-based encodings); rejected on `numerical`. Values occurring fewer than N times in the fetched feature table are dropped from the vocabulary. For `categorical` this is a row count (one value per row); for `multi_label` it counts token **occurrences** — a single row with `a\|a` contributes 2 toward the threshold. | + +### Ids are matched as strings, and zero overlap is fatal + +`id_column` values are matched against the interaction data's +`schema.item_column` / `schema.user_column` **as strings** — both sides are +normalized with `str()` before comparison. So `1` matches `"1"`, but `1.0` does +**not** match `"1"`. + +An interaction id that is absent from the feature table is not an error: it +encodes to the implicit bias column alone, degrading to plain iALS for that one +entity. Partial coverage is expected and legitimate — the vocabulary is built +from the whole fetched table precisely so that entities missing from the +interaction data stay representable for cold-start scoring. + +**Zero** overlap is different: it aborts training with `TrainingError` +(`feature_axis_error`, exit 4). If not one id matches, every entity encodes to +bias-only, and the run would otherwise succeed and sign an artifact whose header +advertises `features` for what is really plain iALS — a silent downgrade. The +error samples ids from both sides so the mismatch is visible. Two causes account +for essentially all of these: + +- An **id dtype mismatch**: one blank cell in an otherwise-integer id column + makes pandas infer `float64`, so `1` reads back as `1.0`. Pin the type at the + source — `dtype: {item_id: str}` on a `csv` source (`dtype` is csv-only; on + `bigquery` / `sql` cast in the query instead). recotem will not coerce it for + you: a column reading `1.0` is indistinguishable from one whose ids are + literally `"1.0"`, so coercion would risk silently rewriting valid ids. +- An `id_column` naming a **wrong-but-existing column**, which passes the + presence check at fetch time and fails only at encode time. + +Coverage is logged per side per phase as `feature_axis_coverage` (`side`, +`matched`, `total`). See +[operations.md](operations.md#recotem-train-exits-4-with-feature_axis_error). + +### Encodings, and their missing/unknown behavior + +| Encoding | Behavior | Row missing entirely | Value missing / unknown | +|---|---|---|---| +| `categorical` | One-hot over the training vocabulary. | All-zero segment. | All-zero segment. | +| `numerical` | Standardized by the training mean/std. | `0` (i.e. the mean). | `0` (i.e. the mean). | +| `multi_label` | Split on `delimiter`, multi-hot. | All-zero segment. | Known tokens are retained; unknown tokens are dropped. | + +The `multi_label` distinction matters: `genres: "Action|Zzz"` with `Action` +known yields `Action=1` and drops `Zzz` — it is not an all-zero segment. +"Row missing" and "value unknown" coincide only for `categorical`. + +At serve time, each cold-start feature value supplied to `:recommend` / +`:recommend-related` (`user_features`, and each `item_features` seed mapping) is +length-capped: a string value longer than **8192 characters** is rejected with +`422` (the error names the offending column, never the value). This bounds the +`multi_label` tokenization work per request — 8192 characters is generous for a +real token list while blocking megabyte-scale amplification — and applies to +the batch verbs too. Non-string scalar values are unaffected. + +If a `numerical` column is constant — or merely **near**-constant — in the +training data, its segment is emitted as zeros and a warning is logged +(`feature_zero_variance_column`). The trigger is not an exact `std == 0.0` +check but a floor relative to the column's own scale: `std <= 1e-8 × +max(abs(mean), 1.0)`. A column whose values differ only by floating-point +rounding noise (std ~1e-15) would survive an exact check and then divide +serve-time standardization by a near-zero denominator, turning an ordinary +request value into an astronomically large standardized one — which trips the +cold-start solver's numerical guard for a reason the client cannot see or +control. Such a column degrades exactly like a missing value instead. See +[api-reference.md](api-reference.md#feature-aware-cold-start). + +An implicit all-ones **bias column** is appended per side (irspack adds no +intercept on its own). It is deliberately collinear with every +`categorical` column's one-hot block — a drop-first encoding was considered +and rejected because it would make an unknown/missing value (all-zero +segment) indistinguishable from the dropped reference level. The ridge +(`lambda_*_feature`, below) absorbs the resulting rank deficiency at the +tuned range. One consequence: if training fails with `Feature ridge +Cholesky decomposition failed`, the message deliberately does not suggest +dropping a column — recotem's own bias column is the more likely structural +cause, and it cannot be removed from the recipe. See +[operations.md](operations.md#feature-aware-ials-sizing) for the remedy +(`min_frequency`). + +### `min_frequency` is the dimension-cap lever + +The encoder vocabulary is built from the **whole fetched feature table**, +not restricted to items/users present in the interaction data — this +maximizes cold-start coverage. Consequently the encoded dimension scales +with **catalog size, not interaction count**: a 1M-item catalog whose +interactions cover only 1k items still pays the full encoded dimension (and +the full training cost — see below) for the other 999k items. Raising +`min_frequency` on high-cardinality columns is the only lever against +`RECOTEM_MAX_FEATURE_DIM` (default 5000; see +[operations.md](operations.md#feature-aware-ials-sizing)); there is no +recipe-level way to restrict the vocabulary to interaction-covered rows. + +Raising it too far fails **loudly but not fatally**. `min_frequency` has no +upper bound and nothing cross-checks it against the catalog, so +`min_frequency: 50` against a 3-row feature table validates happily and +prunes every token. The column then encodes to `width=0` and contributes +nothing — every row falls back to the implicit bias column — while the +`feature_encoder_state_built` INFO event still lists the column as though it +were active. Training logs a `feature_empty_vocabulary_column` **warning** +(carrying the column name, its `encoding` and `min_frequency`, and the +distinct/occurrence counts — never the token values) and continues. An +all-null column reaches the same "contributes nothing" state by a different +route and warns identically. Check the training logs after raising +`min_frequency` aggressively. + +### `lambda_item_feature` / `lambda_user_feature` — the one exception to "not user-tunable" + +`training.algorithms`' hyperparameter ranges normally come from each +recommender's `default_suggest_parameter` in irspack and are **not** +user-tunable from the recipe (see the `algorithms` row above). The +feature-ridge coefficients are the first exception: `lambda_item_feature` +and `lambda_user_feature` are **recotem's own** search range — +`suggest_float(..., 5e-2, 1e6, log=True)` — applied only to the side(s) +that have a `features.item` / `features.user` block, and only when the +trial's class is `IALSRecommender`. They are not present as recipe fields; +they cannot be set explicitly, only tuned. + +Two reasons this range is recotem's own rather than irspack's: irspack ships +no default range for these parameters (`default_suggest_parameter` never +suggests them), and the constructor default of `0.0` is a **hard error** +whenever the matching feature matrix is non-empty (`ValueError: Feature +weight regularization must be positive.`) — so leaving it untuned is not an +option once a features block is present. + +### Validation + +Recipe load rejects, with `RecipeError` (exit 2): + +- An `encoding` outside `categorical` / `numerical` / `multi_label`. +- `delimiter` set on a column whose `encoding` is not `multi_label`. +- `min_frequency` set (to anything other than the default) on a `numerical` column. +- Duplicate column names within one side's `columns` list. +- An `id_column` that also appears as a `columns[].name` on the same side — + the id column is consumed as the index, so a feature column of the same name + would be missing at encode time. Caught at load rather than at train time. +- `features:` present but `training.algorithms` contains no feature-capable + algorithm (today: `IALS`). Either add `IALS` to `algorithms` or remove the + `features` block. +- `features.item.source` / `features.user.source` fail the same + [path-scheme allow-list and mandatory-sha256-for-network-paths rules](#path-rules) + as the top-level `source`. + +`recotem validate` probes `features.item.source` / `features.user.source` +connectivity the same way it probes `source` — each reported line carries a +`[features.item.source]` / `[features.user.source]` label so a failure +names which source failed. + +--- + ## `training` ```yaml @@ -199,7 +407,7 @@ training: | Field | Type | Default | Notes | |-------|------|---------|-------| -| `algorithms` | list[string] | required | `IALS`, `CosineKNN` (alias `CosinekNN`), `TopPop`, `RP3beta`, `DenseSLIM`, `TruncatedSVD`, `BPRFM`. Full irspack class names (e.g. `IALSRecommender`) are also accepted. Hyperparameter ranges come from each recommender's `default_suggest_parameter` in irspack — they are not user-tunable from the recipe. | +| `algorithms` | list[string] | required | `IALS`, `CosineKNN` (alias `CosinekNN`), `TopPop`, `RP3beta`, `DenseSLIM`, `TruncatedSVD`, `BPRFM`. Full irspack class names (e.g. `IALSRecommender`) are also accepted. Hyperparameter ranges come from each recommender's `default_suggest_parameter` in irspack — they are not user-tunable from the recipe, **with one exception**: when a [`features`](#features) block is present, `lambda_item_feature` / `lambda_user_feature` are tuned over recotem's own range (`5e-2`–`1e6`, log-scale), because irspack ships no default range for them and their constructor default of `0.0` is a hard error whenever the matching feature matrix is non-empty. | | `metric` | string | `ndcg` | One of `ndcg`, `map`, `recall`, `hit`. | | `cutoff` | int | `20` | Recommendation list length for evaluation (must be ≥ 1). | | `n_trials` | int | `40` | Total Optuna trial budget (must be ≥ 1). | diff --git a/docs/security.md b/docs/security.md index 0dfa4f80..10978ac8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -164,6 +164,168 @@ Specific operator responsibilities: - **Compute and pin sha256 once, then alert on changes.** A mismatch is the signal. Don't bypass it by silently regenerating during CI. +## Feature-aware iALS + +### Feature-source path and integrity rules + +A recipe's `features.item.source` / `features.user.source` are full +DataSource configs — same registry as the top-level `source` — and are +**not** a lower-trust surface just because they feed side features instead +of interactions. The recipe loader applies the identical rules to +`features.item.source.path` / `features.user.source.path` that it applies +to `source.path`: + +- The same [path-scheme allow-list](recipe-reference.md#path-rules) (bare + local path, `file://`, `s3://`, `gs://`, `az://`, `abfs(s)://`, `http://`, + `https://`; chained fsspec protocols rejected). +- The same mandatory `sha256` integrity pin whenever the scheme is `http://` + or `https://`. +- Embedded URI credentials are rejected on feature-source paths exactly as + on `source.path` / `item_metadata.path`. + +`recotem validate` probes feature-source connectivity the same way it +probes `source` (see [recipe-reference.md](recipe-reference.md#features)), +so a missing extra or an unreachable feature source is caught before +`recotem train` does real work. + +### Feature-encoder version gate + +Every artifact trained with a `features:` block carries a small +`features.version` field in its (unencrypted, HMAC-covered) header. Before +serve deserializes the payload, it checks that field against this build's +known encoder-state version: + +- **`features` key absent** → load proceeds (fail **open**). This is a + pre-feature artifact or a model trained without `features:`; there is no + encoder state to misinterpret. +- **`features` present but `version` missing, non-integer, or not the exact + version this build knows** → refuse to load (fail **closed**), reason + `feature_version`. + +The asymmetry is deliberate, and mirrors the posture of the pre-existing +irspack version-skew guard (see +[operations.md — irspack version skew](operations.md#irspack-version-skew)): +an old serve with no feature code never reads the encoder state and keeps +serving known-user recommendations correctly — safe by ignorance. A serve +that *does* have feature code but does not recognize the state's shape is +the one that must be stopped, because silently proceeding would encode a +request's `user_features` / `item_features` into the wrong vector space and +return **incorrect recommendations that look like correct ones** — the one +failure mode a request-count or error-rate metric cannot catch. See +[operations.md — Feature-aware iALS sizing](operations.md#feature-aware-ials-sizing) +for the operational detail (event name, metric label). + +### Request-side PII: `user_features` / `item_features` + +`user_features` (on `:recommend` and `:recommend-related`) and per-seed +`item_features` (on `:recommend-related`) are attacker- or client-supplied +request fields that carry personal data **by construction** — an age band, +a country, a device category. This is a request-side PII vector distinct +from anything else in the v1 API surface, and recotem's posture is: + +1. **Raw feature values are never logged.** The code paths that touch + feature values (encoding, the unknown-category counter) log column names + and counts only — never the value itself. +2. `log_redaction.py`'s key-based redaction also strips `user_features` / + `item_features` wholesale, as defense in depth in case a future code path + ever logs a raw request body. +3. Feature values are never echoed back in a response body, so no + response-side deny-list is needed for them. `RECOTEM_METADATA_FIELD_DENY` + (see [recipe-reference.md](recipe-reference.md#item_metadata)) is the + existing **response-side** counterpart for a different field: it strips + configured item-metadata columns from `:recommend` / + `:recommend-related` responses. The two controls address opposite + directions of PII flow — one on the way in, one on the way out — and + neither substitutes for the other. + +### Extreme numerical feature values map to a 400, not a 500 + +A client-supplied `numerical` feature value that is extreme but still a +finite float (e.g. `1e22`) is not rejected by schema validation — it is a +legal float. Standardized against the training column's mean/std +(`recotem._features._row_values`), such a value can produce a magnitude +large enough to make irspack's per-request conjugate-gradient cold-start +solve numerically ill-conditioned. irspack's native core raises a bare +`RuntimeError` ("Conjugate-gradient solver encountered a singular system.") +in that case, with no awareness that the offending value came from an +untrusted client rather than a bug. + +`recotem._idmap` catches that `RuntimeError` at each of the three cold-start +call sites that feed a features-derived matrix into irspack's solver +(`get_score_cold_user_from_features`, `get_score_cold_user`, +`compute_item_embedding_from_features`) and re-raises +`ColdStartNumericalError`, which `serving/routes.py` maps to `400 +FEATURE_VALUE_UNUSABLE` (see [api-reference.md](api-reference.md#feature-aware-cold-start)) +rather than letting it surface as an unhandled `500`. + +**What this does and does not guarantee.** The catch is **signature-gated**: +`_is_numerical_cold_start_failure` re-raises only when the `RuntimeError`'s +message matches one of the irspack numerical-failure signatures verified +present in the installed binary and enumerated in +`_NUMERICAL_FAILURE_SIGNATURES` (`recotem/_idmap`). That narrowness is +deliberate — a bare `except RuntimeError` would silently reattribute an +unrelated irspack bug to client input — but it means the mapping is only as +complete as that list. An irspack release that rewords one of those messages +would re-raise past the gate and surface as a `500`. Equally out of scope is +any path where a client value fails as something other than a `RuntimeError` +from the solver — and this PR fixed a live instance of exactly that shape: a +`numerical` value supplied as a JSON integer literal of 309 or more digits +raised `OverflowError` (an `ArithmeticError`, not a `ValueError`) out of +`float()`, escaped the `except (TypeError, ValueError)` around the parse, and +reached the generic 500 handler with nothing but a valid API key +(`_features._parse_number`). The honest claim is therefore narrower than +"cannot crash the request": the **known** ill-conditioning paths are mapped +to a 400, and both the signature list and the parse-path exception handling +are the places to extend when a new one is found. + +The fix is otherwise conservative: it does not change what value a +`numerical` column standardizes to, at either train or serve time — the same +extreme value flowing through training-time `encode()` is untouched, and a +resulting final-refit Cholesky failure on an ill-conditioned *training* +matrix already surfaces as `TrainingError` (exit 4) through an unrelated code +path. Only the three serve-time cold-start solves are wrapped. + +### Why hand-rolled encoding, not scikit-learn preprocessing + +Feature encoding (`recotem._features`) deliberately reimplements one-hot, +standardization, and multi-hot encoding rather than persisting a fitted +`sklearn.preprocessing.OneHotEncoder` / `StandardScaler` inside the artifact. +[operations.md](operations.md#upgrades) already documents scikit-learn as a +**further, unguarded** compatibility axis: `TruncatedSVDRecommender` pickles +an sklearn estimator into the payload, and sklearn's own +`InconsistentVersionWarning` says unpickling across its own minor versions +"might lead to breaking code or invalid results" — recotem range-pins +`scikit-learn` to narrow this window but cannot close it. Pickling +`OneHotEncoder` / `StandardScaler` into the feature-encoder state would +**voluntarily widen** that same unguarded axis, and would do so via private +sklearn module paths (e.g. `sklearn.preprocessing._data`) that have no entry +in the FQCN allow-list's narrow prefix list to absorb a future rename. + +The encoder state is instead plain Python data — nested `dict` / `list`, +`str` vocabularies, and `int` / `float` scalars, with no numpy or pandas +object anywhere in it (`build_encoder_state` constructs every scalar through +`str()` / `float()` / `int()`; the numpy arrays in `_features.py` are built +inside `encode()` at call time and are not part of the persisted state). +Verified to round-trip through the existing `SafeUnpickler` with **no +allow-list change**. + +The allow-list is only a **partial** backstop for that invariant, and the +limit is worth stating precisely, because it is what makes those coercions +load-bearing. A stray `pandas.Index` really would be refused at load time +(`pandas.core.indexes.base._new_Index` is not allow-listed — verified). A +`numpy.str_` would **not**: it pickles via `numpy._core.multiarray.scalar` +plus `numpy.dtype`, both reachable through the narrow `numpy.*` +module-prefix allow-list, so it loads and keeps its type (verified). Nothing +downstream catches it either — `numpy.str_` subclasses `str` and hashes and +compares equal to it, so every vocabulary lookup keeps working and the leak +stays invisible at runtime. The `str()` coercions in `build_encoder_state` +are therefore the only thing keeping numpy's scalar types out of the state, +not a belt-and-braces gesture on top of a gate that would fail closed anyway. +See the module docstring in `recotem/_features.py` for the reasoning, and +`tests/unit/test_features.py::test_vocabulary_keys_are_exactly_str_not_numpy_str` +for the enforcement — it asserts the exact key type, because an `isinstance` +check cannot distinguish the two. + ## Artifact payload and the FQCN allow-list irspack's `IDMappedRecommender` depends on scipy sparse matrices and numpy arrays. These cannot be expressed in JSON without losing structure. The native irspack binary serialization format is required, and it is unavoidable. @@ -521,6 +683,26 @@ recommendation inference; sustained request rates above the recommender's inference throughput will queue under uvicorn and cause request latency to climb. Measure and cap at the proxy. +**Cold-start solves are bounded per request.** Case C of the feature-aware +cold start (a `:recommend-related` seed carrying `item_features`, see +[api-reference.md](api-reference.md#feature-aware-cold-start)) runs one +irspack conjugate-gradient solve **per cold seed** — measured ~0.25–0.45 ms +each. That per-solve cost is effectively **flat in model size**: 0.27 ms at +`n_components=8`, 0.30 ms at 128, 0.45 ms at 256, and flat across encoded +feature dimensions from 3 to 501. The solve is call-overhead-dominated rather +than Cholesky-dominated at every size a recipe can produce, so a +production-sized model does not make this bound materially worse. The +aggregate is capped at 512 solves per request by `BATCH_COLD_SEED_SOLVE_LIMIT` +(`serving/schemas.py`) on `:batch-recommend-related` — roughly 230 ms of +single-threaded CPU in the worst case. An element that would exceed the cap +receives a per-element `VALIDATION_ERROR` inside a 200, matching +`BATCH_AGGREGATE_LIMIT`'s existing posture, rather than failing the whole +request with a 422. The single verbs need no cap of their own: they are +structurally bounded at 100 solves by `seed_items`' `max_length`. As with +everything else in this section, that bounds the work a **single request** can +demand and says nothing about the rate; sustained rates remain the proxy's +job. + **Recommended nginx configuration:** ```nginx diff --git a/examples/feature-aware/README.md b/examples/feature-aware/README.md new file mode 100644 index 00000000..dc5a1b0c --- /dev/null +++ b/examples/feature-aware/README.md @@ -0,0 +1,184 @@ +# Feature-aware iALS example + +Trains an `IALSRecommender` with an item-side `features:` block, then serves +a cold-start recommendation for an item the model never saw during +training. Small enough to run in a few seconds; no network access required. + +Field reference: [docs/recipe-reference.md#features](../../docs/recipe-reference.md#features). + +## Files + +- `recipe.yaml` — interactions from `interactions.csv`, item features from + `items.csv`. +- `interactions.csv` — 144 rows, 30 users, 14 items (`user_id`, `item_id`, + `timestamp`). +- `items.csv` — 15 items, one column per encoding: + - `category` (`categorical`) — `action` / `comedy` / `drama`. + - `release_year` (`numerical`) — standardized at train time. + - `tags` (`multi_label`, `|`-delimited) — e.g. `space|thriller`; some rows + are blank on purpose, to exercise the "missing row" behavior. +- Item **`i15`** appears in `items.csv` but is deliberately **absent** from + `interactions.csv` — it is the cold item used in the cold-start step below. + +## Run + +From the repository root: + +```bash +# 1. Generate keys (once per machine). Copy the values into the exports below. +recotem keygen --type signing --kid dev +recotem keygen --type api --kid dev + +export RECOTEM_SIGNING_KEYS="dev:" # signing: env_entry value +export RECOTEM_API_KEYS="dev:sha256:" # api: env_entry value +export RECOTEM_API_PLAINTEXT="" # api: plaintext, for curl + +# 2. Validate — probes both the interaction source AND features.item.source +recotem validate examples/feature-aware/recipe.yaml +``` + +``` +Recipe 'feature_aware_demo': schema OK +DataSource: probe OK (csv) [source] +DataSource: probe OK (csv) [features.item.source] +Validation passed. +``` + +```bash +# 3. Train +mkdir -p artifacts +recotem train examples/feature-aware/recipe.yaml +# → ./artifacts/feature_aware_demo..recotem (signed) +``` + +```bash +# 4. Inspect the header — confirm the "features" block is present +recotem inspect ./artifacts/feature_aware_demo.recotem +``` + +Example output (the structural fields below — `best_class`, `n_items: 14`, +`features.version: 1`, `n_features: 13`, `columns` — are stable across +runs; `best_params`' numeric values are **not**: `training.split.scheme: +random` picks a fresh item/user vocabulary order per Python process for +string ids, so Optuna explores the search space in a different order each +run and lands on different-but-comparable hyperparameters and score. This +is pre-existing recotem behavior, unrelated to `features:`): + +```json +{ + "best_class": "IALSRecommender", + "best_params": { + "train_epochs": 115, + "n_components": 94, + "alpha0": 0.0632438212511315, + "reg": 0.001976218934028009, + "lambda_item_feature": 6.687175060313052 + }, + "data_stats": {"n_rows": 144, "n_users": 30, "n_items": 14, ...}, + "features": { + "version": 1, + "item": { + "n_features": 13, + "columns": ["category", "release_year", "tags"] + } + } +} +``` + +`best_params` carries `lambda_item_feature` alongside iALS's usual +hyperparameters — this is recotem's own tuned range, not irspack's (see +[recipe-reference.md](../../docs/recipe-reference.md#features)). `n_items: +14` (not 15): item `i15` never appears in `interactions.csv`, so it is not +part of the trained id-map — that is what makes it a genuine cold item for +step 6 below. `n_features: 13` is item `i15`'s only footprint in this +header: it and every other item contributed to the vocabulary that produced +that number, even though it is otherwise invisible to training. + +```bash +# 5. Serve (foreground) +recotem serve --recipes examples/feature-aware/ --port 8080 +``` + +```bash +# 6. Recommend for a known user (ordinary path, unchanged by features) +curl -X POST http://localhost:8080/v1/recipes/feature_aware_demo:recommend \ + -H "X-API-Key: $RECOTEM_API_PLAINTEXT" \ + -H "Content-Type: application/json" \ + -d '{"user_id": "u01", "limit": 5}' +``` + +```bash +# 7. Cold-start :recommend-related for item i15 — never trained on, scored +# purely from its item_features (case C: compute its embedding from +# features, then rank by similarity to that embedding) +curl -X POST http://localhost:8080/v1/recipes/feature_aware_demo:recommend-related \ + -H "X-API-Key: $RECOTEM_API_PLAINTEXT" \ + -H "Content-Type: application/json" \ + -d '{ + "seed_items": ["i15"], + "limit": 5, + "item_features": { + "i15": {"category": "drama", "release_year": 2016, "tags": "crime|period"} + } + }' +``` + +Example output (exact item ids and scores vary run to run — see the +determinism note above — but the shape and the HTTP 200 do not): + +```json +{ + "recipe": "feature_aware_demo", + "items": [ + {"item_id": "i11", "score": 0.000384}, + {"item_id": "i06", "score": 0.000319}, + {"item_id": "i07", "score": 0.000219}, + {"item_id": "i01", "score": 0.000192}, + {"item_id": "i09", "score": 0.000133} + ] +} +``` + +200, with real recommendations, for an item the model was never trained on. +Compare with the same call **without** `item_features`: + +```bash +curl -X POST http://localhost:8080/v1/recipes/feature_aware_demo:recommend-related \ + -H "X-API-Key: $RECOTEM_API_PLAINTEXT" \ + -H "Content-Type: application/json" \ + -d '{"seed_items": ["i15"], "limit": 5}' +# → 404 {"detail":"no known seed_items","code":"UNKNOWN_SEED_ITEMS"} +``` + +Without `item_features`, `i15` is simply an unknown seed — the pre-existing +behavior. Supplying its feature values is what makes the cold-start path +reachable. + +## What it demonstrates + +- A `features.item` block exercising all three encodings + (`categorical` / `numerical` / `multi_label`) plus the missing-row case + (`tags` blank for `i05` / `i10` / `i15`). +- `recotem validate` and `recotem inspect` surfacing feature-source probing + and the `features` header block respectively. +- Case C cold-start (`:recommend-related` + `item_features` for an unseen + seed item) — see + [api-reference.md#feature-aware-cold-start](../../docs/api-reference.md#feature-aware-cold-start) + for the full case A/B/C table, including the user-feature cases this + example does not exercise (no `features.user` block here). + +## What it does not cover + +This example only declares `features.item`. `features.user` follows the +identical shape (`source` + `id_column` + `columns`) and is independently +optional — adding it would additionally enable case A +(`:recommend` + `user_features` for an unknown user) and case B +(`:recommend-related` + `user_features` as a profile prior on top of an +ad-hoc seed history). + +## When to reach for this over the other examples + +Use this example to learn the `features:` block specifically. For a plain +(non-feature) local-CSV walkthrough see +[`examples/csv-local`](../csv-local/README.md); for the smallest possible +recipe see [`examples/quickstart`](../quickstart/README.md). diff --git a/examples/feature-aware/interactions.csv b/examples/feature-aware/interactions.csv new file mode 100644 index 00000000..27805275 --- /dev/null +++ b/examples/feature-aware/interactions.csv @@ -0,0 +1,145 @@ +user_id,item_id,timestamp +u15,i06,1700000000 +u26,i08,1700000060 +u27,i08,1700000120 +u05,i05,1700000180 +u26,i14,1700000240 +u17,i14,1700000300 +u08,i08,1700000360 +u06,i09,1700000420 +u25,i13,1700000480 +u25,i04,1700000540 +u08,i02,1700000600 +u01,i13,1700000660 +u19,i13,1700000720 +u11,i05,1700000780 +u17,i08,1700000840 +u20,i05,1700000900 +u29,i05,1700000960 +u12,i12,1700001020 +u07,i10,1700001080 +u20,i08,1700001140 +u02,i11,1700001200 +u21,i09,1700001260 +u06,i03,1700001320 +u03,i13,1700001380 +u03,i11,1700001440 +u20,i02,1700001500 +u24,i03,1700001560 +u08,i11,1700001620 +u07,i01,1700001680 +u25,i01,1700001740 +u01,i07,1700001800 +u26,i11,1700001860 +u30,i03,1700001920 +u13,i01,1700001980 +u01,i11,1700002040 +u18,i12,1700002100 +u23,i11,1700002160 +u06,i02,1700002220 +u26,i02,1700002280 +u04,i07,1700002340 +u25,i12,1700002400 +u05,i07,1700002460 +u01,i10,1700002520 +u03,i12,1700002580 +u15,i09,1700002640 +u12,i09,1700002700 +u28,i01,1700002760 +u21,i03,1700002820 +u13,i07,1700002880 +u10,i07,1700002940 +u05,i12,1700003000 +u04,i13,1700003060 +u28,i03,1700003120 +u29,i02,1700003180 +u13,i04,1700003240 +u15,i12,1700003300 +u30,i12,1700003360 +u24,i09,1700003420 +u29,i11,1700003480 +u14,i14,1700003540 +u16,i13,1700003600 +u29,i08,1700003660 +u05,i08,1700003720 +u21,i06,1700003780 +u28,i13,1700003840 +u14,i11,1700003900 +u17,i13,1700003960 +u21,i02,1700004020 +u10,i01,1700004080 +u16,i04,1700004140 +u27,i06,1700004200 +u24,i06,1700004260 +u24,i12,1700004320 +u19,i02,1700004380 +u19,i07,1700004440 +u23,i05,1700004500 +u29,i14,1700004560 +u14,i05,1700004620 +u07,i04,1700004680 +u20,i11,1700004740 +u23,i14,1700004800 +u17,i05,1700004860 +u16,i10,1700004920 +u03,i09,1700004980 +u28,i10,1700005040 +u30,i09,1700005100 +u14,i03,1700005160 +u27,i05,1700005220 +u06,i11,1700005280 +u23,i03,1700005340 +u25,i10,1700005400 +u12,i03,1700005460 +u30,i06,1700005520 +u01,i04,1700005580 +u22,i01,1700005640 +u19,i04,1700005700 +u21,i12,1700005760 +u05,i02,1700005820 +u24,i02,1700005880 +u17,i11,1700005940 +u06,i12,1700006000 +u13,i08,1700006060 +u23,i02,1700006120 +u14,i02,1700006180 +u10,i10,1700006240 +u18,i03,1700006300 +u28,i07,1700006360 +u22,i07,1700006420 +u09,i03,1700006480 +u02,i08,1700006540 +u11,i02,1700006600 +u27,i12,1700006660 +u25,i07,1700006720 +u19,i10,1700006780 +u03,i03,1700006840 +u23,i01,1700006900 +u27,i09,1700006960 +u28,i14,1700007020 +u04,i11,1700007080 +u11,i08,1700007140 +u22,i09,1700007200 +u10,i04,1700007260 +u22,i04,1700007320 +u19,i01,1700007380 +u20,i14,1700007440 +u07,i07,1700007500 +u02,i02,1700007560 +u27,i03,1700007620 +u09,i09,1700007680 +u01,i01,1700007740 +u12,i11,1700007800 +u07,i13,1700007860 +u18,i06,1700007920 +u02,i14,1700007980 +u16,i01,1700008040 +u11,i11,1700008100 +u02,i10,1700008160 +u02,i05,1700008220 +u22,i13,1700008280 +u09,i06,1700008340 +u11,i03,1700008400 +u03,i06,1700008460 +u13,i13,1700008520 +u04,i10,1700008580 diff --git a/examples/feature-aware/items.csv b/examples/feature-aware/items.csv new file mode 100644 index 00000000..6c89e404 --- /dev/null +++ b/examples/feature-aware/items.csv @@ -0,0 +1,16 @@ +item_id,category,release_year,tags +i01,action,2002,heist|thriller +i02,comedy,2009,romance|slapstick +i03,drama,2016,romance|period +i04,action,1995,heist|thriller +i05,comedy,2002, +i06,drama,2009,crime|period +i07,action,2016,space|thriller +i08,comedy,1995,satire +i09,drama,2002,crime|period +i10,action,2009, +i11,comedy,2016,romance|slapstick +i12,drama,1995,crime +i13,action,2002,heist|thriller +i14,comedy,2009,satire +i15,drama,2016, diff --git a/examples/feature-aware/recipe.yaml b/examples/feature-aware/recipe.yaml new file mode 100644 index 00000000..30e63c09 --- /dev/null +++ b/examples/feature-aware/recipe.yaml @@ -0,0 +1,64 @@ +# Feature-aware iALS example — trains on interactions.csv plus an item-side +# features table (items.csv) so the model can score cold items at serve +# time from their attributes alone. +# +# Walkthrough: examples/feature-aware/README.md +# +# i15 in items.csv is deliberately absent from interactions.csv — it is the +# cold item used in the README's cold-start :recommend-related demo. + +name: feature_aware_demo + +source: + type: csv + path: examples/feature-aware/interactions.csv + dtype: + user_id: str + item_id: str + +schema: + user_column: user_id + item_column: item_id + # timestamp is present in the CSV but unused: split.scheme is `random`. + +cleansing: + drop_null_ids: true + dedup: keep_last + min_rows: 100 + min_users: 10 + min_items: 10 + +# The features: block is what turns on feature-aware iALS -- there is no +# separate flag. Only features.item is used here; features.user follows the +# same shape (id_column + columns) and is independently optional. See +# docs/recipe-reference.md#features for the full field reference. +features: + item: + source: + type: csv + path: examples/feature-aware/items.csv + id_column: item_id + columns: + # one-hot over the training vocabulary; missing/unknown -> all-zero segment + - {name: category, encoding: categorical} + # standardized by training mean/std; missing -> 0 (the mean) + - {name: release_year, encoding: numerical} + # split on delimiter, multi-hot; unknown tokens dropped, known ones kept + - {name: tags, encoding: multi_label, delimiter: "|"} + +training: + # IALS is the only feature-capable algorithm today; at least one must be + # listed whenever `features:` is present. + algorithms: [IALS] + metric: ndcg + cutoff: 10 + n_trials: 6 + parallelism: 1 + split: + scheme: random + heldout_ratio: 0.2 + seed: 42 + +output: + path: ./artifacts/feature_aware_demo.recotem + versioning: append_sha diff --git a/src/recotem/_features.py b/src/recotem/_features.py new file mode 100644 index 00000000..37cec6a9 --- /dev/null +++ b/src/recotem/_features.py @@ -0,0 +1,824 @@ +"""Neutral home for side-feature encoding. + +Why this module exists +----------------------- +``recotem.training`` and ``recotem.serving`` must never import each other +(CLAUDE.md architecture constraint), but both need to turn a table of raw +attribute values into the numeric matrix irspack's feature-aware iALS expects. +Training builds the encoder state and encodes the training matrices; serving +re-encodes a single request's features with the same state to reach the +cold-start API. Defining the functions here (under ``recotem.*`` -- no +sub-package), both training and serving can import them without violating +the boundary, the same reasoning as ``recotem._idmap``. + +Why the state is plain data +---------------------------- +The state is persisted inside the artifact payload alongside the trained +recommender. It contains only plain Python containers -- dict, list, str, +int, and float, and nothing else: deliberately no numpy, no classes, no +sklearn estimators, and no pandas objects. The numpy arrays live in +``encode`` / ``encode_one``, which build them per call from this state; they +are never persisted. +Keeping the state plain means the artifact FQCN allow-list does not need to +grow to support this feature. + +The artifact allow-list is only a PARTIAL backstop for that invariant, and +it is worth being precise about where it stops. A stray ``pd.Index`` really +would be refused at load time (``pandas.core.indexes.base._new_Index`` is +not allow-listed -- verified). But a ``numpy.str_`` would NOT: it pickles +via ``numpy._core.multiarray.scalar`` + ``numpy.dtype``, both explicitly +allow-listed, so it round-trips through ``SafeUnpickler`` keeping its type +(verified). Nothing downstream catches it either -- ``numpy.str_`` hashes +and compares equal to ``str``, so every vocabulary lookup keeps working and +the leak stays invisible at runtime. + +So the ``str()`` coercions in ``build_encoder_state`` are load-bearing on +their own for the numpy scalar types, not a belt-and-braces gesture on top +of an allow-list that would fail closed anyway. They are total (every +vocabulary key is constructed through ``str()``), and +``tests/unit/test_features.py::test_vocabulary_keys_are_exactly_str_not_ +numpy_str`` enforces the result by asserting the EXACT key type -- an +``isinstance`` check cannot do it, because ``numpy.str_`` subclasses ``str``. + +Why ``encode`` demands ``index_order`` +---------------------------------------- +irspack raises on a feature-matrix row-count mismatch but accepts a +*misordered* matrix silently, and recotem's search phase and final refit do +not share one canonical row ordering (see below). Requiring the caller to +name the row order, and always reindexing onto it, makes *omission* of the +order unrepresentable: there is no convenience overload that infers it and +no "build once, reuse" caching, so a caller cannot forget to think about it. + +That is the honest claim, and it is deliberately narrower than +"misalignment is unrepresentable" -- which would be false. Passing a +wrong-but-same-length permutation of the right ids still returns a +same-shaped, differently-populated matrix, silently. What the signature buys +is that reaching that state requires an explicit wrong argument at a +greppable call site, rather than an omission that reads as correct code. + +Why the row order differs per phase +------------------------------------- +The final refit orders items by ``pd.Categorical`` (sorted). The search +phase does not match it, but the reason is per-scheme: ``random`` and +``time_user`` order items by ``list(set(...))``, which is neither sorted nor +stable across processes for string ids, while ``time_global`` routes to +irspack's ``holdout_specific_interactions``, whose item vocabulary is +``np.unique`` -- sorted and stable. + +The invariant therefore rests on the USER axis, where it holds universally: +``split.py`` builds ``row_user_ids`` as ``train.user_ids + val.user_ids`` +(a concatenation performed after the per-scheme branch, so it applies to +every scheme), and that train-then-val order is pinned to irspack's +Evaluator and never globally sorted. Per-phase re-encoding is mandatory +regardless of scheme. + +Why the bias column is collinear with the categorical one-hots +----------------------------------------------------------------- +Each categorical column's one-hot block sums to 1 for every row (when the +value is known), which is linearly dependent on the always-1 bias column. +This is accepted, not an oversight: drop-first encoding would make an +unknown/missing value (all-zero block) indistinguishable from the dropped +reference level, and the tuned ``lambda >= 5e-2`` search range absorbs the +resulting rank deficiency in irspack's Cholesky solve. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from decimal import Decimal +from typing import Any + +import numpy as np +import pandas as pd +import scipy.sparse as sps +import structlog + +from recotem.config import get_max_feature_dim +from recotem.recipe.models import FeatureColumn + +_logger = structlog.get_logger(__name__) + +# Bump when the state dict's shape changes in a way older readers would +# mis-encode. Serving refuses an artifact whose version it does not know +# (see Task 10's check_artifact_feature_version). +FEATURE_STATE_VERSION: int = 1 + +# Stable message prefix -- serving/watcher.py's `_classify_artifact_error` keys +# the Prometheus `reason` label off it, the same pattern `_irspack_compat.py` +# uses for `SKEW_MSG_PREFIX`. Keep the two in sync: if this string changes, +# update the classifier too, or the failure silently relabels to "parse" +# (the message contains the word "version"). +FEATURE_VERSION_MSG_PREFIX = "feature version check failed:" + +# A numerical column need not be EXACTLY constant (std == 0.0) to behave like +# one. Floating-point rounding noise -- e.g. values that are "the same +# number" up to a few ULPs, giving std ~= 1e-15 -- survives an exact +# std == 0.0 check but still divides serve-time standardization +# ((raw - mean) / std, see `_row_values`) by a near-zero denominator, turning +# an ordinary raw request value (e.g. 1e4) into an astronomically large +# standardized one. That is a false-positive amplifier, not a real signal: +# it makes an unremarkable client value trip the serve-time cold-start +# solver's own numerical-stability guard (see docs/api-reference.md#feature- +# aware-cold-start) for a reason the client cannot see or control. +# +# The floor is RELATIVE to the column's own scale (`max(abs(mean), 1.0)`), +# not an absolute constant, so it means the same thing whether the column's +# values sit near 0 or near 1e9. float64 has ~15-17 significant decimal +# digits (relative machine epsilon ~2.22e-16); 1e-8 is ~8 orders of +# magnitude looser than that -- generous enough to absorb realistic +# floating-point rounding noise accumulated across parsing/aggregation, +# while still many orders of magnitude tighter than any spread an operator +# would call real, intentional variance. +_NUMERICAL_STD_RELATIVE_FLOOR = 1e-8 + +# The encoded matrices are float32 (`encode` / `encode_one` both build with +# `dtype=np.float32`). A standardized value finite as float64 whose magnitude +# exceeds float32's max becomes +-inf on the cast, so it must be caught on the +# float64 side and routed to `unknown` -- see `_row_values`'s numerical branch. +_FLOAT32_MAX = float(np.finfo(np.float32).max) + + +class FeatureEncodeError(Exception): + """Raised for any structural problem building or applying an encoder state.""" + + +def _is_missing(raw: Any) -> bool: + """True if *raw* is a "no value supplied" sentinel. + + A DataFrame row and a hand-built request dict represent "no value" + differently: plain dicts use ``None``, pandas stores missing values as + float ``NaN`` even in object-dtype columns (confirmed: constructing a + DataFrame from a Python list containing ``None`` normalizes it to + ``nan``), and nullable extension dtypes use the ``pandas.NA`` singleton. + Treating all three identically is what keeps ``encode`` and + ``encode_one`` in agreement for a missing value. + """ + if raw is None: + return True + if isinstance(raw, float) and np.isnan(raw): + return True + return raw is pd.NA + + +def _vocabulary(values: Sequence[str], min_frequency: int) -> dict[str, int]: + """Build the kept-token -> index vocabulary, pruned by ``min_frequency``. + + ``min_frequency`` counts occurrences in *values*, not rows of the source + table -- what a "row" means depends on the caller. ``categorical`` + passes one value per row, so the count is a row count. ``multi_label`` + flattens every row's tokens into *values* first (see the ``multi_label`` + branch of ``build_encoder_state``), so the count is a token + **occurrence** count: a single row with ``tags="a|a"`` contributes 2 + toward ``a``'s threshold, not 1. + """ + counts: dict[str, int] = {} + for v in values: + counts[v] = counts.get(v, 0) + 1 + kept = sorted(t for t, n in counts.items() if n >= min_frequency) + return {t: i for i, t in enumerate(kept)} + + +# The scalar (non-text) types `pd.to_numeric(..., errors="coerce")` fits a +# numeric value FROM. `_parse_number` allows `float()` ONLY for these, so it +# can never turn a value the fitting parser dropped into a finite number. +# +# It is a hand-enumerated allow-list, not `numbers.Number`, because that ABC is +# wrong in BOTH directions (verified): it MISSES `np.bool_` (not registered) +# and ADMITS `Fraction` (which `to_numeric` NaNs). `bool` needs no entry -- it +# is an `int` subclass -- but `np.bool_` is neither an `int` nor an `np.number` +# subclass, so it is listed explicitly. `complex` is omitted deliberately: +# `to_numeric` keeps a complex column, but `build_encoder_state` rejects it +# explicitly with `FeatureEncodeError` (via `pd.api.types.is_complex_dtype`) +# BEFORE any `float()` coercion runs -- because under numpy 2.x +# `float(np.complex128)` does NOT raise `TypeError`, it silently discards the +# imaginary part with a `ComplexWarning`. So no state is ever built for a +# complex column and the encode path is unreachable -- there is nothing to +# mirror. +_FIT_NUMERIC_TYPES: tuple[type, ...] = (int, float, Decimal, np.number, np.bool_) + + +# `build_encoder_state` fits mean/std with `pd.to_numeric(..., errors="coerce")` +# while `_row_values` parses a request/row value per-scalar. The two must accept +# exactly the same values, or a value pandas silently dropped from a column's +# own statistics could still be encoded against those statistics: a table of +# ["1_000", "5", "7"] fits mean=6.0/std=1.0 from {5, 7} ALONE, then encodes the +# "1_000" row to 994.0 -- a 994-sigma value the fitted statistics never saw. +# +# The bug reproduces across the WHOLE non-`str` domain, not just `str`. `float()` +# also parses `bytes`/`bytearray`/`memoryview` and any object with +# `__float__`/`__index__`, while `to_numeric` NaNs all of those EXCEPT `bytes` +# (which it treats as text, applying the same grammar it applies to `str`). So a +# BYTES column (a SQL BLOB / parquet binary column declared `numerical`) of +# [b"1_000", b"5", b"7"] reproduces the 994-sigma bug verbatim -- and JSON +# cannot carry any of these types, so this hole is reachable at TRAINING time, +# through the same shared `_row_values`, invisibly (the statistics look healthy). +# +# The gate therefore has two shapes, one per part of `to_numeric`'s domain: +# * TEXT (`str`, `bytes`): `to_numeric` parses both as text with an ASCII-only, +# underscore-free grammar. `float()` is LOOSER on text -- it honours PEP 515 +# underscores ("1_000") and non-ASCII digits (full-width "１２３", +# Arabic-Indic "١٢٣", both of which occur in Japanese CSV exports) -- so +# text outside that grammar is refused. +# * NON-TEXT: allowed only if it is one of `_FIT_NUMERIC_TYPES`. This is what +# rejects bytearray / memoryview / Fraction / `__float__`-objects that +# `float()` would otherwise turn into a finite number the fit never saw. +# +# The invariant restored, across both shapes: a value the FITTING parser could +# not use must not be encodable by the ENCODING parser. It is restored by +# tightening the encoding side only -- loosening the fit would newly admit +# "1_000" into the statistics and silently change every model trained on such a +# table. And the mirror runs BOTH ways: refusing a value the fit DID use (e.g. +# a `bool` / `np.bool_` / `Decimal`) is the same class of bug in the opposite +# direction -- because `_row_values` is shared, it zeroes every training row of +# that column against healthy statistics, with no warning able to fire. That is +# exactly the reverted bool regression; `_FIT_NUMERIC_TYPES` includes `int` +# (covers `bool`) and `np.bool_` precisely so it cannot come back. +# +# Why not just call `pd.to_numeric` on the scalar, which would give parity by +# construction: +# 1. It does not survive its own edge case. `pd.to_numeric(pd.Series([10**309], +# dtype=object), errors="coerce")` raises OverflowError *despite* +# errors="coerce" (verified) -- so routing a request value through it +# would re-introduce the very HTTP 500 this function exists to close. +# 2. It costs ~3.3us per scalar vs ~0.045us for `float()` (measured, ~70x), +# on a path that runs once per served request AND once per catalog row +# per training phase. +# +# Parity is verified by the committed differential fuzz in +# tests/unit/test_features.py (`test_parse_number_mirrors_the_fitting_parser_ +# for_non_text` and `..._never_encodes_text_the_fitting_parser_dropped`), which +# -- unlike the earlier str-only fuzz that structurally could not see the bytes +# hole -- exercises non-`str` types too. The residual divergences are benign or +# in the safe direction: pandas accepts an internal space in an exponent +# ("6E 66") that `float()` rejects (encode is stricter, so the value joins the +# documented unparseable gap), and the two disagree by ~1 ULP on some +# exponent-heavy literals ("22e224"), which standardization renders immaterial. +def _parse_number(raw: Any) -> float | None: + """Parse *raw* the way ``build_encoder_state``'s fitting parser would. + + Returns the parsed float, or ``None`` if this encoder may not use the + value. ``None`` and ``±inf`` are different answers and the caller treats + them differently: ``None`` is the deliberately uncounted "unparseable" + gap, while a non-finite result is routed to ``unknown`` (a counted + signal). A magnitude too large for float64 therefore comes back as + ``±inf`` rather than ``None``. + """ + # A JSON `true` encoding identically to the number 1.0 is the correct + # reading, not a collision to close: for a column the recipe declares + # `numerical`, `true` IS 1.0, which is also what pandas makes of it (a + # bool-dtype column fits its mean/std FROM the bools -- verified: + # [True, False, True, False] fits mean=0.5, std=0.5). `bool` reaches + # `float()` below via the `int` entry in `_FIT_NUMERIC_TYPES`. Note the + # contrast with `check_artifact_feature_version`, which DOES exclude bool + # from its `isinstance(version, int)` check: a state version is an identity + # token, where `True == 1` is a type confusion with no numeric reading at + # all. That exclusion is right there and wrong here. + if isinstance(raw, str | bytes): + # Text: mirror `to_numeric`'s ASCII-only, underscore-free grammar. A + # text literal too large for float64 returns ±inf, never OverflowError + # (verified) -- unlike the int branch below -- so no OverflowError arm + # is needed for this path. + underscore = b"_" if isinstance(raw, bytes) else "_" + if not raw.isascii() or underscore in raw: + return None + try: + return float(raw) + except (TypeError, ValueError): + return None + + # Non-text: allow `float()` ONLY for the scalar types `to_numeric` fits + # from. `float()` would otherwise parse a bytearray/memoryview, a Fraction, + # or any object with `__float__`/`__index__` -- all of which `to_numeric` + # NaNs -- reproducing the 994-sigma bug one type over from "1_000". + if not isinstance(raw, _FIT_NUMERIC_TYPES): + return None + + try: + return float(raw) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + except OverflowError: + # `float()` raises OverflowError -- an ArithmeticError, NOT a + # ValueError -- for a Python int above float64's max, so this escaped + # the caller's original `except (TypeError, ValueError)` and reached + # the generic HTTP 500 handler: `encode_one` does not catch it, + # `_idmap`'s cold-start methods catch only RuntimeError, and + # `routes.py` catches only ColdStartNumericalError/ValueError. Any JSON + # integer literal of >=309 digits triggers it with nothing but a valid + # API key. + # + # Such a magnitude IS ±inf in float64, so report it as such rather + # than returning None: None would silently degrade it to the column + # mean, exactly the "an unknown value must not also be invisible" + # failure the caller's non-finite branch exists to prevent. + return math.inf if raw > 0 else -math.inf + + +def _tokens(raw: Any, delimiter: str) -> list[str]: + if _is_missing(raw): + return [] + text = str(raw) + if not text: + return [] + return [t for t in (p.strip() for p in text.split(delimiter)) if t] + + +def _column_block_varies( + series: pd.Series, vocab: dict[str, int], *, encoding: str, delimiter: str +) -> bool: + """True if the column's encoded one-hot/multi-hot block differs across rows. + + Keys on the same property the numerical branch expresses with ``std``: does + the column actually carry signal, or is every row's block identical (and + therefore collinear with the always-1 bias)? Each row is reduced to the SAME + "kept token set" that ``_row_values`` would encode -- missing, empty, and + unknown (out-of-vocab) values all collapse to the empty set -- so a + null-bearing column like ``[rock, None, rock]`` correctly VARIES + (``{rock}`` vs ``{}``) while a constant one like ``[rock, rock, rock]`` does + not. Returns as soon as a second distinct block is seen, so it is O(rows) + with an early-out on the common (varying) case. + """ + seen: set[frozenset[str]] = set() + for raw in series.tolist(): + if encoding == "multi_label": + kept = frozenset(t for t in _tokens(raw, delimiter) if t in vocab) + else: # categorical + key = "" if _is_missing(raw) else str(raw) + kept = frozenset((key,)) if (key != "" and key in vocab) else frozenset() + seen.add(kept) + if len(seen) > 1: + return True + return False + + +def _warn_if_column_dead( + col: FeatureColumn, + tokens: Sequence[str], + vocab: dict[str, int], + *, + varies: bool, +) -> None: + """Warn when a ``categorical`` / ``multi_label`` column contributes nothing. + + "Contributes nothing" means the encoded block is byte-identical for every + row, so it is collinear with the always-1 bias column and adds no signal. + Three routes reach that state, and this check keys on the shared observable + (the block does not vary across rows) rather than on any one route: + + - an unsatisfiable ``min_frequency`` prunes the vocabulary empty + (``min_frequency`` is ``Field(default=1, ge=1)`` with no upper bound, so + ``min_frequency: 50`` against a 3-row catalog validates happily), + - a column has no usable (non-null, non-empty) values at all, or + - a NON-empty vocabulary is nonetheless CONSTANT across rows (e.g. every + item shares one genre). + + The third route is why the check is keyed on ``varies`` and not on "vocab + empty": the earlier ``if vocab: return`` early-out let a constant column + (non-empty vocab, all-1 one-hot) through as though it were active. A + null-bearing column such as ``[rock, None, rock]`` DOES vary and is genuine + signal, so it stays silent -- that distinction is the whole point. This is + the categorical/multi_label parallel of the numerical branch's + ``feature_zero_variance_column``. + + Logs column names and counts only, never token values: they are catalog + data, and the pipeline's redaction processor is keyed to credentials, not + to arbitrary feature values. + """ + if varies: + return + # Only computed on the warning path, so the set() costs nothing normally. + distinct = len(set(tokens)) + if not vocab: + detail = ( + "every value was pruned by min_frequency; column contributes " + "nothing and every row falls back to the bias column" + if distinct + else "column has no non-empty values; column contributes nothing" + ) + else: + detail = ( + "vocabulary is non-empty but the encoded block is identical for " + "every row, collinear with the bias column; column contributes " + "nothing" + ) + _logger.warning( + "feature_empty_vocabulary_column", + column=col.name, + encoding=col.encoding, + min_frequency=col.min_frequency, + distinct_values=distinct, + occurrences=len(tokens), + detail=detail, + ) + + +def build_encoder_state( + df: pd.DataFrame, + columns: Sequence[FeatureColumn], +) -> dict: + """Build the phase-independent encoder state from a whole feature table. + + *df* must be indexed by entity id. The vocabulary is built from every + row, not only rows that appear in the interaction data, so that + cold-start entities are representable. That means the resulting + dimension scales with catalog size; ``min_frequency`` is the operator's + lever against it. + + Raises + ------ + FeatureEncodeError + If a declared column is absent from *df*, or the encoded dimension + exceeds ``RECOTEM_MAX_FEATURE_DIM``. + """ + specs: list[dict] = [] + offset = 0 + + for col in columns: + if col.name not in df.columns: + raise FeatureEncodeError( + f"feature column {col.name!r} is not present in the feature " + f"table; available columns: {sorted(df.columns)}" + ) + series = df[col.name] + + if col.encoding == "categorical": + present = [str(v) for v in series.dropna().tolist() if str(v) != ""] + vocab = _vocabulary(present, col.min_frequency) + _warn_if_column_dead( + col, + present, + vocab, + varies=_column_block_varies( + series, vocab, encoding="categorical", delimiter="" + ), + ) + spec = { + "name": col.name, + "encoding": "categorical", + "vocab": vocab, + "offset": offset, + "width": len(vocab), + } + + elif col.encoding == "multi_label": + delimiter = col.delimiter or "|" + flat: list[str] = [] + for raw in series.tolist(): + flat.extend(_tokens(raw, delimiter)) + vocab = _vocabulary(flat, col.min_frequency) + _warn_if_column_dead( + col, + flat, + vocab, + varies=_column_block_varies( + series, vocab, encoding="multi_label", delimiter=delimiter + ), + ) + spec = { + "name": col.name, + "encoding": "multi_label", + "delimiter": delimiter, + "vocab": vocab, + "offset": offset, + "width": len(vocab), + } + + else: # numerical + try: + numeric = pd.to_numeric(series, errors="coerce") + # A complex column cannot be standardized. `to_numeric` keeps + # complex dtype, and under numpy 2.x `float(np.complex128)` + # silently discards the imaginary part (ComplexWarning, NOT the + # TypeError older numpy raised), so without this guard a complex + # feature column would train on its real part alone. Reject it + # explicitly, before any float() coercion runs. (Raising here + # bypasses the OverflowError handler below -- FeatureEncodeError + # is not an OverflowError.) + if pd.api.types.is_complex_dtype(numeric): + raise FeatureEncodeError( + f"feature column {col.name!r} has complex values, which " + f"cannot be standardized; declare it categorical or " + f"drop it" + ) + has_values = bool(numeric.notna().any()) + mean = float(numeric.mean()) if has_values else 0.0 + std = float(numeric.std(ddof=0)) if has_values else 0.0 + except OverflowError as exc: + # `pd.to_numeric(..., errors="coerce")` does NOT suppress + # OverflowError for an object-dtype Python int above float64's + # max (a >=309-digit int escapes errors="coerce"). Unhandled it + # escapes `_fetch_side`'s FeatureEncodeError-only catch and + # surfaces as exit 1 (unmapped) instead of the training-domain + # exit 4. Map it to FeatureEncodeError, naming the column. + raise FeatureEncodeError( + f"feature column {col.name!r} contains a value too large to " + f"standardize as float64" + ) from exc + if not np.isfinite(mean): + mean = 0.0 + # See _NUMERICAL_STD_RELATIVE_FLOOR's module-level comment: a + # std that is merely tiny relative to the column's own scale is + # treated the same as an exact 0.0, not just a literal 0.0. + scale = max(abs(mean), 1.0) + if not np.isfinite(std) or std <= _NUMERICAL_STD_RELATIVE_FLOOR * scale: + _logger.warning( + "feature_zero_variance_column", + column=col.name, + detail="standardization would divide by zero; emitting zeros", + ) + std = 0.0 + spec = { + "name": col.name, + "encoding": "numerical", + "mean": mean, + "std": std, + "offset": offset, + "width": 1, + } + + offset += spec["width"] + specs.append(spec) + + # Bias column -- see the module docstring for why it is deliberately + # collinear with the categorical one-hots. + bias_offset = offset + n_features = offset + 1 + + cap = get_max_feature_dim() + if n_features > cap: + raise FeatureEncodeError( + f"encoded feature dimension {n_features} exceeds " + f"RECOTEM_MAX_FEATURE_DIM ({cap}). The vocabulary is built from " + f"the whole feature table, so dimension scales with catalog " + f"size, not interaction count. Raise min_frequency on " + f"high-cardinality columns, drop a column, or raise the cap -- " + f"but note the per-trial Cholesky cost is cubic in this number." + ) + + _logger.info( + "feature_encoder_state_built", + columns=[s["name"] for s in specs], + n_features=n_features, + ) + + return { + "version": FEATURE_STATE_VERSION, + "columns": specs, + "bias_offset": bias_offset, + "n_features": n_features, + } + + +def _row_values(state: dict, values: dict) -> tuple[list[int], list[float], list[str]]: + """Encode one entity's raw feature mapping into COO-style (col, value) pairs. + + Shared by ``encode`` (one call per requested row) and ``encode_one`` (the + serve-time single-row path) so the two can never diverge on how a given + value is turned into numbers -- see the module's parity tests. + """ + cols: list[int] = [] + data: list[float] = [] + unknown: list[str] = [] + + for spec in state["columns"]: + name = spec["name"] + raw = values.get(name) + + if spec["encoding"] == "categorical": + if _is_missing(raw): + continue + key = str(raw) + if key == "": + continue + idx = spec["vocab"].get(key) + if idx is None: + unknown.append(name) + continue + cols.append(spec["offset"] + idx) + data.append(1.0) + + elif spec["encoding"] == "multi_label": + toks = _tokens(raw, spec["delimiter"]) + # Dedupe per row so a repeated token (e.g. "rock|pop|rock") + # contributes exactly one 1.0 to its dimension, not one per + # occurrence. docs/recipe-reference.md documents this encoding as + # "multi-hot" (binary), but scipy's COO->CSR conversion SUMS + # duplicate (row, col) entries, so appending one 1.0 per raw + # token would silently double (or more) the weight of a + # repeated tag -- a count vector, not the documented multi-hot + # one. This is purely a row-encoding concern: it must NOT change + # `_vocabulary`'s occurrence counting, which deliberately counts + # every raw token toward `min_frequency` (a duplicate-heavy row + # legitimately pushes a rare token past the threshold faster). + seen: set[str] = set() + any_unknown = False + for tok in toks: + if tok in seen: + continue + seen.add(tok) + idx = spec["vocab"].get(tok) + if idx is None: + any_unknown = True + continue + cols.append(spec["offset"] + idx) + data.append(1.0) + if any_unknown: + unknown.append(name) + + else: # numerical + std = spec["std"] + if std == 0.0: + continue + if _is_missing(raw): + continue + num = _parse_number(raw) + if num is None: + continue + if not np.isfinite(num): + # A value that WAS supplied and DID parse as a number, but + # is +-inf (directly, via a string like "1e400", or via an + # integer too large for float64 -- see _parse_number's + # OverflowError branch), or NaN reached via a string like + # "nan"/"-nan" (a real float NaN is caught by _is_missing + # above and is a separate, deliberately uncounted gap), is + # not the same kind of gap as "missing" or "unparseable": + # the client sent something that parses as a number but + # cannot be standardized. Recording it as unknown is what + # makes + # `recotem_v1_feature_unknown_value_total` fire instead of + # silently degrading exactly like an omitted column -- + # matching encode_one's own docstring: an unknown value + # must not also be invisible. + unknown.append(name) + continue + scaled = (num - spec["mean"]) / std + # `num` is finite (checked above) but the matrix stores `scaled` + # cast to float32: a standardized magnitude above float32's max + # (or an intermediate float64 overflow from a tiny std) would + # become +-inf on that cast. Count it as unknown here rather than + # let the invisible inf through -- same contract as the non-finite + # branch above ("an unknown value must not also be invisible"). + if not np.isfinite(scaled) or abs(scaled) > _FLOAT32_MAX: + unknown.append(name) + continue + if scaled != 0.0: + cols.append(spec["offset"]) + data.append(scaled) + + cols.append(state["bias_offset"]) + data.append(1.0) + return cols, data, unknown + + +def encode( + state: dict, + df: pd.DataFrame, + index_order: Sequence[str], +) -> sps.csr_matrix: + """Encode *df* into a ``(len(index_order), n_features)`` csr_matrix. + + *df* must be indexed by entity id. Rows are emitted in exactly + *index_order*; ids absent from *df* produce an all-zero row (plus bias), + which irspack treats as "prior at the origin" -- that entity degrades to + plain iALS rather than failing. + + ``index_order`` is required, not optional. See the module docstring for + why: irspack accepts a misordered feature matrix silently, and recotem's + search phase and final refit do not share one canonical item ordering. + """ + order = [str(i) for i in index_order] + frame = df.copy() + frame.index = [str(i) for i in frame.index] + frame = frame[~frame.index.duplicated(keep="first")] + # Bound the row->dict conversion by len(order), not len(df): only rows + # that ``order`` could ever look up below are worth converting, and + # ``to_dict`` is the dominant cost for a large feature table (measured: + # ~4s / ~154MB transient for 400k rows vs. 1k requested ids -- and this + # function runs twice per training run, search + final refit). + # + # ``frame.loc[frame.index.intersection(order)]`` rather than + # ``frame.reindex(order)`` is deliberate, not a style choice: reindex + # introduces a NaN row for every id in *order* absent from *df*, and + # pandas fills that NaN by upcasting the WHOLE column to float64 when + # the column's dtype cannot natively hold NaN (e.g. an int64 + # ``categorical`` column) -- silently turning every PRESENT row's value + # too (``1`` -> ``1.0``), which then fails to match the ``str``-keyed + # vocabulary built from the original dtype at train time, degrading a + # known category to "unknown". reindex also raises on a duplicate + # *target* label (``to_dict(orient="index")`` demands a unique index), + # which a duplicate id in *order* would trigger. ``Index.intersection`` + # only ever selects rows that already exist in *df* -- it cannot + # introduce a NaN row or touch a column's dtype -- and is unaffected by + # duplicates in either operand, so it has neither failure mode. + wanted = frame.index.intersection(order) + lookup = frame.loc[wanted].to_dict(orient="index") + + indptr = [0] + indices: list[int] = [] + data: list[float] = [] + for entity_id in order: + values = lookup.get(entity_id) or {} + cols, vals, _unknown = _row_values(state, values) + indices.extend(cols) + data.extend(vals) + indptr.append(len(indices)) + + return sps.csr_matrix( + ( + np.asarray(data, dtype=np.float32), + np.asarray(indices, dtype=np.int32), + np.asarray(indptr, dtype=np.int32), + ), + shape=(len(order), state["n_features"]), + dtype=np.float32, + ) + + +def encode_one(state: dict, values: dict) -> tuple[sps.csr_matrix, list[str]]: + """Encode a single request's raw feature mapping. + + Returns the ``(1, n_features)`` matrix and the list of column names whose + supplied value was not in the training vocabulary. Callers should count + the unknowns: an unknown category degrades the recommendation silently, + so it must not also be invisible. + """ + cols, data, unknown = _row_values(state, values) + return ( + sps.csr_matrix( + ( + np.asarray(data, dtype=np.float32), + np.asarray(cols, dtype=np.int32), + np.asarray([0, len(cols)], dtype=np.int32), + ), + shape=(1, state["n_features"]), + dtype=np.float32, + ), + unknown, + ) + + +def state_descriptor(state: dict | None) -> dict | None: + """Return the small header summary for *state*, or None.""" + if state is None: + return None + return { + "n_features": state["n_features"], + "columns": [s["name"] for s in state["columns"]], + } + + +def check_artifact_feature_version(header_dict: dict, *, name: str) -> None: + """Refuse an artifact whose feature-encoder state this build cannot read. + + Policy + ------ + - ``features`` absent -> pass. Either the artifact predates the feature or + the model has no features; there is nothing to mis-encode. + - ``version`` equals ``FEATURE_STATE_VERSION`` -> pass. + - anything else (newer, older-and-unknown, missing, malformed) -> refuse. + + Failing CLOSED is the point. If the state's shape changed, serving would + encode a request's features into the wrong vector space and return + silently incorrect recommendations -- the one failure mode no counter can + catch. This mirrors ``_irspack_compat``'s posture on unverified + transitions. + + Note the asymmetry with a pre-feature serve, which is unprotected and does + not need to be: it has no feature code at all, never reads the state, and + serves known-user recommendations that remain correct. It is safe by + ignorance. This gate protects exactly the builds that could mis-encode. + """ + # Deferred import: _features.py is a neutral top-level module imported by + # both recotem.training and recotem.serving (CLAUDE.md architecture + # constraint). A module-level import of recotem.artifact would pull the + # artifact package into every training-only invocation. + from recotem.artifact.format import ArtifactError # noqa: PLC0415 + + raw = header_dict.get("features") + if raw is None: + return + if not isinstance(raw, dict): + raise ArtifactError( + f"{FEATURE_VERSION_MSG_PREFIX} artifact for recipe {name!r} has a " + f"malformed 'features' header (expected an object, got " + f"{type(raw).__name__}); refusing to load" + ) + version = raw.get("version") + # bool is an int subclass in Python -- exclude it explicitly so a stray + # `"version": true` is not silently treated as `1`. + if not isinstance(version, int) or isinstance(version, bool): + raise ArtifactError( + f"{FEATURE_VERSION_MSG_PREFIX} artifact for recipe {name!r} has a " + f"'features' header with a missing or non-integer version " + f"({version!r}); refusing to load" + ) + if version != FEATURE_STATE_VERSION: + raise ArtifactError( + f"{FEATURE_VERSION_MSG_PREFIX} artifact for recipe {name!r} " + f"declares feature encoder version {version}, but this build " + f"implements version {FEATURE_STATE_VERSION}. Loading it could " + f"encode request features into the wrong vector space and " + f"return silently incorrect recommendations. Retrain with this " + f"recotem version, or upgrade serving." + ) diff --git a/src/recotem/_idmap.py b/src/recotem/_idmap.py index 165c1814..64c46917 100644 --- a/src/recotem/_idmap.py +++ b/src/recotem/_idmap.py @@ -29,7 +29,10 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Sequence + +import numpy as np +import structlog # IPython stub: install before any irspack import. Irspack pulls in fastprogress # at import time, which in turn imports IPython.display. The stub provides only @@ -45,6 +48,143 @@ from irspack.utils.id_mapping import IDMapper # noqa: E402 +logger = structlog.get_logger(__name__) + +# --------------------------------------------------------------------------- +# Feature-capable recommender allow-list +# --------------------------------------------------------------------------- +# +# Mirrors ``recotem.training.algorithms.FEATURE_CAPABLE_CLASS_NAMES`` BY +# VALUE. That module cannot be imported here: ``_idmap.py`` is a neutral +# module shared by both ``training`` and ``serving``, and importing +# ``recotem.training`` from it would violate the training/serving boundary +# (CLAUDE.md). The two sets must therefore be kept in sync BY HAND -- if +# irspack ever grows a second feature-aware recommender class, add it to +# BOTH ``FEATURE_CAPABLE_CLASS_NAMES`` and this set in the same change. +# +# This is an explicit allow-list, not duck-typing (``hasattr`` / +# ``inspect.signature``), for the same reason ``training/algorithms.py``, +# ``artifact/signing.py``'s ``_ALLOWED_CLASSES``, and ``_irspack_compat.py`` +# all use hand-enumerated name/FQCN tables rather than deriving the answer at +# runtime: a future irspack class that happens to expose the same method +# names/signatures with different semantics must not be silently accepted +# just because it "looks like" IALSRecommender. +_FEATURE_CAPABLE_CLASS_NAMES: frozenset[str] = frozenset({"IALSRecommender"}) + + +# --------------------------------------------------------------------------- +# Numerical cold-start failure signatures +# --------------------------------------------------------------------------- +# +# Allow-list of substrings (matched case-insensitively) that identify a +# ``RuntimeError`` from irspack's native core as a NUMERICAL condition on the +# client-supplied value, as opposed to a server-side fault. A bare +# ``except RuntimeError`` at the three call sites below would also swallow +# non-numerical ``RuntimeError``s the exact same native calls can raise for +# reasons that have nothing to do with the request (e.g. +# ``irspack/recommenders/ials.py``'s ``trainer_as_ials`` raising +# ``RuntimeError("tried to fetch trainer before the training.")`` when +# ``trainer`` is unexpectedly ``None``) and mislabel them as a 400 client +# error instead of letting them surface as the 500 they actually are. +# +# Hand-enumerated, not duck-typed, for the same reason +# ``_FEATURE_CAPABLE_CLASS_NAMES`` above, ``training.algorithms +# .FEATURE_CAPABLE_CLASS_NAMES``, ``artifact.signing._ALLOWED_CLASSES``, and +# ``_irspack_compat.py``'s verified-transition table all hand-enumerate +# rather than infer: "this RuntimeError happened inside a cold-start solve" +# is not sufficient evidence that it is safe to blame the client. +# +# Verified present in the installed irspack (0.5.0) via ``strings`` on the +# compiled ``recommenders/_ials_core.abi3.so`` (2026-07-17): +# - "Conjugate-gradient solver encountered a singular system." -- the only +# solver recotem's recipes exercise today (recotem never sets +# ``solver_type``, so CG is always the default). +# - "Cholesky decomposition failed." / "Feature ridge Cholesky +# decomposition failed." -- the Cholesky solver's ridge-regression +# failure. Not reachable through any recipe field today, but the literal +# ships in the same binary as the CG one; scoping this allow-list on +# "singular system" alone would silently start 500ing again the moment a +# future recipe field lets an operator pick ``solver_type: cholesky``. +# - "Cholesky solve failed." -- the Cholesky solver's main solve step. +_NUMERICAL_FAILURE_SIGNATURES: tuple[str, ...] = ( + "singular system", + "cholesky decomposition failed", + "cholesky solve failed", +) + + +def _is_numerical_cold_start_failure(exc: RuntimeError) -> bool: + """Return True iff *exc* matches a verified numerical-failure signature. + + See the ``_NUMERICAL_FAILURE_SIGNATURES`` comment above for what this + allow-list covers and why it must not be widened to a bare + ``except RuntimeError``. + """ + message = str(exc).lower() + return any(signature in message for signature in _NUMERICAL_FAILURE_SIGNATURES) + + +class ColdStartNumericalError(Exception): + """A supplied feature value made irspack's cold-start solver numerically + unstable. + + An extreme-but-finite ``numerical`` feature value (e.g. ``1e22``) + standardizes (``recotem._features._row_values``) to a magnitude that + makes the per-request conjugate-gradient system irspack solves for a + cold-start embedding ill-conditioned. irspack's native core raises a + bare ``RuntimeError`` ("Conjugate-gradient solver encountered a singular + system.") with no input-validation semantics of its own -- it has no way + to know the value came from an untrusted client rather than a bug. + + The three cold-start call sites below that feed a features-derived + matrix into irspack's solver (``get_score_cold_user_from_features``, + ``get_score_cold_user``, ``compute_item_embedding_from_features``) catch + ``RuntimeError`` and, ONLY when its message matches + ``_NUMERICAL_FAILURE_SIGNATURES`` above, re-raise this instead -- so + ``serving/routes.py`` can map it to a 400 (bad client input) rather than + let it surface as an unhandled 500. A ``RuntimeError`` that does NOT + match (e.g. a genuinely broken model with ``trainer is None``) is + re-raised unchanged and reaches the router's generic handler as a 500 -- + matching a real server fault is not this client's problem to be blamed + for. Deliberately NOT a ``ValueError`` subclass: ``ValueError`` here + already means "this model / feature side cannot do cold start at all" + (see ``_require_capability`` and the ``*_feature_state is None`` guards + below), a model-capability condition. This is a per-VALUE numerical + condition on an otherwise capable model, so routes.py gives it its own + error code (``FEATURE_VALUE_UNUSABLE``) rather than folding it into + ``FEATURES_NOT_SUPPORTED``, which would incorrectly imply the model can + never serve this recipe's cold start. + + Training is NOT affected by this change, for two independent reasons. + Code-path: ``build_encoder_state`` / ``encode`` / ``_row_values`` + (``recotem._features``) are untouched, so the exact same extreme value + flowing through training-time ``encode()`` still standardizes the same + way -- this class only wraps the three SERVE-time cold-start solves + listed above. Structural (the stronger reason): at train time, a + numerical column's mean/std are computed FROM the same column that + contains the outlier, so the outlier inflates the very std it is later + divided by. That self-referential bound caps the worst-case standardized + magnitude at ``(n - 1) / sqrt(n)`` regardless of how extreme the raw + value is -- verified empirically: an outlier among ``n=20`` training + rows reaches ``max|z| ~= 4.36``, ``n=1000`` reaches ``~= 31.6``, + ``n=400000`` reaches ``~= 632.5`` (a ``1e22`` outlier among 1000 normal + values gives ``max|z| = 31.61``, matching the ``(n-1)/sqrt(n)`` bound of + ``31.59`` almost exactly). Reaching the ``~1e19`` standardized magnitude + needed to break the solver this way would take on the order of ``1e38`` + training rows. At the other extreme, an overflowing outlier (e.g. + ``1e308``) makes the sum-of-squares behind the std computation overflow + to a non-finite value; ``build_encoder_state`` already guards this -- + it warns and pins ``std`` to ``0.0`` whenever the fitted std comes out + non-finite or merely negligible against the column's own scale -- and + ``_row_values``'s numerical branch then skips any column whose std is + ``0.0`` rather than dividing by it. Serve-time cold start has + neither guard: the request's raw value is divided by a std that was fit + WITHOUT it, so nothing bounds how extreme the standardized magnitude can + get. A final-refit Cholesky failure on an ill-conditioned *training* + matrix (a different, already-bounded failure mode) surfaces as + ``TrainingError`` (exit 4) through an entirely different code path. + """ + class IDMappedRecommender: """String-keyed recommender wrapper around irspack IDMapper. @@ -57,15 +197,34 @@ class IDMappedRecommender: to either the training or the serving sub-package. """ + # Class-level defaults, deliberately WITH an assignment. + # + # __setstate__ is what pickle uses to restore state, and pickle constructs + # via cls.__new__(cls) under protocol 2+, so __init__ never runs on + # unpickle. Defaults assigned in __init__ would therefore not protect + # artifacts pickled before these attributes existed -- attribute lookup on + # the class is what does. A bare annotation (no `= None`) would create no + # class attribute at all and would not work. + item_feature_state: dict | None = None + user_feature_state: dict | None = None + def __init__( self, recommender: object, user_ids: Iterable[str], item_ids: Iterable[str], + *, + item_feature_state: dict | None = None, + user_feature_state: dict | None = None, ) -> None: self.recommender = recommender self.user_ids: list[str] = [str(u) for u in user_ids] self.item_ids: list[str] = [str(i) for i in item_ids] + # Assigned as INSTANCE attributes so __getstate__ (which returns + # dict(self.__dict__)) actually persists them. The class defaults above + # cover the read path for older artifacts; they do not persist anything. + self.item_feature_state = item_feature_state + self.user_feature_state = user_feature_state self._mapper: IDMapper = IDMapper(self.user_ids, self.item_ids) # ------------------------------------------------------------------ @@ -81,6 +240,11 @@ def __setstate__(self, state: dict) -> None: self.__dict__.update(state) self.user_ids = [str(u) for u in self.user_ids] self.item_ids = [str(i) for i in self.item_ids] + # Explicit, greppable normalization for artifacts pickled before these + # attributes existed. Redundant with the class-level defaults on + # purpose: the redundancy is cheap and makes the intent searchable. + self.__dict__.setdefault("item_feature_state", None) + self.__dict__.setdefault("user_feature_state", None) self._mapper = IDMapper(self.user_ids, self.item_ids) # ------------------------------------------------------------------ @@ -111,14 +275,306 @@ def get_recommendation_for_known_user_id( cutoff=cutoff, ) + def _require_capability(self, ok: bool, *, missing: str) -> None: + """Raise ``ValueError`` if *ok* is False. + + Guards every feature-based cold-start entry point against a model + whose feature state is present but whose winning recommender cannot + act on it. Task 9 persists ``item_feature_state`` / + ``user_feature_state`` unconditionally so the artifact header always + agrees with the payload -- even when the Optuna search winner is not + feature-capable. ``algorithms: ["TopPop", "IALS"]`` with a + ``features:`` block is valid (``Recipe._validate_features_algorithms`` + requires only that at least one listed algorithm be feature-capable), + and TopPop can win the search. Calling an irspack method that does + not exist on the winner would raise a bare ``AttributeError``; this + turns that into the same ``ValueError`` family as the "no state at + all" checks in the callers below, but with a distinct message so + operators can tell "retrain with features" (no state) apart from + "retrain restricting algorithms to a feature-capable one" (state + present, winner incapable) at a glance. + """ + if not ok: + raise ValueError( + "this model's recommender " + f"({type(self.recommender).__name__}) does not support " + f"feature-based cold-start; it has no {missing}. This " + "happens when a recipe lists a feature-capable algorithm " + "alongside a non-feature-capable one (e.g. TopPop, " + "CosineKNN) and the search winner was the latter -- retrain " + "restricting `algorithms` to a feature-aware one (e.g. IALS) " + "to use this endpoint." + ) + + def _is_feature_capable(self) -> bool: + """Return True iff the wrapped recommender's class is allow-listed. + + Matches on the BARE class name (``type(self.recommender).__name__``), + not the full FQCN (``__module__`` + ``__qualname__``), which is what + the other "which class may do X" decisions of this shape do: + ``training.algorithms.FEATURE_CAPABLE_CLASS_NAMES``, and + ``_irspack_compat.py``'s verified-transition table, whose + ``best_class`` rows are bare names too. + ``artifact.signing._ALLOWED_CLASSES`` is the deliberate exception -- + it keys on the full ``(module, qualname)`` pair because it answers a + different question: which classes the unpickler may construct from + untrusted bytes, where ``SafeUnpickler.find_class`` is handed exactly + that pair. This set instead asks whether an already-deserialized + object can do feature cold start, and exists specifically to mirror + ``FEATURE_CAPABLE_CLASS_NAMES`` value-for-value (see the comment + above ``_FEATURE_CAPABLE_CLASS_NAMES``); matching on FQCN here would + make the two sets structurally different and harder to eyeball as + "in sync". + + A subclass of ``IALSRecommender`` does NOT pass: ``type(x).__name__`` + returns the subclass's own name, not its base's. This is deliberate, + not an oversight -- admitting a subclass automatically via + ``isinstance`` would reintroduce exactly the failure mode this gate + exists to close: a class trusted because it *looks* like + IALSRecommender (same MRO, same inherited method names) rather than + because it was individually verified. ``_irspack_compat.py`` states + the identical rule for version-transition trust: "unproven is not + the same as safe." Admitting a future subclass requires adding its + name here explicitly, in the same change that adds it to + ``FEATURE_CAPABLE_CLASS_NAMES``. + """ + return type(self.recommender).__name__ in _FEATURE_CAPABLE_CLASS_NAMES + def get_recommendation_for_new_user( self, item_ids: Iterable[str], cutoff: int = 20, - ) -> list[tuple[str, float]]: - """Return top-*cutoff* (item_id, score) pairs for a cold-start user.""" - return self._mapper.recommend_for_new_user( - self.recommender, - [str(iid) for iid in item_ids], - cutoff=cutoff, + user_features: dict | None = None, + ) -> list[tuple[str, float]] | tuple[list[tuple[str, float]], list[str]]: + """Recommend for an ad-hoc history, optionally with a user profile. + + Without *user_features* this is unchanged and returns a plain list + -- existing callers depend on this exact type. With *user_features* + (case B) it runs irspack's joint solve over the seed history AND the + feature prior, and returns ``(recommendations, unknown_columns)`` + instead. The return type differs by argument on purpose: the old + signature has existing callers. + + Raises + ------ + ValueError + Only when *user_features* is given: if this model carries no + user feature state, or if the wrapped recommender's + ``get_score_cold_user`` does not accept a ``user_features`` + keyword (the search winner was not feature-capable even though + the artifact carries feature state -- see ``_require_capability``). + ColdStartNumericalError + Only when *user_features* is given: if an extreme-but-finite + supplied value makes irspack's cold-start solver numerically + unstable (see the class docstring). + """ + seeds = [str(iid) for iid in item_ids] + if user_features is None: + return self._mapper.recommend_for_new_user( + self.recommender, seeds, cutoff=cutoff + ) + + from recotem._features import encode_one + + if self.user_feature_state is None: + raise ValueError( + "this model has no user feature state; it was not trained with " + "features.user" + ) + self._require_capability( + self._is_feature_capable(), + missing="a `user_features`-aware `get_score_cold_user`", + ) + matrix, unknown = encode_one(self.user_feature_state, user_features) + X_seed = self._mapper.list_of_user_profile_to_matrix([seeds]) + try: + score = self.recommender.get_score_cold_user(X_seed, user_features=matrix)[ + 0 + ] + except RuntimeError as exc: + if not _is_numerical_cold_start_failure(exc): + raise + logger.warning( + "cold_start_numerical_failure", + method="get_score_cold_user", + irspack_message=str(exc), + ) + raise ColdStartNumericalError(str(exc)) from exc + recs = self._mapper.score_to_recommended_items( + score, cutoff=cutoff, forbidden_item_ids=seeds or None + ) + return recs, unknown + + def get_recommendation_for_cold_user( + self, + user_features: dict, + cutoff: int = 20, + ) -> tuple[list[tuple[str, float]], list[str]]: + """Case A: recommend for an unknown user from their features alone. + + Returns ``(recommendations, unknown_columns)``. *unknown_columns* + names the feature columns whose supplied value was not in the + training vocabulary; the caller must count them, because an unknown + category degrades the result silently. + + Client-requested exclusion is deliberately NOT a parameter here. + ``serving/routes.py``'s ``_build_items`` post-filters + ``exclude_items`` off this list, uniformly for every verb and every + case. Accepting it here and passing it down as ``forbidden_item_ids`` + instead would make the ranker back-fill to a full *cutoff*, so the + same ``exclude_items`` request would return MORE items on this path + than on the pre-existing ones that only post-filter -- one parameter + silently meaning two different things depending on whether features + were supplied. + + Raises + ------ + ValueError + If this model carries no user feature state, or if the wrapped + recommender has no ``get_score_cold_user_from_features`` method. + Passing either through to irspack is unsafe: no state means a + shape mismatch or, for a (1, 0) matrix, silent all-zero scores + with no error; no method means a bare ``AttributeError`` -- a + non-feature-capable search winner (e.g. TopPop, CosineKNN) can + carry non-None feature state (Task 9 persists it + unconditionally) without being able to act on it. + ColdStartNumericalError + If an extreme-but-finite supplied value makes irspack's + cold-start solver numerically unstable (see the class + docstring). + """ + from recotem._features import encode_one + + if self.user_feature_state is None: + raise ValueError( + "this model has no user feature state; it was not trained with " + "features.user" + ) + self._require_capability( + self._is_feature_capable(), + missing="`get_score_cold_user_from_features`", + ) + matrix, unknown = encode_one(self.user_feature_state, user_features) + try: + score = self.recommender.get_score_cold_user_from_features(matrix)[0] + except RuntimeError as exc: + if not _is_numerical_cold_start_failure(exc): + raise + logger.warning( + "cold_start_numerical_failure", + method="get_score_cold_user_from_features", + irspack_message=str(exc), + ) + raise ColdStartNumericalError(str(exc)) from exc + recs = self._mapper.score_to_recommended_items(score, cutoff=cutoff) + return recs, unknown + + def get_recommendation_for_cold_seeds( + self, + seed_items: Sequence[str], + item_features: dict[str, dict], + cutoff: int = 20, + ) -> tuple[list[tuple[str, float]], list[str]]: + """Case C: seeds that include items absent from training. + + Known seeds contribute their learned item embedding; unknown seeds + contribute an embedding computed from their features. The mean is + scored as if it were a user embedding, which is exactly item-item + similarity in this model. + + Removing the seeds from their own related-items result DOES use + ``forbidden_item_ids``, so the ranker back-fills around them: a + client that asked "what goes with i0" should not spend a slot on i0 + itself. Client-requested exclusion is the opposite and is NOT a + parameter here -- ``serving/routes.py``'s ``_build_items`` + post-filters ``exclude_items`` off this list for every verb, and + back-filling it here instead would make ``exclude_items`` mean + something different on this path than on the pre-existing ones. + + This deliberately has DIFFERENT semantics from + ``get_recommendation_for_new_user``, which runs an iALS cold-user + solve treating seeds as an interaction history. Routes must only take + this path when a genuinely new input (item_features for an unknown + seed) is present, so existing clients see no behavior change. + + Raises + ------ + ValueError + If this model carries no item feature state, or if the wrapped + recommender has no item-embedding API. See + ``get_recommendation_for_cold_user`` for why a non-None state + does not guarantee the search winner can act on it. + KeyError + If no seed is either a known item id or accompanied by an entry + in *item_features*. + ColdStartNumericalError + If an extreme-but-finite value in a cold seed's *item_features* + entry makes irspack's cold-item-embedding solver numerically + unstable (see the class docstring). + """ + from recotem._features import encode_one + + if self.item_feature_state is None: + raise ValueError( + "this model has no item feature state; it was not trained with " + "features.item" + ) + self._require_capability( + self._is_feature_capable(), + missing=( + "the item-embedding API (`get_item_embedding` / " + "`compute_item_embedding_from_features` / " + "`get_score_from_user_embedding`)" + ), + ) + # Deliberately OUTSIDE the per-seed try/except below: this reads the + # already-trained item-embedding matrix and does not depend on any + # per-request value at all -- it returns the identical array for + # every call against this model. A RuntimeError here (e.g. the + # ``trainer is None`` fault from irspack's ``trainer_as_ials``) can + # therefore never be the numerical-instability condition this class + # exists to catch; it is unconditionally a server fault and must + # reach the router's generic 500 handler, not be routed through + # ``_is_numerical_cold_start_failure``'s message check only to be + # re-raised anyway. Wrapping it would add a no-op except clause with + # no observable behavior change -- see docs/api-reference.md and the + # task report for the fuller rationale. + item_emb = self.recommender.get_item_embedding() + vectors = [] + unknown: list[str] = [] + for seed in seed_items: + sid = str(seed) + idx = self._mapper.item_id_to_index.get(sid) + if idx is not None: + vectors.append(item_emb[idx]) + continue + raw = item_features.get(sid) + if raw is None: + continue + matrix, unk = encode_one(self.item_feature_state, raw) + unknown.extend(unk) + try: + vectors.append( + self.recommender.compute_item_embedding_from_features(matrix)[0] + ) + except RuntimeError as exc: + if not _is_numerical_cold_start_failure(exc): + raise + logger.warning( + "cold_start_numerical_failure", + method="compute_item_embedding_from_features", + irspack_message=str(exc), + ) + raise ColdStartNumericalError(str(exc)) from exc + if not vectors: + raise KeyError("no usable seed: none known and none carried features") + + mean_emb = np.mean(np.stack(vectors), axis=0, keepdims=True) + score = self.recommender.get_score_from_user_embedding(mean_emb)[0] + forbidden = [ + str(s) for s in seed_items if str(s) in self._mapper.item_id_to_index + ] + recs = self._mapper.score_to_recommended_items( + score, cutoff=cutoff, forbidden_item_ids=forbidden or None ) + return recs, sorted(set(unknown)) diff --git a/src/recotem/cli.py b/src/recotem/cli.py index bc99aa8e..fb1da8b8 100644 --- a/src/recotem/cli.py +++ b/src/recotem/cli.py @@ -29,7 +29,7 @@ import re import uuid from pathlib import Path -from typing import Annotated +from typing import Annotated, Any import structlog import typer @@ -611,32 +611,53 @@ def validate( code = _map_exception_to_exit(exc) _exit(code, f"Recipe validation failed: {exc}") - try: + def _probe_source(source_cfg: Any, where: str) -> None: from recotem.datasource.registry import get_source_class - source_cfg = loaded_recipe.source type_name = getattr(source_cfg, "type", None) if type_name is None: - raise RuntimeError("Recipe source is missing the 'type' discriminator.") + raise RuntimeError(f"{where} is missing the 'type' discriminator.") source_cls = get_source_class(type_name) - # Instantiate the source. Plugins defer optional-dependency imports - # to __init__, so this catches missing extras (e.g. google-cloud-bigquery) - # and config / Config-class mismatches. We do NOT call .fetch() here — - # full data loads can be expensive (BigQuery scans, large CSV reads). + # Instantiate the source. Plugins defer optional-dependency + # imports to __init__, so this catches missing extras (e.g. + # google-cloud-bigquery) and config / Config-class mismatches. + # We do NOT call .fetch() here — full data loads can be + # expensive (BigQuery scans, large CSV reads). source = source_cls(source_cfg) - # Optional connectivity probe — plugin-authoring.md documents this hook. - # Built-ins implement it; third-party plugins may opt in. + # Optional connectivity probe — plugin-authoring.md documents + # this hook. Built-ins implement it; third-party plugins may + # opt in. probe = getattr(source, "probe", None) if callable(probe): probe() - typer.echo(f"DataSource: probe OK ({type_name})") + typer.echo(f"DataSource: probe OK ({type_name}) [{where}]") else: - typer.echo(f"DataSource: extras OK ({type_name}, no probe defined)") - except Exception as exc: - code = _map_exception_to_exit(exc) - _exit(code, f"DataSource probe failed: {exc}") + typer.echo( + f"DataSource: extras OK ({type_name}, no probe defined) [{where}]" + ) + + # (source_cfg, where) pairs to probe: the top-level source plus any + # configured feature-side sources. ``where`` is carried by the *caller* + # (this loop), not by mutating the exception's .args in place -- some + # exception types (e.g. pydantic_core.ValidationError) implement __str__ + # in Rust and ignore .args entirely, which would silently drop the + # ``where`` tag. Building the message here works for any exception type + # from any plugin, regardless of its __str__. + sources: list[tuple[Any, str]] = [(loaded_recipe.source, "source")] + if loaded_recipe.features is not None: + for side_name in ("item", "user"): + side = getattr(loaded_recipe.features, side_name) + if side is not None: + sources.append((side.source, f"features.{side_name}.source")) + + for source_cfg, where in sources: + try: + _probe_source(source_cfg, where) + except Exception as exc: + code = _map_exception_to_exit(exc) + _exit(code, f"DataSource probe failed [{where}]: {exc}") typer.echo("Validation passed.") @@ -668,13 +689,34 @@ def schema() -> None: from pydantic import create_model from recotem.datasource.registry import build_source_config_union - from recotem.recipe.models import Recipe + from recotem.recipe.models import FeaturesConfig, FeatureSideConfig, Recipe source_union = build_source_config_union() + + # ``create_model(__base__=X)`` can only override fields declared + # directly on X — it cannot reach into a nested model's fields. Both + # ``FeatureSideConfig.source`` and ``Recipe.source`` are typed ``Any`` + # for the same circular-import reason (the union is built dynamically + # from entry points), so the nesting has to be rebuilt bottom-up: + # FeatureSideConfig first, then FeaturesConfig referencing the + # rebuilt side config, then Recipe referencing both the top-level + # union and the rebuilt features config. + schema_side = create_model( + "FeatureSideConfig", + __base__=FeatureSideConfig, + source=(source_union, ...), + ) + schema_features = create_model( + "FeaturesConfig", + __base__=FeaturesConfig, + item=(schema_side | None, None), + user=(schema_side | None, None), + ) schema_recipe = create_model( "Recipe", __base__=Recipe, source=(source_union, ...), + features=(schema_features | None, None), ) schema_dict = schema_recipe.model_json_schema() typer.echo(json.dumps(schema_dict, indent=2)) diff --git a/src/recotem/config.py b/src/recotem/config.py index 5918aeec..3089e30f 100644 --- a/src/recotem/config.py +++ b/src/recotem/config.py @@ -35,6 +35,9 @@ accepting private/loopback host addresses. Default refuses RFC1918 / 127.0.0.0/8 to block SSRF via crafted DSNs. + RECOTEM_MAX_FEATURE_DIM Max encoded side-feature dimension for + feature-aware iALS (default 5000; clamped + 16-100000) """ from __future__ import annotations @@ -507,6 +510,31 @@ def get_http_timeout_seconds() -> int: ) +# --------------------------------------------------------------------------- +# Feature-encoding cap (used by recotem._features for feature-aware iALS) +# --------------------------------------------------------------------------- + +DEFAULT_MAX_FEATURE_DIM = 5000 +_MIN_FEATURE_DIM = 16 +_MAX_FEATURE_DIM = 100_000 + + +def get_max_feature_dim() -> int: + """Return RECOTEM_MAX_FEATURE_DIM, clamped to [16, 100000]. + + irspack forms a dense ``F.T @ F`` and solves it by Cholesky, so cost is + cubic in the feature dimension and it never errors -- it only degrades. + Measured per trial: 5k -> 0.6 s / 200 MB; 10k -> 4.2 s / 771 MB; + 20k -> 43 s / 3 GB. Multiplies with training.parallelism. + """ + return _clamped_int_env( + "RECOTEM_MAX_FEATURE_DIM", + DEFAULT_MAX_FEATURE_DIM, + _MIN_FEATURE_DIM, + _MAX_FEATURE_DIM, + ) + + _TRUTHY_ENV_VALUES: frozenset[str] = frozenset({"1", "true", "yes", "on"}) diff --git a/src/recotem/log_redaction.py b/src/recotem/log_redaction.py index ddef359b..7eea2ec0 100644 --- a/src/recotem/log_redaction.py +++ b/src/recotem/log_redaction.py @@ -6,6 +6,9 @@ Redacted key patterns (case-insensitive, matched against the **key name**): - x-api-key, authorization, cookie - recotem_signing_key, recotem_signing_keys, recotem_api_keys + - user_features, item_features (feature-aware iALS cold-start request + attributes -- PII by construction, e.g. age_band, country. Defense in + depth only: callers must never pass a feature dict to a logger.) - any key whose name contains: secret, password, passwd, token, key, auth, bearer, cred, private - any key whose lowercased name starts with: aws_, gcp_, google_, azure_ @@ -32,6 +35,13 @@ # --------------------------------------------------------------------------- # Exact key names (lowercased) to always redact. +# +# user_features / item_features: feature-aware iALS cold-start requests carry +# raw request-supplied attributes (e.g. age_band, country) that are PII by +# construction. This is defense in depth, NOT the primary control -- the +# primary rule is that caller code must never pass a feature dict to a +# logger in the first place (log column names and counts instead). This key +# match is a mechanical backstop for if one does anyway. _EXACT_KEYS: frozenset[str] = frozenset( { "x-api-key", @@ -40,6 +50,8 @@ "recotem_signing_key", "recotem_signing_keys", "recotem_api_keys", + "user_features", + "item_features", } ) @@ -229,7 +241,7 @@ def _redact_value(value: Any) -> Any: return [_redact_value(item) for item in value] if isinstance(value, tuple): return tuple(_redact_value(item) for item in value) - if isinstance(value, (set, frozenset)): + if isinstance(value, set | frozenset): # ``set`` cannot hold unhashable elements; ``_redact_value`` only ever # returns hashables when given hashables (str/bytes/numbers). Return # the same container type so downstream rendering is unchanged. @@ -237,7 +249,7 @@ def _redact_value(value: Any) -> Any: return frozenset(scrubbed) if isinstance(value, frozenset) else scrubbed if isinstance(value, str): return _scrub_string_value(value) - if isinstance(value, (bytes, bytearray)): + if isinstance(value, bytes | bytearray): return _redact_bytes_value(value) return value diff --git a/src/recotem/recipe/loader.py b/src/recotem/recipe/loader.py index 5df00553..a79af34a 100644 --- a/src/recotem/recipe/loader.py +++ b/src/recotem/recipe/loader.py @@ -272,19 +272,127 @@ def _check_recipe_file_containment(recipe_path: Path, recipes_root: Path) -> Non _NO_EXPAND_KEYS: frozenset[str] = frozenset({"query", "query_parameters"}) +# The only three schema positions where a ``source`` mapping is a genuine +# DataSource subtree: the recipe root (top-level ``source``), and +# ``features.item`` / ``features.user`` (``features.item.source`` / +# ``features.user.source``). Each entry is the ancestor-key path of the +# dict that *contains* the ``source`` key, expressed as a tuple from the +# recipe root (``()`` for the root itself). +_SOURCE_NODE_PATHS: frozenset[tuple[str, ...]] = frozenset( + {(), ("features", "item"), ("features", "user")} +) + + +def _is_source_node(key: str, value: Any, path: tuple[str, ...]) -> bool: + """True when *value* is a source mapping at a legitimate schema position. + + *path* is the ancestor-key tuple of the dict currently being walked (the + dict that contains *key*), e.g. ``()`` at the recipe root or + ``("features", "item")`` inside ``features.item``. Matching requires both + the key name (``"source"``) and the position (``path in + _SOURCE_NODE_PATHS``) so that only the recipe's real ``source``, + ``features.item.source``, and ``features.user.source`` subtrees are + treated as DataSource nodes. + + Matching on the key name alone at *any* depth (the previous behaviour) + false-positived on freeform fields such as + ``BigQueryConfig.query_parameters: dict[str, Any]``, where a caller's + query may legitimately bind a parameter literally named ``source`` whose + value is an unrelated nested mapping (e.g. a struct-typed parameter). That + mapping would be mistaken for a DataSource subtree and trigger a spurious + plugin-type lookup, failing recipe load with a confusing "Unknown + DataSource type" error even though no DataSource was ever referenced. + """ + return key == "source" and isinstance(value, dict) and path in _SOURCE_NODE_PATHS + + +def _resolve_extra_no_expand( + source_node: dict[str, Any], where: str, recipe_path: Path | str +) -> frozenset[str]: + """Resolve a source plugin's declared ``no_expand_fields`` for *source_node*. + + *source_node* is a raw (pre-expansion) ``source`` mapping — top-level or + nested under ``features.item`` / ``features.user``. Looks up the plugin + class via its ``type`` discriminator and returns the lower-cased + ``no_expand_fields`` set declared on it. + + *where* is the dotted position of *source_node* and *recipe_path* the file + it came from; both are named in errors. The type name alone identified + the offending source back when a recipe had exactly one, but a recipe now + carries up to three, so two recipes differing only in which subtree holds + a typo would otherwise raise byte-identical errors. + + Returns an empty set when ``type`` is absent (later validation reports the + missing discriminator). A genuinely unknown ``type`` (``DataSourceError``) + or any other lookup failure is NOT swallowed here: silently falling back + to the global ``_NO_EXPAND_KEYS`` baseline would weaken plugin-declared + protection (e.g. SQL injection via env expansion into ``dsn_env`` / + ``query``), so both are re-raised as ``RecipeError``. + """ + type_name = source_node.get("type") + if not type_name: + return frozenset() + try: + from recotem.datasource.registry import get_source_class + + src_cls = get_source_class(str(type_name)) + # Normalise to lowercase so that a plugin declaring 'SQL' and a YAML + # key 'sql:' are both blocked — matching is case-insensitive (see + # _expand_node). + return frozenset( + f.lower() for f in getattr(src_cls, "no_expand_fields", frozenset()) + ) + except DataSourceError as exc: + # Unknown source type during expansion: fail explicitly so the + # operator sees the error at recipe-load time rather than silently + # proceeding with only the global _NO_EXPAND_KEYS baseline. + raise RecipeError( + f"Recipe '{recipe_path}' {where}: plugin source discovery failed " + f"for type {type_name!r}: {exc}", + category="schema", + ) from exc + except Exception as exc: + logger.warning( + "source_class_lookup_failed_during_expand", + type=type_name, + error_class=type(exc).__name__, + where=where, + recipe=str(recipe_path), + ) + raise RecipeError( + f"Recipe '{recipe_path}' {where}: failed to resolve source plugin " + f"{type_name!r} during recipe load: {exc}" + ) from exc + + def _expand_node( node: Any, *, extra_allowed: dict[str, str] | None, + recipe_path: Path | str, _in_no_expand: bool = False, _extra_no_expand: frozenset[str] = frozenset(), + _path: tuple[str, ...] = (), ) -> Any: """Recursively walk a parsed YAML node and expand env-var references. Expansion is skipped entirely inside ``query`` and ``query_parameters`` keys at any nesting level, plus any keys listed in *_extra_no_expand* - (populated from the source plugin's ``no_expand_fields`` class variable - when processing the ``source`` subtree). + (populated from the source plugin's ``no_expand_fields`` class variable). + + *recipe_path* is the file *node* was parsed from, carried solely so that + a plugin-discovery failure can name it (see ``_resolve_extra_no_expand``). + + *_path* tracks the ancestor-key tuple of *node* itself (``()`` at the + recipe root, ``("features", "item")`` inside ``features.item``, etc.). A + mapping keyed literally ``source`` has its plugin's ``no_expand_fields`` + resolved fresh, via ``_resolve_extra_no_expand``, before the walk + descends into it — but only when ``_is_source_node`` confirms both the + key name AND the position (see ``_SOURCE_NODE_PATHS``): the recipe root, + ``features.item``, or ``features.user``. Restricting on position as well + as name prevents a freeform field (e.g. + ``BigQueryConfig.query_parameters``) that happens to contain a key named + ``source`` from being mistaken for a DataSource subtree. """ if isinstance(node, str): if _in_no_expand: @@ -299,12 +407,35 @@ def _expand_node( k.lower() for k in (_NO_EXPAND_KEYS | _extra_no_expand) ) for k, v in node.items(): + if _is_source_node(k, v, _path): + # A 'source' mapping at a legitimate schema position + # (top-level, or nested under features.item/features.user): + # consult the plugin's no_expand_fields before descending so + # protected fields (e.g. SQLSource's dsn_env) are shielded + # regardless of which of the two positions this subtree lives + # at. The parent's _extra_no_expand does not carry into a + # source subtree — each source is governed solely by its own + # plugin's declaration. + source_path = _path + (k,) + result[k] = _expand_node( + v, + extra_allowed=extra_allowed, + recipe_path=recipe_path, + _in_no_expand=_in_no_expand, + _extra_no_expand=_resolve_extra_no_expand( + v, ".".join(source_path), recipe_path + ), + _path=source_path, + ) + continue in_no_expand = _in_no_expand or (k.lower() in combined_no_expand) result[k] = _expand_node( v, extra_allowed=extra_allowed, + recipe_path=recipe_path, _in_no_expand=in_no_expand, _extra_no_expand=_extra_no_expand, + _path=_path + (k,), ) return result if isinstance(node, list): @@ -312,77 +443,16 @@ def _expand_node( _expand_node( item, extra_allowed=extra_allowed, + recipe_path=recipe_path, _in_no_expand=_in_no_expand, _extra_no_expand=_extra_no_expand, + _path=_path, ) for item in node ] return node -def _expand_with_source_no_expand( - raw_data: dict[str, Any], - *, - extra_allowed: dict[str, str] | None, -) -> dict[str, Any]: - """Expand env-var references in *raw_data*, honouring plugin ``no_expand_fields``. - - The ``source`` subtree is expanded separately so that the source plugin's - ``no_expand_fields`` class attribute can be consulted before the recursive - walk descends into source-specific fields. - - All other top-level keys are expanded with no additional restrictions - beyond the global ``_NO_EXPAND_KEYS``. - """ - result: dict[str, Any] = {} - for key, value in raw_data.items(): - if key == "source" and isinstance(value, dict): - # Determine extra protected fields from the plugin's class var. - extra_no_expand: frozenset[str] = frozenset() - type_name = value.get("type") - if type_name: - try: - from recotem.datasource.registry import get_source_class - - src_cls = get_source_class(str(type_name)) - # Normalise to lowercase so that a plugin declaring - # 'SQL' and a YAML key 'sql:' are both blocked — - # matching is case-insensitive (see _expand_node). - extra_no_expand = frozenset( - f.lower() - for f in getattr(src_cls, "no_expand_fields", frozenset()) - ) - except DataSourceError as exc: - # Unknown source type during expansion: fail explicitly so - # the operator sees the error at recipe-load time rather than - # silently proceeding with only the global _NO_EXPAND_KEYS - # baseline. Silent fallback weakens plugin-declared - # no_expand_fields protection and could allow SQL injection - # via env expansion into a query field. - raise RecipeError( - f"plugin source discovery failed for type {type_name!r}: {exc}", - category="schema", - ) from exc - except Exception as exc: - logger.warning( - "source_class_lookup_failed_during_expand", - type=type_name, - error_class=type(exc).__name__, - ) - raise RecipeError( - f"Failed to resolve source plugin {type_name!r} " - f"during recipe load: {exc}" - ) from exc - result[key] = _expand_node( - value, - extra_allowed=extra_allowed, - _extra_no_expand=extra_no_expand, - ) - else: - result[key] = _expand_node(value, extra_allowed=extra_allowed) - return result - - # --------------------------------------------------------------------------- # Pydantic validation error formatting # --------------------------------------------------------------------------- @@ -404,6 +474,72 @@ def _format_pydantic_errors(exc: pydantic.ValidationError) -> str: return "\n".join(lines) +# --------------------------------------------------------------------------- +# Typed source resolution +# --------------------------------------------------------------------------- + + +def _resolve_source_node(raw: Any, where: str, recipe_path: Path | str) -> Any: + """Validate a raw source mapping into its typed DataSource Config. + + *where* is a dotted path used in error messages (``"source"``, + ``"features.item.source"``, ``"features.user.source"``) and + *recipe_path* is the file the mapping came from; every message names + both, since a recipe carries up to three source subtrees and + ``load_recipes_directory`` (public API) does not re-add the filename. + *raw* is returned unchanged when it is not a dict (e.g. ``None`` for a + missing ``source`` key — later Recipe/FeatureSideConfig validation + reports that). + + Shared by the top-level ``source`` and every ``features..source`` + subtree so both get identical typed-resolution treatment: the same + ``type`` discriminator lookup, the same pydantic validation, and the same + error formatting. + + The ``model_validate`` + reassignment shape performed by the caller is + load-bearing: assigning the raw dict onto an already-built Recipe and + relying on ``validate_assignment`` would let an ``object.__setattr__`` + caller bypass re-validation. Building the Recipe once, with every typed + source already in place, avoids that bypass. + """ + if not isinstance(raw, dict): + return raw + try: + from recotem.datasource.registry import get_source_class + + type_name = raw.get("type") + if not type_name: + raise RecipeError( + f"Recipe '{recipe_path}' {where} is missing the 'type' discriminator.", + category="schema", + ) + source_cls = get_source_class(str(type_name)) + config_cls = source_cls.Config + try: + return config_cls.model_validate(raw) + except pydantic.ValidationError as exc: + detail = _format_pydantic_errors(exc) + raise RecipeError( + f"Recipe '{recipe_path}' {where} failed validation:\n{detail}" + ) from exc + except RecipeError: + raise + except (MemoryError, RecursionError): + raise + except Exception as exc: + raise RecipeError( + f"Recipe '{recipe_path}' {where} failed validation: {exc}" + ) from exc + except RecipeError: + raise + except (MemoryError, RecursionError): + raise + except Exception as exc: + raise RecipeError( + f"Recipe '{recipe_path}' {where} resolution failed: {exc}" + ) from exc + + # --------------------------------------------------------------------------- # Line-number extraction from YAML parse errors # --------------------------------------------------------------------------- @@ -481,9 +617,10 @@ def load_recipe( ) # Env expansion (never touches query / query_parameters, and honours - # plugin-declared no_expand_fields for the source subtree). + # plugin-declared no_expand_fields for every source subtree, including + # nested features.item.source / features.user.source). try: - expanded = _expand_with_source_no_expand(raw_data, extra_allowed=extra_allowed) + expanded = _expand_node(raw_data, extra_allowed=extra_allowed, recipe_path=p) except RecipeError: raise except (MemoryError, RecursionError): @@ -511,47 +648,27 @@ def load_recipe( # Path security checks. _validate_path_fields(expanded) - # Resolve the typed DataSource Config BEFORE building the Recipe so that - # pydantic's extra="forbid" is enforced on the source and the Recipe is - # constructed once with the typed source in place (no object.__setattr__ - # bypass of re-validation). + # Resolve the typed DataSource Config(s) BEFORE building the Recipe so + # that pydantic's extra="forbid" is enforced on every source and the + # Recipe is constructed once with all typed sources already in place (no + # object.__setattr__ bypass of re-validation). This covers the top-level + # source and every features..source subtree identically. raw_source = expanded.get("source") - if isinstance(raw_source, dict): - try: - from recotem.datasource.registry import get_source_class - - type_name = raw_source.get("type") - if not type_name: - raise RecipeError( - f"Recipe '{p}' source is missing the 'type' discriminator.", - category="schema", - ) - source_cls = get_source_class(str(type_name)) - config_cls = source_cls.Config - try: - typed_source = config_cls.model_validate(raw_source) - except pydantic.ValidationError as exc: - detail = _format_pydantic_errors(exc) - raise RecipeError( - f"Recipe '{p}' source failed validation:\n{detail}" - ) from exc - except RecipeError: - raise - except (MemoryError, RecursionError): - raise - except Exception as exc: - raise RecipeError( - f"Recipe '{p}' source failed validation: {exc}" - ) from exc - except RecipeError: - raise - except (MemoryError, RecursionError): - raise - except Exception as exc: - raise RecipeError(f"Recipe '{p}' source resolution failed: {exc}") from exc - # Replace the raw dict with the validated typed config before passing - # to Recipe.model_validate — this prevents object.__setattr__ bypass. - expanded = {**expanded, "source": typed_source} + expanded = {**expanded, "source": _resolve_source_node(raw_source, "source", p)} + + raw_features = expanded.get("features") + if isinstance(raw_features, dict): + typed_features = dict(raw_features) + for side in ("item", "user"): + side_node = typed_features.get(side) + if isinstance(side_node, dict) and "source" in side_node: + typed_features[side] = { + **side_node, + "source": _resolve_source_node( + side_node["source"], f"features.{side}.source", p + ), + } + expanded = {**expanded, "features": typed_features} # Build Recipe via pydantic (validates all sub-schemas). try: @@ -587,43 +704,50 @@ def load_recipe( return recipe -def _enforce_sha256_for_network_paths(recipe: Recipe) -> None: - """For source / item_metadata paths using a network scheme, require sha256. +def _require_sha256_for_network_path(node: Any, field_name: str) -> None: + """Raise if *node* (a typed Config with ``.path`` / ``.sha256``) needs a pin. - Raises - ------ - RecipeError - If a network-scheme path is missing the integrity pin. + Shared by ``_enforce_sha256_for_network_paths`` for ``source``, + ``item_metadata``, and every ``features..source`` node. """ - src = recipe.source - src_path = getattr(src, "path", None) + path = getattr(node, "path", None) if ( - isinstance(src_path, str) - and _network_scheme(src_path) - and not getattr(src, "sha256", None) + isinstance(path, str) + and _network_scheme(path) + and not getattr(node, "sha256", None) ): raise RecipeError( - f"'source.path' uses a network scheme " - f"({urlparse(src_path).scheme}://) and requires a 'sha256' " + f"'{field_name}' uses a network scheme " + f"({urlparse(path).scheme}://) and requires a 'sha256' " "integrity pin. Compute it with `shasum -a 256 ` and " - "set `source.sha256: `.", + f"set `{field_name.rsplit('.', 1)[0]}.sha256: `.", category="security", ) + +def _enforce_sha256_for_network_paths(recipe: Recipe) -> None: + """For source / item_metadata / feature-source paths on a network scheme, + require sha256. + + Raises + ------ + RecipeError + If a network-scheme path is missing the integrity pin. + """ + _require_sha256_for_network_path(recipe.source, "source.path") + meta = recipe.item_metadata if meta is not None: - meta_path = getattr(meta, "path", None) - if ( - isinstance(meta_path, str) - and _network_scheme(meta_path) - and not getattr(meta, "sha256", None) - ): - raise RecipeError( - f"'item_metadata.path' uses a network scheme " - f"({urlparse(meta_path).scheme}://) and requires a " - "'sha256' integrity pin.", - category="security", - ) + _require_sha256_for_network_path(meta, "item_metadata.path") + + features = recipe.features + if features is not None: + for side_name in ("item", "user"): + side = getattr(features, side_name) + if side is not None: + _require_sha256_for_network_path( + side.source, f"features.{side_name}.source.path" + ) def _validate_path_fields(data: dict[str, Any]) -> None: @@ -646,6 +770,20 @@ def _validate_path_fields(data: dict[str, Any]) -> None: if isinstance(meta_path, str): _validate_input_path(meta_path, "item_metadata.path") + features = data.get("features") + if isinstance(features, dict): + for side_name in ("item", "user"): + side_node = features.get(side_name) + if not isinstance(side_node, dict): + continue + side_source = side_node.get("source") + if isinstance(side_source, dict): + side_source_path = side_source.get("path") + if isinstance(side_source_path, str): + _validate_input_path( + side_source_path, f"features.{side_name}.source.path" + ) + def load_recipes_directory( path: str | Path, diff --git a/src/recotem/recipe/models.py b/src/recotem/recipe/models.py index 0f94d0bb..c53e858c 100644 --- a/src/recotem/recipe/models.py +++ b/src/recotem/recipe/models.py @@ -3,7 +3,7 @@ from __future__ import annotations import re -from typing import Annotated, Any +from typing import Annotated, Any, Literal import structlog from pydantic import BaseModel, Field, field_validator, model_validator @@ -197,11 +197,216 @@ def _validate_item_id_column(cls, v: str) -> str: return v +# --------------------------------------------------------------------------- +# Feature-aware iALS side features +# --------------------------------------------------------------------------- + +FeatureEncoding = Literal["categorical", "numerical", "multi_label"] + + +class FeatureColumn(BaseModel, extra="forbid"): + """One source column and how to turn it into numeric feature columns. + + ``delimiter`` applies only to ``multi_label`` (defaulted to ``"|"``). + ``min_frequency`` applies only to the vocabulary-based encodings; it is the + operator's only lever against RECOTEM_MAX_FEATURE_DIM, because the + vocabulary is built from the whole feature table and therefore scales with + catalog size rather than interaction size. + """ + + name: str = Field(min_length=1) + encoding: FeatureEncoding + delimiter: str | None = None + min_frequency: int = Field(default=1, ge=1) + + @model_validator(mode="after") + def _validate_encoding_options(self) -> FeatureColumn: + if self.encoding == "multi_label": + if self.delimiter is None: + object.__setattr__(self, "delimiter", "|") + elif not self.delimiter: + raise ValueError("delimiter must not be empty for multi_label") + elif self.delimiter is not None: + raise ValueError( + f"delimiter is only valid for encoding='multi_label', " + f"not {self.encoding!r} (column {self.name!r})" + ) + if self.encoding == "numerical" and self.min_frequency != 1: + raise ValueError( + f"min_frequency is only valid for vocabulary-based encodings " + f"(categorical, multi_label), not 'numerical' " + f"(column {self.name!r})" + ) + return self + + +class FeatureSideConfig(BaseModel, extra="forbid"): + """Feature table for one side (item or user) plus its encoding plan.""" + + # ``source`` is ``Any`` for the same reason as ``Recipe.source``: the + # discriminated union is built dynamically from entry points and importing + # datasource/registry.py here would be circular. recipe/loader.py performs + # the typed construction; ``Recipe._validate_features_sources`` is the + # fallback check for direct construction. + source: Any + id_column: str = Field(min_length=1) + columns: list[FeatureColumn] = Field(min_length=1) + + @field_validator("id_column") + @classmethod + def _validate_id_column(cls, v: str) -> str: + if not v.strip(): + raise ValueError("id_column must not be empty or whitespace-only") + return v + + @field_validator("columns") + @classmethod + def _validate_unique_columns(cls, v: list[FeatureColumn]) -> list[FeatureColumn]: + names = [c.name for c in v] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + raise ValueError(f"duplicate feature column names: {dupes}") + return v + + @model_validator(mode="after") + def _validate_id_column_not_a_feature(self) -> FeatureSideConfig: + """Reject an id_column that also names a feature column. + + ``_fetch_side`` does ``set_index(id_column)``, which consumes that + column, so ``build_encoder_state`` would then raise a misleading + "feature column '...' is not present" at train time (exit 4). The + failure is guaranteed, so surface it early at recipe load instead. + """ + if any(c.name == self.id_column for c in self.columns): + raise ValueError( + f"id_column {self.id_column!r} collides with a feature column " + f"of the same name in columns; the id column is consumed as the " + f"index and cannot also be a feature. Rename one of them." + ) + return self + + +class FeaturesConfig(BaseModel, extra="forbid"): + """Side features for feature-aware iALS. + + The mere presence of this block on a recipe enables feature-aware training + for every feature-capable algorithm in ``training.algorithms``. There is + no separate flag. ``lambda_*_feature`` is a ridge on the feature-weight + matrices ``A``/``B``, not the strength of the feature prior itself; a + large enough value shrinks ``A``/``B`` toward zero and was measured + bit-identical to plain iALS at ``λ=1e8``. The tuned range's upper bound + (``1e6``) is only what upstream's own example exercises, not a verified + "features off" point, so the search space is not guaranteed to contain a + features-off model. Run two recipes if you need a true on/off comparison. + """ + + item: FeatureSideConfig | None = None + user: FeatureSideConfig | None = None + + @model_validator(mode="after") + def _validate_at_least_one_side(self) -> FeaturesConfig: + if self.item is None and self.user is None: + raise ValueError( + "features requires at least one of features.item or features.user" + ) + return self + + # --------------------------------------------------------------------------- # Recipe # --------------------------------------------------------------------------- +def _check_source_value(src: Any, where: str) -> None: + """Reject a source whose ``type`` is not a registered DataSource. + + *where* names the field for the error message ("source", + "features.item.source"). Shared by ``Recipe._validate_source`` and + ``Recipe._validate_features_sources``. + + Behaviour by source kind + ------------------------ + * ``dict`` with a ``type`` key in the registry → allowed (backward + compat for ``Recipe.model_validate(d)`` and test helpers that pass + raw dicts). + * ``dict`` with a ``type`` key NOT in the registry → rejected. + * ``dict`` with no ``type`` key → rejected (missing discriminator). + * ``pydantic.BaseModel`` subclass that is a known Config class → + allowed. + * ``pydantic.BaseModel`` subclass that is NOT a known Config class → + rejected. + * Any other object (e.g. test mock, legacy opaque object) → allowed + silently (registry check skipped to avoid false positives). + + The registry import stays deferred and a registry failure stays + non-fatal (warn and skip) -- both are load-bearing: a module-level + import would trigger plugin-loading side effects at recipe-model import + time, and a broken third-party plugin must not make every recipe + unloadable. + """ + if src is None: + raise ValueError(f"{where} must not be None") + + try: + from pydantic import BaseModel as _BaseModel # noqa: PLC0415 + + from recotem.datasource.registry import get_source_types # noqa: PLC0415 + + types = get_source_types() + known_types_set: set[str] = {cls.type_name for cls in types.values()} # type: ignore[union-attr] + known_config_classes = tuple(cls.Config for cls in types.values()) # type: ignore[union-attr] + except Exception as exc: # noqa: BLE001 + # Registry unavailable (import failure, broken plugin, etc.) — + # emit a structured warning so operators can diagnose broken plugins + # without a traceback, then let the datasource path surface the error + # at training time. + logger.warning( + "source_registry_unavailable_during_validation", + error_class=type(exc).__name__, + error=str(exc), + ) + return + + if isinstance(src, dict): + type_val = src.get("type") + if type_val is None: + raise ValueError( + f"{where} dict is missing the 'type' discriminator field. " + f"Known types: {sorted(known_types_set)}." + ) + if str(type_val) not in known_types_set: + raise ValueError( + f"{where} type {type_val!r} is not a registered DataSource type. " + f"Known types: {sorted(known_types_set)}." + ) + # Known dict type — allow through (will be validated by load_recipe + # or the datasource pipeline). + return + + if isinstance(src, _BaseModel): + # Pydantic model: check it is one of the known Config classes. + if known_config_classes and not isinstance(src, known_config_classes): + type_hint = getattr(src, "type", type(src).__name__) + raise ValueError( + f"{where} type {type_hint!r} is not a known DataSource Config. " + f"Known types: {sorted(known_types_set)}. " + "Pass a typed Config (e.g. CSVConfig, BigQueryConfig) or use " + "load_recipe() to validate from YAML." + ) + return + + # Non-pydantic, non-dict source: reject immediately so library callers + # get a clear ValidationError rather than a cryptic runtime error deep + # inside the training pipeline. Test code that needs to pass a mock + # must use a pydantic BaseModel (e.g. a MagicMock(spec=CSVConfig)) or + # a valid dict with a recognised 'type' key. + raise ValueError( + f"{where} must be a dict with a 'type' key or a registered " + "DataSource Config subclass (pydantic BaseModel). " + f"Got {type(src).__name__!r} which is neither." + ) + + class Recipe(BaseModel, extra="forbid"): """Top-level recipe model. @@ -225,6 +430,7 @@ class Recipe(BaseModel, extra="forbid"): schema_: SchemaConfig = Field(alias="schema") cleansing: CleansingConfig = Field(default_factory=CleansingConfig) item_metadata: ItemMetadataConfig | None = None + features: FeaturesConfig | None = None training: TrainingConfig = Field(default_factory=TrainingConfig) output: OutputConfig @@ -239,94 +445,54 @@ def _validate_name(cls, v: str) -> str: @model_validator(mode="after") def _validate_source(self) -> Recipe: - """Reject sources with a ``type`` that is not registered. + """Reject a ``source`` with a ``type`` that is not registered. When a library caller passes ``source={"type": "unknown_xyz"}`` or a pydantic BaseModel whose type_name is not in the registry, raise immediately rather than allowing the error to surface obscurely at - training time. - - Behaviour by source kind - ------------------------ - * ``dict`` with a ``type`` key in the registry → allowed (backward - compat for ``Recipe.model_validate(d)`` and test helpers that pass - raw dicts). - * ``dict`` with a ``type`` key NOT in the registry → rejected. - * ``dict`` with no ``type`` key → rejected (missing discriminator). - * ``pydantic.BaseModel`` subclass that is a known Config class → - allowed. - * ``pydantic.BaseModel`` subclass that is NOT a known Config class → - rejected. - * Any other object (e.g. test mock, legacy opaque object) → allowed - silently (registry check skipped to avoid false positives). - - The registry import is deferred (not at module level) to avoid - triggering the datasource plugin loading side-effects at recipe-model - import time, and to tolerate environments where the datasource extras - are absent. + training time. See ``_check_source_value`` for the full behaviour + table. """ - src = self.source - if src is None: - raise ValueError("source must not be None") + _check_source_value(self.source, "source") + return self - try: - from pydantic import BaseModel as _BaseModel # noqa: PLC0415 - - from recotem.datasource.registry import get_source_types # noqa: PLC0415 - - types = get_source_types() - known_types_set: set[str] = {cls.type_name for cls in types.values()} # type: ignore[union-attr] - known_config_classes = tuple(cls.Config for cls in types.values()) # type: ignore[union-attr] - except Exception as exc: # noqa: BLE001 - # Registry unavailable (import failure, broken plugin, etc.) — - # emit a structured warning so operators can diagnose broken plugins - # without a traceback, then let the datasource path surface the error - # at training time. - logger.warning( - "source_registry_unavailable_during_validation", - error_class=type(exc).__name__, - error=str(exc), - ) - return self + @model_validator(mode="after") + def _validate_features_sources(self) -> Recipe: + """Reject feature sources with an unregistered ``type``. - if isinstance(src, dict): - type_val = src.get("type") - if type_val is None: - raise ValueError( - "source dict is missing the 'type' discriminator field. " - f"Known types: {sorted(known_types_set)}." - ) - if str(type_val) not in known_types_set: - raise ValueError( - f"source type {type_val!r} is not a registered DataSource type. " - f"Known types: {sorted(known_types_set)}." - ) - # Known dict type — allow through (will be validated by load_recipe - # or the datasource pipeline). + ``load_recipe`` performs the full typed construction; direct + ``Recipe(...)`` calls bypass the YAML pipeline, so this mirrors + ``_validate_source`` for the feature subtrees. + """ + if self.features is None: return self + for side_name in ("item", "user"): + side = getattr(self.features, side_name) + if side is not None: + _check_source_value(side.source, f"features.{side_name}.source") + return self - if isinstance(src, _BaseModel): - # Pydantic model: check it is one of the known Config classes. - if known_config_classes and not isinstance(src, known_config_classes): - type_hint = getattr(src, "type", type(src).__name__) - raise ValueError( - f"source type {type_hint!r} is not a known DataSource Config. " - f"Known types: {sorted(known_types_set)}. " - "Pass a typed Config (e.g. CSVConfig, BigQueryConfig) or use " - "load_recipe() to validate from YAML." - ) - return self + @model_validator(mode="after") + def _validate_features_algorithms(self) -> Recipe: + """Reject a features block that no listed algorithm can consume. - # Non-pydantic, non-dict source: reject immediately so library callers - # get a clear ValidationError rather than a cryptic runtime error deep - # inside the training pipeline. Test code that needs to pass a mock - # must use a pydantic BaseModel (e.g. a MagicMock(spec=CSVConfig)) or - # a valid dict with a recognised 'type' key. - raise ValueError( - "source must be a dict with a 'type' key or a registered " - "DataSource Config subclass (pydantic BaseModel). " - f"Got {type(src).__name__!r} which is neither." - ) + Deliberately tolerant of unresolvable algorithm names: those stay + deferred to train time, matching ``_validate_per_algorithm_trials_keys``. + """ + if self.features is None: + return self + try: + from recotem.training.algorithms import is_feature_capable # noqa: PLC0415 + except ImportError: # pragma: no cover - training extras absent + return self + if not any(is_feature_capable(a) for a in self.training.algorithms): + raise ValueError( + "features is configured but training.algorithms contains no " + "feature-capable algorithm. Feature-aware training requires " + "IALS. Either add IALS to training.algorithms or remove the " + "features block." + ) + return self @model_validator(mode="after") def _validate_time_split(self) -> Recipe: diff --git a/src/recotem/serving/app.py b/src/recotem/serving/app.py index ad406b57..fc9fb581 100644 --- a/src/recotem/serving/app.py +++ b/src/recotem/serving/app.py @@ -39,6 +39,7 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import ASGIApp +from recotem._features import check_artifact_feature_version from recotem._irspack_compat import check_artifact_irspack_version from recotem.artifact.format import ArtifactError, parse_header_from_bytes from recotem.artifact.signing import KeyRing, unpickle_payload, verify_hmac @@ -865,6 +866,21 @@ def _try_load_artifact( ) return _failed_entry(recipe, str(exc)), "version_skew" + # Preflight the feature-encoder state version for the same reason: an + # unknown shape would otherwise be silently mis-encoded rather than + # refused, and the failure would surface as wrong recommendations, not an + # exception. + try: + check_artifact_feature_version(header_dict, name=recipe.name) + except ArtifactError as exc: + logger.warning( + "initial_artifact_feature_version_refused", + name=recipe.name, + kid=hdr.kid, + error=str(exc), + ) + return _failed_entry(recipe, str(exc)), "feature_version" + try: recommender = unpickle_payload(payload_bytes) except ArtifactError as exc: diff --git a/src/recotem/serving/metrics.py b/src/recotem/serving/metrics.py index 343e53cd..197a6480 100644 --- a/src/recotem/serving/metrics.py +++ b/src/recotem/serving/metrics.py @@ -23,6 +23,9 @@ | ``recotem_v1_batch_element_errors_total`` | Counter | recipe, verb, code | | ``recotem_v1_metadata_degraded_items_total`` | Counter | recipe, verb, kind | | ``recotem_v1_validation_errors_outside_verb_total``| Counter | — | +| ``recotem_v1_feature_unknown_value_total`` | Counter | recipe, side, column | +| ``recotem_v1_feature_unknown_column_total`` | Counter | recipe, side | +| ``recotem_v1_cold_start_requests_total`` | Counter | recipe, case | | ``recotem_model_loaded`` | Gauge | recipe | | ``recotem_artifact_load_failures_total`` | Counter | recipe, reason | | ``recotem_active_recipes`` | Gauge | — | @@ -39,7 +42,8 @@ Artifact-load reason taxonomy (``recotem_artifact_load_failures_total``): ``read``, ``parse``, ``hmac``, ``header_json``, ``deserialize``, ``metadata``, -``yaml``, ``unexpected``, ``dir_scan``, ``timeout``, ``version_skew``. +``yaml``, ``unexpected``, ``dir_scan``, ``timeout``, ``version_skew``, +``feature_version``. """ from __future__ import annotations @@ -108,7 +112,7 @@ def _ensure_initialized() -> None: "recotem_artifact_load_failures_total", "Total artifact load failures (initial load and watcher reloads). " "reason ∈ {read, parse, hmac, header_json, deserialize, metadata, " - "yaml, unexpected, dir_scan, timeout, version_skew}.", + "yaml, unexpected, dir_scan, timeout, version_skew, feature_version}.", ["recipe", "reason"], ) _ACTIVE_RECIPES = Gauge( @@ -193,6 +197,11 @@ def set_model_loaded(recipe: str, loaded: bool) -> None: # Distinct from "read" (file could not be opened/parsed) because stat # timeouts are an infrastructure signal rather than a data signal. "timeout", + # Artifact's feature-encoder state version is unknown to this build + # (missing, newer, or malformed). Distinct from "version_skew" + # (irspack pickle compatibility) and from "deserialize" (the payload + # is never touched here -- the header alone is enough to refuse). + "feature_version", } ) @@ -202,8 +211,8 @@ def inc_artifact_load_failure(recipe: str, reason: str = "unexpected") -> None: *reason* must be one of the values in ``_LOAD_FAILURE_REASONS`` (``read | parse | hmac | header_json | deserialize | metadata | yaml | - unexpected | dir_scan | timeout | version_skew``); any other value is - silently coerced + unexpected | dir_scan | timeout | version_skew | feature_version``); any + other value is silently coerced to ``"unexpected"`` so callers cannot accidentally explode the cardinality of the label. """ @@ -296,9 +305,11 @@ def inc_recipe_rescan_error(recipe: str) -> None: def inc_recommender_layout_unexpected(recipe: str) -> None: """Increment the per-recipe recommender-layout-unexpected counter. - Called when ``_any_seed_known`` encounters an ``AttributeError`` accessing - ``recommender._mapper.item_id_to_index``, indicating an unexpected irspack - internal layout. A non-zero rate signals an API incompatibility. + Called when ``_resolve_recommend`` or ``_resolve_recommend_related`` + encounters an ``AttributeError`` accessing + ``recommender._mapper.user_id_to_index`` / ``item_id_to_index``, + indicating an unexpected irspack internal layout. A non-zero rate + signals an API incompatibility. """ _ensure_initialized() if _RECOMMENDER_LAYOUT_UNEXPECTED is None: @@ -330,6 +341,9 @@ def inc_watcher_state_divergence() -> None: _V1_BATCH_ELEMENT_ERRORS: Any = None _V1_METADATA_DEGRADED_ITEMS: Any = None _V1_VALIDATION_ERRORS_OUTSIDE_VERB: Any = None +_V1_FEATURE_UNKNOWN_VALUE: Any = None +_V1_FEATURE_UNKNOWN_COLUMN: Any = None +_V1_COLD_START_REQUESTS: Any = None def _ensure_v1_initialized() -> None: @@ -341,6 +355,8 @@ def _ensure_v1_initialized() -> None: global _V1_REQUEST_COUNTER, _V1_REQUEST_LATENCY, _V1_BATCH_SIZE global _V1_BATCH_ELEMENT_ERRORS global _V1_METADATA_DEGRADED_ITEMS, _V1_VALIDATION_ERRORS_OUTSIDE_VERB + global _V1_FEATURE_UNKNOWN_VALUE, _V1_FEATURE_UNKNOWN_COLUMN + global _V1_COLD_START_REQUESTS if _V1_REQUEST_COUNTER is not None: return if not metrics_enabled(): @@ -382,6 +398,30 @@ def _ensure_v1_initialized() -> None: "422 validation errors on non-v1-verb paths (e.g. /v1/recipes listing " "with bad query parameters).", ) + _V1_FEATURE_UNKNOWN_VALUE = Counter( + "recotem_v1_feature_unknown_value_total", + "Request feature values absent from the training vocabulary, or " + "otherwise unusable. Fires on a categorical miss, a multi_label " + "value where any supplied token misses, or a non-finite numerical " + "value (+-inf, or NaN reached via a string). A missing or " + "unparseable numerical value still degrades silently and is NOT " + "counted here. See docs/api-reference.md#feature-aware-cold-start.", + ["recipe", "side", "column"], + ) + _V1_FEATURE_UNKNOWN_COLUMN = Counter( + "recotem_v1_feature_unknown_column_total", + "Cold-start requests carrying at least one feature key the recipe " + "does not declare. Such a key is never read by the encoder, so the " + "request degrades toward a bias-only profile with an otherwise " + "normal 200. Counted once per request per side -- deliberately NOT " + "labelled by column name, which is unbounded request input.", + ["recipe", "side"], + ) + _V1_COLD_START_REQUESTS = Counter( + "recotem_v1_cold_start_requests_total", + "Cold-start requests served from side features, by case.", + ["recipe", "case"], + ) def record_v1_request( @@ -392,7 +432,14 @@ def record_v1_request( *verb* ∈ {"recommend", "recommend-related", "batch-recommend", "batch-recommend-related"}. *status* ∈ {"ok", "unknown_user", "unknown_seed_items", "no_candidates", "unavailable", - "recipe_not_found", "validation_error", "error"}. + "recipe_not_found", "validation_error", "features_not_supported", + "feature_value_unusable", "error"}. + + ``features_not_supported`` / ``feature_value_unusable`` are the + single-verb counterparts of the identically-named batch ``code`` labels + on ``inc_batch_element_error``. They are client-caused 400s and must + stay OUT of ``"error"``, which docs/operations.md pages on-call for -- + see that file's "Recommend error rate" row. """ _ensure_v1_initialized() if _V1_REQUEST_COUNTER is None: @@ -424,6 +471,10 @@ def inc_batch_element_error(recipe: str, verb: str, code: str) -> None: _DEGRADED_ITEM_KINDS: frozenset[str] = frozenset({"fallback", "dropped", "unexpected"}) +_FEATURE_SIDES: frozenset[str] = frozenset({"item", "user", "unexpected"}) +_COLD_START_CASES: frozenset[str] = frozenset( + {"features_only", "features_and_history", "cold_seeds", "unexpected"} +) def inc_metadata_degraded_items( @@ -445,6 +496,87 @@ def inc_metadata_degraded_items( _V1_METADATA_DEGRADED_ITEMS.labels(recipe=recipe, verb=verb, kind=label).inc(count) +def inc_feature_unknown_value( + recipe: str, side: str, column: str, count: int = 1 +) -> None: + """Increment the unknown-feature-value counter. + + *side* must be ``"item"`` or ``"user"``; anything else is coerced to + ``"unexpected"`` to prevent label-cardinality explosion. *column* is NOT + coerced: it comes from the recipe, so its cardinality is bounded by the + operator's own column list, not by request input. + + An unknown value cannot fail the request, but coverage is per-encoding, + not universal (see ``_row_values`` in ``recotem._features``): a + ``categorical`` miss always increments this counter, and a + ``multi_label`` miss increments it whenever *any* supplied token misses + the vocabulary -- a mixed value such as ``"Action|Thrller"`` still + increments it once for the column, even though the known token's bit is + also set. A ``numerical`` value increments this counter when it is + non-finite (``+inf``, ``-inf``, or a NaN reached via a string like + ``"nan"``) -- these parse successfully but cannot be standardized. A + ``numerical`` value that is missing or fails to parse as a number at all + still degrades to the standardized mean (0) with no signal -- that gap + is deliberate and separate, not fixed by the non-finite case above. See + ``docs/api-reference.md#feature-aware-cold-start`` for the full + breakdown. + """ + _ensure_v1_initialized() + if _V1_FEATURE_UNKNOWN_VALUE is None: + return + label = side if side in _FEATURE_SIDES else "unexpected" + _V1_FEATURE_UNKNOWN_VALUE.labels(recipe=recipe, side=label, column=column).inc( + count + ) + + +def inc_feature_unknown_column(recipe: str, side: str) -> None: + """Increment the unknown-feature-column counter. + + Called once per cold-start request per *side* whose supplied feature + mapping carries at least one key the recipe does not declare. Such a key + is never read (``_features._row_values`` iterates the recipe's columns + and does ``values.get(name)``), so the request silently degrades toward a + bias-only profile while still returning 200 -- a strictly more severe + degradation than an unknown VALUE, which + ``inc_feature_unknown_value`` already covers. + + *side* must be ``"item"`` or ``"user"``; anything else is coerced to + ``"unexpected"``. + + There is deliberately NO ``column`` label, and this is the one place that + asymmetry with ``inc_feature_unknown_value`` matters: that function's + ``column`` is safe precisely BECAUSE it comes from the recipe, so the + operator's own column list bounds its cardinality. An unknown column name + is unbounded request input -- labelling it would let any client mint + arbitrary time series and exhaust the scrape target's memory. The cost is + diagnosability: the counter says a caller is sending undeclared keys but + not which, and the remedy is to diff the client against the recipe's + ``features:`` block. Counted once per request per side rather than once + per key so the number stays normalizable against + ``recotem_v1_requests_total`` and does not scale with cold-seed fan-out. + """ + _ensure_v1_initialized() + if _V1_FEATURE_UNKNOWN_COLUMN is None: + return + label = side if side in _FEATURE_SIDES else "unexpected" + _V1_FEATURE_UNKNOWN_COLUMN.labels(recipe=recipe, side=label).inc() + + +def inc_cold_start_request(recipe: str, case: str) -> None: + """Increment the cold-start-request counter. + + *case* is ``"features_only"`` (A), ``"features_and_history"`` (B), or + ``"cold_seeds"`` (C). Any other value is coerced to ``"unexpected"`` to + prevent label-cardinality explosion. + """ + _ensure_v1_initialized() + if _V1_COLD_START_REQUESTS is None: + return + label = case if case in _COLD_START_CASES else "unexpected" + _V1_COLD_START_REQUESTS.labels(recipe=recipe, case=label).inc() + + def inc_validation_error_outside_verb() -> None: """Increment the counter for 422 errors on non-v1-verb paths. diff --git a/src/recotem/serving/routes.py b/src/recotem/serving/routes.py index b94c65e1..793ec0ff 100644 --- a/src/recotem/serving/routes.py +++ b/src/recotem/serving/routes.py @@ -19,12 +19,14 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response from pydantic import ValidationError +from recotem._idmap import ColdStartNumericalError from recotem.config import ApiKeyEntry from recotem.serving import metrics as _metrics from recotem.serving.auth import verify_api_key from recotem.serving.registry import ModelEntry, ModelRegistry from recotem.serving.schemas import ( BATCH_AGGREGATE_LIMIT, + BATCH_COLD_SEED_SOLVE_LIMIT, BatchRecommendRelatedRequest, BatchRecommendRequest, BatchRecommendResponse, @@ -90,6 +92,335 @@ def _format_batch_validation_message(exc: ValidationError) -> str: return f"{loc_path}: {msg}" if loc_path else msg +def _cold_seed_solve_bound(body: RecommendRelatedRequest) -> int: + """Upper bound on the cold-seed CG solves *body* can drive. + + A seed is solved only when it is BOTH absent from the model's id-map and + named in ``item_features`` (``_idmap.get_recommendation_for_cold_seeds``). + This deliberately does not consult the id-map, so the bound over-counts a + seed that turns out to be known -- the same posture as + ``BATCH_AGGREGATE_LIMIT``, which bounds work by the request's declared + ``limit`` rather than by the items actually returned. Keeping the cap a + pure function of the request makes it a stable client contract: the same + body is accepted or rejected identically regardless of which model happens + to be loaded, and a retrain that warms a seed cannot silently flip the + verdict. + """ + if not body.item_features: + return 0 + return sum(1 for seed in body.seed_items if str(seed) in body.item_features) + + +@contextmanager +def _bind_batch_idx(idx: int) -> Iterator[None]: + """Bind ``idx`` as a structlog contextvar for one batch-loop iteration. + + The shared ``_resolve_recommend`` / ``_resolve_recommend_related`` + resolvers have no batch-index parameter (they serve both the single + verbs and the batch element loops), so any ``logger.warning`` / + ``logger.exception`` call made from *inside* a resolver -- e.g. + ``recommender_layout_unexpected`` or ``recommender_unexpected_key_error`` + -- would otherwise have no way to identify which batch element it came + from. Binding ``idx`` here lets it ride along on every log event emitted + anywhere during this element's processing via + ``structlog.contextvars.merge_contextvars``, without threading an index + parameter through the resolver signatures. + + Unbound in ``finally`` so neither an exception propagating out nor a + ``continue`` to the next loop iteration leaves a stale ``idx`` bound for + the next element. + """ + structlog.contextvars.bind_contextvars(idx=idx) + try: + yield + finally: + structlog.contextvars.unbind_contextvars("idx") + + +# --------------------------------------------------------------------------- +# Cold-start resolution -- shared by the single and batch verbs +# --------------------------------------------------------------------------- +# +# ``_resolve_recommend`` / ``_resolve_recommend_related`` hold the ONLY copy +# of the case A/B/C branch logic (see docstrings below for the case table). +# Both the single handlers and the batch element loops call these; the +# single handlers map the domain exceptions to an ``HTTPException``, the +# batch loops map them to a ``BatchResultErr``. A bare ``KeyError`` / +# ``AttributeError`` propagating out (already logged here where it +# originates) means "unexpected recommender layout" -- both kinds of caller +# must map that to INTERNAL_ERROR too. + + +def _has_undeclared_columns(state: dict, values: dict[str, Any]) -> bool: + """True when *values* carries a key the encoder will never read. + + ``_features._row_values`` drives the encode from ``state["columns"]`` and + does ``values.get(name)``, so any request key outside that list is + silently dropped -- the request degrades toward a bias-only profile and + still returns 200. This is the only detection point for that. + + Derived here rather than reported by ``encode_one`` because the answer is + a pure function of two things this caller already holds: the request's own + mapping, and the recipe's declared column names. The declared names come + from ``_features.state_descriptor`` rather than from ``state["columns"]`` + directly so the state dict's internal shape stays private to + ``_features``. One coupling remains and is deliberate: if ``_row_values`` + ever reads a key NOT among the declared columns (e.g. a derived column), + this would misreport it as undeclared. Returning the undeclared keys from + ``encode_one`` itself would make that unrepresentable. + + Only called on a path where the encode already SUCCEEDED, so *state* is + non-None and well-formed by construction (``encode_one`` just consumed + it); no defensive guard is warranted here. + """ + from recotem._features import state_descriptor # noqa: PLC0415 + + descriptor = state_descriptor(state) + assert descriptor is not None # noqa: S101 — non-None on the encoded path + return not set(values).issubset(descriptor["columns"]) + + +class _ColdStartUnsupported(Exception): + """The model carries no feature state for the side the request supplied, + or its search winner is not on the feature-capable allow-list (Task 11's + ``_require_capability``) despite carrying non-None state.""" + + +class _ColdStartValueUnusable(Exception): + """A supplied feature value made irspack's cold-start solver numerically + unstable (``_idmap.ColdStartNumericalError`` -- e.g. an + extreme-but-finite ``numerical`` value like ``1e22``). + + Deliberately a DIFFERENT error from ``_ColdStartUnsupported``: that one + means the model/feature side can never do cold start at all, whereas + this is a per-value condition on an otherwise-capable model. Folding it + into ``FEATURES_NOT_SUPPORTED`` would incorrectly tell the client the + model cannot serve this recipe's cold start when a different value + would have worked fine. + """ + + +class _NoUsableUser(Exception): + """Unknown user and no user_features to fall back on.""" + + +class _NoUsableSeeds(Exception): + """No seed is known and none carried item_features.""" + + +class _NoCandidates(Exception): + """All seeds known, no features supplied, but the ranker produced no + survivors after its own filtering/score-thresholding.""" + + +def _resolve_recommend( + entry: ModelEntry, name: str, verb: str, body: RecommendRequest +) -> list[tuple[str, float]]: + """Resolve one ``:recommend`` request -- single call or batch element. + + Raises ``_NoUsableUser`` / ``_ColdStartUnsupported`` for conditions the + caller maps to its own error shape. A propagated ``KeyError`` (logged + here) means the recommender layout was unexpected -- callers must map + that to INTERNAL_ERROR. + """ + # S1: determine known-membership BEFORE calling irspack so a genuine + # missing user produces UNKNOWN_USER, not INTERNAL_ERROR. user_known is + # None when the recommender layout is unexpected (F4); irspack itself + # is still given the chance to serve the request in that case. + try: + user_known: bool | None = ( + body.user_id in entry.recommender._mapper.user_id_to_index + ) + except AttributeError as _attr_exc: + logger.warning( + "recommender_layout_unexpected", + recipe=name, + verb=verb, + exc_type=type(_attr_exc).__name__, + ) + _metrics.inc_recommender_layout_unexpected(name) + user_known = None + + if user_known is False and body.user_features is not None: + # Case A: unknown user, cold-started from supplied feature values + # alone (no interaction history exists yet). The ValueError from + # get_recommendation_for_cold_user covers BOTH "model has no user + # feature state" and "the search winner is not feature-capable + # despite carrying state" (Task 9 persists feature state + # unconditionally, so a TopPop artifact can have non-None state + # without being able to act on it) -- Task 11's _require_capability + # is the single source of truth for that distinction, so we relay + # it as _ColdStartUnsupported rather than re-deriving the same + # check here from `user_feature_state is None` alone. + try: + raw_results, unknown_columns = ( + entry.recommender.get_recommendation_for_cold_user( + body.user_features, + cutoff=body.limit, + ) + ) + except ColdStartNumericalError as exc: + raise _ColdStartValueUnusable(str(exc)) from None + except ValueError as exc: + raise _ColdStartUnsupported(str(exc)) from None + _metrics.inc_cold_start_request(name, "features_only") + for column in unknown_columns: + _metrics.inc_feature_unknown_value(name, "user", column) + if _has_undeclared_columns( + entry.recommender.user_feature_state, body.user_features + ): + _metrics.inc_feature_unknown_column(name, "user") + return raw_results + + # Known users always take this path, and a known user's supplied + # user_features are deliberately IGNORED here, not rejected: the + # learned embedding was fit to their real interactions and strictly + # dominates a profile prior, so rejecting would break the natural + # client pattern of always sending the profile and letting the server + # decide. Cross-referenced from docs/api-reference.md#feature-aware-cold-start + # ("A known `user_id` with `user_features` supplied is not an error."). + try: + return entry.recommender.get_recommendation_for_known_user_id( + body.user_id, body.limit + ) + except KeyError: + if user_known is False: + # Deterministic miss: user was not in the id-map, and no + # user_features were supplied to cold-start. + raise _NoUsableUser(body.user_id) from None + # user_known is True or None (unexpected layout): propagate as + # INTERNAL_ERROR so layout surprises are visible, not silent. + logger.exception( + "recommender_unexpected_key_error", + recipe=name, + verb=verb, + user_id_hash=hashlib.sha256(body.user_id.encode()).hexdigest()[:8], + ) + raise + + +def _resolve_recommend_related( + entry: ModelEntry, name: str, verb: str, body: RecommendRelatedRequest +) -> list[tuple[str, float]]: + """Resolve one ``:recommend-related`` request -- single call or batch + element. + + Raises ``_NoUsableSeeds`` / ``_ColdStartUnsupported`` / ``_NoCandidates`` + for conditions the caller maps to its own error shape. A propagated + ``AttributeError`` or ``KeyError`` (logged here) means the recommender + layout was unexpected -- callers must map that to INTERNAL_ERROR. + """ + # Fetch the item id-map once, up front: needed both to find cold seeds + # (case C) and for the plain "any seed known" check. + try: + id_map = entry.recommender._mapper.item_id_to_index + except AttributeError as _attr_exc: + logger.warning( + "recommender_layout_unexpected", + recipe=name, + verb=verb, + exc_type=type(_attr_exc).__name__, + ) + _metrics.inc_recommender_layout_unexpected(name) + raise + + cold_seeds = [s for s in body.seed_items if str(s) not in id_map] + have_cold_features = bool(body.item_features) and any( + str(s) in body.item_features for s in cold_seeds + ) + + if have_cold_features: + # Case C. Must win over case B: a cold seed has no row in the seed + # interaction matrix, so the case-B solve + # (get_recommendation_for_new_user) would silently drop it even + # though user_features may also be present. + try: + raw_results, unknown_columns = ( + entry.recommender.get_recommendation_for_cold_seeds( + body.seed_items, + body.item_features or {}, + cutoff=body.limit, + ) + ) + except ColdStartNumericalError as exc: + raise _ColdStartValueUnusable(str(exc)) from None + except ValueError as exc: + raise _ColdStartUnsupported(str(exc)) from None + except KeyError: + # No seed was usable: none known, and none of the cold ones + # actually carried a feature entry (the `have_cold_features` + # gate above only requires ONE match; defence in depth in case + # that gate and this method's own criteria for "usable" ever + # diverge). + raise _NoUsableSeeds(list(body.seed_items)) from None + _metrics.inc_cold_start_request(name, "cold_seeds") + for column in unknown_columns: + _metrics.inc_feature_unknown_value(name, "item", column) + # Only the seeds irspack actually encoded: a KNOWN seed contributes + # its learned embedding and its item_features entry is never looked + # at (see _idmap.get_recommendation_for_cold_seeds), so a typo in + # that entry is not a degradation and must not be counted. This + # mirrors that loop's own "cold AND carries features" criterion using + # the cold_seeds list already computed above for the case-C gate. + supplied = body.item_features or {} + if any( + _has_undeclared_columns( + entry.recommender.item_feature_state, supplied[str(seed)] + ) + for seed in cold_seeds + if str(seed) in supplied + ): + _metrics.inc_feature_unknown_column(name, "item") + return raw_results + + if not any(str(s) in id_map for s in body.seed_items): + raise _NoUsableSeeds(list(body.seed_items)) + + if body.user_features is not None: + # Case B: the same solve the pre-existing path runs, plus the + # profile prior. + try: + raw_results, unknown_columns = ( + entry.recommender.get_recommendation_for_new_user( + body.seed_items, + cutoff=body.limit, + user_features=body.user_features, + ) + ) + except ColdStartNumericalError as exc: + raise _ColdStartValueUnusable(str(exc)) from None + except ValueError as exc: + raise _ColdStartUnsupported(str(exc)) from None + _metrics.inc_cold_start_request(name, "features_and_history") + for column in unknown_columns: + _metrics.inc_feature_unknown_value(name, "user", column) + if _has_undeclared_columns( + entry.recommender.user_feature_state, body.user_features + ): + _metrics.inc_feature_unknown_column(name, "user") + return raw_results + + # All seeds known, no user_features: byte-for-byte the pre-existing + # path -- unchanged so existing clients see no behavior change. + try: + raw_results = entry.recommender.get_recommendation_for_new_user( + body.seed_items, body.limit + ) + except KeyError: + # Unexpected KeyError despite seed appearing known. + logger.exception( + "recommender_unexpected_key_error", + recipe=name, + verb=verb, + seed_items_count=len(body.seed_items), + ) + raise + + if not raw_results: + raise _NoCandidates() + return raw_results + + def make_router( registry: ModelRegistry, api_keys: list[ApiKeyEntry], @@ -218,32 +549,6 @@ def _apply_build_items_degraded( ) return items - def _any_seed_known( - entry: ModelEntry, seed_items: list[str], name: str - ) -> bool | None: - """Return True if at least one seed is known to the model id-map. - - Returns None when the recommender layout is unexpected (caller must - treat this as INTERNAL_ERROR rather than UNKNOWN_SEED_ITEMS). - - Used to distinguish ``UNKNOWN_SEED_ITEMS`` (no seed in id-map) from - ``NO_CANDIDATES`` (some seeds known but the ranker produced no - survivors after its own filtering / score-thresholding). - """ - try: - mapper = entry.recommender._mapper - id_map = mapper.item_id_to_index - except AttributeError as exc: - # Unexpected recommender layout — log and signal to caller. - logger.warning( - "recommender_layout_unexpected", - recipe=name, - exc_type=type(exc).__name__, - ) - _metrics.inc_recommender_layout_unexpected(name) - return None - return any(str(s) in id_map for s in seed_items) - @router.get("/health", summary="Overall health status (probe-safe)") def health(response: Response) -> dict[str, Any]: # Intentional design difference vs /health/details: this probe endpoint @@ -317,53 +622,43 @@ def recommend( try: entry = _resolve_entry(name, request_id, kid, status_holder) - # S1: determine known-membership BEFORE calling irspack so a - # genuine missing user produces UNKNOWN_USER, not INTERNAL_ERROR. - # Returns None when the recommender layout is unexpected (F4). - try: - user_known: bool | None = ( - body.user_id in entry.recommender._mapper.user_id_to_index - ) - except AttributeError as _attr_exc: - # Unexpected recommender layout — mirror _any_seed_known sentinel. - logger.warning( - "recommender_layout_unexpected", - recipe=name, - verb=verb, - exc_type=type(_attr_exc).__name__, - ) - _metrics.inc_recommender_layout_unexpected(name) - user_known = ( - None # let irspack decide; None → INTERNAL_ERROR on KeyError - ) - try: - raw_results: list[tuple[str, float]] = ( - entry.recommender.get_recommendation_for_known_user_id( - body.user_id, body.limit - ) - ) + raw_results = _resolve_recommend(entry, name, verb, body) + except _NoUsableUser: + status_holder[0] = "unknown_user" + raise HTTPException( + status_code=404, + detail={ + "detail": "user not seen during training", + "code": "UNKNOWN_USER", + }, + ) from None + except _ColdStartUnsupported as exc: + status_holder[0] = "features_not_supported" + raise HTTPException( + status_code=400, + detail={ + "detail": str(exc), + "code": "FEATURES_NOT_SUPPORTED", + }, + ) from None + except _ColdStartValueUnusable as exc: + status_holder[0] = "feature_value_unusable" + raise HTTPException( + status_code=400, + detail={ + "detail": ( + "one or more supplied feature values " + "produced a standardized value that is " + "numerically unusable for this model's " + f"cold-start scoring: {exc}" + ), + "code": "FEATURE_VALUE_UNUSABLE", + }, + ) from None except KeyError: - if user_known is False: - # Deterministic miss: user was not in the id-map. - status_holder[0] = "unknown_user" - raise HTTPException( - status_code=404, - detail={ - "detail": "user not seen during training", - "code": "UNKNOWN_USER", - }, - ) from None - # user_known is True or None (unexpected layout): propagate as - # INTERNAL_ERROR so layout surprises are visible, not silent. - logger.exception( - "recommender_unexpected_key_error", - recipe=name, - verb=verb, - user_id_hash=hashlib.sha256(body.user_id.encode()).hexdigest()[ - :8 - ], - ) + # Unexpected recommender layout (already logged inside + # _resolve_recommend). raise HTTPException( status_code=500, detail={ @@ -418,18 +713,9 @@ def recommend_related( try: entry = _resolve_entry(name, request_id, kid, status_holder) - seed_known = _any_seed_known(entry, body.seed_items, name) - if seed_known is None: - # M1: unexpected recommender layout — propagate as INTERNAL_ERROR. - status_holder[0] = "error" - raise HTTPException( - status_code=500, - detail={ - "detail": "internal error", - "code": "INTERNAL_ERROR", - }, - ) - if not seed_known: + try: + raw_results = _resolve_recommend_related(entry, name, verb, body) + except _NoUsableSeeds: status_holder[0] = "unknown_seed_items" raise HTTPException( status_code=404, @@ -437,29 +723,31 @@ def recommend_related( "detail": "no known seed_items", "code": "UNKNOWN_SEED_ITEMS", }, - ) - - try: - raw_results = entry.recommender.get_recommendation_for_new_user( - body.seed_items, body.limit - ) - except KeyError: - # S1: unexpected KeyError despite seed appearing known. - logger.exception( - "recommender_unexpected_key_error", - recipe=name, - verb=verb, - seed_items_count=len(body.seed_items), - ) + ) from None + except _ColdStartUnsupported as exc: + status_holder[0] = "features_not_supported" raise HTTPException( - status_code=500, + status_code=400, detail={ - "detail": "internal error", - "code": "INTERNAL_ERROR", + "detail": str(exc), + "code": "FEATURES_NOT_SUPPORTED", }, ) from None - - if not raw_results: + except _ColdStartValueUnusable as exc: + status_holder[0] = "feature_value_unusable" + raise HTTPException( + status_code=400, + detail={ + "detail": ( + "one or more supplied feature values " + "produced a standardized value that is " + "numerically unusable for this model's " + f"cold-start scoring: {exc}" + ), + "code": "FEATURE_VALUE_UNUSABLE", + }, + ) from None + except _NoCandidates: status_holder[0] = "no_candidates" raise HTTPException( status_code=404, @@ -467,7 +755,18 @@ def recommend_related( "detail": "no candidates produced by ranker", "code": "NO_CANDIDATES", }, - ) + ) from None + except (AttributeError, KeyError): + # Unexpected recommender layout (already logged inside + # _resolve_recommend_related). + status_holder[0] = "error" + raise HTTPException( + status_code=500, + detail={ + "detail": "internal error", + "code": "INTERNAL_ERROR", + }, + ) from None exclude = ( frozenset(body.exclude_items) if body.exclude_items else frozenset() @@ -520,107 +819,130 @@ def batch_recommend( results: list[BatchResultOk | BatchResultErr] = [] aggregate_limit = 0 for idx, raw in enumerate(body.requests): - if not isinstance(raw, dict): - results.append( - _batch_error_entry( - idx, "VALIDATION_ERROR", "request must be an object" - ) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - try: - single = RecommendRequest.model_validate(raw) - except ValidationError as exc: - _msg = _format_batch_validation_message(exc) - logger.warning( - "batch_element_validation_failed", - recipe=name, - verb=verb, - idx=idx, - errors=_sanitize_validation_errors(exc), - ) - results.append( - _batch_error_entry(idx, "VALIDATION_ERROR", _msg) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - if aggregate_limit + single.limit > BATCH_AGGREGATE_LIMIT: - results.append( - _batch_error_entry( - idx, - "VALIDATION_ERROR", - f"aggregate limit cap exceeded: " - f"{BATCH_AGGREGATE_LIMIT}", - ) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - aggregate_limit += single.limit - # F5: initialize user_known at top of each iteration so - # stale values from a previous iteration cannot leak on - # future refactors. - batch_user_known: bool | None = True - try: - # S1/F4: check membership before calling irspack. - # Returns None when the recommender layout is unexpected. - try: - batch_user_known = ( - single.user_id - in entry.recommender._mapper.user_id_to_index + with _bind_batch_idx(idx): + if not isinstance(raw, dict): + results.append( + _batch_error_entry( + idx, + "VALIDATION_ERROR", + "request must be an object", + ) ) - except AttributeError as _attr_exc: - # Mirror _any_seed_known sentinel: log + metric + None. + _metrics.inc_batch_element_error( + name, verb, "VALIDATION_ERROR" + ) + continue + try: + single = RecommendRequest.model_validate(raw) + except ValidationError as exc: + _msg = _format_batch_validation_message(exc) logger.warning( - "recommender_layout_unexpected", + "batch_element_validation_failed", recipe=name, verb=verb, - exc_type=type(_attr_exc).__name__, - ) - _metrics.inc_recommender_layout_unexpected(name) - batch_user_known = None - - raw_results = ( - entry.recommender.get_recommendation_for_known_user_id( - single.user_id, single.limit - ) - ) - exclude = ( - frozenset(single.exclude_items) - if single.exclude_items - else frozenset() - ) - meta = entry.metadata_index if body.include_metadata else None - items, _fb, _dr = _build_items( - raw_results, exclude, meta, name, verb - ) - if _fb + _dr > 0: - if _fb: - _metrics.inc_metadata_degraded_items( - name, verb, "fallback", _fb - ) - if _dr: - _metrics.inc_metadata_degraded_items( - name, verb, "dropped", _dr + idx=idx, + errors=_sanitize_validation_errors(exc), + ) + results.append( + _batch_error_entry(idx, "VALIDATION_ERROR", _msg) + ) + _metrics.inc_batch_element_error( + name, verb, "VALIDATION_ERROR" + ) + continue + if aggregate_limit + single.limit > BATCH_AGGREGATE_LIMIT: + results.append( + _batch_error_entry( + idx, + "VALIDATION_ERROR", + f"aggregate limit cap exceeded: " + f"{BATCH_AGGREGATE_LIMIT}", ) - results.append( - BatchResultOk(index=idx, status="ok", items=items) - ) - except KeyError: - if batch_user_known is False: + ) + _metrics.inc_batch_element_error( + name, verb, "VALIDATION_ERROR" + ) + continue + aggregate_limit += single.limit + try: + raw_results = _resolve_recommend(entry, name, verb, single) + exclude = ( + frozenset(single.exclude_items) + if single.exclude_items + else frozenset() + ) + meta = ( + entry.metadata_index if body.include_metadata else None + ) + items, _fb, _dr = _build_items( + raw_results, exclude, meta, name, verb + ) + if _fb + _dr > 0: + if _fb: + _metrics.inc_metadata_degraded_items( + name, verb, "fallback", _fb + ) + if _dr: + _metrics.inc_metadata_degraded_items( + name, verb, "dropped", _dr + ) + results.append( + BatchResultOk(index=idx, status="ok", items=items) + ) + except _NoUsableUser: results.append( _batch_error_entry( - idx, "UNKNOWN_USER", "user not seen during training" + idx, + "UNKNOWN_USER", + "user not seen during training", ) ) _metrics.inc_batch_element_error(name, verb, "UNKNOWN_USER") - else: - # batch_user_known is True or None (unexpected layout): - # propagate as INTERNAL_ERROR for observability. + except _ColdStartUnsupported as exc: + results.append( + _batch_error_entry( + idx, "FEATURES_NOT_SUPPORTED", str(exc) + ) + ) + _metrics.inc_batch_element_error( + name, verb, "FEATURES_NOT_SUPPORTED" + ) + except _ColdStartValueUnusable as exc: + results.append( + _batch_error_entry( + idx, + "FEATURE_VALUE_UNUSABLE", + "one or more supplied feature values " + "produced a standardized value that is " + "numerically unusable for this model's " + f"cold-start scoring: {exc}", + ) + ) + _metrics.inc_batch_element_error( + name, verb, "FEATURE_VALUE_UNUSABLE" + ) + except KeyError: + # Unexpected recommender layout (already logged + # inside _resolve_recommend): propagate as + # INTERNAL_ERROR for observability. + results.append( + _batch_error_entry( + idx, "INTERNAL_ERROR", "internal error" + ) + ) + _metrics.inc_batch_element_error( + name, verb, "INTERNAL_ERROR" + ) + except (MemoryError, RecursionError): + raise + except Exception as exc: logger.exception( - "recommender_unexpected_key_error", + "batch_element_error", recipe=name, verb=verb, idx=idx, + exc_type=type(exc).__name__, + exc_module=type(exc).__module__, ) results.append( _batch_error_entry( @@ -630,21 +952,6 @@ def batch_recommend( _metrics.inc_batch_element_error( name, verb, "INTERNAL_ERROR" ) - except (MemoryError, RecursionError): - raise - except Exception as exc: - logger.exception( - "batch_element_error", - recipe=name, - verb=verb, - idx=idx, - exc_type=type(exc).__name__, - exc_module=type(exc).__module__, - ) - results.append( - _batch_error_entry(idx, "INTERNAL_ERROR", "internal error") - ) - _metrics.inc_batch_element_error(name, verb, "INTERNAL_ERROR") status_holder[0] = "ok" response.headers["X-Recotem-Model-Version"] = entry.model_version @@ -684,92 +991,131 @@ def batch_recommend_related( results: list[BatchResultOk | BatchResultErr] = [] aggregate_limit = 0 + cold_seed_solves = 0 for idx, raw in enumerate(body.requests): - if not isinstance(raw, dict): - results.append( - _batch_error_entry( - idx, "VALIDATION_ERROR", "request must be an object" - ) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - try: - single = RecommendRelatedRequest.model_validate(raw) - except ValidationError as exc: - _msg = _format_batch_validation_message(exc) - logger.warning( - "batch_element_validation_failed", - recipe=name, - verb=verb, - idx=idx, - errors=_sanitize_validation_errors(exc), - ) - results.append( - _batch_error_entry(idx, "VALIDATION_ERROR", _msg) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - if aggregate_limit + single.limit > BATCH_AGGREGATE_LIMIT: - results.append( - _batch_error_entry( - idx, - "VALIDATION_ERROR", - f"aggregate limit cap exceeded: " - f"{BATCH_AGGREGATE_LIMIT}", - ) - ) - _metrics.inc_batch_element_error(name, verb, "VALIDATION_ERROR") - continue - aggregate_limit += single.limit - try: - seed_known = _any_seed_known(entry, single.seed_items, name) - if seed_known is None: - # M1: unexpected layout — INTERNAL_ERROR for this element. + with _bind_batch_idx(idx): + if not isinstance(raw, dict): results.append( _batch_error_entry( - idx, "INTERNAL_ERROR", "internal error" + idx, + "VALIDATION_ERROR", + "request must be an object", ) ) _metrics.inc_batch_element_error( - name, verb, "INTERNAL_ERROR" + name, verb, "VALIDATION_ERROR" + ) + continue + try: + single = RecommendRelatedRequest.model_validate(raw) + except ValidationError as exc: + _msg = _format_batch_validation_message(exc) + logger.warning( + "batch_element_validation_failed", + recipe=name, + verb=verb, + idx=idx, + errors=_sanitize_validation_errors(exc), + ) + results.append( + _batch_error_entry(idx, "VALIDATION_ERROR", _msg) + ) + _metrics.inc_batch_element_error( + name, verb, "VALIDATION_ERROR" ) continue - if not seed_known: + if aggregate_limit + single.limit > BATCH_AGGREGATE_LIMIT: results.append( _batch_error_entry( idx, - "UNKNOWN_SEED_ITEMS", - "no known seed_items", + "VALIDATION_ERROR", + f"aggregate limit cap exceeded: " + f"{BATCH_AGGREGATE_LIMIT}", ) ) _metrics.inc_batch_element_error( - name, verb, "UNKNOWN_SEED_ITEMS" + name, verb, "VALIDATION_ERROR" ) continue + # Checked BEFORE either budget is consumed, so a + # rejected element costs neither -- matching the + # aggregate-limit branch above. + _solves = _cold_seed_solve_bound(single) + if cold_seed_solves + _solves > BATCH_COLD_SEED_SOLVE_LIMIT: + results.append( + _batch_error_entry( + idx, + "VALIDATION_ERROR", + f"aggregate cold-seed cap exceeded: " + f"{BATCH_COLD_SEED_SOLVE_LIMIT}", + ) + ) + _metrics.inc_batch_element_error( + name, verb, "VALIDATION_ERROR" + ) + continue + aggregate_limit += single.limit + cold_seed_solves += _solves try: - raw_results = ( - entry.recommender.get_recommendation_for_new_user( - single.seed_items, single.limit + raw_results = _resolve_recommend_related( + entry, name, verb, single + ) + exclude = ( + frozenset(single.exclude_items) + if single.exclude_items + else frozenset() + ) + meta = ( + entry.metadata_index if body.include_metadata else None + ) + items, _fb, _dr = _build_items( + raw_results, exclude, meta, name, verb + ) + if _fb + _dr > 0: + if _fb: + _metrics.inc_metadata_degraded_items( + name, verb, "fallback", _fb + ) + if _dr: + _metrics.inc_metadata_degraded_items( + name, verb, "dropped", _dr + ) + results.append( + BatchResultOk(index=idx, status="ok", items=items) + ) + except _NoUsableSeeds: + results.append( + _batch_error_entry( + idx, "UNKNOWN_SEED_ITEMS", "no known seed_items" ) ) - except KeyError: - # S1: unexpected KeyError despite seed appearing known. - logger.exception( - "recommender_unexpected_key_error", - recipe=name, - verb=verb, - idx=idx, + _metrics.inc_batch_element_error( + name, verb, "UNKNOWN_SEED_ITEMS" ) + except _ColdStartUnsupported as exc: results.append( _batch_error_entry( - idx, "INTERNAL_ERROR", "internal error" + idx, "FEATURES_NOT_SUPPORTED", str(exc) ) ) _metrics.inc_batch_element_error( - name, verb, "INTERNAL_ERROR" + name, verb, "FEATURES_NOT_SUPPORTED" ) - continue - if not raw_results: + except _ColdStartValueUnusable as exc: + results.append( + _batch_error_entry( + idx, + "FEATURE_VALUE_UNUSABLE", + "one or more supplied feature values " + "produced a standardized value that is " + "numerically unusable for this model's " + f"cold-start scoring: {exc}", + ) + ) + _metrics.inc_batch_element_error( + name, verb, "FEATURE_VALUE_UNUSABLE" + ) + except _NoCandidates: results.append( _batch_error_entry( idx, @@ -780,43 +1126,37 @@ def batch_recommend_related( _metrics.inc_batch_element_error( name, verb, "NO_CANDIDATES" ) - continue - exclude = ( - frozenset(single.exclude_items) - if single.exclude_items - else frozenset() - ) - meta = entry.metadata_index if body.include_metadata else None - items, _fb, _dr = _build_items( - raw_results, exclude, meta, name, verb - ) - if _fb + _dr > 0: - if _fb: - _metrics.inc_metadata_degraded_items( - name, verb, "fallback", _fb + except (AttributeError, KeyError): + # Unexpected recommender layout (already logged + # inside _resolve_recommend_related): propagate as + # INTERNAL_ERROR for observability. + results.append( + _batch_error_entry( + idx, "INTERNAL_ERROR", "internal error" ) - if _dr: - _metrics.inc_metadata_degraded_items( - name, verb, "dropped", _dr + ) + _metrics.inc_batch_element_error( + name, verb, "INTERNAL_ERROR" + ) + except (MemoryError, RecursionError): + raise + except Exception as exc: + logger.exception( + "batch_element_error", + recipe=name, + verb=verb, + idx=idx, + exc_type=type(exc).__name__, + exc_module=type(exc).__module__, + ) + results.append( + _batch_error_entry( + idx, "INTERNAL_ERROR", "internal error" ) - results.append( - BatchResultOk(index=idx, status="ok", items=items) - ) - except (MemoryError, RecursionError): - raise - except Exception as exc: - logger.exception( - "batch_element_error", - recipe=name, - verb=verb, - idx=idx, - exc_type=type(exc).__name__, - exc_module=type(exc).__module__, - ) - results.append( - _batch_error_entry(idx, "INTERNAL_ERROR", "internal error") - ) - _metrics.inc_batch_element_error(name, verb, "INTERNAL_ERROR") + ) + _metrics.inc_batch_element_error( + name, verb, "INTERNAL_ERROR" + ) status_holder[0] = "ok" response.headers["X-Recotem-Model-Version"] = entry.model_version diff --git a/src/recotem/serving/schemas.py b/src/recotem/serving/schemas.py index 2c29d5c1..765e905a 100644 --- a/src/recotem/serving/schemas.py +++ b/src/recotem/serving/schemas.py @@ -5,13 +5,38 @@ from typing import Annotated, Any, Literal -from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field # Aggregate ``limit`` cap across all sub-requests in a single batch call. # Documented in docs/api-reference.md. Bounds total candidate work per HTTP # request so a 256-element batch cannot demand 256_000 items in one go. BATCH_AGGREGATE_LIMIT = 5000 +# Aggregate cold-seed cap across all sub-requests in a single +# ``:batch-recommend-related`` call. Documented in docs/api-reference.md and +# docs/operations.md. +# +# Why a SECOND cap rather than reusing BATCH_AGGREGATE_LIMIT: that one caps +# ``sum(limit)`` -- response volume -- which is a different dimension. Case C +# (a cold seed carrying ``item_features``) runs one irspack conjugate-gradient +# solve PER COLD SEED, so ``limit: 1`` x 256 elements x 100 cold seeds keeps +# the aggregate limit at 256 (2% of its cap) while demanding 25_600 solves. +# Every other path costs one solve per element at most. +# +# Why 512: measured ~0.25-0.45 ms/solve, so 512 bounds the worst case at +# ~230 ms of single-threaded CPU per HTTP request -- material on a +# single-process uvicorn but not an outage. The measurement is near-flat in +# ``n_components`` (0.27 ms at 8, 0.30 ms at 128, 0.45 ms at 256) and in the +# encoded feature dimension: the solve is call-overhead-dominated, not +# Cholesky-dominated, at every size a recipe can produce. So this bound does +# NOT need to shrink for a production-sized model. +# +# 512 also leaves the whole existing request space intact: a single +# ``:recommend-related`` tops out at 100 solves (``seed_items`` max_length), +# so the cap only ever binds on batch fan-out, and even then admits five +# maximal elements. +BATCH_COLD_SEED_SOLVE_LIMIT = 512 + # Machine-readable error codes emitted by the v1 API. Kept as a Literal # union so OpenAPI / SDK generation produces an exhaustive enum and any # new code added in routes/auth/app fails type-check until listed here. @@ -25,6 +50,8 @@ "MISSING_API_KEY", "INVALID_API_KEY", "INTERNAL_ERROR", + "FEATURES_NOT_SUPPORTED", + "FEATURE_VALUE_UNUSABLE", ] # --------------------------------------------------------------------------- @@ -33,6 +60,59 @@ _ItemStr = Annotated[str, Field(min_length=1, max_length=256)] +# Per-string-value length cap for cold-start feature values. `Field(max_length=64)` +# on `_FeatureValues` caps only the KEY COUNT; without this a single string +# VALUE was unbounded -- the one request field with no length cap (user_id / +# _ItemStr are 256, seed_items 100, exclude_items 1000). `_tokens` does +# `str(raw).split(delimiter)` unbounded, so a large `multi_label` value +# amplifies (~8x) into a memory-DoS reachable with one API key and multiplied +# by batch/related fan-out. 8192 is generous for a real multi_label token list +# yet blocks MB-scale amplification, restoring parity with every other field. +_MAX_FEATURE_VALUE_CHARS = 8192 + + +def _check_feature_value_lengths( + values: dict[str, Any] | None, +) -> dict[str, Any] | None: + """Reject any string feature value longer than ``_MAX_FEATURE_VALUE_CHARS``. + + Shared by every place a cold-start feature mapping appears: ``user_features`` + on both request models and each nested ``item_features`` mapping (this + validator runs per ``_FeatureValues``, so a nested dict of values is checked + too). Names the offending column key but never echoes the value, which is + treated as personal data. Non-string scalars are unaffected. + """ + if values is None: + return values + for key, val in values.items(): + if isinstance(val, str) and len(val) > _MAX_FEATURE_VALUE_CHARS: + raise ValueError( + f"feature value for column {key!r} exceeds the " + f"{_MAX_FEATURE_VALUE_CHARS}-character limit" + ) + return values + + +# Raw feature values for cold start, shared by the ``user_features`` field on +# both single-request models and the ``item_features`` values on +# ``RecommendRelatedRequest``. Values are encoded server-side against the +# model's training vocabulary (``recotem._features.encode_one``) and are +# treated as personal data: never logged (see ``recotem.log_redaction`` for +# the key-based backstop, which this relies on as defence in depth only). +_FeatureValues = Annotated[ + dict[str, Any], + Field( + max_length=64, + description=( + "Raw feature values for cold start, keyed by the recipe's feature " + "column names. Encoded server-side with the model's training " + "vocabulary. Values are treated as personal data and are never " + "logged." + ), + ), + AfterValidator(_check_feature_value_lengths), +] + class RecommendRequest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -47,6 +127,14 @@ class RecommendRequest(BaseModel): list[_ItemStr] | None, Field(max_length=1000, description="Item IDs to exclude from results"), ] = None + # Cold-start profile (case A -- unknown user, features only). Ignored, + # not rejected, for a KNOWN user_id: the learned embedding was fit to + # their real interactions and strictly dominates a profile prior, so a + # client that always sends the profile keeps working either way. See + # ``routes.py``'s ``recommend`` handler and + # ``docs/api-reference.md#feature-aware-cold-start`` ("A known `user_id` + # with `user_features` supplied is not an error."). + user_features: _FeatureValues | None = None class RecommendRelatedRequest(BaseModel): @@ -67,6 +155,22 @@ class RecommendRelatedRequest(BaseModel): list[_ItemStr] | None, Field(max_length=1000, description="Item IDs to exclude from results"), ] = None + # Case B -- profile prior added to the ad-hoc seed-history solve. + user_features: _FeatureValues | None = None + # Case C -- feature values for seed items absent from training, keyed by + # seed item id. Takes precedence over ``user_features`` when a seed named + # here is also cold: a cold seed has no row in the seed interaction + # matrix, so the case-B solve would silently drop it. + item_features: Annotated[ + dict[str, _FeatureValues] | None, + Field( + max_length=100, + description=( + "Raw feature values for seed items absent from training, keyed " + "by seed item id." + ), + ), + ] = None # --------------------------------------------------------------------------- diff --git a/src/recotem/serving/watcher.py b/src/recotem/serving/watcher.py index 3cca207e..cd283794 100644 --- a/src/recotem/serving/watcher.py +++ b/src/recotem/serving/watcher.py @@ -37,6 +37,10 @@ import fsspec import structlog +from recotem._features import ( + FEATURE_VERSION_MSG_PREFIX, + check_artifact_feature_version, +) from recotem._irspack_compat import ( SKEW_MSG_PREFIX, check_artifact_irspack_version, @@ -1041,6 +1045,12 @@ def _build_entry( # the recipe nor the remedy. check_artifact_irspack_version(header_dict, name=name) + # Preflight the feature-encoder state version for the same reason: an + # unknown shape would otherwise be silently mis-encoded rather than + # refused, and the failure would surface as wrong recommendations, + # not an exception. + check_artifact_feature_version(header_dict, name=name) + recommender = unpickle_payload(payload_bytes) metadata_df = None @@ -1148,6 +1158,11 @@ def _classify_artifact_error(err_msg: str) -> str: # cannot silently drop the message through to that catch-all. if lower.startswith(SKEW_MSG_PREFIX.lower()): return "version_skew" + # Same reasoning as the skew branch immediately above: the feature-version + # gate's message also contains "version", so it must be classified before + # the catch-all below or it silently collapses into "parse". + if lower.startswith(FEATURE_VERSION_MSG_PREFIX.lower()): + return "feature_version" if lower.startswith("deserialization failed:"): return "deserialize" if lower.startswith("metadata load failed:"): diff --git a/src/recotem/training/algorithms.py b/src/recotem/training/algorithms.py index 38b7b143..ed468ff6 100644 --- a/src/recotem/training/algorithms.py +++ b/src/recotem/training/algorithms.py @@ -54,6 +54,27 @@ } ) +# Algorithms that accept `user_features` / `item_features` constructor kwargs. +# As of irspack 0.5.0 feature-aware iALS is not a distinct class -- it is +# IALSRecommender with extra kwargs -- so this set holds exactly one entry. +# Kept explicit (not derived) for the same reason as SUPPORTED_CLASS_NAMES: +# a stable contract across irspack patch releases. +FEATURE_CAPABLE_CLASS_NAMES: frozenset[str] = frozenset({"IALSRecommender"}) + + +def is_feature_capable(alias: str) -> bool: + """Return True when *alias* resolves to a feature-aware-capable class. + + Unknown aliases return False rather than raising: ``training.algorithms`` + has no load-time validation (recipe/models.py deliberately swallows + ``UnknownAlgorithmError``), and this helper must not change that. + """ + try: + class_name = resolve_algorithm_name(alias) + except UnknownAlgorithmError: + return False + return class_name in FEATURE_CAPABLE_CLASS_NAMES + # --------------------------------------------------------------------------- # Public helpers diff --git a/src/recotem/training/features.py b/src/recotem/training/features.py new file mode 100644 index 00000000..694033c1 --- /dev/null +++ b/src/recotem/training/features.py @@ -0,0 +1,464 @@ +"""Fetch feature tables and encode them onto a given axis. + +Training is the only caller. The pure encoding lives in ``recotem._features`` +so serving can reuse it without importing this package. + +Feature tables are fetched through the datasource registry, exactly like the +interaction source. That is sound because ``FetchContext`` carries no +interaction semantics -- it holds only ``recipe_name``, ``run_id``, and an +``extra`` dict -- and BigQuery/SQL sources return the fetched DataFrame +unmodified. + +Do NOT put ``user_column`` / ``item_column`` into ``FetchContext.extra``: that +would wake ``datasource/csv.py``'s ``_validate_required_columns``, which is +currently inert because the production call site (``pipeline.py``) passes no +``extra``, and it would reject a feature table for lacking interaction +columns. + +Error-mapping decision +----------------------- +``build_encoder_state`` (``recotem._features``) raises ``FeatureEncodeError`` +for a missing feature column or a dimension over ``RECOTEM_MAX_FEATURE_DIM``. +That exception is deliberately NOT a ``TrainingError`` subclass -- it is +shared with ``recotem.serving``, which must never import +``recotem.training.errors`` (training/serving isolation). Left unhandled it +would surface as an unmapped exception (exit 1) instead of the documented +training-domain exit code (4), so ``_fetch_side`` catches it here, at the +training/neutral boundary, and re-raises ``TrainingError(code= +"feature_table_error")``. This gives every feature-table failure -- a bad +``id_column``, an unresolvable ``columns`` entry, or a dimension-cap breach +-- the same exit code as every other training-domain error (``SplitError``, +``SearchError``, ...), which is what operators and the CLI exit-code table +expect. + +``DataSourceError`` (unknown source type, CSV parse failure, BigQuery access +failure, ...) is deliberately NOT wrapped: it propagates unchanged, exactly +like the main interaction source's fetch in ``pipeline.py:_fetch_data``, so +it keeps mapping to exit 3. + +Why id coverage is checked at ENCODE time, not fetch time +---------------------------------------------------------- +A feature table whose ids do not match the interaction axis is the one +feature-aware failure that is otherwise SILENT: every entity encodes to the +bias column alone, training completes, and the signed artifact's header +advertises ``features`` for what is really plain iALS. Two independent causes +converge on that same outcome -- an id dtype/format mismatch (a single blank +cell makes pandas infer ``float64``, so ``1`` reads back as ``1.0`` while the +interaction axis carries ``"1"``), and a wrong-but-existing ``id_column``, +which sails through ``_fetch_side``'s presence check. + +``_check_axis_coverage`` catches both, because it keys off the observable +they share rather than the causes they do not: zero id overlap with the axis +being encoded. It necessarily runs in ``encode_for_axis`` and not in +``_fetch_side`` -- ``load_feature_tables`` runs BEFORE the split +(``pipeline.py`` step 2.5 vs. step 4), so the interaction axis does not exist +yet at fetch time. ``encode_for_axis`` is the first point where both sides +are known. + +Coercing the id column defensively at fetch time was considered and rejected +as the primary fix. By the time ``_fetch_side`` sees the frame, ``pd.read_csv`` +has already inferred ``float64``, and the original text is unrecoverable: a +column reading ``1.0`` is indistinguishable from one whose ids are literally +``"1.0"``, so "reformat integral floats as ints" would silently REWRITE ids on +a catalog that legitimately uses that form -- trading a detectable failure for +a quiet corruption. It also would not catch the wrong-``id_column`` case at +all. Reading the id column as a string at the SOURCE is the real remedy, and +it is what the error message points the operator to -- but the mechanism is +source-specific (``dtype: {id: str}`` exists only on ``csv``; ``bigquery`` / +``sql`` need a ``CAST(... AS STRING)`` in the query, and ``parquet`` a schema +fix), so the message links the per-source matrix in ``docs/operations.md`` +rather than naming a key that a non-``csv`` source does not have. The check is +what makes the need for it visible instead of silent. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import pandas as pd +import scipy.sparse as sps +import structlog + +from recotem._features import FeatureEncodeError, build_encoder_state, encode +from recotem.datasource.base import FetchContext +from recotem.datasource.registry import get_source_class +from recotem.recipe.models import FeaturesConfig, FeatureSideConfig +from recotem.training.errors import TrainingError + +logger = structlog.get_logger(__name__) + +# Ids sampled into the zero-overlap error message. Three per side is enough to +# make a systematic format difference ("1.0" vs "1") self-evident. +# +# This is a deliberate, bounded disclosure of ids that ARE otherwise sensitive +# -- and they do NOT stay in the operator's terminal. The message becomes the +# raised TrainingError, which pipeline.py surfaces as ``error=str(exc)`` inside +# the ``train_error`` log event, so these ids travel to whatever sink ingests +# that event (Cloud Logging / DataDog / CI logs), verbatim. ``log_redaction`` is +# keyed to credential/key SHAPES (hex64, cloud creds) and does not touch ids, so +# it does not scrub them. The disclosure is judged worth it -- three sampled ids +# are the only thing that makes a "1.0" vs "1" mismatch diagnosable, and the +# operator already has read access to both tables -- but it is a real export, so +# BOTH the count and each id's LENGTH are bounded: an unbounded id (a stray +# multi-MB cell) would otherwise inflate the event, and the count bound alone +# does not stop that. +_ID_SAMPLE_SIZE = 3 +_ID_SAMPLE_MAX_CHARS = 64 + + +@dataclass(frozen=True) +class FeatureTables: + """Fetched feature frames plus their phase-independent encoder states.""" + + item_state: dict | None = None + item_df: pd.DataFrame | None = None + user_state: dict | None = None + user_df: pd.DataFrame | None = None + + @property + def enabled(self) -> bool: + return self.item_state is not None or self.user_state is not None + + +def _spec_is_live(spec: dict) -> bool: + """True if an encoder-state column spec can emit a non-bias feature. + + A ``numerical`` spec always reserves ``width == 1`` even when it is dead + (std floored to 0.0, emitting nothing at encode time), so its width is not + a reliable liveness signal -- its ``std`` is. A ``categorical`` / + ``multi_label`` spec, by contrast, prunes to ``width == 0`` when dead, so + its width is exactly right. The whole-block-dead guard in ``_fetch_side`` + keys on this rather than on ``n_features``. + """ + if spec["encoding"] == "numerical": + return spec["std"] != 0.0 + return spec["width"] > 0 + + +def _resolve_source(source_cfg: Any, *, which: str) -> tuple[type, Any]: + """Return ``(source_cls, config)`` for a ``FeatureSideConfig.source`` value. + + ``features..source`` arrives already-typed (a ``CSVConfig`` / + ``BigQueryConfig`` / ... instance) when the recipe went through + ``recipe.loader.load_recipe`` -- the production path, since + ``pipeline.py`` passes ``recipe.features`` and the loader performs the + typed construction for every feature subtree. Direct construction of + ``FeatureSideConfig`` -- as unit tests and library callers do -- leaves + ``source`` as a raw dict, because ``FeatureSideConfig.source`` is typed + ``Any``. Both shapes are handled here so this module does not depend on + every caller going through the loader. + """ + type_name = getattr(source_cfg, "type", None) + if type_name is None and isinstance(source_cfg, dict): + type_name = source_cfg.get("type") + if not type_name: + raise TrainingError( + f"features.{which}.source has no discriminator 'type' field.", + code="feature_table_error", + ) + + # Unknown type_name -> DataSourceError, propagated unchanged (exit 3). + source_cls = get_source_class(str(type_name)) + + if isinstance(source_cfg, dict): + try: + config = source_cls.Config.model_validate(source_cfg) + except (MemoryError, RecursionError): + raise + except Exception as exc: + raise TrainingError( + f"features.{which}.source failed validation: {exc}", + code="feature_table_error", + ) from exc + else: + config = source_cfg + + return source_cls, config + + +def _fetch_side( + side: FeatureSideConfig, + *, + which: str, + recipe_name: str, + run_id: str, +) -> tuple[dict, pd.DataFrame]: + """Fetch one side's feature table and build its phase-independent state. + + Raises + ------ + DataSourceError + Propagated unchanged from the datasource fetch (unknown source + type, CSV parse failure, BigQuery access failure, ...) -- exit 3, + same as the main interaction source. + TrainingError + For anything about the *shape* of the fetched table: a missing + ``id_column``, or ``build_encoder_state`` rejecting a ``columns`` + entry / a dimension-cap breach -- exit 4. + """ + source_cls, config = _resolve_source(side.source, which=which) + ctx = FetchContext(recipe_name=recipe_name, run_id=run_id) + df = source_cls(config).fetch(ctx) + + if side.id_column not in df.columns: + raise TrainingError( + f"features.{which}.id_column {side.id_column!r} is not present " + f"in the fetched feature table; available columns: " + f"{sorted(df.columns)}", + code="feature_table_error", + ) + + frame = df.copy() + + # Detect null/empty id BEFORE str-coercion (mirrors metadata/loader.py) + # so that an entity literally named the string "nan" is preserved as a + # real id rather than mistaken for a missing one. + null_mask = frame[side.id_column].isna() | ( + frame[side.id_column].astype(str).str.strip() == "" + ) + null_count = int(null_mask.sum()) + if null_count: + logger.warning( + "feature_table_null_ids_dropped", + side=which, + drop_count=null_count, + ) + frame = frame[~null_mask] + + # Coerce to plain Python str (numpy object dtype), matching pipeline.py's + # id-coercion convention -- avoids ArrowStringArray (pandas 2.x default). + frame[side.id_column] = frame[side.id_column].astype(str).astype(object) + duplicate_count = int(frame[side.id_column].duplicated(keep="first").sum()) + if duplicate_count: + logger.warning( + "feature_table_duplicate_ids_dropped", + side=which, + drop_count=duplicate_count, + ) + frame = frame.drop_duplicates(subset=[side.id_column], keep="first") + frame = frame.set_index(side.id_column) + + try: + state = build_encoder_state(frame, side.columns) + except FeatureEncodeError as exc: + raise TrainingError( + f"features.{which}: {exc}", + code="feature_table_error", + ) from exc + + # Whole-block-dead guard. The block is dead when NO declared column can + # emit a non-bias feature: every entity then encodes to bias alone -- + # byte-identical to plain iALS -- yet training would COMPLETE and sign an + # artifact whose header advertises ``features``. That is the same silent + # outcome ``_check_axis_coverage`` refuses at 0% id overlap, reached by a + # third route, so it is refused here with the same posture. + # ``build_encoder_state`` only WARNS per column (it is a neutral module + # shared with serving and must not raise a training error); this + # training-side check is where the whole-block refusal belongs. + # + # Liveness is per encoding, NOT ``n_features``: a categorical/multi_label + # spec that pruned to width 0 contributes nothing, but a NUMERICAL spec + # always reserves width 1 even when its std was floored to 0.0 (dead) and + # it emits nothing at encode time. Keying on ``n_features == 1`` therefore + # missed an all-dead-NUMERICAL block (n_features stays 2). A single dead + # column among several live ones is NOT refused -- pruning one column via + # ``min_frequency`` is a legitimate operator choice -- so this fires only + # when EVERY spec is dead. + if not any(_spec_is_live(s) for s in state["columns"]): + raise TrainingError( + f"features.{which}: every declared feature column encodes to " + f"nothing, so the whole block collapses to the bias column alone " + f"-- training would otherwise succeed and sign an artifact " + f"advertising features for what is really plain iALS. Usual " + f"causes: a min_frequency higher than any token's count (it prunes " + f"the entire vocabulary), a feature column with no usable values, " + f"or a numerical column with zero (or near-zero) variance. Lower " + f"min_frequency, fix the source column, or drop the empty " + f"features.{which} block.", + code="feature_table_error", + ) + + logger.info( + "feature_table_loaded", + side=which, + n_rows=int(frame.shape[0]), + n_features=state["n_features"], + # Column NAMES only. Feature values are user PII and must never be + # logged. + columns=[s["name"] for s in state["columns"]], + ) + return state, frame + + +def load_feature_tables( + features: FeaturesConfig | None, + *, + recipe_name: str, + run_id: str, +) -> FeatureTables: + """Fetch every configured feature table and build its encoder state. + + The state is built once here and is phase-independent; only the row + order differs between the search phase and the final refit, and that is + supplied per-call to :func:`encode_for_axis`. + """ + if features is None: + return FeatureTables() + + item_state = item_df = user_state = user_df = None + if features.item is not None: + item_state, item_df = _fetch_side( + features.item, which="item", recipe_name=recipe_name, run_id=run_id + ) + if features.user is not None: + user_state, user_df = _fetch_side( + features.user, which="user", recipe_name=recipe_name, run_id=run_id + ) + return FeatureTables( + item_state=item_state, + item_df=item_df, + user_state=user_state, + user_df=user_df, + ) + + +def _id_sample(values: Iterable[Any]) -> list[str]: + """The first few ids, str-normalized exactly as ``encode`` normalizes them. + + Bounds BOTH the count (``_ID_SAMPLE_SIZE``) and each id's length + (``_ID_SAMPLE_MAX_CHARS``): the sample lands in an exception message that + ships to the log stream (see ``_ID_SAMPLE_SIZE``'s comment), so a single + multi-MB id cell would otherwise bloat the event even though only three ids + are taken. Truncation keeps the diagnostic value -- a "1.0" vs "1" prefix + mismatch is still visible in the first characters -- while capping bytes. + """ + out: list[str] = [] + for v in values: + s = str(v) + if len(s) > _ID_SAMPLE_MAX_CHARS: + s = f"{s[:_ID_SAMPLE_MAX_CHARS]}...(+{len(s) - _ID_SAMPLE_MAX_CHARS} chars)" + out.append(s) + if len(out) >= _ID_SAMPLE_SIZE: + break + return out + + +def _check_axis_coverage( + df: pd.DataFrame, + index_order: Sequence[str], + *, + which: str, +) -> None: + """Log how much of *index_order* the feature table covers; refuse at zero. + + See the module docstring for why this lives here rather than in + ``_fetch_side``, and why it is preferred over coercing ids at fetch time. + + Only ZERO overlap is refused. A partially-covering feature table is + legitimate by design -- ``build_encoder_state`` builds its vocabulary from + the whole table precisely so cold-start entities are representable, and an + id absent from the table encodes to bias-only, degrading that one entity to + plain iALS. There is deliberately no low-coverage WARNING threshold: an id + dtype/format is a property of the whole column and an ``id_column`` is + either right or wrong, so every systematic mismatch lands at exactly 0%, + never at 5% or 20%. A nonzero-but-low coverage therefore carries no + evidence of a bug -- it only reflects how much of the catalog the operator's + table happens to cover -- so any threshold above zero would fire on correct + configurations and teach operators to ignore the warning. The INFO line + below carries matched/total for anyone who wants to alert on it themselves. + """ + # Normalize both sides exactly as ``encode`` does, or the coverage + # reported here would not be the coverage ``encode`` actually achieves. + order = {str(i) for i in index_order} + if not order: + # An empty axis has nothing to cover: 0 matched is not a mismatch, and + # 0/0 is not a ratio worth reporting. An interaction table with no + # items/users is a different failure, already caught upstream by the + # min_users / min_items preconditions. + return + + feature_ids = {str(i) for i in df.index} + matched = len(order & feature_ids) + + if matched == 0: + # _fetch_side set the index from side.id_column, so the index carries + # the configured name; fall back for a directly-constructed + # FeatureTables (library/test callers -- see _resolve_source). + id_column = df.index.name or f"features.{which}.id_column" + raise TrainingError( + f"features.{which}: none of the {len(order)} {which} ids in the " + f"interaction data were found in the feature table's " + f"{id_column!r} column, so every {which} would encode to the bias " + f"column alone -- training would otherwise succeed and sign an " + f"artifact advertising features for what is really plain iALS. " + f"feature-table ids look like {_id_sample(df.index)}; interaction " + f"ids look like {_id_sample(index_order)}. Usual causes: an id " + f"dtype mismatch (one blank cell makes pandas read an integer id " + f"column as float, turning 1 into 1.0), or a " + f"features.{which}.id_column naming the wrong column. The remedy is " + f"to ensure the id column is read as a string at the SOURCE; the " + f"exact mechanism is source-specific (csv, bigquery, sql, and " + f"parquet each differ) -- see docs/operations.md#recotem-train-" + f"exits-4-with-feature_axis_error.", + code="feature_axis_error", + ) + + logger.info( + "feature_axis_coverage", + side=which, + matched=matched, + total=len(order), + # Counts only -- ids are user PII, so this healthy-path event carries + # none, the same column-NAMES-only rule as feature_table_loaded above. + # The bounded id sample is confined to the FATAL zero-overlap path: it + # is a deliberate, bounded disclosure (count and per-id length both + # capped) because it is the only thing that makes a "1.0" vs "1" + # mismatch diagnosable. Be precise about where it goes, though -- it is + # NOT "shown once to the operator": that message becomes the raised + # TrainingError, which pipeline.py logs as error=str(exc) in the + # train_error event, so the sampled ids reach the same log sink as this + # event. That export is the reason both bounds exist; see _ID_SAMPLE_SIZE. + ) + + +def encode_for_axis( + tables: FeatureTables, + *, + item_order: Sequence[str] | None, + user_order: Sequence[str] | None, +) -> dict[str, sps.csr_matrix]: + """Encode the configured sides onto the supplied row orders. + + Call once per phase, and never cache or reuse the result across phases: + the search phase and the final refit have DIFFERENT item orderings + (``list(set(...))`` vs ``pd.Categorical``), and the search order is not + even stable across processes for string ids. irspack accepts a + misordered feature matrix silently, so a cached matrix would not be an + optimization -- it would be a silently wrong model. + """ + out: dict[str, sps.csr_matrix] = {} + if tables.item_state is not None: + if item_order is None: + raise TrainingError( + "internal: item features are configured but no item_order " + "was supplied to encode_for_axis", + code="feature_axis_error", + ) + _check_axis_coverage(tables.item_df, item_order, which="item") + out["item_features"] = encode( + tables.item_state, tables.item_df, index_order=item_order + ) + if tables.user_state is not None: + if user_order is None: + raise TrainingError( + "internal: user features are configured but no user_order " + "was supplied to encode_for_axis", + code="feature_axis_error", + ) + _check_axis_coverage(tables.user_df, user_order, which="user") + out["user_features"] = encode( + tables.user_state, tables.user_df, index_order=user_order + ) + return out diff --git a/src/recotem/training/pipeline.py b/src/recotem/training/pipeline.py index 4c0a162b..b20d37f6 100644 --- a/src/recotem/training/pipeline.py +++ b/src/recotem/training/pipeline.py @@ -28,17 +28,27 @@ from irspack.utils import df_to_sparse from recotem._exit_codes import _map_exception_to_exit # shared with cli.py +from recotem._features import FEATURE_STATE_VERSION, state_descriptor from recotem.recipe.errors import RecipeError from recotem.recipe.models import Recipe from recotem.training._compat import IDMappedRecommender -from recotem.training.algorithms import get_recommender_cls, resolve_algorithm_name +from recotem.training.algorithms import ( + get_recommender_cls, + is_feature_capable, + resolve_algorithm_name, +) from recotem.training.errors import ( MinDataViolation, TrainingError, ) from recotem.training.evaluate import build_evaluator +from recotem.training.features import ( + FeatureTables, + encode_for_axis, + load_feature_tables, +) from recotem.training.progress import ProgressReporter -from recotem.training.search import SearchResult, run_search +from recotem.training.search import SearchResult, _construct, run_search from recotem.training.split import split_interactions from recotem.version import __version__ as recotem_version @@ -349,6 +359,19 @@ def _run_training_locked( df: pd.DataFrame = _fetch_data(recipe, run_id=run_id) bound_logger.info("data_fetched", n_rows=len(df)) + # ------------------------------------------------------------------ + # 2.5. Fetch feature tables (feature-aware iALS), if configured. + # + # The fetch + encoder-state build is phase-independent and happens + # once here; ``feature_tables`` is then re-encoded onto each + # phase's OWN axis labels (search vs. final refit) further down, + # because those two phases use different, non-interchangeable + # item/user orderings (see encode_for_axis's docstring). + # ------------------------------------------------------------------ + feature_tables: FeatureTables = load_feature_tables( + recipe.features, recipe_name=recipe.name, run_id=run_id + ) + # ------------------------------------------------------------------ # 3. Cleanse. # ------------------------------------------------------------------ @@ -385,15 +408,32 @@ def _run_training_locked( # 4. Split. # ------------------------------------------------------------------ bound_logger.info("splitting_data") - X_train_full, X_val_test, val_offset = split_interactions( + split_result = split_interactions( df, user_column=user_col, item_column=item_col, time_column=time_col, split_config=recipe.training.split, ) + X_train_full = split_result.X_train_full + X_val_test = split_result.X_val_test + val_offset = split_result.val_offset bound_logger.info("split_done", val_offset=val_offset) + # Encode features onto the SEARCH phase's own axis labels. This matrix + # must never be reused by the final refit: the final refit builds its + # matrix via df_to_sparse -> pd.Categorical (sorted), while this one is + # ordered by split_interactions's list(set(...)) (unsorted, and not even + # stable across processes for string ids) -- a different permutation of + # the same items/users. irspack accepts a misordered feature matrix + # SILENTLY (no shape or value error), so re-encoding per phase is not an + # optional optimization to skip. + search_feature_kwargs = encode_for_axis( + feature_tables, + item_order=split_result.item_ids, + user_order=split_result.row_user_ids, + ) + # ------------------------------------------------------------------ # 5. Build evaluator. # ------------------------------------------------------------------ @@ -439,6 +479,7 @@ def _run_training_locked( recipe_name=recipe.name, run_id=run_id, metric=recipe.training.metric, + feature_kwargs=search_feature_kwargs, ) bound_logger.info( @@ -458,6 +499,7 @@ def _run_training_locked( item_column=item_col, class_name=search_result.best_class_name, best_params=search_result.best_params, + feature_tables=feature_tables, ) bound_logger.info("final_model_trained") @@ -488,6 +530,18 @@ def _run_training_locked( "data_stats": data_stats, } + # Omit the "features" key entirely when features are off, so a + # non-feature artifact's header stays byte-identical to today's. + if feature_tables.enabled: + features_header: dict[str, Any] = {"version": FEATURE_STATE_VERSION} + item_desc = state_descriptor(feature_tables.item_state) + if item_desc is not None: + features_header["item"] = item_desc + user_desc = state_descriptor(feature_tables.user_state) + if user_desc is not None: + features_header["user"] = user_desc + header_dict["features"] = features_header + artifact_path: str = write_artifact_fn( trained_recommender, header_dict, @@ -754,6 +808,7 @@ def _train_final( item_column: str, class_name: str, best_params: dict[str, Any], + feature_tables: FeatureTables | None = None, ) -> IDMappedRecommender: """Train the final model on the full dataset using best hyperparameters. @@ -764,6 +819,26 @@ def _train_final( gets written. Filter to ``__init__``-accepted keys before constructing, and log any dropped names so operators can investigate plugin/version drift. + + ``feature_tables``, when given and enabled, is re-encoded HERE against + THIS function's own ``iids_str`` / ``uids_str`` (derived from + ``df_to_sparse``'s sorted ``pd.Categorical`` ordering). It must never + reuse a matrix built for the search phase's ``list(set(...))`` ordering + (see ``encode_for_axis``'s docstring) -- irspack accepts a misordered + feature matrix silently, so that would train a silently-wrong model + rather than raise. Defaults to ``None`` so existing callers that never + touch features are unaffected. + + The re-encoded kwargs are only built when ``class_name`` actually accepts + them (``is_feature_capable``), mirroring ``search.py``'s per-trial gate + (``trial_features``). A recipe's ``features:`` block only requires that + *at least one* listed algorithm be feature-capable (see + ``Recipe._validate_features_algorithms``); a multi-algorithm search may + still pick a non-feature-capable winner (e.g. TopPop), and that is a + perfectly valid, non-feature artifact -- not an error. Splatting + ``item_features``/``user_features`` into a constructor that does not + declare them (e.g. ``TopPopRecommender.__init__(self, X_train)``) raises + ``TypeError`` unconditionally, so this gate must run BEFORE construction. """ import inspect as _inspect @@ -771,6 +846,24 @@ def _train_final( uids_str = [str(u) for u in uids] iids_str = [str(i) for i in iids] + # Re-encode against THIS phase's own axes -- see the docstring above. + # Gated on is_feature_capable(class_name): a features: recipe only + # requires ONE listed algorithm to be feature-capable, so the search + # winner may legitimately be a non-feature-capable class (e.g. TopPop). + # Splatting item_features/user_features into such a constructor raises + # TypeError unconditionally -- this mirrors search.py's per-trial gate. + final_feature_kwargs: dict[str, Any] = {} + if ( + feature_tables is not None + and feature_tables.enabled + and is_feature_capable(class_name) + ): + final_feature_kwargs = encode_for_axis( + feature_tables, + item_order=iids_str, + user_order=uids_str, + ) + rec_cls = get_recommender_cls(class_name) try: @@ -807,7 +900,9 @@ def _train_final( ) try: - recommender = rec_cls(X_full, **filtered).learn() + recommender = _construct( + rec_cls, X_full, filtered, final_feature_kwargs + ).learn() except TypeError as exc: raise TrainingError( f"Final training of {class_name} failed with params {filtered}: {exc}", @@ -821,5 +916,45 @@ def _train_final( f"Final training of {class_name} rejected params {filtered}: {exc}", code="final_training_error", ) from exc + except RuntimeError as exc: + # Rank-deficient features make the feature-ridge Cholesky solve + # fail. This CAN happen even when every search trial succeeded, + # because the final refit's matrix differs from every trial's + # matrix (full dataset vs. train+val split). Do not tell the user + # to drop a column: recotem's own always-on bias column is + # deliberately collinear with the categorical one-hots (see + # recotem._features's module docstring) and is the most likely + # structural cause, and it cannot be removed from the recipe. + if "Feature ridge Cholesky decomposition failed" in str(exc): + raise TrainingError( + "Feature ridge Cholesky decomposition failed during final " + "training. The feature matrix for the full dataset is rank " + "deficient at the selected lambda. This can happen even " + "when every search trial succeeded, because the final " + "matrix differs from every trial's matrix. Raising " + "min_frequency on high-cardinality feature columns usually " + "resolves it; see docs/operations.md.", + code="feature_cholesky_error", + ) from exc + raise - return IDMappedRecommender(recommender, uids_str, iids_str) + # Deliberately unconditional -- do NOT gate this on is_feature_capable + # the way final_feature_kwargs is gated above. The artifact header's + # "features" key is written whenever feature_tables.enabled (see the + # header-assembly block above, keyed off feature_tables.enabled, not + # is_feature_capable), regardless of which class the search actually + # picked. `recotem inspect` promises the header describes the payload + # without deserializing it, so item_feature_state/user_feature_state + # must be persisted here whenever the header says features are present + # -- even for a non-feature-capable winner (e.g. TopPop) that never + # reads them at serve time. Symmetrizing this return with the + # is_feature_capable gate above would make such an artifact's header + # claim features that the payload does not carry, silently breaking + # header/payload parity. + return IDMappedRecommender( + recommender, + uids_str, + iids_str, + item_feature_state=feature_tables.item_state if feature_tables else None, + user_feature_state=feature_tables.user_state if feature_tables else None, + ) diff --git a/src/recotem/training/search.py b/src/recotem/training/search.py index ed94eda4..cc58b097 100644 --- a/src/recotem/training/search.py +++ b/src/recotem/training/search.py @@ -24,7 +24,11 @@ # _compat applies IPython stub before irspack imports (see _compat.py). import recotem.training._compat # noqa: F401 -from recotem.training.algorithms import get_recommender_cls, resolve_algorithm_name +from recotem.training.algorithms import ( + get_recommender_cls, + is_feature_capable, + resolve_algorithm_name, +) from recotem.training.errors import SearchError, TrainingError, ZeroScoreError from recotem.training.evaluate import get_score from recotem.training.progress import ProgressReporter, make_trial_callback @@ -134,6 +138,57 @@ def _make_storage(storage_path: str) -> optuna.storages.BaseStorage | None: return optuna.storages.RDBStorage(path) +# --------------------------------------------------------------------------- +# Feature-aware construction helper +# --------------------------------------------------------------------------- + + +def _construct( + rec_cls: type, + X: sps.spmatrix, + params: dict[str, Any], + feature_kwargs: dict[str, sps.csr_matrix] | None, +) -> Any: + """Construct a recommender, injecting feature matrices exactly once. + + THE single construction point for feature-aware training: both search + paths here (the per-trial-timeout thread path and the default path) and + the final refit in ``pipeline._train_final``. It exists because irspack + fails asymmetrically: it raises for a feature matrix with lambda=0, but + silently trains PLAIN iALS for a lambda with no feature matrix. Routing + every site through here turns that one silent mistake into a raise. + + The checks below are an explicit ``raise AssertionError`` rather than + ``assert`` because ``assert`` is stripped under ``-O`` / ``PYTHONOPTIMIZE`` + -- which would restore exactly the silent plain-iALS training this helper + exists to prevent. ``AssertionError`` (not ``TrainingError``) is + deliberate: reaching here means recotem suggested a lambda without + plumbing through the matching matrix, which is a bug in this package, not + a bad recipe or bad data, so it must not claim one of the semantic exit + codes (2-8). It is left unmapped by ``_map_exception_to_exit`` and so + surfaces as ``_EXIT_UNKNOWN`` (exit 1, "unhandled / unmapped exception"), + which is the code reserved for bugs -- exit 4 would send the operator + looking at a recipe and data that are not at fault. + + Scope of the guarantee: this catches only lambda-without-matrix. It does + not check that a supplied matrix has the right shape, dtype, or row order + -- axis alignment is covered separately by the alignment tests in + tests/unit/test_training_pipeline.py. + """ + kwargs = dict(feature_kwargs or {}) + if "lambda_item_feature" in params and kwargs.get("item_features") is None: + raise AssertionError( + "lambda_item_feature was suggested but no item_features matrix " + "was supplied; irspack would silently train plain iALS" + ) + if "lambda_user_feature" in params and kwargs.get("user_features") is None: + raise AssertionError( + "lambda_user_feature was suggested but no user_features matrix " + "was supplied; irspack would silently train plain iALS" + ) + return rec_cls(X, **params, **kwargs) + + # --------------------------------------------------------------------------- # Public entry point # --------------------------------------------------------------------------- @@ -155,6 +210,7 @@ def run_search( recipe_name: str, run_id: str, metric: str = "ndcg", + feature_kwargs: dict[str, sps.csr_matrix] | None = None, ) -> SearchResult: """Run an Optuna hyperparameter search and return the best result. @@ -184,6 +240,15 @@ def run_search( ``ProgressReporter`` instance for trial notifications. recipe_name, run_id: Carried in log events. + feature_kwargs: + Optional ``{"item_features": csr_matrix, "user_features": csr_matrix}`` + produced by ``training.features.encode_for_axis`` (only present keys + for configured sides). Defaults to ``None`` so existing callers are + unaffected. Only forwarded to trials whose sampled algorithm is + feature-capable (``is_feature_capable``); every other trial gets + irspack's own ``(n, 0)`` auto-fill. When present for a + feature-capable trial, a ``lambda_*_feature`` search dimension is + added per configured side. Returns ------- @@ -342,8 +407,39 @@ def objective(trial: optuna.Trial) -> float: rec_cls = get_recommender_cls(class_name) + # Feature-aware iALS: only inject matrices when the class sampled + # for THIS trial actually supports them. In a multi-algorithm search + # a non-capable class (e.g. TopPop) may be sampled in the same study + # as IALS; it must never receive item_features/user_features kwargs + # it does not accept. + trial_features: dict[str, sps.csr_matrix] = ( + feature_kwargs + if (feature_kwargs and is_feature_capable(class_name)) + else {} + ) + params: dict[str, Any] = rec_cls.default_suggest_parameter(trial, {}) + # irspack ships no default search space for the feature ridge, and + # the constructor default (0.0) is a hard error whenever a feature + # matrix is present. Recotem defines the range, mirroring upstream's + # own ML-1M example (the only range upstream has exercised). This is + # a ridge on A/B, not the feature-prior strength itself (that's + # reg/nu); a large enough value shrinks A/B toward zero and was + # measured bit-identical to plain iALS at lambda=1e8. 1e6 is just + # upstream's exercised value, NOT a verified features-off bound -- + # the search space is not guaranteed to contain a features-off model. + # Run two recipes for a true on/off comparison. + if trial_features: + if "item_features" in trial_features: + params["lambda_item_feature"] = trial.suggest_float( + "lambda_item_feature", 5e-2, 1e6, log=True + ) + if "user_features" in trial_features: + params["lambda_user_feature"] = trial.suggest_float( + "lambda_user_feature", 5e-2, 1e6, log=True + ) + # Per-trial timeout: run the learn in a thread so we can interrupt. if has_per_trial_timeout: result_holder: list[Any] = [] @@ -353,8 +449,19 @@ def objective(trial: optuna.Trial) -> float: def _learn() -> None: try: try: - rec = rec_cls(X_tv_train, **params) - rec.learn_with_optimizer(evaluator, trial) + rec = _construct(rec_cls, X_tv_train, params, trial_features) + try: + rec.learn_with_optimizer(evaluator, trial) + except RuntimeError as exc: + # Rank-deficient features make the feature-ridge + # Cholesky fail. This is data-dependent, so prune + # the trial rather than failing the run -- + # matching upstream's own feature-aware example. + if "Feature ridge Cholesky decomposition failed" in str( + exc + ): + raise optuna.TrialPruned(str(exc)) from exc + raise result_holder.append(rec) except (MemoryError, RecursionError): # Re-raise without capture so the worker thread dies @@ -362,6 +469,21 @@ def _learn() -> None: # exit code instead of bookkeeping it as a trial # failure that hints at per-trial timeout. raise + except optuna.TrialPruned as exc: + # A pruned trial (e.g. the rank-deficient-feature-Gram + # Cholesky failure above, or any other by-design prune + # raised from inside learn_with_optimizer) is a normal, + # expected outcome -- not a failure. TrialPruned + # subclasses Exception, so without this clause the + # generic handler below would catch it and log a + # spurious trial_learn_failed WARNING, making an + # expected prune look like an error under the + # per-trial-timeout thread path while the + # non-threaded path (which has no such generic catch) + # stays silent for the identical prune. Still route + # through exc_holder so it re-raises in the main + # thread and Optuna records the trial as PRUNED. + exc_holder.append(exc) except Exception as exc: # noqa: BLE001 logger.warning( "trial_learn_failed", @@ -452,8 +574,17 @@ def _learn() -> None: raise exc_holder[0] recommender = result_holder[0] else: - recommender = rec_cls(X_tv_train, **params) - recommender.learn_with_optimizer(evaluator, trial) + recommender = _construct(rec_cls, X_tv_train, params, trial_features) + try: + recommender.learn_with_optimizer(evaluator, trial) + except RuntimeError as exc: + # Rank-deficient features make the feature-ridge Cholesky + # fail. This is data-dependent, so prune the trial rather + # than failing the run -- matching upstream's own + # feature-aware example. + if "Feature ridge Cholesky decomposition failed" in str(exc): + raise optuna.TrialPruned(str(exc)) from exc + raise score = get_score(evaluator, recommender) diff --git a/src/recotem/training/split.py b/src/recotem/training/split.py index f32a5983..9cce4135 100644 --- a/src/recotem/training/split.py +++ b/src/recotem/training/split.py @@ -13,6 +13,8 @@ from __future__ import annotations +from dataclasses import dataclass + import pandas as pd import scipy.sparse as sps from irspack import split_dataframe_partial_user_holdout @@ -24,6 +26,25 @@ from recotem.training.errors import SplitError +@dataclass(frozen=True) +class SplitResult: + """Split matrices plus the axis labels needed to align a feature matrix. + + ``item_ids`` labels ``X_train_full``'s columns; ``row_user_ids`` labels its + rows. Both are load-bearing for feature-aware training: irspack accepts a + misordered feature matrix silently, and the search-phase item order is + ``list(set(...))`` -- neither sorted nor stable across processes for string + ids -- so a feature matrix must be rebuilt in-process from these labels + rather than cached or reused from the final-refit ordering. + """ + + X_train_full: sps.spmatrix + X_val_test: sps.spmatrix + val_offset: int + item_ids: list[str] + row_user_ids: list[str] + + def split_interactions( df: pd.DataFrame, *, @@ -31,18 +52,21 @@ def split_interactions( item_column: str, time_column: str | None, split_config: SplitConfig, -) -> tuple[sps.spmatrix, sps.spmatrix, int]: +) -> SplitResult: """Split *df* into train and validation sparse matrices. Returns ------- - X_train_full: - Full combined train+val train-side matrix (used for final training). - X_val_test: - Held-out test interactions for validation users (used for evaluation). - val_offset: - Row offset into the full user index pointing to the first validation - user (``train.n_users`` in irspack terminology). + SplitResult + ``X_train_full``: full combined train+val train-side matrix (used for + final training). + ``X_val_test``: held-out test interactions for validation users (used + for evaluation). + ``val_offset``: row offset into the full user index pointing to the + first validation user (``train.n_users`` in irspack terminology). + ``item_ids``: item vocabulary labelling ``X_train_full``'s columns. + ``row_user_ids``: user ids labelling ``X_train_full``'s rows, in + train-then-val order. Raises ------ @@ -61,7 +85,7 @@ def split_interactions( try: if scheme == "time_global": assert time_column is not None # narrowed by the check above - dataset = _split_time_global( + dataset, item_all = _split_time_global( df, user_column=user_column, item_column=item_column, @@ -71,7 +95,7 @@ def split_interactions( else: # `random` (time_column is None) and `time_user` (time_column set) # both use partial_user_holdout. - dataset, _ = split_dataframe_partial_user_holdout( + dataset, item_all = split_dataframe_partial_user_holdout( df, user_column=user_column, item_column=item_column, @@ -101,7 +125,30 @@ def split_interactions( X_train_full: sps.spmatrix = sps.vstack([train.X_train, val.X_train]) val_offset: int = train.n_users - return X_train_full, X_val_test, val_offset + # Row order is train-then-val and is NOT a free choice: irspack's Evaluator + # pins rows [val_offset:] to the validation ground truth (evaluator.py:46). + # Do not sort. + row_user_ids = [str(u) for u in train.user_ids] + [str(u) for u in val.user_ids] + item_ids = [str(i) for i in item_all] + + if len(item_ids) != X_train_full.shape[1]: + raise SplitError( + f"internal: item vocabulary size {len(item_ids)} does not match " + f"matrix column count {X_train_full.shape[1]}" + ) + if len(row_user_ids) != X_train_full.shape[0]: + raise SplitError( + f"internal: user row labels {len(row_user_ids)} do not match " + f"matrix row count {X_train_full.shape[0]}" + ) + + return SplitResult( + X_train_full=X_train_full, + X_val_test=X_val_test, + val_offset=val_offset, + item_ids=item_ids, + row_user_ids=row_user_ids, + ) def _split_time_global( @@ -111,7 +158,7 @@ def _split_time_global( item_column: str, time_column: str, split_config: SplitConfig, -) -> dict: +) -> tuple[dict, list]: """Hold out every interaction at or after the global timestamp quantile. The cutoff is the ``1 - heldout_ratio`` quantile of ``df[time_column]``. @@ -133,7 +180,7 @@ def _split_time_global( f"cutoff={cutoff!r}; check heldout_ratio and time_column values." ) - _, dataset = holdout_specific_interactions( + item_all, dataset = holdout_specific_interactions( df, user_column=user_column, item_column=item_column, @@ -142,4 +189,7 @@ def _split_time_global( validatable_user_ratio_test=0.0, random_state=split_config.seed, ) - return dataset + # holdout_specific_interactions returns (item_ids, dataset). This first + # value is the ONLY source of the item vocabulary for this scheme: both + # returned datasets have item_ids is None (specified.py:118,138-140). + return dataset, list(item_all) diff --git a/tests/integration/test_serve_predict_e2e.py b/tests/integration/test_serve_predict_e2e.py index 2f5a8276..87480d06 100644 --- a/tests/integration/test_serve_predict_e2e.py +++ b/tests/integration/test_serve_predict_e2e.py @@ -10,6 +10,7 @@ import hashlib import time from pathlib import Path +from typing import Any from unittest.mock import MagicMock import pytest @@ -338,6 +339,30 @@ def test_train_then_serve_full_artifact_roundtrip(tmp_path: Path) -> None: ] +def test_roundtrip_algos_covers_every_supported_class_name() -> None: + """Pin ``_ROUNDTRIP_ALGOS`` to the real source of truth. + + The list above is hand-maintained and its comment claims completeness + ("every algorithm recotem advertises") without deriving from + ``training.algorithms.SUPPORTED_CLASS_NAMES`` -- so an algorithm added to + the supported set but forgotten here would silently stop being covered by + ``test_every_algorithm_artifact_serve_roundtrip`` below. Resolving each + alias to its canonical class name and comparing sets closes that gap + cheaply without touching the (working) parametrized test itself. + """ + from recotem.training.algorithms import ( + SUPPORTED_CLASS_NAMES, + resolve_algorithm_name, + ) + + resolved = {resolve_algorithm_name(alias) for alias in _ROUNDTRIP_ALGOS} + assert resolved == SUPPORTED_CLASS_NAMES, ( + f"_ROUNDTRIP_ALGOS (resolved: {sorted(resolved)}) has drifted from " + f"SUPPORTED_CLASS_NAMES ({sorted(SUPPORTED_CLASS_NAMES)}); update " + "_ROUNDTRIP_ALGOS to add/remove the alias." + ) + + def _irspack_has_recommender(class_name: str) -> bool: """True when the installed irspack build exposes ``class_name``. @@ -897,3 +922,500 @@ def test_recipes_discovery_list_and_detail() -> None: headers={"x-api-key": plaintext}, ) assert missing_resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Task 16: feature-aware iALS -- integration and compatibility coverage +# --------------------------------------------------------------------------- +# +# The three tests below prove, at the integration level, that the whole +# feature-aware stack (recipe -> training -> artifact -> serving) works +# TOGETHER, and that old and new artifacts interoperate: +# +# 1. test_feature_aware_artifact_serve_roundtrip -- a features.item AND +# features.user recipe trains a real IALS, the resulting artifact is +# served through the real v1 HTTP surface, and a known user, a cold user +# (case A), and a cold seed (case C) all succeed. +# 2. test_old_artifact_loads_on_feature_aware_serve -- an artifact with no +# "features" header key at all (pre-Task-9 shape) loads and serves +# normally through create_app()'s real startup loader. +# 3. test_feature_version_2_artifact_fails_closed -- a hand-written artifact +# declaring features.version=2 is refused with reason "feature_version", +# visible through /v1/health and /v1/health/details. + + +def _make_item_features_csv(tmp_path: Path, n_items: int = 40) -> Path: + """Item feature table: alternating categorical "genre" by item parity. + + Matches the id space of ``_make_clustered_synthetic_csv`` (item ids + ``i0``..``i{n_items - 1}``). + """ + rows = ["item_id,genre"] + for i in range(n_items): + genre = "action" if i % 2 else "drama" + rows.append(f"i{i},{genre}") + p = tmp_path / "item_features.csv" + p.write_text("\n".join(rows) + "\n") + return p + + +def _make_user_features_csv(tmp_path: Path, n_users: int = 60) -> Path: + """User feature table: alternating categorical "band" by user parity. + + Matches the id space of ``_make_clustered_synthetic_csv`` (user ids + ``u0``..``u{n_users - 1}``). + """ + rows = ["user_id,band"] + for u in range(n_users): + band = "young" if u % 2 else "old" + rows.append(f"u{u},{band}") + p = tmp_path / "user_features.csv" + p.write_text("\n".join(rows) + "\n") + return p + + +def test_feature_aware_artifact_serve_roundtrip( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Train an IALS recipe with features.item AND features.user end to end, + serve the resulting artifact through the real v1 HTTP surface, and + exercise a known user, a cold user with ``user_features`` (case A), and + a cold seed with ``item_features`` (case C). + + Mutation guard + -------------- + ``lambda_item_feature`` / ``lambda_user_feature`` are tuned over a + log-uniform ``[5e-2, 1e6]`` range (``training/search.py``); the top of + that range drives the feature contribution toward zero -- close to plain + iALS. That means "the cold-start calls below returned 200" proves + nothing about features actually being used: the exact same 200s would + come back from a model that quietly trained close to plain iALS (e.g. + because Optuna's sampler landed near the top of the range), or from a + future regression that silently stopped threading the encoded feature + matrix through to irspack -- routes.py's case A/C branches only check + that the model *carries* feature state, not that the request's feature + values reach it. To make the assertions mean something, ``suggest_float`` + is patched to pin *only* those two parameter names to ``0.1`` -- the same + order of magnitude ``tests/unit/test_idmap.py``'s ``fa_model`` fixture + uses to get a genuinely feature-sensitive model -- while every other + hyperparameter is still sampled by the real (seeded) TPESampler. The + fetch, cleanse, split, search loop, final refit, artifact write, HMAC + signing, and deserialization all run for real, unmocked. + + The proof itself: cold-start recommendations are requested twice, once + per category value (``band: young`` vs ``band: old``; ``genre: action`` + vs ``genre: drama``), and the ITEM-ID sequences (not the raw score- + bearing dicts) are asserted to DIFFER. Comparing item ids rather than + scores matters: irspack's underlying BLAS calls are not bit-reproducible + across two separate calls even with an UNCHANGED encoded feature vector + (observed empirically -- repeated identical calls differ at the float32 + rounding level, ~1e-7 relative), so comparing raw scores would make this + assertion pass on noise alone regardless of whether the feature value + was ever used. Confirmed by temporarily forcing + ``recotem._features.encode_one`` to ignore its ``values`` argument + (simulating a request's feature dict never reaching the encoder): every + call in this test still returned 200, but the item-id sequences for + band='young' vs band='old' and genre='action' vs genre='drama' came out + IDENTICAL while the raw per-item scores still jittered in the 7th + decimal digit -- exactly the false-negative a naive "200 and the dicts + differ" check would have missed. + """ + import json + + import optuna + + from recotem.artifact.io import read_artifact + from recotem.artifact.signing import KeyRing, unpickle_payload + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + FeatureColumn, + FeaturesConfig, + FeatureSideConfig, + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + from recotem.training.pipeline import run_training + + original_suggest_float = optuna.trial.Trial.suggest_float + + def _pinned_suggest_float( + self: Any, name: str, low: float, high: float, *args: Any, **kwargs: Any + ) -> float: + if name in ("lambda_item_feature", "lambda_user_feature"): + # Collapse the sampled range to a single point (0.1) rather than + # short-circuiting the call entirely: a bare early-return skips + # Optuna's own bookkeeping, so the value would never land in + # ``trial.params`` / ``best_trial.params`` -- and pipeline.py's + # final refit builds its params from exactly that dict. A refit + # missing lambda_item_feature/lambda_user_feature while still + # receiving item_features/user_features hits irspack's "Feature + # weight regularization must be positive" (its constructor + # default is 0.0, invalid whenever a feature matrix is given) -- + # a real failure this monkeypatch must not manufacture. Routing + # through the REAL suggest_float with low==high keeps every bit + # of that bookkeeping intact while still being deterministic. + return original_suggest_float(self, name, 0.1, 0.1, *args, **kwargs) + return original_suggest_float(self, name, low, high, *args, **kwargs) + + monkeypatch.setattr(optuna.trial.Trial, "suggest_float", _pinned_suggest_float) + + csv_file = _make_clustered_synthetic_csv(tmp_path) + items_csv = _make_item_features_csv(tmp_path) + users_csv = _make_user_features_csv(tmp_path) + artifact_path = str(tmp_path / "feature_roundtrip.recotem") + + recipe = Recipe( + name="feature-aware-roundtrip", + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + features=FeaturesConfig( + item=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(items_csv)), + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ), + user=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(users_csv)), + id_column="user_id", + columns=[FeatureColumn(name="band", encoding="categorical")], + ), + ), + training=TrainingConfig( + algorithms=["IALS"], + n_trials=2, + cutoff=5, # must be < n_items to avoid irspack ValueError + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig(path=artifact_path, versioning="always_overwrite"), + ) + + kr = KeyRing("dev:" + ("0" * 64)) + result = run_training( + recipe, + key_ring=kr, + signing_key="dev", + no_lock=True, + dev_allow_unsigned=True, + quiet=True, + ) + assert result is not None + assert result.best_class == "IALSRecommender", ( + "test setup invariant: IALS is the only listed algorithm, so it must win" + ) + + header, payload_bytes = read_artifact(result.artifact_path, kr) + header_dict = json.loads(header.header_data) + assert header_dict["features"]["version"] == 1 + assert header_dict["features"]["item"]["columns"] == ["genre"] + assert header_dict["features"]["user"]["columns"] == ["band"] + + recommender = unpickle_payload(payload_bytes) + assert recommender.item_feature_state is not None + assert recommender.user_feature_state is not None + + # --- serve the REAL trained artifact through the real v1 HTTP surface --- + entry = ModelEntry( + name=recipe.name, + recommender=recommender, + header=header_dict, + kid=header.kid, + _loaded_marker=(None, hashlib.sha256(payload_bytes).hexdigest()), + loaded_at_unix=time.time(), + ) + registry = ModelRegistry() + registry.replace(recipe.name, entry) + + plaintext = "feature_roundtrip_api_key_32byte" + api_entry = _make_api_entry(plaintext) + app = build_v1_app(registry, api_keys=[api_entry]) + client = TestClient(app) + headers = {"x-api-key": plaintext} + + # 1. Known user -> 200 (unchanged path; not a cold start at all). + known = client.post( + f"/v1/recipes/{recipe.name}:recommend", + json={"user_id": "u0", "limit": 5}, + headers=headers, + ) + assert known.status_code == 200, known.text + + # 2. Unknown user + user_features -> 200 (case A). + cold_young = client.post( + f"/v1/recipes/{recipe.name}:recommend", + json={ + "user_id": "brand_new_user", + "limit": 5, + "user_features": {"band": "young"}, + }, + headers=headers, + ) + assert cold_young.status_code == 200, cold_young.text + + # 3. :recommend-related with a cold seed + item_features -> 200 (case C). + cold_action = client.post( + f"/v1/recipes/{recipe.name}:recommend-related", + json={ + "seed_items": ["brand_new_item"], + "limit": 5, + "item_features": {"brand_new_item": {"genre": "action"}}, + }, + headers=headers, + ) + assert cold_action.status_code == 200, cold_action.text + + # --- mutation guard: prove genuine feature-dependence, not just plumbing --- + cold_old = client.post( + f"/v1/recipes/{recipe.name}:recommend", + json={ + "user_id": "brand_new_user", + "limit": 5, + "user_features": {"band": "old"}, + }, + headers=headers, + ) + assert cold_old.status_code == 200, cold_old.text + + # Compare item-ID SEQUENCES, not the raw score-bearing dicts: irspack's + # underlying BLAS calls are not guaranteed bit-reproducible across two + # separate calls (observed empirically: repeated calls with an + # UNCHANGED encoded feature vector still differ at the float32 rounding + # level, ~1e-7 relative). Comparing full score dicts would make this + # assertion pass on noise alone regardless of whether the feature value + # was used. The item-id ranking, in contrast, only reorders when the + # underlying embeddings differ by much more than that noise floor -- + # confirmed by re-running this exact scenario with the feature-blind + # mutation described in this test's docstring: the item-id sequences + # came out IDENTICAL while the scores still jittered in the 7th decimal + # digit. + young_ids = [item["item_id"] for item in cold_young.json()["items"]] + old_ids = [item["item_id"] for item in cold_old.json()["items"]] + assert young_ids != old_ids, ( + "cold-start recommendations for band='young' vs band='old' must " + "differ -- otherwise the served model is not actually using " + "user_feature_state (e.g. lambda_user_feature landed at the top of " + "its tuned range, or the feature vector never reached irspack)." + ) + + cold_drama = client.post( + f"/v1/recipes/{recipe.name}:recommend-related", + json={ + "seed_items": ["brand_new_item"], + "limit": 5, + "item_features": {"brand_new_item": {"genre": "drama"}}, + }, + headers=headers, + ) + assert cold_drama.status_code == 200, cold_drama.text + action_ids = [item["item_id"] for item in cold_action.json()["items"]] + drama_ids = [item["item_id"] for item in cold_drama.json()["items"]] + assert action_ids != drama_ids, ( + "cold-seed recommendations for genre='action' vs genre='drama' must " + "differ -- otherwise the served model is not actually using " + "item_feature_state." + ) + + +def test_old_artifact_loads_on_feature_aware_serve(tmp_path: Path) -> None: + """An artifact with NO feature state (pre-Task-9 shape) must load and + serve normally through this feature-aware build. + + Trains a real, features-less TopPop recipe (its header omits the + "features" key entirely -- see ``test_no_features_recipe_omits_header_key`` + in ``tests/unit/test_training_pipeline.py``), then drives the artifact + through ``create_app()``'s real startup loader -- the same code path + ``test_feature_version_2_artifact_fails_closed`` below proves fails + CLOSED for a version mismatch. This is the positive control: the + "features absent -> pass" branch of ``check_artifact_feature_version`` + must not regress backward compatibility for the (huge) population of + already-deployed artifacts trained before this feature existed. + """ + import json + + from recotem.artifact.io import read_artifact + from recotem.artifact.signing import KeyRing, unpickle_payload + from recotem.config import ServeConfig + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + from recotem.serving.app import create_app + from recotem.training.pipeline import run_training + + # A dense "every user rates every item" grid (_make_tiny_synthetic_csv) + # would leave a known user with nothing left to recommend (every item + # already seen), so the final :recommend assertion below needs the + # partial-density clustered dataset instead. + csv_file = _make_clustered_synthetic_csv(tmp_path) + artifact_path = str(tmp_path / "old_style.recotem") + recipe_name = "old-style-no-features" + + recipe = Recipe( + name=recipe_name, + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + training=TrainingConfig( + algorithms=["TopPop"], + n_trials=1, + cutoff=5, # must be < n_items to avoid irspack ValueError + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig(path=artifact_path, versioning="always_overwrite"), + ) + + signing_hex = "0" * 64 + kr = KeyRing("dev:" + signing_hex) + result = run_training( + recipe, + key_ring=kr, + signing_key="dev", + no_lock=True, + dev_allow_unsigned=True, + quiet=True, + ) + assert result is not None + + header, payload_bytes = read_artifact(result.artifact_path, kr) + header_dict = json.loads(header.header_data) + assert "features" not in header_dict, ( + "test setup invariant: a features-less recipe must omit the " + "'features' header key entirely" + ) + # Sanity: the artifact really does deserialize (would raise otherwise). + assert unpickle_payload(payload_bytes) is not None + + # Drive it through the REAL startup loader, not a hand-built ModelEntry -- + # this is what actually calls check_artifact_feature_version. + recipes_dir = tmp_path / "recipes" + recipes_dir.mkdir() + _write_minimal_recipe_yaml(recipes_dir, recipe_name, result.artifact_path) + + cfg = ServeConfig() + cfg.signing_keys_raw = f"dev:{signing_hex}" + cfg.recipes_dir = str(recipes_dir) + cfg.env = "development" + cfg.insecure_no_auth = True + cfg.allowed_hosts = ["testserver", "localhost", "127.0.0.1", "*"] + + app = create_app(cfg) + client = TestClient(app) + + health = client.get("/v1/health") + assert health.status_code == 200 + assert health.json() == {"status": "ok", "total": 1, "loaded": 1} + + details = client.get("/v1/health/details") + assert details.status_code == 200 + details_body = details.json() + assert details_body["recipes"][recipe_name]["loaded"] is True + assert not details_body["recipes"][recipe_name].get("error") + + predict = client.post( + f"/v1/recipes/{recipe_name}:recommend", + json={"user_id": "u0", "limit": 3}, + ) + assert predict.status_code == 200, predict.text + assert len(predict.json()["items"]) == 3 + + +def test_feature_version_2_artifact_fails_closed(tmp_path: Path) -> None: + """A hand-written artifact declaring ``features.version: 2`` must be + refused by serve's startup loader with reason ``"feature_version"``, and + that refusal must be visible through the real ``/v1/health`` and + ``/v1/health/details`` endpoints -- not just as a raised Python + exception from calling ``check_artifact_feature_version`` directly + (``tests/unit/test_features_compat.py`` already covers the gate function + and both loader call sites in isolation; this proves the wiring holds + all the way through ``create_app()``). + + Fails for the RIGHT reason, not merely fails: asserts the /health/details + error text names the version-check gate specifically (not e.g. an HMAC + or a deserialize failure), and separately asserts + ``_classify_artifact_error`` -- the same function the watcher's hot-swap + path uses to label the ``recotem_artifact_load_failures_total`` metric -- + maps that exact message to ``"feature_version"``, not the "parse" + catch-all (the message contains the word "version", the same trap the + irspack skew guard's message has). + """ + from recotem.artifact.io import write_artifact + from recotem.artifact.signing import KeyRing + from recotem.config import ServeConfig + from recotem.serving.app import create_app + from recotem.serving.watcher import _classify_artifact_error + + kid_hex = "ab" * 32 + kr = KeyRing("probe:" + kid_hex) + + artifact_path = str(tmp_path / "feature_v2.recotem") + recipe_name = "feature_v2_recipe" + header_dict = { + "recipe_name": recipe_name, + "best_class": "TopPopRecommender", + "trained_at": "2026-01-01T00:00:00Z", + # No "irspack_version" key: the irspack skew guard (which runs + # BEFORE the feature-version gate in the real loader) fails OPEN on + # a missing version, so this artifact is refused for exactly one + # reason -- the feature-version gate -- not two entangled ones. + "features": { + "version": 2, + "item": {"n_features": 4, "columns": ["genre"]}, + }, + } + write_artifact( + {"dummy": "payload"}, + header_dict, + kr, + artifact_path, + versioning="always_overwrite", + ) + + recipes_dir = tmp_path / "recipes" + recipes_dir.mkdir() + _write_minimal_recipe_yaml(recipes_dir, recipe_name, artifact_path) + + cfg = ServeConfig() + cfg.signing_keys_raw = f"probe:{kid_hex}" + cfg.recipes_dir = str(recipes_dir) + cfg.env = "development" + cfg.insecure_no_auth = True + cfg.allowed_hosts = ["testserver", "localhost", "127.0.0.1", "*"] + + app = create_app(cfg) + client = TestClient(app) + + health = client.get("/v1/health") + assert health.status_code == 503 + health_body = health.json() + assert health_body["status"] == "degraded" + assert health_body["loaded"] == 0 + assert health_body["total"] == 1 + + details = client.get("/v1/health/details") + assert details.status_code == 503 + details_body = details.json() + recipe_health = details_body["recipes"][recipe_name] + assert recipe_health["loaded"] is False + error = (recipe_health.get("error") or "").lower() + assert "feature version check failed" in error, ( + f"expected the feature-version gate's message prefix; got {error!r}" + ) + assert "declares feature encoder version 2" in error, ( + f"expected the refusal to name the offending version; got {error!r}" + ) + + # The reason must classify as "feature_version" specifically -- proving + # the failure is the version gate, not e.g. a coincidental deserialize + # or HMAC failure that also happens to yield loaded=False. + assert _classify_artifact_error(error) == "feature_version" + + predict = client.post( + f"/v1/recipes/{recipe_name}:recommend", + json={"user_id": "u1", "limit": 5}, + ) + assert predict.status_code == 503 diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 4e3b38e1..4641d40f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -112,6 +112,206 @@ def test_validate_probe_ok_for_existing_csv(tmp_path: Path) -> None: assert "probe OK (csv)" in result.stdout +def _feature_recipe_yaml( + tmp_path: Path, feature_source_path: Path, side: str = "item" +) -> Path: + """A recipe with a features. source pointing at *feature_source_path*. + + ``side`` is ``"item"`` or ``"user"`` -- both share the single + ``FeatureSideConfig`` model, so the generated block only differs in its + key and ``id_column``. + """ + interactions = tmp_path / "i.csv" + interactions.write_text("user_id,item_id\nu1,i1\nu2,i2\n") + id_column = "item_id" if side == "item" else "user_id" + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + f"""\ +name: probe +source: + type: csv + path: {interactions} +schema: + user_column: user_id + item_column: item_id +features: + {side}: + source: + type: csv + path: {feature_source_path} + id_column: {id_column} + columns: + - {{name: genre, encoding: categorical}} +training: + algorithms: [IALS] +output: + path: {tmp_path / "out.recotem"} +""" + ) + return recipe + + +def test_validate_probes_feature_sources(tmp_path: Path) -> None: + """validate must probe features.item.source, not just the top-level source. + + A missing feature-source CSV must fail validate (exit 3) with the + ``features.item.source`` label so an operator with multiple sources + knows exactly which one failed -- rather than surviving validate only to + die mid-train. + """ + recipe = _feature_recipe_yaml(tmp_path, tmp_path / "does_not_exist.csv") + result = runner.invoke(app, ["validate", str(recipe)]) + assert result.exit_code == 3, ( + f"missing features.item.source CSV must exit 3; got {result.exit_code}. " + f"Output: {result.stdout}" + ) + assert "features.item.source" in (result.stdout + result.stderr) + + +def test_validate_passes_with_reachable_feature_source(tmp_path: Path) -> None: + """validate exits 0 and probes features.item.source when it is reachable.""" + items = tmp_path / "items.csv" + items.write_text("item_id,genre\ni1,action\ni2,drama\n") + recipe = _feature_recipe_yaml(tmp_path, items) + result = runner.invoke(app, ["validate", str(recipe)]) + assert result.exit_code == 0, result.stdout + assert "Validation passed." in result.stdout + assert "probe OK (csv) [features.item.source]" in result.stdout + + +def test_validate_probes_user_feature_source(tmp_path: Path) -> None: + """validate must probe features.user.source symmetrically with item. + + Only ``features.item.source`` was covered above; ``features.user.source`` + shares the exact same ``FeatureSideConfig`` code path (same loop body in + ``validate``) but was previously unpinned by any test. + """ + recipe = _feature_recipe_yaml( + tmp_path, tmp_path / "does_not_exist.csv", side="user" + ) + result = runner.invoke(app, ["validate", str(recipe)]) + assert result.exit_code == 3, ( + f"missing features.user.source CSV must exit 3; got {result.exit_code}. " + f"Output: {result.stdout}" + ) + assert "features.user.source" in (result.stdout + result.stderr) + + +def test_validate_passes_with_reachable_user_feature_source(tmp_path: Path) -> None: + """validate exits 0 and probes features.user.source when it is reachable.""" + users = tmp_path / "users.csv" + users.write_text("user_id,genre\nu1,action\nu2,drama\n") + recipe = _feature_recipe_yaml(tmp_path, users, side="user") + result = runner.invoke(app, ["validate", str(recipe)]) + assert result.exit_code == 0, result.stdout + assert "Validation passed." in result.stdout + assert "probe OK (csv) [features.user.source]" in result.stdout + + +def test_validate_reports_where_for_exception_whose_str_ignores_args( + tmp_path: Path, +) -> None: + """``where`` must reach the operator even when str(exc) ignores .args. + + ``pydantic_core.ValidationError.__str__`` is implemented in Rust and + reads the error's own internal list, not ``.args`` -- so tagging + ``where`` by mutating ``exc.args`` in place (as an earlier version of + this fix did) is silently invisible for this exception type. A plugin + author who re-validates its config via a nested pydantic model inside + ``__init__`` (a plausible pattern per ``docs/plugin-authoring.md`` -- + nothing enforces the "always raise DataSourceError" convention) would + trigger exactly this. The fix must carry ``where`` in the caller's error + message instead, which works regardless of what ``str(exc)`` returns. + """ + from unittest.mock import MagicMock, patch + + from pydantic import BaseModel, ValidationError + + from recotem.datasource.bigquery import BigQueryConfig + from recotem.datasource.registry import get_source_class as _real_get_source_class + + class _NestedConfig(BaseModel): + x: int + + class _StubFeatureSource: + extras_required: list = [] + # Real Config so ``load_recipe`` (which also calls + # ``get_source_class``) resolves ``features.item.source`` into a + # valid ``BigQueryConfig`` and never itself raises -- the + # ValidationError below must come from *this* class's __init__, + # reached only via the CLI's own probe step, not from recipe loading. + Config = BigQueryConfig + + def __init__(self, config: object) -> None: + _NestedConfig(x="not-an-int") # raises pydantic_core.ValidationError + + # Sanity-check the test's own premise: this exception type's __str__ + # really does ignore .args, so the test cannot pass for the wrong reason + # (e.g. against a pydantic version that changed this behaviour). + try: + _NestedConfig(x="not-an-int") + except ValidationError as probe_exc: + before = str(probe_exc) + probe_exc.args = ("MUTATED-MARKER",) + assert "MUTATED-MARKER" not in str(probe_exc), ( + "test premise broken: this pydantic version's ValidationError." + "__str__ now reads .args -- the blind spot this test targets no " + "longer exists as designed" + ) + assert str(probe_exc) == before + + interactions = tmp_path / "i.csv" + interactions.write_text("user_id,item_id\nu1,i1\nu2,i2\n") + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + f"""\ +name: probe_validation_error +source: + type: csv + path: {interactions} +schema: + user_column: user_id + item_column: item_id +features: + item: + source: + type: bigquery + query: "SELECT item_id, genre FROM t" + id_column: item_id + columns: + - {{name: genre, encoding: categorical}} +training: + algorithms: [IALS] +output: + path: {tmp_path / "out.recotem"} +""" + ) + + # Only intercept the "bigquery" lookup (the feature source) so the + # top-level "csv" source resolves and probes normally through the real + # registry -- isolating the failure to features.item.source alone. + def _fake_get_source_class(type_name: str) -> type: + if type_name == "bigquery": + return _StubFeatureSource + return _real_get_source_class(type_name) + + with patch( + "recotem.datasource.registry.get_source_class", + MagicMock(side_effect=_fake_get_source_class), + ): + result = runner.invoke(app, ["validate", str(recipe)]) + + combined = result.stdout + (result.stderr or "") + assert result.exit_code != 0, ( + f"validate must exit non-zero when a feature source's __init__ " + f"raises a ValidationError; got {result.exit_code}. Output: {combined!r}" + ) + assert "features.item.source" in combined, ( + "validate must name WHICH source failed even when the raised " + f"exception's __str__ ignores .args; got: {combined!r}" + ) + + # --------------------------------------------------------------------------- # recotem schema # --------------------------------------------------------------------------- @@ -125,6 +325,42 @@ def test_schema_command_emits_valid_jsonschema() -> None: assert "properties" in schema_dict or "title" in schema_dict +def test_schema_emits_discriminated_union_for_feature_sources() -> None: + """features.item.source (via FeatureSideConfig) must be the same + discriminated source union as the top-level ``source`` field, not the + bare ``{"title": "Source"}`` that ``create_model(__base__=Recipe, ...)`` + emits for a nested ``Any``-typed field it cannot override. + + Asserts the actual discriminator mapping is present and lists every + registered built-in source type -- a weaker assertion (e.g. merely + ``"source" in properties``) would also pass against the unfixed bug. + """ + result = runner.invoke(app, ["schema"]) + assert result.exit_code == 0, result.stdout + schema_dict = json.loads(result.stdout) + + defs = schema_dict.get("$defs", {}) + assert "FeatureSideConfig" in defs, sorted(defs.keys()) + side_source = defs["FeatureSideConfig"]["properties"]["source"] + + # Must be a real discriminated union, not a bare {"title": "Source"}. + assert "oneOf" in side_source or "anyOf" in side_source, side_source + assert "discriminator" in side_source, side_source + assert side_source["discriminator"]["propertyName"] == "type" + + mapping = side_source["discriminator"]["mapping"] + for expected_type in ("csv", "parquet", "bigquery", "sql"): + assert expected_type in mapping, ( + f"expected {expected_type!r} in features.item.source discriminator " + f"mapping; got {sorted(mapping.keys())}" + ) + + # The mapping must match the top-level source union exactly (same + # registered types, not a coincidentally-similar unrelated union). + top_level_source = schema_dict["properties"]["source"] + assert mapping == top_level_source["discriminator"]["mapping"] + + # --------------------------------------------------------------------------- # recotem keygen # --------------------------------------------------------------------------- diff --git a/tests/unit/test_config_feature_dim.py b/tests/unit/test_config_feature_dim.py new file mode 100644 index 00000000..6076d70c --- /dev/null +++ b/tests/unit/test_config_feature_dim.py @@ -0,0 +1,33 @@ +"""Tests for RECOTEM_MAX_FEATURE_DIM (recotem._features's dimension cap).""" + +from __future__ import annotations + +import pytest + +from recotem.config import DEFAULT_MAX_FEATURE_DIM, get_max_feature_dim + + +def test_max_feature_dim_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("RECOTEM_MAX_FEATURE_DIM", raising=False) + assert get_max_feature_dim() == DEFAULT_MAX_FEATURE_DIM + assert DEFAULT_MAX_FEATURE_DIM == 5000 + + +def test_max_feature_dim_custom(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", "1234") + assert get_max_feature_dim() == 1234 + + +def test_max_feature_dim_below_min_clamped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", "0") + assert get_max_feature_dim() == 16 + + +def test_max_feature_dim_above_max_clamped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", str(10_000_000)) + assert get_max_feature_dim() == 100_000 + + +def test_max_feature_dim_invalid_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", "not-a-number") + assert get_max_feature_dim() == DEFAULT_MAX_FEATURE_DIM diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py new file mode 100644 index 00000000..4c9dbbb0 --- /dev/null +++ b/tests/unit/test_features.py @@ -0,0 +1,1656 @@ +"""Unit tests for recotem._features, the neutral side-feature encoder. + +Covers: +- build_encoder_state: versioning, plain-data state, dimension cap, missing + source columns, min_frequency vocabulary pruning. +- encode: required index_order reindexing, bias column, missing rows, + standardization, unknown-value handling for each encoding kind. +- encode_one: single-row encoding and unknown-column reporting. +- Parity between encode (DataFrame path) and encode_one (dict path) for + equivalent inputs, including divergent missing-value and numeric-string + representations across the two call paths. +- state_descriptor. +""" + +from __future__ import annotations + +import json +import math +from decimal import Decimal +from fractions import Fraction +from typing import Any + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from recotem._features import ( + FEATURE_STATE_VERSION, + FeatureEncodeError, + _parse_number, + build_encoder_state, + encode, + encode_one, + state_descriptor, +) +from recotem.recipe.models import FeatureColumn + + +@pytest.fixture +def df() -> pd.DataFrame: + return pd.DataFrame( + { + "item_id": ["i1", "i2", "i3"], + "genre": ["action", "drama", "action"], + "year": [2000.0, 2010.0, 2020.0], + "tags": ["a|b", "b", None], + } + ).set_index("item_id") + + +@pytest.fixture +def columns() -> list[FeatureColumn]: + return [ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="year", encoding="numerical"), + FeatureColumn(name="tags", encoding="multi_label", delimiter="|"), + ] + + +def test_state_is_versioned_and_json_shaped( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + assert state["version"] == FEATURE_STATE_VERSION + # 2 genres + 1 year + 2 tags + 1 bias + assert state["n_features"] == 6 + + +def test_state_contains_no_pandas_objects( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """The state must contain only plain Python containers and numpy arrays. + + The key assertion is ``type(k) is str``, NOT ``isinstance(k, str)``. + ``numpy.str_`` subclasses ``str``, so an ``isinstance`` check (this + test's original form) passes for a ``numpy.str_`` key and cannot detect + the exact leak this test exists to prevent -- see + ``test_vocabulary_keys_are_exactly_str_not_numpy_str`` for why nothing + downstream catches it either. + """ + state = build_encoder_state(df, columns) + + def walk(o: object) -> None: + assert not isinstance(o, pd.Index | pd.Series | pd.DataFrame), o + if isinstance(o, dict): + for k, v in o.items(): + assert type(k) is str, f"{k!r} is {type(k)}, not exactly str" + walk(v) + elif isinstance(o, list | tuple): + for v in o: + walk(v) + + walk(state) + + +def test_vocabulary_keys_are_exactly_str_not_numpy_str() -> None: + """``build_encoder_state``'s ``str()`` coercions are load-bearing on their + own -- the artifact FQCN allow-list does NOT back them up. + + The module docstring used to imply that a stray non-plain object in the + state would be refused at load time. That is true for ``pd.Index`` + (``pandas.core.indexes.base._new_Index`` is genuinely not allow-listed) + but FALSE for ``numpy.str_``, which pickles via + ``numpy._core.multiarray.scalar`` + ``numpy.dtype`` -- both explicitly + allow-listed -- and so round-trips through ``SafeUnpickler`` keeping its + type. Nothing downstream catches it either: ``numpy.str_`` hashes and + compares equal to ``str``, so vocabulary lookups keep working and the + leak is invisible at runtime. + + This test is therefore the only enforcement of the "state is plain data" + invariant for the numpy scalar types, which is exactly why it asserts the + EXACT type. + """ + d = pd.DataFrame( + { + "item_id": ["i1", "i2"], + "genre": pd.Series([np.str_("action"), np.str_("drama")], dtype=object), + "tags": pd.Series([np.str_("a|b"), np.str_("b")], dtype=object), + } + ).set_index("item_id") + # Premise: the cells really are numpy.str_, not plain str. An object-dtype + # column preserves them (a native numpy str-dtype column would normalize + # to plain str on .tolist(), and would not exercise the coercion at all). + assert type(d["genre"].iloc[0]) is np.str_ + assert type(d["tags"].iloc[0]) is np.str_ + + state = build_encoder_state( + d, + [ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="tags", encoding="multi_label", delimiter="|"), + ], + ) + for spec in state["columns"]: + for key in spec["vocab"]: + assert type(key) is str, ( + f"vocab key {key!r} of column {spec['name']!r} is " + f"{type(key)}, not exactly str; the str() coercion in " + f"build_encoder_state is the ONLY thing keeping the state " + f"plain -- the artifact allow-list loads numpy.str_ happily" + ) + + +def test_encode_reindexes_to_requested_order( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + forward = encode(state, df, index_order=["i1", "i2", "i3"]) + reverse = encode(state, df, index_order=["i3", "i2", "i1"]) + assert forward.shape == (3, 6) + np.testing.assert_allclose(forward.toarray()[::-1], reverse.toarray()) + + +def test_encode_appends_bias_column( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + m = encode(state, df, index_order=["i1", "i2", "i3"]).toarray() + np.testing.assert_allclose(m[:, -1], np.ones(3)) + + +def test_missing_row_is_all_zero_except_bias( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + m = encode(state, df, index_order=["i1", "absent"]).toarray() + np.testing.assert_allclose(m[1, :-1], np.zeros(5)) + assert m[1, -1] == 1.0 + + +def test_encode_handles_duplicate_id_in_index_order( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A repeated id in ``index_order`` must produce the same row each time + it appears, not raise. + + Regression guard for the row-count optimization in ``encode``: an + earlier draft narrowed the lookup via ``frame.reindex(index_order)``, + which raises ``ValueError`` from the subsequent + ``to_dict(orient="index")`` whenever *index_order* itself contains a + duplicate (reindexing onto a repeated label yields a non-unique result + index). No production caller currently repeats an id in ``index_order``, + but ``encode`` is a general-purpose neutral function and must not crash + if one ever does. + """ + state = build_encoder_state(df, columns) + m = encode(state, df, index_order=["i1", "i1", "i2"]).toarray() + np.testing.assert_allclose(m[0], m[1]) + + +def test_encode_does_not_upcast_int_dtype_column_when_reindexing() -> None: + """A non-nullable-int-dtype ``categorical`` column must encode a + present row identically whether or not *index_order* also asks for an + id absent from the table. + + Regression guard: narrowing the lookup via ``frame.reindex(index_order)`` + fills every id absent from the table with NaN, which forces pandas to + upcast an int64 column to float64 for EVERY row (present ones + included) -- turning ``1`` into ``1.0``. The categorical branch keys the + vocabulary by ``str(raw)``, so ``"1"`` (built at train time from the + original int64 column) no longer matches ``"1.0"`` (read back after the + upcast), silently degrading a known category to unknown. + """ + d = pd.DataFrame({"item_id": ["i1", "i2"], "genre": [1, 2]}).set_index("item_id") + assert d["genre"].dtype == np.int64 + state = build_encoder_state( + d, [FeatureColumn(name="genre", encoding="categorical")] + ) + + # No absent id requested: nothing to regress against even on the buggy + # reindex-based implementation, but pins the base case. + m_clean = encode(state, d, index_order=["i1", "i2"]).toarray() + + # An absent id ("ghost") IS requested alongside a present one: this is + # the case that silently broke under `frame.reindex(index_order)`. + m_with_absent = encode(state, d, index_order=["i1", "ghost", "i2"]).toarray() + + genre_spec = state["columns"][0] + lo, hi = genre_spec["offset"], genre_spec["offset"] + genre_spec["width"] + # "i1" (genre=1) must one-hot to vocab index 0 in both cases. + np.testing.assert_allclose(m_clean[0, lo:hi], m_with_absent[0, lo:hi]) + assert m_with_absent[0, lo:hi].sum() == 1.0, ( + "known int-valued category must not degrade to unknown (all-zero) " + "just because index_order also contains an absent id" + ) + + +def test_multi_label_partial_unknown_keeps_known_tokens( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + probe = pd.DataFrame( + {"item_id": ["x"], "genre": ["action"], "year": [2000.0], "tags": ["a|zzz"]} + ).set_index("item_id") + m = encode(state, probe, index_order=["x"]).toarray() + tag_slice = state["columns"][2] + lo, hi = tag_slice["offset"], tag_slice["offset"] + tag_slice["width"] + # 'a' known -> 1; 'zzz' unknown -> dropped, not an all-zero segment. + assert m[0, lo:hi].sum() == 1.0 + + +def test_categorical_unknown_is_all_zero( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + probe = pd.DataFrame( + {"item_id": ["x"], "genre": ["comedy"], "year": [2000.0], "tags": ["a"]} + ).set_index("item_id") + m = encode(state, probe, index_order=["x"]).toarray() + g = state["columns"][0] + assert m[0, g["offset"] : g["offset"] + g["width"]].sum() == 0.0 + + +def test_numerical_is_standardized( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + m = encode(state, df, index_order=["i1", "i2", "i3"]).toarray() + y = state["columns"][1] + col = m[:, y["offset"]] + assert abs(col.mean()) < 1e-6 + assert abs(col.std() - 1.0) < 1e-6 + + +def test_zero_variance_numerical_emits_zeros() -> None: + d = pd.DataFrame({"item_id": ["i1", "i2"], "year": [5.0, 5.0]}).set_index("item_id") + state = build_encoder_state(d, [FeatureColumn(name="year", encoding="numerical")]) + m = encode(state, d, index_order=["i1", "i2"]).toarray() + np.testing.assert_allclose(m[:, 0], np.zeros(2)) + + +# --------------------------------------------------------------------------- +# Review finding: a column need not be EXACTLY constant (std == 0.0) to +# behave like one. Floating-point rounding noise (values that are "the same +# number" up to a few ULPs) survives an exact std == 0.0 check but still +# divides serve-time standardization by a near-zero denominator, turning an +# ordinary raw request value into an astronomically large standardized one +# -- the mechanism behind a false FEATURE_VALUE_UNUSABLE 400 for a value that +# is not, in any meaningful sense, extreme. +# --------------------------------------------------------------------------- + + +def test_near_constant_numerical_std_is_floored_to_zero() -> None: + """A column whose values differ only by ULP-scale floating-point noise + (std ~1e-15, not exactly 0.0) must be treated as zero-variance, exactly + like ``test_zero_variance_numerical_emits_zeros`` above -- otherwise + standardizing an ordinary request value against a near-zero std produces + an astronomically large standardized value. + """ + base = 5.0 + values = [ + base, + np.nextafter(base, np.inf), + np.nextafter(base, -np.inf), + np.nextafter(np.nextafter(base, np.inf), np.inf), + ] + d = pd.DataFrame({"item_id": ["i1", "i2", "i3", "i4"], "year": values}).set_index( + "item_id" + ) + + # Test-setup invariant: the raw std must be genuinely nonzero (not + # exactly 0.0, which the pre-existing `std == 0.0` check already + # handled) but tiny -- ULP-scale -- or this test would not exercise the + # floor at all. + raw_std = float(pd.Series(values).std(ddof=0)) + assert 0.0 < raw_std < 1e-10, ( + f"test setup invariant violated: need a genuinely nonzero, " + f"ULP-scale std; got {raw_std!r}" + ) + + state = build_encoder_state(d, [FeatureColumn(name="year", encoding="numerical")]) + assert state["columns"][0]["std"] == 0.0, ( + f"a std of {raw_std!r} (relative to a column scale of {base!r}) is " + f"well below the relative floor and must be treated as zero-variance" + ) + + # An ordinary-looking request value must standardize to 0 (degrade like + # a missing value), not explode. + m, unknown = encode_one(state, {"year": 10000.0}) + assert m.toarray()[0, 0] == 0.0 + assert unknown == [] + + +def test_small_but_real_variance_numerical_std_is_not_floored() -> None: + """A column with genuine (not rounding-noise) small variance, comfortably + above the relative floor, must still standardize normally -- the floor + must not swallow real variance just because it is small. + """ + d = pd.DataFrame( + {"item_id": ["i1", "i2", "i3"], "year": [4.999, 5.0, 5.001]} + ).set_index("item_id") + raw_std = float(pd.Series([4.999, 5.0, 5.001]).std(ddof=0)) + assert raw_std > 1e-4, ( + f"test setup invariant violated: need a std well above the " + f"relative floor (1e-8 * scale); got {raw_std!r}" + ) + state = build_encoder_state(d, [FeatureColumn(name="year", encoding="numerical")]) + assert state["columns"][0]["std"] == pytest.approx(raw_std) + + +def test_numerical_missing_becomes_mean( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + probe = pd.DataFrame( + {"item_id": ["x"], "genre": ["action"], "year": [np.nan], "tags": ["a"]} + ).set_index("item_id") + m = encode(state, probe, index_order=["x"]).toarray() + y = state["columns"][1] + assert m[0, y["offset"]] == 0.0 # standardized mean == 0 + + +# --------------------------------------------------------------------------- +# Review finding: a directly-supplied non-finite numerical value (+-inf, or a +# NaN reached via a string) was a silent no-op -- byte-identical to omitting +# the column entirely, with no `unknown` entry and no counter fired. This is +# exactly the failure mode encode_one's own docstring says must not happen: +# "an unknown category degrades the recommendation silently, so it must not +# also be invisible." A *missing* (None/NaN/pd.NA) or *unparseable* +# (non-numeric string) value is a separate, deliberately uncounted gap and +# must stay that way -- these tests pin both sides of that distinction. +# --------------------------------------------------------------------------- + + +def test_encode_one_reports_unknown_for_infinite_numerical_value( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """+inf and -inf must both fire the unknown-value signal, and must not + contribute to the standardized column (same zero contribution as + before -- only the signal is new).""" + state = build_encoder_state(df, columns) + y = state["columns"][1] + + m_pos, unknown_pos = encode_one( + state, {"genre": "action", "year": float("inf"), "tags": "a"} + ) + assert unknown_pos == ["year"] + assert m_pos.toarray()[0, y["offset"]] == 0.0 + + m_neg, unknown_neg = encode_one( + state, {"genre": "action", "year": float("-inf"), "tags": "a"} + ) + assert unknown_neg == ["year"] + assert m_neg.toarray()[0, y["offset"]] == 0.0 + + +def test_encode_one_infinite_numerical_value_is_identical_to_omitted_column( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """Pre-fix, a supplied +inf was byte-identical to omitting the column + entirely -- this pins that the encoded MATRIX stays identical (dropping + an unusable value must still degrade the same way), while the `unknown` + signal now differs (the whole point of the fix).""" + state = build_encoder_state(df, columns) + + m_inf, unknown_inf = encode_one( + state, {"genre": "action", "year": float("inf"), "tags": "a"} + ) + m_omitted, unknown_omitted = encode_one(state, {"genre": "action", "tags": "a"}) + np.testing.assert_allclose(m_inf.toarray(), m_omitted.toarray()) + assert unknown_inf == ["year"] + assert unknown_omitted == [], ( + "omitting the column entirely must NOT report it as unknown -- " + "'missing' and 'non-finite' are different, deliberately different, " + "signals" + ) + + +def test_encode_one_missing_numerical_value_does_not_report_unknown( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A genuinely missing numerical value (None) must stay uncounted -- + this deliberate gap is separate from the non-finite fix above and must + not be widened by it.""" + state = build_encoder_state(df, columns) + _, unknown = encode_one(state, {"genre": "action", "year": None, "tags": "a"}) + assert unknown == [] + + +def test_encode_one_unparseable_numerical_string_does_not_report_unknown( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A numerical value that fails to parse as a number at all (e.g. a + non-numeric string) must stay uncounted -- also a deliberate, + unaffected gap, distinct from the non-finite case above where + ``float()`` succeeds but the result cannot be standardized.""" + state = build_encoder_state(df, columns) + _, unknown = encode_one( + state, {"genre": "action", "year": "not-a-number", "tags": "a"} + ) + assert unknown == [] + + +# --------------------------------------------------------------------------- +# Review finding (MERGE BLOCKER): `float()` raises OverflowError -- an +# ArithmeticError, NOT a ValueError -- for an int above float64's max, so it +# escaped `_row_values`'s `except (TypeError, ValueError)`, escaped +# `encode_one`, escaped `_idmap`'s RuntimeError-only cold-start catch, escaped +# `routes.py`'s ColdStartNumericalError/ValueError catch, and reached the +# generic HTTP 500 handler. Any JSON integer literal of >=309 digits triggers +# it with nothing but a valid API key. +# +# Note the asymmetry that made this subtle: the STRING "1e400" and the FLOAT +# literal 1e309 both yield `inf` and were already handled (counted unknown, +# HTTP 200). Only the INTEGER token raises. +# --------------------------------------------------------------------------- + + +def test_encode_one_reports_unknown_for_oversized_integer( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """An integer too large for float64 must be counted as unknown, exactly + like the +-inf it conceptually is -- not raise, and not silently degrade + to the column mean.""" + state = build_encoder_state(df, columns) + y = state["columns"][1] + huge = 10**309 + + # Test-setup invariant: this is the token that raises, and it raises + # OverflowError specifically -- NOT the ValueError the pre-fix except + # clause was written for. If a future Python makes float() return inf + # here instead, this test would silently stop covering the 500 path. + with pytest.raises(OverflowError): + float(huge) + + m, unknown = encode_one(state, {"genre": "action", "year": huge, "tags": "a"}) + assert unknown == ["year"], ( + "an oversized integer must fire the unknown-value signal, like +-inf" + ) + assert m.toarray()[0, y["offset"]] == 0.0 + + +def test_encode_one_reports_unknown_for_oversized_negative_integer( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """The negative side of the same token must behave identically -- it is + conceptually -inf, and -inf is already a counted unknown.""" + state = build_encoder_state(df, columns) + y = state["columns"][1] + m, unknown = encode_one(state, {"genre": "action", "year": -(10**309), "tags": "a"}) + assert unknown == ["year"] + assert m.toarray()[0, y["offset"]] == 0.0 + + +def test_oversized_integer_arrives_as_a_plain_json_integer_literal( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """Pins the REACHABILITY of the 500, not just the arithmetic. + + A >=309-digit integer is a valid JSON number literal; Python's json + parser yields an arbitrary-precision `int` for it, and pydantic's + `dict[str, Any]` passes that through uncoerced -- so the value reaching + `_row_values` really is a Python int, with no float conversion anywhere + in between to blunt it. This test builds the value the way a client + actually would rather than writing `10**309` by hand. + """ + payload = json.loads('{"year": ' + "9" * 400 + "}") + assert type(payload["year"]) is int, ( + "premise: a long JSON integer literal must parse to a Python int, " + "not a float -- otherwise this attack surface would not exist" + ) + state = build_encoder_state(df, columns) + _, unknown = encode_one(state, {"genre": "action", **payload, "tags": "a"}) + assert unknown == ["year"] + + +# --------------------------------------------------------------------------- +# Fix B: the non-finite guard tested the RAW parsed value (`num`), but the +# matrix stores `scaled = (num - mean) / std` cast to float32. A value finite +# as float64 whose STANDARDIZED magnitude exceeds float32's max (~3.4e38) +# becomes +/-inf when the matrix is cast to float32 and was NOT appended to +# `unknown` -- violating the branch's own contract ("an unknown value must not +# also be invisible"). The right variable to check is `scaled`, not `num`. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("raw", "why"), + [ + (1e39, "float finite in float64 but standardized magnitude > float32 max"), + (1e300, "large float, standardizes far above float32 max"), + ("1e39", "same magnitude arriving as a numeric string"), + ], +) +def test_encode_one_reports_unknown_for_float32_overflow_value( + raw: object, why: str +) -> None: + """A value that parses finite in float64 but standardizes to a magnitude + float32 cannot hold must be counted as unknown, and must NOT inject a + non-finite value into the matrix -- pre-fix it was a silent `inf` with + `unknown == []`.""" + # A state fitted to mean ~ 0 / std ~ 1, so a raw value that is finite in + # float64 (1e39) still standardizes above float32's max. + d = pd.DataFrame({"item_id": ["i1", "i2", "i3"], "x": [-1.0, 0.0, 1.0]}).set_index( + "item_id" + ) + state = build_encoder_state(d, [FeatureColumn(name="x", encoding="numerical")]) + spec = state["columns"][0] + # Premise: mean ~ 0, std ~ 0.816 -- so 1e39 standardizes to ~1.2e39, which + # is finite in float64 but overflows float32 (~3.4e38). + assert abs(spec["mean"]) < 1e-9, f"premise ({why})" + assert spec["std"] == pytest.approx(0.816, abs=1e-2), f"premise ({why})" + + m, unknown = encode_one(state, {"x": raw}) + assert unknown == ["x"], f"{raw!r} must fire the unknown-value signal ({why})" + # The matrix must carry NO non-finite value -- only the bias 1.0. + assert all(np.isfinite(v) for v in m.data), ( + f"{raw!r} injected a non-finite value into the matrix ({list(m.data)}) -- {why}" + ) + # And the standardized column contributes nothing, like an omitted value. + assert m.toarray()[0, spec["offset"]] == 0.0 + + +# --------------------------------------------------------------------------- +# Review finding: `build_encoder_state` fits mean/std with +# `pd.to_numeric(..., errors="coerce")` but `_row_values` parsed the raw value +# with `float()`. The two do not accept the same strings, and `float()` is the +# LOOSER of the pair -- so a value pandas silently dropped from a column's own +# statistics could still be encoded against those statistics. +# +# The invariant these tests pin: a value the FITTING parser could not use must +# not be encodable by the ENCODING parser. +# --------------------------------------------------------------------------- + + +def test_value_excluded_from_statistics_is_not_encodable() -> None: + """The headline divergence, end to end. + + ``"1_000"`` is accepted by ``float()`` (PEP 515 underscores) but coerced + to NaN by ``pd.to_numeric``. So the column fits mean=6.0/std=1.0 from + {5, 7} ALONE -- the "1_000" row is excluded from its own column's + statistics -- and then pre-fix encoded that same row to (1000-6)/1 == + 994.0: a 994-sigma value the fitted statistics never saw, with no + warning and no counter. + """ + d = pd.DataFrame( + {"item_id": ["a", "b", "c"], "price": ["1_000", "5", "7"]} + ).set_index("item_id") + state = build_encoder_state(d, [FeatureColumn(name="price", encoding="numerical")]) + spec = state["columns"][0] + + # Premise: pandas really did exclude "1_000" from the column's own stats. + assert spec["mean"] == pytest.approx(6.0), ( + "premise: the fitting parser must have coerced '1_000' to NaN and " + "fitted from {5, 7} only" + ) + assert spec["std"] == pytest.approx(1.0) + + m = encode(state, d, index_order=["a"]).toarray() + assert m[0, spec["offset"]] == 0.0, ( + "a value the fitting parser coerced to NaN must not be encodable; " + "pre-fix this was 994.0" + ) + + +@pytest.mark.parametrize( + ("raw", "why"), + [ + ("1_000", "PEP 515 underscores: float() accepts, to_numeric rejects"), + ("１２３", "full-width digits (common in Japanese CSV exports)"), + ("١٢٣", "Arabic-Indic digits"), + ], +) +def test_encode_one_rejects_values_the_fitting_parser_cannot_use( + df: pd.DataFrame, columns: list[FeatureColumn], raw: str, why: str +) -> None: + """Each string form that ``float()`` accepts but ``pd.to_numeric`` coerces + to NaN must join the documented unparseable path on the request side too: + encoded identically to omitting the column, and (like every other + unparseable value) deliberately uncounted.""" + # Premise: this really is a divergence between the two parsers, or the + # case proves nothing. + assert isinstance(float(raw), float), why + assert pd.isna(pd.to_numeric(pd.Series([raw], dtype=object), errors="coerce")[0]), ( + f"premise for {raw!r}: the fitting parser must coerce it to NaN" + ) + + state = build_encoder_state(df, columns) + m, unknown = encode_one(state, {"genre": "action", "year": raw, "tags": "a"}) + m_omitted, _ = encode_one(state, {"genre": "action", "tags": "a"}) + np.testing.assert_allclose(m.toarray(), m_omitted.toarray()) + assert unknown == [] + + +def test_ascii_numeric_strings_are_still_accepted( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """The tightening must not overshoot: ordinary ASCII numeric strings -- + including signs, exponents and surrounding whitespace, all of which the + fitting parser accepts -- must still encode normally.""" + state = build_encoder_state(df, columns) + y = state["columns"][1] + baseline, _ = encode_one(state, {"genre": "action", "year": 2000.0, "tags": "a"}) + expected = baseline.toarray()[0, y["offset"]] + assert expected != 0.0, "premise: 2000.0 must standardize to a nonzero value" + + for raw in ["2000", " 2000 ", "+2000", "2e3", "2000.0"]: + m, unknown = encode_one(state, {"genre": "action", "year": raw, "tags": "a"}) + assert m.toarray()[0, y["offset"]] == pytest.approx(expected), ( + f"{raw!r} is accepted by the fitting parser and must stay encodable" + ) + assert unknown == [] + + +# --------------------------------------------------------------------------- +# Review finding: the parity gate above was `str`-only, but `_parse_number`'s +# domain is `Any`. `float()` also parses `bytes`, `bytearray`, `memoryview` and +# any object with `__float__` / `__index__`, while `pd.to_numeric` NaNs all of +# those EXCEPT `bytes` (which it parses as text, with the same grammar it +# applies to `str`). So the "1_000" worked example reproduced verbatim one type +# over: a BYTES column (a SQL BLOB / parquet binary column declared +# `numerical`) of [b"1_000", b"5", b"7"] fitted mean=6.0/std=1.0 from {5, 7} +# ALONE and then encoded the b"1_000" row to 994.0. +# +# JSON cannot carry any of these types, so this is a TRAINING-path bug -- which +# is why these tests drive `encode`, not only `encode_one`. That is the level +# at which the bool regression hid. +# +# Both directions are pinned below, because tightening only direction 1 is how +# the bool regression happened: +# 1. a value the FITTING parser could not use must not be encodable, and +# 2. a value the FITTING parser DID use must stay encodable. +# --------------------------------------------------------------------------- + + +class _HasFloat: + """An object `float()` accepts and `pd.to_numeric` NaNs.""" + + def __float__(self) -> float: + return 1000.0 + + +class _HasIndex: + """`float()` accepts `__index__` too; `pd.to_numeric` still NaNs it.""" + + def __index__(self) -> int: + return 1000 + + +@pytest.mark.parametrize( + ("poison", "why"), + [ + (b"1_000", "bytes: float() honours PEP 515 underscores, to_numeric does not"), + (bytearray(b"1000"), "bytearray: float() parses it, to_numeric NaNs it"), + (memoryview(b"1000"), "memoryview: float() parses it, to_numeric NaNs it"), + (Fraction(1000, 1), "Fraction: IS a numbers.Number, yet to_numeric NaNs it"), + (_HasFloat(), "custom __float__: float() calls it, to_numeric NaNs it"), + (_HasIndex(), "custom __index__: float() uses it, to_numeric NaNs it"), + ], +) +def test_non_str_value_excluded_from_statistics_is_not_encodable( + poison: object, why: str +) -> None: + """The 994-sigma bug, reproduced across the whole non-``str`` domain. + + Exactly the shape of ``test_value_excluded_from_statistics_is_not_encodable`` + above, one type over: each ``poison`` is a value ``pd.to_numeric`` coerces + to NaN -- so the column fits mean=6.0/std=1.0 from {5, 7} ALONE -- but that + ``float()`` happily turns into 1000.0, encoding the excluded row to + (1000-6)/1 == 994.0 against statistics that never saw it. + + Drives ``encode`` (the training path), because none of these types can + arrive over JSON. + """ + d = pd.DataFrame( + { + "item_id": ["a", "b", "c"], + "price": pd.Series([poison, "5", "7"], dtype=object), + } + ).set_index("item_id") + state = build_encoder_state(d, [FeatureColumn(name="price", encoding="numerical")]) + spec = state["columns"][0] + + # Premise: the fitting parser really did exclude the poison value from the + # column's own statistics, and really did fit from {5, 7}. Without this the + # case proves nothing. + assert spec["mean"] == pytest.approx(6.0), f"premise for {why}" + assert spec["std"] == pytest.approx(1.0), f"premise for {why}" + # Premise: `float()` -- the encoding parser -- does accept it. This is the + # divergence itself; if a future Python/numpy stops accepting it, this case + # would silently stop covering anything. + assert float(poison) == 1000.0, f"premise for {why}" # type: ignore[arg-type] + + m = encode(state, d, index_order=["a"]).toarray() + assert m[0, spec["offset"]] == 0.0, ( + f"a value the fitting parser coerced to NaN must not be encodable " + f"({why}); pre-fix this encoded to 994.0" + ) + + +@pytest.mark.parametrize( + ("value", "why"), + [ + (b"1000", "bytes ARE text to to_numeric: it parses b'5' -> 5 (measured)"), + (np.bytes_(b"1000"), "np.bytes_ subclasses bytes and parses the same"), + ( + Decimal("1000"), + "a SQL NUMERIC/DECIMAL column yields Decimal; the fit uses it", + ), + (np.float32(1000.0), "numpy scalars reach encode via an object-dtype column"), + (np.int64(1000), "same for the integer scalars"), + ], +) +def test_non_str_value_the_fitting_parser_used_stays_encodable( + value: object, why: str +) -> None: + """Direction 2: refusing a value the FIT used is the bool regression again. + + ``_row_values`` is shared by ``encode`` and ``encode_one``, so a rule that + rejects one of these silently encodes 0.0 for every TRAINING row of the + column, against perfectly healthy statistics, with no warning able to fire + (``feature_zero_variance_column`` cannot see a healthy std). This is why the + fix is an allow-list measured against ``to_numeric`` rather than a + convenient ``isinstance(raw, numbers.Number)`` -- which would MISS + ``np.bool_`` and ``bytes`` and land exactly here. + """ + d = pd.DataFrame( + { + "item_id": ["a", "b", "c"], + "price": pd.Series([value, 5.0, 7.0], dtype=object), + } + ).set_index("item_id") + # Premise: the object dtype really preserved the exotic type (a native + # dtype would normalize it to a plain float and prove nothing). + assert type(d["price"].iloc[0]) is type(value) + + state = build_encoder_state(d, [FeatureColumn(name="price", encoding="numerical")]) + spec = state["columns"][0] + + # Premise: the fitting parser USED the value, so the statistics include it. + assert spec["mean"] == pytest.approx((1000.0 + 5.0 + 7.0) / 3.0), ( + f"premise: to_numeric must have fitted FROM the value ({why})" + ) + + expected = (1000.0 - spec["mean"]) / spec["std"] + assert expected != 0.0, "premise: the value must standardize to something visible" + + m = encode(state, d, index_order=["a"]).toarray() + assert m[0, spec["offset"]] == pytest.approx(expected), ( + f"the fit used this value, so encode must too, or the column is " + f"silently dead at training time ({why})" + ) + + +def test_numpy_bool_object_column_encodes_real_values_at_training_time() -> None: + """REGRESSION guard, ``np.bool_`` edition. + + ``test_bool_column_encodes_real_values_at_training_time`` covers a native + bool-dtype column, whose cells arrive at ``_row_values`` as plain Python + ``bool`` (measured). An OBJECT-dtype column preserves ``np.bool_`` instead + -- and ``np.bool_`` is NOT registered with ``numbers.Number`` (measured), + while ``pd.to_numeric`` uses it exactly like a bool. So the tidy-looking + ``isinstance(raw, numbers.Number)`` gate would zero this column at training + time: the bool regression, one type over. + """ + d = pd.DataFrame( + { + "item_id": ["i1", "i2", "i3", "i4"], + "in_stock": pd.Series( + [np.bool_(True), np.bool_(False), np.bool_(True), np.bool_(False)], + dtype=object, + ), + } + ).set_index("item_id") + # Premise: the cells really are np.bool_, and really are not numbers.Number. + import numbers + + assert type(d["in_stock"].iloc[0]) is np.bool_ + assert not isinstance(np.bool_(True), numbers.Number) + + state = build_encoder_state( + d, [FeatureColumn(name="in_stock", encoding="numerical")] + ) + spec = state["columns"][0] + assert (spec["mean"], spec["std"]) == (0.5, 0.5), ( + "premise: to_numeric must fit mean/std from the np.bool_ values themselves" + ) + + m = encode(state, d, ["i1", "i2", "i3", "i4"]) + np.testing.assert_allclose( + m.toarray()[:, spec["offset"]], + [1.0, -1.0, 1.0, -1.0], + err_msg="an np.bool_ column declared `numerical` must standardize to +-1.0", + ) + + +# --- differential fuzz: _parse_number vs. the fitting parser --------------- +# +# The gate's original justification cited "differential fuzzing over 20k +# numeric-flavored inputs" -- but that fuzz was str-only AND was never +# committed, so nothing re-ran it and nothing could have caught the non-str +# hole above. These two properties are the committed replacement. + + +def _fitting_parser(value: object) -> float | None: + """What ``build_encoder_state`` would fit from *value*, or None if it can't. + + Mirrors the ``pd.to_numeric(series, errors="coerce")`` call in + ``build_encoder_state`` exactly -- one object-dtype cell at a time. + """ + try: + fitted = pd.to_numeric(pd.Series([value], dtype=object), errors="coerce")[0] + except (OverflowError, TypeError, ValueError): + # `errors="coerce"` does NOT suppress OverflowError for an int above + # float64's max (measured) -- the documented reason _parse_number does + # not simply delegate to to_numeric. Nothing to compare against here. + return None + if pd.isna(fitted): + return None + # Normalize to a plain float: `to_numeric` returns numpy scalars + # (``np.True_`` for a bool, ``np.int64`` for an int), and ``pytest.approx`` + # mishandles ``np.True_`` (``1.0 == approx(np.True_)`` is False). What + # `build_encoder_state` actually fits from is ``float(numeric.mean())``, + # so a plain float is the honest comparison target. + return float(fitted) + + +# Non-text values only: `str`/`bytes` carry a documented residual divergence +# (pandas tolerates an internal space in an exponent, "6E 66" -> 5.99e66, which +# `float()` rejects -- measured), so they get the weaker property below. +_non_text_values = st.one_of( + st.floats(allow_nan=True, allow_infinity=True), + st.integers(), + st.booleans(), + st.decimals(allow_nan=True, allow_infinity=True), + st.fractions(), + st.binary(max_size=8).map(bytearray), + st.binary(max_size=8).map(memoryview), + st.builds(_HasFloat), + st.builds(_HasIndex), + st.datetimes(), + st.just(np.bool_(True)), + st.just(np.bool_(False)), + st.floats(allow_nan=False, allow_infinity=False, width=32).map(np.float32), + st.integers(min_value=-(2**63), max_value=2**63 - 1).map(np.int64), +) + + +@settings( + max_examples=2000, deadline=None, suppress_health_check=[HealthCheck.too_slow] +) +@given(value=_non_text_values) +def test_parse_number_mirrors_the_fitting_parser_for_non_text(value: Any) -> None: + """Full parity, both directions, over the non-text domain. + + Direction 1 (a value the fit dropped must not be encodable) is the + 994-sigma invariant. Direction 2 (a value the fit used must stay encodable) + is the bool/np.bool_/Decimal regression guard. `complex` is excluded from + the strategy on purpose: `to_numeric` keeps a complex, but + `build_encoder_state` then raises TypeError at `float(numeric.mean())`, so + no state is ever built and the invariant is vacuous for it. + """ + fitted = _fitting_parser(value) + parsed = _parse_number(value) + + if fitted is None: + assert parsed is None or not math.isfinite(parsed), ( + f"{value!r} ({type(value).__name__}) was dropped from the column's " + f"statistics by the fitting parser, so it must not encode to a " + f"finite number -- got {parsed!r}" + ) + else: + assert parsed is not None, ( + f"{value!r} ({type(value).__name__}) WAS used by the fitting " + f"parser, so refusing it here silently zeroes the column for every " + f"training row (the bool regression's exact shape)" + ) + if math.isfinite(fitted): + assert parsed == pytest.approx(fitted, rel=1e-12, nan_ok=True) + + +@settings( + max_examples=2000, deadline=None, suppress_health_check=[HealthCheck.too_slow] +) +@given( + value=st.one_of( + st.text(max_size=12), + st.from_regex( + r"[+-]?[0-9_]{1,6}(\.[0-9]{0,4})?([eE][+-]?[0-9]{1,3})?", fullmatch=True + ), + st.binary(max_size=8), + st.from_regex(r"[+-]?[0-9_]{1,6}", fullmatch=True).map(str.encode), + ) +) +def test_parse_number_never_encodes_text_the_fitting_parser_dropped(value: Any) -> None: + """Direction 1 over the text domain (`str` AND `bytes`). + + Only direction 1: the pair has a known, documented residual in the other + direction (pandas tolerates whitespace inside an exponent where `float()` + does not), which makes `encode` STRICTER than the fit -- the safe side of + the invariant, and the same "deliberately uncounted unparseable gap" the + tests above pin. + """ + fitted = _fitting_parser(value) + parsed = _parse_number(value) + if fitted is None: + assert parsed is None or not math.isfinite(parsed), ( + f"{value!r} ({type(value).__name__}) was NaN'd by the fitting " + f"parser, so it must not encode to a finite number -- got {parsed!r}" + ) + + +# --------------------------------------------------------------------------- +# `bool` is a *numerical* value, deliberately. It is an `int` subclass, so +# `float(True)` is 1.0 -- and that is the right answer, because the fitting +# parser (`pd.to_numeric`) uses bools too: a bool-dtype column fits its +# mean/std FROM them. Excluding bool from `_parse_number` was tried once, on +# the theory that a JSON `true` should not be indistinguishable from the +# number 1.0. It was a regression: `_row_values` is shared by `encode` and +# `encode_one`, so the exclusion also fired on every TRAINING row and silently +# zeroed out any bool column declared `numerical`, against healthy statistics +# and with no warning. These tests pin the accepted behaviour on both paths so +# the exclusion cannot come back unnoticed. +# +# The contrast is `check_artifact_feature_version`, which DOES exclude bool +# from its `isinstance(version, int)` check (see test_features_compat.py) -- a +# state version is an identity token with no numeric reading, not a +# measurement. +# --------------------------------------------------------------------------- + + +def test_encode_one_boolean_is_a_number( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A JSON boolean must encode as the number it is, not join the + unparseable path. + + Asserts against the encoding of the equivalent *number* rather than a + hardcoded cell value: the encoded number for 1.0 is not 1.0, it is the + state-dependent standardized `(1.0 - mean) / std`. + """ + state = build_encoder_state(df, columns) + y = state["columns"][1] + m_omitted, _ = encode_one(state, {"genre": "action", "tags": "a"}) + + for raw, equivalent in [(True, 1.0), (False, 0.0)]: + m, unknown = encode_one(state, {"genre": "action", "year": raw, "tags": "a"}) + m_num, _ = encode_one( + state, {"genre": "action", "year": equivalent, "tags": "a"} + ) + cell = m.toarray()[0, y["offset"]] + + # Premise: the equivalent number must standardize to something + # visible, or "encodes like the number" would be trivially true of the + # all-zero unparseable path and prove nothing. + assert m_num.toarray()[0, y["offset"]] != 0.0, ( + f"premise: {equivalent} must standardize to a nonzero value " + f"against this fixture's statistics" + ) + np.testing.assert_allclose( + m.toarray(), + m_num.toarray(), + err_msg=f"{raw!r} must encode exactly like the number {equivalent}", + ) + assert cell != m_omitted.toarray()[0, y["offset"]], ( + f"{raw!r} must not degrade to the omitted-column path" + ) + assert unknown == [] + + +def test_encode_one_true_matches_the_number_one( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """`true` encoding like the number 1.0 is correct, not a collision. + + For a column the recipe declares `numerical`, `true` IS 1.0 -- which is + also what the fitting parser makes of it. This is the assertion that was + previously inverted; it is pinned explicitly because the inverted form + reads plausible in isolation. + """ + state = build_encoder_state(df, columns) + y = state["columns"][1] + m_true, _ = encode_one(state, {"genre": "action", "year": True, "tags": "a"}) + m_one, _ = encode_one(state, {"genre": "action", "year": 1.0, "tags": "a"}) + + cell_one = m_one.toarray()[0, y["offset"]] + # Premise: 1.0 must itself standardize to something visible, or "matches + # 1.0" would be satisfied by two all-zero rows and prove nothing. + assert cell_one != 0.0, ( + "premise: the number 1.0 must standardize to a nonzero value against " + "this fixture's statistics" + ) + assert m_true.toarray()[0, y["offset"]] == cell_one, ( + "JSON `true` must encode as the number 1.0 for a `numerical` column" + ) + + +def test_bool_column_encodes_real_values_at_training_time() -> None: + """REGRESSION: a bool-dtype column declared `numerical` must encode to + real standardized values on the TRAINING path, not zeros. + + This is the coverage that was missing when `_parse_number` excluded bool. + The whole failure was invisible from the serve-side tests: the fit was + healthy (std=0.5, so `feature_zero_variance_column` never fired) and + `feature_encoder_state_built` still listed the column as active, so a + retrain of an existing recipe silently degraded the model. + + Asserts the exact encoded values, not merely "not zero": `(x - 0.5) / 0.5` + is +1.0 for True and -1.0 for False, and the sign carries the meaning. + """ + d = pd.DataFrame( + { + "item_id": ["i1", "i2", "i3", "i4"], + "in_stock": [True, False, True, False], + } + ).set_index("item_id") + assert d["in_stock"].dtype == bool, "premise: pandas must infer a bool dtype" + + cols = [FeatureColumn(name="in_stock", encoding="numerical")] + state = build_encoder_state(d, cols) + spec = state["columns"][0] + + # Premise: the fitting parser used the bools, so the statistics are + # healthy and the column is not zero-variance. + assert (spec["mean"], spec["std"]) == (0.5, 0.5), ( + "premise: pd.to_numeric must fit mean/std from the bools themselves" + ) + + m = encode(state, d, ["i1", "i2", "i3", "i4"]) + np.testing.assert_allclose( + m.toarray()[:, spec["offset"]], + [1.0, -1.0, 1.0, -1.0], + err_msg="a bool column declared `numerical` must standardize to +-1.0", + ) + + +def test_encode_and_encode_one_agree_for_a_bool() -> None: + """PARITY: the shared `_row_values` must give `encode` (training) and + `encode_one` (serve) the same answer for a bool. + + The bool exclusion broke exactly this: it was reasoned about as a + serve-side concern only, but `_row_values` is shared, so it silently + changed training too. A serve-side-only bool rule would break this test by + construction, which is the point. + """ + d = pd.DataFrame( + { + "item_id": ["i1", "i2", "i3", "i4"], + "in_stock": [True, False, True, False], + } + ).set_index("item_id") + state = build_encoder_state( + d, [FeatureColumn(name="in_stock", encoding="numerical")] + ) + + for entity_id, raw in [("i1", True), ("i2", False)]: + from_training = encode(state, d, [entity_id]).toarray() + from_serving, unknown = encode_one(state, {"in_stock": raw}) + np.testing.assert_allclose( + from_training, + from_serving.toarray(), + err_msg=f"encode and encode_one must agree for {raw!r}", + ) + assert unknown == [] + + +# --------------------------------------------------------------------------- +# Review finding: `min_frequency` is `Field(default=1, ge=1)` with no upper +# bound, so `min_frequency: 50` against a 3-row catalog validates happily and +# prunes EVERY token -- the column comes back width=0 and the `features:` +# block becomes a complete no-op, with `feature_encoder_state_built` at INFO +# listing the column as though it were active. The numerical branch already +# warns loudly on the identical "this column contributes nothing" condition +# (`feature_zero_variance_column`); the vocabulary branches did not. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("encoding", ["categorical", "multi_label"]) +def test_min_frequency_that_empties_a_vocabulary_warns(encoding: str) -> None: + """An emptied vocabulary must warn, mirroring feature_zero_variance_column.""" + import structlog.testing + + # Distinctive sentinel values: a single letter like "a" would collide with + # ordinary English prose in the event's own `detail` text, making the PII + # assertion below fail for a reason that has nothing to do with the values. + values = ["zqxsentinel1", "zqxsentinel2", "zqxsentinel3"] + d = pd.DataFrame({"item_id": ["i1", "i2", "i3"], "g": values}).set_index("item_id") + with structlog.testing.capture_logs() as cap: + state = build_encoder_state( + d, [FeatureColumn(name="g", encoding=encoding, min_frequency=50)] + ) + + # Premise: the column really did collapse to nothing. + assert state["columns"][0]["width"] == 0 + assert state["n_features"] == 1 # bias only + + events = [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] + assert events, ( + f"an emptied vocabulary must warn; got events: {[e.get('event') for e in cap]}" + ) + ev = events[0] + assert ev["log_level"] == "warning" + assert ev["column"] == "g" + assert ev["min_frequency"] == 50 + assert ev["distinct_values"] == 3 + # PII rule: column names and counts only -- never the values themselves. + rendered = str(ev) + for value in values: + assert value not in rendered, ( + f"vocabulary values must never be logged; found {value!r} in {ev}" + ) + + +def test_empty_vocabulary_warning_names_every_emptied_column() -> None: + """Each emptied column must be named individually -- the operator has to + know WHICH column is a no-op, not merely that one of them is. + + Updated for the dead-column broadening (see the ``feature_dead_column`` + tests below): ``keep`` must genuinely VARY across rows, or it too would be + a (correctly) warned dead column. A prior single-row form left ``keep`` + constant, which the broadened check now flags -- so this table has two + rows and a varying ``keep`` to isolate the min_frequency-emptied columns. + """ + import structlog.testing + + d = pd.DataFrame( + { + "item_id": ["i1", "i2"], + "genre": ["a", "b"], + "tags": ["x|y", "z"], + "keep": ["k", "m"], + } + ).set_index("item_id") + with structlog.testing.capture_logs() as cap: + build_encoder_state( + d, + [ + FeatureColumn(name="genre", encoding="categorical", min_frequency=9), + FeatureColumn(name="tags", encoding="multi_label", min_frequency=9), + FeatureColumn(name="keep", encoding="categorical"), + ], + ) + warned = { + e["column"] for e in cap if e.get("event") == "feature_empty_vocabulary_column" + } + assert warned == {"genre", "tags"}, ( + f"expected exactly the emptied columns to warn; got {warned}" + ) + + +# --------------------------------------------------------------------------- +# Review finding: the empty-vocabulary warning keyed on "vocab has no entries", +# but the real property is "the encoded one-hot block is byte-identical to the +# bias column for every row" -- i.e. the column contributes nothing. A CONSTANT +# categorical column (``["rock","rock","rock"]``) has a NON-empty vocab +# (``{"rock"}``) yet its one-hot is all-1 for every row, collinear with the +# bias column -- equally dead, but the ``if vocab: return`` early-out skipped +# it. The numerical branch already catches its constant case +# (``feature_zero_variance_column``); this brings categorical / multi_label to +# parity. +# +# The trap: the null-bearing variant ``["rock", None, "rock"]`` -> [1,0,1] IS +# genuinely informative (it distinguishes has-rock from missing) and must stay +# silent. So the property is whether the encoded column VARIES across rows, not +# whether the vocabulary has exactly one entry. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("encoding", "values"), + [ + # Distinctive tokens: a bare "a"/"b" would collide with ordinary prose + # in the warning's own `detail` text ("vocabulary", "collinear"), + # making the PII assertion below fail for an unrelated reason -- the + # same trap test_min_frequency_that_empties_a_vocabulary_warns avoids. + ("categorical", ["zqxrock", "zqxrock", "zqxrock"]), + # multi_label: every row carries the identical token set, so every + # row's multi-hot block is identical -- collinear with bias. + ("multi_label", ["zqxalpha|zqxbeta", "zqxalpha|zqxbeta", "zqxalpha|zqxbeta"]), + ], +) +def test_constant_vocabulary_column_warns_as_dead( + encoding: str, values: list[str] +) -> None: + """A constant (non-empty-vocab) column encodes identically for every row, + so it is as dead as an emptied one and must warn -- the false negative the + ``if vocab: return`` early-out used to let through.""" + import structlog.testing + + d = pd.DataFrame( + {"item_id": [f"i{i}" for i in range(len(values))], "g": values} + ).set_index("item_id") + col = ( + FeatureColumn(name="g", encoding=encoding, delimiter="|") + if (encoding == "multi_label") + else FeatureColumn(name="g", encoding=encoding) + ) + with structlog.testing.capture_logs() as cap: + state = build_encoder_state(d, [col]) + + # Premise: the vocab really is NON-empty (so the old early-out would have + # skipped it) yet the block is constant across rows. + assert state["columns"][0]["vocab"], "premise: this case has a non-empty vocab" + m = encode(state, d, index_order=list(d.index)).toarray() + spec = state["columns"][0] + block = m[:, spec["offset"] : spec["offset"] + spec["width"]] + assert len({tuple(r) for r in block.tolist()}) == 1, ( + "premise: every row's encoded block must be identical (dead column)" + ) + + events = [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] + assert events, ( + f"a constant (non-empty-vocab) column must warn as dead; " + f"got events {[e.get('event') for e in cap]}" + ) + assert events[0]["column"] == "g" + # PII rule: never the token values themselves. + rendered = str(events[0]) + for v in set(values): + for tok in v.split("|"): + assert tok not in rendered, f"dead-column warning must not log {tok!r}" + + +@pytest.mark.parametrize( + ("encoding", "values"), + [ + # [1,0,1]: has-rock vs missing is real signal, must stay silent. + ("categorical", ["rock", None, "rock"]), + # one row {a,b}, one row {b}: the blocks differ -> informative. + ("multi_label", ["a|b", "b", "a|b"]), + ], +) +def test_null_bearing_or_varying_vocabulary_column_does_not_warn( + encoding: str, values: list[str | None] +) -> None: + """The trap: a null-bearing / partially-covering column has a one-entry (or + small) vocab but its encoded block VARIES across rows, so it is genuinely + informative and must NOT be warned as dead -- exactly the property the + numerical branch expresses with ``std``.""" + import structlog.testing + + d = pd.DataFrame( + {"item_id": [f"i{i}" for i in range(len(values))], "g": values} + ).set_index("item_id") + col = ( + FeatureColumn(name="g", encoding=encoding, delimiter="|") + if (encoding == "multi_label") + else FeatureColumn(name="g", encoding=encoding) + ) + with structlog.testing.capture_logs() as cap: + state = build_encoder_state(d, [col]) + # Premise: the block really does vary across rows. + m = encode(state, d, index_order=list(d.index)).toarray() + spec = state["columns"][0] + block = m[:, spec["offset"] : spec["offset"] + spec["width"]] + assert len({tuple(r) for r in block.tolist()}) > 1, ( + "premise: an informative column's block must differ across rows" + ) + assert not [ + e for e in cap if e.get("event") == "feature_empty_vocabulary_column" + ], "an informative (varying) column must not be warned as dead" + + +def test_healthy_vocabulary_does_not_warn() -> None: + """The warning must not cry wolf on a normal column.""" + import structlog.testing + + d = pd.DataFrame({"item_id": ["i1", "i2"], "g": ["a", "b"]}).set_index("item_id") + with structlog.testing.capture_logs() as cap: + build_encoder_state(d, [FeatureColumn(name="g", encoding="categorical")]) + assert not [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] + + +def test_all_null_column_warns_as_empty_vocabulary() -> None: + """A column with no usable values at all is the same "contributes + nothing" condition, reached by a different route than min_frequency, and + must warn too.""" + import structlog.testing + + d = pd.DataFrame({"item_id": ["i1", "i2"], "g": [None, None]}).set_index("item_id") + with structlog.testing.capture_logs() as cap: + build_encoder_state(d, [FeatureColumn(name="g", encoding="categorical")]) + events = [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] + assert events, "an all-null vocabulary column must warn" + assert events[0]["distinct_values"] == 0 + + +def test_min_frequency_prunes_vocabulary() -> None: + d = pd.DataFrame( + {"item_id": ["i1", "i2", "i3"], "g": ["a", "a", "rare"]} + ).set_index("item_id") + state = build_encoder_state( + d, [FeatureColumn(name="g", encoding="categorical", min_frequency=2)] + ) + assert state["columns"][0]["vocab"] == {"a": 0} + assert state["n_features"] == 2 # 'a' + bias + + +def test_min_frequency_counts_multi_label_occurrences_not_rows() -> None: + """``min_frequency`` is a row count for ``categorical`` but an + *occurrence* count for ``multi_label``: a single row's repeated tokens + all count. ``tags="a|a|a"`` in ONE row must satisfy ``min_frequency=2`` + and keep ``a`` -- pins docs/recipe-reference.md's documented semantics + and guards against reverting to a row-count model for this encoding. + """ + d = pd.DataFrame({"item_id": ["i1"], "tags": ["a|a|a"]}).set_index("item_id") + state = build_encoder_state( + d, + [FeatureColumn(name="tags", encoding="multi_label", min_frequency=2)], + ) + assert state["columns"][0]["vocab"] == {"a": 0} + assert state["n_features"] == 2 # 'a' + bias + + +def test_dimension_cap_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", "16") + d = pd.DataFrame( + {"item_id": [f"i{i}" for i in range(40)], "g": [f"c{i}" for i in range(40)]} + ).set_index("item_id") + with pytest.raises(FeatureEncodeError, match="exceeds"): + build_encoder_state(d, [FeatureColumn(name="g", encoding="categorical")]) + + +def test_missing_source_column_raises(df: pd.DataFrame) -> None: + with pytest.raises(FeatureEncodeError, match="not present"): + build_encoder_state(df, [FeatureColumn(name="nope", encoding="categorical")]) + + +# --------------------------------------------------------------------------- +# Fix D: build_encoder_state's numerical branch had two escape hatches. +# (1) `pd.to_numeric(..., errors="coerce")` does NOT suppress OverflowError for +# an object-dtype Python int above float64's max (>=309 digits) -- it +# escaped `_fetch_side`'s FeatureEncodeError-only catch and surfaced as +# exit 1 (unmapped) instead of the documented exit 4. +# (2) A complex column: `to_numeric` keeps complex dtype and (under numpy 2.x) +# `float(np.complex128)` silently discards the imaginary part +# (ComplexWarning, not TypeError), so a complex feature column trained on +# its real part with no error. Both must raise FeatureEncodeError naming +# the column. +# --------------------------------------------------------------------------- + + +def test_build_encoder_state_oversized_int_column_raises_feature_encode_error() -> None: + """An object-dtype numerical column carrying a Python int too large for + float64 must raise FeatureEncodeError (which _fetch_side maps to exit 4), + not the raw OverflowError that escaped to exit 1.""" + big = int("1" + "0" * 309) + # Premise: this really is the token that makes the FIT's own parser raise + # OverflowError despite errors="coerce" -- an ArithmeticError, so it + # escaped every ValueError/TypeError-shaped catch on the path. + with pytest.raises(OverflowError): + pd.to_numeric(pd.Series([big], dtype=object), errors="coerce") + + d = pd.DataFrame( + {"item_id": ["a", "b", "c"], "amount": pd.Series([big, 5, 7], dtype=object)} + ).set_index("item_id") + with pytest.raises(FeatureEncodeError, match="amount"): + build_encoder_state(d, [FeatureColumn(name="amount", encoding="numerical")]) + + +def test_build_encoder_state_complex_column_raises_feature_encode_error() -> None: + """A complex-dtype numerical column must be rejected explicitly. Pre-fix it + built a state silently, training on the real part alone -- `float(complex)` + discards the imaginary part under numpy 2.x with only a ComplexWarning.""" + d = pd.DataFrame( + { + "item_id": ["a", "b", "c"], + "amount": pd.Series([1 + 2j, 3 + 4j, 5 + 6j], dtype=object), + } + ).set_index("item_id") + # Premise: to_numeric keeps it complex (so float() would silently discard + # the imaginary part rather than raise a clean TypeError). + assert pd.api.types.is_complex_dtype(pd.to_numeric(d["amount"], errors="coerce")) + with pytest.raises(FeatureEncodeError, match="complex"): + build_encoder_state(d, [FeatureColumn(name="amount", encoding="numerical")]) + + +def test_encode_one_reports_unknown_columns( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + state = build_encoder_state(df, columns) + m, unknown = encode_one(state, {"genre": "comedy", "year": 2000.0, "tags": "a"}) + assert m.shape == (1, 6) + assert unknown == ["genre"] + + +def test_state_descriptor_shape(df: pd.DataFrame, columns: list[FeatureColumn]) -> None: + state = build_encoder_state(df, columns) + d = state_descriptor(state) + assert d == {"n_features": 6, "columns": ["genre", "year", "tags"]} + assert state_descriptor(None) is None + + +# --------------------------------------------------------------------------- +# encode / encode_one parity +# +# encode() goes through a DataFrame row (values may be numpy scalars, NaN, +# pandas NA); encode_one() goes through a plain request dict (values may be +# Python str/int/float/None). A cold-start request must be encoded exactly +# the way training saw the same entity, so these two paths must agree. +# --------------------------------------------------------------------------- + + +def test_encode_and_encode_one_agree_on_dataframe_row( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """The row encode() produces for a known id must equal encode_one() fed + the same values pulled out of the DataFrame by hand.""" + state = build_encoder_state(df, columns) + from_df = encode(state, df, index_order=["i1"]).toarray() + row = df.loc["i1"] + from_dict, unknown = encode_one( + state, {"genre": row["genre"], "year": row["year"], "tags": row["tags"]} + ) + np.testing.assert_allclose(from_df, from_dict.toarray()) + assert unknown == [] + + +def test_encode_and_encode_one_agree_on_missing_values( + columns: list[FeatureColumn], +) -> None: + """A DataFrame row with a real NaN (encode's missing convention) must + produce the same vector as a dict with None (encode_one's missing + convention), for the categorical and multi_label columns too -- not just + numerical. + + The probe frame mixes a string row in with the missing row so pandas + promotes the missing cell to a real float ``NaN``. A single-row, + all-``None`` object column is *not* promoted -- pandas leaves it as + literal Python ``None`` -- so a naive one-row probe would silently + compare None-vs-None instead of the NaN-vs-None case this test exists to + cover (confirmed: ``pd.DataFrame({"g": [None]})["g"][0]`` is ``None``, + but ``pd.DataFrame({"g": [None, "x"]})["g"][0]`` is ``nan``). The + assertions right after building ``probe_df`` guard against that trap + recurring silently if pandas' promotion rules ever change. + """ + train_df = pd.DataFrame( + { + "item_id": ["i1", "i2", "i3"], + "genre": ["action", "drama", "action"], + "year": [2000.0, 2010.0, 2020.0], + "tags": ["a|b", "b", None], + } + ).set_index("item_id") + state = build_encoder_state(train_df, columns) + + probe_df = pd.DataFrame( + { + "item_id": ["x", "y"], + "genre": [None, "action"], + "year": [np.nan, 2000.0], + "tags": [None, "a|b"], + } + ).set_index("item_id") + # Guard the premise: row "x"'s missing cells must be real float NaN, not + # Python None, or this test would not exercise what it claims to. + assert isinstance(probe_df.loc["x", "genre"], float) + assert np.isnan(probe_df.loc["x", "genre"]) + assert isinstance(probe_df.loc["x", "tags"], float) + assert np.isnan(probe_df.loc["x", "tags"]) + + from_df = encode(state, probe_df, index_order=["x"]).toarray() + + from_dict, unknown_dict = encode_one( + state, {"genre": None, "year": None, "tags": None} + ) + np.testing.assert_allclose(from_df, from_dict.toarray()) + # A genuinely missing value must not be reported as an unknown category. + assert unknown_dict == [] + + +def test_encode_one_agrees_on_nan_vs_none_categorical( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """encode_one must treat a float NaN identically to None for a + categorical column -- both mean "missing", not "unknown category". + + This talks to encode_one directly (no DataFrame involved) so there is no + dtype-promotion subtlety to get wrong: it is the most direct possible + proof that the categorical branch of ``_row_values`` uses ``_is_missing`` + rather than the narrower ``raw is None or str(raw) == ""`` check. + """ + state = build_encoder_state(df, columns) + from_nan, unknown_nan = encode_one( + state, {"genre": float("nan"), "year": 2000.0, "tags": "a"} + ) + from_none, unknown_none = encode_one( + state, {"genre": None, "year": 2000.0, "tags": "a"} + ) + np.testing.assert_allclose(from_nan.toarray(), from_none.toarray()) + assert unknown_nan == unknown_none == [] + + +def test_encode_one_agrees_on_nan_vs_none_multi_label( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """Same guarantee as the categorical case above, for multi_label. + + Direct encode_one-vs-encode_one comparison, fed straight to _tokens via + _row_values, so it needs no pandas dtype promotion to expose a broken + ``_is_missing`` check. + """ + state = build_encoder_state(df, columns) + from_nan, unknown_nan = encode_one( + state, {"genre": "action", "year": 2000.0, "tags": float("nan")} + ) + from_none, unknown_none = encode_one( + state, {"genre": "action", "year": 2000.0, "tags": None} + ) + np.testing.assert_allclose(from_nan.toarray(), from_none.toarray()) + assert unknown_nan == unknown_none == [] + + +def test_encode_one_reports_unknown_multi_label_columns( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """Mirrors test_encode_one_reports_unknown_columns (categorical) for the + multi_label path, which had no dedicated coverage before this test.""" + state = build_encoder_state(df, columns) + _, unknown_empty = encode_one( + state, {"genre": "action", "year": 2000.0, "tags": ""} + ) + assert unknown_empty == [] + + _, unknown_all = encode_one( + state, {"genre": "action", "year": 2000.0, "tags": "zzz|qqq"} + ) + assert unknown_all == ["tags"] + + +def test_encode_one_numeric_string_matches_numeric_value( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A numeric feature passed as the string "2000" must standardize + identically to the same value passed as a float.""" + state = build_encoder_state(df, columns) + m_numeric, _ = encode_one(state, {"genre": "action", "year": 2000.0, "tags": "a"}) + m_string, _ = encode_one(state, {"genre": "action", "year": "2000", "tags": "a"}) + np.testing.assert_allclose(m_numeric.toarray(), m_string.toarray()) + + +# --------------------------------------------------------------------------- +# Review finding: multi_label must emit multi-HOT, not counts (a repeated +# token must not double its dimension's weight), and the unknown-value +# counter must fire on ANY supplied token miss, not only a total miss. +# --------------------------------------------------------------------------- + + +def test_multi_label_duplicate_tokens_encode_as_binary_not_count( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """``tags="a|b|a"`` must put 1.0 on 'a', not 2.0. + + docs/recipe-reference.md documents ``multi_label`` as "multi-hot" + (binary), but scipy's COO->CSR conversion SUMS duplicate (row, col) + entries -- appending one 1.0 per raw token occurrence (the pre-fix + behavior) would silently turn a doubled tag into a weight of 2.0. This + talks to ``encode`` (the DataFrame/training path); the paired + ``encode_one`` test below covers the cold-start request path. + """ + state = build_encoder_state(df, columns) + probe = pd.DataFrame( + {"item_id": ["x"], "genre": ["action"], "year": [2000.0], "tags": ["a|b|a"]} + ).set_index("item_id") + m = encode(state, probe, index_order=["x"]).toarray() + tag_spec = state["columns"][2] + a_idx = tag_spec["vocab"]["a"] + assert m[0, tag_spec["offset"] + a_idx] == 1.0 + + +def test_encode_one_multi_label_duplicate_tokens_encode_as_binary_not_count( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """Same guarantee as the ``encode`` test above, for the cold-start + request path (``encode_one``) -- a duplicated token in + ``item_features`` / ``user_features`` must not double-weight its + dimension either.""" + state = build_encoder_state(df, columns) + m, unknown = encode_one(state, {"genre": "action", "year": 2000.0, "tags": "a|b|a"}) + tag_spec = state["columns"][2] + a_idx = tag_spec["vocab"]["a"] + assert m.toarray()[0, tag_spec["offset"] + a_idx] == 1.0 + assert unknown == [] + + +def test_encode_one_reports_unknown_for_partial_multi_label_miss( + df: pd.DataFrame, columns: list[FeatureColumn] +) -> None: + """A MIXED multi_label value (one known token, one unknown token) must + still report the column as unknown. + + Before this fix, ``_row_values`` only appended to ``unknown`` when + EVERY supplied token missed the vocabulary (``toks and not hit``), so + ``"a|zzz"`` (with 'a' known) silently escaped the counter -- exactly + the partial-typo shape (``"Action|Thrller"``) the spec's Observability + section commits to catching: "Fires ... when a multi_label token is + dropped" (any token), not "when every token is dropped". + """ + state = build_encoder_state(df, columns) + _, unknown = encode_one(state, {"genre": "action", "year": 2000.0, "tags": "a|zzz"}) + assert unknown == ["tags"] + + +# --------------------------------------------------------------------------- +# Characterization: `_tokens` strips each split piece and drops the empty ones, +# so trailing / leading / doubled delimiters and surrounding whitespace all +# tokenize to the same set. Each variant below must encode identically to the +# canonical "a|b". (No production code -- documents existing behavior.) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("raw", ["a|b|", "|a|b", "a||b", " a | b "]) +def test_multi_label_tokenization_ignores_empty_and_whitespace_tokens( + df: pd.DataFrame, columns: list[FeatureColumn], raw: str +) -> None: + """Each delimiter/whitespace variant tokenizes to {a, b} and so encodes + identically to the canonical "a|b".""" + state = build_encoder_state(df, columns) + m_canonical, _ = encode_one( + state, {"genre": "action", "year": 2000.0, "tags": "a|b"} + ) + m, unknown = encode_one(state, {"genre": "action", "year": 2000.0, "tags": raw}) + np.testing.assert_allclose(m.toarray(), m_canonical.toarray()) + assert unknown == [] diff --git a/tests/unit/test_features_compat.py b/tests/unit/test_features_compat.py new file mode 100644 index 00000000..b683c432 --- /dev/null +++ b/tests/unit/test_features_compat.py @@ -0,0 +1,276 @@ +"""Tests for the artifact feature-encoder version gate (Task 10). + +``check_artifact_feature_version`` closes a payload-shape gap +``_irspack_compat`` does not cover: recotem has no ``recotem_version`` gate at +serve time, so this descriptor is the only thing standing between a shape +change in the feature-encoder state and silently wrong recommendations (a +request's features encoded into the wrong vector space). + +The wiring tests below follow the shape of +``tests/unit/test_irspack_compat_wiring.py``: the gate is only useful if it is +actually reached from BOTH load paths -- ``app.py``'s startup loader and +``watcher.py``'s hot-swap loader -- and its ``ArtifactError`` is classified +under its own ``"feature_version"`` reason rather than falling into a +neighbouring bucket (the message contains the word "version", so it would +otherwise be swallowed by the "parse" catch-all). +""" + +from __future__ import annotations + +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import structlog.testing + +from recotem._features import ( + FEATURE_VERSION_MSG_PREFIX, + check_artifact_feature_version, +) +from recotem.artifact.format import ArtifactError +from recotem.config import ServeConfig +from recotem.serving.app import _try_load_artifact +from recotem.serving.metrics import _LOAD_FAILURE_REASONS +from recotem.serving.registry import ModelRegistry +from recotem.serving.watcher import ( + ArtifactWatcher, + _classify_artifact_error, + _RecipeWatchState, +) + +# --------------------------------------------------------------------------- +# Gate logic +# --------------------------------------------------------------------------- + + +def test_absent_features_key_passes() -> None: + """Old artifact or a non-feature model: nothing to gate.""" + check_artifact_feature_version({"recipe_name": "r"}, name="r") + + +def test_known_version_passes() -> None: + check_artifact_feature_version( + {"features": {"version": 1, "item": {"n_features": 3}}}, name="r" + ) + + +def test_newer_version_refused() -> None: + with pytest.raises(ArtifactError, match="feature encoder version"): + check_artifact_feature_version({"features": {"version": 2}}, name="r") + + +def test_non_int_version_refused() -> None: + with pytest.raises(ArtifactError): + check_artifact_feature_version({"features": {"version": "1"}}, name="r") + + +def test_bool_version_refused() -> None: + """``isinstance(True, int)`` is True in Python; the guard must exclude bools.""" + with pytest.raises(ArtifactError): + check_artifact_feature_version({"features": {"version": True}}, name="r") + + +def test_missing_version_refused() -> None: + """A features block with no version is malformed -- fail closed.""" + with pytest.raises(ArtifactError): + check_artifact_feature_version({"features": {"item": {}}}, name="r") + + +def test_non_dict_features_refused() -> None: + with pytest.raises(ArtifactError): + check_artifact_feature_version({"features": "nope"}, name="r") + + +# --------------------------------------------------------------------------- +# Classification -- mirrors test_irspack_compat_wiring.py's structure +# --------------------------------------------------------------------------- + + +def _feature_version_message() -> str: + """Return a real refusal message, produced by the guard itself. + + Built from the guard rather than hand-written so the test cannot drift + away from the wording the guard actually emits. + """ + with pytest.raises(ArtifactError) as excinfo: + check_artifact_feature_version({"features": {"version": 2}}, name="news") + return str(excinfo.value) + + +def test_feature_version_message_classifies_as_feature_version() -> None: + assert _classify_artifact_error(_feature_version_message()) == "feature_version" + + +def test_feature_version_message_is_not_misclassified_as_parse() -> None: + """Regression: the "parse" branch claims any message containing "version". + + The refusal message contains "feature encoder version 2", so ordering in + ``_classify_artifact_error`` is load-bearing, exactly as it is for the + irspack skew guard's message. + """ + msg = _feature_version_message() + assert "version" in msg.lower(), "precondition: message contains 'version'" + assert _classify_artifact_error(msg) != "parse" + + +def test_feature_version_is_an_allowed_metric_label() -> None: + """Otherwise inc_artifact_load_failure silently coerces it to "unexpected".""" + assert "feature_version" in _LOAD_FAILURE_REASONS + + +def test_classifier_prefix_matches_guard_prefix() -> None: + """The classifier keys off the guard's prefix; keep them in sync.""" + assert ( + _feature_version_message() + .lower() + .startswith(FEATURE_VERSION_MSG_PREFIX.lower()) + ) + + +# --------------------------------------------------------------------------- +# Startup path (app.py) -- its reason is a hardcoded literal, not classified +# --------------------------------------------------------------------------- + +_REFUSED_HEADER = { + "recipe_name": "news", + "best_class": "TopPopRecommender", + "trained_at": "2026-01-01T00:00:00Z", + "features": {"version": 2}, +} + + +def _load_with_header(tmp_path: Path, make_artifact, key_ring, header: dict): + """Run serve's startup loader over an artifact carrying *header*.""" + data = make_artifact(header_dict=header) + path = tmp_path / "feature_version.recotem" + path.write_bytes(data) + recipe = types.SimpleNamespace( + name="news", + output=types.SimpleNamespace(path=str(path)), + item_metadata=None, + ) + return _try_load_artifact(recipe, key_ring, ServeConfig()) + + +def test_startup_path_reports_feature_version( + tmp_path: Path, make_artifact, single_key_ring +) -> None: + entry, reason = _load_with_header( + tmp_path, make_artifact, single_key_ring, dict(_REFUSED_HEADER) + ) + assert reason == "feature_version" + assert entry.loaded is False + assert "feature encoder version" in (entry.last_load_error or "").lower() + + +def test_startup_path_loads_when_feature_version_matches( + tmp_path: Path, make_artifact, single_key_ring +) -> None: + """Positive control: a matching feature version must load, not just fail + to be refused. + + Without this, a gate that unconditionally refused every artifact would + still pass ``test_startup_path_reports_feature_version`` above. + """ + header = dict(_REFUSED_HEADER) + header["features"] = {"version": 1} + entry, reason = _load_with_header(tmp_path, make_artifact, single_key_ring, header) + assert reason == "ok", f"matching feature version must load; got {reason!r}" + assert entry.loaded is True + + +# --------------------------------------------------------------------------- +# Hot-swap path (watcher.py) -- a gate wired into only ONE of app.py/watcher.py +# is a half-fix that a naive test (covering app.py alone) would not catch. +# --------------------------------------------------------------------------- + + +def _make_watcher_serve_config() -> ServeConfig: + cfg = ServeConfig() + cfg.max_artifact_bytes = 100 * 1024 * 1024 + return cfg + + +def test_watcher_path_reports_feature_version( + tmp_path: Path, make_artifact, single_key_ring +) -> None: + """``ArtifactWatcher._build_entry`` (the hot-swap loader) must also refuse. + + Drives ``_load_recipe`` directly -- the same synchronous pattern + ``tests/unit/test_serving_watcher.py`` uses for its failure-path tests -- + rather than starting the watcher thread, to avoid a timing-dependent test. + """ + artifact_path = tmp_path / "model.recotem" + data = make_artifact(header_dict=dict(_REFUSED_HEADER)) + artifact_path.write_bytes(data) + + recipe = types.SimpleNamespace(item_metadata=None) + state = _RecipeWatchState(recipe=recipe, artifact_path=str(artifact_path)) + + registry = ModelRegistry() + stub_entry = MagicMock() + stub_entry.last_load_error = None + stub_entry.loaded = False + registry.replace("news", stub_entry) + + watcher = ArtifactWatcher( + registry=registry, + recipes_dir=tmp_path, + serve_config=_make_watcher_serve_config(), + key_ring=single_key_ring, + initial_states={"news": state}, + ) + + with structlog.testing.capture_logs() as cap: + watcher._load_recipe("news", state, force=True) + + failed = [e for e in cap if e.get("event") == "artifact_load_failed"] + assert failed, "watcher must log artifact_load_failed for a refused feature version" + assert failed[0].get("reason") == "feature_version" + + entry = registry.get("news") + assert entry is not None + assert "feature encoder version" in (entry.last_load_error or "").lower() + + +def test_watcher_path_loads_when_feature_version_matches( + tmp_path: Path, make_artifact, single_key_ring +) -> None: + """Positive control for the hot-swap path, mirroring the startup-path one. + + Without this, a watcher-side gate that unconditionally refused every + artifact would still pass ``test_watcher_path_reports_feature_version``. + """ + artifact_path = tmp_path / "model_ok.recotem" + header = dict(_REFUSED_HEADER) + header["features"] = {"version": 1} + data = make_artifact(header_dict=header) + artifact_path.write_bytes(data) + + recipe = types.SimpleNamespace(item_metadata=None) + state = _RecipeWatchState(recipe=recipe, artifact_path=str(artifact_path)) + + registry = ModelRegistry() + stub_entry = MagicMock() + stub_entry.last_load_error = None + stub_entry.loaded = False + registry.replace("news", stub_entry) + + watcher = ArtifactWatcher( + registry=registry, + recipes_dir=tmp_path, + serve_config=_make_watcher_serve_config(), + key_ring=single_key_ring, + initial_states={"news": state}, + ) + + with structlog.testing.capture_logs() as cap: + watcher._load_recipe("news", state, force=True) + + failed = [e for e in cap if e.get("event") == "artifact_load_failed"] + assert not failed, f"matching feature version must not fail load; got {failed!r}" + + entry = registry.get("news") + assert entry is not None + assert entry.loaded is True diff --git a/tests/unit/test_idmap.py b/tests/unit/test_idmap.py index a80a0c39..a112a40e 100644 --- a/tests/unit/test_idmap.py +++ b/tests/unit/test_idmap.py @@ -4,13 +4,59 @@ - Fix 4: unknown user_id raises KeyError without calling underlying recommender. - Fix 4: known user_id that causes RuntimeError in the underlying recommender propagates as RuntimeError (not masked to KeyError). +- Task 8: item_feature_state / user_feature_state class-level default + + round-trip persistence, including through the real signed-artifact path. + +NOTE: ``IDMappedRecommender`` is pickled directly (via ``pickle.dumps`` / +``pickle.loads``) because that is production behaviour: irspack recommenders +carry scipy sparse matrices / numpy arrays that only pickle supports. This is +the same intentional, defence-in-depth-guarded usage documented in +``tests/conftest.py`` -- the artifact-path test below additionally goes +through ``recotem.artifact.signing.unpickle_payload``'s HMAC verification and +FQCN allow-list (``SafeUnpickler``), not bare ``pickle.loads``, for anything +that also needs to be safe against untrusted input. """ from __future__ import annotations +import pickle +from pathlib import Path from unittest.mock import MagicMock +import numpy as np import pytest +import scipy.sparse as sps + + +class _FakeRecommender: + """Minimal picklable stand-in for a trained irspack recommender. + + A bare ``unittest.mock.MagicMock``/``Mock`` is NOT picklable (raises + ``PicklingError: Can't pickle ``), so it + cannot stand in for the recommender in tests that pickle + ``IDMappedRecommender`` end-to-end (the feature-state round-trip tests + below, and every real artifact). This class exposes just the surface + ``irspack.utils.id_mapping.IDMapper.recommend_for_known_user_id`` needs + (``n_users`` and ``get_score_remove_seen``) so those round trips can also + exercise the real recommend path, not just attribute persistence. + """ + + def __init__(self, n_users: int = 1, n_items: int = 1) -> None: + self.n_users = n_users + self.n_items = n_items + + def get_score_remove_seen(self, user_indices: np.ndarray) -> np.ndarray: + return np.zeros((len(user_indices), self.n_items), dtype=np.float64) + + +@pytest.fixture() +def mock_rec() -> _FakeRecommender: + """A minimal picklable stand-in for a trained irspack recommender. + + Sized for the single-user/single-item fixtures used throughout this + module (``IDMappedRecommender(mock_rec, ["u1"], ["i1"])``). + """ + return _FakeRecommender(n_users=1, n_items=1) def _make_idmapped(user_ids: list[str], item_ids: list[str]) -> object: @@ -184,3 +230,622 @@ def test_ipython_stub_noop_when_both_present( assert sys.modules["IPython.display"] is existing_display, ( "install() must not replace an already-present 'IPython.display' module" ) + + +# --------------------------------------------------------------------------- +# Task 8: item_feature_state / user_feature_state persistence +# --------------------------------------------------------------------------- + + +def test_feature_state_defaults_to_none(mock_rec: _FakeRecommender) -> None: + from recotem._idmap import IDMappedRecommender + + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"]) + assert idm.item_feature_state is None + assert idm.user_feature_state is None + + +def test_feature_state_is_class_level_not_init_only() -> None: + """__setstate__ bypasses __init__, so the default must resolve via the class.""" + from recotem._idmap import IDMappedRecommender + + assert "item_feature_state" in IDMappedRecommender.__dict__ + assert IDMappedRecommender.item_feature_state is None + assert "user_feature_state" in IDMappedRecommender.__dict__ + assert IDMappedRecommender.user_feature_state is None + + +def test_feature_state_round_trips(mock_rec: _FakeRecommender) -> None: + from recotem._idmap import IDMappedRecommender + + state = {"version": 1, "n_features": 2, "columns": []} + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"], item_feature_state=state) + back = pickle.loads(pickle.dumps(idm)) # noqa: S301 + assert back.item_feature_state == state + assert back.user_feature_state is None + + +def test_old_pickle_without_state_resolves_default(mock_rec: _FakeRecommender) -> None: + """Simulate an artifact pickled before these attributes existed. + + ``__getstate__`` output with the two keys stripped stands in for a state + dict produced by an old build of ``IDMappedRecommender``. Restoring it + via ``__setstate__`` must still resolve both attributes to ``None`` + without raising ``AttributeError``. + """ + from recotem._idmap import IDMappedRecommender + + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"]) + raw = idm.__getstate__() + raw.pop("item_feature_state", None) + raw.pop("user_feature_state", None) + revived = IDMappedRecommender.__new__(IDMappedRecommender) + revived.__setstate__(raw) + assert revived.item_feature_state is None + assert revived.get_recommendation_for_known_user_id("u1", 1) is not None + + +def test_feature_state_round_trips_through_real_artifact_path( + tmp_path: Path, +) -> None: + """Prove the encoder state survives the REAL artifact path, not just a + bare ``pickle.dumps``/``pickle.loads`` round trip. + + ``write_artifact`` -> ``read_artifact`` -> ``unpickle_payload`` is the + production path, and ``unpickle_payload`` enforces + ``recotem.artifact.signing.SafeUnpickler``'s hand-enumerated FQCN + allow-list. A plain dict + numpy array value (what + ``recotem._features.build_encoder_state`` produces, per the brief) is + expected to clear that allow-list with NO change to it -- that is the + claim this whole design rests on, so it is worth pinning here. + + The ``recommender`` slot uses a plain ``dict`` (an allow-listed builtin) + rather than a mock: mocks are not on the FQCN allow-list and would make + this fail for a reason unrelated to feature-state persistence. + """ + from recotem._idmap import IDMappedRecommender + from recotem.artifact.io import read_artifact, write_artifact + from recotem.artifact.signing import KeyRing, unpickle_payload + + state = {"version": 1, "n_features": 2, "weights": np.array([1.0, 2.0, 3.0])} + idm = IDMappedRecommender({}, ["u1"], ["i1"], item_feature_state=state) + + key_ring = KeyRing("probe:" + "ab" * 32) + output_path = str(tmp_path / "probe.recotem") + write_artifact( + payload_obj=idm, + header_dict={"recipe_name": "probe"}, + key_ring=key_ring, + fs_path=output_path, + versioning="always_overwrite", + ) + + _, payload_bytes = read_artifact(output_path, key_ring) + revived = unpickle_payload(payload_bytes) + + assert revived.item_feature_state["version"] == 1 + assert revived.item_feature_state["n_features"] == 2 + np.testing.assert_array_equal( + revived.item_feature_state["weights"], state["weights"] + ) + assert revived.user_feature_state is None + + +# --------------------------------------------------------------------------- +# Task 11: feature-based cold-start methods +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fa_model() -> object: + """A real, tiny 2-epoch feature-aware IALS wrapped in IDMappedRecommender. + + Benchmarked at ~1-2ms to train (40 users x 20 items, 2 epochs, K=8) -- + far under the threshold for @pytest.mark.slow, so this runs in the + default suite. + """ + import pandas as pd + from irspack import IALSRecommender + + from recotem._features import build_encoder_state, encode + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + rng = np.random.default_rng(0) + n_users, n_items = 40, 20 + X = sps.csr_matrix((rng.random((n_users, n_items)) > 0.6).astype(np.float64)) + item_df = pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(n_items)], + "genre": ["action" if i % 2 else "drama" for i in range(n_items)], + } + ).set_index("item_id") + user_df = pd.DataFrame( + { + "user_id": [f"u{u}" for u in range(n_users)], + "band": ["young" if u % 2 else "old" for u in range(n_users)], + } + ).set_index("user_id") + + istate = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + ustate = build_encoder_state( + user_df, [FeatureColumn(name="band", encoding="categorical")] + ) + F = encode(istate, item_df, index_order=[f"i{i}" for i in range(n_items)]) + U = encode(ustate, user_df, index_order=[f"u{u}" for u in range(n_users)]) + + rec = IALSRecommender( + X, + n_components=8, + alpha0=0.1, + train_epochs=2, + random_seed=42, + item_features=F, + user_features=U, + lambda_item_feature=1e-1, + lambda_user_feature=1e-1, + ).learn() + return IDMappedRecommender( + rec, + [f"u{u}" for u in range(n_users)], + [f"i{i}" for i in range(n_items)], + item_feature_state=istate, + user_feature_state=ustate, + ) + + +def test_cold_user_from_features(fa_model: object) -> None: + recs, unknown = fa_model.get_recommendation_for_cold_user( + {"band": "young"}, cutoff=5 + ) + assert len(recs) == 5 + assert unknown == [] + assert all(isinstance(r[0], str) for r in recs) + + +def test_cold_user_reports_unknown_category(fa_model: object) -> None: + _, unknown = fa_model.get_recommendation_for_cold_user( + {"band": "martian"}, cutoff=3 + ) + assert unknown == ["band"] + + +def test_cold_start_methods_take_no_exclude_items_parameter() -> None: + """Client-requested exclusion is the router's post-filter, never a + ranker argument. + + These two methods used to accept ``exclude_items`` and pass it to + irspack as ``forbidden_item_ids``, which makes the ranker BACK-FILL to a + full ``cutoff``. Every pre-existing path instead post-filters in + ``routes._build_items``, which truncates -- so the same + ``exclude_items`` request returned a full page here and a short one + everywhere else, and merely adding ``user_features`` to a request + changed how many items came back. + ``tests/unit/test_serving_cold_start.py``'s + ``test_exclude_items_truncates_and_never_backfills`` proves the three + cold-start cases now agree end-to-end; this pins the signature, because + that test cannot see a re-added parameter that nothing happens to pass. + """ + import inspect + + from recotem._idmap import IDMappedRecommender + + for method in ( + IDMappedRecommender.get_recommendation_for_cold_user, + IDMappedRecommender.get_recommendation_for_cold_seeds, + ): + assert "exclude_items" not in inspect.signature(method).parameters, ( + f"{method.__name__} must not take exclude_items: exclusion " + "post-filters in routes._build_items so limit stays a ceiling" + ) + + +def test_new_user_with_features_differs_from_without(fa_model: object) -> None: + without = fa_model.get_recommendation_for_new_user(["i0", "i2"], cutoff=5) + with_f, _ = fa_model.get_recommendation_for_new_user( + ["i0", "i2"], cutoff=5, user_features={"band": "young"} + ) + assert without != with_f # the joint solve is genuinely different + + +def test_new_user_without_features_is_backward_compatible(fa_model: object) -> None: + recs = fa_model.get_recommendation_for_new_user(["i0"], cutoff=3) + assert isinstance(recs, list) # NOT a tuple: old signature preserved + + +def test_new_user_with_features_reports_unknown_category(fa_model: object) -> None: + """Case B: an out-of-vocabulary ``band`` value must surface in `unknown`. + + ``test_new_user_with_features_differs_from_without`` only ever feeds a + KNOWN category ("young"), so it cannot tell "correctly empty" apart from + "silently discarded" -- `unknown` could be hardcoded to `[]` and that + test would still pass. This test feeds a value absent from the `band` + vocabulary built in the `fa_model` fixture ("young"/"old" only) and + requires it to be reported, since `unknown_columns` is the only signal + serving gets that a category silently degraded to an all-zero segment. + """ + _, unknown = fa_model.get_recommendation_for_new_user( + ["i0", "i2"], cutoff=5, user_features={"band": "martian"} + ) + assert unknown == ["band"] + + +def test_cold_seeds_from_item_features(fa_model: object) -> None: + recs, unknown = fa_model.get_recommendation_for_cold_seeds( + ["i0", "brand_new"], + {"brand_new": {"genre": "action"}}, + cutoff=5, + ) + assert len(recs) == 5 + assert unknown == [] + # A known seed must not recommend itself back. This is the "remove seen" + # behavior at _idmap.py's `forbidden.extend(...)` -- deleting that line + # leaves this assertion as the ONLY thing in the suite that would catch + # the regression (verified: see task-11-report.md's mutation-proof log). + assert "i0" not in {r[0] for r in recs} + + +def test_cold_seeds_removes_all_known_seeds_from_own_output( + fa_model: object, +) -> None: + """Dedicated proof for the seed-removal behavior, independent of ranking. + + Seeds with every item in the (small, 20-item) catalog at a cutoff that + would otherwise return the whole catalog: if seed removal were broken, + every seed would have to appear somewhere in the output since there is + nothing else to return instead. Using only known seeds (no unknown ones) + isolates the exact line under review (`forbidden.extend` over known + seeds) from the unrelated "unknown seed" code path. + """ + all_items = [f"i{i}" for i in range(20)] + recs, _ = fa_model.get_recommendation_for_cold_seeds(all_items, {}, cutoff=20) + rec_ids = {r[0] for r in recs} + assert rec_ids.isdisjoint(set(all_items)) + + +def test_cold_seeds_reports_unknown_category(fa_model: object) -> None: + """Case C: an out-of-vocabulary ``genre`` for the UNKNOWN seed must + surface in `unknown`. + + ``test_cold_seeds_from_item_features`` only ever feeds a KNOWN category + ("action") for its unknown seed, so it cannot distinguish "correctly + empty" from "silently discarded" -- `unknown` could be hardcoded to `[]` + and that test would still pass. This test feeds a `genre` value absent + from the vocabulary built in the `fa_model` fixture ("action"/"drama" + only) for the unknown seed "brand_new", and requires it to be reported. + """ + _, unknown = fa_model.get_recommendation_for_cold_seeds( + ["i0", "brand_new"], + {"brand_new": {"genre": "sci-fi"}}, + cutoff=5, + ) + assert unknown == ["genre"] + + +# --------------------------------------------------------------------------- +# Review finding 1 (MEDIUM): an extreme-but-finite numerical feature value +# standardizes to a magnitude that makes irspack's per-request +# conjugate-gradient cold-start solve ill-conditioned. irspack's native core +# raises a bare RuntimeError ("Conjugate-gradient solver encountered a +# singular system.") with no awareness that the input came from an +# untrusted client. Each of the three cold-start call sites below must catch +# that RuntimeError and re-raise ColdStartNumericalError -- a distinct, +# non-RuntimeError type -- so serving/routes.py can map it to a 400 instead +# of letting a bare RuntimeError surface as an unhandled 500. +# +# Mocked (not a genuine 1e22 value through a real solve) so this pins the +# WRAPPING contract deterministically, independent of BLAS/threading +# variance in exactly which magnitude triggers the native solver's failure. +# The reproduction with a REAL trained model and a real extreme value lives +# in tests/unit/test_serving_cold_start.py (route-level, end-to-end). +# --------------------------------------------------------------------------- + +_SINGULAR_SYSTEM_MSG = "Conjugate-gradient solver encountered a singular system." + + +def test_cold_user_from_features_wraps_runtime_error(fa_model: object) -> None: + from unittest.mock import patch + + from recotem._idmap import ColdStartNumericalError + + with patch.object( + fa_model.recommender, + "get_score_cold_user_from_features", + side_effect=RuntimeError(_SINGULAR_SYSTEM_MSG), + ): + with pytest.raises(ColdStartNumericalError): + fa_model.get_recommendation_for_cold_user({"band": "young"}, cutoff=3) + + +def test_new_user_with_features_wraps_runtime_error(fa_model: object) -> None: + """Case B: ``get_score_cold_user`` (joint history + feature-prior solve) + must have the same wrapping as the features-only case A path above.""" + from unittest.mock import patch + + from recotem._idmap import ColdStartNumericalError + + with patch.object( + fa_model.recommender, + "get_score_cold_user", + side_effect=RuntimeError(_SINGULAR_SYSTEM_MSG), + ): + with pytest.raises(ColdStartNumericalError): + fa_model.get_recommendation_for_new_user( + ["i0"], cutoff=3, user_features={"band": "young"} + ) + + +def test_cold_seeds_wraps_runtime_error(fa_model: object) -> None: + """Case C: ``compute_item_embedding_from_features`` -- the exact call + site in the review's reproduction (``:recommend-related`` with a cold + seed's ``item_features`` carrying an extreme numerical value).""" + from unittest.mock import patch + + from recotem._idmap import ColdStartNumericalError + + with patch.object( + fa_model.recommender, + "compute_item_embedding_from_features", + side_effect=RuntimeError(_SINGULAR_SYSTEM_MSG), + ): + with pytest.raises(ColdStartNumericalError): + fa_model.get_recommendation_for_cold_seeds( + ["i0", "brand_new"], {"brand_new": {"genre": "action"}}, cutoff=3 + ) + + +# --------------------------------------------------------------------------- +# Review round 2, Important 1: the wrap above was previously BLANKET (any +# RuntimeError, no message check), which also swallowed non-numerical +# RuntimeErrors the exact same irspack calls can raise for reasons that have +# nothing to do with the request -- e.g. irspack/recommenders/ials.py's +# ``trainer_as_ials`` raising RuntimeError("tried to fetch trainer before +# the training.") when ``trainer`` is unexpectedly None. Each of the three +# call sites must now re-raise a non-matching RuntimeError UNCHANGED (not +# wrap it in ColdStartNumericalError), so it surfaces as a 500 at the route +# layer rather than a mislabeled 400. See +# tests/unit/test_serving_cold_start.py for the route-level 500 proof. +# --------------------------------------------------------------------------- + +_NON_NUMERICAL_TRAINER_MSG = "tried to fetch trainer before the training." + + +def test_cold_user_from_features_propagates_non_numerical_runtime_error( + fa_model: object, +) -> None: + from unittest.mock import patch + + with patch.object( + fa_model.recommender, + "get_score_cold_user_from_features", + side_effect=RuntimeError(_NON_NUMERICAL_TRAINER_MSG), + ): + with pytest.raises(RuntimeError, match=_NON_NUMERICAL_TRAINER_MSG): + fa_model.get_recommendation_for_cold_user({"band": "young"}, cutoff=3) + + +def test_new_user_with_features_propagates_non_numerical_runtime_error( + fa_model: object, +) -> None: + from unittest.mock import patch + + with patch.object( + fa_model.recommender, + "get_score_cold_user", + side_effect=RuntimeError(_NON_NUMERICAL_TRAINER_MSG), + ): + with pytest.raises(RuntimeError, match=_NON_NUMERICAL_TRAINER_MSG): + fa_model.get_recommendation_for_new_user( + ["i0"], cutoff=3, user_features={"band": "young"} + ) + + +def test_cold_seeds_propagates_non_numerical_runtime_error(fa_model: object) -> None: + from unittest.mock import patch + + with patch.object( + fa_model.recommender, + "compute_item_embedding_from_features", + side_effect=RuntimeError(_NON_NUMERICAL_TRAINER_MSG), + ): + with pytest.raises(RuntimeError, match=_NON_NUMERICAL_TRAINER_MSG): + fa_model.get_recommendation_for_cold_seeds( + ["i0", "brand_new"], {"brand_new": {"genre": "action"}}, cutoff=3 + ) + + +def test_cold_user_without_state_raises(mock_rec: _FakeRecommender) -> None: + from recotem._idmap import IDMappedRecommender + + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"]) + with pytest.raises(ValueError, match="no user feature state"): + idm.get_recommendation_for_cold_user({"band": "young"}, cutoff=1) + + +def test_cold_seeds_without_state_raises(mock_rec: _FakeRecommender) -> None: + from recotem._idmap import IDMappedRecommender + + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"]) + with pytest.raises(ValueError, match="no item feature state"): + idm.get_recommendation_for_cold_seeds(["i1"], {}, cutoff=1) + + +def test_new_user_with_features_without_state_raises( + mock_rec: _FakeRecommender, +) -> None: + from recotem._idmap import IDMappedRecommender + + idm = IDMappedRecommender(mock_rec, ["u1"], ["i1"]) + with pytest.raises(ValueError, match="no user feature state"): + idm.get_recommendation_for_new_user( + ["i1"], cutoff=1, user_features={"band": "young"} + ) + + +# --------------------------------------------------------------------------- +# Task 11 capability gate: non-None feature state does NOT imply the winning +# recommender can act on it (TopPop/CosineKNN/etc. can carry feature state +# unconditionally persisted by Task 9, but have no cold-start-from-features +# API at all). This must fail with a clean ValueError, not AttributeError. +# --------------------------------------------------------------------------- + + +def test_cold_start_methods_reject_non_feature_capable_recommender() -> None: + """A real TopPopRecommender wrapped with non-None item/user feature state. + + TopPop is a legal search winner even when the recipe lists a + feature-capable algorithm alongside it (``Recipe._validate_features_algorithms`` + requires only one feature-capable entry), and Task 9 persists feature + state unconditionally so the header always agrees with the payload. + TopPopRecommender has no ``get_score_cold_user_from_features``, no + ``get_item_embedding``/``compute_item_embedding_from_features``, and its + ``get_score_cold_user`` does not accept ``user_features`` -- so the ONLY + guard against a bare ``AttributeError``/``TypeError`` escaping to a caller + is the capability check in ``IDMappedRecommender._require_capability``. + """ + import pandas as pd + from irspack import TopPopRecommender + + from recotem._features import build_encoder_state + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + n_users, n_items = 5, 4 + X = sps.csr_matrix(np.ones((n_users, n_items))) + rec = TopPopRecommender(X).learn() + + item_df = pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(n_items)], + "genre": ["a", "b", "a", "b"], + } + ).set_index("item_id") + user_df = pd.DataFrame( + { + "user_id": [f"u{u}" for u in range(n_users)], + "band": ["x"] * n_users, + } + ).set_index("user_id") + istate = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + ustate = build_encoder_state( + user_df, [FeatureColumn(name="band", encoding="categorical")] + ) + + idm = IDMappedRecommender( + rec, + [f"u{u}" for u in range(n_users)], + [f"i{i}" for i in range(n_items)], + item_feature_state=istate, + user_feature_state=ustate, + ) + # The precondition this test exists to cover: state IS present. + assert idm.item_feature_state is not None + assert idm.user_feature_state is not None + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_cold_user({"band": "x"}, cutoff=2) + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_cold_seeds( + ["i0", "brand_new"], {"brand_new": {"genre": "a"}}, cutoff=2 + ) + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_new_user( + ["i0"], cutoff=2, user_features={"band": "x"} + ) + + # The untouched path (no user_features) must still work -- TopPop's + # existing cold-start-from-history behavior is unaffected by this gate. + recs = idm.get_recommendation_for_new_user(["i0"], cutoff=2) + assert isinstance(recs, list) + + +class _LookAlikeRecommender: + """Exposes the exact method names/signatures the capability gate looks + for, but is NOT ``IALSRecommender`` and must NOT be treated as capable. + + This is the class the allow-list gate (as opposed to duck-typing) exists + to defend against: a `hasattr(rec, "get_score_cold_user_from_features")` + / `inspect.signature(rec.get_score_cold_user)` check would happily accept + this class -- every symbol it looks for is present. Each method raises + `AssertionError` rather than returning a real score so that a test + reaching into the method body (i.e. the gate failed to stop it) fails + loudly with an unambiguous, distinguishable error, not a passing test + with garbage data. + """ + + def get_score_cold_user_from_features(self, matrix: object) -> object: + raise AssertionError("must not be called: not on the allow-list") + + def get_item_embedding(self) -> object: + raise AssertionError("must not be called: not on the allow-list") + + def compute_item_embedding_from_features(self, matrix: object) -> object: + raise AssertionError("must not be called: not on the allow-list") + + def get_score_from_user_embedding(self, matrix: object) -> object: + raise AssertionError("must not be called: not on the allow-list") + + def get_score_cold_user(self, X: object, user_features: object = None) -> object: + raise AssertionError("must not be called: not on the allow-list") + + +def test_cold_start_gate_rejects_lookalike_class_not_on_allowlist() -> None: + """A class exposing the right method names/signatures must still be + refused if its class name is not on the explicit allow-list. + + This is the test that only makes sense once the gate is an allow-list + rather than duck-typing: `_LookAlikeRecommender` above would sail through + the OLD `hasattr`/`inspect.signature` checks (it defines every symbol + they look for) and fail deep inside the method body with a bare + `AssertionError` instead of the clean `ValueError` this gate exists to + guarantee. Under the fixed allow-list gate, `type(self.recommender).__name__` + is `"_LookAlikeRecommender"`, which is absent from + `_idmap._FEATURE_CAPABLE_CLASS_NAMES`, so it must be refused BEFORE any + of the methods above are ever invoked. + """ + from recotem._idmap import IDMappedRecommender + + rec = _LookAlikeRecommender() + state = {"version": 1, "columns": [], "bias_offset": 0, "n_features": 1} + idm = IDMappedRecommender( + rec, + ["u1"], + ["i1"], + item_feature_state=state, + user_feature_state=state, + ) + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_cold_user({}, cutoff=1) + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_cold_seeds(["i1"], {}, cutoff=1) + + with pytest.raises(ValueError, match="does not support"): + idm.get_recommendation_for_new_user(["i1"], cutoff=1, user_features={}) + + +def test_feature_capable_class_names_stay_in_sync_with_training() -> None: + """`_idmap._FEATURE_CAPABLE_CLASS_NAMES` must equal + `training.algorithms.FEATURE_CAPABLE_CLASS_NAMES` value-for-value. + + The two sets are hand-duplicated (`_idmap.py` is a neutral module that + must not import `recotem.training` -- see the comment above + `_FEATURE_CAPABLE_CLASS_NAMES`), so nothing at import time enforces they + stay identical. A test is allowed to import both sides (tests are not + bound by the training/serving import boundary), so this is the one + place that can catch a silent divergence -- e.g. a new feature-capable + irspack class added to only one of the two sets, which would otherwise + fail closed (a capable model 400s on cold start at serve time) with no + signal until an operator noticed. + """ + from recotem._idmap import _FEATURE_CAPABLE_CLASS_NAMES + from recotem.training.algorithms import FEATURE_CAPABLE_CLASS_NAMES + + assert _FEATURE_CAPABLE_CLASS_NAMES == FEATURE_CAPABLE_CLASS_NAMES diff --git a/tests/unit/test_log_redaction.py b/tests/unit/test_log_redaction.py index 13a234ff..6239e397 100644 --- a/tests/unit/test_log_redaction.py +++ b/tests/unit/test_log_redaction.py @@ -886,3 +886,34 @@ def test_redact_value_recurses_into_sets() -> None: scrubbed = [s for s in out if s.startswith("postgresql://")] assert scrubbed and "***" in scrubbed[0] assert all(":p@" not in s for s in out) + + +# --------------------------------------------------------------------------- +# Feature-aware iALS cold-start: user_features / item_features are PII by +# construction (e.g. age_band, country) and must be redacted by key name. +# +# This is defence in depth, not the primary control -- the primary rule is +# that callers must never pass a feature dict to a logger in the first +# place. This backstop exists in case one does anyway. +# --------------------------------------------------------------------------- + + +def test_user_features_values_are_redacted() -> None: + event = {"event": "x", "user_features": {"band": "35-44", "country": "JP"}} + out = _invoke(dict(event)) + assert "35-44" not in repr(out) + assert "JP" not in repr(out) + + +def test_item_features_values_are_redacted() -> None: + event = {"event": "x", "item_features": {"new1": {"genre": "action"}}} + out = _invoke(dict(event)) + assert "action" not in repr(out) + + +def test_unrelated_keys_still_pass_through() -> None: + """The new redaction must not swallow ordinary fields.""" + event = {"event": "x", "recipe": "movies", "limit": 10} + out = _invoke(dict(event)) + assert out["recipe"] == "movies" + assert out["limit"] == 10 diff --git a/tests/unit/test_recipe_loader.py b/tests/unit/test_recipe_loader.py index 2981015a..24c5fe4b 100644 --- a/tests/unit/test_recipe_loader.py +++ b/tests/unit/test_recipe_loader.py @@ -2352,3 +2352,716 @@ def test_gs_project_at_bucket_accepted(tmp_path: Path) -> None: f"gs://project@bucket/key must not trigger credentials rejection; " f"got: {exc}" ) + + +# --------------------------------------------------------------------------- +# Task 3: features..source — typed resolution + security validation +# --------------------------------------------------------------------------- +# +# ``features`` requires a feature-capable algorithm (IALS) per +# ``Recipe._validate_features_algorithms``, so every recipe below lists IALS. + +_FEATURES_BASE_RECIPE = """\ +name: {name} +source: + type: csv + path: ./interactions.csv +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [IALS] + n_trials: 1 +output: + path: {output_path} +""" + + +def _write_features_recipe(tmp_path: Path, name: str, features_block: str) -> Path: + content = ( + _FEATURES_BASE_RECIPE.format( + name=name, + output_path=str(tmp_path / f"{name}.recotem"), + ) + + features_block + ) + return _write_recipe(tmp_path, content, filename=f"{name}.yaml") + + +def test_feature_source_is_typed_after_load(tmp_path: Path) -> None: + """After load_recipe, features.item.source must be a typed CSVConfig.""" + from recotem.datasource.csv import CSVConfig + + p = _write_features_recipe( + tmp_path, + "feat_typed", + """\ +features: + item: + source: + type: csv + path: ./items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + recipe = load_recipe(p) + assert isinstance(recipe.features.item.source, CSVConfig), ( + f"Expected CSVConfig, got {type(recipe.features.item.source)}" + ) + assert recipe.features.item.source.path == "./items.csv" + + +def test_feature_source_rejects_disallowed_scheme(tmp_path: Path) -> None: + """The source-path scheme allow-list must also cover features.item.source.""" + p = _write_features_recipe( + tmp_path, + "feat_bad_scheme", + """\ +features: + item: + source: + type: csv + path: ftp://evil.example/items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + with pytest.raises(RecipeError, match="scheme"): + load_recipe(p) + + +def test_feature_source_https_requires_sha256(tmp_path: Path) -> None: + """An unpinned https:// feature source must be rejected, same as source.""" + p = _write_features_recipe( + tmp_path, + "feat_https_no_sha", + """\ +features: + item: + source: + type: csv + path: https://example.com/items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + with pytest.raises(RecipeError, match="sha256"): + load_recipe(p) + + +def test_feature_source_https_with_sha256_accepted(tmp_path: Path) -> None: + sha = "0" * 64 + p = _write_features_recipe( + tmp_path, + "feat_https_ok", + f"""\ +features: + item: + source: + type: csv + path: https://example.com/items.csv + sha256: "{sha}" + id_column: item_id + columns: + - {{name: genre, encoding: categorical}} +""", + ) + recipe = load_recipe(p) + assert recipe.features.item.source.sha256 == sha + + +def test_feature_source_rejects_embedded_credentials(tmp_path: Path) -> None: + p = _write_features_recipe( + tmp_path, + "feat_creds", + f"""\ +features: + item: + source: + type: csv + path: https://user:pw@example.com/items.csv + sha256: "{"0" * 64}" + id_column: item_id + columns: + - {{name: genre, encoding: categorical}} +""", + ) + with pytest.raises(RecipeError, match="credentials"): + load_recipe(p) + + +def test_feature_source_unknown_type_rejected(tmp_path: Path) -> None: + """An unregistered features.item.source.type must be rejected at load time. + + Complements (does not duplicate) the models-layer + ``Recipe._validate_features_sources`` fallback exercised in + test_recipe_models.py — this test exercises the real YAML → load_recipe + path, which must fail no later than that fallback would. + """ + p = _write_features_recipe( + tmp_path, + "feat_unknown_type", + """\ +features: + item: + source: + type: no_such_source + path: ./items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + with pytest.raises(RecipeError): + load_recipe(p) + + +def test_feature_sql_source_dsn_env_not_expanded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A features..source plugin's no_expand_fields must reach feature + subtrees, proven with a real ``${...}`` reference (not a bare string). + + Before this task, the no_expand_fields hook was gated on the ``source`` + key appearing at the recipe's top level, so a feature source living at + features.item.source would silently lose that protection. + + A bare string with no ``${...}`` marker (e.g. plain + ``RECOTEM_RECIPE_DB``) is never touched by ``expand_env_vars`` regardless + of whether no_expand_fields is honoured -- so asserting such a value + round-trips unchanged is vacuous; it would pass identically even if + feature sources got zero plugin-declared protection. SQLConfig.dsn_env's + own pattern (``^RECOTEM_RECIPE_[A-Z0-9_]+$``) also rejects any string + containing ``${`` or ``}``, so the real SQLSource cannot be used to embed + a literal reference and still validate. + + This mirrors + ``test_plugin_no_expand_fields_prevents_expansion_in_custom_field``'s + approach (mock ``get_source_class`` with a fake Config that has no + pattern constraint, embed a real ``${RECOTEM_RECIPE_X}`` reference, + assert the literal survives) but under ``features.item.source`` instead + of the top-level ``source``. It additionally mocks ``get_source_types`` + so ``Recipe.model_validate``'s ``_check_source_value`` accepts the fake + Config and the recipe loads all the way through, letting this test + assert on the actual field value instead of merely tolerating a + ``RecipeError``. + + Non-vacuity verified manually: monkeypatching + ``recotem.recipe.loader._resolve_extra_no_expand`` to return + ``frozenset()`` (simulating feature sources getting zero plugin-declared + protection -- the exact bug this task exists to prevent) makes this test + fail, because ``dsn_env`` is then expanded to the injected env value + instead of staying literal. + """ + from unittest.mock import patch + + from pydantic import BaseModel + + from recotem.datasource.registry import get_source_class as real_get_source_class + from recotem.datasource.registry import get_source_types as real_get_source_types + + dsn_ref = "${RECOTEM_RECIPE_X}" + monkeypatch.setenv("RECOTEM_RECIPE_X", "injected_value") + + class _FeatureSqlConfig(BaseModel, extra="ignore"): + type: str = "test_feature_sql_source" + dsn_env: str = "" + + class _FeatureSqlSource: + type_name: str = "test_feature_sql_source" + Config = _FeatureSqlConfig + extras_required: list = [] + no_expand_fields: frozenset = frozenset({"dsn_env"}) + + def __init__(self, config: _FeatureSqlConfig) -> None: # pragma: no cover + self.config = config + + def fetch(self, ctx): # pragma: no cover + raise NotImplementedError + + # Merge the fake type into the *real* registry snapshot (rather than + # replacing it outright) so the top-level csv source in + # _FEATURES_BASE_RECIPE still resolves normally through + # _check_source_value. + real_types = dict(real_get_source_types()) + merged_types = {**real_types, "test_feature_sql_source": _FeatureSqlSource} + + def _fake_get_source_class(type_name: str): + if type_name == "test_feature_sql_source": + return _FeatureSqlSource + return real_get_source_class(type_name) + + p = _write_features_recipe( + tmp_path, + "feat_plugin_no_expand", + f"""\ +features: + item: + source: + type: test_feature_sql_source + dsn_env: "{dsn_ref}" + id_column: item_id + columns: + - {{name: genre, encoding: categorical}} +""", + ) + + with ( + patch( + "recotem.datasource.registry.get_source_class", + side_effect=_fake_get_source_class, + ), + patch( + "recotem.datasource.registry.get_source_types", + return_value=merged_types, + ), + ): + recipe = load_recipe(p) + + # dsn_env must remain the literal ${...} reference; if no_expand_fields + # was not honoured for this feature subtree it would instead be expanded + # to the injected env value asserted against below. + assert recipe.features.item.source.dsn_env == dsn_ref, ( + f"no_expand_fields guard must preserve literal {dsn_ref!r} in " + "features.item.source.dsn_env; got: " + f"{recipe.features.item.source.dsn_env!r}" + ) + assert "injected_value" not in recipe.features.item.source.dsn_env, ( + "features.item.source.dsn_env must not receive env expansion; " + f"found injected value in: {recipe.features.item.source.dsn_env!r}" + ) + + +def test_feature_user_side_sql_source_dsn_env_not_expanded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Same as ``test_feature_sql_source_dsn_env_not_expanded`` above, but for + ``features.user.source`` instead of ``features.item.source``. + + ``_SOURCE_NODE_PATHS`` lists ``("features", "item")`` and + ``("features", "user")`` as the two symmetric feature-side positions + where a ``source`` mapping is treated as a genuine DataSource subtree + (see ``_is_source_node``). No existing test proved the user-side entry + is load-bearing: ``test_feature_sql_source_dsn_env_not_expanded`` only + covers ``features.item.source``, and + ``test_feature_user_side_source_also_typed_and_validated`` only covers + ``_validate_path_fields``'s scheme check, a mechanism entirely + independent of ``_is_source_node`` / ``_SOURCE_NODE_PATHS``. This test + closes that gap by mirroring the item-side test under + ``features.user.source``. + + Non-vacuity verified manually: removing ``("features", "user")`` from + ``_SOURCE_NODE_PATHS`` makes this test fail, because ``dsn_env`` is then + expanded to the injected env value instead of staying literal. + """ + from unittest.mock import patch + + from pydantic import BaseModel + + from recotem.datasource.registry import get_source_class as real_get_source_class + from recotem.datasource.registry import get_source_types as real_get_source_types + + dsn_ref = "${RECOTEM_RECIPE_X}" + monkeypatch.setenv("RECOTEM_RECIPE_X", "injected_value") + + class _FeatureSqlConfig(BaseModel, extra="ignore"): + type: str = "test_feature_sql_source" + dsn_env: str = "" + + class _FeatureSqlSource: + type_name: str = "test_feature_sql_source" + Config = _FeatureSqlConfig + extras_required: list = [] + no_expand_fields: frozenset = frozenset({"dsn_env"}) + + def __init__(self, config: _FeatureSqlConfig) -> None: # pragma: no cover + self.config = config + + def fetch(self, ctx): # pragma: no cover + raise NotImplementedError + + # Merge the fake type into the *real* registry snapshot (rather than + # replacing it outright) so the top-level csv source in + # _FEATURES_BASE_RECIPE still resolves normally through + # _check_source_value. + real_types = dict(real_get_source_types()) + merged_types = {**real_types, "test_feature_sql_source": _FeatureSqlSource} + + def _fake_get_source_class(type_name: str): + if type_name == "test_feature_sql_source": + return _FeatureSqlSource + return real_get_source_class(type_name) + + p = _write_features_recipe( + tmp_path, + "feat_user_plugin_no_expand", + f"""\ +features: + user: + source: + type: test_feature_sql_source + dsn_env: "{dsn_ref}" + id_column: user_id + columns: + - {{name: segment, encoding: categorical}} +""", + ) + + with ( + patch( + "recotem.datasource.registry.get_source_class", + side_effect=_fake_get_source_class, + ), + patch( + "recotem.datasource.registry.get_source_types", + return_value=merged_types, + ), + ): + recipe = load_recipe(p) + + # dsn_env must remain the literal ${...} reference; if no_expand_fields + # was not honoured for this feature subtree it would instead be expanded + # to the injected env value asserted against below. + assert recipe.features.user.source.dsn_env == dsn_ref, ( + f"no_expand_fields guard must preserve literal {dsn_ref!r} in " + "features.user.source.dsn_env; got: " + f"{recipe.features.user.source.dsn_env!r}" + ) + assert "injected_value" not in recipe.features.user.source.dsn_env, ( + "features.user.source.dsn_env must not receive env expansion; " + f"found injected value in: {recipe.features.user.source.dsn_env!r}" + ) + + +def test_feature_sql_query_not_env_expanded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """features.item.source.query must not receive ${...} env expansion. + + query is already covered by the global _NO_EXPAND_KEYS baseline (applies + at any nesting depth), independent of the plugin-declared no_expand_fields + generalisation exercised above. + """ + monkeypatch.setenv("RECOTEM_RECIPE_TBL", "evil") + monkeypatch.setenv("RECOTEM_RECIPE_DB", "postgresql://h/db") + p = _write_features_recipe( + tmp_path, + "feat_sql_query", + """\ +features: + item: + source: + type: sql + dsn_env: RECOTEM_RECIPE_DB + query: SELECT item_id FROM ${RECOTEM_RECIPE_TBL} + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + recipe = load_recipe(p) + assert "${RECOTEM_RECIPE_TBL}" in recipe.features.item.source.query + + +def test_feature_user_side_source_also_typed_and_validated(tmp_path: Path) -> None: + """features.user.source must get the same typed-resolution treatment.""" + p = _write_features_recipe( + tmp_path, + "feat_user_side", + """\ +features: + user: + source: + type: csv + path: ftp://evil.example/users.csv + id_column: user_id + columns: + - {name: segment, encoding: categorical} +""", + ) + with pytest.raises(RecipeError, match="scheme"): + load_recipe(p) + + +# --------------------------------------------------------------------------- +# Task 3 review fix: _is_source_node must not false-positive on a freeform +# field (e.g. BigQueryConfig.query_parameters) that happens to contain a key +# literally named 'source'. +# --------------------------------------------------------------------------- + + +def test_query_parameters_named_source_not_misdetected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A query_parameters entry literally named 'source' is not a DataSource. + + BigQueryConfig.query_parameters is genuinely freeform + (dict[str, Any] | None) -- a query may legitimately bind a parameter + named 'source' whose value is an unrelated nested mapping (e.g. a + struct-typed parameter). Before the fix, ``_is_source_node`` matched on + the key name 'source' at *any* nesting depth, so this nested mapping + (whose own 'type' field happens to look like a DataSource discriminator) + was mistaken for a features.*.source subtree and triggered a spurious + plugin-type lookup, failing recipe load with a confusing "Unknown + DataSource type 'organic'" error even though no DataSource was ever + referenced. + """ + monkeypatch.setenv("RECOTEM_RECIPE_CHANNEL", "leaked_value") + content = f"""\ +name: bq_query_param_source +source: + type: bigquery + query: "SELECT * FROM t WHERE traffic_source = @source" + query_parameters: + source: {{type: organic, channel: "${{RECOTEM_RECIPE_CHANNEL}}"}} +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [TopPop] + n_trials: 1 +output: + path: {tmp_path / "bq_query_param_source.recotem"} +""" + p = _write_recipe(tmp_path, content) + + # Must not raise -- the nested 'source' mapping under query_parameters is + # not a DataSource subtree and must not trigger plugin-type discovery. + recipe = load_recipe(p) + + # query_parameters is globally no-expand (_NO_EXPAND_KEYS), so the nested + # mapping -- including its 'channel' string -- must survive completely + # unexpanded and untouched, proving no accidental env-var substitution + # occurred while walking the (correctly non-)detected node. + assert recipe.source.query_parameters == { + "source": {"type": "organic", "channel": "${RECOTEM_RECIPE_CHANNEL}"} + } + + +# Note: proof that _is_source_node's position-based narrowing does not +# over-narrow away from the two legitimate features..source positions +# lives in test_feature_sql_source_dsn_env_not_expanded above (Task 3 +# section) — it embeds a real ${...} reference in a plugin-declared +# no_expand field under features.item.source and asserts the literal +# survives. That assertion is sensitive to exactly this: manually reverting +# _SOURCE_NODE_PATHS to omit ("features", "item") makes it fail (dsn_env +# gets expanded), confirming detection still fires at that position. An +# additional test asserting only that an unregistered type under +# features.item.source raises a RecipeError would NOT be a useful over- +# narrowing check here: _resolve_source_node's typed-resolution step (a +# separate, unconditional lookup made regardless of _is_source_node) raises +# an equivalent error on its own, so such a test would pass identically +# whether or not _is_source_node detects the position. + + +# --------------------------------------------------------------------------- +# Source-resolution errors must name BOTH the recipe file and the subtree +# +# A recipe now has up to three source subtrees (top-level, features.item, +# features.user), so an error must say which file AND which subtree. The +# file name matters most for load_recipes_directory (public API), which does +# not re-add it: without it a failing 20-file directory names no file at all. +# --------------------------------------------------------------------------- + + +def _bad_field_source_recipe(tmp_path: Path, name: str) -> Path: + """Write a recipe whose *top-level* source carries an unknown field. + + ``bogus_field`` trips CSVConfig's ``extra="forbid"``, driving + ``_resolve_source_node``'s pydantic-ValidationError branch. + """ + content = f"""\ +name: {name} +source: + type: csv + path: /tmp/data.csv + bogus_field: 1 +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [TopPop] + n_trials: 1 +output: + path: {tmp_path / f"{name}.recotem"} +""" + return _write_recipe(tmp_path, content, filename=f"{name}.yaml") + + +def test_source_validation_error_names_recipe_file(tmp_path: Path) -> None: + """A bad top-level source must name the recipe file and the subtree. + + Restores the file path that ``Recipe '{p}' source failed validation:`` + carried before the ``_resolve_source_node`` refactor, which reduced the + message to a bare ``source failed validation:``. + """ + p = _bad_field_source_recipe(tmp_path, "prod_events") + + with pytest.raises(RecipeError) as excinfo: + load_recipe(p) + + msg = str(excinfo.value) + assert str(p) in msg, f"message must name the recipe file {str(p)!r}; got: {msg}" + assert "source failed validation" in msg, msg + # The underlying pydantic detail must survive the prefixing. + assert "bogus_field" in msg, msg + + +def test_feature_source_validation_error_names_recipe_file_and_subtree( + tmp_path: Path, +) -> None: + """A bad features.item.source names the file *and* the subtree. + + The subtree location alone is not enough: load_recipes_directory raises + straight through, so an operator loading a directory needs the file too. + """ + p = _write_features_recipe( + tmp_path, + "feat_bad_field", + """\ +features: + item: + source: + type: csv + path: ./items.csv + bogus_field: 1 + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + + with pytest.raises(RecipeError) as excinfo: + load_recipe(p) + + msg = str(excinfo.value) + assert str(p) in msg, f"message must name the recipe file {str(p)!r}; got: {msg}" + assert "features.item.source failed validation" in msg, ( + f"message must locate the failing subtree; got: {msg}" + ) + + +def test_source_missing_type_error_names_recipe_file(tmp_path: Path) -> None: + """The missing-'type'-discriminator message must name the recipe file. + + Complements test_load_recipe_source_missing_type_raises_recipe_error, + which only asserts that *some* RecipeError is raised. + """ + content = f"""\ +name: no_type_named +source: + path: /tmp/data.csv +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [TopPop] + n_trials: 1 +output: + path: {tmp_path / "no_type_named.recotem"} +""" + p = _write_recipe(tmp_path, content, filename="no_type_named.yaml") + + with pytest.raises(RecipeError) as excinfo: + load_recipe(p) + + msg = str(excinfo.value) + assert str(p) in msg, f"message must name the recipe file {str(p)!r}; got: {msg}" + assert "missing the 'type' discriminator" in msg, msg + + +def test_load_recipes_directory_source_error_names_offending_file( + tmp_path: Path, +) -> None: + """load_recipes_directory is public API and does not re-add the filename. + + With several recipes on disk, the raised error must still identify which + one failed -- the concrete reason the path belongs in the message rather + than being left to the CLI's single-file framing. + """ + good = tmp_path / "good.yaml" + good.write_text( + MINIMAL_RECIPE_TEMPLATE.format( + name="good_recipe", + output_path=str(tmp_path / "good_recipe.recotem"), + ) + ) + bad = _bad_field_source_recipe(tmp_path, "bad_recipe") + + with pytest.raises(RecipeError) as excinfo: + load_recipes_directory(tmp_path) + + msg = str(excinfo.value) + assert str(bad) in msg, ( + f"directory load must name the offending file {str(bad)!r}; got: {msg}" + ) + + +def test_plugin_discovery_error_distinguishes_which_source_is_broken( + tmp_path: Path, +) -> None: + """An unknown 'type' must say WHICH of the three sources carries it. + + ``_resolve_extra_no_expand`` reports the type name only, so two recipes + differing solely in which subtree holds the typo produced byte-identical + output. The type name was self-identifying on main (one source per + recipe); with features there are three, so the subtree is required to + tell them apart. + """ + top_level = _write_recipe( + tmp_path, + f"""\ +name: top_typo +source: + type: no_such_source_type + path: /tmp/data.csv +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [IALS] + n_trials: 1 +output: + path: {tmp_path / "top_typo.recotem"} +""", + filename="top_typo.yaml", + ) + feature_side = _write_features_recipe( + tmp_path, + "feat_typo", + """\ +features: + item: + source: + type: no_such_source_type + path: ./items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + + with pytest.raises(RecipeError) as top_exc: + load_recipe(top_level) + with pytest.raises(RecipeError) as feat_exc: + load_recipe(feature_side) + + top_msg = str(top_exc.value) + feat_msg = str(feat_exc.value) + + assert str(top_level) in top_msg, top_msg + assert str(feature_side) in feat_msg, feat_msg + assert "features.item.source" in feat_msg, ( + f"feature-side typo must name the subtree; got: {feat_msg}" + ) + assert "features" not in top_msg, ( + f"top-level typo must not be attributed to a feature subtree; got: {top_msg}" + ) + assert top_msg != feat_msg, ( + "the two recipes differ only in which source carries the typo, so " + f"their errors must not be identical; both were: {top_msg}" + ) diff --git a/tests/unit/test_recipe_models.py b/tests/unit/test_recipe_models.py index 6af4907a..f9bf9c15 100644 --- a/tests/unit/test_recipe_models.py +++ b/tests/unit/test_recipe_models.py @@ -15,6 +15,9 @@ from recotem.recipe.models import ( CleansingConfig, + FeatureColumn, + FeaturesConfig, + FeatureSideConfig, ItemMetadataConfig, OutputConfig, Recipe, @@ -619,3 +622,203 @@ def test_validate_source_warning_contains_error_class_and_message() -> None: warn = warnings[0] assert warn.get("error_class") == "RuntimeError" assert "plugin entry_point collision" in warn.get("error", "") + + +# --------------------------------------------------------------------------- +# Task 1: features block — FeatureColumn / FeatureSideConfig / FeaturesConfig +# --------------------------------------------------------------------------- + + +def test_feature_column_defaults(): + col = FeatureColumn(name="genre", encoding="categorical") + assert col.delimiter is None + assert col.min_frequency == 1 + + +def test_feature_column_rejects_unknown_encoding(): + with pytest.raises(ValidationError): + FeatureColumn(name="genre", encoding="one_hot") + + +def test_delimiter_requires_multi_label(): + with pytest.raises(ValidationError, match="delimiter is only valid"): + FeatureColumn(name="genre", encoding="categorical", delimiter="|") + + +def test_multi_label_defaults_delimiter(): + col = FeatureColumn(name="genres", encoding="multi_label") + assert col.delimiter == "|" + + +def test_min_frequency_rejected_on_numerical(): + with pytest.raises(ValidationError, match="min_frequency is only valid"): + FeatureColumn(name="year", encoding="numerical", min_frequency=5) + + +def test_feature_side_requires_at_least_one_column(): + with pytest.raises(ValidationError): + FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, id_column="item_id", columns=[] + ) + + +def test_feature_side_rejects_duplicate_column_names(): + with pytest.raises(ValidationError, match="duplicate"): + FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="genre", encoding="categorical"), + ], + ) + + +# --------------------------------------------------------------------------- +# Fix H: an id_column equal to one of the columns[].name is guaranteed to fail +# at train time (set_index consumes it, then build_encoder_state raises a +# misleading "feature column '...' is not present"). Surface it early at recipe +# load with a clear message. +# --------------------------------------------------------------------------- + + +def test_feature_side_rejects_id_column_colliding_with_feature_column(): + with pytest.raises(ValidationError, match="id_column"): + FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, + id_column="x", + columns=[FeatureColumn(name="x", encoding="categorical")], + ) + + +def test_feature_side_id_column_distinct_from_columns_ok(): + """The common case -- id_column not among the feature columns -- must still + construct, so the collision guard is not over-broad.""" + cfg = FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + assert cfg.id_column == "item_id" + + +# --------------------------------------------------------------------------- +# Characterization: an empty delimiter is rejected for BOTH multi_label (empty +# is meaningless as a split token) and any other encoding (delimiter is only +# valid for multi_label at all). Documents the existing FeatureColumn behavior. +# --------------------------------------------------------------------------- + + +def test_feature_column_multi_label_empty_delimiter_rejected(): + with pytest.raises(ValidationError, match="delimiter must not be empty"): + FeatureColumn(name="tags", encoding="multi_label", delimiter="") + + +def test_feature_column_categorical_empty_delimiter_rejected(): + with pytest.raises(ValidationError, match="delimiter is only valid"): + FeatureColumn(name="genre", encoding="categorical", delimiter="") + + +def test_features_config_requires_a_side(): + with pytest.raises(ValidationError, match="at least one of"): + FeaturesConfig() + + +def test_features_config_item_only_is_valid(): + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + assert cfg.user is None + + +# --------------------------------------------------------------------------- +# Task 2: features block — Recipe-level validation +# --------------------------------------------------------------------------- + + +@pytest.fixture +def minimal_recipe_kwargs() -> dict: + return { + "name": "probe", + "source": {"type": "csv", "path": "./x.csv"}, + "schema": {"user_column": "user_id", "item_column": "item_id"}, + "training": TrainingConfig(algorithms=["IALS"]), + "output": {"path": "./out.recotem"}, + } + + +def _feature_side() -> FeatureSideConfig: + return FeatureSideConfig( + source={"type": "csv", "path": "./x.csv"}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + + +def test_features_without_feature_capable_algorithm_rejected( + minimal_recipe_kwargs, +) -> None: + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig(item=_feature_side()) + kwargs["training"] = TrainingConfig(algorithms=["TopPop", "CosineKNN"]) + with pytest.raises(ValidationError, match="no feature-capable algorithm"): + Recipe(**kwargs) + + +def test_features_with_ials_accepted(minimal_recipe_kwargs) -> None: + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig(item=_feature_side()) + kwargs["training"] = TrainingConfig(algorithms=["TopPop", "IALS"]) + assert Recipe(**kwargs).features is not None + + +def test_unknown_algorithm_with_features_still_deferred_to_train_time( + minimal_recipe_kwargs, +) -> None: + # An unresolvable name must not turn into a features error; it stays + # deferred to train time, matching the existing tolerance. + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig(item=_feature_side()) + kwargs["training"] = TrainingConfig(algorithms=["IALS", "NoSuchAlgo"]) + assert Recipe(**kwargs).features is not None + + +def test_feature_source_with_unregistered_type_rejected( + minimal_recipe_kwargs, +) -> None: + """Direct Recipe(...) construction bypasses load_recipe's typed resolution. + Recipe.source is guarded by _validate_source; features.*.source must be too. + """ + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "unknown_xyz"}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(ValidationError, match="unknown_xyz"): + Recipe(**kwargs) + + +def test_feature_source_with_registered_type_accepted(minimal_recipe_kwargs) -> None: + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig(item=_feature_side()) + assert Recipe(**kwargs).features.item is not None + + +def test_user_side_feature_source_also_validated(minimal_recipe_kwargs) -> None: + kwargs = dict(minimal_recipe_kwargs) + kwargs["features"] = FeaturesConfig( + user=FeatureSideConfig( + source={"type": "unknown_xyz"}, + id_column="user_id", + columns=[FeatureColumn(name="band", encoding="categorical")], + ) + ) + with pytest.raises(ValidationError, match="unknown_xyz"): + Recipe(**kwargs) diff --git a/tests/unit/test_serving_cold_start.py b/tests/unit/test_serving_cold_start.py new file mode 100644 index 00000000..d4685659 --- /dev/null +++ b/tests/unit/test_serving_cold_start.py @@ -0,0 +1,1640 @@ +# tests/unit/test_serving_cold_start.py +"""Cold-start coverage for the ``:recommend`` / ``:recommend-related`` verbs. + +Case table (see ``routes.py`` docstrings for the full rationale): + +| Case | Verb | Trigger | Method | +|------|---------------------|-----------------------------------|--------------------------------------| +| A | :recommend | unknown user + user_features | get_recommendation_for_cold_user | +| B | :recommend-related | known/mixed seeds + user_features | get_recommendation_for_new_user(...) | +| C | :recommend-related | a cold seed carries item_features | get_recommendation_for_cold_seeds | + +Two fixtures back every test: + +- ``client`` / ``fa`` recipe: a REAL, tiny (2-epoch) feature-aware IALS -- + same construction as ``tests/unit/test_idmap.py``'s ``fa_model`` fixture + (40 users ``u0..u39``, 20 items ``i0..i19``, categorical ``band``/``genre`` + features) -- so cold-start calls exercise the genuine irspack API surface, + not a mock standing in for it. +- ``plain_client`` / ``plain`` recipe: a TopPop model with NO feature state + at all, and a user-id space that deliberately excludes ``"u1"`` so a + request naming ``"u1"`` is a genuine cold-start attempt (not the + known-user path) that must hit the "no feature state" guard and 400. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +import pytest +import scipy.sparse as sps +import structlog.contextvars +import structlog.testing +from fastapi.testclient import TestClient + +from recotem.serving.registry import ModelEntry, ModelRegistry +from tests.conftest import build_v1_app + +_FAKE_SHA256_HEX = "d" * 64 # 64 lowercase hex chars for a valid Sha256Hex marker + + +def _fa_recommender(*, n_threads: int | None = None) -> object: + """Build the same real feature-aware IALS as test_idmap.py's fa_model. + + Not imported from there because that fixture is function-scoped and + private to test_idmap.py; duplicated here (benchmarked at ~1-2ms to + train) so this module has no cross-test-module fixture dependency. + + *n_threads* defaults to irspack's own auto-sizing -- the production + default, and what every test here that does not assert an exact ranking + should use. Pass ``1`` only to make the ranking itself reproducible; see + ``stable_ranking_client``. + """ + from irspack import IALSRecommender + + from recotem._features import build_encoder_state, encode + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + rng = np.random.default_rng(0) + n_users, n_items = 40, 20 + X = sps.csr_matrix((rng.random((n_users, n_items)) > 0.6).astype(np.float64)) + item_df = pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(n_items)], + "genre": ["action" if i % 2 else "drama" for i in range(n_items)], + } + ).set_index("item_id") + user_df = pd.DataFrame( + { + "user_id": [f"u{u}" for u in range(n_users)], + "band": ["young" if u % 2 else "old" for u in range(n_users)], + } + ).set_index("user_id") + + istate = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + ustate = build_encoder_state( + user_df, [FeatureColumn(name="band", encoding="categorical")] + ) + F = encode(istate, item_df, index_order=[f"i{i}" for i in range(n_items)]) + U = encode(ustate, user_df, index_order=[f"u{u}" for u in range(n_users)]) + + rec = IALSRecommender( + X, + n_components=8, + alpha0=0.1, + train_epochs=2, + random_seed=42, + item_features=F, + user_features=U, + lambda_item_feature=1e-1, + lambda_user_feature=1e-1, + n_threads=n_threads, + ).learn() + return IDMappedRecommender( + rec, + [f"u{u}" for u in range(n_users)], + [f"i{i}" for i in range(n_items)], + item_feature_state=istate, + user_feature_state=ustate, + ) + + +def _fa_recommender_with_numerical_item_feature() -> object: + """Same construction as ``_fa_recommender``, plus a numerical ``tight`` + item column with a realistic small std. + + Reproduces review finding 1: a client-supplied ``numerical`` feature + value that is extreme but finite (e.g. ``1e22``) standardizes + (``recotem._features._row_values``) to a magnitude that makes irspack's + per-request conjugate-gradient cold-start solve ill-conditioned, + independently confirmed against this exact fixture shape to raise + ``RuntimeError: Conjugate-gradient solver encountered a singular + system.`` pre-fix. + """ + from irspack import IALSRecommender + + from recotem._features import build_encoder_state, encode + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + rng = np.random.default_rng(0) + n_users, n_items = 40, 20 + X = sps.csr_matrix((rng.random((n_users, n_items)) > 0.6).astype(np.float64)) + item_df = pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(n_items)], + "genre": ["action" if i % 2 else "drama" for i in range(n_items)], + "tight": rng.normal(loc=0.0, scale=0.5, size=n_items), + } + ).set_index("item_id") + + istate = build_encoder_state( + item_df, + [ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="tight", encoding="numerical"), + ], + ) + F = encode(istate, item_df, index_order=[f"i{i}" for i in range(n_items)]) + + rec = IALSRecommender( + X, + n_components=8, + alpha0=0.1, + train_epochs=2, + random_seed=42, + item_features=F, + lambda_item_feature=1e-1, + ).learn() + return IDMappedRecommender( + rec, + [f"u{u}" for u in range(n_users)], + [f"i{i}" for i in range(n_items)], + item_feature_state=istate, + ) + + +def _fa_recommender_with_near_constant_numerical_item_feature() -> object: + """Same construction as ``_fa_recommender_with_numerical_item_feature``, + except the SERVE-TIME ``tight`` encoder state has its ``std`` overridden + to a hand-crafted, near-zero-but-nonzero value (``1.36e-15``, matching + the review's reported reproduction) after training completes. + + Training itself uses the item feature matrix built from the ORIGINAL, + normally-scaled ``std`` (so this override cannot make training + ill-conditioned); only ``item_feature_state`` -- read exclusively at + serve-time cold-start scoring -- is mutated afterwards. This exists to + prove the route-level ``FEATURE_VALUE_UNUSABLE`` `detail` message is + truthful independent of *how* a tiny std ends up in a feature state: an + ordinary-looking raw request value (e.g. ``1e4``, not ``1e22``) against + a near-constant column must still 400 with a message that blames the + STANDARDIZED value, not the raw one -- the raw value here is not, by any + reasonable definition, extreme. + """ + rec = _fa_recommender_with_numerical_item_feature() + tight_spec = next( + s for s in rec.item_feature_state["columns"] if s["name"] == "tight" + ) + assert tight_spec["std"] > 1e-3, ( + "test setup invariant: the original std must be a normal, non-tiny " + "value so this override is an observable change, not a no-op" + ) + tight_spec["std"] = 1.36e-15 + return rec + + +def _fa_recommender_with_numerical_user_feature() -> object: + """Same construction as ``_fa_recommender_with_numerical_item_feature``, + but the ``tight`` numerical column lives on the USER side. + + Exercises case A (``get_score_cold_user_from_features``, reached via + ``:recommend``) and case B (``get_score_cold_user``, reached via + ``:recommend-related``) with a genuine numerical cold-start failure -- + the item-only fixture above only reaches case C. Independently confirmed + against this exact fixture shape: ``{"band": "young", "tight": 1e22}`` + raises ``RuntimeError: Conjugate-gradient solver encountered a singular + system.`` pre-fix for both ``get_score_cold_user_from_features`` and + ``get_score_cold_user``. + """ + from irspack import IALSRecommender + + from recotem._features import build_encoder_state, encode + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + rng = np.random.default_rng(0) + n_users, n_items = 40, 20 + X = sps.csr_matrix((rng.random((n_users, n_items)) > 0.6).astype(np.float64)) + user_df = pd.DataFrame( + { + "user_id": [f"u{u}" for u in range(n_users)], + "band": ["young" if u % 2 else "old" for u in range(n_users)], + "tight": rng.normal(loc=0.0, scale=0.5, size=n_users), + } + ).set_index("user_id") + + ustate = build_encoder_state( + user_df, + [ + FeatureColumn(name="band", encoding="categorical"), + FeatureColumn(name="tight", encoding="numerical"), + ], + ) + U = encode(ustate, user_df, index_order=[f"u{u}" for u in range(n_users)]) + + rec = IALSRecommender( + X, + n_components=8, + alpha0=0.1, + train_epochs=2, + random_seed=42, + user_features=U, + lambda_user_feature=1e-1, + ).learn() + return IDMappedRecommender( + rec, + [f"u{u}" for u in range(n_users)], + [f"i{i}" for i in range(n_items)], + user_feature_state=ustate, + ) + + +def _entry(name: str, recommender: object) -> ModelEntry: + return ModelEntry( + name=name, + recommender=recommender, + header={}, + kid="t", + metadata_df=None, + metadata_index=None, + loaded=True, + _loaded_marker=(None, _FAKE_SHA256_HEX), + loaded_at_unix=1747800000.0, + ) + + +def _plain_recommender() -> object: + """A TopPop model with no feature state and no "u1" in its user space.""" + from irspack import TopPopRecommender + + from recotem._idmap import IDMappedRecommender + + n_users, n_items = 3, 3 + X = sps.csr_matrix(np.ones((n_users, n_items))) + rec = TopPopRecommender(X).learn() + return IDMappedRecommender(rec, ["p1", "p2", "p3"], ["x1", "x2", "x3"]) + + +def _toppop_with_feature_state_recommender() -> object: + """A TopPop model that carries a non-None ``user_feature_state``. + + Task 9 persists the feature encoder state unconditionally -- even when + the Optuna search winner is not feature-capable -- so the artifact + header and payload agree. That means ``state is not None`` does NOT + imply cold start is available: TopPop is not in + ``_FEATURE_CAPABLE_CLASS_NAMES``, so Task 11's ``_require_capability`` + must still refuse it. This is a genuinely different code path from + ``_plain_recommender`` (no state at all), which hits the earlier + ``user_feature_state is None`` guard instead. + """ + from irspack import TopPopRecommender + + from recotem._features import build_encoder_state + from recotem._idmap import IDMappedRecommender + from recotem.recipe.models import FeatureColumn + + n_users, n_items = 3, 3 + X = sps.csr_matrix(np.ones((n_users, n_items))) + rec = TopPopRecommender(X).learn() + user_df = pd.DataFrame( + {"user_id": ["p1", "p2", "p3"], "band": ["young", "old", "young"]} + ).set_index("user_id") + ustate = build_encoder_state( + user_df, [FeatureColumn(name="band", encoding="categorical")] + ) + return IDMappedRecommender( + rec, + ["p1", "p2", "p3"], + ["x1", "x2", "x3"], + user_feature_state=ustate, + ) + + +@pytest.fixture() +def client() -> TestClient: + registry = ModelRegistry() + registry.replace("fa", _entry("fa", _fa_recommender())) + return TestClient(build_v1_app(registry)) + + +@pytest.fixture() +def stable_ranking_client() -> TestClient: + """``client``'s model, but with a reproducible ranking. + + irspack sizes its thread pool automatically, and its cold-start solve + reduces in whatever order the threads finish, so two identical calls can + return slightly different scores. This fixture's catalog has genuinely + near-tied items (measured: ranks 2-4 sit within 1.4e-6 of each other in + float32), which that jitter is more than enough to reorder -- measured + at 2 distinct top-5 orderings across 50 identical in-process calls, and + it reordered ACROSS the cutoff boundary, so the top-5 membership itself + changed, not just the order within it. Any test that asserts which items + come back is therefore flaky against the default fixture. + + ``n_threads=1`` removes the reduction-order jitter and nothing else: it + is the same model, same seed, same scores. Measured deterministic at 200 + identical calls per case (A, B, and C) vs. the 2 orderings above. + + Use ``client`` for anything that does not pin an exact ranking; this + trades irspack's production thread default away for reproducibility, so + it earns its keep only where the ranking IS the assertion. + """ + registry = ModelRegistry() + registry.replace("fa", _entry("fa", _fa_recommender(n_threads=1))) + return TestClient(build_v1_app(registry)) + + +@pytest.fixture() +def plain_client() -> TestClient: + registry = ModelRegistry() + registry.replace("plain", _entry("plain", _plain_recommender())) + return TestClient(build_v1_app(registry)) + + +@pytest.fixture() +def toppop_with_state_client() -> TestClient: + registry = ModelRegistry() + registry.replace( + "toppop_state", + _entry("toppop_state", _toppop_with_feature_state_recommender()), + ) + return TestClient(build_v1_app(registry)) + + +# --------------------------------------------------------------------------- +# :recommend -- case A (features only) + the "ignored, not rejected" rule +# --------------------------------------------------------------------------- + + +def test_known_user_unchanged_without_features(client: TestClient) -> None: + r = client.post("/v1/recipes/fa:recommend", json={"user_id": "u1", "limit": 3}) + assert r.status_code == 200 + + +def test_known_user_ignores_supplied_features(client: TestClient) -> None: + """The learned embedding was fit to real interactions and strictly + dominates the profile prior. Rejecting with 400 would break the natural + client pattern of always sending the profile.""" + plain = client.post("/v1/recipes/fa:recommend", json={"user_id": "u1", "limit": 3}) + with_f = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "u1", "limit": 3, "user_features": {"band": "young"}}, + ) + assert with_f.status_code == 200 + assert with_f.json()["items"] == plain.json()["items"] + + +def test_unknown_user_with_features_is_served(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "never_seen", "limit": 3, "user_features": {"band": "young"}}, + ) + assert r.status_code == 200 + assert len(r.json()["items"]) == 3 + + +def test_unknown_user_without_features_still_404(client: TestClient) -> None: + r = client.post("/v1/recipes/fa:recommend", json={"user_id": "never_seen"}) + assert r.status_code == 404 + # Response body is flat (``{"detail": ..., "code": ...}``), not nested + # under an "error" key -- verified against the unmodified handler via + # ``_http_exception_handler`` in ``tests/conftest.py``'s ``build_v1_app``. + assert r.json()["code"] == "UNKNOWN_USER" + + +def test_features_on_model_without_state_is_400(plain_client: TestClient) -> None: + r = plain_client.post( + "/v1/recipes/plain:recommend", + json={"user_id": "u1", "user_features": {"band": "young"}}, + ) + assert r.status_code == 400 + + +def test_features_on_toppop_with_state_is_400_not_500( + toppop_with_state_client: TestClient, +) -> None: + """Task 9 persists feature state unconditionally, so a TopPop artifact + can carry a non-None ``user_feature_state`` despite TopPop not being + feature-capable. ``state is not None`` must NOT be read as "cold start + is available" -- only ``_require_capability``'s class-name allow-list + decides that (see ``_idmap.py``). This is a different code path from + ``test_features_on_model_without_state_is_400`` above, which covers "no + state at all" and never reaches ``_require_capability``. Must produce a + clean 400 FEATURES_NOT_SUPPORTED, never a 500, and must not leak the raw + feature value into the response body. + """ + r = toppop_with_state_client.post( + "/v1/recipes/toppop_state:recommend", + json={ + "user_id": "never_seen", + "user_features": {"band": "super_secret_value"}, + }, + ) + assert r.status_code == 400 + assert r.json()["code"] == "FEATURES_NOT_SUPPORTED" + assert "super_secret_value" not in r.text + + +# --------------------------------------------------------------------------- +# :recommend-related -- case B (features + history) and case C (cold seed) +# --------------------------------------------------------------------------- + + +def test_related_with_user_features_is_case_b(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:recommend-related", + json={"seed_items": ["i0"], "limit": 3, "user_features": {"band": "young"}}, + ) + assert r.status_code == 200 + + +def test_related_with_cold_seed_is_case_c(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:recommend-related", + json={ + "seed_items": ["brand_new"], + "limit": 3, + "item_features": {"brand_new": {"genre": "action"}}, + }, + ) + assert r.status_code == 200 + + +def test_related_all_known_seeds_unchanged(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:recommend-related", json={"seed_items": ["i0"], "limit": 3} + ) + assert r.status_code == 200 + + +def test_unknown_seed_without_features_still_404(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:recommend-related", json={"seed_items": ["nope"], "limit": 3} + ) + assert r.status_code == 404 + assert r.json()["code"] == "UNKNOWN_SEED_ITEMS" + + +# --------------------------------------------------------------------------- +# Precedence proof -- case C must win over case B when both are supplied. +# --------------------------------------------------------------------------- +# +# A cold seed has no row in the seed interaction matrix, so if case B's +# solve (get_recommendation_for_new_user) ran on a request that names a +# cold seed, that seed would be silently dropped from the interaction +# history it is supposed to seed. This test names ONE cold seed with +# item_features AND supplies user_features, so both case B's and case C's +# trigger conditions are simultaneously true -- only the precedence rule +# decides which one runs. +# --------------------------------------------------------------------------- + + +def test_cold_seed_with_item_and_user_features_prefers_case_c( + client: TestClient, +) -> None: + r = client.post( + "/v1/recipes/fa:recommend-related", + json={ + "seed_items": ["brand_new"], + "limit": 3, + "user_features": {"band": "young"}, + "item_features": {"brand_new": {"genre": "action"}}, + }, + ) + assert r.status_code == 200 + + +# --------------------------------------------------------------------------- +# Metrics coverage -- every call site on all three cold-start paths must +# actually fire (Task 12 wiring). +# +# The reviewer mutated each of the five ``_metrics`` call sites in +# ``routes.py`` individually (case A ~:368-370, case C ~:531-533, case B +# ~:562-564) and found that deleting ``inc_cold_start_request`` on ANY of +# the three paths caused zero test failures, and deleting +# ``inc_feature_unknown_value`` on cases B/C also caused zero test +# failures -- only case A's ``inc_feature_unknown_value`` was pinned (by +# the single-case test this one replaces). An unknown feature value cannot +# fail the request -- it silently degrades to an all-zero segment -- so +# these counters are the only operator-facing signal; a metrics call that +# no test can kill is free to be deleted by any future refactor. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("path", "body", "expected_case", "expected_unknown"), + [ + pytest.param( + "/v1/recipes/fa:recommend", + { + "user_id": "never_seen", + "limit": 3, + "user_features": {"band": "martian"}, + }, + "features_only", + [("fa", "user", "band")], + id="case-a-features-only", + ), + pytest.param( + "/v1/recipes/fa:recommend-related", + { + "seed_items": ["i0"], + "limit": 3, + "user_features": {"band": "martian"}, + }, + "features_and_history", + [("fa", "user", "band")], + id="case-b-features-and-history", + ), + pytest.param( + "/v1/recipes/fa:recommend-related", + { + "seed_items": ["brand_new"], + "limit": 3, + "item_features": {"brand_new": {"genre": "martian_genre"}}, + }, + "cold_seeds", + [("fa", "item", "genre")], + id="case-c-cold-seeds", + ), + ], +) +def test_cold_start_metrics_fire_for_every_case( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: dict, + expected_case: str, + expected_unknown: list[tuple[str, str, str]], +) -> None: + cold_start_calls: list[tuple[str, str]] = [] + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_cold_start_request", + lambda recipe, case: cold_start_calls.append((recipe, case)), + ) + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + r = client.post(path, json=body) + assert r.status_code == 200 + assert cold_start_calls == [("fa", expected_case)] + assert unknown_calls == expected_unknown + + +# --------------------------------------------------------------------------- +# Unknown feature COLUMN (a request key outside the recipe) +# --------------------------------------------------------------------------- +# +# ``_features._row_values`` iterates ``state["columns"]`` and does +# ``values.get(name)``, so a request key the recipe never declared is simply +# never read: a fully typo'd body encodes bias-only and is byte-identical to +# sending ``{}``. That is a strictly MORE severe silent degradation than an +# unknown VALUE (which already has a counter), so it must not be the one +# place ``encode_one``'s own rule -- "an unknown category degrades the +# recommendation silently, so it must not also be invisible" -- goes +# unapplied. The response stays 200 (deliberately: rejecting would break +# clients that legitimately send a superset profile); the counter is the +# signal. +# +# The column name is deliberately NOT a metric label. Unlike +# ``inc_feature_unknown_value``'s ``column`` (bounded by the operator's own +# recipe), an unknown column name comes from request input -- an unbounded +# label there is a metrics-cardinality DoS. + + +def test_unknown_user_feature_column_is_counted( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "never_seen", "limit": 3, "user_features": {"bandd": "young"}}, + ) + assert r.status_code == 200 + assert calls == [("fa", "user")] + + +def test_fully_typoed_feature_body_is_counted_not_silent( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """The reported case: every key typo'd, so the encode is bias-only and the + response is byte-identical to sending ``user_features: {}``. Before the + counter this was indistinguishable from a correct request.""" + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + typoed = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "never_seen", "limit": 3, "user_features": {"bandd": "young"}}, + ) + empty = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "never_seen", "limit": 3, "user_features": {}}, + ) + assert typoed.json()["items"] == empty.json()["items"], ( + "precondition: a typo'd column is invisible in the RESPONSE -- the " + "counter is the only place it can surface" + ) + assert calls == [("fa", "user")], "only the typo'd request may increment" + + +def test_multiple_unknown_columns_count_once_per_request( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Per-request, not per-key: without a ``column`` label, n increments are + an uninterpretable magnitude (3 typos once vs 1 typo 3 times collide), + whereas one-per-request stays normalizable against + ``recotem_v1_requests_total``.""" + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend", + json={ + "user_id": "never_seen", + "limit": 3, + "user_features": {"bandd": "young", "agee": 30, "cityy": "tokyo"}, + }, + ) + assert r.status_code == 200 + assert calls == [("fa", "user")] + + +def test_declared_column_does_not_count_as_unknown( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend", + json={"user_id": "never_seen", "limit": 3, "user_features": {"band": "young"}}, + ) + assert r.status_code == 200 + assert calls == [] + + +def test_unknown_value_is_not_an_unknown_column( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """``band`` IS declared; only its value is out-of-vocabulary. That is the + pre-existing ``inc_feature_unknown_value`` signal and must not also trip + the column counter -- the two conditions have different remedies.""" + column_calls: list[tuple[str, str]] = [] + value_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: column_calls.append((recipe, side)), + ) + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: value_calls.append((recipe, side, column)), + ) + r = client.post( + "/v1/recipes/fa:recommend", + json={ + "user_id": "never_seen", + "limit": 3, + "user_features": {"band": "martian"}, + }, + ) + assert r.status_code == 200 + assert column_calls == [] + assert value_calls == [("fa", "user", "band")] + + +def test_unknown_user_feature_column_counted_on_case_b( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend-related", + json={"seed_items": ["i0"], "limit": 3, "user_features": {"bandd": "young"}}, + ) + assert r.status_code == 200 + assert calls == [("fa", "user")] + + +def test_unknown_item_feature_column_counted_on_case_c( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend-related", + json={ + "seed_items": ["brand_new"], + "limit": 3, + "item_features": {"brand_new": {"genree": "action"}}, + }, + ) + assert r.status_code == 200 + assert calls == [("fa", "item")] + + +def test_cold_seed_fanout_counts_once_per_request( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Many cold seeds sharing one typo'd key must increment once, not once + per seed -- otherwise a 100-seed request drowns the signal.""" + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend-related", + json={ + "seed_items": ["new_a", "new_b", "new_c"], + "limit": 3, + "item_features": { + "new_a": {"genree": "action"}, + "new_b": {"genree": "drama"}, + "new_c": {"genree": "action"}, + }, + }, + ) + assert r.status_code == 200 + assert calls == [("fa", "item")] + + +def test_item_features_for_known_seed_are_not_inspected( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A KNOWN seed's features entry is never encoded (``_idmap`` uses its + learned embedding and skips the dict entirely), so a typo inside it is + not a degradation and must not be counted. ``i0`` is known; ``brand_new`` + is what actually reaches the encoder.""" + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_column", + lambda recipe, side: calls.append((recipe, side)), + ) + r = client.post( + "/v1/recipes/fa:recommend-related", + json={ + "seed_items": ["i0", "brand_new"], + "limit": 3, + "item_features": { + "i0": {"totally_bogus": "x"}, + "brand_new": {"genre": "action"}, + }, + }, + ) + assert r.status_code == 200 + assert calls == [] + + +# --------------------------------------------------------------------------- +# Batch verbs -- same cold-start branch logic, reached through the shared +# ``_resolve_recommend`` / ``_resolve_recommend_related`` helpers so a +# per-element failure degrades that element only (``BatchResultErr``), +# never the whole batch. +# --------------------------------------------------------------------------- + + +def test_batch_recommend_mixes_known_and_cold(client: TestClient) -> None: + r = client.post( + "/v1/recipes/fa:batch-recommend", + json={ + "requests": [ + {"user_id": "u1", "limit": 2}, + { + "user_id": "never_seen", + "limit": 2, + "user_features": {"band": "young"}, + }, + ] + }, + ) + assert r.status_code == 200 + results = r.json()["results"] + assert results[0]["status"] == "ok" + assert results[1]["status"] == "ok" + + +def test_batch_cold_element_without_features_errors_only_that_element( + client: TestClient, +) -> None: + r = client.post( + "/v1/recipes/fa:batch-recommend", + json={ + "requests": [ + {"user_id": "u1", "limit": 2}, + {"user_id": "never_seen", "limit": 2}, + ] + }, + ) + assert r.status_code == 200 + results = r.json()["results"] + assert results[0]["status"] == "ok" + assert results[1]["status"] == "error" + assert results[1]["error"]["code"] == "UNKNOWN_USER" + + +def test_batch_features_on_model_without_state_errors_only_that_element( + plain_client: TestClient, +) -> None: + r = plain_client.post( + "/v1/recipes/plain:batch-recommend", + json={ + "requests": [ + {"user_id": "p1", "limit": 2}, + { + "user_id": "u1", + "limit": 2, + "user_features": {"band": "young"}, + }, + ] + }, + ) + assert r.status_code == 200 + results = r.json()["results"] + # "p1" is a known user in the plain recipe's own space -- unaffected by + # this change. + assert results[0]["status"] == "ok" + # "u1" is deliberately excluded from plain's user space (see module + # docstring), so this is a genuine cold-start attempt that must hit the + # "no feature state" guard rather than the known-user path. + assert results[1]["status"] == "error" + assert results[1]["error"]["code"] == "FEATURES_NOT_SUPPORTED" + + +def test_batch_related_cold_seed( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Case C on the batch verb must fire both ``inc_cold_start_request`` + (always) and ``inc_feature_unknown_value`` (for the out-of-vocabulary + ``genre`` value below) -- mirroring the single-verb + ``test_cold_start_metrics_fire_for_every_case``'s ``case-c-cold-seeds`` + parametrization. Without an out-of-vocabulary value, ``unknown_columns`` + would be empty and ``inc_feature_unknown_value`` would never fire, + leaving that call site unpinned by this test. + """ + cold_start_calls: list[tuple[str, str]] = [] + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_cold_start_request", + lambda recipe, case: cold_start_calls.append((recipe, case)), + ) + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + r = client.post( + "/v1/recipes/fa:batch-recommend-related", + json={ + "requests": [ + { + "seed_items": ["brand_new"], + "limit": 2, + "item_features": {"brand_new": {"genre": "martian_genre"}}, + }, + ] + }, + ) + assert r.status_code == 200 + assert r.json()["results"][0]["status"] == "ok" + assert cold_start_calls == [("fa", "cold_seeds")] + assert unknown_calls == [("fa", "item", "genre")] + + +def test_batch_related_with_user_features_is_case_b( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A known seed plus ``user_features`` must take case B (the joint + solve), not silently fall back to the plain seed-only path -- which + would also return ``status: "ok"`` (since "i0" is a known seed) without + ever exercising the feature prior. Asserting on the ``inc_cold_start_request`` + call, not just on ``status == "ok"``, is what makes this test able to + fail against that bug. + + Uses an out-of-vocabulary ``band`` value (``"martian"``) rather than a + valid one so that ``unknown_columns`` is non-empty and + ``inc_feature_unknown_value`` actually fires -- pinning that call site + too, not just ``inc_cold_start_request``. + """ + cold_start_calls: list[tuple[str, str]] = [] + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_cold_start_request", + lambda recipe, case: cold_start_calls.append((recipe, case)), + ) + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + r = client.post( + "/v1/recipes/fa:batch-recommend-related", + json={ + "requests": [ + { + "seed_items": ["i0"], + "limit": 3, + "user_features": {"band": "martian"}, + }, + ] + }, + ) + assert r.json()["results"][0]["status"] == "ok" + assert cold_start_calls == [("fa", "features_and_history")] + assert unknown_calls == [("fa", "user", "band")] + + +def test_batch_related_cold_seed_with_user_features_prefers_case_c( + client: TestClient, +) -> None: + """Batch counterpart of ``test_cold_seed_with_item_and_user_features_prefers_case_c``. + + Both case B's and case C's trigger conditions are simultaneously true; + only the precedence rule (case C wins) decides which solve runs. If case + B ran instead, it would treat ``"brand_new"`` (absent from the id-map) as + part of the seed interaction history and the underlying irspack call + would raise -- surfacing as a batch element error rather than "ok". + """ + r = client.post( + "/v1/recipes/fa:batch-recommend-related", + json={ + "requests": [ + { + "seed_items": ["brand_new"], + "limit": 3, + "user_features": {"band": "young"}, + "item_features": {"brand_new": {"genre": "action"}}, + }, + ] + }, + ) + result = r.json()["results"][0] + assert result["status"] == "ok", result + + +def test_batch_related_unknown_seed_without_features_errors_only_that_element( + client: TestClient, +) -> None: + r = client.post( + "/v1/recipes/fa:batch-recommend-related", + json={ + "requests": [ + {"seed_items": ["i0"], "limit": 2}, + {"seed_items": ["nope"], "limit": 2}, + ] + }, + ) + assert r.status_code == 200 + results = r.json()["results"] + assert results[0]["status"] == "ok" + assert results[1]["status"] == "error" + assert results[1]["error"]["code"] == "UNKNOWN_SEED_ITEMS" + + +def test_batch_related_item_features_on_model_without_state_is_error( + plain_client: TestClient, +) -> None: + r = plain_client.post( + "/v1/recipes/plain:batch-recommend-related", + json={ + "requests": [ + { + "seed_items": ["brand_new"], + "limit": 2, + "item_features": {"brand_new": {"genre": "action"}}, + }, + ] + }, + ) + assert r.status_code == 200 + result = r.json()["results"][0] + assert result["status"] == "error" + assert result["error"]["code"] == "FEATURES_NOT_SUPPORTED" + + +def test_batch_cold_start_metrics_fire( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Same mutation-pinning intent as ``test_cold_start_metrics_fire_for_every_case``, + but through the batch wiring -- proves the shared helper's metrics calls + fire identically regardless of which verb's loop invoked it.""" + cold_start_calls: list[tuple[str, str]] = [] + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_cold_start_request", + lambda recipe, case: cold_start_calls.append((recipe, case)), + ) + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + r = client.post( + "/v1/recipes/fa:batch-recommend", + json={ + "requests": [ + { + "user_id": "never_seen", + "limit": 3, + "user_features": {"band": "martian"}, + }, + ] + }, + ) + assert r.status_code == 200 + assert r.json()["results"][0]["status"] == "ok" + assert cold_start_calls == [("fa", "features_only")] + assert unknown_calls == [("fa", "user", "band")] + + +# --------------------------------------------------------------------------- +# Finding 1 (Task 14 review): ``idx`` must ride along on the +# ``recommender_unexpected_key_error`` event for BOTH batch verbs. +# +# Pre-refactor, ``batch_recommend`` / ``batch_recommend_related`` each had +# their own ``except KeyError:`` block that logged this event with +# ``idx=idx``, so an operator could map the log line straight to +# ``results[idx]``. The refactor moved the log call into the shared +# ``_resolve_recommend`` / ``_resolve_recommend_related`` resolvers, which +# have no batch-index parameter and log only the single-verb's fields +# (``user_id_hash`` / ``seed_items_count``) -- silently dropping ``idx`` for +# the batch call sites. ``_bind_batch_idx`` (routes.py) restores it by +# binding ``idx`` as a structlog contextvar for the duration of each batch +# element's processing, so it is merged into any log event emitted anywhere +# during that element -- including ones raised deep inside the resolvers -- +# without the resolvers' signatures needing to change. +# +# Each test below uses TWO elements, both of which independently trigger +# the "recommender layout unexpected" KeyError path, and asserts each +# element's log event carries ITS OWN idx (0 and 1 respectively, not the +# other element's) -- this is what would catch a regression where binding +# happened once outside the loop (or leaked from the previous iteration) +# instead of being freshly bound per element. +# --------------------------------------------------------------------------- + + +def test_batch_recommend_unexpected_key_error_logs_idx_per_element() -> None: + """Simulates an internal irspack layout bug: both users ARE in the + id-map (so this is not the "genuine unknown user" 404 path), but + ``get_recommendation_for_known_user_id`` raises ``KeyError`` anyway for + both. Each element's ``recommender_unexpected_key_error`` event must + carry that element's own ``idx`` -- 0 and 1, never the other's. + """ + rec = MagicMock() + rec._mapper.user_id_to_index = {"u-known0": 0, "u-known1": 1} + rec.get_recommendation_for_known_user_id.side_effect = KeyError("irspack-internal") + entry = _entry("demo", rec) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + with structlog.testing.capture_logs( + processors=(structlog.contextvars.merge_contextvars,) + ) as cap: + r = client.post( + "/v1/recipes/demo:batch-recommend", + json={ + "requests": [ + {"user_id": "u-known0", "limit": 2}, + {"user_id": "u-known1", "limit": 2}, + ] + }, + ) + + assert r.status_code == 200, r.text + results = r.json()["results"] + assert results[0]["status"] == "error" + assert results[0]["error"]["code"] == "INTERNAL_ERROR" + assert results[1]["status"] == "error" + assert results[1]["error"]["code"] == "INTERNAL_ERROR" + + key_error_events = [ + e for e in cap if e.get("event") == "recommender_unexpected_key_error" + ] + assert len(key_error_events) == 2, ( + f"expected one recommender_unexpected_key_error event per element; " + f"got: {key_error_events!r}" + ) + # Each event must carry ITS OWN idx -- not the other element's (proves + # no cross-iteration contextvar leak). + assert key_error_events[0]["idx"] == 0 + assert key_error_events[1]["idx"] == 1 + + +def test_batch_recommend_related_unexpected_key_error_logs_idx_per_element() -> None: + """Same as above for ``:batch-recommend-related``'s "all seeds known, + no user_features" path: both seeds ARE in the id-map, but + ``get_recommendation_for_new_user`` raises ``KeyError`` anyway for both. + """ + rec = MagicMock() + rec._mapper.item_id_to_index = {"s0": 0, "s1": 1} + rec.get_recommendation_for_new_user.side_effect = KeyError("irspack-internal") + entry = _entry("demo", rec) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + with structlog.testing.capture_logs( + processors=(structlog.contextvars.merge_contextvars,) + ) as cap: + r = client.post( + "/v1/recipes/demo:batch-recommend-related", + json={ + "requests": [ + {"seed_items": ["s0"], "limit": 2}, + {"seed_items": ["s1"], "limit": 2}, + ] + }, + ) + + assert r.status_code == 200, r.text + results = r.json()["results"] + assert results[0]["status"] == "error" + assert results[0]["error"]["code"] == "INTERNAL_ERROR" + assert results[1]["status"] == "error" + assert results[1]["error"]["code"] == "INTERNAL_ERROR" + + key_error_events = [ + e for e in cap if e.get("event") == "recommender_unexpected_key_error" + ] + assert len(key_error_events) == 2, ( + f"expected one recommender_unexpected_key_error event per element; " + f"got: {key_error_events!r}" + ) + assert key_error_events[0]["idx"] == 0 + assert key_error_events[1]["idx"] == 1 + + +# --------------------------------------------------------------------------- +# Review finding 1 (MEDIUM): a client-supplied extreme-but-finite numerical +# feature value must never produce an HTTP 500. +# +# Reproduced end-to-end against a REAL trained feature-aware IALS model (not +# mocked): ``item_features={"tight": 1e22}`` on a cold seed makes irspack's +# per-request conjugate-gradient cold-start solve ill-conditioned, and +# irspack's native core raises a bare ``RuntimeError`` with no awareness the +# input came from an untrusted client. Pre-fix, that propagated unhandled +# through ``_resolve_recommend_related`` to the router's bare ``except +# Exception`` and became ``500 {"detail": "internal error", "code": +# "INTERNAL_ERROR"}``. +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def numerical_client() -> TestClient: + registry = ModelRegistry() + registry.replace( + "fa_num", _entry("fa_num", _fa_recommender_with_numerical_item_feature()) + ) + return TestClient(build_v1_app(registry)) + + +def test_extreme_numerical_item_feature_value_returns_4xx_not_500( + numerical_client: TestClient, +) -> None: + """The exact reproduction from the review report: POST + ``:recommend-related`` with a cold seed whose ``item_features`` carries + an extreme-but-finite ``numerical`` value must return a 4xx with an + actionable code -- never a 500. + """ + r = numerical_client.post( + "/v1/recipes/fa_num:recommend-related", + json={ + "seed_items": ["zzz"], + "limit": 2, + "item_features": {"zzz": {"tight": 1e22}}, + }, + ) + assert 400 <= r.status_code < 500, ( + f"a client-supplied value must never produce a 500; got " + f"{r.status_code}: {r.text}" + ) + body = r.json() + assert body["code"] == "FEATURE_VALUE_UNUSABLE", body + # Review finding 2: the detail message must not claim the RAW supplied + # value (1e22 here) was extreme in some absolute sense -- it must blame + # the STANDARDIZED value, which is the thing that is actually unusable. + assert "extreme magnitude" not in body["detail"], body["detail"] + assert "standardized value" in body["detail"], body["detail"] + + +def test_over_cap_feature_value_is_rejected_at_validation_422( + client: TestClient, +) -> None: + """A cold-start feature value longer than the per-value character cap is + rejected at request validation (422) -- the same status every other + request-schema cap returns -- before it can reach the multi_label + tokenizer. Pins the HTTP status the schema-level ``AfterValidator`` + produces (the unit tests assert only the model-level ValidationError), and + matches docs/recipe-reference.md + CHANGELOG. + """ + from recotem.serving.schemas import _MAX_FEATURE_VALUE_CHARS + + r = client.post( + "/v1/recipes/fa:recommend", + json={ + "user_id": "unseen_u", + "limit": 3, + "user_features": {"band": "a" * (_MAX_FEATURE_VALUE_CHARS + 1)}, + }, + ) + assert r.status_code == 422, r.text + # A value one char under the cap is NOT rejected by the cap (premise guard + # so this cannot pass vacuously by rejecting everything). + r_ok = client.post( + "/v1/recipes/fa:recommend", + json={ + "user_id": "unseen_u", + "limit": 3, + "user_features": {"band": "a" * _MAX_FEATURE_VALUE_CHARS}, + }, + ) + assert r_ok.status_code != 422, r_ok.text + + +@pytest.fixture() +def near_constant_numerical_client() -> TestClient: + registry = ModelRegistry() + registry.replace( + "fa_near_constant", + _entry( + "fa_near_constant", + _fa_recommender_with_near_constant_numerical_item_feature(), + ), + ) + return TestClient(build_v1_app(registry)) + + +# --------------------------------------------------------------------------- +# Review finding 2 (IMPORTANT): the 400's message is false for a +# near-constant column. A column whose training std is tiny (e.g. 1.36e-15, +# not exactly 0.0) turns an entirely ORDINARY raw request value into an +# astronomically large standardized one -- the same 400 as the 1e22-on-a- +# normal-column case above, but the raw value itself (1e5 here) is not +# extreme by any reasonable definition. The message must describe the +# STANDARDIZED value as unusable, not claim the client's raw value was +# extreme. +# +# 1e5, not 1e4: independently verified (see the mutation proof in the task +# report) that against THIS fixture's exact dimensionality/lambda, the +# solver singularity crossover for std=1.36e-15 falls between 1e4 (still +# 200) and 1e5 (400) -- docs/api-reference.md already discloses that this +# crossover is not a fixed constant across models, so the test uses a value +# empirically confirmed to trip it here rather than assuming the review's +# reported number transfers exactly to this fixture's shape. +# --------------------------------------------------------------------------- + + +def test_ordinary_numerical_value_on_near_constant_column_returns_4xx_with_honest_message( + near_constant_numerical_client: TestClient, +) -> None: + """An unremarkable raw value (1e5) against a near-constant column (std + 1.36e-15) must still 400 -- via the exact same solver-singularity + mechanism as the 1e22-on-a-normal-column case -- but the message must + not blame the raw magnitude, which is not extreme here.""" + r = near_constant_numerical_client.post( + "/v1/recipes/fa_near_constant:recommend-related", + json={ + "seed_items": ["zzz"], + "limit": 2, + "item_features": {"zzz": {"tight": 1e5}}, + }, + ) + assert 400 <= r.status_code < 500, ( + f"a client-supplied ordinary value must never produce a 500; got " + f"{r.status_code}: {r.text}" + ) + body = r.json() + assert body["code"] == "FEATURE_VALUE_UNUSABLE", body + assert "extreme magnitude" not in body["detail"], body["detail"] + assert "standardized value" in body["detail"], body["detail"] + + +# --------------------------------------------------------------------------- +# Review finding 3 (IMPORTANT): a directly-supplied +inf/-inf numerical value +# was a silent no-op -- byte-identical to omitting the column entirely, with +# no `unknown` entry recorded and no `recotem_v1_feature_unknown_value_total` +# increment. Contrast with an unknown CATEGORICAL value (already covered +# elsewhere, e.g. ``test_cold_start_metrics_fire_for_every_case``), which +# correctly fires the counter today. +# --------------------------------------------------------------------------- + + +def test_infinite_numerical_item_feature_fires_unknown_value_counter( + numerical_client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """+inf on a numerical feature must still degrade like a missing value + (200, the feature contributes nothing to the row) but MUST fire + ``inc_feature_unknown_value`` -- pre-fix this was a silent no-op, + indistinguishable from omitting the column (see the contrast test + below). + + Sends the raw JSON body as bytes with a literal ``1e309`` number token + (the review's exact reproduction) rather than via the ``json=`` kwarg's + Python-side encoding: standard-compliant JSON has no ``Infinity`` + literal, so httpx's own encoder (``allow_nan=False``) refuses to + serialize a Python ``float("inf")`` object -- but ``1e309`` is an + ordinary, spec-legal JSON number token that merely overflows float64 on + the *server's* parse, which is exactly the real-world attack surface + this finding describes. + """ + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + raw_body = ( + b'{"seed_items": ["zzz"], "limit": 2, ' + b'"item_features": {"zzz": {"tight": 1e309}}}' + ) + r = numerical_client.post( + "/v1/recipes/fa_num:recommend-related", + content=raw_body, + headers={"Content-Type": "application/json"}, + ) + assert r.status_code == 200, r.text + assert unknown_calls == [("fa_num", "item", "tight")] + + +def test_omitted_numerical_item_feature_does_not_fire_unknown_value_counter( + numerical_client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Contrast case: simply OMITTING the numerical column entirely (rather + than supplying +inf) must still NOT fire the counter -- a missing value + is a separate, deliberately uncounted gap that this fix must not widen. + """ + unknown_calls: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + "recotem.serving.routes._metrics.inc_feature_unknown_value", + lambda recipe, side, column: unknown_calls.append((recipe, side, column)), + ) + r = numerical_client.post( + "/v1/recipes/fa_num:recommend-related", + json={ + "seed_items": ["zzz"], + "limit": 2, + "item_features": {"zzz": {}}, + }, + ) + assert r.status_code == 200, r.text + assert unknown_calls == [] + + +def test_extreme_numerical_user_feature_value_on_incapable_side_stays_400( + numerical_client: TestClient, +) -> None: + """``fa_num`` carries item features only (no ``user_feature_state``), so + a request combining a known seed with an extreme ``user_features`` value + must still hit the pre-existing "no user feature state" + ``FEATURES_NOT_SUPPORTED`` guard, not the new numerical-error path -- + proving the new exception handler did not shadow the existing + capability check for an unrelated side. The genuinely + numerically-unstable case B path (``get_score_cold_user`` with a real + user feature state) is covered at the ``_idmap`` layer by + ``test_new_user_with_features_wraps_runtime_error`` in + ``tests/unit/test_idmap.py``. + """ + r = numerical_client.post( + "/v1/recipes/fa_num:recommend-related", + json={ + "seed_items": ["i0"], + "limit": 2, + "user_features": {"tight": 1e22}, + }, + ) + assert 400 <= r.status_code < 500 + assert r.json()["code"] == "FEATURES_NOT_SUPPORTED" + + +# --------------------------------------------------------------------------- +# Review round 2, Important 1: the ``except RuntimeError`` at the three +# ``_idmap.py`` call sites was previously BLANKET (no message check), so a +# non-numerical server fault (e.g. ``trainer is None``) was mislabeled as a +# 400 ``FEATURE_VALUE_UNUSABLE`` telling the client their feature value was +# at fault. The fix scopes the catch to a verified allow-list of numerical +# failure signatures; anything else must still reach the router's generic +# handler as a 500 -- proven end-to-end (route layer, real trained model) +# here. The mock-level proof for each of the three ``_idmap.py`` call sites +# lives in ``tests/unit/test_idmap.py``. +# --------------------------------------------------------------------------- + + +def test_non_numerical_runtime_error_from_cold_start_returns_500_not_400( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The tell that the pre-fix breadth was accidental: with + ``recommender.trainer = None``, irspack's ``trainer_as_ials`` raises + ``RuntimeError("tried to fetch trainer before the training.")`` -- a + server fault, not a client-value problem. A blanket ``except + RuntimeError`` at ``get_score_cold_user_from_features``'s call site + (``_idmap.py``) mapped this to ``400 FEATURE_VALUE_UNUSABLE`` with a + message actively blaming the client. Post-fix, only messages matching + the verified numerical-failure signatures are wrapped; everything else + must reach the router's generic handler as ``500 INTERNAL_ERROR``. + """ + idmapped = _fa_recommender() + registry = ModelRegistry() + registry.replace("fa", _entry("fa", idmapped)) + local_client = TestClient(build_v1_app(registry), raise_server_exceptions=False) + + monkeypatch.setattr( + idmapped.recommender, + "get_score_cold_user_from_features", + MagicMock( + side_effect=RuntimeError("tried to fetch trainer before the training.") + ), + ) + r = local_client.post( + "/v1/recipes/fa:recommend", + json={ + "user_id": "never_seen", + "limit": 2, + "user_features": {"band": "young"}, + }, + ) + assert r.status_code == 500, ( + f"a non-numerical server fault must never be mislabeled as a client " + f"4xx; got {r.status_code}: {r.text}" + ) + assert r.json()["code"] == "INTERNAL_ERROR", r.json() + + +# --------------------------------------------------------------------------- +# Review round 2, Minor 5: 3 of the 4 new cold-start route handlers had no +# test at all -- only ``:recommend-related`` (single) did, above. The +# reviewer verified all three empirically; this pins that verification as a +# standing regression guard: ``:recommend`` (routes.py:583, case A), +# ``:batch-recommend`` (routes.py:838/845, batch case A), and +# ``:batch-recommend-related`` (routes.py:1012/1019, batch case C). +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def numerical_user_client() -> TestClient: + registry = ModelRegistry() + registry.replace( + "fa_num_user", + _entry("fa_num_user", _fa_recommender_with_numerical_user_feature()), + ) + return TestClient(build_v1_app(registry)) + + +def test_extreme_numerical_user_feature_value_on_recommend_returns_4xx_not_500( + numerical_user_client: TestClient, +) -> None: + """Case A through the single ``:recommend`` verb (routes.py:583, + previously untested): an unknown user cold-started from an + extreme-but-finite ``numerical`` ``user_features`` value must return a + 4xx with ``FEATURE_VALUE_UNUSABLE`` -- never a 500. + """ + r = numerical_user_client.post( + "/v1/recipes/fa_num_user:recommend", + json={ + "user_id": "never_seen", + "limit": 2, + "user_features": {"band": "young", "tight": 1e22}, + }, + ) + assert 400 <= r.status_code < 500, ( + f"a client-supplied value must never produce a 500; got " + f"{r.status_code}: {r.text}" + ) + assert r.json()["code"] == "FEATURE_VALUE_UNUSABLE", r.json() + + +def test_batch_extreme_numerical_user_feature_value_errors_only_that_element( + numerical_user_client: TestClient, +) -> None: + """Batch counterpart on ``:batch-recommend`` (routes.py:838/845, + previously untested): the numerical failure must degrade only the + offending element, leaving the batch response itself ``200`` and the + other element ``ok``. + """ + r = numerical_user_client.post( + "/v1/recipes/fa_num_user:batch-recommend", + json={ + "requests": [ + { + "user_id": "never_seen_1", + "limit": 2, + "user_features": {"band": "young", "tight": 1e22}, + }, + { + "user_id": "never_seen_2", + "limit": 2, + "user_features": {"band": "old"}, + }, + ] + }, + ) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert results[0]["status"] == "error" + assert results[0]["error"]["code"] == "FEATURE_VALUE_UNUSABLE" + assert results[1]["status"] == "ok", results[1] + + +def test_batch_extreme_numerical_item_feature_value_on_related_errors_only_that_element( + numerical_client: TestClient, +) -> None: + """Batch counterpart on ``:batch-recommend-related`` (routes.py:1012/1019, + previously untested): case C's numerical failure must degrade only the + offending element. + """ + r = numerical_client.post( + "/v1/recipes/fa_num:batch-recommend-related", + json={ + "requests": [ + { + "seed_items": ["zzz"], + "limit": 2, + "item_features": {"zzz": {"tight": 1e22}}, + }, + {"seed_items": ["i0"], "limit": 2}, + ] + }, + ) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert results[0]["status"] == "error" + assert results[0]["error"]["code"] == "FEATURE_VALUE_UNUSABLE" + assert results[1]["status"] == "ok", results[1] + + +# --------------------------------------------------------------------------- +# exclude_items means the same thing in every case +# --------------------------------------------------------------------------- +# +# ``exclude_items`` is post-filtered off the ranker's output +# (``routes._build_items``) and is never pushed down into the ranker as +# ``forbidden_item_ids``. It therefore removes items from a page instead of +# freeing a slot for a replacement: ``limit`` is a ceiling, not a promise. +# That is what every pre-existing verb has always done, so no cold-start case +# may read it differently -- otherwise adding ``user_features`` to an +# otherwise identical request would silently change how many items come back. + +_EXCLUDE_ITEMS_CASES = ( + pytest.param( + "recommend", + {"user_id": "never_seen", "user_features": {"band": "young"}}, + id="case-a-cold-user-features-only", + ), + pytest.param( + "recommend-related", + {"seed_items": ["i0"], "user_features": {"band": "young"}}, + id="case-b-known-seed-plus-user-features", + ), + pytest.param( + "recommend-related", + { + "seed_items": ["brand_new"], + "item_features": {"brand_new": {"genre": "action"}}, + }, + id="case-c-cold-seed-item-features", + ), +) + + +@pytest.mark.parametrize(("verb", "body"), _EXCLUDE_ITEMS_CASES) +def test_exclude_items_truncates_and_never_backfills( + stable_ranking_client: TestClient, verb: str, body: dict +) -> None: + """Excluding 3 of a case's own top 5 must leave exactly those 5 minus 3. + + The ids to exclude are derived from each case's OWN unexcluded page + rather than hardcoded, so the test states the invariant -- "excluding k + of your own top-`limit` returns the other `limit - k`, unchanged and in + order" -- independently of how any one case happens to rank the catalog. + + Asserting the surviving ids, not just their count, is what separates + "post-filtered" from "back-filled": a ranker handed ``forbidden_item_ids`` + returns a FULL page of 5 whose last 3 entries are items the unexcluded + page never contained, which would still satisfy a count-only check the + moment someone "fixed" the count by re-ranking. + + Runs on ``stable_ranking_client`` because it compares two separate + scoring calls: on the default fixture the ranker's own jitter can change + the second call's top 5, which fails this assertion for a reason that + has nothing to do with exclusion (see that fixture). + """ + url = f"/v1/recipes/fa:{verb}" + baseline = stable_ranking_client.post(url, json={**body, "limit": 5}) + assert baseline.status_code == 200, baseline.text + top5 = [item["item_id"] for item in baseline.json()["items"]] + assert len(top5) == 5, "fixture invariant: a 20-item catalog must fill limit=5" + + r = stable_ranking_client.post( + url, json={**body, "limit": 5, "exclude_items": top5[:3]} + ) + assert r.status_code == 200, r.text + assert [item["item_id"] for item in r.json()["items"]] == top5[3:] diff --git a/tests/unit/test_serving_metrics.py b/tests/unit/test_serving_metrics.py index 285f9f15..0e2a7e4c 100644 --- a/tests/unit/test_serving_metrics.py +++ b/tests/unit/test_serving_metrics.py @@ -186,6 +186,9 @@ def _teardown() -> None: "recotem_v1_batch_element_errors", "recotem_v1_metadata_degraded_items", "recotem_v1_validation_errors_outside_verb", + "recotem_v1_feature_unknown_value", + "recotem_v1_feature_unknown_column", + "recotem_v1_cold_start_requests", } for collector in list(REGISTRY._names_to_collectors.values()): describe = getattr(collector, "describe", None) @@ -209,6 +212,9 @@ def _teardown() -> None: "_V1_BATCH_ELEMENT_ERRORS", "_V1_METADATA_DEGRADED_ITEMS", "_V1_VALIDATION_ERRORS_OUTSIDE_VERB", + "_V1_FEATURE_UNKNOWN_VALUE", + "_V1_FEATURE_UNKNOWN_COLUMN", + "_V1_COLD_START_REQUESTS", ): setattr(_m, attr, None) @@ -261,3 +267,112 @@ def test_inc_metadata_degraded_items_coerces_unknown_kind(reset_metrics_registry assert "arbitrary_future_kind" not in text, ( "raw unknown kind must not appear in Prometheus output" ) + + +# --------------------------------------------------------------------------- +# Feature-aware iALS cold-start metrics: recotem_v1_feature_unknown_value_total, +# recotem_v1_cold_start_requests_total +# --------------------------------------------------------------------------- + + +def test_inc_feature_unknown_value_emits_labels(reset_metrics_registry): + _m.inc_feature_unknown_value("r1", "user", "band") + _m.inc_feature_unknown_value("r1", "item", "genre", 3) + + out, _ = _m.generate_latest() + text = out.decode() + + assert "recotem_v1_feature_unknown_value_total" in text + assert 'side="user"' in text + assert 'column="band"' in text + assert 'side="item"' in text + assert 'column="genre"' in text + + +def test_inc_feature_unknown_column_emits_labels(reset_metrics_registry): + _m.inc_feature_unknown_column("r1", "user") + _m.inc_feature_unknown_column("r1", "item") + + out, _ = _m.generate_latest() + text = out.decode() + + assert "recotem_v1_feature_unknown_column_total" in text + assert 'side="user"' in text + assert 'side="item"' in text + + +def test_inc_feature_unknown_column_has_no_column_label(reset_metrics_registry): + """The column name is request input, not recipe content, so it must never + become a label: an unbounded label value is a metrics-cardinality DoS. + This is the deliberate asymmetry with ``inc_feature_unknown_value``, + whose ``column`` label is bounded by the operator's own recipe.""" + _m.inc_feature_unknown_column("r1", "user") + + out, _ = _m.generate_latest() + line = next( + ln + for ln in out.decode().splitlines() + if ln.startswith("recotem_v1_feature_unknown_column_total{") + ) + assert "column=" not in line, f"unknown column name must not be a label: {line!r}" + assert _m.inc_feature_unknown_column.__code__.co_argcount == 2, ( + "signature must stay (recipe, side) so a column name cannot be passed" + ) + + +def test_inc_feature_unknown_column_coerces_unknown_side(reset_metrics_registry): + _m.inc_feature_unknown_column("r1", "user") + _m.inc_feature_unknown_column("r1", "sideways") + + out, _ = _m.generate_latest() + text = out.decode() + + assert 'side="user"' in text + assert 'side="unexpected"' in text + assert 'side="sideways"' not in text + + +def test_inc_feature_unknown_value_coerces_unknown_side(reset_metrics_registry): + """Unknown side values must be coerced to 'unexpected' to prevent label + cardinality explosion; 'item' and 'user' pass through.""" + _m.inc_feature_unknown_value("r1", "user", "band") + _m.inc_feature_unknown_value("r1", "sideways", "band") + + out, _ = _m.generate_latest() + text = out.decode() + + assert 'side="user"' in text + assert 'side="unexpected"' in text + assert "sideways" not in text, "raw unknown side must not reach Prometheus" + + +def test_inc_cold_start_request_coerces_unknown_case(reset_metrics_registry): + _m.inc_cold_start_request("r1", "features_only") + _m.inc_cold_start_request("r1", "cold_seeds") + _m.inc_cold_start_request("r1", "arbitrary_future_case") + + out, _ = _m.generate_latest() + text = out.decode() + + assert 'case="features_only"' in text + assert 'case="cold_seeds"' in text + assert 'case="unexpected"' in text + assert "arbitrary_future_case" not in text + + +def test_feature_counters_are_noop_when_metrics_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With metrics off the helpers must be inert, not raise.""" + from recotem.serving import metrics as m + + monkeypatch.setattr(m, "metrics_enabled", lambda: False) + monkeypatch.setattr(m, "_V1_FEATURE_UNKNOWN_VALUE", None) + monkeypatch.setattr(m, "_V1_COLD_START_REQUESTS", None) + monkeypatch.setattr(m, "_V1_REQUEST_COUNTER", None) + + m.inc_feature_unknown_value("r1", "user", "band") + m.inc_cold_start_request("r1", "features_only") + + assert m._V1_FEATURE_UNKNOWN_VALUE is None + assert m._V1_COLD_START_REQUESTS is None diff --git a/tests/unit/test_serving_schemas.py b/tests/unit/test_serving_schemas.py index 452b930d..15801a83 100644 --- a/tests/unit/test_serving_schemas.py +++ b/tests/unit/test_serving_schemas.py @@ -318,6 +318,82 @@ def test_recommend_request_extra_fields_rejected() -> None: RecommendRequest(user_id="u1", context={"a": 1}) # type: ignore[call-arg] +# --------------------------------------------------------------------------- +# Fix C: cold-start feature request VALUES were the only uncapped request +# field. `_FeatureValues`' `Field(max_length=64)` caps the KEY COUNT, but a +# string VALUE was unbounded -- and `_tokens` does `str(raw).split(delimiter)` +# with ~8x amplification, so a large multi_label value is a memory-DoS +# reachable with one API key, multiplied by batch/related fan-out. Every other +# request field is length-capped; this restores parity. +# --------------------------------------------------------------------------- + +# Mirrors schemas._MAX_FEATURE_VALUE_CHARS; the sync test below pins them equal. +_FEATURE_VALUE_CAP = 8192 + + +def test_feature_value_cap_matches_module_constant() -> None: + from recotem.serving.schemas import _MAX_FEATURE_VALUE_CHARS + + assert _MAX_FEATURE_VALUE_CHARS == _FEATURE_VALUE_CAP + + +def test_recommend_request_user_features_value_over_cap_rejected() -> None: + with pytest.raises(ValidationError): + RecommendRequest( + user_id="u1", user_features={"g": "a" * (_FEATURE_VALUE_CAP + 1)} + ) + + +def test_recommend_request_user_features_value_at_cap_accepted() -> None: + req = RecommendRequest(user_id="u1", user_features={"g": "a" * _FEATURE_VALUE_CAP}) + assert len(req.user_features["g"]) == _FEATURE_VALUE_CAP + + +def test_recommend_related_request_user_features_value_over_cap_rejected() -> None: + with pytest.raises(ValidationError): + RecommendRelatedRequest( + seed_items=["s1"], + user_features={"g": "a" * (_FEATURE_VALUE_CAP + 1)}, + ) + + +def test_recommend_related_request_item_features_nested_value_over_cap_rejected() -> ( + None +): + """`item_features` is a nested dict (keyed by seed id); the cap must reach + into each cold-seed feature mapping's values, not only the outer keys.""" + with pytest.raises(ValidationError): + RecommendRelatedRequest( + seed_items=["s1"], + item_features={"s1": {"g": "a" * (_FEATURE_VALUE_CAP + 1)}}, + ) + + +def test_feature_value_cap_does_not_echo_the_value_or_touch_non_strings() -> None: + """The error names the offending column key but must NOT echo the value + (PII / log-safety), and a non-string scalar value is unaffected by the + char cap.""" + blob = "a" * (_FEATURE_VALUE_CAP + 1) + with pytest.raises(ValidationError) as exc_info: + RecommendRequest(user_id="u1", user_features={"country": blob}) + msg = str(exc_info.value) + assert "country" in msg, "the error must name the offending column key" + assert blob not in msg, "the error must not echo the (PII) value" + + # A huge non-string scalar is not a char-cap violation. + req = RecommendRequest(user_id="u1", user_features={"n": 10**400}) + assert req.user_features["n"] == 10**400 + + +def test_feature_value_cap_covers_batch_reparse_path() -> None: + """Batch verbs re-parse each element through these same request models via + `model_validate`, so the cap covers batch too -- pin that exact path.""" + with pytest.raises(ValidationError): + RecommendRequest.model_validate( + {"user_id": "u1", "user_features": {"g": "a" * (_FEATURE_VALUE_CAP + 1)}} + ) + + # --------------------------------------------------------------------------- # Finding 6: Discriminated union extra-field enforcement # --------------------------------------------------------------------------- diff --git a/tests/unit/test_training_algorithms.py b/tests/unit/test_training_algorithms.py index 7e85c12a..6e8a1899 100644 --- a/tests/unit/test_training_algorithms.py +++ b/tests/unit/test_training_algorithms.py @@ -5,7 +5,9 @@ import pytest from recotem.training.algorithms import ( + FEATURE_CAPABLE_CLASS_NAMES, SUPPORTED_CLASS_NAMES, + is_feature_capable, resolve_algorithm_name, ) from recotem.training.errors import UnknownAlgorithmError @@ -55,3 +57,24 @@ def test_unsupported_irspack_recommender_rejected(alias: str) -> None: def test_garbage_alias_rejected() -> None: with pytest.raises(UnknownAlgorithmError): resolve_algorithm_name("not-an-algorithm") + + +# --------------------------------------------------------------------------- +# Task 2: feature-capable algorithm registry +# --------------------------------------------------------------------------- + + +def test_only_ials_is_feature_capable() -> None: + assert frozenset({"IALSRecommender"}) == FEATURE_CAPABLE_CLASS_NAMES + + +def test_is_feature_capable_accepts_alias() -> None: + assert is_feature_capable("IALS") is True + assert is_feature_capable("ials") is True + assert is_feature_capable("TopPop") is False + + +def test_is_feature_capable_unknown_name_is_false_not_raise() -> None: + # Unknown names must NOT raise here: training.algorithms has no load-time + # validation and models.py:136-141 deliberately tolerates them. + assert is_feature_capable("NoSuchThing") is False diff --git a/tests/unit/test_training_features.py b/tests/unit/test_training_features.py new file mode 100644 index 00000000..e18e5a7a --- /dev/null +++ b/tests/unit/test_training_features.py @@ -0,0 +1,674 @@ +"""Unit tests for recotem.training.features. + +Tests: +- load_feature_tables(None, ...) returns an empty FeatureTables. +- load_feature_tables builds an encoder state from a fetched CSV feature + table (item side); n_features accounts for the bias column. +- numeric feature columns keep their dtype through the fetch -> state path + (only the id_column is string-coerced). +- encode_for_axis reindexes onto the supplied item_order and omits absent + sides from its result dict. +- a missing id_column raises with the offending name in the message. +- rows with a null/empty id are dropped before the vocabulary is built. +- a dimension-cap breach (recotem._features.FeatureEncodeError) is wrapped + into TrainingError so it maps to exit 4, not exit 1. +- an unregistered source type raises DataSourceError (exit 3) unwrapped, + same as the main interaction source. +- encode_for_axis raises TrainingError if a configured side's order is + omitted (internal-misuse guard, not reachable through the public pipeline). +- encode_for_axis refuses a feature table with ZERO id overlap against the + interaction axis (the silent all-bias bug), logs coverage at INFO, and + still permits a legitimately partial-covering table. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +# _compat applies the IPython stub required by irspack's transitive import chain. +import recotem.training._compat # noqa: F401 +from recotem.datasource.base import DataSourceError +from recotem.recipe.models import FeatureColumn, FeaturesConfig, FeatureSideConfig +from recotem.training.errors import TrainingError +from recotem.training.features import encode_for_axis, load_feature_tables + + +@pytest.fixture +def items_csv(tmp_path: Path) -> str: + p = tmp_path / "items.csv" + pd.DataFrame( + { + "item_id": ["i_a", "i_b", "i_c"], + "genre": ["action", "drama", "action"], + "year": [2000, 2010, 2020], + } + ).to_csv(p, index=False) + return str(p) + + +def _features(items_csv: str) -> FeaturesConfig: + return FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": items_csv}, + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="year", encoding="numerical"), + ], + ) + ) + + +def test_load_feature_tables_none_returns_empty() -> None: + t = load_feature_tables(None, recipe_name="r", run_id="run") + assert t.item_state is None and t.user_state is None + assert t.enabled is False + + +def test_load_feature_tables_builds_state(items_csv: str) -> None: + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + # 2 genre one-hots + 1 standardized year + 1 bias + assert t.item_state["n_features"] == 4 + assert t.user_state is None + assert t.enabled is True + + +def test_numeric_column_is_not_stringified(items_csv: str) -> None: + """The CSV datasource must preserve dtypes; metadata/loader forces str.""" + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + spec = next(s for s in t.item_state["columns"] if s["name"] == "year") + assert spec["encoding"] == "numerical" + assert spec["std"] > 0 + + +def test_encode_for_axis_respects_order(items_csv: str) -> None: + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + fwd = encode_for_axis(t, item_order=["i_a", "i_b", "i_c"], user_order=None) + rev = encode_for_axis(t, item_order=["i_c", "i_b", "i_a"], user_order=None) + np.testing.assert_allclose( + fwd["item_features"].toarray()[::-1], rev["item_features"].toarray() + ) + assert "user_features" not in fwd + + +def test_id_column_missing_raises(items_csv: str) -> None: + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": items_csv}, + id_column="nope", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(Exception, match="nope"): + load_feature_tables(cfg, recipe_name="r", run_id="run") + + +def test_null_and_empty_ids_are_dropped(tmp_path: Path) -> None: + """A blank id must not become a spurious feature-table row. + + Detecting null BEFORE str-coercion (mirroring metadata/loader.py) also + means an entity literally named the string "nan" is preserved rather + than mistaken for a missing id. + """ + p = tmp_path / "items_with_nulls.csv" + pd.DataFrame( + { + "item_id": ["i_a", None, "i_c", ""], + "genre": ["action", "drama", "action", "comedy"], + "year": [2000, 2010, 2020, 2030], + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + encoded = encode_for_axis(t, item_order=["i_a", "i_c"], user_order=None) + # Only the two valid ids should have been encoded; "comedy" (the dropped + # empty-id row) must not appear in the vocabulary. + spec = next(s for s in t.item_state["columns"] if s["name"] == "genre") + assert "comedy" not in spec["vocab"] + assert encoded["item_features"].shape[0] == 2 + + +def test_duplicate_ids_are_dropped_and_logged(tmp_path: Path) -> None: + """A repeated id must keep only its first row (``keep="first"``) AND + emit a ``feature_table_duplicate_ids_dropped`` warning naming the drop + count -- mirroring the adjacent null-id path's + ``feature_table_null_ids_dropped``. Before this fix, the null-id path + logged its drop count but the duplicate-id path (``drop_duplicates``) + dropped silently: a 28-row table with 14 unique ids logged ``n_rows: 14`` + and nothing else, giving an operator no signal that half the table was + discarded. The log must carry only the count -- never the ids/values, + which are user PII. + """ + import structlog.testing + + p = tmp_path / "items_with_dupes.csv" + pd.DataFrame( + { + "item_id": ["i_a", "i_b", "i_a", "i_c", "i_b", "i_b"], + "genre": ["action", "drama", "comedy", "action", "horror", "sci-fi"], + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with structlog.testing.capture_logs() as cap: + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + + # 6 rows, 3 unique ids -> 3 duplicates dropped, first occurrence kept. + encoded = encode_for_axis(t, item_order=["i_a", "i_b", "i_c"], user_order=None) + assert encoded["item_features"].shape[0] == 3 + spec = next(s for s in t.item_state["columns"] if s["name"] == "genre") + # "i_a"'s FIRST row ("action") must have won, not "comedy" (its second, + # dropped row) -- pins keep="first", not just "some row survived". + assert "comedy" not in spec["vocab"] + assert "action" in spec["vocab"] + + dupe_events = [ + e for e in cap if e.get("event") == "feature_table_duplicate_ids_dropped" + ] + assert dupe_events, ( + "Expected 'feature_table_duplicate_ids_dropped' warning; " + f"got events: {[e.get('event') for e in cap]}" + ) + ev = dupe_events[0] + assert ev["drop_count"] == 3, f"Expected drop_count=3; got {ev['drop_count']!r}" + assert ev["side"] == "item" + # Never log the raw ids/values (user PII) -- only the count and side. + for e in cap: + for value in e.values(): + assert "i_a" not in str(value) + assert "comedy" not in str(value) + + +def test_dimension_cap_becomes_training_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """FeatureEncodeError is not a TrainingError subclass; load_feature_tables + must wrap it so a dimension-cap breach maps to exit 4 like every other + training-domain error, not exit 1 (unmapped exception). + """ + monkeypatch.setenv("RECOTEM_MAX_FEATURE_DIM", "16") + p = tmp_path / "wide_items.csv" + pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(40)], + "genre": [f"g{i}" for i in range(40)], + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(TrainingError, match="exceeds") as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + assert exc_info.value.code == "feature_table_error" + + +def test_unregistered_source_type_raises_datasource_error(items_csv: str) -> None: + """An unknown source type is a datasource problem (exit 3), not a + training-domain one (exit 4) -- same treatment as the main interaction + source in pipeline.py's _fetch_data. + """ + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "not_a_real_source", "path": items_csv}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(DataSourceError): + load_feature_tables(cfg, recipe_name="r", run_id="run") + + +def test_encode_for_axis_missing_item_order_raises(items_csv: str) -> None: + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + with pytest.raises(TrainingError, match="item_order"): + encode_for_axis(t, item_order=None, user_order=None) + + +@pytest.fixture +def blank_id_cell_csv(tmp_path: Path) -> str: + """A feature table whose id column contains ONE blank cell. + + That single blank is enough for pandas to infer ``float64`` for the whole + column, so the surviving integer ids read back as ``1.0`` / ``2.0`` while + the interaction axis carries ``"1"`` / ``"2"``. + """ + p = tmp_path / "items_blank_id.csv" + p.write_text("item_id,genre\n1,action\n2,drama\n,comedy\n") + return str(p) + + +def _int_id_features(path: str, **source_extra: object) -> FeaturesConfig: + return FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": path, **source_extra}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + + +def test_zero_overlap_from_blank_cell_dtype_coercion_raises( + blank_id_cell_csv: str, +) -> None: + """A blank id cell must not silently train an all-bias (feature-less) model. + + One blank cell makes pandas infer float64 for the id column, so ids become + "1.0" while the interaction axis has "1". Overlap is empty, every item + encodes to the bias column only, and training used to COMPLETE and sign an + artifact whose header advertises `features` for what is really plain iALS. + The null-id handling in _fetch_side does fire for the blank row, but the + dtype damage to the SURVIVING rows is already done before it runs. + """ + t = load_feature_tables( + _int_id_features(blank_id_cell_csv), recipe_name="r", run_id="run" + ) + # Precondition: the table itself loaded fine -- the damage is invisible + # until the ids meet the interaction axis. + assert list(t.item_df.index) == ["1.0", "2.0"] + + with pytest.raises(TrainingError) as exc_info: + encode_for_axis(t, item_order=["1", "2"], user_order=None) + + msg = str(exc_info.value) + # Name the side and the id_column ... + assert "item" in msg + assert "item_id" in msg + # ... and show both samples, so the 1.0-vs-1 mismatch is self-evident. + assert "1.0" in msg + assert "'1'" in msg + # TrainingError -> exit 4 (see CLAUDE.md's exit-code table). Only the + # "signing_key_missing" code diverts to exit 8, so any other code is 4. + assert exc_info.value.code != "signing_key_missing" + + +def test_zero_overlap_from_wrong_id_column_raises(tmp_path: Path) -> None: + """A wrong-but-EXISTING id_column passes _fetch_side's presence check. + + `sku` exists, so the missing-column guard never fires; the ids simply have + nothing to do with the interaction axis. Same silent all-bias outcome as + the dtype case, and the same overlap check catches it. + """ + p = tmp_path / "items_sku.csv" + pd.DataFrame( + { + "sku": ["SKU-1", "SKU-2"], + "product_id": ["p1", "p2"], + "genre": ["action", "drama"], + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="sku", # should have been product_id + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + with pytest.raises(TrainingError) as exc_info: + encode_for_axis(t, item_order=["p1", "p2"], user_order=None) + + msg = str(exc_info.value) + assert "sku" in msg + assert "SKU-1" in msg + + +def test_zero_overlap_names_the_user_side(tmp_path: Path) -> None: + """The message must name the offending SIDE -- a user-side mismatch must + not report itself as an item-side one.""" + p = tmp_path / "users.csv" + p.write_text("user_id,country\n1,jp\n2,us\n,fr\n") + cfg = FeaturesConfig( + user=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="user_id", + columns=[FeatureColumn(name="country", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + with pytest.raises(TrainingError) as exc_info: + encode_for_axis(t, item_order=None, user_order=["1", "2"]) + + msg = str(exc_info.value) + assert "user" in msg + assert "user_id" in msg + assert "item" not in msg + + +def test_dtype_override_restores_overlap(blank_id_cell_csv: str) -> None: + """The remedy the error message recommends must actually work. + + Pinning the id column to `str` on the feature source defeats the float64 + inference, so the ids line up with the interaction axis again and the genre + one-hot is really encoded (not just the bias column). + """ + t = load_feature_tables( + _int_id_features(blank_id_cell_csv, dtype={"item_id": "str"}), + recipe_name="r", + run_id="run", + ) + assert list(t.item_df.index) == ["1", "2"] + + encoded = encode_for_axis(t, item_order=["1", "2"], user_order=None) + dense = encoded["item_features"].toarray() + # Each row must carry a real genre one-hot ALONGSIDE the bias column; + # an all-bias row would sum to exactly 1.0. + assert dense.sum(axis=1).tolist() == [2.0, 2.0] + + +def test_axis_coverage_is_logged(items_csv: str) -> None: + """Coverage is the observability that was missing: a healthy run must say + how much of the axis the feature table actually covers. Ids are NOT logged + -- they are user PII (see test_duplicate_ids_are_dropped_and_logged); the + bounded id sample belongs only to the fatal zero-overlap message. + """ + import structlog.testing + + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + with structlog.testing.capture_logs() as cap: + encode_for_axis(t, item_order=["i_a", "i_b", "i_zzz"], user_order=None) + + events = [e for e in cap if e.get("event") == "feature_axis_coverage"] + assert events, f"Expected coverage log; got {[e.get('event') for e in cap]}" + ev = events[0] + assert ev["side"] == "item" + assert ev["matched"] == 2 + assert ev["total"] == 3 + for e in cap: + for value in e.values(): + assert "i_a" not in str(value) + + +def test_partial_coverage_is_allowed(items_csv: str) -> None: + """A partially-covering feature table is legitimate, not an error. + + Cold-start entities are representable by design (see build_encoder_state's + docstring): an id absent from the table encodes to bias-only and degrades + to plain iALS for that entity alone. Only ZERO overlap is refused. + """ + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + encoded = encode_for_axis( + t, item_order=["i_a", "cold_1", "cold_2", "cold_3"], user_order=None + ) + dense = encoded["item_features"].toarray() + assert dense.shape[0] == 4 + # Count NONZEROS, not the row sum: `year` is standardized, so a covered + # row can legitimately sum below 1.0 via a negative z-score. + nonzero = (dense != 0).sum(axis=1).tolist() + # i_a keeps genre + year + bias; the three cold ids are bias-only. + assert nonzero == [3, 1, 1, 1] + + +def test_empty_axis_does_not_raise(items_csv: str) -> None: + """An empty axis has nothing to cover, so 0 matched is not a mismatch -- + and must not trip a 0/0 ratio. An itemless interaction table is a + different problem, already caught by the min_items precondition.""" + t = load_feature_tables(_features(items_csv), recipe_name="r", run_id="run") + encoded = encode_for_axis(t, item_order=[], user_order=None) + assert encoded["item_features"].shape[0] == 0 + + +# --------------------------------------------------------------------------- +# Review finding (Gap 2): `_check_axis_coverage` RAISES on 0% id overlap +# because "training completes and the artifact advertises `features` for what +# is really plain iALS". A whole `features:` block that prunes to bias-only +# (`n_features == 1` -- every categorical/multi_label vocab emptied, no +# numerical column) reaches the SAME end state by a third route, and +# `_features.py`'s `build_encoder_state` only WARNS there (it is a neutral +# module and must not raise a training error). The training-side refusal +# therefore lives here, matching the coverage check's posture: a signed +# artifact must not advertise features it does not actually carry. +# --------------------------------------------------------------------------- + + +def test_whole_block_pruned_to_bias_only_raises(tmp_path: Path) -> None: + """A `features:` block whose every column is emptied (here by an + unsatisfiable `min_frequency`) collapses to `n_features == 1` (bias only). + Training must refuse it, not sign an artifact advertising a no-op block.""" + p = tmp_path / "items.csv" + pd.DataFrame( + {"item_id": ["i_a", "i_b", "i_c"], "genre": ["action", "drama", "comedy"]} + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + # min_frequency=50 against a 3-row catalog prunes every token. + columns=[ + FeatureColumn(name="genre", encoding="categorical", min_frequency=50) + ], + ) + ) + with pytest.raises(TrainingError) as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + # Exit 4 (any TrainingError code except signing_key_missing -> exit 4). + assert exc_info.value.code != "signing_key_missing" + msg = str(exc_info.value) + assert "item" in msg + assert "bias" in msg + + +def test_all_null_feature_column_raises_as_bias_only(tmp_path: Path) -> None: + """The all-null route to the same bias-only end state must also refuse -- + a single categorical column with no usable values leaves `n_features == 1`. + """ + p = tmp_path / "items_null_feat.csv" + p.write_text("item_id,genre\ni_a,\ni_b,\ni_c,\n") + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(TrainingError, match="bias"): + load_feature_tables(cfg, recipe_name="r", run_id="run") + + +# --------------------------------------------------------------------------- +# Fix A: the whole-block-dead guard keyed on `n_features == 1`, which a dead +# NUMERICAL block escapes: a numerical spec always reserves width 1 (so +# n_features stays 2) even when its std was floored to 0.0 (dead) and it emits +# nothing at encode time. So an all-dead-numerical block signs an artifact +# advertising `features` that serves bias-only == plain iALS -- the exact +# silent downgrade this guard exists to refuse. The guard must key on whether +# ANY spec can emit a non-bias feature, not on n_features. +# --------------------------------------------------------------------------- + + +def test_all_dead_numerical_block_raises_as_bias_only(tmp_path: Path) -> None: + """A block whose ONLY column is a constant numerical column is dead: its + std floors to 0.0 and it emits nothing, so every entity encodes to bias + alone -- yet n_features stays 2 (width-1 numerical + bias), so the old + `n_features == 1` guard never fired. It must be refused like the + all-categorical-dead block above.""" + p = tmp_path / "items_constant_num.csv" + # ids overlap the interaction axis so the overlap check passes; the only + # feature column is a constant numerical one (std -> 0.0, dead). + pd.DataFrame({"item_id": ["i_a", "i_b", "i_c"], "score": [5, 5, 5]}).to_csv( + p, index=False + ) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="score", encoding="numerical")], + ) + ) + with pytest.raises(TrainingError) as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + assert exc_info.value.code != "signing_key_missing" # -> exit 4 + msg = str(exc_info.value) + assert "item" in msg + assert "bias" in msg + + +def test_all_null_numerical_column_raises_as_bias_only(tmp_path: Path) -> None: + """The all-null numerical route to the same bias-only end state must also + refuse: no usable values leaves std == 0.0 (dead), width still 1, so the + old `n_features == 1` guard again missed it.""" + p = tmp_path / "items_null_num.csv" + p.write_text("item_id,score\ni_a,\ni_b,\ni_c,\n") + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="score", encoding="numerical")], + ) + ) + with pytest.raises(TrainingError, match="bias"): + load_feature_tables(cfg, recipe_name="r", run_id="run") + + +def test_one_dead_column_among_several_does_not_raise(tmp_path: Path) -> None: + """The whole-block refusal must fire ONLY when the ENTIRE block is dead. + Pruning one column of several via `min_frequency` is the operator's + legitimate choice: it warns per-column (see test_features.py) but must not + abort, because the surviving column keeps `n_features > 1`.""" + p = tmp_path / "items_mixed.csv" + pd.DataFrame( + { + "item_id": ["i_a", "i_b", "i_c"], + "genre": ["action", "drama", "comedy"], + "brand": ["x", "y", "z"], + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[ + # genre pruned to nothing, brand survives. + FeatureColumn(name="genre", encoding="categorical", min_frequency=50), + FeatureColumn(name="brand", encoding="categorical"), + ], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + # brand's 3 one-hots + bias == 4; the block is NOT dead. + assert t.item_state["n_features"] == 4 + + +def test_constant_feature_column_warns_at_training_time(tmp_path: Path) -> None: + """Gap 3 at the TRAINING level: a constant categorical column (every item + the same genre) is dead (collinear with bias) and must warn when the + feature table is loaded, not only when `build_encoder_state` is called by + hand. n_features stays 2 (one one-hot + bias), so it warns but does not + raise.""" + import structlog.testing + + p = tmp_path / "items_constant.csv" + pd.DataFrame( + {"item_id": ["i_a", "i_b", "i_c"], "genre": ["action", "action", "action"]} + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with structlog.testing.capture_logs() as cap: + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + assert t.item_state["n_features"] == 2 # one one-hot + bias, not a raise + events = [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] + assert events, ( + f"a constant column must warn as dead at load time; " + f"got {[e.get('event') for e in cap]}" + ) + # PII: the genre value must not be logged. + for e in cap: + for value in e.values(): + assert "action" not in str(value) + + +# --------------------------------------------------------------------------- +# Review finding (Gap 4): the zero-overlap message samples real ids -- which +# are PII. `_id_sample` bounded the COUNT (3) but not the per-id BYTES, so two +# multi-MB ids produced a multi-MB exception message that then travels into the +# `train_error` event (`error=str(exc)` in pipeline.py). Bound each sampled +# id's length too. +# --------------------------------------------------------------------------- + + +def test_zero_overlap_message_bounds_each_sampled_id_length(tmp_path: Path) -> None: + """A pathologically long id must be truncated in the sample, so the fatal + message (and the `train_error` event carrying it) stays bounded.""" + huge = "z" * 1_000_000 + p = tmp_path / "items_huge_id.csv" + pd.DataFrame({"item_id": [huge, "other"], "genre": ["a", "b"]}).to_csv( + p, index=False + ) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + with pytest.raises(TrainingError) as exc_info: + encode_for_axis(t, item_order=["nope1", "nope2"], user_order=None) + + msg = str(exc_info.value) + # The full 1 MB id must NOT appear verbatim, and the message must stay + # small even though a sampled id was a megabyte. + assert huge not in msg + assert len(msg) < 2000, f"message length {len(msg)} not bounded" + + +# --------------------------------------------------------------------------- +# Review finding (Gap 5): the zero-overlap message hardcoded +# `dtype: {item_id: str}` as the remedy -- but `dtype` exists only on +# `CSVConfig`. A bigquery/sql/parquet operator hitting this abort was told to +# set a key their source does not have. The message must not name a +# source-specific key; it points at the docs' per-source remedy matrix instead. +# --------------------------------------------------------------------------- + + +def test_zero_overlap_message_does_not_hardcode_csv_only_dtype_key( + tmp_path: Path, +) -> None: + """The remedy must not name `dtype: {...}`, which only a `csv` source has; + a non-csv operator would be misdirected. It must instead point at the docs + that carry the per-source-type matrix.""" + p = tmp_path / "items_sku.csv" + pd.DataFrame({"sku": ["SKU-1", "SKU-2"], "genre": ["a", "b"]}).to_csv( + p, index=False + ) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="sku", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + with pytest.raises(TrainingError) as exc_info: + encode_for_axis(t, item_order=["p1", "p2"], user_order=None) + msg = str(exc_info.value) + assert "dtype:" not in msg, ( + "must not hardcode the csv-only `dtype:` remedy; a bigquery/sql/parquet " + "operator would be told to set a key their source does not have" + ) + # Still diagnosable: names the side, the id_column, and points at the docs. + assert "sku" in msg + assert "operations.md" in msg diff --git a/tests/unit/test_training_pipeline.py b/tests/unit/test_training_pipeline.py index ae62cc07..2f007c3f 100644 --- a/tests/unit/test_training_pipeline.py +++ b/tests/unit/test_training_pipeline.py @@ -10,6 +10,11 @@ - zero-score -> ZeroScoreError - per_algorithm_trials partitioning - one structured log per trial +- Task 9: feature-aware iALS wiring -- header/payload carry the encoder + state when a features: block is configured, the header omits the key + entirely otherwise, and the final refit re-encodes onto its OWN item + order rather than reusing the search phase's (a regression guard for the + one bug irspack will not raise an error for: a misordered feature matrix) """ from __future__ import annotations @@ -86,6 +91,204 @@ def _make_recipe( return recipe +# --------------------------------------------------------------------------- +# Task 9: feature-aware iALS fixtures +# +# IALS's default tune range samples n_components from [4, 300] and needs a +# matrix with real (non-degenerate) low-rank structure to factorise cleanly +# without divide-by-zero warnings; a couple of interaction rows is not +# enough. This mirrors the clustered synthetic dataset that +# tests/integration/test_serve_predict_e2e.py already uses to exercise IALS +# outside the `slow` mark. +# --------------------------------------------------------------------------- + + +def _make_clustered_synthetic_csv(tmp_path: Path) -> Path: + """Deterministic interaction matrix with real low-rank cluster structure. + + A fully-dense grid is rank-deficient and makes IALS warn/divide-by-zero; + laying users out in overlapping clusters with a few idiosyncratic items + each gives a matrix every algorithm (including IALS) factorises cleanly. + """ + n_users, n_items, n_clusters, band = 60, 40, 6, 12 + pairs: set[tuple[str, str]] = set() + for u in range(n_users): + cluster = u % n_clusters + for k in range(band): + pairs.add( + (f"u{u}", f"i{(cluster * (n_items // n_clusters) + k) % n_items}") + ) + pairs.add((f"u{u}", f"i{(u * 7) % n_items}")) + pairs.add((f"u{u}", f"i{(u * 13 + 3) % n_items}")) + rows = ["user_id,item_id"] + rows.extend(f"{u},{i}" for u, i in sorted(pairs)) + csv_file = tmp_path / "clustered.csv" + csv_file.write_text("\n".join(rows) + "\n") + return csv_file + + +@pytest.fixture +def plain_recipe(tmp_path: Path): + """A features-less recipe: the header must omit "features" entirely.""" + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + + csv_file = _make_clustered_synthetic_csv(tmp_path) + return Recipe( + name="plain_pipeline_test", + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + training=TrainingConfig( + algorithms=["TopPop"], + n_trials=1, + cutoff=5, # must be < n_items to avoid irspack ValueError + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig( + path=str(tmp_path / "plain_pipeline_test.recotem"), + versioning="always_overwrite", + ), + ) + + +@pytest.fixture +def feature_recipe(tmp_path: Path): + """An IALS recipe with an item `features:` block (genre + year).""" + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + FeatureColumn, + FeaturesConfig, + FeatureSideConfig, + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + + csv_file = _make_clustered_synthetic_csv(tmp_path) + + items_csv = tmp_path / "item_features.csv" + genres = ["action", "drama", "comedy"] + pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(40)], + "genre": [genres[i % len(genres)] for i in range(40)], + "year": [2000 + i for i in range(40)], + } + ).to_csv(items_csv, index=False) + + return Recipe( + name="feature_pipeline_test", + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + features=FeaturesConfig( + item=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(items_csv)), + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="year", encoding="numerical"), + ], + ) + ), + training=TrainingConfig( + algorithms=["IALS"], + n_trials=2, + cutoff=5, # must be < n_items to avoid irspack ValueError + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig( + path=str(tmp_path / "feature_pipeline_test.recotem"), + versioning="always_overwrite", + ), + ) + + +@pytest.fixture +def feature_recipe_both_axes(tmp_path: Path): + """An IALS recipe with BOTH an item and a USER `features:` block. + + ``feature_recipe`` above (used by most Task 9 tests) is item-only, which + means no pipeline fixture ever exercised the search-phase alignment + test's ``user_features`` row order -- a mutation of the search-phase + call site's ``user_order`` argument (e.g. reversing it) was therefore + invisible to the whole suite. This fixture exists specifically to close + that gap: it is deliberately NOT a drop-in replacement for + ``feature_recipe`` (several other tests assert ``"user" not in + parsed["features"]`` against that fixture and must keep passing). + """ + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + FeatureColumn, + FeaturesConfig, + FeatureSideConfig, + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + + csv_file = _make_clustered_synthetic_csv(tmp_path) + + items_csv = tmp_path / "item_features_both_axes.csv" + genres = ["action", "drama", "comedy"] + pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(40)], + "genre": [genres[i % len(genres)] for i in range(40)], + "year": [2000 + i for i in range(40)], + } + ).to_csv(items_csv, index=False) + + users_csv = tmp_path / "user_features_both_axes.csv" + bands = ["young", "old"] + pd.DataFrame( + { + "user_id": [f"u{u}" for u in range(60)], + "band": [bands[u % len(bands)] for u in range(60)], + } + ).to_csv(users_csv, index=False) + + return Recipe( + name="feature_pipeline_both_axes_test", + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + features=FeaturesConfig( + item=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(items_csv)), + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="year", encoding="numerical"), + ], + ), + user=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(users_csv)), + id_column="user_id", + columns=[FeatureColumn(name="band", encoding="categorical")], + ), + ), + training=TrainingConfig( + algorithms=["IALS"], + n_trials=2, + cutoff=5, # must be < n_items to avoid irspack ValueError + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig( + path=str(tmp_path / "feature_pipeline_both_axes_test.recotem"), + versioning="always_overwrite", + ), + ) + + # --------------------------------------------------------------------------- # end-to-end on small MovieLens slice with n_trials=2 # --------------------------------------------------------------------------- @@ -1845,3 +2048,656 @@ def _oom(*args, **kwargs): with patch("recotem.training.pipeline.pd.to_datetime", side_effect=_oom): with pytest.raises(MemoryError): _cleanse(df, recipe) + + +# --------------------------------------------------------------------------- +# Task 9: feature-aware iALS end-to-end wiring +# --------------------------------------------------------------------------- + + +def test_feature_aware_training_writes_state_and_header( + tmp_path: Path, feature_recipe, key_ring +) -> None: + """End-to-end: a features: recipe produces an artifact carrying both the + encoder state (payload) and the descriptor (header).""" + import json + + from recotem.artifact.io import read_artifact + from recotem.artifact.signing import unpickle_payload + from recotem.training.pipeline import run_training + + result = run_training( + feature_recipe, + key_ring=key_ring, + signing_key="active", + no_lock=True, + quiet=True, + ) + assert result is not None + + header, payload = read_artifact(result.artifact_path, key_ring) + parsed = json.loads(header.header_data) + + assert parsed["features"]["version"] == 1 + assert parsed["features"]["item"]["columns"] == ["genre", "year"] + assert "user" not in parsed["features"] + + model = unpickle_payload(payload) + assert model.item_feature_state["version"] == 1 + assert model.user_feature_state is None + # The matrix must NOT ride in best_params: that is JSON-serialized into a + # 64 KiB-capped header. + assert "item_features" not in parsed["best_params"] + assert "lambda_item_feature" in parsed["best_params"] + + +def test_no_features_recipe_omits_header_key( + tmp_path: Path, plain_recipe, key_ring +) -> None: + """A recipe with no features: block must keep the header byte-identical + to today's -- no "features" key at all, not even null.""" + import json + + from recotem.artifact.io import read_artifact + from recotem.training.pipeline import run_training + + result = run_training( + plain_recipe, + key_ring=key_ring, + signing_key="active", + no_lock=True, + quiet=True, + ) + assert result is not None + + header, _ = read_artifact(result.artifact_path, key_ring) + assert "features" not in json.loads(header.header_data) + + +def test_train_final_reencodes_features_for_its_own_axis_not_search_phase() -> None: + """Regression / mutation guard for the single most dangerous mistake in + feature-aware training: irspack raises on a feature-matrix ROW-COUNT + mismatch but accepts a MISORDERED matrix silently -- no shape error, no + value error, just a silently-wrong model. That means the header/payload + smoke test above cannot detect it: ``item_feature_state`` and the header + descriptor are built from ``feature_tables`` directly and do not depend + on whether ``_train_final`` actually re-encoded onto the right axis. + + This test makes the row order itself observable. Each item's "genre" + value IS its own item id, so the encoder's vocabulary index for item X + equals X's rank in the *sorted* item id list -- exactly + ``df_to_sparse``'s own row/column order (``pd.Categorical`` sorts its + categories). A correctly re-encoded matrix is therefore the identity + permutation: row i's lone non-bias one-hot sits at column i. Reusing a + matrix built for any other ordering -- e.g. the search phase's + ``list(set(...))`` order, which is neither sorted nor stable across + processes for string ids -- breaks that identity and fails the + assertion below. + """ + from recotem._features import build_encoder_state + from recotem.recipe.models import FeatureColumn + from recotem.training.features import FeatureTables + from recotem.training.pipeline import _train_final + + item_ids = ["i5", "i1", "i9", "i3"] # deliberately unsorted input order + df = pd.DataFrame( + { + "user_id": ["u1", "u2", "u1", "u2"], + "item_id": item_ids, + } + ) + # The feature table's "genre" value for each item IS its own id, so the + # vocab index is a stand-in for "which item is this row". + item_df = pd.DataFrame({"genre": item_ids}, index=item_ids) + item_state = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + tables = FeatureTables(item_state=item_state, item_df=item_df) + + captured: dict = {} + + class FakeIALS: + """Stand-in for IALSRecommender: records the matrix it was given.""" + + def __init__( + self, X, lambda_item_feature: float = 0.0, item_features=None + ) -> None: + captured["item_features"] = item_features + + def learn(self): + return self + + with patch( + "recotem.training.pipeline.get_recommender_cls", + return_value=FakeIALS, + ): + # class_name must be a REAL, feature-capable canonical class name + # ("IALSRecommender") -- not an arbitrary fake string -- because + # _train_final now gates final_feature_kwargs on + # is_feature_capable(class_name) (Finding 1 fix), which resolves the + # string through the real alias table. get_recommender_cls is + # patched above so the actual class instantiated is still FakeIALS + # regardless of this string. + result = _train_final( + df, + user_column="user_id", + item_column="item_id", + class_name="IALSRecommender", + best_params={"lambda_item_feature": 0.1}, + feature_tables=tables, + ) + + dense = captured["item_features"].toarray() + vocab = item_state["columns"][0]["vocab"] + bias_col = item_state["bias_offset"] + + for row_idx, iid in enumerate(result.item_ids): + expected_col = vocab[iid] + nonzero_cols = set(dense[row_idx].nonzero()[0].tolist()) + assert nonzero_cols == {expected_col, bias_col}, ( + f"row {row_idx} (item {iid!r}): expected the one-hot at column " + f"{expected_col} (plus the always-on bias column {bias_col}), " + f"got nonzero columns {nonzero_cols}. _train_final must " + "re-encode features against df_to_sparse's OWN item order -- " + "it must never reuse/cache a matrix built for a different " + "ordering (e.g. the search phase's)." + ) + + +def test_train_final_feature_cholesky_error_message_does_not_blame_a_column() -> None: + """A Cholesky failure during the FINAL refit must map to TrainingError + with an actionable message -- and that message must NOT tell the user to + drop a column. recotem's own always-on bias column is deliberately + collinear with the categorical one-hots (see recotem._features's module + docstring) and is the single most likely structural cause, and the user + cannot remove it from the recipe. + """ + from recotem._features import build_encoder_state + from recotem.recipe.models import FeatureColumn + from recotem.training.errors import TrainingError + from recotem.training.features import FeatureTables + from recotem.training.pipeline import _train_final + + df = pd.DataFrame( + { + "user_id": ["u1", "u2"], + "item_id": ["i1", "i2"], + } + ) + item_df = pd.DataFrame({"genre": ["a", "b"]}, index=["i1", "i2"]) + item_state = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + tables = FeatureTables(item_state=item_state, item_df=item_df) + + class RankDeficientRec: + def __init__( + self, X, lambda_item_feature: float = 0.0, item_features=None + ) -> None: + pass + + def learn(self): + raise RuntimeError( + "Feature ridge Cholesky decomposition failed: matrix is not " + "positive definite" + ) + + with patch( + "recotem.training.pipeline.get_recommender_cls", + return_value=RankDeficientRec, + ): + # class_name must be a REAL, feature-capable canonical class name + # ("IALSRecommender") so is_feature_capable(class_name) (Finding 1 + # fix) lets final_feature_kwargs be built; get_recommender_cls is + # patched above so RankDeficientRec is still what actually gets + # instantiated. + with pytest.raises(TrainingError, match="Cholesky") as exc_info: + _train_final( + df, + user_column="user_id", + item_column="item_id", + class_name="IALSRecommender", + best_params={"lambda_item_feature": 0.1}, + feature_tables=tables, + ) + + assert exc_info.value.code == "feature_cholesky_error" + message = str(exc_info.value) + assert "drop" not in message.lower(), ( + f"the error message must not tell the user to drop a column " + f"(recotem's own bias column is the most likely structural cause " + f"and cannot be removed from the recipe); got: {message!r}" + ) + assert "min_frequency" in message + + +def test_train_final_without_features_is_unaffected_by_feature_tables_param() -> None: + """Back-compat: omitting ``feature_tables`` (the pre-Task-9 call shape) + must behave exactly as before -- no feature kwargs, no feature state on + the returned wrapper.""" + from recotem.training.pipeline import _train_final + + df = pd.DataFrame( + {"user_id": ["u1", "u2", "u1", "u3"], "item_id": ["i1", "i1", "i2", "i2"]} + ) + + class PlainRec: + def __init__(self, X) -> None: + self.X = X + + def learn(self): + return self + + with patch("recotem.training.pipeline.get_recommender_cls", return_value=PlainRec): + result = _train_final( + df, + user_column="user_id", + item_column="item_id", + class_name="PlainRec", + best_params={}, + ) + + assert result.item_feature_state is None + assert result.user_feature_state is None + + +# --------------------------------------------------------------------------- +# Review finding 1 (CRITICAL): _train_final must not crash when the search +# winner is a features:-enabled recipe's non-feature-capable algorithm. +# +# Recipe._validate_features_algorithms only requires that AT LEAST ONE +# listed algorithm be feature-capable (see recipe/models.py); an +# `algorithms: ["TopPop", "IALS"]` recipe with a `features:` block is +# explicitly valid, and the search may legitimately pick TopPop as the +# winner. Pre-fix, `_train_final` built `final_feature_kwargs` whenever +# `feature_tables.enabled`, with no check on whether `class_name` accepts +# feature kwargs -- splatting `item_features` into TopPop's constructor +# (which only accepts `X_train`) raised +# `TypeError: TopPopRecommender.__init__() got an unexpected keyword +# argument 'item_features'`, wrapped as +# TrainingError(code="final_training_error"). search.py's run_search +# already gated its analogous `trial_features` on `is_feature_capable` +# (search.py:398-402); that gate was never replicated for the final refit. +# --------------------------------------------------------------------------- + + +def test_train_final_with_non_feature_capable_winner_does_not_crash() -> None: + """``_train_final`` must produce a valid (non-feature) artifact, not + raise, when ``class_name`` resolves to a real, non-patched, + non-feature-capable irspack class (``TopPopRecommender``) alongside an + enabled ``feature_tables``. + + Uses the REAL irspack ``TopPopRecommender`` (not a mock/fake class) so + the constructor-signature mismatch this test guards against is genuine, + matching the reviewer's exact reproduction. + """ + from recotem._features import build_encoder_state + from recotem.recipe.models import FeatureColumn + from recotem.training.features import FeatureTables + from recotem.training.pipeline import _train_final + + df = pd.DataFrame( + { + "user_id": ["u1", "u2", "u1", "u2"], + "item_id": ["i1", "i2", "i3", "i4"], + } + ) + item_df = pd.DataFrame( + {"genre": ["a", "b", "a", "b"]}, index=["i1", "i2", "i3", "i4"] + ) + item_state = build_encoder_state( + item_df, [FeatureColumn(name="genre", encoding="categorical")] + ) + tables = FeatureTables(item_state=item_state, item_df=item_df) + + # class_name="TopPopRecommender" resolves via get_recommender_cls to the + # real irspack class -- no patch("...get_recommender_cls", ...) here. + result = _train_final( + df, + user_column="user_id", + item_column="item_id", + class_name="TopPopRecommender", + best_params={}, + feature_tables=tables, + ) + + assert result is not None + assert sorted(result.item_ids) == ["i1", "i2", "i3", "i4"] + + # Decision (see task report for full rationale): item_feature_state is + # STILL persisted even though TopPop never received item_features at + # construction time. The header's "features" descriptor is already + # unconditional on feature_tables.enabled (independent of best_class -- + # see _run_training_locked's header-building step below the search), + # so nulling the payload side out here would make the payload silently + # disagree with the header for the very same artifact. The encoder + # state is descriptive metadata about what the recipe configured, not a + # claim that the winning model consumed it. + assert result.item_feature_state is not None + assert result.item_feature_state["version"] == 1 + + +def test_multi_algorithm_features_recipe_with_toppop_winner_produces_valid_artifact( + tmp_path: Path, key_ring +) -> None: + """Full run_training proof of the reviewer's exact scenario: a legal + multi-algorithm recipe (``algorithms: ["TopPop", "IALS"]``) with a + ``features:`` block, where TopPop -- not IALS -- wins the search. + + ``per_algorithm_trials={"IALS": 0, "TopPop": 1}`` forces TopPop to be + the only algorithm that actually runs (budget 0 means "skip", per + ``_compute_budgets``), while the recipe-level validator is satisfied + because IALS is still *listed* in ``training.algorithms``. Pre-fix, this + is a 100%-reproducible hard failure for any operator running exactly + this configuration; post-fix it must produce a normal, non-feature + artifact. + """ + from recotem.datasource.csv import CSVConfig + from recotem.recipe.models import ( + FeatureColumn, + FeaturesConfig, + FeatureSideConfig, + OutputConfig, + Recipe, + SchemaConfig, + SplitConfig, + TrainingConfig, + ) + from recotem.training.pipeline import run_training + + csv_file = _make_clustered_synthetic_csv(tmp_path) + + items_csv = tmp_path / "item_features_toppop_winner.csv" + genres = ["action", "drama", "comedy"] + pd.DataFrame( + { + "item_id": [f"i{i}" for i in range(40)], + "genre": [genres[i % len(genres)] for i in range(40)], + } + ).to_csv(items_csv, index=False) + + recipe = Recipe( + name="toppop_winner_features_test", + source=CSVConfig(type="csv", path=str(csv_file)), + schema=SchemaConfig(user_column="user_id", item_column="item_id"), + features=FeaturesConfig( + item=FeatureSideConfig( + source=CSVConfig(type="csv", path=str(items_csv)), + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ), + training=TrainingConfig( + algorithms=["TopPop", "IALS"], + n_trials=1, + per_algorithm_trials={"IALS": 0, "TopPop": 1}, + cutoff=5, + split=SplitConfig(scheme="random", heldout_ratio=0.2, seed=0), + ), + output=OutputConfig( + path=str(tmp_path / "toppop_winner_features_test.recotem"), + versioning="always_overwrite", + ), + ) + + result = run_training( + recipe, + key_ring=key_ring, + signing_key="active", + no_lock=True, + quiet=True, + ) + + assert result is not None + assert result.best_class == "TopPopRecommender", ( + "test setup invariant: IALS must have budget 0 so TopPop is " + f"guaranteed to win; got best_class={result.best_class!r}" + ) + + +# --------------------------------------------------------------------------- +# Review finding 2 (Important): the SEARCH-phase feature encoding call site +# (pipeline.py, immediately after split_interactions, before run_search) had +# no misordering regression test of its own -- only the final-refit +# re-encoding did (test_train_final_reencodes_features_for_its_own_axis_not_ +# search_phase above). Because the final refit re-encodes independently, a +# search-phase-only misordering bug (e.g. `sorted(split_result.item_ids)` +# instead of `split_result.item_ids`) would not corrupt the shipped model, +# but WOULD silently corrupt the hyperparameter search: lambda tuned against +# a mismatched item<->feature correspondence, with no error and (pre-fix) no +# failing test. +# --------------------------------------------------------------------------- + + +def test_search_phase_feature_kwargs_match_split_result_axis_order_exactly( + feature_recipe_both_axes, key_ring +) -> None: + """Spy on run_search's ``feature_kwargs`` argument and assert its row + order matches ``split_result.item_ids`` / ``row_user_ids`` EXACTLY -- by + order, not merely by set -- for BOTH the item and the user axis. + + Strategy: wrap (not replace) ``split_interactions`` and + ``load_feature_tables`` so the REAL split result and REAL feature tables + are captured alongside whatever ``run_search`` actually received, then + independently recompute the expected encoding from the captured + ``split_result.item_ids`` / ``row_user_ids`` and assert it is row-for-row + identical to what ``run_search`` was given. ``run_search`` itself is + replaced with a stub that raises immediately after capturing its + ``feature_kwargs``, so this test does not need a real, successful Optuna + search to complete. + + A same-SET-but-different-ORDER mutation of the search-phase call site + (e.g. ``item_order=sorted(split_result.item_ids)`` or + ``user_order=list(reversed(split_result.row_user_ids))``) changes the + actual encoding's row order without changing which items/users are + present, so it is invisible to any assertion that only checks set + membership or matrix shape -- exactly the gap this test targets. + + Uses ``feature_recipe_both_axes`` (not the item-only ``feature_recipe``): + a prior version of this test only configured ``features.item``, so + ``feature_kwargs`` never contained ``user_features`` at all and no + assertion here could ever have observed a misordered user axis -- + verified by mutating ``pipeline.py``'s search-phase call site from + ``user_order=split_result.row_user_ids`` to + ``user_order=list(reversed(split_result.row_user_ids))`` and confirming + the full suite still passed (the exact mirror of the item-axis hazard + this whole feature is built to prevent). + """ + from recotem.training import pipeline as pipeline_mod + from recotem.training.features import encode_for_axis + from recotem.training.pipeline import run_training + from recotem.training.split import split_interactions as real_split_interactions + + captured: dict = {} + + def _spy_split_interactions(*args, **kwargs): + result = real_split_interactions(*args, **kwargs) + captured["split_result"] = result + return result + + real_load_feature_tables = pipeline_mod.load_feature_tables + + def _spy_load_feature_tables(*args, **kwargs): + tables = real_load_feature_tables(*args, **kwargs) + captured["feature_tables"] = tables + return tables + + class _StopAfterCapture(Exception): + """Sentinel raised to short-circuit the pipeline right after + run_search is invoked, so no real search need complete.""" + + def _spy_run_search(*args, feature_kwargs=None, **kwargs): + captured["feature_kwargs"] = feature_kwargs + raise _StopAfterCapture + + with ( + patch( + "recotem.training.pipeline.split_interactions", + side_effect=_spy_split_interactions, + ), + patch( + "recotem.training.pipeline.load_feature_tables", + side_effect=_spy_load_feature_tables, + ), + patch( + "recotem.training.pipeline.run_search", + side_effect=_spy_run_search, + ), + ): + with pytest.raises(_StopAfterCapture): + run_training( + feature_recipe_both_axes, + key_ring=key_ring, + signing_key="active", + no_lock=True, + quiet=True, + ) + + split_result = captured["split_result"] + feature_tables = captured["feature_tables"] + actual_kwargs = captured["feature_kwargs"] + + assert actual_kwargs is not None and "item_features" in actual_kwargs, ( + f"run_search must have received item_features; got {actual_kwargs!r}" + ) + assert "user_features" in actual_kwargs, ( + f"run_search must have received user_features; got {actual_kwargs!r}" + ) + + # Test-setup invariant: split_result.item_ids must not already be sorted, + # or a sorted(...) mutation would be unobservable by coincidence and this + # test would pass whether or not the mutation guard is present. + assert split_result.item_ids != sorted(split_result.item_ids), ( + "test setup invariant violated: split_result.item_ids is already in " + "sorted order, so this test cannot distinguish correct code from " + "the sorted(...) mutation it targets" + ) + + # Test-setup invariant for the USER axis. Unlike item_ids (built from + # irspack's `list(set(...))`, hash-order-dependent and not sorted for a + # typical string-id fixture), row_user_ids is built from pandas + # Categorical-backed user indexing and comes out ALREADY sorted for a + # typical "u0".."u59" fixture -- confirmed empirically, not assumed. That + # makes a `sorted(...)` mutation of user_order a value-level NO-OP here + # (indistinguishable from correct code), so the mutation this guards + # against is `reversed(...)`, not `sorted(...)`, and the vacuity check + # must match: assert the row order differs from its own reversal. + assert split_result.row_user_ids != list(reversed(split_result.row_user_ids)), ( + "test setup invariant violated: split_result.row_user_ids is a " + "palindrome under reversal, so this test cannot distinguish correct " + "code from the reversed(...) mutation it targets" + ) + + # Recompute independently from the SAME feature_tables and the split + # phase's OWN (real, unsorted) axis labels. + expected_kwargs = encode_for_axis( + feature_tables, + item_order=split_result.item_ids, + user_order=split_result.row_user_ids, + ) + + actual_dense = actual_kwargs["item_features"].toarray() + expected_dense = expected_kwargs["item_features"].toarray() + assert actual_dense.shape == expected_dense.shape + assert (actual_dense == expected_dense).all(), ( + "run_search's feature_kwargs['item_features'] must be row-for-row " + "identical to encode_for_axis(feature_tables, " + "item_order=split_result.item_ids, user_order=split_result." + "row_user_ids) -- a same-SET-but-different-ORDER item_order (e.g. " + "sorted(split_result.item_ids)) must fail this assertion." + ) + + actual_user_dense = actual_kwargs["user_features"].toarray() + expected_user_dense = expected_kwargs["user_features"].toarray() + assert actual_user_dense.shape == expected_user_dense.shape + assert (actual_user_dense == expected_user_dense).all(), ( + "run_search's feature_kwargs['user_features'] must be row-for-row " + "identical to encode_for_axis(feature_tables, " + "item_order=split_result.item_ids, user_order=split_result." + "row_user_ids) -- a same-SET-but-different-ORDER user_order (e.g. " + "list(reversed(split_result.row_user_ids))) must fail this " + "assertion. This is the exact mirror of the item-axis hazard above, " + "on the axis that had no coverage at all before this test." + ) + + +# --------------------------------------------------------------------------- +# Review finding 4: the feature-aware iALS design spec ("Testing" + "Risks") +# promises the alignment test above "must run under multiple PYTHONHASHSEED +# values to catch any +# reintroduced 'build once, reuse' shortcut". That promise was fulfilled by a +# manual `for seed in 0 1 2; do PYTHONHASHSEED=$seed uv run pytest ...` +# shell loop run once during implementation -- not a standing guard a future +# regression would ever trip, and outside the test suite entirely. +# +# Worse: the vacuity guard inside the alignment test itself (`assert +# split_result.item_ids != sorted(split_result.item_ids)`, a few lines +# above) is itself hash-order-dependent. Python randomizes PYTHONHASHSEED +# per process by default, so whether that guard's precondition is even +# satisfied -- and therefore whether the alignment test exercises a +# non-sorted order at all -- varies run to run, uncontrolled. +# +# This test converts the promise into a standing, in-suite guard: it +# re-executes the alignment test above, as a subprocess, under each of the +# three PYTHONHASHSEED values the spec's own worked example (Hazard 1, +# "Silent misalignment") demonstrates produce three DIFFERENT +# `list(set(...))` orderings for string item ids -- so this is not merely +# "run under several arbitrary seeds", it specifically covers the seeds the +# design doc already showed to be non-sorted, guaranteeing real coverage +# rather than leaving it to chance on whatever seed pytest happens to start +# with. +# +# Re-invokes pytest on the single node id rather than duplicating the +# alignment test's spy/compare logic here: any future edit to that test +# keeps being exactly what this guard re-runs, with no second copy that +# could silently drift out of sync (the same "kept in sync by hand" risk +# this codebase calls out elsewhere, e.g. +# `_idmap._FEATURE_CAPABLE_CLASS_NAMES`). +# +# A CI matrix entry was considered instead (running the alignment test +# under a PYTHONHASHSEED matrix in .github/workflows/test.yml) and rejected: +# a workflow YAML edit is invisible to `uv run pytest` and easy for someone +# tuning CI runtime to delete without realizing what guarantee it was +# providing, whereas this test fails the same `pytest tests` invocation +# every contributor already runs locally and in CI. +# --------------------------------------------------------------------------- + + +def test_search_phase_feature_alignment_holds_across_hash_seeds() -> None: + """Standing guard: re-run the alignment test above under three fixed + PYTHONHASHSEED values so a reintroduced 'build once, reuse' shortcut + (e.g. ``item_order=sorted(split_result.item_ids)`` at the search-phase + feature-encoding call site in ``pipeline.py``) cannot slip back in + unnoticed just because one particular process's random hash seed + happened to produce an order the vacuity guard was happy with. + """ + import os + import subprocess + import sys + + node_id = ( + f"{__file__}::" + "test_search_phase_feature_kwargs_match_split_result_axis_order_exactly" + ) + # The exact three seeds the feature-aware iALS design spec's Hazard 1 + # demonstrates produce three DIFFERENT `list(set(...))` orderings for + # string item ids -- not an arbitrary choice of "a few seeds". + failures: list[str] = [] + for seed in ("0", "1", "2"): + env = {**os.environ, "PYTHONHASHSEED": seed} + result = subprocess.run( + [sys.executable, "-m", "pytest", node_id, "-q", "--no-header"], + capture_output=True, + text=True, + env=env, + timeout=120, + ) + if result.returncode != 0: + failures.append( + f"PYTHONHASHSEED={seed} FAILED (exit {result.returncode}):\n" + f"{result.stdout}\n{result.stderr}" + ) + assert not failures, ( + "alignment test failed under one or more fixed hash seeds:\n\n" + + "\n\n".join(failures) + ) diff --git a/tests/unit/test_training_pipeline_train_error.py b/tests/unit/test_training_pipeline_train_error.py index 71e1d97e..5dfdd0e9 100644 --- a/tests/unit/test_training_pipeline_train_error.py +++ b/tests/unit/test_training_pipeline_train_error.py @@ -246,11 +246,21 @@ def _mock_write(payload_obj, header_dict, key_ring, fs_path, *, versioning): X_sparse = sps.csr_matrix(np.ones((2, 2))) + from recotem.training.split import SplitResult + + fake_split_result = SplitResult( + X_train_full=X_sparse, + X_val_test=X_sparse, + val_offset=1, + item_ids=["i1", "i2"], + row_user_ids=["u1", "u2"], + ) + with ( patch("recotem.training.pipeline._fetch_data", return_value=mock_df), patch( "recotem.training.pipeline.split_interactions", - return_value=(X_sparse, X_sparse, 1), + return_value=fake_split_result, ), patch("recotem.training.pipeline.build_evaluator", return_value=MagicMock()), patch("recotem.training.pipeline.run_search", return_value=fake_search_result), diff --git a/tests/unit/test_training_search.py b/tests/unit/test_training_search.py index e7da2aa5..6812a973 100644 --- a/tests/unit/test_training_search.py +++ b/tests/unit/test_training_search.py @@ -24,7 +24,7 @@ import scipy.sparse as sps from recotem.training.errors import SearchError, TrainingError -from recotem.training.search import _compute_budgets, _make_storage +from recotem.training.search import _compute_budgets, _construct, _make_storage # --------------------------------------------------------------------------- # B1. n_trials smaller than number of explicit-positive classes @@ -1180,6 +1180,121 @@ def _optimize_one_fail(objective, n_trials, **kwargs): assert "deliberate trial failure" in evt.get("error", "") +# --------------------------------------------------------------------------- +# Task 7 finding: a Cholesky-triggered TrialPruned must not be logged as a +# trial_learn_failed WARNING under the threaded (per_trial_timeout_seconds) +# path. optuna.TrialPruned subclasses Exception, so the generic +# `except Exception` handler in `_learn()` used to catch it on its way out +# and log it exactly like a genuine failure, even though search.py converts +# the rank-deficient-feature-Gram RuntimeError into TrialPruned by design. +# --------------------------------------------------------------------------- + + +def test_cholesky_prune_does_not_log_trial_learn_failed() -> None: + """A Cholesky-triggered prune in the threaded path must stay silent. + + Reproduces the reviewer's finding with a fake recommender whose + ``learn_with_optimizer`` raises the exact Cholesky RuntimeError message + that search.py's RuntimeError handler recognises and converts into + ``optuna.TrialPruned``. With ``per_trial_timeout_seconds`` set (the + threaded ``_learn()`` path), the prune must propagate to the caller + without a spurious ``trial_learn_failed`` WARNING -- the by-design prune + must read identically to the non-threaded path, which logs nothing. + """ + import structlog.testing + + from recotem.training.progress import ProgressReporter + from recotem.training.search import run_search + + class _CholeskyFailingRecommender: + learnt_config: dict = {} + + def __init__(self, X, **kwargs): + pass + + @staticmethod + def default_suggest_parameter(trial, space): + return {} + + def learn_with_optimizer(self, evaluator, trial): + raise RuntimeError( + "Feature ridge Cholesky decomposition failed: rank deficient" + ) + + def learn(self): + return self + + def _make_fake_completed(number: int) -> MagicMock: + t = MagicMock(spec=optuna.trial.FrozenTrial) + t.state = optuna.trial.TrialState.COMPLETE + t.value = -0.5 + t.number = number + t.params = {"recommender_class_name": "TopPopRecommender"} + t.user_attrs = {"recommender_class_name": "TopPopRecommender"} + return t + + fake_completed = [_make_fake_completed(i) for i in range(3)] + pruned_raised = [False] + + with patch( + "recotem.training.search.get_recommender_cls", + return_value=_CholeskyFailingRecommender, + ): + with patch("recotem.training.search.optuna.create_study") as mock_study_fn: + mock_study = MagicMock() + + def _optimize_one_prune(objective, n_trials, **kwargs): + fake_t = MagicMock(spec=optuna.Trial) + fake_t.number = 0 + fake_t.suggest_categorical.return_value = "TopPopRecommender" + fake_t.set_user_attr = MagicMock() + try: + objective(fake_t) + except optuna.TrialPruned: + pruned_raised[0] = True + except Exception: # noqa: BLE001 + pass + mock_study.trials = fake_completed + mock_study.best_trial = fake_completed[0] + + mock_study.trials = [] + mock_study.optimize = _optimize_one_prune + mock_study_fn.return_value = mock_study + + X = sps.csr_matrix(np.ones((5, 3))) + evaluator = MagicMock() + + with structlog.testing.capture_logs() as cap: + with ProgressReporter( + n_trials=1, recipe_name="cholesky_prune_test", run_id="rcp1" + ) as rep: + run_search( + algorithms=["TopPopRecommender"], + X_tv_train=X, + evaluator=evaluator, + n_trials=1, + per_algorithm_trials=None, + per_trial_timeout_seconds=1, # use thread path + timeout_seconds=None, + parallelism=1, + storage_path="", + random_seed=42, + reporter=rep, + recipe_name="cholesky_prune_test", + run_id="rcp1", + ) + + assert pruned_raised[0], ( + "objective must raise optuna.TrialPruned for the Cholesky RuntimeError" + ) + + fail_events = [e for e in cap if e.get("event") == "trial_learn_failed"] + assert not fail_events, ( + f"Cholesky-triggered prune must NOT emit trial_learn_failed; " + f"got events: {fail_events}" + ) + + def test_unknown_algorithm_in_per_algorithm_trials_raises() -> None: """A typo in per_algorithm_trials (e.g. 'IALSS') must raise TrainingError with code='unknown_algorithm_in_budget' — not silently drop to zero budget. @@ -1310,3 +1425,312 @@ def _mock_get_score(evaluator, recommender): f"Expected exactly {BUDGET_PER_ALGO * len(ALGO_NAMES)} total trials " f"(O(N) budget enforcement), got {result.n_completed}" ) + + +# --------------------------------------------------------------------------- +# Task 7: feature-aware construction -- single construction-site guard +# +# search.py constructs a recommender in TWO places (the per-trial-timeout +# thread path and the default else-branch), and irspack fails asymmetrically: +# a feature matrix with lambda=0 raises loudly, but a lambda with NO feature +# matrix trains silently as plain iALS. _construct is the single point both +# sites must route through so that asymmetry becomes an unconditional +# AssertionError instead of a silently-wrong shipped model. +# --------------------------------------------------------------------------- + + +class _FakeRec: + def __init__(self, X, **kwargs): + self.X = X + self.kwargs = kwargs + + +def test_construct_injects_feature_matrices() -> None: + X = sps.csr_matrix(np.ones((2, 3))) + F = sps.csr_matrix(np.ones((3, 2), dtype=np.float32)) + rec = _construct(_FakeRec, X, {"lambda_item_feature": 0.1}, {"item_features": F}) + assert rec.kwargs["item_features"] is F + + +def test_construct_rejects_lambda_without_matrix() -> None: + """irspack does NOT raise for lambda-without-features: it silently trains + plain iALS. Convert that silent asymmetry into a loud one. + """ + X = sps.csr_matrix(np.ones((2, 3))) + with pytest.raises(AssertionError, match="lambda_item_feature"): + _construct(_FakeRec, X, {"lambda_item_feature": 0.1}, {}) + + +def test_construct_rejects_user_lambda_without_matrix() -> None: + X = sps.csr_matrix(np.ones((2, 3))) + with pytest.raises(AssertionError, match="lambda_user_feature"): + _construct(_FakeRec, X, {"lambda_user_feature": 0.1}, None) + + +def test_construct_without_features_is_unchanged() -> None: + X = sps.csr_matrix(np.ones((2, 3))) + rec = _construct(_FakeRec, X, {"n_components": 4}, None) + assert rec.kwargs == {"n_components": 4} + + +# --------------------------------------------------------------------------- +# The guard above is only worth anything if it cannot be compiled away. A bare +# `assert` is exactly the statement `-O` / PYTHONOPTIMIZE strips, which would +# silently restore the plain-iALS bug _construct exists to prevent -- the +# failure mode is a wrong model shipped with no error and no warning, so a +# guard against it must not be one environment variable away from a no-op. +# +# The three tests above run under the suite's own interpreter, which never has +# -O set, so they cannot see that difference: they pass identically whether the +# guard is a strippable `assert` or an unconditional `raise`. This test closes +# that gap by re-checking the guard in a subprocess that really is optimized, +# mirroring the subprocess-under-fixed-interpreter-flags pattern already used +# by test_search_phase_feature_alignment_holds_across_hash_seeds in +# tests/unit/test_training_pipeline.py. +# +# The script drives _construct directly rather than re-running a pytest node +# under -O: pytest's assertion rewriting compiles test-module asserts into +# explicit raises, so a rewritten `pytest.raises(AssertionError)` keeps passing +# under -O regardless of what src/ does. That would make the whole test vacuous +# -- it would pass even against the stripped `assert` it is meant to catch. +# --------------------------------------------------------------------------- + +_MINUS_O_GUARD_SCRIPT = """ +import sys + +import numpy as np +import scipy.sparse as sps + +from recotem.training.search import _construct + + +class _FakeRec: + def __init__(self, X, **kwargs): + self.kwargs = kwargs + + +# Vacuity guard: prove -O actually took effect in this process, so that a +# subprocess that silently ran unoptimized cannot report a false pass. This +# deliberately is not an `assert` -- that is the very statement -O removes, +# which would make the vacuity guard itself vanish under the condition it is +# checking for. +if __debug__: + print("NOT_OPTIMIZED", file=sys.stderr) + sys.exit(2) + +X = sps.csr_matrix(np.ones((2, 3))) + +for lam in ("lambda_item_feature", "lambda_user_feature"): + try: + _construct(_FakeRec, X, {lam: 0.5}, {}) + except AssertionError: + continue + # Reached only if the guard did not fire: _construct returned a + # recommender built with the lambda and no feature matrix, which is + # exactly the silent plain-iALS training the helper exists to prevent. + print("GUARD_STRIPPED:" + lam, file=sys.stderr) + sys.exit(3) + +print("OK") +""" + + +def test_construct_guard_survives_python_optimize() -> None: + """The lambda-without-matrix guard must still fire under ``-O``. + + Standing guard against the guard itself being written as a bare ``assert``: + under ``-O`` those are stripped at compile time, and _construct would then + hand ``lambda_item_feature`` to irspack with no matrix and silently train + plain iALS. + """ + import subprocess + import sys + + result = subprocess.run( + [sys.executable, "-O", "-c", _MINUS_O_GUARD_SCRIPT], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, ( + f"_construct's guard did not hold under -O (exit {result.returncode}); " + "exit 2 = subprocess was not actually optimized (test would have been " + "vacuous), exit 3 = guard was stripped and construction silently " + f"succeeded.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +@pytest.mark.parametrize("per_trial_timeout", [None, 1]) +def test_both_search_paths_receive_feature_matrices(per_trial_timeout) -> None: + """search.py constructs in two places and per_trial_timeout_seconds picks + which. Patching only one is silent: irspack trains plain iALS for a + lambda with no matrix, so the search would score models unlike the one + shipped. + + Parametrizing over per_trial_timeout drives both construction sites -- + ``None`` takes the default else-branch (search.py's non-threaded path), + ``1`` takes the per-trial-timeout thread path. A guard that only patched + one site would pass for one parametrization and fail for the other. + """ + from recotem.training.progress import ProgressReporter + from recotem.training.search import run_search + + seen: list[dict] = [] + + class _RecordingRecommender: + learnt_config: dict = {} + + def __init__(self, X, **kwargs): + seen.append(dict(kwargs)) + + @staticmethod + def default_suggest_parameter(trial, space): + return {} + + def learn_with_optimizer(self, evaluator, trial): + return None + + def learn(self): + return self + + F = sps.csr_matrix(np.ones((4, 2), dtype=np.float32)) + X = sps.csr_matrix(np.eye(4, dtype=np.float64)) + + with patch( + "recotem.training.search.get_recommender_cls", + return_value=_RecordingRecommender, + ): + with patch("recotem.training.search.optuna.create_study") as mock_study_fn: + mock_study = MagicMock() + + def _optimize(objective, n_trials, **kwargs): + fake_t = MagicMock(spec=optuna.Trial) + fake_t.number = 0 + fake_t.suggest_categorical.return_value = "IALSRecommender" + fake_t.suggest_float.return_value = 0.5 + fake_t.set_user_attr = MagicMock() + try: + objective(fake_t) + except Exception: # noqa: BLE001 + pass + + mock_study.optimize.side_effect = _optimize + mock_study_fn.return_value = mock_study + + with ProgressReporter( + n_trials=1, recipe_name="feature_construct_test", run_id="fc1" + ) as rep: + try: + run_search( + algorithms=["IALSRecommender"], + X_tv_train=X, + evaluator=MagicMock(), + n_trials=1, + per_algorithm_trials=None, + per_trial_timeout_seconds=per_trial_timeout, + timeout_seconds=None, + parallelism=1, + storage_path="", + random_seed=42, + reporter=rep, + recipe_name="feature_construct_test", + run_id="fc1", + feature_kwargs={"item_features": F}, + ) + except Exception: # noqa: BLE001 + pass # we assert on construction, not on a usable SearchResult + + assert seen, "no trial was constructed" + assert all("item_features" in kwargs for kwargs in seen), ( + f"per_trial_timeout={per_trial_timeout} path did not inject features: {seen}" + ) + + +def test_non_feature_capable_algorithm_does_not_receive_feature_kwargs() -> None: + """A class outside FEATURE_CAPABLE_CLASS_NAMES (e.g. TopPop) must never + receive item_features/user_features kwargs, even when feature_kwargs is + supplied at the run_search level (e.g. a multi-algorithm search that + also includes IALS). + """ + from recotem.training.progress import ProgressReporter + from recotem.training.search import run_search + + seen: list[dict] = [] + + class _RecordingRecommender: + learnt_config: dict = {} + + def __init__(self, X, **kwargs): + seen.append(dict(kwargs)) + + @staticmethod + def default_suggest_parameter(trial, space): + return {} + + def learn_with_optimizer(self, evaluator, trial): + return None + + def learn(self): + return self + + F = sps.csr_matrix(np.ones((4, 2), dtype=np.float32)) + X = sps.csr_matrix(np.eye(4, dtype=np.float64)) + + with patch( + "recotem.training.search.get_recommender_cls", + return_value=_RecordingRecommender, + ): + with patch("recotem.training.search.optuna.create_study") as mock_study_fn: + mock_study = MagicMock() + + def _optimize(objective, n_trials, **kwargs): + fake_t = MagicMock(spec=optuna.Trial) + fake_t.number = 0 + fake_t.suggest_categorical.return_value = "TopPopRecommender" + fake_t.set_user_attr = MagicMock() + try: + objective(fake_t) + except Exception: # noqa: BLE001 + pass + + mock_study.optimize.side_effect = _optimize + mock_study_fn.return_value = mock_study + + with ProgressReporter( + n_trials=1, recipe_name="non_feature_capable_test", run_id="nfc1" + ) as rep: + try: + run_search( + algorithms=["TopPopRecommender"], + X_tv_train=X, + evaluator=MagicMock(), + n_trials=1, + per_algorithm_trials=None, + per_trial_timeout_seconds=None, + timeout_seconds=None, + parallelism=1, + storage_path="", + random_seed=42, + reporter=rep, + recipe_name="non_feature_capable_test", + run_id="nfc1", + feature_kwargs={"item_features": F}, + ) + except Exception: # noqa: BLE001 + pass + + assert seen, "no trial was constructed" + assert all("item_features" not in kwargs for kwargs in seen), ( + f"non-feature-capable class must not receive item_features: {seen}" + ) + + +def test_run_search_feature_kwargs_defaults_to_none() -> None: + """Existing callers that omit feature_kwargs must keep working unchanged.""" + import inspect + + from recotem.training.search import run_search + + sig = inspect.signature(run_search) + assert "feature_kwargs" in sig.parameters + assert sig.parameters["feature_kwargs"].default is None diff --git a/tests/unit/test_training_split.py b/tests/unit/test_training_split.py index ad400482..36cf0c45 100644 --- a/tests/unit/test_training_split.py +++ b/tests/unit/test_training_split.py @@ -9,6 +9,7 @@ from __future__ import annotations +import numpy as np import pandas as pd import pytest @@ -69,7 +70,7 @@ def test_random_split_is_deterministic_for_same_seed() -> None: split_config=config, ) - assert _matrix_fingerprint(a[1]) == _matrix_fingerprint(b[1]) + assert _matrix_fingerprint(a.X_val_test) == _matrix_fingerprint(b.X_val_test) def test_random_split_differs_for_different_seeds() -> None: @@ -89,7 +90,7 @@ def test_random_split_differs_for_different_seeds() -> None: split_config=SplitConfig(scheme="random", heldout_ratio=0.2, seed=999), ) - assert _matrix_fingerprint(a[1]) != _matrix_fingerprint(b[1]) + assert _matrix_fingerprint(a.X_val_test) != _matrix_fingerprint(b.X_val_test) def test_time_user_split_is_deterministic_for_same_seed() -> None: @@ -110,7 +111,7 @@ def test_time_user_split_is_deterministic_for_same_seed() -> None: split_config=config, ) - assert _matrix_fingerprint(a[1]) == _matrix_fingerprint(b[1]) + assert _matrix_fingerprint(a.X_val_test) == _matrix_fingerprint(b.X_val_test) # --------------------------------------------------------------------------- @@ -124,7 +125,7 @@ def test_time_global_held_out_interactions_are_after_global_cutoff() -> None: heldout_ratio = 0.2 cutoff = df["ts"].quantile(1.0 - heldout_ratio) - _, X_val_test, _ = split_interactions( + res = split_interactions( df, user_column="user_id", item_column="item_id", @@ -136,7 +137,7 @@ def test_time_global_held_out_interactions_are_after_global_cutoff() -> None: ), ) - csr = X_val_test.tocsr() + csr = res.X_val_test.tocsr() item_ids = sorted(df["item_id"].unique()) item_idx_to_name = dict(enumerate(item_ids)) held_out_item_names = {item_idx_to_name[c] for c in csr.indices} @@ -255,7 +256,7 @@ def test_time_global_and_time_user_produce_different_splits() -> None: time_column="ts", ) - _, X_user, _ = split_interactions( + res_user = split_interactions( df, **user_args, split_config=SplitConfig( @@ -264,7 +265,7 @@ def test_time_global_and_time_user_produce_different_splits() -> None: seed=42, ), ) - _, X_global, _ = split_interactions( + res_global = split_interactions( df, **user_args, split_config=SplitConfig( @@ -277,4 +278,111 @@ def test_time_global_and_time_user_produce_different_splits() -> None: # Held-out counts can match coincidentally, but the structural fingerprint # must differ because time_user holds each user's most recent k% while # time_global holds the global tail (some users contribute zero). - assert _matrix_fingerprint(X_user) != _matrix_fingerprint(X_global) + assert _matrix_fingerprint(res_user.X_val_test) != _matrix_fingerprint( + res_global.X_val_test + ) + + +# --------------------------------------------------------------------------- +# SplitResult axis labels — item_ids / row_user_ids must align to the +# returned matrices' columns/rows. Feature-aware training (Task 6) relies on +# these labels to build a correctly ordered feature matrix; irspack accepts a +# misordered feature matrix silently, so these axes must be verified rather +# than assumed. +# --------------------------------------------------------------------------- + + +def _df() -> pd.DataFrame: + rng = np.random.default_rng(0) + rows = [] + # STRING ids on purpose: integer ids would pass by accident because + # hash(int) == int makes list(set(...)) come out sorted. + for u in range(12): + for i in rng.choice(8, size=4, replace=False): + rows.append( + { + "user_id": f"u{u:02d}", + "item_id": f"i_{'abcdefgh'[i]}", + "ts": 1000 + u * 10 + int(i), + } + ) + df = pd.DataFrame(rows) + # Match _synth_df: force plain-object string dtype. Pandas' default + # (Arrow-backed) string dtype produces an ArrowStringArray, which + # irspack's `_split_list` cannot shuffle (not a Sequence subclass). + df["user_id"] = df["user_id"].astype(object) + df["item_id"] = df["item_id"].astype(object) + return df + + +@pytest.mark.parametrize( + "scheme,time_col", + [("random", None), ("time_user", "ts"), ("time_global", "ts")], +) +def test_split_returns_axes(scheme: str, time_col: str | None) -> None: + # heldout_ratio=0.25: with 4 items/user (as in _synth_df's time_user + # test), the default 0.1 rounds down to 0 held-out interactions per user + # for the random/time_user schemes and raises SplitError. + # test_user_ratio=0.5: the repo default of 1.0 sends every user into the + # validation split, leaving train.n_users == 0. That degenerates + # row_user_ids (== train.user_ids + val.user_ids) into literally + # val.user_ids, so a swapped concatenation order would go undetected. + # 0.5 guarantees a genuine non-empty train block for every scheme. + res = split_interactions( + _df(), + user_column="user_id", + item_column="item_id", + time_column=time_col, + split_config=SplitConfig( + scheme=scheme, heldout_ratio=0.25, test_user_ratio=0.5 + ), + ) + assert len(res.item_ids) == res.X_train_full.shape[1] + assert len(res.row_user_ids) == res.X_train_full.shape[0] + assert all(isinstance(i, str) for i in res.item_ids) + assert all(isinstance(u, str) for u in res.row_user_ids) + + +@pytest.mark.parametrize( + "scheme,time_col", + [("random", None), ("time_user", "ts"), ("time_global", "ts")], +) +def test_item_ids_label_the_columns(scheme: str, time_col: str | None) -> None: + """Reconstructing interactions from the returned axes must match the input.""" + df = _df() + # test_user_ratio=0.5: see test_split_returns_axes for why the default + # 1.0 would make this test blind to a swapped row_user_ids concatenation + # order (train.n_users == 0 collapses train+val into just val). + res = split_interactions( + df, + user_column="user_id", + item_column="item_id", + time_column=time_col, + split_config=SplitConfig( + scheme=scheme, heldout_ratio=0.25, test_user_ratio=0.5 + ), + ) + X = res.X_train_full.tocoo() + recon = { + (res.row_user_ids[r], res.item_ids[c]) + for r, c in zip(X.row, X.col, strict=True) + } + truth = set(zip(df["user_id"], df["item_id"], strict=True)) + assert recon <= truth, "reconstructed pairs must all be real interactions" + assert recon, "reconstruction must not be empty" + + +def test_val_offset_still_points_at_validation_users() -> None: + # test_user_ratio=0.5: see test_split_returns_axes for why the default + # 1.0 would make this test blind to a swapped row_user_ids concatenation + # order (train.n_users == 0 collapses train+val into just val). + res = split_interactions( + _df(), + user_column="user_id", + item_column="item_id", + time_column=None, + split_config=SplitConfig( + scheme="random", heldout_ratio=0.25, test_user_ratio=0.5 + ), + ) + assert res.X_val_test.shape[0] == res.X_train_full.shape[0] - res.val_offset diff --git a/tests/unit/test_v1_batch_recommend_related.py b/tests/unit/test_v1_batch_recommend_related.py index f74b7e30..84365f12 100644 --- a/tests/unit/test_v1_batch_recommend_related.py +++ b/tests/unit/test_v1_batch_recommend_related.py @@ -186,6 +186,121 @@ def test_batch_related_aggregate_limit_cap_exceeded() -> None: assert results[-1]["error"]["code"] == "VALIDATION_ERROR" +# --------------------------------------------------------------------------- +# Cold-seed solve cap +# --------------------------------------------------------------------------- +# +# Case C runs one irspack CG solve PER COLD SEED (``_idmap``'s +# ``get_recommendation_for_cold_seeds`` loop), unlike every pre-existing path +# where one solve served the whole element. The schema caps seed_items <= 100 +# and requests <= 256 independently, so their PRODUCT -- 25_600 solves, ~11s +# of single-threaded CPU on a single-process uvicorn -- was reachable in one +# HTTP request. BATCH_AGGREGATE_LIMIT cannot catch this: it caps sum(limit) +# (response volume), a different dimension entirely. + + +def _cold_element(n_seeds: int, limit: int = 1) -> dict: + """One batch element driving exactly *n_seeds* cold-seed solves.""" + seeds = [f"c{i}" for i in range(n_seeds)] + return { + "seed_items": seeds, + "limit": limit, + "item_features": {s: {"genre": "action"} for s in seeds}, + } + + +def _cold_seed_client() -> TestClient: + rec = MagicMock() + # A real encoder state, not a bare MagicMock: the router counts undeclared + # columns via ``_features.state_descriptor``, which reads both keys. + rec.item_feature_state = { + "columns": [{"name": "genre"}], + "n_features": 1, + } + rec.get_recommendation_for_cold_seeds.return_value = ([("i1", 0.9)], []) + return _client(rec, known_items=["s1"]) + + +def test_batch_related_cold_seed_solve_cap_exceeded() -> None: + """6 elements x 100 cold seeds = 600 solves. The 6th element pushes the + running total to 600 > 512 and is rejected; the first five still serve.""" + r = _cold_seed_client().post( + "/v1/recipes/demo:batch-recommend-related", + json={"requests": [_cold_element(100) for _ in range(6)]}, + ) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert [e["status"] for e in results] == ["ok"] * 5 + ["error"] + assert results[-1]["error"]["code"] == "VALIDATION_ERROR" + assert "cold-seed" in results[-1]["error"]["message"] + + +def test_batch_related_cold_seed_solve_cap_boundary() -> None: + """Exactly 512 solves is allowed; one more solve is not.""" + at_cap = _cold_seed_client().post( + "/v1/recipes/demo:batch-recommend-related", + json={"requests": [_cold_element(100) for _ in range(5)] + [_cold_element(12)]}, + ) + assert at_cap.status_code == 200, at_cap.text + assert [e["status"] for e in at_cap.json()["results"]] == ["ok"] * 6 + + over_cap = _cold_seed_client().post( + "/v1/recipes/demo:batch-recommend-related", + json={"requests": [_cold_element(100) for _ in range(5)] + [_cold_element(13)]}, + ) + assert over_cap.status_code == 200, over_cap.text + assert [e["status"] for e in over_cap.json()["results"]] == ["ok"] * 5 + ["error"] + + +def test_batch_related_cold_seed_cap_fires_where_aggregate_limit_cannot() -> None: + """The reported DoS shape: limit=1 x 256 elements keeps sum(limit) at 256, + far under BATCH_AGGREGATE_LIMIT (5000), while still demanding 25_600 + solves. Proves the two caps guard genuinely different dimensions.""" + r = _cold_seed_client().post( + "/v1/recipes/demo:batch-recommend-related", + json={"requests": [_cold_element(100, limit=1) for _ in range(256)]}, + ) + assert r.status_code == 200, r.text + statuses = [e["status"] for e in r.json()["results"]] + assert statuses[:5] == ["ok"] * 5 + assert set(statuses[5:]) == {"error"}, ( + "every element past the solve budget must be rejected" + ) + assert sum(s == "ok" for s in statuses) == 5, ( + "at most 512 solves may run; 5 x 100 is the most that fits" + ) + + +def test_batch_related_cold_seed_cap_counts_only_seeds_with_features() -> None: + """A seed with no item_features entry can never be solved from features, + so it must not consume solve budget. 256 elements x 100 featureless seeds + would otherwise trip the cap and reject a legitimate batch.""" + rec = MagicMock() + rec.get_recommendation_for_new_user.return_value = [("i1", 0.9)] + r = _client(rec, known_items=["s1"]).post( + "/v1/recipes/demo:batch-recommend-related", + json={ + "requests": [ + {"seed_items": ["s1"] + [f"c{i}" for i in range(99)], "limit": 1} + for _ in range(256) + ] + }, + ) + assert r.status_code == 200, r.text + assert set(e["status"] for e in r.json()["results"]) == {"ok"} + + +def test_single_related_maximal_cold_seeds_is_not_capped() -> None: + """The single verb is already bounded at 100 solves by seed_items' + max_length=100 -- well under the 512 batch budget -- so a maximal single + request must still serve. The cap is a batch-only concern by construction.""" + r = _cold_seed_client().post( + "/v1/recipes/demo:recommend-related", + json=_cold_element(100, limit=10), + ) + assert r.status_code == 200, r.text + + def test_batch_recommend_related_sets_model_version_response_header(): rec = MagicMock() rec.get_recommendation_for_new_user.return_value = [("i9", 0.7)] diff --git a/tests/unit/test_v1_error_handling.py b/tests/unit/test_v1_error_handling.py index 37797659..250cb558 100644 --- a/tests/unit/test_v1_error_handling.py +++ b/tests/unit/test_v1_error_handling.py @@ -304,6 +304,8 @@ def test_validation_error_handler_records_metric_when_enabled( monkeypatch.setattr(_metrics, "_V1_BATCH_ELEMENT_ERRORS", None) monkeypatch.setattr(_metrics, "_V1_METADATA_DEGRADED_ITEMS", None) monkeypatch.setattr(_metrics, "_V1_VALIDATION_ERRORS_OUTSIDE_VERB", None) + monkeypatch.setattr(_metrics, "_V1_FEATURE_UNKNOWN_VALUE", None) + monkeypatch.setattr(_metrics, "_V1_COLD_START_REQUESTS", None) client = _client_with(_loaded_entry(name="metric_recipe")) @@ -698,6 +700,8 @@ def test_422_on_path_parameter_validation_no_metric_recorded( monkeypatch.setattr(_metrics, "_V1_BATCH_ELEMENT_ERRORS", None) monkeypatch.setattr(_metrics, "_V1_METADATA_DEGRADED_ITEMS", None) monkeypatch.setattr(_metrics, "_V1_VALIDATION_ERRORS_OUTSIDE_VERB", None) + monkeypatch.setattr(_metrics, "_V1_FEATURE_UNKNOWN_VALUE", None) + monkeypatch.setattr(_metrics, "_V1_COLD_START_REQUESTS", None) client = _client_with(_loaded_entry()) # 'has spaces' contains a space and so does not match the {1,64} pattern. diff --git a/tests/unit/test_v1_metrics_cardinality.py b/tests/unit/test_v1_metrics_cardinality.py index 16e2a67b..b53b7a27 100644 --- a/tests/unit/test_v1_metrics_cardinality.py +++ b/tests/unit/test_v1_metrics_cardinality.py @@ -47,6 +47,8 @@ def _make_client_with_metrics( monkeypatch.setattr(_m, "_V1_BATCH_ELEMENT_ERRORS", None) monkeypatch.setattr(_m, "_V1_METADATA_DEGRADED_ITEMS", None) monkeypatch.setattr(_m, "_V1_VALIDATION_ERRORS_OUTSIDE_VERB", None) + monkeypatch.setattr(_m, "_V1_FEATURE_UNKNOWN_VALUE", None) + monkeypatch.setattr(_m, "_V1_COLD_START_REQUESTS", None) registry = ModelRegistry() return TestClient(build_v1_app(registry)) diff --git a/tests/unit/test_v1_recommend.py b/tests/unit/test_v1_recommend.py index d24c1fae..d16c0bd1 100644 --- a/tests/unit/test_v1_recommend.py +++ b/tests/unit/test_v1_recommend.py @@ -277,7 +277,7 @@ def test_recommend_sets_model_version_response_header(): # --------------------------------------------------------------------------- -# F4: user_known AttributeError path — mirrors _any_seed_known sentinel +# F4: user_known AttributeError path (unexpected recommender layout) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_v1_recommend_related.py b/tests/unit/test_v1_recommend_related.py index 4a0287db..23c2a6dc 100644 --- a/tests/unit/test_v1_recommend_related.py +++ b/tests/unit/test_v1_recommend_related.py @@ -157,7 +157,7 @@ def test_recommend_related_rejects_oversized_seed_item() -> None: # --------------------------------------------------------------------------- -# Finding 10: _any_seed_known AttributeError → INTERNAL_ERROR +# Finding 10: _resolve_recommend_related AttributeError → INTERNAL_ERROR # --------------------------------------------------------------------------- @@ -222,8 +222,9 @@ def test_batch_recommend_related_attribute_error_only_affects_element() -> None: ) # spec=[] means NO attributes allowed → AttributeError # We need ONE entry with two different requests. The handler calls - # _any_seed_known per-element, which calls entry.recommender._mapper.item_id_to_index - # Since entry.recommender is fixed, we can't simulate mixed per-element mapper failure. + # _resolve_recommend_related per-element, which accesses + # entry.recommender._mapper.item_id_to_index. Since entry.recommender is + # fixed, we can't simulate mixed per-element mapper failure. # Instead, test that a wholly broken mapper yields all INTERNAL_ERROR in a batch. broken_entry = ModelEntry( name="broken", diff --git a/tests/unit/test_v1_status_labels.py b/tests/unit/test_v1_status_labels.py index 44edf61e..ae74894b 100644 --- a/tests/unit/test_v1_status_labels.py +++ b/tests/unit/test_v1_status_labels.py @@ -15,6 +15,7 @@ import pytest from fastapi.testclient import TestClient +from recotem._idmap import ColdStartNumericalError from recotem.serving import metrics as _metrics from recotem.serving.registry import ModelEntry, ModelRegistry from tests.conftest import build_v1_app @@ -45,6 +46,9 @@ def _enable_metrics(monkeypatch: pytest.MonkeyPatch) -> None: "recotem_v1_batch_element_errors", "recotem_v1_metadata_degraded_items", "recotem_v1_validation_errors_outside_verb", + "recotem_v1_feature_unknown_value", + "recotem_v1_feature_unknown_column", + "recotem_v1_cold_start_requests", } for collector in list(prometheus_client.REGISTRY._collector_to_names): names = prometheus_client.REGISTRY._collector_to_names.get(collector, set()) @@ -61,6 +65,9 @@ def _enable_metrics(monkeypatch: pytest.MonkeyPatch) -> None: "_V1_BATCH_ELEMENT_ERRORS", "_V1_METADATA_DEGRADED_ITEMS", "_V1_VALIDATION_ERRORS_OUTSIDE_VERB", + "_V1_FEATURE_UNKNOWN_VALUE", + "_V1_FEATURE_UNKNOWN_COLUMN", + "_V1_COLD_START_REQUESTS", ): monkeypatch.setattr(_metrics, attr, None, raising=False) @@ -195,6 +202,101 @@ def test_recommend_related_records_no_candidates() -> None: assert _label_value("recommend-related", "no_candidates") == 1.0 +# --------------------------------------------------------------------------- +# Feature-aware cold-start 400s are client-caused, not server errors +# --------------------------------------------------------------------------- +# +# Both branches are reachable purely from request content: sending +# ``user_features``/``item_features`` to a model that cannot act on them +# (FEATURES_NOT_SUPPORTED), or sending a value that cannot be standardized +# (FEATURE_VALUE_UNUSABLE). ``_request_metrics`` defaults the label to +# "error", which docs/operations.md's "Recommend error rate" row pages +# on-call at 10% — a threshold reserved for genuine 500s. These tests pin +# the two branches to their own labels so a client cannot page on-call. + + +def test_recommend_records_features_not_supported_status() -> None: + entry = _loaded_entry() + entry.recommender.get_recommendation_for_cold_user.side_effect = ValueError( + "this model has no user feature state; it was not trained with features.user" + ) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + r = client.post( + "/v1/recipes/demo:recommend", + json={"user_id": "u-cold", "user_features": {"band": "young"}}, + ) + assert r.status_code == 400 + assert r.json()["code"] == "FEATURES_NOT_SUPPORTED" + assert _label_value("recommend", "features_not_supported") == 1.0 + assert _label_value("recommend", "error") == 0.0 + + +def test_recommend_records_feature_value_unusable_status() -> None: + entry = _loaded_entry() + entry.recommender.get_recommendation_for_cold_user.side_effect = ( + ColdStartNumericalError("singular system") + ) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + r = client.post( + "/v1/recipes/demo:recommend", + json={"user_id": "u-cold", "user_features": {"tight": 1e22}}, + ) + assert r.status_code == 400 + assert r.json()["code"] == "FEATURE_VALUE_UNUSABLE" + assert _label_value("recommend", "feature_value_unusable") == 1.0 + assert _label_value("recommend", "error") == 0.0 + + +def test_recommend_related_records_features_not_supported_status() -> None: + entry = _loaded_entry() + entry.recommender.get_recommendation_for_cold_seeds.side_effect = ValueError( + "this model has no item feature state; it was not trained with features.item" + ) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + r = client.post( + "/v1/recipes/demo:recommend-related", + json={ + "seed_items": ["cold-seed"], + "item_features": {"cold-seed": {"genre": "action"}}, + }, + ) + assert r.status_code == 400 + assert r.json()["code"] == "FEATURES_NOT_SUPPORTED" + assert _label_value("recommend-related", "features_not_supported") == 1.0 + assert _label_value("recommend-related", "error") == 0.0 + + +def test_recommend_related_records_feature_value_unusable_status() -> None: + entry = _loaded_entry() + entry.recommender.get_recommendation_for_cold_seeds.side_effect = ( + ColdStartNumericalError("singular system") + ) + registry = ModelRegistry() + registry.replace("demo", entry) + client = TestClient(build_v1_app(registry)) + + r = client.post( + "/v1/recipes/demo:recommend-related", + json={ + "seed_items": ["cold-seed"], + "item_features": {"cold-seed": {"tight": 1e22}}, + }, + ) + assert r.status_code == 400 + assert r.json()["code"] == "FEATURE_VALUE_UNUSABLE" + assert _label_value("recommend-related", "feature_value_unusable") == 1.0 + assert _label_value("recommend-related", "error") == 0.0 + + def test_validation_error_records_metric_for_matching_v1_path() -> None: registry = ModelRegistry() registry.replace("demo", _loaded_entry()) From c82a977f5360b53b2b9365aba1191753cf6f7af0 Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Sun, 19 Jul 2026 14:16:45 +0900 Subject: [PATCH 2/5] fix(recipe): report missing `source:` alongside other schema errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the `isinstance(raw_source, dict)` guard in `load_recipe` so an absent `source:` key stays absent and pydantic emits an aggregatable "field required" error, instead of being coerced to `None` — which routed the check to a `mode="after"` validator that pydantic v2 skips whenever another field already errored. A recipe missing `source:` plus any other schema error now surfaces both problems in one validation round again (regression from the source-resolution refactor on this branch). Tests: - regression test for the combined missing-`source:` + other-error case - `::`-chain rejection test for `features..source` (parity with the existing top-level source-path validator) - strengthen the feature-aware serve roundtrip with a Case B differential guard (`:recommend-related` known seed + `user_features`), mirroring the Case A/C young-vs-old assertions so a feature-blind regression cannot pass on a bare 200 Also document that feature matrices are shared read-only across parallel Optuna trials (safe in irspack 0.5.0; do not add a per-trial copy). --- src/recotem/recipe/loader.py | 3 +- src/recotem/training/search.py | 7 +++ tests/integration/test_serve_predict_e2e.py | 45 ++++++++++++++ tests/unit/test_recipe_loader.py | 68 +++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/recotem/recipe/loader.py b/src/recotem/recipe/loader.py index a79af34a..d2f743a8 100644 --- a/src/recotem/recipe/loader.py +++ b/src/recotem/recipe/loader.py @@ -654,7 +654,8 @@ def load_recipe( # object.__setattr__ bypass of re-validation). This covers the top-level # source and every features..source subtree identically. raw_source = expanded.get("source") - expanded = {**expanded, "source": _resolve_source_node(raw_source, "source", p)} + if isinstance(raw_source, dict): + expanded = {**expanded, "source": _resolve_source_node(raw_source, "source", p)} raw_features = expanded.get("features") if isinstance(raw_features, dict): diff --git a/src/recotem/training/search.py b/src/recotem/training/search.py index cc58b097..d4109ef9 100644 --- a/src/recotem/training/search.py +++ b/src/recotem/training/search.py @@ -417,6 +417,13 @@ def objective(trial: optuna.Trial) -> float: if (feature_kwargs and is_feature_capable(class_name)) else {} ) + # NB: this reuses the ONE feature-matrix object built per phase, so + # under parallelism>1 (Optuna n_jobs = threads, one process) every + # concurrent trial shares it. Safe because irspack 0.5.0 reads the + # feature matrices read-only -- unlike the interaction matrix X, which + # it defensively copies via .astype(). Do NOT add a per-trial .copy() + # (pure cost, no correctness gain today); if a future irspack ever + # mutates the feature buffers in place, that reliance breaks here. params: dict[str, Any] = rec_cls.default_suggest_parameter(trial, {}) diff --git a/tests/integration/test_serve_predict_e2e.py b/tests/integration/test_serve_predict_e2e.py index 87480d06..8de2973e 100644 --- a/tests/integration/test_serve_predict_e2e.py +++ b/tests/integration/test_serve_predict_e2e.py @@ -1166,6 +1166,51 @@ def _pinned_suggest_float( ) assert cold_action.status_code == 200, cold_action.text + # 4. :recommend-related with a KNOWN seed + user_features -> 200 (case B). + # This endpoint carries no user_id at all, so "cold user" here just means + # the profile prior is not backed by any known user's learned embedding + # -- routes.py's case B branch (_resolve_recommend_related) adds it as a + # prior alongside the ad-hoc seed-history solve. "i0" is a known/ + # in-training item id from _make_clustered_synthetic_csv (u0's cluster). + known_seed_young = client.post( + f"/v1/recipes/{recipe.name}:recommend-related", + json={ + "seed_items": ["i0"], + "limit": 5, + "user_features": {"band": "young"}, + }, + headers=headers, + ) + assert known_seed_young.status_code == 200, known_seed_young.text + + # Paired request, same known seed, opposite user_features value -- same + # differential guard as cases A/C (see docstring): 200 alone proves + # nothing here, since a feature-blind regression (encode_one silently + # dropping its ``values`` argument) would return 200 too. Case B mixes a + # strong known-seed signal with the user_features signal, so this also + # confirms empirically that the seed doesn't drown out the profile prior. + known_seed_old = client.post( + f"/v1/recipes/{recipe.name}:recommend-related", + json={ + "seed_items": ["i0"], + "limit": 5, + "user_features": {"band": "old"}, + }, + headers=headers, + ) + assert known_seed_old.status_code == 200, known_seed_old.text + known_seed_young_ids = [ + item["item_id"] for item in known_seed_young.json()["items"] + ] + known_seed_old_ids = [item["item_id"] for item in known_seed_old.json()["items"]] + assert known_seed_young_ids != known_seed_old_ids, ( + "case-B recommendations (:recommend-related, known seed 'i0') for " + "band='young' vs band='old' must differ -- otherwise the served " + "model is not actually using user_feature_state for the case-B " + "profile-prior solve (get_recommendation_for_new_user with " + "user_features=...)." + ) + # --- mutation guard: prove genuine feature-dependence, not just plumbing --- cold_old = client.post( f"/v1/recipes/{recipe.name}:recommend", diff --git a/tests/unit/test_recipe_loader.py b/tests/unit/test_recipe_loader.py index 24c5fe4b..4742f7d9 100644 --- a/tests/unit/test_recipe_loader.py +++ b/tests/unit/test_recipe_loader.py @@ -583,6 +583,48 @@ def test_recipe_with_no_source_rejected(tmp_path: Path) -> None: load_recipe(p) +def test_missing_and_other_schema_error_both_reported(tmp_path: Path) -> None: + """A recipe with no 'source:' key AND an unrelated schema error must + report BOTH problems in a single validation round, not just one. + + Regression guard: pydantic v2's aggregatable field-required error for a + genuinely-absent key is what surfaces the "source" complaint here. If the + loader instead writes ``source: None`` into the expanded dict before + validation, the complaint moves to the ``_check_source_value`` + model_validator(mode="after") in models.py, which pydantic v2 SKIPS + whenever another field already has a validation error -- silently + dropping the source complaint from this combined-error case. + + Note: this asserts on the formatted "- source:" error line rather than a + bare substring match for "source", because pytest's ``tmp_path`` embeds + the test's own function name in the recipe's file path, and that name + itself contains "source" -- a bare match would pass even when the loader + drops the real complaint. + """ + content = """\ +name: missing_and_bogus +schema: + user_column: user_id + item_column: item_id +training: + algorithms: [TopPop] + n_trials: 1 + bogus_unknown_field: 123 +output: + path: /tmp/out.recotem +""" + p = _write_recipe(tmp_path, content, filename="missing_and_bogus.yaml") + with pytest.raises(RecipeError) as exc_info: + load_recipe(p) + message = str(exc_info.value) + assert "- source:" in message, ( + f"expected the missing 'source' field to be reported; got: {message}" + ) + assert "bogus_unknown_field" in message, ( + f"expected the OTHER schema error to be reported too; got: {message}" + ) + + def test_recipe_name_revalidated_before_filesystem_use(tmp_path: Path) -> None: """validate_for_filesystem raises ValueError for names with slashes.""" from recotem.recipe.models import validate_for_filesystem @@ -2433,6 +2475,32 @@ def test_feature_source_rejects_disallowed_scheme(tmp_path: Path) -> None: load_recipe(p) +def test_feature_source_rejects_chained_scheme(tmp_path: Path) -> None: + """The ``::``-chain rejection must also cover features.item.source. + + Mirrors ``test_input_source_disallowed_scheme_rejected``'s + ``simplecache::https://...`` case for the top-level source: the same + shared path-scheme validator is reused for ``features.*.source.path``, so + a chained fsspec protocol string must be rejected there too. + """ + p = _write_features_recipe( + tmp_path, + "feat_chained_scheme", + """\ +features: + item: + source: + type: csv + path: simplecache::https://example.com/items.csv + id_column: item_id + columns: + - {name: genre, encoding: categorical} +""", + ) + with pytest.raises(RecipeError, match="chain"): + load_recipe(p) + + def test_feature_source_https_requires_sha256(tmp_path: Path) -> None: """An unpinned https:// feature source must be rejected, same as source.""" p = _write_features_recipe( From a71ec9cc380945263168afa43fc42e778b30ccea Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Mon, 20 Jul 2026 14:26:14 +0900 Subject: [PATCH 3/5] fix(training): refuse an all-constant categorical/multi_label feature block The whole-block-dead refusal in `_fetch_side` keyed on `_spec_is_live`, which used width for categorical/multi_label specs. A constant-but-present column (non-empty vocab, width > 0, but the same one-hot on every row) is collinear with the always-1 bias and carries no signal, yet passed the refusal -- signing an artifact advertising `features` for what is really plain iALS. This was inconsistent with both the per-column dead-column warning (varies-based) and the numerical branch (std-based), which already treat a constant column as dead. Replace the width-based check with a shared `spec_is_live(series, spec)` helper that keys on the same "does the encoded block vary across rows" signal, so the refusal now agrees with the warning and the numerical zero-variance refusal. A single dead column among live ones is still allowed (min_frequency pruning is legitimate); only an entirely-dead block is refused. Also correct three documentation inaccuracies found in review: - security.md: `numpy.dtype` is admitted via its explicit FQCN entry, not the `numpy.*` module-prefix list (only `numpy._core.multiarray.scalar` is). - recipe-reference.md: the 8192-char feature-value cap surfaces on the batch verbs as a per-element VALIDATION_ERROR inside the 200 response, not a whole-batch 422. - recipe-reference.md: document that the id-column dtype-pinning caveat also applies to a categorical value column (a float64-inferred column trains its vocabulary as "1990.0" and silently counts a request's "1990" as unknown). Tests: add constant categorical / multi_label refusal cases plus varying and mixed controls; update two pre-existing tests that encoded the old behavior. --- docs/recipe-reference.md | 16 ++- docs/security.md | 5 +- src/recotem/_features.py | 24 +++++ src/recotem/training/features.py | 48 +++++---- tests/unit/test_training_features.py | 144 +++++++++++++++++++++++++-- 5 files changed, 202 insertions(+), 35 deletions(-) diff --git a/docs/recipe-reference.md b/docs/recipe-reference.md index b68549c0..9218bc4c 100644 --- a/docs/recipe-reference.md +++ b/docs/recipe-reference.md @@ -279,13 +279,25 @@ The `multi_label` distinction matters: `genres: "Action|Zzz"` with `Action` known yields `Action=1` and drops `Zzz` — it is not an all-zero segment. "Row missing" and "value unknown" coincide only for `categorical`. +The same `str()`-matching caveat that applies to `id_column` above also +applies to a `categorical` **value** column. If a blank cell makes pandas +infer `float64` for an otherwise-integer column, its vocabulary is trained as +`"1990.0"`, and a serve-time request sending the JSON integer `1990` (matched +as `"1990"`) misses every key and is silently counted as an unknown value. +Unlike the id axis, this is **not** refused at train time — the column varies +across rows, so training stays self-consistent — so pin the type at the source +(`dtype: {year: str}` on `csv`; `CAST(... AS STRING)` on `bigquery` / `sql`; +fix the schema on `parquet`) exactly as for the id column. + At serve time, each cold-start feature value supplied to `:recommend` / `:recommend-related` (`user_features`, and each `item_features` seed mapping) is length-capped: a string value longer than **8192 characters** is rejected with `422` (the error names the offending column, never the value). This bounds the `multi_label` tokenization work per request — 8192 characters is generous for a -real token list while blocking megabyte-scale amplification — and applies to -the batch verbs too. Non-string scalar values are unaffected. +real token list while blocking megabyte-scale amplification. The same cap +applies on the batch verbs, but a violation there surfaces as a per-element +`VALIDATION_ERROR` inside the `200` batch response rather than failing the +whole batch with `422`. Non-string scalar values are unaffected. If a `numerical` column is constant — or merely **near**-constant — in the training data, its segment is emitted as zeros and a warning is logged diff --git a/docs/security.md b/docs/security.md index 10978ac8..91815a0d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -314,8 +314,9 @@ limit is worth stating precisely, because it is what makes those coercions load-bearing. A stray `pandas.Index` really would be refused at load time (`pandas.core.indexes.base._new_Index` is not allow-listed — verified). A `numpy.str_` would **not**: it pickles via `numpy._core.multiarray.scalar` -plus `numpy.dtype`, both reachable through the narrow `numpy.*` -module-prefix allow-list, so it loads and keeps its type (verified). Nothing +plus `numpy.dtype`, both allow-listed (the former via the `numpy._core.*` +module-prefix list, the latter via its explicit FQCN entry), so it loads and +keeps its type (verified). Nothing downstream catches it either — `numpy.str_` subclasses `str` and hashes and compares equal to it, so every vocabulary lookup keeps working and the leak stays invisible at runtime. The `str()` coercions in `build_encoder_state` diff --git a/src/recotem/_features.py b/src/recotem/_features.py index 37cec6a9..918b344d 100644 --- a/src/recotem/_features.py +++ b/src/recotem/_features.py @@ -352,6 +352,30 @@ def _column_block_varies( return False +def spec_is_live(series: pd.Series, spec: dict) -> bool: + """True if an encoded column *spec* can contribute signal beyond the bias. + + Unifies the two liveness signals ``build_encoder_state`` already computes: + a ``numerical`` spec is live iff its recorded ``std`` is non-zero; a + ``categorical`` / ``multi_label`` spec is live iff its encoded block + varies across rows (``_column_block_varies``) -- i.e. it is not constant, + and therefore not collinear with the always-1 bias. A width-only test + cannot see a constant-but-present categorical column (non-empty vocab, + ``width > 0``, yet every row emits the same one-hot); this can, because it + re-reads *series*. The training-side whole-block-dead refusal keys on this + so it agrees with the per-column dead-column warning and with the + numerical branch's zero-variance refusal. + """ + if spec["encoding"] == "numerical": + return spec["std"] != 0.0 + return _column_block_varies( + series, + spec["vocab"], + encoding=spec["encoding"], + delimiter=spec.get("delimiter", ""), + ) + + def _warn_if_column_dead( col: FeatureColumn, tokens: Sequence[str], diff --git a/src/recotem/training/features.py b/src/recotem/training/features.py index 694033c1..c5a78241 100644 --- a/src/recotem/training/features.py +++ b/src/recotem/training/features.py @@ -81,7 +81,12 @@ import scipy.sparse as sps import structlog -from recotem._features import FeatureEncodeError, build_encoder_state, encode +from recotem._features import ( + FeatureEncodeError, + build_encoder_state, + encode, + spec_is_live, +) from recotem.datasource.base import FetchContext from recotem.datasource.registry import get_source_class from recotem.recipe.models import FeaturesConfig, FeatureSideConfig @@ -122,21 +127,6 @@ def enabled(self) -> bool: return self.item_state is not None or self.user_state is not None -def _spec_is_live(spec: dict) -> bool: - """True if an encoder-state column spec can emit a non-bias feature. - - A ``numerical`` spec always reserves ``width == 1`` even when it is dead - (std floored to 0.0, emitting nothing at encode time), so its width is not - a reliable liveness signal -- its ``std`` is. A ``categorical`` / - ``multi_label`` spec, by contrast, prunes to ``width == 0`` when dead, so - its width is exactly right. The whole-block-dead guard in ``_fetch_side`` - keys on this rather than on ``n_features``. - """ - if spec["encoding"] == "numerical": - return spec["std"] != 0.0 - return spec["width"] > 0 - - def _resolve_source(source_cfg: Any, *, which: str) -> tuple[type, Any]: """Return ``(source_cls, config)`` for a ``FeatureSideConfig.source`` value. @@ -258,15 +248,23 @@ def _fetch_side( # shared with serving and must not raise a training error); this # training-side check is where the whole-block refusal belongs. # - # Liveness is per encoding, NOT ``n_features``: a categorical/multi_label - # spec that pruned to width 0 contributes nothing, but a NUMERICAL spec - # always reserves width 1 even when its std was floored to 0.0 (dead) and - # it emits nothing at encode time. Keying on ``n_features == 1`` therefore - # missed an all-dead-NUMERICAL block (n_features stays 2). A single dead - # column among several live ones is NOT refused -- pruning one column via - # ``min_frequency`` is a legitimate operator choice -- so this fires only - # when EVERY spec is dead. - if not any(_spec_is_live(s) for s in state["columns"]): + # Liveness is VARIES-based, NOT ``n_features`` and NOT width. ``spec_is_live`` + # keys on exactly what the per-column dead-column warning and the numerical + # branch already compute: a numerical spec is live iff its std is non-zero, + # and a categorical/multi_label spec is live iff its encoded block VARIES + # across rows. Keying on ``n_features == 1`` missed an all-dead-NUMERICAL + # block, because a numerical spec reserves width 1 even when its std was + # floored to 0.0 and it emits nothing at encode time (n_features stays 2). + # A width-only test would miss the symmetric categorical case: a + # constant-but-present column (e.g. every item shares one genre) keeps a + # non-empty vocab and ``width > 0``, yet every row emits the SAME one-hot, + # collinear with the bias -- signal-free, identical to plain iALS. Reading + # ``frame[s["name"]]`` per column (the same series ``build_encoder_state`` + # saw) catches it, so the refusal now agrees with the per-column warning and + # the numerical branch. A single dead column among several live ones is NOT + # refused -- pruning one column via ``min_frequency`` is a legitimate + # operator choice -- so this fires only when EVERY spec is dead. + if not any(spec_is_live(frame[s["name"]], s) for s in state["columns"]): raise TrainingError( f"features.{which}: every declared feature column encodes to " f"nothing, so the whole block collapses to the bias column alone " diff --git a/tests/unit/test_training_features.py b/tests/unit/test_training_features.py index e18e5a7a..3830aab7 100644 --- a/tests/unit/test_training_features.py +++ b/tests/unit/test_training_features.py @@ -115,10 +115,14 @@ def test_null_and_empty_ids_are_dropped(tmp_path: Path) -> None: than mistaken for a missing id. """ p = tmp_path / "items_with_nulls.csv" + # The two SURVIVING rows (i_a, i_c) carry DISTINCT genres so the block stays + # live -- were they identical, the (correct) whole-block-dead guard would + # refuse a bias-only block, which is a different test's concern + # (test_constant_categorical_whole_block_raises), not this one's. pd.DataFrame( { "item_id": ["i_a", None, "i_c", ""], - "genre": ["action", "drama", "action", "comedy"], + "genre": ["action", "drama", "sci-fi", "comedy"], "year": [2000, 2010, 2020, 2030], } ).to_csv(p, index=False) @@ -567,33 +571,161 @@ def test_one_dead_column_among_several_does_not_raise(tmp_path: Path) -> None: assert t.item_state["n_features"] == 4 +# --------------------------------------------------------------------------- +# Constant-categorical refusal: the whole-block-dead guard was WIDTH-based, so +# a constant-but-present categorical/multi_label column (non-empty vocab, +# `width > 0`, yet every row emits the SAME one-hot) counted as live and slipped +# past the refusal -- even though it is collinear with the bias and byte- +# identical to plain iALS, exactly what the per-column warning already flags and +# what the numerical branch's zero-variance refusal already refuses. The guard +# now keys on whether the encoded block VARIES across rows, so it agrees with +# both. A varying column, and a mixed block with one live column, must still +# load: the refusal fires only when EVERY spec is dead. +# --------------------------------------------------------------------------- + + +def test_constant_categorical_whole_block_raises(tmp_path: Path) -> None: + """A block whose ONLY column is a CONSTANT categorical (every item the same + value; full id overlap so this is not the coverage check) keeps a non-empty + vocab and `width == 1`, so the old width-based guard passed it -- yet every + row emits the same one-hot, collinear with the bias, so the model is plain + iALS. It must be refused like the all-dead-numerical block.""" + p = tmp_path / "items_constant_cat.csv" + pd.DataFrame( + {"item_id": ["i_a", "i_b", "i_c"], "genre": ["book", "book", "book"]} + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + with pytest.raises(TrainingError) as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + assert exc_info.value.code == "feature_table_error" # -> exit 4 + msg = str(exc_info.value) + assert "item" in msg + assert "bias" in msg + + +def test_constant_multi_label_whole_block_raises(tmp_path: Path) -> None: + """The multi_label analogue: every row carries the same single token, so its + multi-hot block is identical across rows (constant), collinear with the + bias. Non-empty vocab and `width == 1` again fooled the width-based guard; + the varies-based guard refuses it.""" + p = tmp_path / "items_constant_ml.csv" + pd.DataFrame( + {"item_id": ["i_a", "i_b", "i_c"], "tags": ["rock", "rock", "rock"]} + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="tags", encoding="multi_label")], + ) + ) + with pytest.raises(TrainingError) as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + assert exc_info.value.code == "feature_table_error" # -> exit 4 + msg = str(exc_info.value) + assert "item" in msg + assert "bias" in msg + + +def test_varying_categorical_whole_block_loads(tmp_path: Path) -> None: + """Vacuity control: a VARYING categorical column carries real signal and + must load, so the refusal cannot be trivially satisfied (it must not fire + on every categorical-only block).""" + p = tmp_path / "items_varying_cat.csv" + pd.DataFrame( + {"item_id": ["i_a", "i_b", "i_c"], "genre": ["action", "drama", "comedy"]} + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[FeatureColumn(name="genre", encoding="categorical")], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + # 3 one-hots + bias == 4; the block is live. + assert t.item_state["n_features"] == 4 + + +def test_mixed_constant_and_varying_categorical_does_not_raise( + tmp_path: Path, +) -> None: + """A constant categorical column ALONGSIDE a varying one: the varying column + is a live spec, so the whole block is not dead and must load -- the refusal + fires only when EVERY spec is dead. This is the constant-column route to + "one dead column among several", distinct from the min_frequency-pruning + route in test_one_dead_column_among_several_does_not_raise (the dead column + here keeps a non-empty vocab).""" + p = tmp_path / "items_mixed_constant.csv" + pd.DataFrame( + { + "item_id": ["i_a", "i_b", "i_c"], + "genre": ["book", "book", "book"], # constant -> dead + "brand": ["x", "y", "z"], # varying -> live + } + ).to_csv(p, index=False) + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "csv", "path": str(p)}, + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="brand", encoding="categorical"), + ], + ) + ) + t = load_feature_tables(cfg, recipe_name="r", run_id="run") + # constant genre one-hot (1) + varying brand's 3 one-hots + bias == 5. + assert t.item_state["n_features"] == 5 + + def test_constant_feature_column_warns_at_training_time(tmp_path: Path) -> None: """Gap 3 at the TRAINING level: a constant categorical column (every item the same genre) is dead (collinear with bias) and must warn when the feature table is loaded, not only when `build_encoder_state` is called by - hand. n_features stays 2 (one one-hot + bias), so it warns but does not - raise.""" + hand. Here the constant `genre` sits ALONGSIDE a varying `brand`, so the + block as a whole stays live (the varying column keeps it off the + whole-block-dead refusal) and the load succeeds -- letting us observe the + per-column warning in isolation from the whole-block guard. A block whose + ONLY column is constant is refused instead; see + test_constant_categorical_whole_block_raises.""" import structlog.testing p = tmp_path / "items_constant.csv" pd.DataFrame( - {"item_id": ["i_a", "i_b", "i_c"], "genre": ["action", "action", "action"]} + { + "item_id": ["i_a", "i_b", "i_c"], + "genre": ["action", "action", "action"], + "brand": ["x", "y", "z"], + } ).to_csv(p, index=False) cfg = FeaturesConfig( item=FeatureSideConfig( source={"type": "csv", "path": str(p)}, id_column="item_id", - columns=[FeatureColumn(name="genre", encoding="categorical")], + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="brand", encoding="categorical"), + ], ) ) with structlog.testing.capture_logs() as cap: t = load_feature_tables(cfg, recipe_name="r", run_id="run") - assert t.item_state["n_features"] == 2 # one one-hot + bias, not a raise + # constant genre one-hot (1) + varying brand's 3 one-hots + bias == 5. + assert t.item_state["n_features"] == 5 # not a raise: brand keeps it live events = [e for e in cap if e.get("event") == "feature_empty_vocabulary_column"] assert events, ( f"a constant column must warn as dead at load time; " f"got {[e.get('event') for e in cap]}" ) + # The warning must name the constant column, not the varying one. + assert any(e.get("column") == "genre" for e in events) # PII: the genre value must not be logged. for e in cap: for value in e.values(): From 5a50c189e356e83abc4f30f55e6793a31db9b3d3 Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Mon, 20 Jul 2026 16:28:17 +0900 Subject: [PATCH 4/5] fix(features): harden numerical encoding and unify cold-start empty results Follow-up fixes from a multi-agent review of the feature-aware iALS branch. - A non-finite numerical value no longer silently drops a usable column. pd.to_numeric maps an overflow token like "1e400" to +inf, and pandas mean/std do not skip +-inf, so one such cell made the column's std non-finite and routed the whole column to the zero-variance path -- silently dropping a column that still held usable finite values while the artifact kept advertising features, with a warning that misattributed the cause as "divide by zero". build_encoder_state now computes mean/std over the finite values only (consistent with encode(), which already routes a per-row non-finite value to unknown); a column with no finite value at all is still dropped, now with a distinct, accurate warning. - :recommend-related cold-start paths now return 404 NO_CANDIDATES consistently. The all-seeds-known path already raised NO_CANDIDATES on an empty ranker result, but the user_features and item_features branches returned 200 with an empty items list for the identical condition; both now raise the same error, single and batch alike. - Correct the _features.py design comment: the float() coercions on the numerical mean/std are load-bearing plainness guards alongside the vocabulary str() coercions (a np.float64 would otherwise leak into the persisted encoder state), so neither may be refactored away. - Document the new [source] / [features..source] tags in recotem validate output in the changelog. Full suite: 2154 passed, 1 skipped; ruff and ruff format clean. --- CHANGELOG.md | 29 ++++++ src/recotem/_features.py | 73 +++++++++++--- src/recotem/serving/routes.py | 17 ++++ tests/unit/test_features.py | 123 ++++++++++++++++++++++++ tests/unit/test_v1_recommend_related.py | 47 +++++++++ 5 files changed, 276 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353f7bb7..8cfcb8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 sklearn axis at all. If you need TruncatedSVD artifacts to be reproducible bit-exact, pin sklearn exactly or build train and serve from the same lock file. +- **`recotem validate` labels each probed data source.** Because a recipe may + now declare feature-side sources (`features.item.source` / + `features.user.source`) alongside the top-level `source:`, the probe output + tags which one it is (`DataSource: probe OK (csv) [source]`, `DataSource probe + failed [features.item.source]: ...`) and the missing-discriminator message + reads `source is missing the 'type' discriminator.` rather than `Recipe + source is missing the 'type' discriminator.`. Exit codes are unchanged; + tooling that greps the exact `validate` output lines should update. ### Fixed @@ -171,6 +179,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Recipe load rejects a `features..id_column` that also names a feature column.** The collision is guaranteed to fail at train time (the id column is consumed as the index); it is now caught at recipe load with a clear message. +- **Feature-aware iALS: a non-finite value no longer silently kills an + otherwise-usable `numerical` column.** `pd.to_numeric` maps an overflow token + like `1e400` to `+inf`, and pandas `mean` / `std` do not skip `±inf`, so a + single such cell made the column's `std` non-finite and routed the whole + column to the zero-variance path — silently dropping a column that still held + usable finite values (like `[1, 2, 3]`) while the artifact continued to + advertise `features`, and emitting a `feature_zero_variance_column` warning + that misattributed the cause as "divide by zero." `build_encoder_state` now + computes mean/std over the finite values only, so a stray overflow cell + degrades to `unknown` at encode time — exactly as it already did per request — + instead of killing the column at fit time. A column that parses to no finite + value at all is still dropped, now with a distinct, accurate warning detail. + This changes training-time encoding for any feature table with such a column; + retrain to pick it up. +- **`:recommend-related` cold-start paths now return `404 NO_CANDIDATES` + consistently.** The pre-existing all-seeds-known path raised `NO_CANDIDATES` + when the ranker produced no survivors, but the two cold-start branches (the + `user_features` profile prior, and `item_features` for a seed absent from + training) returned `200` with an empty `items` list for the identical + condition. Both branches now raise the same `NO_CANDIDATES`, so every path of + the verb — single and batch — reports an empty result the same way. ### Migrating to irspack 0.5.0 diff --git a/src/recotem/_features.py b/src/recotem/_features.py index 918b344d..79b4d2bc 100644 --- a/src/recotem/_features.py +++ b/src/recotem/_features.py @@ -32,13 +32,22 @@ and compares equal to ``str``, so every vocabulary lookup keeps working and the leak stays invisible at runtime. -So the ``str()`` coercions in ``build_encoder_state`` are load-bearing on +So the plainness coercions in ``build_encoder_state`` are load-bearing on their own for the numpy scalar types, not a belt-and-braces gesture on top -of an allow-list that would fail closed anyway. They are total (every -vocabulary key is constructed through ``str()``), and -``tests/unit/test_features.py::test_vocabulary_keys_are_exactly_str_not_ -numpy_str`` enforces the result by asserting the EXACT key type -- an -``isinstance`` check cannot do it, because ``numpy.str_`` subclasses ``str``. +of an allow-list that would fail closed anyway. There are two, one per scalar +kind, and neither is cosmetic: + +- the ``str()`` coercions on vocabulary keys (categorical / multi_label): + total -- every key is constructed through ``str()`` -- and enforced by + ``tests/unit/test_features.py::test_vocabulary_keys_are_exactly_str_not_ + numpy_str``, which asserts the EXACT key type (an ``isinstance`` check + cannot, because ``numpy.str_`` subclasses ``str``). +- the ``float()`` coercions on the numerical branch's ``mean`` / ``std``: + ``numeric.mean()`` / ``numeric.std()`` return ``np.float64`` scalars, which + leak into the state and unpickle via the same allow-listed + ``numpy._core.multiarray.scalar`` if the wrapping ``float(...)`` is dropped. + Do NOT refactor ``float(numeric.mean())`` back to ``numeric.mean()``: it + silently reintroduces a numpy scalar the allow-list will happily load. Why ``encode`` demands ``index_order`` ---------------------------------------- @@ -525,9 +534,31 @@ def build_encoder_state( f"cannot be standardized; declare it categorical or " f"drop it" ) - has_values = bool(numeric.notna().any()) - mean = float(numeric.mean()) if has_values else 0.0 - std = float(numeric.std(ddof=0)) if has_values else 0.0 + # Compute mean/std over the FINITE values only. A single + # non-finite cell must not poison the whole column's + # statistics: `pd.to_numeric(..., errors="coerce")` maps an + # overflow token like "1e400" to +inf (NOT NaN -- + # `float("1e400") == inf`), and pandas `mean()`/`std()` do NOT + # skip +-inf, so without this restriction one such cell would + # make mean=+inf (reset below) and std=nan (floored below), + # silently marking a column with usable finite values like + # [1,2,3] dead. `encode()` already routes a per-row non-finite + # standardized value to `unknown` (see the non-finite guard in + # `_row_values`), so restricting the fit to the finite values + # keeps the two paths consistent -- the "1e400" cell degrades + # to `unknown` at encode time, not the whole column at fit time. + # Excluding non-finite values also subsumes the NaN exclusion + # `notna()` gave before. + finite = numeric[np.isfinite(numeric)] + has_finite = bool(finite.size) + # The `float()` coercions are load-bearing plainness guards, + # not cosmetic -- see the module docstring. `mean()` / `std()` + # return `np.float64` scalars that would otherwise leak into the + # persisted state and unpickle via the allow-listed + # `numpy._core.multiarray.scalar`; they are the numerical-branch + # counterpart of the vocabulary `str()` coercions. + mean = float(finite.mean()) if has_finite else 0.0 + std = float(finite.std(ddof=0)) if has_finite else 0.0 except OverflowError as exc: # `pd.to_numeric(..., errors="coerce")` does NOT suppress # OverflowError for an object-dtype Python int above float64's @@ -541,11 +572,27 @@ def build_encoder_state( ) from exc if not np.isfinite(mean): mean = 0.0 - # See _NUMERICAL_STD_RELATIVE_FLOOR's module-level comment: a - # std that is merely tiny relative to the column's own scale is - # treated the same as an exact 0.0, not just a literal 0.0. scale = max(abs(mean), 1.0) - if not np.isfinite(std) or std <= _NUMERICAL_STD_RELATIVE_FLOOR * scale: + if not has_finite: + # Distinct cause from zero variance: the column parsed to NO + # finite value at all -- every cell was NaN/unparseable or a + # non-finite overflow like "1e400". Name that cause rather than + # blaming "divide by zero" (finding M2): the two are different, + # and a stray non-finite cell must not misattribute the death of + # a column that in fact has no usable values. + _logger.warning( + "feature_zero_variance_column", + column=col.name, + detail="no finite parseable values to standardize; emitting zeros", + ) + std = 0.0 + # See _NUMERICAL_STD_RELATIVE_FLOOR's module-level comment: a std + # that is merely tiny relative to the column's own scale is treated + # the same as an exact 0.0, not just a literal 0.0. This is genuine + # (near-)zero variance AMONG the finite values -- a distinct cause + # from the no-finite-values case above, so it keeps the original + # divide-by-zero wording. + elif not np.isfinite(std) or std <= _NUMERICAL_STD_RELATIVE_FLOOR * scale: _logger.warning( "feature_zero_variance_column", column=col.name, diff --git a/src/recotem/serving/routes.py b/src/recotem/serving/routes.py index 793ec0ff..4f828e0c 100644 --- a/src/recotem/serving/routes.py +++ b/src/recotem/serving/routes.py @@ -371,6 +371,17 @@ def _resolve_recommend_related( if str(seed) in supplied ): _metrics.inc_feature_unknown_column(name, "item") + # Empty ranker result is NO_CANDIDATES, same as the plain path below. + # Placed after the metric increments (not before): a cold-start + # request that produced no survivors was still a cold-start attempt + # and may still have carried an unknown feature value/column -- those + # counters describe the request's inputs, which are true regardless of + # whether the ranker returned anything, so suppressing them on an empty + # result would under-count real cold-start traffic and diverge from the + # non-empty case. The plain path has no such counters to preserve, so + # "mirror the plain path" means only "raise _NoCandidates on empty". + if not raw_results: + raise _NoCandidates() return raw_results if not any(str(s) in id_map for s in body.seed_items): @@ -398,6 +409,12 @@ def _resolve_recommend_related( entry.recommender.user_feature_state, body.user_features ): _metrics.inc_feature_unknown_column(name, "user") + # Empty ranker result is NO_CANDIDATES, same as the plain path below. + # Placed after the metric increments for the same reason as case C: + # the cold-start / unknown-feature counters describe request inputs + # that hold whether or not the ranker returned survivors. + if not raw_results: + raise _NoCandidates() return raw_results # All seeds known, no user_features: byte-for-byte the pre-existing diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py index 4c9dbbb0..b6cb352c 100644 --- a/tests/unit/test_features.py +++ b/tests/unit/test_features.py @@ -340,6 +340,129 @@ def test_small_but_real_variance_numerical_std_is_not_floored() -> None: assert state["columns"][0]["std"] == pytest.approx(raw_std) +# --------------------------------------------------------------------------- +# Review finding M2: a single non-finite/overflow cell poisoned the WHOLE +# column's statistics. `pd.to_numeric(..., errors="coerce")` maps a token like +# "1e400" to +inf (NOT NaN, because `float("1e400") == inf`), and pandas +# `mean()`/`std()` do NOT skip +-inf -- so one such cell made mean=inf (reset +# to 0.0) and std=nan (routed into the zero-variance branch), silently marking +# a column with usable finite values like [1,2,3] DEAD (std=0.0, emits zeros) +# and MISATTRIBUTING the cause as "standardization would divide by zero". The +# fit must instead compute mean/std over the FINITE values only -- consistent +# with `encode()`, which already routes a per-row non-finite standardized +# value to `unknown` (see the `not isfinite`/`abs > FLOAT32_MAX` guard in +# `_row_values`). Only a column with NO finite values, or genuine zero variance +# AMONG the finite values, is dead, and the two causes must warn distinctly. +# --------------------------------------------------------------------------- + + +def test_numerical_column_with_one_overflow_cell_stays_live() -> None: + """A numerical column with usable finite values plus one overflow cell + ("1e400" -> +inf) must stay LIVE, with mean/std computed over the finite + values only. + + Pre-fix, `pd.to_numeric` mapped "1e400" to +inf, which pandas did NOT skip + in `mean()`/`std()`: mean came back +inf (reset to 0.0) and std came back + nan (routed into the zero-variance branch), so `std` was floored to 0.0 and + the whole column emitted zeros -- as though [1,2,3] carried no signal -- + while the warning blamed "divide by zero". This test pins the fixed + behavior: the finite rows [1,2,3] set mean=2.0/std=sqrt(2/3), the column is + live, and no zero-variance warning fires. + """ + import structlog.testing + + d = pd.DataFrame( + {"item_id": ["a", "b", "c", "d"], "price": ["1.0", "2.0", "3.0", "1e400"]} + ).set_index("item_id") + with structlog.testing.capture_logs() as cap: + state = build_encoder_state( + d, [FeatureColumn(name="price", encoding="numerical")] + ) + spec = state["columns"][0] + + # Statistics are computed over the finite values [1,2,3] alone. + assert spec["mean"] == pytest.approx(2.0) + assert spec["std"] == pytest.approx(math.sqrt(2.0 / 3.0)) + assert spec["std"] != 0.0, ( + "one overflow cell must not mark a column with finite values dead; " + "pre-fix std was floored to 0.0" + ) + + # The column is live -> NO zero-variance warning. + assert not [e for e in cap if e.get("event") == "feature_zero_variance_column"], ( + "a live column must not warn as zero-variance" + ) + + # End-to-end consistency with encode(): the "1e400" row degrades like a + # missing value (contributes 0) while a finite row contributes its + # standardized value. + m = encode(state, d, index_order=["a", "d"]).toarray() + assert m[0, spec["offset"]] == pytest.approx((1.0 - 2.0) / math.sqrt(2.0 / 3.0)) + assert m[1, spec["offset"]] == 0.0, "the overflow row must encode to zero" + + # And encode_one routes the same overflow token to `unknown` -- an unknown + # value must not also be invisible. + _, unknown = encode_one(state, {"price": "1e400"}) + assert unknown == ["price"] + + +def test_numerical_column_all_nonfinite_is_dead_with_accurate_message() -> None: + """A column with NO finite parseable values is dead, and warns with a + message that names the real cause -- not "divide by zero". + + Every cell is either an overflow token ("1e400" -> +inf) or unparseable + ("nope" -> NaN), so there is nothing finite to fit. Pre-fix this still + reported the zero-variance "standardization would divide by zero" detail, + misattributing the cause; the fix emits a distinct, accurate message. + """ + import structlog.testing + + d = pd.DataFrame({"item_id": ["a", "b"], "price": ["1e400", "nope"]}).set_index( + "item_id" + ) + with structlog.testing.capture_logs() as cap: + state = build_encoder_state( + d, [FeatureColumn(name="price", encoding="numerical")] + ) + spec = state["columns"][0] + + assert spec["std"] == 0.0, "no finite values -> dead column (std == 0.0)" + + events = [e for e in cap if e.get("event") == "feature_zero_variance_column"] + assert events, "a no-finite-values column must still warn" + detail = events[0]["detail"] + assert "divide by zero" not in detail, ( + f"an all-non-finite column must not blame zero variance; got {detail!r}" + ) + assert "finite" in detail, ( + f"the warning must name the real cause (no finite values); got {detail!r}" + ) + + +def test_constant_finite_numerical_column_uses_zero_variance_message() -> None: + """A genuinely constant finite column [5,5,5] still hits the zero-variance + path and keeps the ORIGINAL "divide by zero" wording -- the fix must not + relabel real zero-variance-among-finite-values as the no-finite case. + """ + import structlog.testing + + d = pd.DataFrame({"item_id": ["a", "b", "c"], "price": [5.0, 5.0, 5.0]}).set_index( + "item_id" + ) + with structlog.testing.capture_logs() as cap: + state = build_encoder_state( + d, [FeatureColumn(name="price", encoding="numerical")] + ) + assert state["columns"][0]["std"] == 0.0 + + events = [e for e in cap if e.get("event") == "feature_zero_variance_column"] + assert events, "a constant finite column must warn as zero-variance" + assert "divide by zero" in events[0]["detail"], ( + "genuine zero variance among finite values must keep the original " + "divide-by-zero wording" + ) + + def test_numerical_missing_becomes_mean( df: pd.DataFrame, columns: list[FeatureColumn] ) -> None: diff --git a/tests/unit/test_v1_recommend_related.py b/tests/unit/test_v1_recommend_related.py index 23c2a6dc..1bffe16c 100644 --- a/tests/unit/test_v1_recommend_related.py +++ b/tests/unit/test_v1_recommend_related.py @@ -89,6 +89,53 @@ def test_related_404_when_seeds_known_but_ranker_empty(): assert body["code"] == "NO_CANDIDATES" +def test_related_404_when_case_b_ranker_empty(): + """Case B (known seed + user_features) with an empty ranker result must + return 404 NO_CANDIDATES -- the same contract the plain "all seeds known, + no user_features" path already enforces -- not 200 with an empty items + list. Regression guard for the L1 inconsistency fix. + + ``get_recommendation_for_new_user`` here is the case-B overload that + returns the ``(raw_results, unknown_columns)`` tuple; stub it to + ``([], [])`` so the ranker yields no survivors. + """ + rec = MagicMock() + rec.get_recommendation_for_new_user.return_value = ([], []) + rec.user_feature_state = {"n_features": 1, "columns": [{"name": "band"}]} + r = _client_with_recommender(rec, known_items=["i1"]).post( + "/v1/recipes/demo:recommend-related", + json={"seed_items": ["i1"], "limit": 5, "user_features": {"band": "young"}}, + ) + assert r.status_code == 404, r.text + body = r.json() + assert body["code"] == "NO_CANDIDATES" + + +def test_related_404_when_case_c_cold_seed_ranker_empty(): + """Case C (cold seed + item_features) with an empty ranker result must + return 404 NO_CANDIDATES -- matching the plain path -- not 200 with an + empty items list. Regression guard for the L1 inconsistency fix. + + ``brand_new`` is absent from the id-map (cold) and carries item_features, + so the request takes the ``get_recommendation_for_cold_seeds`` branch; + stub it to ``([], [])`` so the ranker yields no survivors. + """ + rec = MagicMock() + rec.get_recommendation_for_cold_seeds.return_value = ([], []) + rec.item_feature_state = {"n_features": 1, "columns": [{"name": "genre"}]} + r = _client_with_recommender(rec, known_items=[]).post( + "/v1/recipes/demo:recommend-related", + json={ + "seed_items": ["brand_new"], + "limit": 5, + "item_features": {"brand_new": {"genre": "action"}}, + }, + ) + assert r.status_code == 404, r.text + body = r.json() + assert body["code"] == "NO_CANDIDATES" + + def test_related_404_when_recipe_missing_from_registry(): rec = MagicMock() r = _client_with_recommender(rec).post( From 0a8313a8b37d9737cd298bddbd7b41412548e1b8 Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Wed, 22 Jul 2026 07:35:39 +0900 Subject: [PATCH 5/5] fix(serving): cap request body size and feature-dict key lengths Address second-round review findings: - Add BodySizeLimitMiddleware bounding each HTTP request body at RECOTEM_MAX_BODY_BYTES (default 128 MiB, clamped [1 MiB, 2 GiB]). Over-cap requests get 413 PAYLOAD_TOO_LARGE before the body is buffered/parsed, enforced on both a declared Content-Length and chunked bodies with no length header. - Cap cold-start feature-dict key lengths at 1-256 chars: user_features column-name keys, item_features outer seed-id keys (now typed _ItemStr), and nested per-seed keys. Previously only string values were length-capped and max_length bounded the key count only, leaving key length unbounded. Over-length keys 422 reporting only the length, never the key text. - Docs: README gains the feature-aware iALS / cold-start bullets; api-reference documents the silently-ignored unknown feature key, the 413 code, and all length/size bounds; recipe-reference extends the dtype-trap note to value columns with source-side type guidance; operations/security/CLAUDE.md gain the env-var and threat-model entries. - Tests: fuzz corpus now exercises the features: block (seed YAML, column-field mutations, arbitrary-value injection at the features key); a 0-row feature table is pinned to feature_table_error; new body-cap (Content-Length, chunked, boundary) and key-length tests. --- CHANGELOG.md | 18 ++++ CLAUDE.md | 1 + README.md | 2 + docs/api-reference.md | 54 +++++++++++- docs/operations.md | 2 + docs/recipe-reference.md | 29 ++++-- docs/security.md | 31 +++++++ src/recotem/config.py | 26 ++++++ src/recotem/serving/app.py | 122 +++++++++++++++++++++++++- src/recotem/serving/schemas.py | 34 +++++-- tests/fuzz/test_recipe_loader.py | 111 +++++++++++++++++++++++ tests/unit/test_config_body.py | 35 ++++++++ tests/unit/test_serving_body_limit.py | 100 +++++++++++++++++++++ tests/unit/test_serving_schemas.py | 81 +++++++++++++++++ tests/unit/test_training_features.py | 58 ++++++++++++ 15 files changed, 685 insertions(+), 19 deletions(-) create mode 100644 tests/unit/test_config_body.py create mode 100644 tests/unit/test_serving_body_limit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cfcb8d8..0c288bee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New example: `examples/feature-aware/` — a small interactions CSV, an item feature table exercising all three encodings, and a README walking train → serve → cold-start `:recommend-related`. +- **Request-body size cap.** `serve` now bounds the raw HTTP request body via a + `BodySizeLimitMiddleware` before Starlette buffers and JSON-parses it: a + declared `Content-Length` over the cap is rejected outright, and bodies with + no `Content-Length` (chunked / streamed) are counted as they arrive so the + header cannot be omitted to bypass the limit. Over-cap requests get a + `413 PAYLOAD_TOO_LARGE` in the standard error envelope. Previously an + authenticated client could make the process buffer and parse a multi-GB body. +- `RECOTEM_MAX_BODY_BYTES` (default 128 MiB, clamped [1 MiB, 2 GiB]) tunes the + cap. The default clears the largest well-formed request `serve` already + accepts (~72 MiB) with headroom while blocking GB-scale bodies. A new + `PAYLOAD_TOO_LARGE` error code is added to the v1 API's `ErrorCode` union. +- **Cold-start feature-dict key-length caps.** Every feature-mapping KEY is now + bounded to 1–256 characters (parity with other identifier fields): + `user_features` column names, the `item_features` outer seed-id keys, and the + nested per-seed feature keys. Previously only string VALUES were capped and + `Field(max_length=64)` bounded only the key COUNT, leaving key length + unbounded. Over-length or empty keys now get a `422`; an over-length key + reports only its length, never its (possibly huge) text. ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 3e5c68a6..d4fbef03 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,6 +219,7 @@ uv run ruff format --check src tests | `RECOTEM_DRAIN_SECONDS` | 30 | SIGTERM grace window. Clamped [1, 300]. | | `RECOTEM_LOG_FORMAT` | auto | `auto` / `json` / `console`. | | `RECOTEM_MAX_PAYLOAD_BYTES` | 512 MiB | Per-payload cap (post-HMAC-verify) for serve-side deserialization. Clamped [1 MiB, 16 GiB]. Smaller than `RECOTEM_MAX_ARTIFACT_BYTES` to bound deserialization memory expansion. | +| `RECOTEM_MAX_BODY_BYTES` | 128 MiB | Max serve-side HTTP **request** body size. Clamped [1 MiB, 2 GiB]. A `BodySizeLimitMiddleware` returns `413 PAYLOAD_TOO_LARGE` when the declared `Content-Length` exceeds the cap, and enforces a running byte count on chunked/streamed bodies with no `Content-Length` so the header cannot be omitted to bypass it. Default preserves the entire legitimate request space (largest well-formed body main accepts is ~72 MiB) while blocking GB-scale bodies that Starlette would buffer and parse before validation. | | `RECOTEM_ARTIFACT_ROOT` | (empty) | If set, local `output.path` must lie under it. | | `RECOTEM_RECIPE_*` | — | Allow-listed for `${...}` recipe expansion. | | `RECOTEM_METADATA_FIELD_DENY` | (empty) | Comma-separated columns stripped from `/v1/recipes/{name}:recommend` and `:recommend-related` responses. | diff --git a/README.md b/README.md index c9a6b252..9a4e92d4 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ moving parts to a recipe file and a binary artifact: - Recipe-driven: 1 YAML = 1 model = 1 `/v1/recipes/{name}:recommend` endpoint (with related/batch verbs) - Hyperparameter search across irspack algorithms via Optuna +- Feature-aware iALS: attach item/user side features (categorical / numerical / multi_label) via a `features:` recipe block +- Cold-start serving: recommend for unknown users and unseen seed items from their attributes alone via `user_features` / `item_features` - Pluggable data sources (built-in: CSV / Parquet / BigQuery / SQL; extend via Python entry points) - HMAC-signed artifacts with multi-key rotation and a deterministic FQCN allow-list at deserialization time diff --git a/docs/api-reference.md b/docs/api-reference.md index 60a84ceb..18febcbf 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -26,7 +26,7 @@ recipe-name constraint enforced by the recipe loader). **Response body:** see `RecommendResponse` in `src/recotem/serving/schemas.py`. -**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_USER` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_USER` | `RECIPE_NOT_FOUND`), 413 (`PAYLOAD_TOO_LARGE`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). ### `POST /v1/recipes/{name}:recommend-related` Seed-item → items. @@ -41,7 +41,7 @@ Seed-item → items. | `user_features` | object \| null | no | null | Raw feature values, keyed by the recipe's `features.user` column names. Adds a profile prior to the seed-history solve. See [Feature-aware cold start](#feature-aware-cold-start). ≤64 keys. | | `item_features` | object[string, object] \| null | no | null | Raw feature values for seed items absent from training, keyed by seed item id. ≤100 keys; each value ≤64 keys. See [Feature-aware cold start](#feature-aware-cold-start). | -**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_SEED_ITEMS` | `NO_CANDIDATES` | `RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 400 (`FEATURES_NOT_SUPPORTED` | `FEATURE_VALUE_UNUSABLE`), 401, 404 (`UNKNOWN_SEED_ITEMS` | `NO_CANDIDATES` | `RECIPE_NOT_FOUND`), 413 (`PAYLOAD_TOO_LARGE`), 422 (`VALIDATION_ERROR`), 503 (`RECIPE_UNAVAILABLE`). `UNKNOWN_SEED_ITEMS` means none of the supplied `seed_items` were known to the model id-map (typically a client-side data issue). @@ -77,7 +77,7 @@ list size cap (1..256) is enforced at the schema level (whole-request 422 if violated); per-element schema failures are surfaced per-element so a single bad entry never 422s the whole batch. -**Status codes:** 200, 401, 404 (`RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR` — only for whole-request shape, e.g. missing `requests` key, list too large), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 401, 404 (`RECIPE_NOT_FOUND`), 413 (`PAYLOAD_TOO_LARGE`), 422 (`VALIDATION_ERROR` — only for whole-request shape, e.g. missing `requests` key, list too large), 503 (`RECIPE_UNAVAILABLE`). > **Note:** batch endpoints return `{item_id, score}` only by default > (`include_metadata=false`). Set `include_metadata: true` to include @@ -115,7 +115,7 @@ precedence rules and the `200 {"items": []}` vs `NO_CANDIDATES` asymmetry described in [Feature-aware cold start](#feature-aware-cold-start) — both apply per-element here. -**Status codes:** 200, 401, 404 (`RECIPE_NOT_FOUND`), 422 (`VALIDATION_ERROR` — only for whole-request shape), 503 (`RECIPE_UNAVAILABLE`). +**Status codes:** 200, 401, 404 (`RECIPE_NOT_FOUND`), 413 (`PAYLOAD_TOO_LARGE`), 422 (`VALIDATION_ERROR` — only for whole-request shape), 503 (`RECIPE_UNAVAILABLE`). ### `GET /v1/recipes` Authenticated. Returns `RecipesListResponse` with one entry per loaded @@ -175,6 +175,26 @@ dominates a profile prior, so the server always prefers it and simply This lets a client always send the user's profile on every request without needing to know in advance whether the user is new or returning. +**A feature key that names no declared column is silently ignored — it is +not an error.** `_row_values` (`_features.py`) drives the encode from the +model's *declared* `features:` columns and does `values.get(name)`, so a key +in `user_features` / `item_features` that matches no declared column on that +side is never read. The request returns `200` with no error field and nothing +in the body marking the key as rejected. The only server-side signal is the +`recotem_v1_feature_unknown_column_total` metric (see +[operations.md](operations.md#feature-aware-ials-sizing)), labelled by recipe +and **side only — never by the key name** — and incremented once per side per +request that carried at least one such key. This is distinct from an unknown +*value* in a *declared* column (next section), which also returns `200` but is +counted separately, by `recotem_v1_feature_unknown_value_total`. A mapping in +which *every* key is mistyped (or is aimed at the wrong side) therefore +encodes to the bias column alone and comes back with **population-prior +results** — the same output an empty `user_features` would produce, and +indistinguishable from it in the response. **This is current behavior: clients +must not rely on the API to validate feature keys.** A silently-ignored key is +byte-for-byte identical, in the response, to a correct request that happened +to add no signal. + **Unknown feature values degrade, they do not fail the request.** What "degrade" means, and whether `recotem_v1_feature_unknown_value_total` (see [operations.md](operations.md#feature-aware-ials-sizing)) actually catches @@ -307,6 +327,31 @@ the more defensible response. If your client treats an empty popularity-based recommendations), branch on `items == []` rather than on HTTP status for this verb. +**Length and size bounds on cold-start fields.** A cold-start feature mapping +is bounded on three axes, each rejected before the model is consulted: + +- **Key count** — each `user_features` / `item_features` mapping accepts at + most **64 keys** (`item_features` additionally caps its outer seed-id keys at + **100**). Over the cap is `422 VALIDATION_ERROR`. +- **Key length** — each feature-dict key (a `user_features` column name, an + `item_features` outer seed id, or a nested per-seed feature key) must be + **1..256 characters**. Over the cap is `422`; the error reports only the + offending length, never the key text. +- **Value length** — each *string* feature value must be **≤ 8192 characters** + (this bounds `multi_label` tokenization work). Over the cap is `422`; the + error names the offending column but never echoes the value. Non-string + scalar values are unaffected. + +On the batch verbs a key- or value-length violation surfaces as a per-element +`VALIDATION_ERROR` inside the `200` batch response rather than failing the +whole batch. + +Independently of these per-field caps, the **entire request body** is bounded +by `RECOTEM_MAX_BODY_BYTES` (default **128 MiB**, clamped to +`[1 MiB, 2 GiB]`). A body over that limit is rejected with `413 +PAYLOAD_TOO_LARGE` **before** the JSON is parsed, so it applies to every POST +endpoint regardless of which fields the body carries. + ## Headers - `X-Request-ID` — accepted (regex `^[A-Za-z0-9_-]{1,128}$`) or generated; @@ -369,6 +414,7 @@ in every error case is one of the three forms above. | `VALIDATION_ERROR` | 422 | Pydantic schema rejected the request (also used per-element inside batch responses) | | `FEATURES_NOT_SUPPORTED` | 400 | `user_features` / `item_features` supplied but the model has no matching feature state, or its search winner is not feature-capable (also used per-element inside batch responses) | | `FEATURE_VALUE_UNUSABLE` | 400 | a supplied `numerical` feature value, once standardized against the column's training mean/std, is large enough to make irspack's cold-start solver itself fail (the exact threshold is std/BLAS-dependent, not a fixed constant, and depends on the column's std as much as the raw value — see [Feature-aware cold start](#feature-aware-cold-start)) — the model and feature side both support cold start, but this particular value does not. Values large enough to be meaningless but not large enough to break the solver degrade silently as `200` instead (also used per-element inside batch responses) | +| `PAYLOAD_TOO_LARGE` | 413 | request body exceeds `RECOTEM_MAX_BODY_BYTES` (default 128 MiB, clamped `[1 MiB, 2 GiB]`); rejected before the body is parsed, so it applies to every POST endpoint | | `MISSING_API_KEY` | 401 | `X-API-Key` header missing | | `INVALID_API_KEY` | 401 | `X-API-Key` header present but did not match any configured digest (also covers short-key / oversize-key rejections so callers cannot fingerprint the guard) | | `INTERNAL_ERROR` | 500 / batch | unhandled server-side exception, or unexpected recommender internal layout (`recommender_layout_unexpected`) — status=500 on single endpoints; per-element `status=error` inside batch responses | diff --git a/docs/operations.md b/docs/operations.md index 9dfca4d2..1b548dd0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -368,6 +368,7 @@ Each model replica holds every loaded model in RAM. Plan accordingly. |--------|--------| | `RECOTEM_MAX_ARTIFACT_BYTES` | Hard cap per artifact file (default 2 GiB, clamped [1 MiB, 16 GiB]). Reduce this if you have many small models. | | `RECOTEM_MAX_PAYLOAD_BYTES` | Cap on the deserialised payload per artifact (default 512 MiB, post-HMAC-verify). Must be ≤ `RECOTEM_MAX_ARTIFACT_BYTES`; if not, `recotem serve` fails at startup with `ConfigError` (exit 8). Reduces the memory spike from deserialization relative to the raw file size. | +| `RECOTEM_MAX_BODY_BYTES` | Hard cap on each HTTP **request** body (default 128 MiB, clamped [1 MiB, 2 GiB]). A `413 PAYLOAD_TOO_LARGE` is returned before Starlette buffers/parses the body, so a single authenticated client cannot make the process allocate a multi-GB request. The default clears the largest well-formed request `serve` accepts (~72 MiB: a 256-element batch, each carrying 1000 exclude_items of up to 256 chars) with headroom. Reduce it if your legitimate batch sizes are small and you want a tighter bound; the cap applies both to a declared `Content-Length` and to chunked bodies with no length header. | | Number of recipes | Each recipe loads one model. 10 recipes × 500 MiB = 5 GiB baseline. | | Number of replicas | Each replica is independent. 2 replicas = 2× memory. | | Item metadata | DataFrame in-memory per recipe. Size ≈ rows × columns × 8 bytes. | @@ -494,6 +495,7 @@ Full list of environment variables recognised by Recotem. Variables marked `serv | `RECOTEM_WATCH_INTERVAL` | 5 | serve | Artifact watcher poll interval in seconds (clamped 1–30). | | `RECOTEM_MAX_ARTIFACT_BYTES` | 2 GiB | serve | Per-artifact size cap (clamped [1 MiB, 16 GiB]). | | `RECOTEM_MAX_PAYLOAD_BYTES` | 512 MiB | serve | Per-payload cap post-HMAC-verify (clamped [1 MiB, 16 GiB]). Must be ≤ `RECOTEM_MAX_ARTIFACT_BYTES`. | +| `RECOTEM_MAX_BODY_BYTES` | 128 MiB | serve | Max HTTP request body size (clamped [1 MiB, 2 GiB]). Over-cap requests get `413 PAYLOAD_TOO_LARGE` before the body is buffered/parsed. See [Sizing `recotem serve` memory](#sizing-recotem-serve-memory). | | `RECOTEM_MAX_DOWNLOAD_BYTES` | 256 MiB | train | Raw I/O bytes cap for HTTP/HTTPS, local, and object-store source reads (clamped [1 MiB, 16 GiB]). Does **not** cap the decompressed DataFrame. | | `RECOTEM_HTTP_TIMEOUT_SECONDS` | 30 | train | Connect/read timeout for HTTP/HTTPS source fetch (clamped [1, 600]). | | `RECOTEM_HTTP_ALLOW_PRIVATE` | (unset) | train | Truthy (`1`/`true`/`yes`/`on`) allows HTTP fetches to private/loopback/link-local destinations. Leave unset in production to block SSRF against cloud-metadata services. | diff --git a/docs/recipe-reference.md b/docs/recipe-reference.md index 9218bc4c..a2a9bdda 100644 --- a/docs/recipe-reference.md +++ b/docs/recipe-reference.md @@ -280,14 +280,27 @@ known yields `Action=1` and drops `Zzz` — it is not an all-zero segment. "Row missing" and "value unknown" coincide only for `categorical`. The same `str()`-matching caveat that applies to `id_column` above also -applies to a `categorical` **value** column. If a blank cell makes pandas -infer `float64` for an otherwise-integer column, its vocabulary is trained as -`"1990.0"`, and a serve-time request sending the JSON integer `1990` (matched -as `"1990"`) misses every key and is silently counted as an unknown value. -Unlike the id axis, this is **not** refused at train time — the column varies -across rows, so training stays self-consistent — so pin the type at the source -(`dtype: {year: str}` on `csv`; `CAST(... AS STRING)` on `bigquery` / `sql`; -fix the schema on `parquet`) exactly as for the id column. +applies to a `categorical` or `multi_label` **value** column — the vocabulary +is fit from each value's string rendering, and a serve-time request value is +matched the same way. If a blank cell makes pandas infer `float64` for an +otherwise-integer column, its vocabulary is trained from `"1990.0"` (a +`multi_label` column's tokens the same way), and a serve-time request sending +the JSON integer `1990` (matched as `"1990"`) misses every key. So **declare +id-like or numeric-looking attribute columns as strings at the source** — one +blank cell is enough to flip the whole column to `float64` inference. Prefer +consistent types over relying on the counter after the fact. Unlike the id +axis, this is **not** refused at train time — the column varies across rows, so +training stays self-consistent — so pin the type at the source (`dtype: {year: +str}` on `csv`; `CAST(... AS STRING)` on `bigquery` / `sql`; fix the schema on +`parquet`) exactly as for the id column. The mismatch is not silent, though: at +serve time each such miss increments +`recotem_v1_feature_unknown_value_total` (labelled by recipe / side / column — +see [operations.md](operations.md#feature-aware-ials-sizing)), so a spike on a +column you expected to match is the signal to check its source dtype. The **id +axis** dtype trap is the stricter, train-time analogue: there the same `"1.0"` +vs `"1"` mismatch drives coverage to 0% and aborts training with the +[zero-overlap refusal](#ids-are-matched-as-strings-and-zero-overlap-is-fatal) +(`feature_axis_error`, exit 4) rather than degrading silently at serve time. At serve time, each cold-start feature value supplied to `:recommend` / `:recommend-related` (`user_features`, and each `item_features` seed mapping) is diff --git a/docs/security.md b/docs/security.md index 91815a0d..eeec4640 100644 --- a/docs/security.md +++ b/docs/security.md @@ -39,6 +39,7 @@ The internet-facing boundary is `recotem serve`. `recotem train` has no inbound | Malicious artifact file (serialization RCE) | HMAC-SHA256 verify before any deserialization; signing key required; no legacy unsigned fallback | | HMAC bypass leading to arbitrary class construction | Hand-enumerated FQCN allow-list as backstop (see below) | | Artifact-size DoS | `RECOTEM_MAX_ARTIFACT_BYTES` cap (default 2 GiB); header length cap (64 KiB); both enforced before deserialization | +| Request-body DoS (multi-GB body buffered before validation) | `RECOTEM_MAX_BODY_BYTES` cap (default 128 MiB) enforced by `BodySizeLimitMiddleware` before Starlette buffers/parses the body — on both `Content-Length` and chunked bodies; over-cap → `413 PAYLOAD_TOO_LARGE`. All request fields (ids, `exclude_items`, `seed_items`, batch size, and feature-dict key/value lengths + key count) are individually bounded. See [Rate limiting and DoS](#rate-limiting-and-dos) | | Stat-then-read TOCTOU on artifact | Read-once protocol: bytes read into memory once, sha256 computed, then HMAC-verified from the same buffer | | Key material in logs | structlog redaction processor runs first in chain; unit test asserts no key material at any log level | | API key brute-force / timing attack | `hmac.compare_digest` constant-time compare; no logging of plaintext or hash | @@ -704,6 +705,36 @@ everything else in this section, that bounds the work a **single request** can demand and says nothing about the rate; sustained rates remain the proxy's job. +**Request body is size-capped before it is parsed.** A `BodySizeLimitMiddleware` +(`serving/app.py`) rejects any request body larger than `RECOTEM_MAX_BODY_BYTES` +(default 128 MiB, clamped [1 MiB, 2 GiB]) with a `413 PAYLOAD_TOO_LARGE` +**before** Starlette buffers and JSON-parses it. Without this an authenticated +client could send a multi-GB body and force the process to allocate and parse it +in full ahead of any pydantic validation. The middleware enforces the cap at two +points so the header cannot be omitted to bypass it: a declared `Content-Length` +over the cap is refused outright, and a chunked/streamed body with no +`Content-Length` is counted as it arrives and cut off the moment the running +total crosses the cap. The default preserves the entire legitimate request space +— the largest well-formed body the API accepts is ~72 MiB (a 256-element batch, +each sub-request carrying 1000 `exclude_items` of up to 256 chars) — while +blocking GB-scale bodies. This bounds a **single request**; sustained rates are +still the proxy's job. + +**Per-request input fields are all length- and count-bounded.** Every +client-controlled request field has an explicit cap so a well-formed but huge +body cannot amplify inside validation or the recommender: `user_id` / item ids +are 1–256 chars (`_ItemStr`), `exclude_items` ≤ 1000, `seed_items` ≤ 100, batch +`requests` ≤ 256. The cold-start feature mappings are bounded on all three axes: +`Field(max_length=64)` caps the number of keys, each string **value** is capped +at 8192 chars (`_MAX_FEATURE_VALUE_CHARS`), and each **key** is capped at 1–256 +chars (`_MAX_FEATURE_KEY_CHARS`) — covering `user_features` column names, the +`item_features` outer seed-id keys (typed `_ItemStr`), and the nested per-seed +feature keys. Before the key cap the dict keys were the one length-unbounded +field left: `max_length` bounded only the key *count*, and only *values* were +length-checked, so an attacker could send megabyte-scale keys. An over-length +key now yields a `422` reporting only its length, never its text, so it cannot +amplify into the error body or logs. + **Recommended nginx configuration:** ```nginx diff --git a/src/recotem/config.py b/src/recotem/config.py index 3089e30f..9d6544fd 100644 --- a/src/recotem/config.py +++ b/src/recotem/config.py @@ -26,6 +26,8 @@ (default 256 MiB; clamped 1 MiB–16 GiB) RECOTEM_HTTP_TIMEOUT_SECONDS Timeout in seconds for HTTP/HTTPS datasource fetch (default 30; clamped 1–600) + RECOTEM_MAX_BODY_BYTES Max serve-side request body size in bytes + (default 128 MiB; clamped 1 MiB–2 GiB) RECOTEM_STARTUP_PARALLELISM Number of parallel threads used to load artifacts at startup (default min(recipes, 8); clamped 1–32) @@ -510,6 +512,30 @@ def get_http_timeout_seconds() -> int: ) +# --------------------------------------------------------------------------- +# Request-body cap (used by serving/app.py's body-size middleware) +# --------------------------------------------------------------------------- + +# Default is chosen to preserve the entire existing legitimate request space: +# the largest well-formed body serve already accepts is ~72 MiB (a 256-element +# batch, each sub-request carrying 1000 exclude_items of up to 256 chars). 128 +# MiB clears that with headroom while still blocking GB-scale bodies that would +# otherwise be buffered and parsed in full before validation. +DEFAULT_MAX_BODY_BYTES = 128 * 1024 * 1024 # 128 MiB +_MIN_BODY_BYTES = 1 * 1024 * 1024 # 1 MiB +_MAX_BODY_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB + + +def get_max_body_bytes() -> int: + """Return RECOTEM_MAX_BODY_BYTES, clamped to [1 MiB, 2 GiB].""" + return _clamped_int_env( + "RECOTEM_MAX_BODY_BYTES", + DEFAULT_MAX_BODY_BYTES, + _MIN_BODY_BYTES, + _MAX_BODY_BYTES, + ) + + # --------------------------------------------------------------------------- # Feature-encoding cap (used by recotem._features for feature-aware iALS) # --------------------------------------------------------------------------- diff --git a/src/recotem/serving/app.py b/src/recotem/serving/app.py index fc9fb581..866ed013 100644 --- a/src/recotem/serving/app.py +++ b/src/recotem/serving/app.py @@ -37,13 +37,13 @@ from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.responses import JSONResponse from starlette.middleware.base import BaseHTTPMiddleware -from starlette.types import ASGIApp +from starlette.types import ASGIApp, Message, Receive, Scope, Send from recotem._features import check_artifact_feature_version from recotem._irspack_compat import check_artifact_irspack_version from recotem.artifact.format import ArtifactError, parse_header_from_bytes from recotem.artifact.signing import KeyRing, unpickle_payload, verify_hmac -from recotem.config import ConfigError, ServeConfig +from recotem.config import ConfigError, ServeConfig, get_max_body_bytes from recotem.recipe.loader import load_recipes_directory_lenient from recotem.serving import metrics as _metrics from recotem.serving._header_utils import extract_algorithms, normalize_config_digest @@ -134,6 +134,116 @@ async def dispatch(self, request: Request, call_next): # type: ignore[override] structlog.contextvars.unbind_contextvars("request_id") +# --------------------------------------------------------------------------- +# Request-body size cap middleware +# --------------------------------------------------------------------------- + + +async def _noop_receive() -> Message: # pragma: no cover - Response never calls it + return {"type": "http.request", "body": b"", "more_body": False} + + +class BodySizeLimitMiddleware: + """Pure ASGI middleware capping the REQUEST body at ``max_body_bytes``. + + Starlette buffers and JSON-parses the whole request body before pydantic + validation runs, so without this an authenticated client can make the + process allocate a multi-GB body. Enforced at two points: + + - A declared ``Content-Length`` above the cap is rejected outright, before + any body byte is read. + - Bodies with no ``Content-Length`` (chunked / streamed) are counted as + they arrive by wrapping ``receive``; the cap fires the moment the running + total crosses it, so omitting the header cannot bypass the limit. + + Only REQUEST bodies are bounded: the ``send`` side is untouched on the + normal path, so streaming large RESPONSES is unaffected. Written as a pure + ASGI middleware (not ``BaseHTTPMiddleware``) because the running-count guard + needs to wrap ``receive`` itself. + """ + + def __init__(self, app: ASGIApp, max_body_bytes: int) -> None: + self.app = app + self.max_body_bytes = max_body_bytes + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + declared = _declared_content_length(scope) + if declared is not None and declared > self.max_body_bytes: + await self._send_too_large(scope, send) + return + + received = 0 + overflowed = False + response_started = False + + async def rcv() -> Message: + nonlocal received, overflowed + if overflowed: + # Already over the cap: stop draining the oversized stream and + # tell the inner app the client is gone so it unwinds. + return {"type": "http.disconnect"} + message = await receive() + if message["type"] == "http.request": + received += len(message.get("body", b"")) + if received > self.max_body_bytes: + overflowed = True + return {"type": "http.disconnect"} + return message + + async def snd(message: Message) -> None: + nonlocal response_started + if overflowed: + # We own the response once the cap is breached; swallow whatever + # the inner app emits and reply 413 exactly once. + if not response_started: + response_started = True + await self._send_too_large(scope, send) + return + await send(message) + + try: + await self.app(scope, rcv, snd) + except Exception: + # The inner app may raise ClientDisconnect (or similar) because we + # injected http.disconnect after the cap was hit. That is a + # consequence of our own guard, not a real error, so absorb it and + # emit the 413. Anything raised without an overflow is a genuine + # error and must propagate. + if not overflowed: + raise + if overflowed and not response_started: + response_started = True + await self._send_too_large(scope, send) + + async def _send_too_large(self, scope: Scope, send: Send) -> None: + response = JSONResponse( + status_code=413, + content={ + "detail": ( + "Request body exceeds the " + f"{self.max_body_bytes}-byte limit (RECOTEM_MAX_BODY_BYTES)" + ), + "code": "PAYLOAD_TOO_LARGE", + }, + ) + await response(scope, _noop_receive, send) + + +def _declared_content_length(scope: Scope) -> int | None: + """Return the request's ``Content-Length`` as an int, or None if absent/bad.""" + for name, value in scope.get("headers", []): + if name == b"content-length": + try: + return int(value) + except ValueError: + return None + return None + + # --------------------------------------------------------------------------- # Application factory # --------------------------------------------------------------------------- @@ -570,6 +680,14 @@ async def _unhandled_exception_handler( allowed_hosts=serve_config.allowed_hosts, ) + # Cap the request body before Starlette buffers/parses it. Added after + # TrustedHost/CORS (so it wraps them and runs before the body is read) but + # before RequestIDMiddleware, so a 413 still carries X-Request-ID. + app.add_middleware( + BodySizeLimitMiddleware, + max_body_bytes=get_max_body_bytes(), + ) + if serve_config.allowed_origins: app.add_middleware( CORSMiddleware, diff --git a/src/recotem/serving/schemas.py b/src/recotem/serving/schemas.py index 765e905a..9cfd0680 100644 --- a/src/recotem/serving/schemas.py +++ b/src/recotem/serving/schemas.py @@ -52,6 +52,7 @@ "INTERNAL_ERROR", "FEATURES_NOT_SUPPORTED", "FEATURE_VALUE_UNUSABLE", + "PAYLOAD_TOO_LARGE", ] # --------------------------------------------------------------------------- @@ -70,21 +71,40 @@ # yet blocks MB-scale amplification, restoring parity with every other field. _MAX_FEATURE_VALUE_CHARS = 8192 +# Per-KEY length cap for cold-start feature mappings. `Field(max_length=64)` on +# `_FeatureValues` caps the key COUNT; `_MAX_FEATURE_VALUE_CHARS` caps each +# string VALUE; this caps each KEY's length. Without it the dict keys +# (`user_features` column names and the nested `item_features` per-seed feature +# keys) were unbounded even while every other identifier field is length-capped +# (_ItemStr is 1..256). 256 keeps parity with `_ItemStr`. An over-length key +# reports only its length, never its text, so a multi-MB key cannot amplify into +# the 422 body / logs; the `item_features` OUTER keys (seed ids) are capped +# separately by typing that dict's keys as `_ItemStr`. +_MAX_FEATURE_KEY_CHARS = 256 + def _check_feature_value_lengths( values: dict[str, Any] | None, ) -> dict[str, Any] | None: - """Reject any string feature value longer than ``_MAX_FEATURE_VALUE_CHARS``. + """Reject over-length feature-dict KEYS and string VALUES. Shared by every place a cold-start feature mapping appears: ``user_features`` on both request models and each nested ``item_features`` mapping (this - validator runs per ``_FeatureValues``, so a nested dict of values is checked - too). Names the offending column key but never echoes the value, which is - treated as personal data. Non-string scalars are unaffected. + validator runs per ``_FeatureValues``, so a nested dict of keys/values is + checked too). KEYS are bounded to ``1..._MAX_FEATURE_KEY_CHARS``; string + VALUES to ``_MAX_FEATURE_VALUE_CHARS``. The value check names the offending + column key but never echoes the value (treated as personal data); the key + check reports only the length, never the key text. Non-string scalar values + are unaffected by the value cap. """ if values is None: return values for key, val in values.items(): + if not 1 <= len(key) <= _MAX_FEATURE_KEY_CHARS: + raise ValueError( + f"feature key length {len(key)} is outside the permitted " + f"1..{_MAX_FEATURE_KEY_CHARS} characters" + ) if isinstance(val, str) and len(val) > _MAX_FEATURE_VALUE_CHARS: raise ValueError( f"feature value for column {key!r} exceeds the " @@ -161,8 +181,12 @@ class RecommendRelatedRequest(BaseModel): # seed item id. Takes precedence over ``user_features`` when a seed named # here is also cold: a cold seed has no row in the seed interaction # matrix, so the case-B solve would silently drop it. + # Outer keys are seed item ids, so they are typed ``_ItemStr`` (1..256) to + # bound key length exactly like ``seed_items`` -- ``Field(max_length=100)`` + # caps only the key COUNT. The nested ``_FeatureValues`` validator bounds + # each cold-seed mapping's own keys and values. item_features: Annotated[ - dict[str, _FeatureValues] | None, + dict[_ItemStr, _FeatureValues] | None, Field( max_length=100, description=( diff --git a/tests/fuzz/test_recipe_loader.py b/tests/fuzz/test_recipe_loader.py index 4efe3693..fe51b9ba 100644 --- a/tests/fuzz/test_recipe_loader.py +++ b/tests/fuzz/test_recipe_loader.py @@ -10,6 +10,7 @@ from pathlib import Path import pytest +import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st @@ -35,6 +36,45 @@ """ +# A recipe that DOES carry a `features:` block, so mutations reach the nested +# FeatureSide / FeatureColumn validation (and the nested `source` checks) that +# MINIMAL_VALID_YAML never exercises. Exercises all three encodings, a +# multi_label `delimiter`, and a `min_frequency`, on both the item and user +# sides. `training.algorithms` must name a feature-capable algorithm (IALS) or +# recipe load rejects the pairing before the nested feature validation runs. +FEATURES_VALID_YAML = """\ +name: fuzz_features +source: + type: csv + path: /tmp/data.csv +schema: + user_column: user_id + item_column: item_id +features: + item: + source: + type: csv + path: /tmp/items.csv + id_column: item_id + columns: + - {name: genres, encoding: multi_label, delimiter: "|", min_frequency: 2} + - {name: release_year, encoding: numerical} + - {name: country, encoding: categorical, min_frequency: 5} + user: + source: + type: csv + path: /tmp/users.csv + id_column: user_id + columns: + - {name: age_band, encoding: categorical} +training: + algorithms: [IALS] + n_trials: 1 +output: + path: /tmp/out.recotem +""" + + def _try_load_yaml(content: str, tmp_path: Path) -> None: """Attempt to load a YAML string; accept RecipeError, raise on anything else.""" from recotem.recipe.loader import load_recipe @@ -137,6 +177,77 @@ def test_loader_handles_unknown_source_type(source_type: str, tmp_path: Path) -> _try_load_yaml(content, tmp_path) +# --------------------------------------------------------------------------- +# Hypothesis: mutation of the `features:` block (nested FeatureSide / +# FeatureColumn validation, absent from MINIMAL_VALID_YAML) +# --------------------------------------------------------------------------- + + +@given( + encoding=st.text(alphabet=string.printable, min_size=0, max_size=20), + min_frequency=st.one_of(st.integers(), st.text(min_size=0, max_size=6)), + delimiter=st.one_of(st.none(), st.text(min_size=0, max_size=4)), +) +@settings( + max_examples=100, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.function_scoped_fixture], +) +def test_loader_handles_mutated_features_column( + encoding: str, min_frequency: object, delimiter: str | None, tmp_path: Path +) -> None: + """Mutating a FeatureColumn's encoding/min_frequency/delimiter must still + yield only a Recipe or a RecipeError -- never an unhandled exception from + the nested feature-column validation.""" + doc = yaml.safe_load(FEATURES_VALID_YAML) + col = doc["features"]["item"]["columns"][0] + col["encoding"] = encoding + col["min_frequency"] = min_frequency + if delimiter is not None: + col["delimiter"] = delimiter + _try_load_yaml(yaml.safe_dump(doc), tmp_path) + + +# Arbitrary JSON-ish values (scalars, lists, nested dicts) for injection AT a +# recipe key. Keeps the leaves to YAML-dumpable scalar types so the resulting +# document is always syntactically valid YAML -- the point is to fuzz the +# pydantic *shape* at `features:`, not the YAML parser (that is what +# test_loader_handles_arbitrary_text already covers). +_JSON_ISH = st.recursive( + st.none() + | st.booleans() + | st.integers() + | st.text(alphabet=string.printable, min_size=0, max_size=20), + lambda children: ( + st.lists(children, max_size=4) + | st.dictionaries( + st.text(alphabet=string.ascii_letters, min_size=0, max_size=8), + children, + max_size=4, + ) + ), + max_leaves=15, +) + + +@given(features_value=_JSON_ISH) +@settings( + max_examples=200, + derandomize=True, # deterministic corpus across runs + suppress_health_check=[HealthCheck.too_slow, HealthCheck.function_scoped_fixture], +) +def test_loader_handles_arbitrary_features_value( + features_value: object, tmp_path: Path +) -> None: + """An arbitrary value injected AT the `features:` key -- a list, an int, or + nested garbage where a mapping is expected -- must resolve to a Recipe or a + RecipeError, never an unhandled exception. Builds on FEATURES_VALID_YAML so + the surrounding recipe (including the IALS algorithm the features block + requires) stays valid and the fuzzed value is the only variable.""" + doc = yaml.safe_load(FEATURES_VALID_YAML) + doc["features"] = features_value + _try_load_yaml(yaml.safe_dump(doc), tmp_path) + + # --------------------------------------------------------------------------- # Edge cases: empty/null content # --------------------------------------------------------------------------- diff --git a/tests/unit/test_config_body.py b/tests/unit/test_config_body.py new file mode 100644 index 00000000..df1ca7f2 --- /dev/null +++ b/tests/unit/test_config_body.py @@ -0,0 +1,35 @@ +"""Tests for RECOTEM_MAX_BODY_BYTES (serve-side request body cap).""" + +from __future__ import annotations + +import pytest + +from recotem.config import DEFAULT_MAX_BODY_BYTES, get_max_body_bytes + + +def test_max_body_bytes_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("RECOTEM_MAX_BODY_BYTES", raising=False) + assert get_max_body_bytes() == DEFAULT_MAX_BODY_BYTES + assert DEFAULT_MAX_BODY_BYTES == 128 * 1024 * 1024 + + +def test_max_body_bytes_custom(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_BODY_BYTES", str(4 * 1024 * 1024)) + assert get_max_body_bytes() == 4 * 1024 * 1024 + + +def test_max_body_bytes_below_min_clamped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_BODY_BYTES", "0") + # Clamp to 1 MiB minimum + assert get_max_body_bytes() == 1024 * 1024 + + +def test_max_body_bytes_above_max_clamped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_BODY_BYTES", str(64 * 1024 * 1024 * 1024)) + # Clamp to 2 GiB maximum + assert get_max_body_bytes() == 2 * 1024 * 1024 * 1024 + + +def test_max_body_bytes_invalid_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECOTEM_MAX_BODY_BYTES", "not-a-number") + assert get_max_body_bytes() == DEFAULT_MAX_BODY_BYTES diff --git a/tests/unit/test_serving_body_limit.py b/tests/unit/test_serving_body_limit.py new file mode 100644 index 00000000..3705a009 --- /dev/null +++ b/tests/unit/test_serving_body_limit.py @@ -0,0 +1,100 @@ +"""Tests for BodySizeLimitMiddleware — the serve-side request body cap. + +Covers both enforcement points: +- a declared Content-Length above the cap is rejected before the body is read; +- a chunked/streamed body with no Content-Length is rejected on a running count. +A normal-sized body must pass the gate untouched. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from recotem.config import ServeConfig +from recotem.serving.app import create_app + +_CAP = 1024 * 1024 # 1 MiB (the clamp minimum), keeps the oversized body small. + + +def _app_with_cap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + recipes_dir = tmp_path / "recipes" + recipes_dir.mkdir() + monkeypatch.setenv("RECOTEM_MAX_BODY_BYTES", str(_CAP)) + cfg = ServeConfig() + cfg.signing_keys_raw = "active:" + "aa" * 32 + cfg.recipes_dir = str(recipes_dir) + cfg.env = "development" + cfg.insecure_no_auth = True + cfg.allowed_hosts = ["testserver", "localhost", "127.0.0.1", "*"] + return create_app(cfg) + + +def test_oversized_content_length_returns_413( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + app = _app_with_cap(tmp_path, monkeypatch) + client = TestClient(app, raise_server_exceptions=False) + # A bytes body sets an explicit Content-Length > cap → rejected outright. + resp = client.post( + "/v1/recipes/demo:recommend", + content=b"a" * (_CAP + 1), + headers={"content-type": "application/json"}, + ) + assert resp.status_code == 413 + body = resp.json() + assert body["code"] == "PAYLOAD_TOO_LARGE" + assert "detail" in body + + +def test_oversized_chunked_body_without_content_length_returns_413( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + app = _app_with_cap(tmp_path, monkeypatch) + client = TestClient(app, raise_server_exceptions=False) + + def _gen() -> Iterator[bytes]: + # 16 chunks of 128 KiB = 2 MiB, streamed with Transfer-Encoding: chunked + # (no Content-Length), so only the running-count guard can catch it. + for _ in range(16): + yield b"a" * (128 * 1024) + + resp = client.post( + "/v1/recipes/demo:recommend", + content=_gen(), + headers={"content-type": "application/json"}, + ) + assert resp.status_code == 413 + assert resp.json()["code"] == "PAYLOAD_TOO_LARGE" + + +def test_normal_body_passes_the_size_gate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + app = _app_with_cap(tmp_path, monkeypatch) + client = TestClient(app, raise_server_exceptions=False) + # Well under the cap: the gate must let it through to routing, which 404s + # because no recipe named "demo" is loaded — proving it was NOT a 413. + resp = client.post("/v1/recipes/demo:recommend", json={"user_id": "u1"}) + assert resp.status_code != 413 + assert resp.status_code == 404 + assert resp.json()["code"] == "RECIPE_NOT_FOUND" + + +def test_413_carries_request_id_header( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The body cap sits inside RequestIDMiddleware, so its 413 still carries + an X-Request-ID for correlation.""" + app = _app_with_cap(tmp_path, monkeypatch) + client = TestClient(app, raise_server_exceptions=False) + resp = client.post( + "/v1/recipes/demo:recommend", + content=b"a" * (_CAP + 1), + headers={"content-type": "application/json"}, + ) + assert resp.status_code == 413 + assert resp.headers.get("x-request-id") diff --git a/tests/unit/test_serving_schemas.py b/tests/unit/test_serving_schemas.py index 15801a83..b9731c88 100644 --- a/tests/unit/test_serving_schemas.py +++ b/tests/unit/test_serving_schemas.py @@ -394,6 +394,87 @@ def test_feature_value_cap_covers_batch_reparse_path() -> None: ) +# --------------------------------------------------------------------------- +# Feature-dict KEY length caps. `_FeatureValues`' `Field(max_length=64)` caps +# the key COUNT and `_MAX_FEATURE_VALUE_CHARS` caps each string VALUE, but the +# KEYS themselves were length-unbounded: `user_features` column names, the +# `item_features` outer seed-id keys, and the nested per-seed feature keys. +# All three are now capped at 256 chars (parity with `_ItemStr`), empty keys +# rejected. +# --------------------------------------------------------------------------- + +# Mirrors schemas._MAX_FEATURE_KEY_CHARS; the sync test below pins them equal. +_FEATURE_KEY_CAP = 256 + + +def test_feature_key_cap_matches_module_constant() -> None: + from recotem.serving.schemas import _MAX_FEATURE_KEY_CHARS + + assert _MAX_FEATURE_KEY_CHARS == _FEATURE_KEY_CAP + + +def test_recommend_request_user_features_key_over_cap_rejected() -> None: + with pytest.raises(ValidationError): + RecommendRequest( + user_id="u1", user_features={"k" * (_FEATURE_KEY_CAP + 1): "v"} + ) + + +def test_recommend_request_user_features_key_at_cap_accepted() -> None: + req = RecommendRequest(user_id="u1", user_features={"k" * _FEATURE_KEY_CAP: "v"}) + assert "k" * _FEATURE_KEY_CAP in req.user_features + + +def test_recommend_request_user_features_empty_key_rejected() -> None: + with pytest.raises(ValidationError): + RecommendRequest(user_id="u1", user_features={"": "v"}) + + +def test_recommend_related_request_item_features_outer_key_over_cap_rejected() -> None: + """The `item_features` OUTER keys are seed item ids, capped like seed_items + (`_ItemStr`, 1..256).""" + with pytest.raises(ValidationError): + RecommendRelatedRequest( + seed_items=["s1"], + item_features={"s" * (_FEATURE_KEY_CAP + 1): {"g": "v"}}, + ) + + +def test_recommend_related_request_item_features_empty_outer_key_rejected() -> None: + with pytest.raises(ValidationError): + RecommendRelatedRequest( + seed_items=["s1"], + item_features={"": {"g": "v"}}, + ) + + +def test_recommend_related_request_item_features_inner_key_over_cap_rejected() -> None: + """The nested per-seed feature mapping's KEYS are also capped.""" + with pytest.raises(ValidationError): + RecommendRelatedRequest( + seed_items=["s1"], + item_features={"s1": {"k" * (_FEATURE_KEY_CAP + 1): "v"}}, + ) + + +def test_feature_key_over_cap_error_does_not_echo_key_text() -> None: + """An over-length key must report only its length, never its (possibly huge) + text, so it cannot amplify into the 422 body / logs.""" + huge_key = "k" * (_FEATURE_KEY_CAP + 5000) + with pytest.raises(ValidationError) as exc_info: + RecommendRequest(user_id="u1", user_features={huge_key: "v"}) + assert huge_key not in str(exc_info.value) + + +def test_feature_key_cap_covers_batch_reparse_path() -> None: + """Batch verbs re-parse each element via `model_validate`; the key cap must + cover that path too.""" + with pytest.raises(ValidationError): + RecommendRequest.model_validate( + {"user_id": "u1", "user_features": {"k" * (_FEATURE_KEY_CAP + 1): "v"}} + ) + + # --------------------------------------------------------------------------- # Finding 6: Discriminated union extra-field enforcement # --------------------------------------------------------------------------- diff --git a/tests/unit/test_training_features.py b/tests/unit/test_training_features.py index 3830aab7..1f5753ed 100644 --- a/tests/unit/test_training_features.py +++ b/tests/unit/test_training_features.py @@ -804,3 +804,61 @@ def test_zero_overlap_message_does_not_hardcode_csv_only_dtype_key( # Still diagnosable: names the side, the id_column, and points at the docs. assert "sku" in msg assert "operations.md" in msg + + +# --------------------------------------------------------------------------- +# Zero-row feature table: a header-only table (columns declared, no data rows) +# flowing into the training feature path. The built-in csv/parquet sources +# reject a header-only file at FETCH time with DataSourceError (exit 3), so +# they never hand a 0-row frame to the encoder. To pin what the training +# feature path ITSELF does with an empty frame -- the state a 0-row sql query +# or a custom plugin can still reach -- a stub source that returns an empty +# (but correctly-columned) frame is injected via the registry lookup +# ``load_feature_tables`` uses. The encoder then finds every declared column +# dead (empty categorical vocab, zero-variance numerical), so the whole-block- +# dead guard refuses it exactly as the constant/all-null blocks above do. +# --------------------------------------------------------------------------- + + +def test_zero_row_feature_table_refused_as_bias_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A 0-row (header-only) feature table reaching the encoder collapses to the + bias column alone and is refused with ``feature_table_error`` (exit 4) -- + NOT signed as a features-advertising plain-iALS artifact.""" + import recotem.training.features as tf + + class _EmptyFrameSource: + class Config: + @classmethod + def model_validate(cls, raw: object) -> _EmptyFrameSource.Config: + return cls() + + def __init__(self, config: object) -> None: + pass + + def fetch(self, ctx: object) -> pd.DataFrame: + # Columns declared (header present), zero data rows. + return pd.DataFrame({"item_id": [], "genre": [], "year": []}) + + monkeypatch.setattr(tf, "get_source_class", lambda _name: _EmptyFrameSource) + + cfg = FeaturesConfig( + item=FeatureSideConfig( + source={"type": "stub_empty"}, + id_column="item_id", + columns=[ + FeatureColumn(name="genre", encoding="categorical"), + FeatureColumn(name="year", encoding="numerical"), + ], + ) + ) + with pytest.raises(TrainingError) as exc_info: + load_feature_tables(cfg, recipe_name="r", run_id="run") + # Whole-block-dead guard fires in _fetch_side, before any axis is known -- + # so this is feature_table_error, not the encode-time feature_axis_error. + assert exc_info.value.code == "feature_table_error" # -> exit 4 + assert exc_info.value.code != "signing_key_missing" + msg = str(exc_info.value) + assert "item" in msg + assert "bias" in msg