Add better guidance and amend feature to webui

.github/workflows/validate_data.yaml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        name: [ json-files, folder-names, store-ids, logo-files, fiber-consistency ]
+        name: [ json-files, folder-names, store-ids, logo-files, fiber-consistency, data-quality ]
     name: Validate data - ${{ matrix.name }}
     steps:
       - name: Checkout

ofd/commands/validate.py

@@ -93,6 +93,12 @@ def register_subcommand(subparsers: argparse._SubParsersAction) -> None:
         action="store_true",
         help="Validate no filament mixes carbon fiber and glass fiber across its variants",
     )
+    scope_group.add_argument(
+        "--data-quality",
+        action="store_true",
+        help="Check for placeholder values, duplicate spool rows, name casing/whitespace, "
+        "missing fiber traits, filaments with no variants, and word-order duplicates",
+    )
 
     # Output options
     output_group = parser.add_argument_group("output options")
@@ -172,15 +178,16 @@ def run_validate(args: argparse.Namespace) -> int:
             args.store_ids,
             args.gtin,
             args.fiber_consistency,
+            args.data_quality,
         ]
     )
 
     if not specific_validations:
         # Run all validations
         if not args.json and not args.progress:
             print(_bold("Running all validations..."))
-        # validate_all() already includes the native fiber-consistency check
-        # (skipped automatically under a changes overlay).
+        # validate_all() already includes the native fiber-consistency and
+        # data-quality checks (skipped automatically under a changes overlay).
         result = orchestrator.validate_all(changes_json=changes_json)
     else:
         # Run specific validations
@@ -207,6 +214,18 @@ def run_validate(args: argparse.Namespace) -> int:
                     ),
                     file=sys.stderr,
                 )
+        if args.data_quality:
+            # Same constraint as fiber-consistency: on-disk only.
+            if changes_json is None:
+                result.merge(orchestrator.validate_data_quality())
+            else:
+                print(
+                    _yellow(
+                        "Skipping data-quality: it runs against on-disk data and "
+                        "cannot apply --apply-changes."
+                    ),
+                    file=sys.stderr,
+                )
 
     # Output results
     if args.json:

ofd/scripts/style_data.py

@@ -104,8 +104,14 @@
     "refinements",
     "_encoding",
     "pldnsite",
+    # SimplyPrint storefront recommender context. Seen on 3dhojor datasheet URLs merged in
+    # #461; the token is per-session and hundreds of characters long.
+    "_su_rec",
+    "_su_rec_id",
+    # Shopify affiliate referral (seen across SUNLU's purchase links).
+    "sca_ref",
 }
-_TRACKING_PREFIXES = ("utm_", "mc_", "pk_", "pd_rd_", "pf_rd_")
+_TRACKING_PREFIXES = ("utm_", "mc_", "pk_", "pd_rd_", "pf_rd_", "_su_")
 # Hosts where the product identity is entirely in the path, so the whole query is disposable.
 _WHOLE_QUERY_HOSTS = re.compile(r"(^|\.)(amazon|ebay)\.[a-z.]+$")
 _SHORTLINK_HOSTS = {"a.co", "amzn.to", "amzn.eu"}
@@ -122,6 +128,26 @@ def _is_tracking_fragment(fragment: str) -> bool:
     return _is_tracking_key(re.split(r"[&=]", fragment, maxsplit=1)[0])
 
 
+_AMAZON_HOST = re.compile(r"(^|\.)amazon\.[a-z.]+$")
+
+
+def _strip_amazon_ref_path(base: str, host: str) -> str:
+    """Drop Amazon's trailing `/ref=<how-you-got-here>` path breadcrumb.
+
+    Amazon puts a tracker in the *path*, not only the query. PR #408 was submitted as
+    `.../dp/B0FB93XQLM/ref=sr_1_6` ("search result, page 1, position 6") and had to be
+    shortened by hand in review. The product-name slug before `/dp/<ASIN>` is kept — Amazon
+    ignores it and it makes the link readable.
+    """
+    if not _AMAZON_HOST.search(host):
+        return base
+    path = urlsplit(base).path
+    idx = path.find("/ref=")
+    if idx == -1:
+        return base
+    return base[: len(base) - len(path) + idx]
+
+
 def strip_tracking_params(url: str) -> str:
     """Remove known tracking/affiliate params (plus an empty '?' and tracking-only fragments)
     from a URL, preserving everything else. Idempotent. Mirrors the ofd-validator Rust rule
@@ -139,6 +165,9 @@ def strip_tracking_params(url: str) -> str:
         host = ""
     drop_all = bool(_WHOLE_QUERY_HOSTS.search(host)) or host in _SHORTLINK_HOSTS
 
+    # Amazon carries tracking in the path, not only the query.
+    base = _strip_amazon_ref_path(base, host)
+
     out = base
     if query is not None and not drop_all:
         kept = [

ofd/validation/__init__.py

@@ -87,13 +87,25 @@ def validate_fiber_consistency(self) -> ValidationResult:
             result.add_error(error)
         return result
 
+    def validate_data_quality(self) -> ValidationResult:
+        """Report the recurring data-quality defects reviewers fixed by hand.
+
+        This is a native (non-Rust) check; see data_quality.py for the rules.
+        """
+        from ofd.validation.data_quality import check_data_quality
+
+        result = ValidationResult()
+        for error in check_data_quality(self.data_dir):
+            result.add_error(error)
+        return result
+
     def validate_all(self, changes_json: str | None = None) -> ValidationResult:
         """Run all validations, optionally with pending changes applied.
 
-        Includes the native cross-variant fiber-consistency check, except under a
-        changes overlay: that check reads on-disk data and can't see pending edits,
-        so running it there could report stale conflicts (the webui enforces the
-        rule pre-export).
+        Includes the native fiber-consistency and data-quality checks, except under a
+        changes overlay: those read on-disk data and can't see pending edits, so
+        running them there could report stale findings (the webui enforces the same
+        rules pre-export).
         """
         if changes_json and _validate_all_with_changes is not None:
             return _validate_all_with_changes(
@@ -107,6 +119,7 @@ def validate_all(self, changes_json: str | None = None) -> ValidationResult:
         result = _validate_all(self.data_dir, self.stores_dir, max_workers=self.max_workers)
         if not changes_json:
             result.merge(self.validate_fiber_consistency())
+            result.merge(self.validate_data_quality())
         return result