-
Notifications
You must be signed in to change notification settings - Fork 0
fix: make reviewer coverage resilient (exempt lockfiles; degrade over-long constraints) #567
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
41fcdf5
6d0772e
7157adb
5fdbceb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -272,10 +272,7 @@ func DecodeFindings(data []byte, opts FindingsOptions) (Findings, error) { | |
| if err := validateCoverageFileDisjoint(inspected, skipped); err != nil { | ||
| return Findings{}, err | ||
| } | ||
| constraints, err := decodeCoverageStrings("constraints", wire.Constraints) | ||
| if err != nil { | ||
| return Findings{}, err | ||
| } | ||
| constraints := decodeCoverageStrings(wire.Constraints) | ||
|
|
||
| result := Findings{ | ||
| AgentID: wire.AgentID, | ||
|
|
@@ -352,27 +349,31 @@ func decodeCoverageFiles(name string, files []string, changedFiles map[string]bo | |
| return out, nil | ||
| } | ||
|
|
||
| func decodeCoverageStrings(name string, values []string) ([]string, error) { | ||
| // decodeCoverageStrings cleans reviewer coverage constraints. These are | ||
| // informational notes ("couldn't verify X against source-of-truth docs"), not | ||
| // a contract, so a malformed or verbose entry is degraded — the count is | ||
| // capped, an over-long entry is truncated, and empties/duplicates are dropped — | ||
| // rather than failing the decode. Failing here sinks the whole reviewer as | ||
| // "completed without a result file" and blocks approval on an otherwise-clean | ||
| // review, which a single legitimate ~300-rune constraint once did. | ||
| func decodeCoverageStrings(values []string) []string { | ||
| if len(values) > defaultMaxCoverageConstraints { | ||
| return nil, fmt.Errorf("llm: %s cap exceeded", name) | ||
| values = values[:defaultMaxCoverageConstraints] | ||
| } | ||
| out := make([]string, 0, len(values)) | ||
| seen := map[string]bool{} | ||
| for _, value := range values { | ||
| if utf8.RuneCountInString(value) > defaultMaxCoverageConstraintRunes { | ||
| return nil, fmt.Errorf("llm: %s entry length out of bounds", name) | ||
| value = truncateRunes(value, defaultMaxCoverageConstraintRunes) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. decodeCoverageStrings truncates an over-long constraint to defaultMaxCoverageConstraintRunes (300) and then appends "..." via truncateRunes, so the value that survives sanitize() can be up to 303 runes — 3 runes past the documented/enforced cap (DefaultFindingsConstraintLimits().MaxRunesPerEntry = 300, which the orchestrator prompt tells reviewers is the hard limit). TestDecodeFindingsConstraintRuneBoundaries only asserts the result ends with "...", not that it respects the rune cap, so this drift isn't caught. Fix by reserving the ellipsis width before truncating, e.g. truncateRunes(value, defaultMaxCoverageConstraintRunes-3), or by post-truncating the whole result (including the suffix) down to defaultMaxCoverageConstraintRunes. Reply inline to this comment. |
||
| } | ||
| value = sanitize(value) | ||
| if strings.TrimSpace(value) == "" { | ||
| return nil, fmt.Errorf("llm: %s entries must be non-empty", name) | ||
| } | ||
| if seen[value] { | ||
| return nil, fmt.Errorf("llm: duplicate %s entry %q", name, value) | ||
| if strings.TrimSpace(value) == "" || seen[value] { | ||
| continue | ||
| } | ||
| seen[value] = true | ||
| out = append(out, value) | ||
| } | ||
| return out, nil | ||
| return out | ||
| } | ||
|
|
||
| func validateCoverageFileDisjoint(inspected, skipped []string) error { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2437,10 +2437,56 @@ func reviewerToolEvidenceByAgent(sessions []sessionDraft) map[string]*llm.Review | |
| return out | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. File-level note: internal/pipeline/pipeline.go docs/checkout-native-review-contract.md is the documented source of truth for reviewer coverage semantics: it defines "readable files" as "all changed files in the workbench" and states unassigned/skipped files "must not turn into a clean approval silently." This diff's buildReviewerCoverage now filters generated lockfiles out of changedFiles (and each agent's scope) before computing coverage, so lockfiles are silently excluded from the readable-files universe the doc describes — but the doc's coverage-status section (lines ~337-354) was not updated to mention the lockfile exemption or the new filterReviewableFiles/isGeneratedLockfile step. A future agent or engineer reading that doc to understand what "complete" coverage guarantees will get a materially misleading picture of what the harness actually enforces. Update docs/checkout-native-review-contract.md to document the lockfile exemption alongside the existing coverage-status definitions. Reply inline to this comment. |
||
| } | ||
|
|
||
| // generatedLockfiles are dependency lockfiles: machine-written by a package | ||
| // manager and reviewed (if at all) through the manifest change that produced | ||
| // them, never line by line. A reviewer that skips one is behaving correctly, so | ||
| // they are excluded from the coverage universe — otherwise a skipped lockfile | ||
| // marks the reviewer incomplete_skipped and blocks approval on an otherwise | ||
| // clean review. (A real PR stalled exactly this way: a Cargo.lock churned by a | ||
| // dependency bump was the only file left "unreviewed".) | ||
| var generatedLockfiles = map[string]bool{ | ||
| "Cargo.lock": true, | ||
| "package-lock.json": true, | ||
| "npm-shrinkwrap.json": true, | ||
| "yarn.lock": true, | ||
| "pnpm-lock.yaml": true, | ||
| "bun.lockb": true, | ||
| "go.sum": true, | ||
| "Gemfile.lock": true, | ||
| "poetry.lock": true, | ||
| "Pipfile.lock": true, | ||
| "composer.lock": true, | ||
| "Podfile.lock": true, | ||
| "flake.lock": true, | ||
| "mix.lock": true, | ||
| } | ||
|
|
||
| // isGeneratedLockfile reports whether path is a dependency lockfile a reviewer | ||
| // is not expected to read line by line. | ||
| func isGeneratedLockfile(path string) bool { | ||
| return generatedLockfiles[filepath.Base(path)] | ||
| } | ||
|
|
||
| // filterReviewableFiles drops generated lockfiles from a file list so they do | ||
| // not become a coverage obligation. | ||
| func filterReviewableFiles(files []string) []string { | ||
| out := make([]string, 0, len(files)) | ||
| for _, file := range files { | ||
| if isGeneratedLockfile(file) { | ||
| continue | ||
| } | ||
| out = append(out, file) | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| func buildReviewerCoverage(selected []llm.SelectedAgent, results []llm.Findings, failures []ReviewerFailure, changedFiles []string, toolEvidence ...map[string]*llm.ReviewerToolEvidence) []reviewplan.ReviewerCoverageSummary { | ||
| if len(selected) == 0 && len(changedFiles) == 0 { | ||
| return nil | ||
| } | ||
| // Generated lockfiles are not a review obligation: exclude them so neither a | ||
| // reviewer that skips one nor an unassigned lockfile blocks approval. | ||
| changedFiles = filterReviewableFiles(changedFiles) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. U-S1: the exemption is applied at the accounting layer only, so the repo now answers "what is in the coverage universe?" in two places that disagree. Choosing the accounting layer is the right call — filtering Reply inline to this comment. |
||
| resultByAgent := make(map[string]llm.Findings, len(results)) | ||
| for _, result := range results { | ||
| resultByAgent[result.AgentID] = result | ||
|
|
@@ -2452,7 +2498,9 @@ func buildReviewerCoverage(selected []llm.SelectedAgent, results []llm.Findings, | |
| assigned := map[string]bool{} | ||
| out := make([]reviewplan.ReviewerCoverageSummary, 0, len(selected)+1) | ||
| for _, agent := range selected { | ||
| scope := reviewerAssignmentScope(agent, changedFiles) | ||
| // A lockfile explicitly assigned to an agent is exempt too — the scope | ||
| // is what the reviewer is held to, and lockfiles are not reviewable. | ||
| scope := filterReviewableFiles(reviewerAssignmentScope(agent, changedFiles)) | ||
| for _, file := range scope { | ||
| assigned[file] = true | ||
| } | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
U-L2 (naming, not behavior): the degrade-don't-reject decision is sound and explicitly commented, so the swallowed-error rule is satisfied. Two labels are now stale around it.
decodeCoverageStringssits next todecodeCoverageFiles(line 335) with a near-identical name and the opposite error contract — one rejects, one silently repairs;decodeCoverageConstraintswould carry the distinction, since constraints are its only caller (line 275). AndDefaultFindingsConstraintLimits's doc comment still reads "limits enforced by DecodeFindings" (line 41), which now overstates: entries are capped and truncated, never enforced by failure. Prompt text at prompts.go:716-717 can keep saying "must" — instructing the model harder than the validator fails is deliberate.Reply inline to this comment.