Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions internal/llm/contracts.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,10 +272,7 @@ func DecodeFindings(data []byte, opts FindingsOptions) (Findings, error) {
if err := validateCoverageFileDisjoint(inspected, skipped); err != nil {
return Findings{}, err
}
constraints, err := decodeCoverageStrings("constraints", wire.Constraints)
if err != nil {
return Findings{}, err
}
constraints := decodeCoverageStrings(wire.Constraints)

result := Findings{
AgentID: wire.AgentID,
Expand Down Expand Up @@ -352,27 +349,31 @@ func decodeCoverageFiles(name string, files []string, changedFiles map[string]bo
return out, nil
}

func decodeCoverageStrings(name string, values []string) ([]string, error) {
// decodeCoverageStrings cleans reviewer coverage constraints. These are
// informational notes ("couldn't verify X against source-of-truth docs"), not
// a contract, so a malformed or verbose entry is degraded — the count is
// capped, an over-long entry is truncated, and empties/duplicates are dropped —
// rather than failing the decode. Failing here sinks the whole reviewer as
// "completed without a result file" and blocks approval on an otherwise-clean
// review, which a single legitimate ~300-rune constraint once did.
func decodeCoverageStrings(values []string) []string {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

U-L2 (naming, not behavior): the degrade-don't-reject decision is sound and explicitly commented, so the swallowed-error rule is satisfied. Two labels are now stale around it. decodeCoverageStrings sits next to decodeCoverageFiles (line 335) with a near-identical name and the opposite error contract — one rejects, one silently repairs; decodeCoverageConstraints would carry the distinction, since constraints are its only caller (line 275). And DefaultFindingsConstraintLimits's doc comment still reads "limits enforced by DecodeFindings" (line 41), which now overstates: entries are capped and truncated, never enforced by failure. Prompt text at prompts.go:716-717 can keep saying "must" — instructing the model harder than the validator fails is deliberate.

Reply inline to this comment.

if len(values) > defaultMaxCoverageConstraints {
return nil, fmt.Errorf("llm: %s cap exceeded", name)
values = values[:defaultMaxCoverageConstraints]
}
out := make([]string, 0, len(values))
seen := map[string]bool{}
for _, value := range values {
if utf8.RuneCountInString(value) > defaultMaxCoverageConstraintRunes {
return nil, fmt.Errorf("llm: %s entry length out of bounds", name)
value = truncateRunes(value, defaultMaxCoverageConstraintRunes)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

decodeCoverageStrings truncates an over-long constraint to defaultMaxCoverageConstraintRunes (300) and then appends "..." via truncateRunes, so the value that survives sanitize() can be up to 303 runes — 3 runes past the documented/enforced cap (DefaultFindingsConstraintLimits().MaxRunesPerEntry = 300, which the orchestrator prompt tells reviewers is the hard limit). TestDecodeFindingsConstraintRuneBoundaries only asserts the result ends with "...", not that it respects the rune cap, so this drift isn't caught. Fix by reserving the ellipsis width before truncating, e.g. truncateRunes(value, defaultMaxCoverageConstraintRunes-3), or by post-truncating the whole result (including the suffix) down to defaultMaxCoverageConstraintRunes.

Reply inline to this comment.

}
value = sanitize(value)
if strings.TrimSpace(value) == "" {
return nil, fmt.Errorf("llm: %s entries must be non-empty", name)
}
if seen[value] {
return nil, fmt.Errorf("llm: duplicate %s entry %q", name, value)
if strings.TrimSpace(value) == "" || seen[value] {
continue
}
seen[value] = true
out = append(out, value)
}
return out, nil
return out
}

func validateCoverageFileDisjoint(inspected, skipped []string) error {
Expand Down
69 changes: 55 additions & 14 deletions internal/llm/contracts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,6 @@ func TestDecodeFindings(t *testing.T) {
assertFindingsError(t, baseOpts, `{"schema_version":1,"agent_id":"agent-1","inspected_files":["main.go","main.go"],"findings":[]}`, "duplicate inspected_files")
assertFindingsError(t, baseOpts, `{"schema_version":1,"agent_id":"agent-1","inspected_files":["main.go"],"skipped_files":["other.go"],"findings":[]}`, "skipped_files entry")
assertFindingsError(t, baseOpts, `{"schema_version":1,"agent_id":"agent-1","inspected_files":["main.go"],"skipped_files":["main.go"],"findings":[]}`, "both inspected and skipped")
assertFindingsError(t, baseOpts, `{"schema_version":1,"agent_id":"agent-1","inspected_files":["main.go"],"constraints":[" "],"findings":[]}`, "constraints")
assertFindingsError(t, baseOpts, `{"schema_version":1,"agent_id":"agent-1","inspected_files":["main.go"],"constraints":["one","two","three","four","five","six","seven","eight","nine","ten","eleven"],"findings":[]}`, "constraints cap exceeded")
assertFindingsError(t, baseOpts, findingsFixture(`"schema_version":2,"agent_id":"agent-1","findings":[]`), "schema_version")
assertFindingsError(t, baseOpts, findingsFixture(`"schema_version":1,"agent_id":"agent-1","findings":[],"extra":true`), "unknown field")
assertFindingsError(t, baseOpts, findingsFixture(`"schema_version":1,"agent_id":"missing","findings":[]`), "unknown findings agent")
Expand Down Expand Up @@ -187,35 +185,78 @@ func TestDecodeFindingsConstraintRuneBoundaries(t *testing.T) {
multibyteAtLimit := strings.Repeat("界", limits.MaxRunesPerEntry)

for _, tt := range []struct {
name string
constraint string
wantErr string
wantClean bool
name string
constraint string
wantTruncated bool
}{
{name: "marker opening at limit", constraint: markerAtLimit, wantClean: true},
{name: "marker opening over limit", constraint: markerAtLimit + "x", wantErr: "constraints entry length"},
{name: "marker opening at limit", constraint: markerAtLimit},
// Over-limit entries are truncated, not rejected: a verbose (but valid)
// coverage note must not fail the whole reviewer and block approval.
{name: "marker opening over limit", constraint: markerAtLimit + "x", wantTruncated: true},
{name: "multibyte at limit", constraint: multibyteAtLimit},
{name: "multibyte over limit", constraint: multibyteAtLimit + "界", wantErr: "constraints entry length"},
{name: "multibyte over limit", constraint: multibyteAtLimit + "界", wantTruncated: true},
} {
t.Run(tt.name, func(t *testing.T) {
got, err := decodeFindingsWithConstraint(t, tt.constraint)
if tt.wantErr != "" {
assertErrContains(t, err, tt.wantErr)
return
}
if err != nil {
t.Fatalf("DecodeFindings: %v", err)
}
if len(got.Constraints) != 1 {
t.Fatalf("constraints = %#v, want one value", got.Constraints)
}
if tt.wantClean && strings.Contains(got.Constraints[0], markerOpening) {
if strings.Contains(got.Constraints[0], markerOpening) {
t.Fatalf("constraint = %q, want sanitized marker opening", got.Constraints[0])
}
if tt.wantTruncated && !strings.HasSuffix(got.Constraints[0], "...") {
t.Fatalf("constraint = %q, want truncated (ends with ...), not rejected", got.Constraints[0])
}
})
}
}

func TestDecodeFindingsConstraintsDegradeInsteadOfFailing(t *testing.T) {
// Coverage constraints are informational: an over-count list, a
// whitespace-only entry, and a duplicate are cleaned rather than failing
// the whole reviewer (which surfaced as "completed without a result file"
// and blocked approval on an otherwise-clean review).
payload := map[string]any{
"schema_version": 1,
"agent_id": "agent-1",
"inspected_files": []string{"main.go"},
"constraints": []string{
"a", " ", "b", "a", // whitespace-only and duplicate, within the cap
"c", "d", "e", "f", "g", "h", "i", "j", // pushes the list over the cap of 10
},
"findings": []any{},
}
data, err := json.Marshal(payload)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
got, err := DecodeFindings(data, FindingsOptions{
KnownAgents: map[string]bool{"agent-1": true},
ChangedFiles: map[string]bool{"main.go": true},
NewFindingID: newIDQueue("f-1").next,
})
if err != nil {
t.Fatalf("DecodeFindings degraded to an error: %v", err)
}
lim := DefaultFindingsConstraintLimits()
if len(got.Constraints) > lim.MaxEntries {
t.Fatalf("constraints = %#v, want ≤ %d after capping", got.Constraints, lim.MaxEntries)
}
seen := map[string]bool{}
for _, c := range got.Constraints {
if strings.TrimSpace(c) == "" {
t.Fatalf("kept a whitespace-only constraint: %#v", got.Constraints)
}
if seen[c] {
t.Fatalf("kept a duplicate constraint: %#v", got.Constraints)
}
seen[c] = true
}
}

func decodeFindingsWithConstraint(t *testing.T, constraint string) (Findings, error) {
t.Helper()
payload := map[string]any{
Expand Down
50 changes: 49 additions & 1 deletion internal/pipeline/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -2437,10 +2437,56 @@ func reviewerToolEvidenceByAgent(sessions []sessionDraft) map[string]*llm.Review
return out

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

File-level note: internal/pipeline/pipeline.go

docs/checkout-native-review-contract.md is the documented source of truth for reviewer coverage semantics: it defines "readable files" as "all changed files in the workbench" and states unassigned/skipped files "must not turn into a clean approval silently." This diff's buildReviewerCoverage now filters generated lockfiles out of changedFiles (and each agent's scope) before computing coverage, so lockfiles are silently excluded from the readable-files universe the doc describes — but the doc's coverage-status section (lines ~337-354) was not updated to mention the lockfile exemption or the new filterReviewableFiles/isGeneratedLockfile step. A future agent or engineer reading that doc to understand what "complete" coverage guarantees will get a materially misleading picture of what the harness actually enforces. Update docs/checkout-native-review-contract.md to document the lockfile exemption alongside the existing coverage-status definitions.

Reply inline to this comment.

}

// generatedLockfiles are dependency lockfiles: machine-written by a package
// manager and reviewed (if at all) through the manifest change that produced
// them, never line by line. A reviewer that skips one is behaving correctly, so
// they are excluded from the coverage universe — otherwise a skipped lockfile
// marks the reviewer incomplete_skipped and blocks approval on an otherwise
// clean review. (A real PR stalled exactly this way: a Cargo.lock churned by a
// dependency bump was the only file left "unreviewed".)
var generatedLockfiles = map[string]bool{
"Cargo.lock": true,
"package-lock.json": true,
"npm-shrinkwrap.json": true,
"yarn.lock": true,
"pnpm-lock.yaml": true,
"bun.lockb": true,
"go.sum": true,
"Gemfile.lock": true,
"poetry.lock": true,
"Pipfile.lock": true,
"composer.lock": true,
"Podfile.lock": true,
"flake.lock": true,
"mix.lock": true,
}

// isGeneratedLockfile reports whether path is a dependency lockfile a reviewer
// is not expected to read line by line.
func isGeneratedLockfile(path string) bool {
return generatedLockfiles[filepath.Base(path)]
}

// filterReviewableFiles drops generated lockfiles from a file list so they do
// not become a coverage obligation.
func filterReviewableFiles(files []string) []string {
out := make([]string, 0, len(files))
for _, file := range files {
if isGeneratedLockfile(file) {
continue
}
out = append(out, file)
}
return out
}

func buildReviewerCoverage(selected []llm.SelectedAgent, results []llm.Findings, failures []ReviewerFailure, changedFiles []string, toolEvidence ...map[string]*llm.ReviewerToolEvidence) []reviewplan.ReviewerCoverageSummary {
if len(selected) == 0 && len(changedFiles) == 0 {
return nil
}
// Generated lockfiles are not a review obligation: exclude them so neither a
// reviewer that skips one nor an unassigned lockfile blocks approval.
changedFiles = filterReviewableFiles(changedFiles)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

U-S1: the exemption is applied at the accounting layer only, so the repo now answers "what is in the coverage universe?" in two places that disagree. ensureSelectedGlobCoverage (pipeline.go:1561-1594) still iterates the unfiltered changedFiles and force-assigns an uncovered lockfile to the first glob-matching agent, widening that agent's AllowedFiles/Files. The reviewer is therefore still told to cover Cargo.lock in its prompt scope, and the obligation is then silently discarded here. A smaller observable version of the same split: entry.Scope is filtered (line 2503) but entry.InspectedFiles (line 2524) is not, so a reviewer that did read a lockfile emits a coverage row listing an inspected file that is not in its scope.

Choosing the accounting layer is the right call — filtering prepared.changedFiles wholesale would also drop lockfiles from FindingsOptions.ChangedFiles and make a legitimate finding on a swapped registry URL undecodable — so the fix is not to move the filter but to give the universe one owner. Concretely: export the predicate as a single helper used both here and in ensureSelectedGlobCoverage's covered/assignment loops (e.g. have that function skip files where isGeneratedLockfile(file) before looking for an owner), and apply the same filter to entry.InspectedFiles so scope and coverage rows are drawn from one set. Without that, the next place that computes an obligation from raw changedFiles reintroduces the stall this PR fixes.

Reply inline to this comment.

resultByAgent := make(map[string]llm.Findings, len(results))
for _, result := range results {
resultByAgent[result.AgentID] = result
Expand All @@ -2452,7 +2498,9 @@ func buildReviewerCoverage(selected []llm.SelectedAgent, results []llm.Findings,
assigned := map[string]bool{}
out := make([]reviewplan.ReviewerCoverageSummary, 0, len(selected)+1)
for _, agent := range selected {
scope := reviewerAssignmentScope(agent, changedFiles)
// A lockfile explicitly assigned to an agent is exempt too — the scope
// is what the reviewer is held to, and lockfiles are not reviewable.
scope := filterReviewableFiles(reviewerAssignmentScope(agent, changedFiles))
for _, file := range scope {
assigned[file] = true
}
Expand Down
25 changes: 25 additions & 0 deletions internal/pipeline/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5097,6 +5097,31 @@ func TestBuildReviewerCoverageStatuses(t *testing.T) {
}
}

func TestBuildReviewerCoverageExemptsGeneratedLockfiles(t *testing.T) {
// A reviewer that inspects the real change and skips only the churned
// Cargo.lock is complete, not incomplete_skipped — a lockfile is not a
// review obligation. An unassigned lockfile (yarn.lock) likewise must not
// surface as incomplete_unassigned and block approval.
selected := []llm.SelectedAgent{
{AgentID: "rust:impl", Files: []string{"main.go", "Cargo.lock"}, AllowedFiles: []string{"main.go", "Cargo.lock"}},
}
results := []llm.Findings{
{AgentID: "rust:impl", InspectedFiles: []string{"main.go"}, SkippedFiles: []string{"Cargo.lock"}},
}
got := buildReviewerCoverage(selected, results, nil, []string{"main.go", "Cargo.lock", "yarn.lock"})
if len(got) != 1 {
t.Fatalf("coverage = %#v, want a single reviewer entry (no lockfile coverage rows)", got)
}
if got[0].AgentID != "rust:impl" || got[0].Status != reviewerCoverageCompleteConstrained {
t.Fatalf("coverage = %#v, want rust:impl complete_constrained", got)
}
if len(got[0].SkippedFiles) != 0 {
t.Fatalf("skipped files = %#v, want none (Cargo.lock is exempt from coverage)", got[0].SkippedFiles)
}
// complete_constrained is an approvable status; had Cargo.lock counted, this
// would be reviewerCoverageIncompleteSkipped, which blocks approval.
}

func TestBuildReviewerCoverageUsesTypedToolEvidenceInsteadOfModelConstraint(t *testing.T) {
got := buildReviewerCoverage(
[]llm.SelectedAgent{{AgentID: "harness:reviewer", Files: []string{"main.go"}}},
Expand Down
Loading