diff --git a/web/packages/agenta-annotation-ui/src/components/AnnotationQueuesView/cells/EvaluatorNamesCell.tsx b/web/packages/agenta-annotation-ui/src/components/AnnotationQueuesView/cells/EvaluatorNamesCell.tsx index 22eaefca27..159d9e9112 100644 --- a/web/packages/agenta-annotation-ui/src/components/AnnotationQueuesView/cells/EvaluatorNamesCell.tsx +++ b/web/packages/agenta-annotation-ui/src/components/AnnotationQueuesView/cells/EvaluatorNamesCell.tsx @@ -9,48 +9,80 @@ interface EvaluatorNamesCellProps { runId: string | undefined } +interface EvaluatorEntry { + evaluatorId: string | null + evaluatorRevisionId: string | null + evaluatorSlug: string | null +} + +function getEvaluatorEntryKey(entry: EvaluatorEntry) { + return entry.evaluatorId ?? entry.evaluatorRevisionId ?? entry.evaluatorSlug ?? "unknown" +} + /** * Cell that renders evaluator names for a queue. * * Resolution chain: * 1. Read evaluation run data via runId (batch-fetched) - * 2. Extract evaluator workflow IDs from annotation step references + * 2. Extract evaluator workflow IDs + slugs from annotation step references * 3. Read evaluator entity data for each ID (batch-fetched) - * 4. Display evaluator names + * 4. Display evaluator names, falling back to slug from step refs then truncated ID */ const EvaluatorNamesCell = memo(function EvaluatorNamesCell({runId}: EvaluatorNamesCellProps) { if (!runId) return null return }) -/** Reads evaluation run → extracts evaluator IDs → delegates to name resolution */ +/** Reads evaluation run → extracts evaluator IDs + slugs → delegates to name resolution */ const EvaluatorIdsBridge = memo(function EvaluatorIdsBridge({runId}: {runId: string}) { const rawQuery = useAtomValue(evaluationRunMolecule.atoms.query(runId)) - const evaluatorIds = useAtomValue(evaluationRunMolecule.selectors.evaluatorIds(runId)) + const columnDefs = useAtomValue(evaluationRunMolecule.selectors.annotationColumnDefs(runId)) + + // Deduplicate by evaluatorId, preserving order + const evaluatorEntries: EvaluatorEntry[] = [] + const seen = new Set() + for (const col of columnDefs) { + const key = col.evaluatorId ?? col.evaluatorRevisionId ?? col.evaluatorSlug + if (key && !seen.has(key)) { + seen.add(key) + evaluatorEntries.push({ + evaluatorId: col.evaluatorId, + evaluatorRevisionId: col.evaluatorRevisionId, + evaluatorSlug: col.evaluatorSlug, + }) + } + } - if (rawQuery.isPending && evaluatorIds.length === 0) { + if (rawQuery.isPending && evaluatorEntries.length === 0) { return } - if (evaluatorIds.length === 0) return null + if (evaluatorEntries.length === 0) return null - return + return }) -/** Resolves evaluator names from IDs and renders tags */ +/** Resolves evaluator names from IDs+slugs and renders tags */ const EvaluatorNamesList = memo(function EvaluatorNamesList({ - evaluatorIds, + evaluatorEntries, }: { - evaluatorIds: string[] + evaluatorEntries: EvaluatorEntry[] }) { - const names = evaluatorIds.map((id) => ) + const names = evaluatorEntries.map((entry) => ( + + )) if (names.length <= 2) { return
{names}
} const visible = names.slice(0, 2) - const remainingIds = evaluatorIds.slice(2) + const remainingEntries = evaluatorEntries.slice(2) return (
@@ -58,32 +90,57 @@ const EvaluatorNamesList = memo(function EvaluatorNamesList({ - {remainingIds.map((id) => ( - + {remainingEntries.map((entry) => ( + ))}
} > - +{remainingIds.length} + +{remainingEntries.length} ) }) /** Single evaluator name tag — subscribes to evaluator entity for its name */ -const EvaluatorNameTag = memo(function EvaluatorNameTag({evaluatorId}: {evaluatorId: string}) { - const name = useAtomValue(workflowMolecule.selectors.name(evaluatorId)) - const slug = useAtomValue(workflowMolecule.selectors.slug(evaluatorId)) +const EvaluatorNameTag = memo(function EvaluatorNameTag({ + evaluatorId, + evaluatorRevisionId, + fallbackSlug, +}: { + evaluatorId: string | null + evaluatorRevisionId: string | null + fallbackSlug: string | null +}) { + const lookupId = evaluatorRevisionId ?? evaluatorId ?? "" + const name = useAtomValue(workflowMolecule.selectors.name(lookupId)) + const slug = useAtomValue(workflowMolecule.selectors.slug(lookupId)) + const fallbackId = evaluatorId ?? lookupId - return {name || slug || evaluatorId.slice(0, 8)} + return {name || fallbackSlug || slug || fallbackId.slice(0, 8)} }) /** Single evaluator name span (for tooltip) */ -const EvaluatorNameSpan = memo(function EvaluatorNameSpan({evaluatorId}: {evaluatorId: string}) { - const name = useAtomValue(workflowMolecule.selectors.name(evaluatorId)) - const slug = useAtomValue(workflowMolecule.selectors.slug(evaluatorId)) +const EvaluatorNameSpan = memo(function EvaluatorNameSpan({ + evaluatorId, + evaluatorRevisionId, + fallbackSlug, +}: { + evaluatorId: string | null + evaluatorRevisionId: string | null + fallbackSlug: string | null +}) { + const lookupId = evaluatorRevisionId ?? evaluatorId ?? "" + const name = useAtomValue(workflowMolecule.selectors.name(lookupId)) + const slug = useAtomValue(workflowMolecule.selectors.slug(lookupId)) + const fallbackId = evaluatorId ?? lookupId - return {name || slug || evaluatorId.slice(0, 8)} + return {name || fallbackSlug || slug || fallbackId.slice(0, 8)} }) export default EvaluatorNamesCell diff --git a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/AnnotationFormField.tsx b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/AnnotationFormField.tsx index 35de135008..5037c7470c 100644 --- a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/AnnotationFormField.tsx +++ b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/AnnotationFormField.tsx @@ -194,11 +194,12 @@ const StringField = memo(function StringField({ className={`flex flex-col gap-1 playground-property-control ${readOnly ? READONLY_CLASS : ""}`} > {label} - onChange(e.target.value || null)} disabled={isDisabled} placeholder="Enter value" + autoSize={{minRows: 2, maxRows: 6}} /> ) diff --git a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/ScenarioListView.tsx b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/ScenarioListView.tsx index b51f7caec5..d486925518 100644 --- a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/ScenarioListView.tsx +++ b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/ScenarioListView.tsx @@ -397,12 +397,14 @@ const AnnotationColumnHeader = memo(function AnnotationColumnHeader({ }: { def: AnnotationColumnDef }) { - const name = useAtomValue(workflowMolecule.selectors.name(def.evaluatorId ?? "")) - const slug = useAtomValue(workflowMolecule.selectors.slug(def.evaluatorId ?? "")) - const displayName = name || slug || def.evaluatorSlug || def.columnName || def.stepKey + const evaluatorLookupId = def.evaluatorRevisionId ?? def.evaluatorId ?? "" + const name = useAtomValue(workflowMolecule.selectors.name(evaluatorLookupId)) + const slug = useAtomValue(workflowMolecule.selectors.slug(evaluatorLookupId)) + const displaySlug = def.evaluatorSlug || slug + const displayName = name || displaySlug || def.columnName || def.stepKey return ( - + {displayName} ) @@ -423,9 +425,11 @@ const AnnotationGroupHeader = memo(function AnnotationGroupHeader({ isCollapsed: boolean onToggle: () => void }) { - const name = useAtomValue(workflowMolecule.selectors.name(def.evaluatorId ?? "")) - const slug = useAtomValue(workflowMolecule.selectors.slug(def.evaluatorId ?? "")) - const displayName = name || slug || def.evaluatorSlug || def.columnName || def.stepKey + const evaluatorLookupId = def.evaluatorRevisionId ?? def.evaluatorId ?? "" + const name = useAtomValue(workflowMolecule.selectors.name(evaluatorLookupId)) + const slug = useAtomValue(workflowMolecule.selectors.slug(evaluatorLookupId)) + const displaySlug = def.evaluatorSlug || slug + const displayName = name || displaySlug || def.columnName || def.stepKey const handleClick = useCallback( (e: React.MouseEvent) => { @@ -1166,16 +1170,17 @@ function resolveExportColumnLabel( if (def) { if (def.columnType === "annotation") { const annotationDef = def.annotationDef - const name = annotationDef.evaluatorId - ? store.get(workflowMolecule.selectors.name(annotationDef.evaluatorId)) + const evaluatorLookupId = annotationDef.evaluatorRevisionId ?? annotationDef.evaluatorId + const name = evaluatorLookupId + ? store.get(workflowMolecule.selectors.name(evaluatorLookupId)) : null - const slug = annotationDef.evaluatorId - ? store.get(workflowMolecule.selectors.slug(annotationDef.evaluatorId)) + const slug = evaluatorLookupId + ? store.get(workflowMolecule.selectors.slug(evaluatorLookupId)) : null return ( name || - slug || annotationDef.evaluatorSlug || + slug || annotationDef.columnName || annotationDef.stepKey ) diff --git a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/index.tsx b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/index.tsx index bd569a776a..fea878c47d 100644 --- a/web/packages/agenta-annotation-ui/src/components/AnnotationSession/index.tsx +++ b/web/packages/agenta-annotation-ui/src/components/AnnotationSession/index.tsx @@ -186,7 +186,7 @@ const AnnotationSession = ({ try { await addScenariosToTestset({ targetMode: params.mode === "new" ? "new" : "existing", - commitMessage: params.message, + commitMessage: params.message ?? "", newTestsetName: params.entityName, newTestsetSlug: params.entitySlug, }) diff --git a/web/packages/agenta-annotation/src/state/controllers/annotationFormController.ts b/web/packages/agenta-annotation/src/state/controllers/annotationFormController.ts index 1df371d4e4..edff0214b8 100644 --- a/web/packages/agenta-annotation/src/state/controllers/annotationFormController.ts +++ b/web/packages/agenta-annotation/src/state/controllers/annotationFormController.ts @@ -683,7 +683,7 @@ function normalizeResolvedEvaluator(ref: EvaluatorStepRef, evaluator: Workflow): const variantId = evaluator.workflow_variant_id ?? evaluator.variant_id ?? ref.variantId ?? null return { ...evaluator, - slug: evaluator.slug ?? ref.slug ?? null, + slug: ref.slug ?? evaluator.slug ?? null, workflow_id: evaluator.workflow_id ?? ref.workflowId ?? null, workflow_variant_id: variantId, variant_id: variantId, diff --git a/web/packages/agenta-annotation/src/state/controllers/annotationSessionController.ts b/web/packages/agenta-annotation/src/state/controllers/annotationSessionController.ts index aa35fcf44a..0219ab8a9c 100644 --- a/web/packages/agenta-annotation/src/state/controllers/annotationSessionController.ts +++ b/web/packages/agenta-annotation/src/state/controllers/annotationSessionController.ts @@ -506,6 +506,12 @@ const evaluatorRevisionIdsAtom = atom((get) => { return get(evaluationRunMolecule.selectors.evaluatorRevisionIds(runId)) }) +function deriveEvaluatorSlugFromStepKey(stepKey: string | null | undefined): string | null { + if (!stepKey) return null + const parts = stepKey.split(".").filter(Boolean) + return parts.at(-1) ?? null +} + /** * Ordered evaluator references from annotation steps. * Each entry preserves the queue's pinned evaluator revision while keeping the @@ -524,6 +530,8 @@ const evaluatorStepRefsAtom = atom((get) => { revisionId: step.references?.evaluator_revision?.id ?? null, slug: step.references?.evaluator?.slug ?? + step.references?.evaluator_variant?.slug ?? + deriveEvaluatorSlugFromStepKey(step.key) ?? step.references?.evaluator_revision?.slug ?? null, stepKey: step.key ?? null, @@ -545,6 +553,8 @@ const testsetSyncEvaluatorsAtom = atom((get) => { const name = evaluatorEntity?.name?.trim() || null const slug = step.references?.evaluator?.slug ?? + step.references?.evaluator_variant?.slug ?? + deriveEvaluatorSlugFromStepKey(step.key) ?? evaluatorEntity?.slug ?? step.references?.evaluator_revision?.slug ?? workflowId @@ -753,11 +763,14 @@ const META_KEYS = new Set(["tags", "meta"]) type TestcaseColumnGroup = "input" | "output" | "expected" function getAnnotationDisplayTitle(get: Getter, def: AnnotationColumnDef): string { - const evaluator = def.evaluatorId ? get(workflowMolecule.selectors.data(def.evaluatorId)) : null + const evaluatorLookupId = def.evaluatorRevisionId ?? def.evaluatorId + const evaluator = evaluatorLookupId + ? get(workflowMolecule.selectors.data(evaluatorLookupId)) + : null return ( evaluator?.name?.trim() || - evaluator?.slug?.trim() || def.evaluatorSlug?.trim() || + evaluator?.slug?.trim() || def.columnName?.trim() || def.stepKey?.trim() || "" @@ -2444,20 +2457,29 @@ async function waitForStoreAtomValue( function resolveScenarioIdsForAddToTestset(get: Getter): string[] { const scope = get(addToTestsetScopeAtom) + const queueKind = get(queueKindAtom) - if (scope === "all") { - return get(scenarioIdsAtom) - } - - if (scope === "complete") { + if (queueKind === "testcases" && (scope === "all" || scope === "complete")) { const completed = get(completedScenarioIdsAtom) const records = get(scenarioRecordsAtom) return get(scenarioIdsAtom).filter((id) => isScenarioCompleted(id, completed, records)) } + if (scope === "all" || scope === "complete") { + return get(scenarioIdsAtom) + } return get(addToTestsetScenarioIdsAtom) } +function resolveCompletedScenarioIdsForAnnotationExport( + get: Getter, + scenarioIds: string[], +): Set { + const completed = get(completedScenarioIdsAtom) + const records = get(scenarioRecordsAtom) + return new Set(scenarioIds.filter((id) => isScenarioCompleted(id, completed, records))) +} + function extractExistingColumns( rows: {data?: Record | null}[] | null | undefined, ): Set { @@ -2988,7 +3010,8 @@ const addScenariosToTestsetAtom = atom( }), queueId, evaluators, - requireAnnotationOutputScenarioIds: new Set(), + requireAnnotationOutputScenarioIds: + resolveCompletedScenarioIdsForAnnotationExport(get, scenarioIds), setProcessed, }) : await prepareTestcaseExportRows({ @@ -3096,8 +3119,14 @@ const canSyncToTestsetAtom = atom((get) => { }) const canAddToTestsetAtom = atom((get) => { + const queueKind = get(queueKindAtom) const ids = get(scenarioIdsAtom) - return ids.length > 0 + if (ids.length === 0) return false + if (queueKind === "traces") return true + + const completed = get(completedScenarioIdsAtom) + const records = get(scenarioRecordsAtom) + return ids.some((id) => isScenarioCompleted(id, completed, records)) }) async function buildTestsetSyncPreviewForSession(get: Getter) { diff --git a/web/packages/agenta-annotation/src/state/testsetSync.ts b/web/packages/agenta-annotation/src/state/testsetSync.ts index 8dfd20160b..2ecfc92b25 100644 --- a/web/packages/agenta-annotation/src/state/testsetSync.ts +++ b/web/packages/agenta-annotation/src/state/testsetSync.ts @@ -485,6 +485,7 @@ export function buildTestcaseExportRows(params: TestcaseExportRowBuilderParams): evaluators: params.evaluators, queueId: params.queueId, }) + if (entries.length === 0) continue applyAnnotationOutputEntries(data, entries) diff --git a/web/packages/agenta-annotation/src/state/types.ts b/web/packages/agenta-annotation/src/state/types.ts index 5f2c4c2e0e..3c8e82aad2 100644 --- a/web/packages/agenta-annotation/src/state/types.ts +++ b/web/packages/agenta-annotation/src/state/types.ts @@ -72,7 +72,9 @@ export interface AnnotationColumnDef { path: string | null /** Evaluator workflow ID from the annotation step's references */ evaluatorId: string | null - /** Evaluator slug from the annotation step's references */ + /** Evaluator revision ID from the annotation step's references */ + evaluatorRevisionId: string | null + /** Evaluator slug from step refs, step key, or mapping column fallback */ evaluatorSlug: string | null } diff --git a/web/packages/agenta-entities/src/evaluationRun/state/molecule.ts b/web/packages/agenta-entities/src/evaluationRun/state/molecule.ts index 7b419f107e..c47a1d3c4f 100644 --- a/web/packages/agenta-entities/src/evaluationRun/state/molecule.ts +++ b/web/packages/agenta-entities/src/evaluationRun/state/molecule.ts @@ -263,6 +263,33 @@ interface ScenarioStepsKey { scenarioId: string } +function normalizeString(value: unknown): string | null { + if (typeof value !== "string") return null + const trimmed = value.trim() + return trimmed.length > 0 ? trimmed : null +} + +function getReferenceValue( + step: EvaluationRunDataStep, + refName: string, + key: "id" | "slug", +): string | null { + return normalizeString(step.references?.[refName]?.[key]) +} + +function stripOutputSuffix(value: string | null): string | null { + if (!value) return null + const parts = value.split(".").filter(Boolean) + if (parts.length < 2) return value + return parts.slice(0, -1).join(".") || value +} + +function lastSegment(value: string | null): string | null { + if (!value) return null + const parts = value.split(".").filter(Boolean) + return parts.at(-1) ?? value +} + // ============================================================================ // CONVENIENCE SELECTORS (compound derived data) // ============================================================================ @@ -278,9 +305,25 @@ export interface AnnotationColumnDef { columnKind: string | null path: string | null evaluatorId: string | null + evaluatorRevisionId: string | null evaluatorSlug: string | null } +function getAnnotationEvaluatorSlug( + step: EvaluationRunDataStep, + mapping: EvaluationRunDataMapping, +): string | null { + const candidates = [ + getReferenceValue(step, "evaluator", "slug"), + getReferenceValue(step, "evaluator_variant", "slug"), + lastSegment(normalizeString(step.key)), + stripOutputSuffix(normalizeString(mapping.column?.name)), + getReferenceValue(step, "evaluator_revision", "slug"), + ] + + return candidates.find((candidate) => Boolean(candidate)) ?? null +} + /** * Annotation column definitions derived from run annotation steps + mappings. * Joins mappings to steps by key and extracts evaluator references. @@ -301,8 +344,9 @@ const annotationColumnDefsAtomFamily = atomFamily((runId: string) => columnName: m.column?.name ?? null, columnKind: m.column?.kind ?? null, path: m.step!.path ?? null, - evaluatorId: step.references?.evaluator?.id ?? null, - evaluatorSlug: step.references?.evaluator?.slug ?? null, + evaluatorId: getReferenceValue(step, "evaluator", "id"), + evaluatorRevisionId: getReferenceValue(step, "evaluator_revision", "id"), + evaluatorSlug: getAnnotationEvaluatorSlug(step, m), } }) }),