Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions web/packages/agenta-annotation/src/state/testsetSync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ function buildAnnotationOutputEntries(params: {
annotations: Annotation[]
evaluators: TestsetSyncEvaluator[]
queueId: string
existingData?: Record<string, unknown> | null
}): {columnKey: string; outputs: Record<string, unknown>}[] {
const entries: {columnKey: string; outputs: Record<string, unknown>}[] = []
const slugByEvaluatorId = buildSlugByEvaluatorId(params.evaluators)
Expand All @@ -220,16 +221,20 @@ function buildAnnotationOutputEntries(params: {

if (!selection.annotation || selection.conflictCode) continue

const outputs = getAnnotationOutputs(selection.annotation)
if (Object.keys(outputs).length === 0) continue

const columnKey = getTestsetSyncEvaluatorColumnKey({
evaluator,
annotation: selection.annotation,
slugByEvaluatorId,
})
if (!columnKey) continue

const outputs = getAnnotationOutputs(selection.annotation)
const hasExistingColumn =
isPlainRecord(params.existingData?.[columnKey]) &&
Object.keys(params.existingData[columnKey]).length > 0

if (Object.keys(outputs).length === 0 && !hasExistingColumn) continue

entries.push({
columnKey,
outputs,
Expand All @@ -248,9 +253,7 @@ function applyAnnotationOutputEntries(
entries: {columnKey: string; outputs: Record<string, unknown>}[],
): void {
for (const entry of entries) {
const existingValue = data[entry.columnKey]
data[entry.columnKey] = {
...(isPlainRecord(existingValue) ? existingValue : {}),
...entry.outputs,
}
}
Expand Down Expand Up @@ -503,6 +506,7 @@ export function buildTestcaseExportRows(params: TestcaseExportRowBuilderParams):
annotations,
evaluators: params.evaluators,
queueId: params.queueId,
existingData: testcase.data,
})
if (entries.length === 0) continue

Expand Down Expand Up @@ -645,16 +649,20 @@ export function buildTestsetSyncPreview(params: BuildTestsetSyncPreviewParams):

if (!selection.annotation) continue

const outputs = getAnnotationOutputs(selection.annotation)

if (Object.keys(outputs).length === 0) continue
const columnKey = getTestsetSyncEvaluatorColumnKey({
evaluator,
annotation: selection.annotation,
slugByEvaluatorId,
})
if (!columnKey) continue

const outputs = getAnnotationOutputs(selection.annotation)
const hasExistingColumn =
isPlainRecord(testcase.data?.[columnKey]) &&
Object.keys(testcase.data[columnKey]).length > 0
Comment thread
TbhVishal marked this conversation as resolved.
Resolved

if (Object.keys(outputs).length === 0 && !hasExistingColumn) continue

selected.push({
columnKey,
outputs,
Expand Down
131 changes: 131 additions & 0 deletions web/packages/agenta-annotation/tests/unit/testset-sync.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,72 @@ describe("buildTestcaseExportRows", () => {
})
expect(rows).toHaveLength(0)
})

it("replaces existing evaluator column data so cleared fields are removed", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello", quality: {score: 8, note: "stale note"}},
} as unknown as Testcase
const ann = makeAnnotation({
evaluatorSlug: "quality",
tags: [queueTag("q-1"), TESTCASE_QUEUE_KIND_TAG],
outputs: {score: 9},
})
const rows = buildTestcaseExportRows({
scenarioIds: ["s-1"],
testcasesByScenarioId: new Map([["s-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
queueId: "q-1",
})
expect(rows).toHaveLength(1)
expect((rows[0].data as Record<string, unknown>).prompt).toBe("hello")
expect((rows[0].data as Record<string, unknown>).quality).toEqual({score: 9})
})

it("clears evaluator column when all fields are removed and existing column exists", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello", quality: {score: 8}},
} as unknown as Testcase
const ann = makeAnnotation({
evaluatorSlug: "quality",
tags: [queueTag("q-1"), TESTCASE_QUEUE_KIND_TAG],
outputs: {},
})
const rows = buildTestcaseExportRows({
scenarioIds: ["s-1"],
testcasesByScenarioId: new Map([["s-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
queueId: "q-1",
})
expect(rows).toHaveLength(1)
expect((rows[0].data as Record<string, unknown>).quality).toEqual({})
})

it("skips annotation with empty outputs when testcase has no existing evaluator data", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello"},
} as unknown as Testcase
const ann = makeAnnotation({
evaluatorSlug: "quality",
tags: [queueTag("q-1"), TESTCASE_QUEUE_KIND_TAG],
outputs: {},
})
const rows = buildTestcaseExportRows({
scenarioIds: ["s-1"],
testcasesByScenarioId: new Map([["s-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
queueId: "q-1",
})
expect(rows).toHaveLength(0)
})
})

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -599,6 +665,71 @@ describe("buildTestsetSyncPreview", () => {
expect(preview.hasBlockingConflicts).toBe(false)
})

it("replaces existing evaluator outputs so cleared fields do not persist into preview rows", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello", quality: {score: 7, note: "stale note"}},
} as unknown as Testcase
const ann = makeQueueAnn()
const preview = buildTestsetSyncPreview({
queueId: "q-1",
completedScenarios: [{scenarioId: "s-1", testcaseId: "tc-1"}],
testcasesById: new Map([["tc-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
latestRevisionIdsByTestsetId: new Map([["ts-1", "rev-1"]]),
})
expect(preview.targets[0].rows[0].data.quality).toEqual({score: 7})
expect(preview.targets[0].rows[0].data.quality).not.toHaveProperty("note")
})

it("replaces existing evaluator column with empty record when all fields are cleared", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello", quality: {score: 7}},
} as unknown as Testcase
const ann = makeAnnotation({
evaluatorSlug: "quality",
tags: [queueTag("q-1"), TESTCASE_QUEUE_KIND_TAG],
outputs: {},
})
const preview = buildTestsetSyncPreview({
queueId: "q-1",
completedScenarios: [{scenarioId: "s-1", testcaseId: "tc-1"}],
testcasesById: new Map([["tc-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
latestRevisionIdsByTestsetId: new Map([["ts-1", "rev-1"]]),
})
expect(preview.exportableRows).toBe(1)
expect(preview.targets[0].rows[0].data.quality).toEqual({})
})

it("skips row with empty outputs when testcase has no existing evaluator data", () => {
const testcase = {
id: "tc-1",
testset_id: "ts-1",
data: {prompt: "hello"},
} as unknown as Testcase
const ann = makeAnnotation({
evaluatorSlug: "quality",
tags: [queueTag("q-1"), TESTCASE_QUEUE_KIND_TAG],
outputs: {},
})
const preview = buildTestsetSyncPreview({
queueId: "q-1",
completedScenarios: [{scenarioId: "s-1", testcaseId: "tc-1"}],
testcasesById: new Map([["tc-1", testcase]]),
annotationsByTestcaseId: new Map([["tc-1", [ann]]]),
evaluators: [evaluator],
latestRevisionIdsByTestsetId: new Map([["ts-1", "rev-1"]]),
})
expect(preview.exportableRows).toBe(0)
expect(preview.targets).toHaveLength(0)
})

it("records duplicate_queue_annotations conflict and skips the row", () => {
const ann1 = makeQueueAnn("trace-1")
const ann2 = makeQueueAnn("trace-2")
Expand Down
Loading