From 6640bec92087f1bac099aba16b2cb06c9b3fe09d Mon Sep 17 00:00:00 2001 From: Sean Teramae Date: Wed, 19 Aug 2026 10:21:48 -0700 Subject: [PATCH] feat(studio): Transform through Data Designer Processors Signed-off-by: Sean Teramae --- .../studio/src/api/datasets/constants.ts | 2 - .../api/datasets/useDatasetFileTransform.ts | 211 ++++--------- .../buildTransformJobRequest.test.ts | 73 +++++ .../buildTransformJobRequest.ts | 86 ++++++ .../DataDesignerTransformModal/index.test.tsx | 179 +++++++++++ .../DataDesignerTransformModal/index.tsx | 284 ++++++++++++++++++ .../TransformFileModal/TransformPreview.tsx | 53 ---- .../TransformFileModal/index.test.tsx | 133 ++++++++ .../FilesTable/TransformFileModal/index.tsx | 185 +++++------- .../FilesTable/TransformFileModal/types.ts | 59 ---- .../useTransformPreview.test.ts | 93 ------ .../TransformFileModal/useTransformPreview.ts | 121 -------- .../transform/CustomTemplateRows.tsx | 88 ++++++ .../transform/DiscardTransformModal.tsx | 37 +++ .../components/transform/FieldMappingRow.tsx | 136 +++++++++ .../src/components/transform/FormatPicker.tsx | 42 +++ .../components/transform/MappingSection.tsx | 74 +++++ .../transform/TemplateSyntaxTooltip.tsx | 56 ++++ .../components/transform/TransformPreview.tsx | 68 +++++ .../src/components/transform/draft.test.ts | 83 +++++ .../studio/src/components/transform/draft.ts | 70 +++++ .../src/components/transform/formats.ts | 121 ++++++++ .../transform/renderTemplate.test.ts | 62 ++++ .../components/transform/renderTemplate.ts | 130 ++++++++ .../src/components/transform/template.test.ts | 237 +++++++++++++++ .../src/components/transform/template.ts | 231 ++++++++++++++ .../transform/useTransformMapping.ts | 161 ++++++++++ .../transform/useTransformPreview.test.ts | 70 +++++ .../transform/useTransformPreview.ts | 71 +++++ .../DataDesignerJobDetailsRoute/index.tsx | 46 ++- 30 files changed, 2659 insertions(+), 603 deletions(-) create mode 100644 web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.test.ts create mode 100644 web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.ts create mode 100644 web/packages/studio/src/components/DataDesignerTransformModal/index.test.tsx create mode 100644 web/packages/studio/src/components/DataDesignerTransformModal/index.tsx delete mode 100644 web/packages/studio/src/components/FilesTable/TransformFileModal/TransformPreview.tsx create mode 100644 web/packages/studio/src/components/FilesTable/TransformFileModal/index.test.tsx delete mode 100644 web/packages/studio/src/components/FilesTable/TransformFileModal/types.ts delete mode 100644 web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.test.ts delete mode 100644 web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.ts create mode 100644 web/packages/studio/src/components/transform/CustomTemplateRows.tsx create mode 100644 web/packages/studio/src/components/transform/DiscardTransformModal.tsx create mode 100644 web/packages/studio/src/components/transform/FieldMappingRow.tsx create mode 100644 web/packages/studio/src/components/transform/FormatPicker.tsx create mode 100644 web/packages/studio/src/components/transform/MappingSection.tsx create mode 100644 web/packages/studio/src/components/transform/TemplateSyntaxTooltip.tsx create mode 100644 web/packages/studio/src/components/transform/TransformPreview.tsx create mode 100644 web/packages/studio/src/components/transform/draft.test.ts create mode 100644 web/packages/studio/src/components/transform/draft.ts create mode 100644 web/packages/studio/src/components/transform/formats.ts create mode 100644 web/packages/studio/src/components/transform/renderTemplate.test.ts create mode 100644 web/packages/studio/src/components/transform/renderTemplate.ts create mode 100644 web/packages/studio/src/components/transform/template.test.ts create mode 100644 web/packages/studio/src/components/transform/template.ts create mode 100644 web/packages/studio/src/components/transform/useTransformMapping.ts create mode 100644 web/packages/studio/src/components/transform/useTransformPreview.test.ts create mode 100644 web/packages/studio/src/components/transform/useTransformPreview.ts diff --git a/web/packages/studio/src/api/datasets/constants.ts b/web/packages/studio/src/api/datasets/constants.ts index b1017a7eb7..d2de0c1773 100644 --- a/web/packages/studio/src/api/datasets/constants.ts +++ b/web/packages/studio/src/api/datasets/constants.ts @@ -39,5 +39,3 @@ export const BINARY_FILE_EXTENSIONS = new Set([ // Documents 'pdf', ]); - -export const COMPLETION_PROMPT_KEY_ORDER = ['prompt', 'instruction', 'question']; // Searches for a prompt in the following keys diff --git a/web/packages/studio/src/api/datasets/useDatasetFileTransform.ts b/web/packages/studio/src/api/datasets/useDatasetFileTransform.ts index bd27c30e93..8f9d04c3d4 100644 --- a/web/packages/studio/src/api/datasets/useDatasetFileTransform.ts +++ b/web/packages/studio/src/api/datasets/useDatasetFileTransform.ts @@ -1,188 +1,85 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { useChatCompletions } from '@nemo/common/src/hooks/useChatCompletions'; -import { getEntityReference } from '@nemo/common/src/namedEntity'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { isDefined } from '@nemo/common/src/utils/isDefined'; import { filesUploadFile } from '@nemo/sdk/generated/platform/api'; -import type { FilesetFileOutput, ModelEntity } from '@nemo/sdk/generated/platform/schema'; -import { COMPLETION_PROMPT_KEY_ORDER } from '@studio/api/datasets/constants'; +import type { FilesetFileOutput } from '@nemo/sdk/generated/platform/schema'; import { invalidateDatasetCaches } from '@studio/api/datasets/invalidateDatasetCaches'; -import { TransformFileFormFields } from '@studio/components/FilesTable/TransformFileModal/types'; -import { parseFileContent, Row } from '@studio/util/files'; +import { renderTemplate } from '@studio/components/transform/renderTemplate'; +import { parseFileContent } from '@studio/util/files'; import { useMutation, UseMutationOptions } from '@tanstack/react-query'; -import Handlebars from 'handlebars'; -import { ChatCompletion, ChatCompletionCreateParams } from 'openai/resources/index.mjs'; -import { useCallback, useState } from 'react'; +import { useCallback } from 'react'; -type MutationProps = { +interface MutationProps { workspace: string; datasetName: string; - filepath: TransformFileFormFields['filepath']; - mappings: TransformFileFormFields['mappings']; + filepath: string; + /** The `schema_transform` template each row is rewritten through. */ + template: Record; fileContent: string; - model?: ModelEntity; -}; + /** + * Column the template references but the source file does not have. A Data + * Designer job would declare it as a UUID sampler; here each row gets its own + * short identifier as it is rewritten. + */ + generatedIdColumn?: string; +} type Props = Omit, 'mutationFn'>; +/** + * Rewrites a fileset file in place through a transform template. The mapping is + * applied in the browser with the same renderer that drives the preview, so what + * the user approved is exactly what is uploaded. + */ export const useDatasetFileTransform = ({ onError, onSuccess }: Props) => { const toast = useToast(); - const { mutateAsync: createChatCompletions } = useChatCompletions(); - const [progressLabel, setProgressLabel] = useState(''); - const [progressValue, setProgressValue] = useState(0); - const progPreInferValue = 20; - const progPostInferValue = 90; const mutationFn = useCallback( - async ({ fileContent, filepath, model, mappings, workspace, datasetName }: MutationProps) => { - setProgressValue(10); - - // Parse JSON objects - const content = parseFileContent({ + async ({ + fileContent, + filepath, + template, + workspace, + datasetName, + generatedIdColumn, + }: MutationProps) => { + const { rows, failures } = parseFileContent({ content: fileContent, fileType: filepath.split('.').at(-1), }); - const { failures } = content; - let { rows } = content; if (failures?.length) { toast.error(`${failures.length} Line(s) had parsing errors.`); } - // Re-map each row to the described mappings - if (mappings) { - rows = rows - .map((row) => { - const newRow: Record = {}; - let skipInvalidRow = false; - mappings.forEach(({ key, value }) => { - // Pre-process the row to stringify arrays and objects - const processedRow = Object.fromEntries( - Object.entries(row).map(([k, v]) => [ - k, - Array.isArray(v) || (typeof v === 'object' && v !== null) ? JSON.stringify(v) : v, - ]) - ); - - const template = Handlebars.compile(value); - // Handle nested keys (e.g. "user.profile.name") - const keyParts = key.split('.'); - let current = newRow; - - // Process all parts except the last one - for (let i = 0; i < keyParts.length - 1; i++) { - const part = keyParts[i]; - if (!(part in current)) { - current[part] = {}; - } - current = current[part] as Record; - } - - // Handle the last part of the key - const lastPart = keyParts[keyParts.length - 1]; - const compiledValue = template(processedRow); - - // Try to parse as JSON if it looks like an array or object - try { - if (compiledValue.trim().startsWith('[') || compiledValue.trim().startsWith('{')) { - current[lastPart] = JSON.parse(compiledValue); - } else { - current[lastPart] = compiledValue; - } - } catch { - skipInvalidRow = true; - } - }); - return skipInvalidRow ? undefined : newRow; - }) - .filter(Boolean) as Row[]; - } - setProgressValue(progPreInferValue); - - // Generate completions if necessary - if (model) { - const chatCompletionRequests = rows - .map((row) => { - let userMsg = ''; - for (const key of COMPLETION_PROMPT_KEY_ORDER) { - if (key in row) { - userMsg = row[key] as string; - break; - } - } - if (!userMsg) { - return undefined; - } - const messages = [ - { - role: 'user', - content: userMsg, - }, - ]; - if (model.prompt?.system_prompt) { - messages.unshift({ role: 'system', content: model.prompt.system_prompt }); - } - return { - messages, - model: getEntityReference(model), - }; - }) - .filter(isDefined) as ChatCompletionCreateParams[]; - setProgressLabel(`Inferencing ${chatCompletionRequests.length} rows...`); - const completions = (await createChatCompletions({ - requests: chatCompletionRequests, - onTaskComplete: ({ completedTasks }) => { - const ratioComplete = completedTasks / chatCompletionRequests.length; - const inferProgress = ratioComplete * 70; - setProgressLabel(`Inferencing... (${completedTasks}/${chatCompletionRequests.length})`); - setProgressValue(progPreInferValue + inferProgress); - }, - })) as ChatCompletion[]; - rows = completions.map((completion, index) => { - return { - input: { category: '', ...rows[index] }, - response: completion.choices[0].message.content, - llm_name: model.name, - }; - }); - } - setProgressValue(progPostInferValue); - - // Upload file to fileset - setProgressLabel('Uploading...'); - setProgressValue(95); - const fileContent2 = rows.map((row) => JSON.stringify(row)).join('\n'); - const blob = new Blob([fileContent2], { type: 'application/json' }); + const transformed = rows.map((row) => { + const input = generatedIdColumn + ? { ...row, [generatedIdColumn]: crypto.randomUUID().replaceAll('-', '').slice(0, 8) } + : row; + return renderTemplate(template, input).row; + }); + const blob = new Blob([transformed.map((row) => JSON.stringify(row)).join('\n')], { + type: 'application/json', + }); return filesUploadFile(workspace, datasetName, filepath, blob); }, - [createChatCompletions, toast] + [toast] ); - return { - ...useMutation({ - mutationFn, - onError: (data, variables, onMutateResult, context) => { - onError?.(data, variables, onMutateResult, context); - }, - onSuccess: (data, variables, onMutateResult, context) => { - invalidateDatasetCaches( - variables.workspace, - variables.datasetName, - ['files', 'content'], - variables.filepath - ); - onSuccess?.(data, variables, onMutateResult, context); - }, - onMutate: () => { - setProgressLabel(''); - }, - onSettled: () => { - setProgressLabel(''); - }, - }), - progressLabel, - progressValue, - }; + return useMutation({ + mutationFn, + onError: (data, variables, onMutateResult, context) => { + onError?.(data, variables, onMutateResult, context); + }, + onSuccess: (data, variables, onMutateResult, context) => { + invalidateDatasetCaches( + variables.workspace, + variables.datasetName, + ['files', 'content'], + variables.filepath + ); + onSuccess?.(data, variables, onMutateResult, context); + }, + }); }; diff --git a/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.test.ts b/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.test.ts new file mode 100644 index 0000000000..0c3035a8b9 --- /dev/null +++ b/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildSeedPath, + buildTransformJobRequest, +} from '@studio/components/DataDesignerTransformModal/buildTransformJobRequest'; + +const input = { + jobName: 'support-evals-agent-eval-tasks', + processorName: 'agent_eval_tasks', + filesetWorkspace: 'default', + filesetName: 'support-evals-artifacts', + filePath: 'dataset.parquet', + numRecords: 500, + template: { id: '{{ task_id }}' }, +}; + +describe('buildSeedPath', () => { + it('addresses a single file inside a workspace-qualified fileset', () => { + expect(buildSeedPath('default', 'my-fileset', 'nested/dataset.parquet')).toBe( + 'default/my-fileset#nested/dataset.parquet' + ); + }); +}); + +describe('buildTransformJobRequest', () => { + it('declares no columns so the job generates nothing', () => { + expect(buildTransformJobRequest(input).spec.config.columns).toEqual([]); + }); + + it('declares a UUID sampler when an id column has to be generated', () => { + const request = buildTransformJobRequest({ ...input, generatedIdColumn: 'row_id' }); + expect(request.spec.config.columns).toEqual([ + { + name: 'row_id', + column_type: 'sampler', + sampler_type: 'uuid', + params: { short_form: true }, + }, + ]); + // A sampler needs no model, so the job is still inference-free. + expect(request.spec.config.model_configs).toBeUndefined(); + }); + + it('seeds from the source file in order', () => { + const { seed_config: seedConfig } = buildTransformJobRequest(input).spec.config; + expect(seedConfig).toEqual({ + source: { seed_type: 'nmp', path: 'default/support-evals-artifacts#dataset.parquet' }, + sampling_strategy: 'ordered', + }); + }); + + it('carries the template through on a schema_transform processor', () => { + expect(buildTransformJobRequest(input).spec.config.processors).toEqual([ + { + processor_type: 'schema_transform', + name: 'agent_eval_tasks', + template: { id: '{{ task_id }}' }, + }, + ]); + }); + + it('requests no models, since nothing is generated', () => { + expect(buildTransformJobRequest(input).spec.config.model_configs).toBeUndefined(); + }); + + it('passes the row count and job name through', () => { + const request = buildTransformJobRequest(input); + expect(request.name).toBe('support-evals-agent-eval-tasks'); + expect(request.spec.num_records).toBe(500); + }); +}); diff --git a/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.ts b/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.ts new file mode 100644 index 0000000000..95c8c3b8ae --- /dev/null +++ b/web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { SamplingStrategy, type CreateJobRequest } from '@nemo/sdk/generated/data-designer/schema'; + +export interface TransformJobRequestInput { + readonly jobName: string; + readonly processorName: string; + /** Workspace owning the source fileset. */ + readonly filesetWorkspace: string; + readonly filesetName: string; + /** Path of the source file inside that fileset. */ + readonly filePath: string; + /** Rows to read from the source file. */ + readonly numRecords: number; + readonly template: Record; + /** + * When set, the job declares this column as a UUID sampler so the template can + * reference it. Used to give each row an identifier the source file lacks. + */ + readonly generatedIdColumn?: string; +} + +/** + * `FilesetFileSeedSource` addresses a single file as `/#`. + * The workspace prefix is optional server-side but we always send it, since the + * fileset a job wrote to is not necessarily the workspace the user is browsing. + */ +export const buildSeedPath = ( + filesetWorkspace: string, + filesetName: string, + filePath: string +): string => `${filesetWorkspace}/${filesetName}#${filePath}`; + +/** + * Builds a transform job: the engine resolves the seed file's own columns into + * passthrough seed columns, then the `schema_transform` processor rewrites each + * row into `processors-files//`. + * + * Only `generatedIdColumn` is ever declared as a column, and a sampler costs no + * inference, so the job needs no model either way. + * + * `num_records` must not exceed the source file's row count — the seed reader + * restarts at the top when it runs dry, which silently duplicates rows. + */ +export const buildTransformJobRequest = ({ + jobName, + processorName, + filesetWorkspace, + filesetName, + filePath, + numRecords, + template, + generatedIdColumn, +}: TransformJobRequestInput): CreateJobRequest => ({ + name: jobName, + spec: { + num_records: numRecords, + config: { + columns: generatedIdColumn + ? [ + { + name: generatedIdColumn, + column_type: 'sampler', + sampler_type: 'uuid', + params: { short_form: true }, + }, + ] + : [], + seed_config: { + source: { + seed_type: 'nmp', + path: buildSeedPath(filesetWorkspace, filesetName, filePath), + }, + sampling_strategy: SamplingStrategy.ordered, + }, + processors: [ + { + processor_type: 'schema_transform', + name: processorName, + template, + }, + ], + }, + }, +}); diff --git a/web/packages/studio/src/components/DataDesignerTransformModal/index.test.tsx b/web/packages/studio/src/components/DataDesignerTransformModal/index.test.tsx new file mode 100644 index 0000000000..eb6a587b2c --- /dev/null +++ b/web/packages/studio/src/components/DataDesignerTransformModal/index.test.tsx @@ -0,0 +1,179 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DataDesignerTransformModal } from '@studio/components/DataDesignerTransformModal'; +import { TestProviders } from '@studio/tests/util/TestProviders'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { MemoryRouter } from 'react-router'; + +let SOURCE_ROWS: Record[] = []; + +const withIdColumn = [ + { task_id: 'a1', category: 'billing', user_request: 'Where is my refund?', ideal_response: '..' }, +]; +const withoutIdColumn = [ + { category: 'billing', user_request: 'Where is my refund?', ideal_response: '..' }, +]; + +const createJobMock = vi.fn(); +const onCloseMock = vi.fn(); + +vi.mock('@studio/api/datasets/useDatasetFileContent', () => ({ + useDatasetFileContent: () => ({ + data: SOURCE_ROWS.map((row) => JSON.stringify(row)).join('\n'), + isLoading: false, + }), +})); + +vi.mock('@nemo/sdk/generated/data-designer/api', () => ({ + useDataDesignerCreateJob: () => ({ + mutateAsync: createJobMock, + isPending: false, + error: null, + }), +})); + +const renderModal = () => + render( + + + + + + ); + +describe('DataDesignerTransformModal', () => { + beforeEach(() => { + createJobMock.mockReset(); + onCloseMock.mockReset(); + createJobMock.mockResolvedValue({ name: 'support-evals-agent-eval-tasks' }); + SOURCE_ROWS = withIdColumn; + }); + + it('auto-maps the default format from the source columns', () => { + renderModal(); + + expect(screen.getByText('inputs.instruction')).toBeInTheDocument(); + expect( + screen.getByRole('combobox', { name: 'inputs.instruction source column' }) + ).toHaveTextContent('user_request'); + }); + + it('shows a row preview once the source file is read', async () => { + renderModal(); + + // Rendering of the row itself is covered by the renderTemplate unit tests; + // CodeSnippet highlights asynchronously, so only the panel is asserted here. + expect(await screen.findByText('Preview output')).toBeInTheDocument(); + }); + + it('creates a processor-only job seeded from the source file', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('button', { name: 'Create transform job' })); + + expect(createJobMock).toHaveBeenCalledTimes(1); + const { workspace, data } = createJobMock.mock.calls[0][0]; + expect(workspace).toBe('default'); + expect(data.spec.config.columns).toEqual([]); + expect(data.spec.config.seed_config.source.path).toBe( + 'default/support-evals-artifacts#dataset.parquet' + ); + expect(data.spec.config.processors[0].name).toBe('agent_eval_tasks'); + expect(data.spec.num_records).toBe(500); + }); + + it('closes without a prompt when nothing has been edited', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('button', { name: 'Cancel' })); + + expect(screen.queryByText('Discard this transform?')).not.toBeInTheDocument(); + expect(onCloseMock).toHaveBeenCalledTimes(1); + }); + + it('warns before discarding an edited mapping, and keeps it on Keep editing', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.clear(screen.getByLabelText('Output name')); + await user.type(screen.getByLabelText('Output name'), 'my_tasks'); + await user.click(screen.getByRole('button', { name: 'Cancel' })); + + expect(screen.getByText('Discard this transform?')).toBeInTheDocument(); + expect(onCloseMock).not.toHaveBeenCalled(); + + await user.click(screen.getByRole('button', { name: 'Keep editing' })); + + expect(screen.queryByText('Discard this transform?')).not.toBeInTheDocument(); + expect(onCloseMock).not.toHaveBeenCalled(); + expect(screen.getByLabelText('Output name')).toHaveValue('my_tasks'); + }); + + it('closes once the discard is confirmed', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.clear(screen.getByLabelText('Output name')); + await user.type(screen.getByLabelText('Output name'), 'my_tasks'); + await user.click(screen.getByRole('button', { name: 'Cancel' })); + await user.click(screen.getByRole('button', { name: 'Discard' })); + + expect(onCloseMock).toHaveBeenCalledTimes(1); + }); + + it('generates a UUID id column when the source has no unique key', async () => { + SOURCE_ROWS = withoutIdColumn; + const user = userEvent.setup(); + renderModal(); + + expect(screen.getByText(/adds it as a\s+UUID sampler column/)).toBeInTheDocument(); + + await user.click(screen.getByRole('button', { name: 'Create transform job' })); + + const { data } = createJobMock.mock.calls[0][0]; + expect(data.spec.config.columns).toEqual([ + { + name: 'row_id', + column_type: 'sampler', + sampler_type: 'uuid', + params: { short_form: true }, + }, + ]); + expect(data.spec.config.processors[0].template.id).toBe('{{ row_id }}'); + }); + + it('declares no id column when the source already has one', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('button', { name: 'Create transform job' })); + + const { data } = createJobMock.mock.calls[0][0]; + expect(data.spec.config.columns).toEqual([]); + expect(data.spec.config.processors[0].template.id).toBe('{{ task_id }}'); + }); + + it('blocks submission while a required field is unmapped', async () => { + const user = userEvent.setup(); + renderModal(); + + // Switching to Preference Pairs leaves `rejected` with no matching column. + await user.click(screen.getByRole('radio', { name: 'Preference Pairs' })); + + expect(screen.getByText(/must have a source/)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Create transform job' })).toBeDisabled(); + }); +}); diff --git a/web/packages/studio/src/components/DataDesignerTransformModal/index.tsx b/web/packages/studio/src/components/DataDesignerTransformModal/index.tsx new file mode 100644 index 0000000000..09097f9190 --- /dev/null +++ b/web/packages/studio/src/components/DataDesignerTransformModal/index.tsx @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getErrorMessage } from '@nemo/common/src/api/common/utils'; +import { FormModal } from '@nemo/common/src/components/FormModal'; +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { useDataDesignerCreateJob } from '@nemo/sdk/generated/data-designer/api'; +import { + Banner, + Divider, + Flex, + Label, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Stack, + Text, + TextInput, +} from '@nvidia/foundations-react-core'; +import { useDatasetFileContent } from '@studio/api/datasets/useDatasetFileContent'; +import { buildTransformJobRequest } from '@studio/components/DataDesignerTransformModal/buildTransformJobRequest'; +import { DiscardTransformModal } from '@studio/components/transform/DiscardTransformModal'; +import { slugify } from '@studio/components/transform/draft'; +import { FormatPicker } from '@studio/components/transform/FormatPicker'; +import { OUTPUT_FORMATS } from '@studio/components/transform/formats'; +import { MappingSection } from '@studio/components/transform/MappingSection'; +import { TransformPreview } from '@studio/components/transform/TransformPreview'; +import { useTransformMapping } from '@studio/components/transform/useTransformMapping'; +import { getDataDesignerJobDetailsRoute } from '@studio/routes/utils'; +import { getContentColumns, getFileNameFromPath } from '@studio/util/files'; +import { GitBranch } from 'lucide-react'; +import { useCallback, useMemo, useState, type FC } from 'react'; +import { useNavigate } from 'react-router'; + +export interface DataDesignerTransformModalProps { + open: boolean; + onClose: () => void; + /** Workspace the new job is created in. */ + workspace: string; + /** Name of the job whose output is being transformed, used to seed the new job's name. */ + sourceJobName: string; + /** Location of the source job's artifacts fileset. */ + filesetWorkspace: string; + filesetName: string; + /** Data files in that fileset the transform can read. */ + fileOptions: readonly string[]; + /** Row count of the source job — the transform reads at most this many rows. */ + defaultNumRecords: number; +} + +const pickDefaultFile = (fileOptions: readonly string[]): string => + fileOptions.find((path) => /\.parquet$/i.test(path)) ?? fileOptions[0] ?? ''; + +/** + * Rewrites a finished Data Designer dataset into another schema by launching a + * second, generation-free Data Designer job: the source file becomes the seed, + * no columns are declared, and a `schema_transform` processor does the mapping. + * Because nothing is generated, the job needs no model and costs no inference. + */ +export const DataDesignerTransformModal: FC = ({ + open, + onClose, + workspace, + sourceJobName, + filesetWorkspace, + filesetName, + fileOptions, + defaultNumRecords, +}) => { + const toast = useToast(); + const navigate = useNavigate(); + + const [filePath, setFilePath] = useState(() => pickDefaultFile(fileOptions)); + const [processorName, setProcessorName] = useState(OUTPUT_FORMATS[0].defaultProcessorName); + const [jobName, setJobName] = useState( + () => `${slugify(sourceJobName)}-${OUTPUT_FORMATS[0].defaultProcessorName}` + ); + const [numRecords, setNumRecords] = useState(String(defaultNumRecords)); + const [isDiscardOpen, setIsDiscardOpen] = useState(false); + + // Parquet is decoded to JSONL by the hook, so every file parses as JSONL here. + const { data: fileContent, isLoading: isLoadingContent } = useDatasetFileContent({ + workspace: filesetWorkspace, + name: filesetName, + path: filePath, + enabled: open && Boolean(filesetWorkspace && filesetName && filePath), + }); + + const fileType = /\.parquet$/i.test(filePath) ? 'jsonl' : (filePath.split('.').at(-1) ?? ''); + const columns = useMemo(() => getContentColumns(fileContent, fileType), [fileContent, fileType]); + + const handleFormatChange = useCallback( + (format: (typeof OUTPUT_FORMATS)[number]) => { + setProcessorName(format.defaultProcessorName); + setJobName(`${slugify(sourceJobName)}-${format.defaultProcessorName}`); + }, + [sourceJobName] + ); + const mapping = useTransformMapping({ columns, onFormatChange: handleFormatChange }); + + const parsedRows = Number(numRecords); + const isRowCountValid = Number.isInteger(parsedRows) && parsedRows > 0; + const exceedsSource = isRowCountValid && parsedRows > defaultNumRecords; + + const isDirty = + mapping.isDirty || + jobName !== `${slugify(sourceJobName)}-${mapping.format.defaultProcessorName}` || + processorName !== mapping.format.defaultProcessorName || + numRecords !== String(defaultNumRecords); + + const createJob = useDataDesignerCreateJob(); + const submitError = createJob.error ? getErrorMessage(createJob.error) : null; + + const canSubmit = + Boolean(filePath) && + Boolean(jobName.trim()) && + Boolean(processorName.trim()) && + isRowCountValid && + mapping.isComplete; + + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); + if (!canSubmit) { + return; + } + try { + const created = await createJob.mutateAsync({ + workspace, + data: buildTransformJobRequest({ + jobName: jobName.trim(), + processorName: processorName.trim(), + filesetWorkspace, + filesetName, + filePath, + numRecords: parsedRows, + template: mapping.template, + generatedIdColumn: mapping.needsGeneratedId ? mapping.generatedIdColumn : undefined, + }), + }); + toast.success(`Transform job ${created?.name ?? jobName} created.`); + onClose(); + if (created?.name) { + navigate(getDataDesignerJobDetailsRoute(workspace, created.name)); + } + } catch { + // Surfaced through `submitError` in the modal footer. + } + }; + + /** + * Every dismissal — Cancel, Escape, the close affordance, a click outside — + * arrives here through `FormModal`'s single `onClose`, so the field mapping + * cannot be lost to a stray keypress. + */ + const handleCloseRequest = () => { + if (isDirty) { + setIsDiscardOpen(true); + return; + } + onClose(); + }; + + return ( + <> + + + Transform + + } + instruction="Rewrite this dataset into another schema. This starts a second Data Designer job that only maps fields — no rows are generated and no model is called." + submitButtonText="Create transform job" + errorText={submitError} + submitDisabled={!canSubmit} + loading={createJob.isPending} + disabled={createJob.isPending} + onSubmit={handleSubmit} + onClose={handleCloseRequest} + className="w-[860px]" + > + + + + + + typeof value === 'string' ? getFileNameFromPath(value) : null + } + /> + + + {fileOptions.map((path) => ( + + {path} + + ))} + + + + + + + + + + + + + + + Written to processors-files/{processorName || 'output'}/ in the new + job's fileset. + + {mapping.needsGeneratedId && ( + + {mapping.generatedIdColumn} is not in the source file — the job adds it + as a UUID sampler column, one value per row. Samplers run without a model, so this + still costs no inference. + + )} + + + + + + + + setJobName(event.currentTarget.value)} + /> + + + + setProcessorName(event.currentTarget.value)} + /> + + + + setNumRecords(event.currentTarget.value)} + /> + + + + {exceedsSource && ( + + The source job produced {defaultNumRecords} rows. Reading more than that restarts at + the top of the file, duplicating rows in the output. + + )} + + + + {isDiscardOpen && ( + setIsDiscardOpen(false)} + onConfirm={onClose} + description="Your field mapping has not been submitted. Closing now discards it — no job is created." + /> + )} + + ); +}; diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/TransformPreview.tsx b/web/packages/studio/src/components/FilesTable/TransformFileModal/TransformPreview.tsx deleted file mode 100644 index 486a1719ba..0000000000 --- a/web/packages/studio/src/components/FilesTable/TransformFileModal/TransformPreview.tsx +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { TransformFileFormFields } from '@studio/components/FilesTable/TransformFileModal/types'; -import { useTransformPreview } from '@studio/components/FilesTable/TransformFileModal/useTransformPreview'; -import { PreviewOutputPanel } from '@studio/components/PreviewOutputPanel'; -import { useMemo, type FC } from 'react'; -import { useWatch, type Control } from 'react-hook-form'; -import { useDebounce } from 'use-debounce'; - -const PREVIEW_DEBOUNCE_MS = 250; - -interface Props { - control: Control; - fileContent: string | undefined; - fileType: string; -} - -/** - * Isolates the mapping subscription so typing in a mapping field only re-renders - * (and re-runs the Handlebars templating for) the preview, not the whole modal. - */ -export const TransformPreview: FC = ({ control, fileContent, fileType }) => { - const mappings = useWatch({ control, name: 'mappings' }); - const [debouncedMappings] = useDebounce(mappings, PREVIEW_DEBOUNCE_MS); - - const { currentRow, totalRows, sourceRow, afterRow, onRowChange } = useTransformPreview({ - fileContent, - fileType, - mappings: debouncedMappings ?? [], - }); - - const beforeValue = useMemo(() => JSON.stringify(sourceRow, null, 2), [sourceRow]); - const afterValue = useMemo( - () => - afterRow - ? JSON.stringify(afterRow, null, 2) - : '// Add mappings above to see the transformed output', - [afterRow] - ); - - if (!sourceRow) return null; - - return ( - - ); -}; diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/index.test.tsx b/web/packages/studio/src/components/FilesTable/TransformFileModal/index.test.tsx new file mode 100644 index 0000000000..f46b8aafad --- /dev/null +++ b/web/packages/studio/src/components/FilesTable/TransformFileModal/index.test.tsx @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TransformFileModal } from '@studio/components/FilesTable/TransformFileModal'; +import { TestProviders } from '@studio/tests/util/TestProviders'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +let SOURCE_ROWS: Record[] = []; + +const withIdColumn = [ + { task_id: 'a1', category: 'billing', user_request: 'Where is my refund?', ideal_response: '..' }, +]; +const withoutIdColumn = [ + { category: 'billing', user_request: 'Where is my refund?', ideal_response: '..' }, +]; + +const transformMock = vi.fn(); +const onCloseMock = vi.fn(); + +vi.mock('@studio/hooks/useSelectedDatasetId', () => ({ + useSelectedDatasetId: () => 'default/test', +})); + +vi.mock('@studio/api/datasets/useDatasetFileContent', () => ({ + useDatasetFileContent: () => ({ + data: SOURCE_ROWS.map((row) => JSON.stringify(row)).join('\n'), + isLoading: false, + }), +})); + +vi.mock('@studio/api/datasets/useDatasetFileTransform', () => ({ + useDatasetFileTransform: () => ({ mutate: transformMock, isPending: false }), +})); + +const renderModal = () => + render( + + + + ); + +describe('TransformFileModal', () => { + beforeEach(() => { + transformMock.mockReset(); + onCloseMock.mockReset(); + SOURCE_ROWS = withIdColumn; + }); + + it('auto-maps the default format from the file columns', () => { + renderModal(); + + expect( + screen.getByRole('combobox', { name: 'inputs.instruction source column' }) + ).toHaveTextContent('user_request'); + }); + + it('transforms the file in place with the built template', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('button', { name: 'Transform file' })); + + expect(transformMock).toHaveBeenCalledTimes(1); + const { workspace, datasetName, filepath, template } = transformMock.mock.calls[0][0]; + expect({ workspace, datasetName, filepath }).toEqual({ + workspace: 'default', + datasetName: 'test', + filepath: 'data.jsonl', + }); + expect(template).toMatchObject({ + id: '{{ task_id }}', + intent: '{{ category }}', + inputs: { instruction: '{{ user_request }}' }, + }); + }); + + it('generates the identifier client-side when the file has no unique key', async () => { + SOURCE_ROWS = withoutIdColumn; + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('button', { name: 'Transform file' })); + + expect(transformMock.mock.calls[0][0].generatedIdColumn).toBe('row_id'); + }); + + it('blocks submission while a required field is unmapped', async () => { + const user = userEvent.setup(); + renderModal(); + + // Switching to Preference Pairs leaves `rejected` with no matching column. + await user.click(screen.getByRole('radio', { name: 'Preference Pairs' })); + + expect(screen.getByText(/must have a source/)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Transform file' })).toBeDisabled(); + }); + + it('appends a fresh custom row once the last one is filled in', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('radio', { name: 'Custom' })); + + // The four source columns pass through, leaving a fifth row blank. + expect(screen.queryByLabelText('Output key 6')).not.toBeInTheDocument(); + + await user.type(screen.getByLabelText('Output key 5'), 'summary'); + expect(screen.queryByLabelText('Output key 6')).not.toBeInTheDocument(); + + await user.type(screen.getByLabelText('Template for key 5'), '{{{{ category }}'); + + expect(screen.getByLabelText('Output key 6')).toHaveValue(''); + expect(screen.queryByRole('button', { name: 'Add key' })).not.toBeInTheDocument(); + }); + + it('warns before discarding an edited mapping', async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole('radio', { name: 'Custom' })); + await user.type(screen.getByLabelText('Output key 1'), 'summary'); + await user.click(screen.getByRole('button', { name: 'Cancel' })); + + expect(screen.getByText('Discard this transform?')).toBeInTheDocument(); + expect(onCloseMock).not.toHaveBeenCalled(); + }); +}); diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/index.tsx b/web/packages/studio/src/components/FilesTable/TransformFileModal/index.tsx index 161eae907b..9ef7efce41 100644 --- a/web/packages/studio/src/components/FilesTable/TransformFileModal/index.tsx +++ b/web/packages/studio/src/components/FilesTable/TransformFileModal/index.tsx @@ -1,29 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { zodResolver } from '@hookform/resolvers/zod'; -import { MappingFields } from '@nemo/common/src/components/form/MappingFields'; import { FormModal } from '@nemo/common/src/components/FormModal'; -import { ModelSelect } from '@nemo/common/src/components/ModelSelect'; -import { getEntityReference, getPartsFromReference } from '@nemo/common/src/namedEntity'; +import { getPartsFromReference } from '@nemo/common/src/namedEntity'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { handleFormErrorsGeneric } from '@nemo/common/src/utils/forms/error'; -import { useModelsListModels } from '@nemo/sdk/generated/platform/api'; -import { Divider, Flex, Label, Spinner, Stack, Text } from '@nvidia/foundations-react-core'; +import { Divider, Flex, Stack } from '@nvidia/foundations-react-core'; import { useDatasetFileContent } from '@studio/api/datasets/useDatasetFileContent'; import { useDatasetFileTransform } from '@studio/api/datasets/useDatasetFileTransform'; -import { TransformPreview } from '@studio/components/FilesTable/TransformFileModal/TransformPreview'; -import { - type TransformFileFormFields, - transformFileSchema, -} from '@studio/components/FilesTable/TransformFileModal/types'; +import { DiscardTransformModal } from '@studio/components/transform/DiscardTransformModal'; +import { FormatPicker } from '@studio/components/transform/FormatPicker'; +import { MappingSection } from '@studio/components/transform/MappingSection'; +import { TransformPreview } from '@studio/components/transform/TransformPreview'; +import { useTransformMapping } from '@studio/components/transform/useTransformMapping'; import { ValueWithLabel } from '@studio/components/ValueWithLabel'; import { useSelectedDatasetId } from '@studio/hooks/useSelectedDatasetId'; -import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getContentSchema } from '@studio/util/files'; +import { getContentColumns } from '@studio/util/files'; import { GitBranch } from 'lucide-react'; -import { useMemo, type ComponentProps, type FC } from 'react'; -import { useForm } from 'react-hook-form'; +import { useMemo, useState, type ComponentProps, type FC } from 'react'; interface Props extends Pick, 'open' | 'onClose'> { filepath?: string; @@ -31,35 +24,15 @@ interface Props extends Pick, 'open' | 'onClose } /** - * This modal is used to handle transforms to a file's schema - * such as manipulating columns and adding model completions. + * Rewrites one fileset file into another schema, in place. Same field mapping as + * the Data Designer transform, but applied in the browser: the file is small + * enough to read, remap, and re-upload without a job. */ export const TransformFileModal: FC = ({ open, onClose, filepath, datasetId }) => { const toast = useToast(); const resolvedDatasetId = useSelectedDatasetId({ datasetId }); const datasetNameSplit = getPartsFromReference(resolvedDatasetId); - const workspace = useWorkspaceFromPath(); - - const { control, reset, handleSubmit } = useForm({ - mode: 'onChange', - resolver: zodResolver(transformFileSchema), - defaultValues: { - filepath, - mappings: [], - }, - }); - const resetAndClose = () => { - reset(); - onClose(); - }; - - const { data: modelsResponse, isFetching: isFetchingModels } = useModelsListModels(workspace, { - page_size: 1000, - sort: 'created_at', - }); - const models = useMemo(() => { - return modelsResponse?.data; - }, [modelsResponse]); + const [isDiscardOpen, setIsDiscardOpen] = useState(false); const resolvedFilepath = filepath ?? ''; const filepathParts = resolvedFilepath.split('.'); @@ -69,93 +42,89 @@ export const TransformFileModal: FC = ({ open, onClose, filepath, dataset ...datasetNameSplit, path: resolvedFilepath, }); - const { schema } = useMemo(() => { - return getContentSchema(fileContent, { fileType }); - }, [fileType, fileContent]); + + const columns = useMemo(() => getContentColumns(fileContent, fileType), [fileContent, fileType]); + const mapping = useTransformMapping({ columns }); const { mutate: transformFile, isPending } = useDatasetFileTransform({ onSuccess: () => { toast.success('Successfully finished file transformation!'); - resetAndClose(); + onClose(); }, }); - const onSubmit = (data: TransformFileFormFields) => { - const model = models?.find((model) => getEntityReference(model) === data.model); + const handleSubmit = (event: React.FormEvent) => { + event.preventDefault(); if (!fileContent) { toast.error('File content not found'); return; } - if (!model && data.model) { - toast.error('Model not found'); - return; - } transformFile({ workspace: datasetNameSplit.workspace, datasetName: datasetNameSplit.name, filepath: resolvedFilepath, - mappings: data.mappings.filter((m) => m.key.trim() !== ''), - fileContent: fileContent, - model, + template: mapping.template, + fileContent, + generatedIdColumn: mapping.needsGeneratedId ? mapping.generatedIdColumn : undefined, }); }; + const handleCloseRequest = () => { + if (mapping.isDirty) { + setIsDiscardOpen(true); + return; + } + onClose(); + }; + return ( - - - Transform - - } - submitButtonText="Confirm" - onSubmit={handleSubmit( - onSubmit, - handleFormErrorsGeneric({ title: 'Transform File Form Errors' }) - )} - className="w-[960px] overflow-hidden" - onClose={resetAndClose} - disabled={isPending} - submitDisabled={isLoadingFileContent} - loading={isPending} - > - - - Map existing columns to new names, add computed fields, and optionally run inference - models on each row to enhance your data with AI-generated content. - - - - {isLoadingFileContent ? ( - - + <> + + + Transform - ) : ( - - - - Model for Inference, - }} - useControllerProps={{ control, name: 'model' }} - /> - - )} - - + } + instruction="Rewrite this file into another schema. Every row is remapped in place — the file is overwritten with the result." + submitButtonText="Transform file" + onSubmit={handleSubmit} + className="w-[860px] overflow-hidden" + onClose={handleCloseRequest} + disabled={isPending} + submitDisabled={isLoadingFileContent || !mapping.isComplete} + loading={isPending} + > + + + + + + + + + + + + + + {isDiscardOpen && ( + setIsDiscardOpen(false)} + onConfirm={onClose} + description="Your field mapping has not been submitted. Closing now discards it — the file is left unchanged." + /> + )} + ); }; diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/types.ts b/web/packages/studio/src/components/FilesTable/TransformFileModal/types.ts deleted file mode 100644 index 57c61cf615..0000000000 --- a/web/packages/studio/src/components/FilesTable/TransformFileModal/types.ts +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { z } from 'zod'; - -export const mappingSchema = z.object({ - /** Empty key is allowed for the trailing draft row in MappingFields. */ - key: z.string(), - value: z.string().optional(), -}); - -export const transformFileSchema = z - .object({ - filepath: z.string().nonempty('Filepath is required'), - model: z.string().optional(), - mappings: z.array(mappingSchema), - }) - .superRefine((data, ctx) => { - const keys = new Set(); - for (let i = 0; i < data.mappings.length; i++) { - const k = data.mappings[i].key.trim(); - if (!k) continue; - if (keys.has(k)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: 'Mapping keys must be unique.', - path: ['mappings', i, 'key'], - }); - break; - } - keys.add(k); - } - }) - .refine( - (data) => { - if (!data.model) { - return true; - } - const hasUserMsg = data.mappings.some( - (mapping) => - mapping.key === 'prompt' || mapping.key === 'instruction' || mapping.key === 'question' - ); - if (!hasUserMsg) { - return false; - } - return true; - }, - { - path: ['model'], - message: - 'Missing user message for model inference. Please add a mapping with a key of "prompt", "instruction", or "question".', - } - ); - -export type TransformFileFormFields = { - filepath: string; - model?: string; - mappings: z.infer[]; -}; diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.test.ts b/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.test.ts deleted file mode 100644 index 8d9b3d0646..0000000000 --- a/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { useTransformPreview } from '@studio/components/FilesTable/TransformFileModal/useTransformPreview'; -import { act, renderHook } from '@testing-library/react'; - -const fileContent = JSON.stringify([{ name: 'Ada', role: 'engineer' }]); - -describe('useTransformPreview', () => { - it('applies mappings to nested keys', () => { - const { result } = renderHook(() => - useTransformPreview({ - fileContent, - fileType: 'json', - mappings: [{ key: 'user.name', value: '{{name}}' }], - }) - ); - - expect(result.current.afterRow).toEqual({ user: { name: 'Ada' } }); - }); - - it('does not throw when a later mapping nests under a key already set to a primitive', () => { - const { result } = renderHook(() => - useTransformPreview({ - fileContent, - fileType: 'json', - mappings: [ - { key: 'user', value: '{{name}}' }, - { key: 'user.role', value: '{{role}}' }, - ], - }) - ); - - expect(result.current.afterRow).toEqual({ user: { role: 'engineer' } }); - }); - - it('ignores mappings whose keys traverse the prototype chain', () => { - const { result } = renderHook(() => - useTransformPreview({ - fileContent, - fileType: 'json', - mappings: [ - { key: '__proto__.polluted', value: 'yes' }, - { key: 'constructor', value: 'yes' }, - { key: 'user..name', value: '{{name}}' }, - { key: 'name', value: '{{name}}' }, - ], - }) - ); - - expect(result.current.afterRow).toEqual({ name: 'Ada' }); - expect(({} as Record).polluted).toBeUndefined(); - }); - - it('keeps rendering when a mapping holds an invalid template', () => { - const { result } = renderHook(() => - useTransformPreview({ - fileContent, - fileType: 'json', - mappings: [ - { key: 'broken', value: '{{#if}}' }, - { key: 'name', value: '{{name}}' }, - ], - }) - ); - - expect(result.current.afterRow).toEqual({ broken: '{{#if}}', name: 'Ada' }); - }); - - it('reports the row actually displayed when the row count shrinks', () => { - const { result, rerender } = renderHook( - (props: { fileContent: string }) => - useTransformPreview({ - fileContent: props.fileContent, - fileType: 'json', - mappings: [{ key: 'name', value: '{{name}}' }], - }), - { - initialProps: { - fileContent: JSON.stringify([{ name: 'Ada' }, { name: 'Grace' }, { name: 'Katherine' }]), - }, - } - ); - - act(() => result.current.onRowChange(3)); - expect(result.current.currentRow).toBe(3); - - rerender({ fileContent: JSON.stringify([{ name: 'Ada' }]) }); - - expect(result.current.currentRow).toBe(1); - expect(result.current.totalRows).toBe(1); - }); -}); diff --git a/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.ts b/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.ts deleted file mode 100644 index fb0ada7886..0000000000 --- a/web/packages/studio/src/components/FilesTable/TransformFileModal/useTransformPreview.ts +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { TransformFileFormFields } from '@studio/components/FilesTable/TransformFileModal/types'; -import { parseFileContent, type Row } from '@studio/util/files'; -import Handlebars from 'handlebars'; -import { useMemo, useState } from 'react'; - -type Mapping = TransformFileFormFields['mappings'][number]; - -const UNSAFE_KEY_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); - -/** - * Splits a free-text mapping key into path segments, rejecting keys with empty - * or prototype-sensitive segments so traversal can never reach Object.prototype. - */ -const parseKeyParts = (key: string): string[] | null => { - const parts = key - .trim() - .split('.') - .map((part) => part.trim()); - - if (parts.some((part) => part === '' || UNSAFE_KEY_SEGMENTS.has(part))) return null; - - return parts; -}; - -const renderMapping = (value: string | undefined, row: Record): string => { - try { - return Handlebars.compile(value ?? '')(row); - } catch { - // An in-progress template (e.g. `{{name`) must not break the whole preview. - return value ?? ''; - } -}; - -const applyMappings = (row: Row, mappings: Mapping[]): Row => { - const newRow: Record = {}; - - const processedRow = Object.fromEntries( - Object.entries(row).map(([k, v]) => [ - k, - Array.isArray(v) || (typeof v === 'object' && v !== null) ? JSON.stringify(v) : v, - ]) - ); - - for (const { key, value } of mappings) { - const keyParts = parseKeyParts(key); - if (!keyParts) continue; - - let current = newRow; - - for (let i = 0; i < keyParts.length - 1; i++) { - const part = keyParts[i]; - const existing = current[part]; - if (typeof existing !== 'object' || existing === null || Array.isArray(existing)) { - current[part] = {}; - } - current = current[part] as Record; - } - - const lastPart = keyParts[keyParts.length - 1]; - const compiledValue = renderMapping(value, processedRow); - - try { - if (compiledValue.trim().startsWith('[') || compiledValue.trim().startsWith('{')) { - current[lastPart] = JSON.parse(compiledValue); - } else { - current[lastPart] = compiledValue; - } - } catch { - current[lastPart] = compiledValue; - } - } - - return newRow; -}; - -interface Props { - fileContent: string | undefined; - fileType: string; - mappings: Mapping[]; -} - -export const useTransformPreview = ({ fileContent, fileType, mappings }: Props) => { - const [currentRow, setCurrentRow] = useState(1); - - const rows = useMemo(() => { - if (!fileContent) return []; - try { - return parseFileContent({ content: fileContent, fileType }).rows; - } catch { - return []; - } - }, [fileContent, fileType]); - - const activeMappings = useMemo(() => mappings.filter((m) => m.key.trim() !== ''), [mappings]); - - const rowIndex = Math.min(currentRow - 1, rows.length - 1); - const sourceRow = rows[rowIndex] ?? null; - - const afterRow = useMemo(() => { - if (!sourceRow || activeMappings.length === 0) return null; - return applyMappings(sourceRow, activeMappings); - }, [sourceRow, activeMappings]); - - const totalRows = rows.length; - - // `currentRow` can outlive the file it was chosen for, so report the row actually shown. - const displayedRow = totalRows === 0 ? currentRow : rowIndex + 1; - - const onRowChange = (row: number) => setCurrentRow(Math.min(Math.max(1, row), totalRows)); - - return { - currentRow: displayedRow, - totalRows, - sourceRow, - afterRow, - onRowChange, - }; -}; diff --git a/web/packages/studio/src/components/transform/CustomTemplateRows.tsx b/web/packages/studio/src/components/transform/CustomTemplateRows.tsx new file mode 100644 index 0000000000..bc4e6abc37 --- /dev/null +++ b/web/packages/studio/src/components/transform/CustomTemplateRows.tsx @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Badge, Button, Flex, Stack, Text, TextInput } from '@nvidia/foundations-react-core'; +import { columnReference, type CustomTemplateRow } from '@studio/components/transform/template'; +import { Trash2 } from 'lucide-react'; +import { type FC } from 'react'; + +export interface CustomTemplateRowsProps { + rows: readonly CustomTemplateRow[]; + columns: readonly string[]; + onChange: (rows: CustomTemplateRow[]) => void; +} + +const isBlank = (row: CustomTemplateRow | undefined): boolean => + !row || (!row.key.trim() && !row.value.trim()); + +const isFilled = (row: CustomTemplateRow | undefined): boolean => + !!row && !!row.key.trim() && !!row.value.trim(); + +/** Grows the grid so an unfinished row is always available to type into. */ +const withTrailingBlank = (rows: CustomTemplateRow[]): CustomTemplateRow[] => { + const last = rows[rows.length - 1]; + return rows.length === 0 || isFilled(last) ? [...rows, { key: '', value: '' }] : rows; +}; + +/** + * The escape hatch behind every preset: the raw `schema_transform` template as a + * key/template grid. Keys accept dot paths (`inputs.instruction`) and numeric + * segments (`messages.0.content`) to build nested objects and arrays. Filling in + * the last row grows the grid, so no explicit add control is needed. + */ +export const CustomTemplateRows: FC = ({ rows, columns, onChange }) => { + const update = (index: number, patch: Partial) => { + onChange(withTrailingBlank(rows.map((row, i) => (i === index ? { ...row, ...patch } : row)))); + }; + + const remove = (index: number) => { + onChange(withTrailingBlank(rows.filter((_, i) => i !== index))); + }; + + return ( + + {rows.map((row, index) => ( + + update(index, { key: event.currentTarget.value })} + /> + update(index, { value: event.currentTarget.value })} + /> + + + ))} + + {columns.length > 0 && ( + + + Available columns: + + {columns.map((column) => ( + + {columnReference(column)} + + ))} + + )} + + ); +}; diff --git a/web/packages/studio/src/components/transform/DiscardTransformModal.tsx b/web/packages/studio/src/components/transform/DiscardTransformModal.tsx new file mode 100644 index 0000000000..3249f1d424 --- /dev/null +++ b/web/packages/studio/src/components/transform/DiscardTransformModal.tsx @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ConfirmationModal } from '@nemo/common/src/components/ConfirmationModal'; +import { type FC } from 'react'; + +interface Props { + onClose: () => void; + onConfirm: () => void; + description?: string; +} + +/** + * Guards an edited field mapping against a stray dismissal. Mount only while + * needed: it carries its own form state, which would otherwise settle + * asynchronously behind the still-open transform modal. + */ +export const DiscardTransformModal: FC = ({ + onClose, + onConfirm, + description = 'Your field mapping has not been submitted. Closing now discards it.', +}) => ( + { + onConfirm(); + return true; + }} + title="Discard this transform?" + description={description} + submitButtonText="Discard" + cancelButtonText="Keep editing" + submitButtonColor="danger" + suppressResultToasts + /> +); diff --git a/web/packages/studio/src/components/transform/FieldMappingRow.tsx b/web/packages/studio/src/components/transform/FieldMappingRow.tsx new file mode 100644 index 0000000000..d8f1aee0b1 --- /dev/null +++ b/web/packages/studio/src/components/transform/FieldMappingRow.tsx @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + Button, + Flex, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Stack, + Tag, + Text, + TextInput, +} from '@nvidia/foundations-react-core'; +import type { OutputFormatField } from '@studio/components/transform/formats'; +import { columnReference } from '@studio/components/transform/template'; +import { Braces } from 'lucide-react'; +import { type FC } from 'react'; + +export interface FieldMappingRowProps { + field: OutputFormatField; + /** The Jinja2 template currently mapped to this field, `''` when unmapped. */ + value: string; + /** Source column names available in the file being transformed. */ + columns: readonly string[]; + /** When true the row shows a raw template input instead of the column picker. */ + isRaw: boolean; + /** + * Name of the UUID column the job would generate. Offered in the picker for + * identity fields, so a source with no unique key still gets a per-row id. + */ + generatedIdColumn?: string; + onChange: (path: string, value: string) => void; + onToggleRaw: (path: string) => void; +} + +/** + * One field of an output format. The common case — "this field is that column" — + * is a column picker; the `{ }` toggle drops to a raw Jinja2 input for anything + * the picker cannot express (filters, concatenation, literals). + */ +export const FieldMappingRow: FC = ({ + field, + value, + columns, + isRaw, + generatedIdColumn, + onChange, + onToggleRaw, +}) => { + const offersGeneratedId = Boolean(field.identity && generatedIdColumn); + const options = offersGeneratedId ? [...columns, generatedIdColumn as string] : columns; + const selectedColumn = options.find((column) => columnReference(column) === value) ?? ''; + const isGenerated = offersGeneratedId && selectedColumn === generatedIdColumn; + const isUnmappedRequired = field.required && !value.trim(); + + return ( + + + + {field.label} + + + {field.required ? 'Required' : 'Optional'} + + {isGenerated && ( + + Generated + + )} + + + + {isRaw ? ( + onChange(field.path, event.currentTarget.value)} + /> + ) : ( + + onChange(field.path, column ? columnReference(column) : '') + } + > + + + + {columns.map((column) => ( + + {column} + + ))} + {offersGeneratedId && ( + + {`${generatedIdColumn} — generate a UUID per row`} + + )} + + + + )} + + + + + {isUnmappedRequired + ? `${field.description} This field is required and has no source.` + : isGenerated + ? `${field.description} The source has no unique key, so the job adds a ${generatedIdColumn} column.` + : field.description} + + + ); +}; diff --git a/web/packages/studio/src/components/transform/FormatPicker.tsx b/web/packages/studio/src/components/transform/FormatPicker.tsx new file mode 100644 index 0000000000..32daf39d90 --- /dev/null +++ b/web/packages/studio/src/components/transform/FormatPicker.tsx @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { RadioCard } from '@nemo/common/src/components/RadioCard'; +import { Label, RadioGroupRoot, Stack, Text } from '@nvidia/foundations-react-core'; +import { OUTPUT_FORMATS } from '@studio/components/transform/formats'; +import type { TransformMapping } from '@studio/components/transform/useTransformMapping'; +import { type FC } from 'react'; + +interface Props { + mapping: TransformMapping; + label?: string; +} + +/** Target-format selector: the first decision of every transform. */ +export const FormatPicker: FC = ({ mapping, label = 'Target format' }) => ( + + + +
+ {OUTPUT_FORMATS.map((option) => ( + {option.label}} + description={ + + {option.description} + + } + labelSide="left" + /> + ))} +
+ + +); diff --git a/web/packages/studio/src/components/transform/MappingSection.tsx b/web/packages/studio/src/components/transform/MappingSection.tsx new file mode 100644 index 0000000000..7a769b690c --- /dev/null +++ b/web/packages/studio/src/components/transform/MappingSection.tsx @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Banner, Flex, Label, Spinner, Stack } from '@nvidia/foundations-react-core'; +import { CustomTemplateRows } from '@studio/components/transform/CustomTemplateRows'; +import { FieldMappingRow } from '@studio/components/transform/FieldMappingRow'; +import { TemplateSyntaxTooltip } from '@studio/components/transform/TemplateSyntaxTooltip'; +import type { TransformMapping } from '@studio/components/transform/useTransformMapping'; +import { type FC } from 'react'; + +interface Props { + mapping: TransformMapping; + /** True while the source file is still being read for its columns. */ + isLoadingColumns?: boolean; +} + +/** + * The field mapping itself: one row per field of the chosen format, or the raw + * key/template grid behind the custom format. + */ +export const MappingSection: FC = ({ mapping, isLoadingColumns }) => { + if (isLoadingColumns) { + return ( + + + + ); + } + + return ( + + + + + + + {mapping.columns.length === 0 && ( + + The selected file could not be read, so columns cannot be suggested. You can still write + templates by hand. + + )} + + {mapping.isCustom ? ( + + ) : ( + + {mapping.format.fields.map((field) => ( + + ))} + + )} + + {mapping.missingRequired.length > 0 && ( + + {mapping.missingRequired.join(', ')} must have a source before this transform can run. + + )} + + ); +}; diff --git a/web/packages/studio/src/components/transform/TemplateSyntaxTooltip.tsx b/web/packages/studio/src/components/transform/TemplateSyntaxTooltip.tsx new file mode 100644 index 0000000000..0db26e7042 --- /dev/null +++ b/web/packages/studio/src/components/transform/TemplateSyntaxTooltip.tsx @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Flex, Stack, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { tooltipClassName } from '@studio/styles/common'; +import { HelpCircle } from 'lucide-react'; +import { type FC } from 'react'; + +const Tip: FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +); + +const TemplateSyntaxTooltipContent: FC = () => ( +
+ + Template syntax + + Values are Jinja2. {'{{ column }}'} inserts that column's value for the + row; text outside the braces is kept as-is, so{' '} + {'Ticket {{ id }}: {{ summary }}'} is one field. + + Text with no braces is a constant — every row gets the same value. + + Filters transform a value: {'{{ topic | upper }}'},{' '} + {'{{ text | trim }}'}, {'{{ score | int }}'}. + + + Fallbacks cover empty cells: {"{{ notes | default('none') }}"} + . + + + Key syntax + + + A dot in a key nests an object: reference.expected becomes{' '} + {'{ "reference": { "expected": … } }'}. + + + A number in a key builds an array: messages.0.content and{' '} + messages.1.content become a two-item messages list. + + + Leaving a field blank drops it from the output entirely rather than writing an empty string. + + +
+); + +/** Help affordance explaining Jinja2 template and key syntax for the field mapping. */ +export const TemplateSyntaxTooltip: FC = () => ( + } side="right"> + + + + +); diff --git a/web/packages/studio/src/components/transform/TransformPreview.tsx b/web/packages/studio/src/components/transform/TransformPreview.tsx new file mode 100644 index 0000000000..a580651556 --- /dev/null +++ b/web/packages/studio/src/components/transform/TransformPreview.tsx @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Stack, Text } from '@nvidia/foundations-react-core'; +import { PreviewOutputPanel } from '@studio/components/PreviewOutputPanel'; +import { useTransformPreview } from '@studio/components/transform/useTransformPreview'; +import { useMemo, type FC } from 'react'; +import { useDebounce } from 'use-debounce'; + +const PREVIEW_DEBOUNCE_MS = 250; + +interface Props { + fileContent: string | undefined; + fileType: string; + template: Record; + /** Column the transform generates rather than reads, e.g. a per-row identifier. */ + generatedIdColumn?: string; +} + +/** + * Before/after view of a single source row. The template is debounced so typing + * a raw template re-renders the preview rather than the whole modal on every + * keystroke. + */ +export const TransformPreview: FC = ({ + fileContent, + fileType, + template, + generatedIdColumn, +}) => { + const [debouncedTemplate] = useDebounce(template, PREVIEW_DEBOUNCE_MS); + + const { currentRow, totalRows, sourceRow, afterRow, approximated, onRowChange } = + useTransformPreview({ + fileContent, + fileType, + template: debouncedTemplate, + generatedIdColumn, + }); + + const beforeValue = useMemo(() => JSON.stringify(sourceRow, null, 2), [sourceRow]); + const afterValue = useMemo( + () => + afterRow + ? JSON.stringify(afterRow, null, 2) + : '// Map a field above to see the transformed output', + [afterRow] + ); + + if (!sourceRow) return null; + + return ( + + + {approximated && ( + + This preview ignores template filters — the transform applies them when it runs. + + )} + + ); +}; diff --git a/web/packages/studio/src/components/transform/draft.test.ts b/web/packages/studio/src/components/transform/draft.test.ts new file mode 100644 index 0000000000..0baa552c97 --- /dev/null +++ b/web/packages/studio/src/components/transform/draft.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + isMappingDraftDirty, + slugify, + type MappingBaseline, + type MappingDraft, +} from '@studio/components/transform/draft'; +import { findOutputFormat, type OutputFormat } from '@studio/components/transform/formats'; +import { autoMapFields } from '@studio/components/transform/template'; + +const agentEvalTask = findOutputFormat('agent-eval-task') as OutputFormat; +const columns = ['task_id', 'category', 'user_request', 'ideal_response']; + +const baseline: MappingBaseline = { + mappings: autoMapFields(agentEvalTask, columns, 'row_id'), + customRows: [{ key: '', value: '' }], +}; + +const pristine: MappingDraft = { + mappings: baseline.mappings, + rawPaths: new Set(), + customRows: baseline.customRows, +}; + +describe('slugify', () => { + it('lowercases and hyphenates, trimming stray separators', () => { + expect(slugify(' Support Evals (v2) ')).toBe('support-evals-v2'); + }); +}); + +describe('isMappingDraftDirty', () => { + it('is false for an untouched draft', () => { + expect(isMappingDraftDirty(pristine, baseline)).toBe(false); + }); + + it('treats a blank mapping as equivalent to an absent one', () => { + const draft = { ...pristine, mappings: { ...baseline.mappings, 'reference.other': ' ' } }; + expect(isMappingDraftDirty(draft, baseline)).toBe(false); + }); + + it('is true once a mapping is changed', () => { + const draft = { ...pristine, mappings: { ...baseline.mappings, intent: '{{ other }}' } }; + expect(isMappingDraftDirty(draft, baseline)).toBe(true); + }); + + it('is true once an auto-mapped field is cleared', () => { + const draft = { ...pristine, mappings: { ...baseline.mappings, id: '' } }; + expect(isMappingDraftDirty(draft, baseline)).toBe(true); + }); + + it('is true once a field is switched to a raw template', () => { + expect(isMappingDraftDirty({ ...pristine, rawPaths: new Set(['id']) }, baseline)).toBe(true); + }); + + it('ignores empty custom rows but not filled ones', () => { + expect( + isMappingDraftDirty({ ...pristine, customRows: [{ key: '', value: '' }] }, baseline) + ).toBe(false); + expect( + isMappingDraftDirty({ ...pristine, customRows: [{ key: 'id', value: '' }] }, baseline) + ).toBe(true); + }); + + it('ignores custom rows that still match a seeded passthrough baseline', () => { + const seeded: MappingBaseline = { + ...baseline, + customRows: [ + { key: 'task_id', value: '{{ task_id }}' }, + { key: '', value: '' }, + ], + }; + + expect(isMappingDraftDirty({ ...pristine, customRows: seeded.customRows }, seeded)).toBe(false); + expect( + isMappingDraftDirty( + { ...pristine, customRows: [{ key: 'renamed', value: '{{ task_id }}' }] }, + seeded + ) + ).toBe(true); + }); +}); diff --git a/web/packages/studio/src/components/transform/draft.ts b/web/packages/studio/src/components/transform/draft.ts new file mode 100644 index 0000000000..e6179d9780 --- /dev/null +++ b/web/packages/studio/src/components/transform/draft.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { CustomTemplateRow } from '@studio/components/transform/template'; + +/** The editable mapping state shared by every transform surface. */ +export interface MappingDraft { + readonly mappings: Readonly>; + /** Fields switched from the column picker to a raw template input. */ + readonly rawPaths: ReadonlySet; + readonly customRows: readonly CustomTemplateRow[]; +} + +/** What the mapping would hold with no user input, for the current format and file. */ +export interface MappingBaseline { + readonly mappings: Readonly>; + readonly customRows: readonly CustomTemplateRow[]; +} + +/** Turns a name into the slug used for generated job names. */ +export const slugify = (value: string): string => + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + +/** Blank and absent mean the same thing for a mapping, so both normalize away. */ +const withoutBlanks = (mappings: Readonly>): Record => + Object.fromEntries(Object.entries(mappings).filter(([, value]) => value.trim() !== '')); + +const sameMappings = ( + a: Readonly>, + b: Readonly> +): boolean => { + const left = withoutBlanks(a); + const right = withoutBlanks(b); + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every((key) => left[key] === right[key]); +}; + +/** Trailing blank rows are an editing affordance, not content. */ +const filledRows = (rows: readonly CustomTemplateRow[]): CustomTemplateRow[] => + rows.filter((row) => row.key.trim() !== '' || row.value.trim() !== ''); + +const sameCustomRows = ( + a: readonly CustomTemplateRow[], + b: readonly CustomTemplateRow[] +): boolean => { + const left = filledRows(a); + const right = filledRows(b); + return ( + left.length === right.length && + left.every( + (row, index) => + row.key.trim() === right[index].key.trim() && row.value.trim() === right[index].value.trim() + ) + ); +}; + +/** + * Whether the mapping holds work the user would lose by closing. + * + * Compared against the baseline rather than tracked as an "edited" flag, so + * merely switching target format — which re-derives every default — does not + * count, and neither does typing a value back to what it already was. + */ +export const isMappingDraftDirty = (draft: MappingDraft, baseline: MappingBaseline): boolean => + !sameMappings(draft.mappings, baseline.mappings) || + draft.rawPaths.size > 0 || + !sameCustomRows(draft.customRows, baseline.customRows); diff --git a/web/packages/studio/src/components/transform/formats.ts b/web/packages/studio/src/components/transform/formats.ts new file mode 100644 index 0000000000..d915246a2b --- /dev/null +++ b/web/packages/studio/src/components/transform/formats.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Output formats a dataset can be rewritten into, shared by every transform + * surface: the Data Designer job transform and the in-place file transform. + * + * A format is purely a Studio-side convenience: it names the fields the target + * consumer expects and pre-fills the template the transform actually receives. + * Anything a format cannot express is still reachable through the `custom` + * format, which is the raw key/template grid. + */ + +/** One mappable field in an output format. */ +export interface OutputFormatField { + /** + * Dot path into the output record. Numeric segments build arrays, so + * `messages.0.content` produces `{ messages: [{ content: ... }] }`. + */ + readonly path: string; + readonly label: string; + readonly description: string; + readonly required: boolean; + /** Source column-name fragments used to guess a mapping, best match first. */ + readonly hints: readonly string[]; + /** + * Marks a field that must be unique per row. When no source column matches, + * the transform job generates one instead of leaving it unmapped — a constant + * would be identical on every row, which silently collapses the output. + */ + readonly identity?: boolean; +} + +export interface OutputFormat { + readonly id: string; + readonly label: string; + readonly description: string; + /** Default name of the processor, and of the directory its output is written to. */ + readonly defaultProcessorName: string; + readonly fields: readonly OutputFormatField[]; + /** Literal values emitted at fixed paths — not user-mappable (e.g. a chat role). */ + readonly constants?: Readonly>; +} + +export const CUSTOM_FORMAT_ID = 'custom'; + +export const OUTPUT_FORMATS: readonly OutputFormat[] = [ + { + id: 'agent-eval-task', + label: 'Evaluation Tasks', + description: 'Tasks the Evaluator can run an agent against.', + defaultProcessorName: 'agent_eval_tasks', + fields: [ + { + path: 'id', + label: 'id', + description: 'Stable task identifier, unique within the task collection.', + required: true, + hints: ['task_id', 'id', 'uuid'], + identity: true, + }, + { + path: 'intent', + label: 'intent', + description: 'Human-readable description of the desired agent behavior.', + required: true, + hints: ['intent', 'goal', 'objective', 'category', 'topic'], + }, + { + path: 'inputs.instruction', + label: 'inputs.instruction', + description: 'The task input handed to the agent.', + required: true, + hints: ['instruction', 'prompt', 'question', 'request', 'input'], + }, + { + path: 'reference.expected', + label: 'reference.expected', + description: 'Grader-only ground truth. Never shown to the agent.', + required: false, + hints: ['expected', 'reference', 'answer', 'response', 'ideal'], + }, + ], + }, + { + id: 'chat-messages', + label: 'Messages', + description: 'A two-turn messages array, the usual shape for SFT.', + defaultProcessorName: 'chat_messages', + fields: [ + { + path: 'messages.0.content', + label: 'user message', + description: 'Content of the user turn.', + required: true, + hints: ['prompt', 'question', 'instruction', 'request', 'input'], + }, + { + path: 'messages.1.content', + label: 'assistant message', + description: 'Content of the assistant turn.', + required: true, + hints: ['response', 'answer', 'completion', 'output', 'ideal'], + }, + ], + constants: { + 'messages.0.role': 'user', + 'messages.1.role': 'assistant', + }, + }, + { + id: CUSTOM_FORMAT_ID, + label: 'Custom', + description: 'Write the output schema yourself, one key at a time.', + defaultProcessorName: 'transformed', + fields: [], + }, +]; + +export const findOutputFormat = (id: string): OutputFormat | undefined => + OUTPUT_FORMATS.find((format) => format.id === id); diff --git a/web/packages/studio/src/components/transform/renderTemplate.test.ts b/web/packages/studio/src/components/transform/renderTemplate.test.ts new file mode 100644 index 0000000000..a925f8b8af --- /dev/null +++ b/web/packages/studio/src/components/transform/renderTemplate.test.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { renderTemplate } from '@studio/components/transform/renderTemplate'; + +const row = { + name: 'Ada & Grace', + tags: ['a', 'b'], + score: 3, + reference: { expected: 'a refund', notes: null }, + messages: [{ content: 'hello' }], +}; + +describe('renderTemplate', () => { + it('does not HTML-escape a column reference', () => { + expect(renderTemplate({ who: '{{ name }}' }, row).row).toEqual({ who: 'Ada & Grace' }); + }); + + it('keeps surrounding text and combines references', () => { + expect(renderTemplate({ label: 'by {{ name }} ({{ score }})' }, row).row).toEqual({ + label: 'by Ada & Grace (3)', + }); + }); + + it('reads a dot path into a nested source column', () => { + expect(renderTemplate({ expected: '{{ reference.expected }}' }, row).row).toEqual({ + expected: 'a refund', + }); + }); + + it('reads an indexed path into an array column', () => { + expect(renderTemplate({ first: '{{ messages.0.content }}' }, row).row).toEqual({ + first: 'hello', + }); + }); + + it('renders an unresolved path as empty rather than the raw braces', () => { + expect(renderTemplate({ missing: '{{ reference.absent }}' }, row).row).toEqual({ missing: '' }); + }); + + it('renders a nested object or array column as its JSON', () => { + expect(renderTemplate({ tags: '{{ tags }}' }, row).row).toEqual({ tags: ['a', 'b'] }); + }); + + it('preserves the template shape, including arrays', () => { + expect(renderTemplate({ messages: [{ content: '{{ name }}' }] }, row).row).toEqual({ + messages: [{ content: 'Ada & Grace' }], + }); + }); + + it('drops a Jinja2 filter and flags the result as approximate', () => { + const { row: output, approximated } = renderTemplate({ who: '{{ name | upper }}' }, row); + expect(output).toEqual({ who: 'Ada & Grace' }); + expect(approximated).toBe(true); + }); + + it('leaves a constant untouched', () => { + const { row: output, approximated } = renderTemplate({ role: 'user' }, row); + expect(output).toEqual({ role: 'user' }); + expect(approximated).toBe(false); + }); +}); diff --git a/web/packages/studio/src/components/transform/renderTemplate.ts b/web/packages/studio/src/components/transform/renderTemplate.ts new file mode 100644 index 0000000000..b6422224f5 --- /dev/null +++ b/web/packages/studio/src/components/transform/renderTemplate.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Row } from '@studio/util/files'; +import Handlebars from 'handlebars'; + +/** Matches a bare `{{ column }}`, `{{ nested.path }}`, or either with Jinja2 filters. */ +const SIMPLE_REFERENCE = /\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*(\|[^}]*)?\}\}/g; + +export interface RenderedTemplate { + row: Row; + /** + * True when part of a value could not be rendered faithfully — a Jinja2 filter + * was dropped, or the template used a construct only Handlebars understands. + * Data Designer transforms run server-side, so the browser can only + * approximate anything beyond a plain reference. + */ + approximated: boolean; +} + +/** + * Reads a dot path out of a row: `reference.expected` and `messages.0.content` + * both resolve, so a source column holding an object or array is reachable. + */ +const resolvePath = (row: Row, path: string): unknown => { + let cursor: unknown = row; + for (const segment of path.split('.')) { + if (cursor === null || typeof cursor !== 'object') { + return undefined; + } + cursor = Array.isArray(cursor) + ? cursor[Number(segment)] + : (cursor as Record)[segment]; + } + return cursor; +}; + +/** A resolved value as template text. Objects and arrays become their JSON. */ +const asText = (value: unknown): string => { + if (value === undefined || value === null) return ''; + return typeof value === 'object' ? JSON.stringify(value) : String(value); +}; + +/** Values for the Handlebars fallback, where a container has to be pre-rendered. */ +const flattenRow = (row: Row): Row => + Object.fromEntries( + Object.entries(row).map(([key, value]) => [ + key, + typeof value === 'object' && value !== null ? JSON.stringify(value) : value, + ]) + ); + +const parseIfJson = (value: string): unknown => { + const trimmed = value.trim(); + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + return JSON.parse(trimmed); + } catch { + // Not JSON after all — keep the rendered text. + } + } + return value; +}; + +/** + * Renders one template value against a row. + * + * Plain references are resolved directly rather than handed to Handlebars: that + * keeps dot paths working (Handlebars cannot see into a column it has already + * been given as JSON text) and avoids HTML-escaping the result. Anything else — + * a block, a helper — still falls back to Handlebars, and is flagged as + * approximate since the real transform speaks Jinja2. + */ +const renderValue = (value: string, row: Row): { rendered: unknown; approximated: boolean } => { + if (!value.includes('{{')) { + return { rendered: value, approximated: false }; + } + + let approximated = false; + const substituted = value.replace(SIMPLE_REFERENCE, (_match, path: string, filter?: string) => { + if (filter) { + approximated = true; + } + return asText(resolvePath(row, path)); + }); + + // Braces left after removing every plain reference mean a block or a helper. + const hasComplexConstruct = value.replace(SIMPLE_REFERENCE, '').includes('{{'); + if (!hasComplexConstruct) { + return { rendered: parseIfJson(substituted), approximated }; + } + + try { + return { + rendered: parseIfJson(Handlebars.compile(value)(flattenRow(row))), + approximated: true, + }; + } catch { + // An in-progress template (e.g. `{{name`) must not break the whole preview. + return { rendered: value, approximated: true }; + } +}; + +/** + * Applies a `schema_transform` template to one source row, preserving the + * template's own shape: nested objects stay nested and numeric key segments + * (already materialized as arrays by `setAtPath`) stay arrays. + */ +export const renderTemplate = (template: Record, row: Row): RenderedTemplate => { + let approximated = false; + + const visit = (value: unknown): unknown => { + if (typeof value === 'string') { + const result = renderValue(value, row); + approximated = approximated || result.approximated; + return result.rendered; + } + if (Array.isArray(value)) { + return value.map(visit); + } + if (typeof value === 'object' && value !== null) { + return Object.fromEntries( + Object.entries(value as Record).map(([key, entry]) => [key, visit(entry)]) + ); + } + return value; + }; + + return { row: visit(template) as Row, approximated }; +}; diff --git a/web/packages/studio/src/components/transform/template.test.ts b/web/packages/studio/src/components/transform/template.test.ts new file mode 100644 index 0000000000..c840184078 --- /dev/null +++ b/web/packages/studio/src/components/transform/template.test.ts @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + CUSTOM_FORMAT_ID, + findOutputFormat, + type OutputFormat, +} from '@studio/components/transform/formats'; +import { + autoMapFields, + buildTemplate, + columnReference, + missingRequiredPaths, + resolveGeneratedIdColumn, + setAtPath, + templateReferences, + usesGeneratedIdColumn, +} from '@studio/components/transform/template'; + +const agentEvalTask = findOutputFormat('agent-eval-task') as OutputFormat; +const chatMessages = findOutputFormat('chat-messages') as OutputFormat; +const custom = findOutputFormat(CUSTOM_FORMAT_ID) as OutputFormat; + +describe('setAtPath', () => { + it('writes a top-level key', () => { + expect(setAtPath({}, 'id', '{{ task_id }}')).toEqual({ id: '{{ task_id }}' }); + }); + + it('creates nested objects for dot paths', () => { + expect(setAtPath({}, 'inputs.instruction', '{{ q }}')).toEqual({ + inputs: { instruction: '{{ q }}' }, + }); + }); + + it('creates arrays for numeric segments', () => { + const target = {}; + setAtPath(target, 'messages.0.content', '{{ q }}'); + setAtPath(target, 'messages.1.content', '{{ a }}'); + expect(target).toEqual({ messages: [{ content: '{{ q }}' }, { content: '{{ a }}' }] }); + }); + + it('merges into an existing container rather than replacing it', () => { + const target = { inputs: { instruction: '{{ q }}' } }; + setAtPath(target, 'inputs.context', '{{ c }}'); + expect(target).toEqual({ inputs: { instruction: '{{ q }}', context: '{{ c }}' } }); + }); + + it('replaces a container whose shape does not match the next segment', () => { + const target = { messages: { content: 'wrong shape' } }; + setAtPath(target, 'messages.0.content', '{{ q }}'); + expect(target).toEqual({ messages: [{ content: '{{ q }}' }] }); + }); + + it('drops a path that traverses the prototype chain', () => { + const target: Record = {}; + setAtPath(target, '__proto__.polluted', 'yes'); + setAtPath(target, 'constructor', 'yes'); + expect(target).toEqual({}); + expect(({} as Record).polluted).toBeUndefined(); + }); + + it('ignores an empty path', () => { + expect(setAtPath({ a: 1 }, '', 'x')).toEqual({ a: 1 }); + }); +}); + +describe('buildTemplate', () => { + it('nests preset fields at their declared paths', () => { + const template = buildTemplate( + agentEvalTask, + { + id: '{{ task_id }}', + intent: '{{ category }}', + 'inputs.instruction': '{{ user_request }}', + 'reference.expected': '{{ ideal_response }}', + }, + [] + ); + expect(template).toEqual({ + id: '{{ task_id }}', + intent: '{{ category }}', + inputs: { instruction: '{{ user_request }}' }, + reference: { expected: '{{ ideal_response }}' }, + }); + }); + + it('omits blank mappings instead of emitting empty strings', () => { + const template = buildTemplate(agentEvalTask, { id: '{{ task_id }}', intent: ' ' }, []); + expect(template).toEqual({ id: '{{ task_id }}' }); + }); + + it('emits format constants alongside mapped fields', () => { + const template = buildTemplate( + chatMessages, + { 'messages.0.content': '{{ q }}', 'messages.1.content': '{{ a }}' }, + [] + ); + expect(template).toEqual({ + messages: [ + { content: '{{ q }}', role: 'user' }, + { content: '{{ a }}', role: 'assistant' }, + ], + }); + }); + + it('keeps constants even when the paired field is unmapped', () => { + const template = buildTemplate(chatMessages, {}, []); + expect(template).toEqual({ messages: [{ role: 'user' }, { role: 'assistant' }] }); + }); + + it('builds custom rows and skips ones with no key', () => { + const template = buildTemplate(custom, {}, [ + { key: 'instruction', value: '{{ prompt }}' }, + { key: '', value: '{{ ignored }}' }, + { key: 'meta.source', value: 'data-designer' }, + ]); + expect(template).toEqual({ + instruction: '{{ prompt }}', + meta: { source: 'data-designer' }, + }); + }); +}); + +describe('autoMapFields', () => { + it('prefers an exact hint match over a substring match', () => { + const mappings = autoMapFields(agentEvalTask, ['row_id', 'id', 'intent']); + expect(mappings.id).toBe('{{ id }}'); + }); + + it('falls back to a whole-word match inside a compound name', () => { + const mappings = autoMapFields(agentEvalTask, ['task_id', 'user_request']); + expect(mappings['inputs.instruction']).toBe('{{ user_request }}'); + }); + + it('matches camelCase names too', () => { + const mappings = autoMapFields(agentEvalTask, ['taskId', 'userRequest']); + expect(mappings.id).toBe('{{ taskId }}'); + expect(mappings['inputs.instruction']).toBe('{{ userRequest }}'); + }); + + it('does not let a short hint swallow an unrelated column', () => { + // `ideal_response` contains the substring "id" but has no `id` word. + const mappings = autoMapFields(agentEvalTask, ['category', 'ideal_response']); + expect(mappings.id).toBeUndefined(); + expect(mappings['reference.expected']).toBe('{{ ideal_response }}'); + }); + + it('never assigns the same column to two fields', () => { + const mappings = autoMapFields(chatMessages, ['response']); + const used = Object.values(mappings); + expect(new Set(used).size).toBe(used.length); + }); + + it('leaves a field unmapped when nothing matches', () => { + const mappings = autoMapFields(agentEvalTask, ['alpha', 'beta']); + expect(mappings['inputs.instruction']).toBeUndefined(); + }); +}); + +describe('resolveGeneratedIdColumn', () => { + it('uses the plain name when the source has no such column', () => { + expect(resolveGeneratedIdColumn(['task_id', 'prompt'])).toBe('row_id'); + }); + + it('suffixes past a collision, since a declared column cannot shadow a seed column', () => { + expect(resolveGeneratedIdColumn(['row_id'])).toBe('row_id_2'); + expect(resolveGeneratedIdColumn(['row_id', 'row_id_2'])).toBe('row_id_3'); + }); +}); + +describe('autoMapFields with a generated id', () => { + it('falls back to the generated column for an identity field with no match', () => { + const mappings = autoMapFields(agentEvalTask, ['prompt', 'answer'], 'row_id'); + expect(mappings.id).toBe('{{ row_id }}'); + }); + + it('prefers a real source column over generating one', () => { + const mappings = autoMapFields(agentEvalTask, ['task_id', 'prompt'], 'row_id'); + expect(mappings.id).toBe('{{ task_id }}'); + }); + + it('does not generate for non-identity fields', () => { + const mappings = autoMapFields(agentEvalTask, ['task_id'], 'row_id'); + expect(mappings['inputs.instruction']).toBeUndefined(); + }); +}); + +describe('templateReferences', () => { + it('collects root names through nesting, arrays, and filters', () => { + const refs = templateReferences({ + id: '{{ row_id }}', + inputs: { instruction: '{{ prompt | trim }}' }, + messages: [{ content: '{{ a.b.c }}' }], + literal: 'no references here', + }); + expect([...refs].sort()).toEqual(['a', 'prompt', 'row_id']); + }); +}); + +describe('usesGeneratedIdColumn', () => { + it('is true when the template references the generated column', () => { + expect(usesGeneratedIdColumn({ id: '{{ row_id }}' }, 'row_id', ['prompt'])).toBe(true); + }); + + it('is false once the reference is gone', () => { + expect(usesGeneratedIdColumn({ id: '{{ prompt }}' }, 'row_id', ['prompt'])).toBe(false); + }); + + it('is false when the name is a real source column, which needs no sampler', () => { + expect(usesGeneratedIdColumn({ id: '{{ row_id }}' }, 'row_id', ['row_id'])).toBe(false); + }); +}); + +describe('missingRequiredPaths', () => { + it('lists only required fields with no template', () => { + expect(missingRequiredPaths(agentEvalTask, { id: '{{ task_id }}' })).toEqual([ + 'intent', + 'inputs.instruction', + ]); + }); + + it('is empty once every required field is mapped', () => { + expect( + missingRequiredPaths(agentEvalTask, { + id: '{{ a }}', + intent: '{{ b }}', + 'inputs.instruction': '{{ c }}', + }) + ).toEqual([]); + }); +}); + +describe('columnReference', () => { + it('wraps a column in Jinja2 delimiters', () => { + expect(columnReference('user_request')).toBe('{{ user_request }}'); + }); +}); diff --git a/web/packages/studio/src/components/transform/template.ts b/web/packages/studio/src/components/transform/template.ts new file mode 100644 index 0000000000..45f04991e9 --- /dev/null +++ b/web/packages/studio/src/components/transform/template.ts @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OutputFormat } from '@studio/components/transform/formats'; + +/** A single row of the custom format's key/template grid. */ +export interface CustomTemplateRow { + readonly key: string; + readonly value: string; +} + +/** Segments that would reach Object.prototype, so a key can never pollute it. */ +const UNSAFE_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); + +/** + * Writes `value` at a dot path inside `target`, creating containers as it goes. + * A numeric segment creates (or extends) an array, so `messages.0.content` + * yields `{ messages: [{ content: value }] }`. Paths that traverse a prototype + * are dropped whole — the template is user-authored and is also applied to rows + * in the browser for the preview. + */ +export const setAtPath = ( + target: Record, + path: string, + value: unknown +): Record => { + const segments = path.split('.').filter(Boolean); + if (segments.length === 0 || segments.some((segment) => UNSAFE_SEGMENTS.has(segment))) { + return target; + } + + let cursor: Record | unknown[] = target; + for (let index = 0; index < segments.length - 1; index += 1) { + const segment = segments[index]; + const nextIsIndex = /^\d+$/.test(segments[index + 1]); + const existing = readSegment(cursor, segment); + const container = + isContainer(existing) && Array.isArray(existing) === nextIsIndex + ? existing + : nextIsIndex + ? [] + : {}; + writeSegment(cursor, segment, container); + cursor = container; + } + + writeSegment(cursor, segments[segments.length - 1], value); + return target; +}; + +const isContainer = (value: unknown): value is Record | unknown[] => + typeof value === 'object' && value !== null; + +const readSegment = (cursor: Record | unknown[], segment: string): unknown => + Array.isArray(cursor) ? cursor[Number(segment)] : cursor[segment]; + +const writeSegment = ( + cursor: Record | unknown[], + segment: string, + value: unknown +): void => { + if (Array.isArray(cursor)) { + cursor[Number(segment)] = value; + } else { + cursor[segment] = value; + } +}; + +/** + * Builds the `schema_transform` template. Preset fields are written at their dot + * paths in declaration order, then the format's constants, so a literal (like a + * chat role) can never be clobbered by a mapping. Blank mappings are skipped — + * an unmapped optional field simply does not appear in the output. + */ +export const buildTemplate = ( + format: OutputFormat, + mappings: Readonly>, + customRows: readonly CustomTemplateRow[] +): Record => { + const template: Record = {}; + + for (const field of format.fields) { + const value = mappings[field.path]?.trim(); + if (value) { + setAtPath(template, field.path, value); + } + } + + for (const [path, value] of Object.entries(format.constants ?? {})) { + setAtPath(template, path, value); + } + + for (const row of customRows) { + const key = row.key.trim(); + if (key) { + setAtPath(template, key, row.value.trim()); + } + } + + return template; +}; + +/** Wraps a source column name as the Jinja2 reference the processor expects. */ +export const columnReference = (column: string): string => `{{ ${column} }}`; + +/** Preferred name for the identifier column the transform job generates. */ +const GENERATED_ID_BASE = 'row_id'; + +/** + * Name for the generated identifier column. Data Designer rejects a declared + * column whose name collides with a seed column, so suffix until it is free. + */ +export const resolveGeneratedIdColumn = (columns: readonly string[]): string => { + const taken = new Set(columns); + if (!taken.has(GENERATED_ID_BASE)) { + return GENERATED_ID_BASE; + } + let suffix = 2; + while (taken.has(`${GENERATED_ID_BASE}_${suffix}`)) { + suffix += 1; + } + return `${GENERATED_ID_BASE}_${suffix}`; +}; + +/** + * Guesses a mapping for each field from the source columns: an exact hint match + * wins over a substring match, and earlier hints win over later ones. Columns + * are never reused, so two fields cannot both claim `response`. + * + * An identity field with no match falls back to `generatedIdColumn`, which the + * job creates as a UUID sampler rather than reading from the source. + */ +export const autoMapFields = ( + format: OutputFormat, + columns: readonly string[], + generatedIdColumn?: string +): Record => { + const claimed = new Set(); + const mappings: Record = {}; + + for (const field of format.fields) { + const match = findColumnMatch(field.hints, columns, claimed); + if (match) { + claimed.add(match); + mappings[field.path] = columnReference(match); + } else if (field.identity && generatedIdColumn) { + mappings[field.path] = columnReference(generatedIdColumn); + } + } + + return mappings; +}; + +/** Root variable names referenced by any template value, e.g. `{{ a.b | upper }}` → `a`. */ +export const templateReferences = (template: Record): Set => { + const found = new Set(); + const visit = (value: unknown): void => { + if (typeof value === 'string') { + for (const match of value.matchAll(/\{\{\s*([A-Za-z_][A-Za-z0-9_]*)/g)) { + found.add(match[1]); + } + } else if (Array.isArray(value)) { + value.forEach(visit); + } else if (typeof value === 'object' && value !== null) { + Object.values(value).forEach(visit); + } + }; + visit(template); + return found; +}; + +/** + * Whether the finished template actually uses the generated identifier. Checked + * against the rendered template rather than the mapping state so that clearing + * or overwriting the field also drops the column from the job. + */ +export const usesGeneratedIdColumn = ( + template: Record, + generatedIdColumn: string, + columns: readonly string[] +): boolean => + !columns.includes(generatedIdColumn) && templateReferences(template).has(generatedIdColumn); + +/** + * Splits a column name into words, treating `_`, `-` and camelCase humps as + * boundaries: `userRequest` and `user_request` both become `['user', 'request']`. + */ +const tokenize = (column: string): string[] => + column + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); + +/** + * Matches on whole words rather than substrings. A plain `includes` lets a short + * hint swallow an unrelated column — `id` matches `ideal_response` — which is + * worse than no guess, since a wrong auto-map looks deliberate. + */ +const findColumnMatch = ( + hints: readonly string[], + columns: readonly string[], + claimed: ReadonlySet +): string | undefined => { + const available = columns + .filter((column) => !claimed.has(column)) + .map((column) => ({ column, tokens: tokenize(column) })); + + for (const hint of hints) { + const exact = available.find(({ tokens }) => tokens.join('_') === hint); + if (exact) { + return exact.column; + } + } + for (const hint of hints) { + const word = available.find(({ tokens }) => tokens.includes(hint)); + if (word) { + return word.column; + } + } + return undefined; +}; + +/** Paths of required fields that have no template yet. */ +export const missingRequiredPaths = ( + format: OutputFormat, + mappings: Readonly> +): string[] => + format.fields + .filter((field) => field.required && !mappings[field.path]?.trim()) + .map((field) => field.path); diff --git a/web/packages/studio/src/components/transform/useTransformMapping.ts b/web/packages/studio/src/components/transform/useTransformMapping.ts new file mode 100644 index 0000000000..f89d637925 --- /dev/null +++ b/web/packages/studio/src/components/transform/useTransformMapping.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isMappingDraftDirty } from '@studio/components/transform/draft'; +import { + CUSTOM_FORMAT_ID, + findOutputFormat, + OUTPUT_FORMATS, + type OutputFormat, +} from '@studio/components/transform/formats'; +import { + autoMapFields, + buildTemplate, + columnReference, + missingRequiredPaths, + resolveGeneratedIdColumn, + usesGeneratedIdColumn, + type CustomTemplateRow, +} from '@studio/components/transform/template'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +const BLANK_ROW: CustomTemplateRow = { key: '', value: '' }; + +export interface TransformMapping { + format: OutputFormat; + isCustom: boolean; + setFormat: (id: string) => void; + mappings: Record; + setMapping: (path: string, value: string) => void; + rawPaths: ReadonlySet; + toggleRaw: (path: string) => void; + customRows: CustomTemplateRow[]; + setCustomRows: (rows: CustomTemplateRow[]) => void; + /** Source columns the mapping was built against. */ + columns: readonly string[]; + /** The finished `schema_transform` template. */ + template: Record; + missingRequired: string[]; + generatedIdColumn: string; + /** Whether the template still references the generated identifier column. */ + needsGeneratedId: boolean; + /** No required field is unmapped and the template writes at least one key. */ + isComplete: boolean; + /** Whether the mapping holds work the user would lose by closing. */ + isDirty: boolean; +} + +interface Options { + /** Column names of the source file, as read from its first row. */ + columns: readonly string[]; + /** Called when the user picks a different target format, for dependent defaults. */ + onFormatChange?: (format: OutputFormat) => void; +} + +/** + * The custom grid starts as a passthrough of the source columns, so renaming or + * dropping a couple of fields is an edit rather than a from-scratch schema. The + * trailing blank row is what `CustomTemplateRows` appends to. + */ +const baselineCustomRows = (isCustom: boolean, columns: readonly string[]): CustomTemplateRow[] => + isCustom + ? [...columns.map((column) => ({ key: column, value: columnReference(column) })), BLANK_ROW] + : [BLANK_ROW]; + +/** + * Owns the field mapping shared by every transform surface: the chosen output + * format, the per-field templates, and the custom key/template grid. The + * mapping is re-guessed from the source columns whenever either changes, so a + * caller only has to supply the columns. + */ +export const useTransformMapping = ({ columns, onFormatChange }: Options): TransformMapping => { + const [formatId, setFormatId] = useState(OUTPUT_FORMATS[0].id); + const [mappings, setMappings] = useState>({}); + const [rawPaths, setRawPaths] = useState>(new Set()); + const [customRows, setCustomRows] = useState([BLANK_ROW]); + + const format = findOutputFormat(formatId) ?? OUTPUT_FORMATS[0]; + const isCustom = format.id === CUSTOM_FORMAT_ID; + const generatedIdColumn = useMemo(() => resolveGeneratedIdColumn(columns), [columns]); + + // Re-guess the mapping whenever the target format or the source columns change. + const columnsKey = columns.join(','); + useEffect(() => { + setMappings(autoMapFields(format, columns, generatedIdColumn)); + setRawPaths(new Set()); + setCustomRows(baselineCustomRows(format.id === CUSTOM_FORMAT_ID, columns)); + // `columnsKey` stands in for `columns`, which is a new array on every render. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [format, columnsKey]); + + const setFormat = useCallback( + (id: string) => { + const next = findOutputFormat(id); + if (!next) { + return; + } + setFormatId(next.id); + onFormatChange?.(next); + }, + [onFormatChange] + ); + + const setMapping = useCallback((path: string, value: string) => { + setMappings((prev) => ({ ...prev, [path]: value })); + }, []); + + const toggleRaw = useCallback((path: string) => { + setRawPaths((prev) => { + const next = new Set(prev); + if (next.has(path)) { + next.delete(path); + } else { + next.add(path); + } + return next; + }); + }, []); + + const template = useMemo( + () => buildTemplate(format, mappings, isCustom ? customRows : []), + [format, mappings, isCustom, customRows] + ); + const missingRequired = useMemo( + () => (isCustom ? [] : missingRequiredPaths(format, mappings)), + [isCustom, format, mappings] + ); + + const isDirty = useMemo( + () => + isMappingDraftDirty( + { mappings, rawPaths, customRows }, + { + mappings: autoMapFields(format, columns, generatedIdColumn), + customRows: baselineCustomRows(format.id === CUSTOM_FORMAT_ID, columns), + } + ), + // eslint-disable-next-line react-hooks/exhaustive-deps + [mappings, rawPaths, customRows, format, columnsKey, generatedIdColumn] + ); + + return { + format, + isCustom, + setFormat, + mappings, + setMapping, + rawPaths, + toggleRaw, + customRows, + setCustomRows, + columns, + template, + missingRequired, + generatedIdColumn, + // Only true while the finished template still references it, so clearing or + // overwriting the id field also drops the column from the job. + needsGeneratedId: usesGeneratedIdColumn(template, generatedIdColumn, columns), + isComplete: missingRequired.length === 0 && Object.keys(template).length > 0, + isDirty, + }; +}; diff --git a/web/packages/studio/src/components/transform/useTransformPreview.test.ts b/web/packages/studio/src/components/transform/useTransformPreview.test.ts new file mode 100644 index 0000000000..78a04c37a0 --- /dev/null +++ b/web/packages/studio/src/components/transform/useTransformPreview.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useTransformPreview } from '@studio/components/transform/useTransformPreview'; +import { act, renderHook } from '@testing-library/react'; + +const fileContent = JSON.stringify([{ name: 'Ada', role: 'engineer' }]); + +describe('useTransformPreview', () => { + it('renders the template against the current row, keeping its shape', () => { + const { result } = renderHook(() => + useTransformPreview({ + fileContent, + fileType: 'json', + template: { user: { name: '{{ name }}' }, messages: [{ content: '{{ role }}' }] }, + }) + ); + + expect(result.current.afterRow).toEqual({ + user: { name: 'Ada' }, + messages: [{ content: 'engineer' }], + }); + }); + + it('has no output until the template writes a key', () => { + const { result } = renderHook(() => + useTransformPreview({ fileContent, fileType: 'json', template: {} }) + ); + + expect(result.current.sourceRow).toEqual({ name: 'Ada', role: 'engineer' }); + expect(result.current.afterRow).toBeNull(); + }); + + it('keeps rendering when one value holds an invalid template', () => { + const { result } = renderHook(() => + useTransformPreview({ + fileContent, + fileType: 'json', + template: { broken: '{{#if}}', name: '{{ name }}' }, + }) + ); + + expect(result.current.afterRow).toEqual({ broken: '{{#if}}', name: 'Ada' }); + expect(result.current.approximated).toBe(true); + }); + + it('reports the row actually displayed when the row count shrinks', () => { + const { result, rerender } = renderHook( + (props: { fileContent: string }) => + useTransformPreview({ + fileContent: props.fileContent, + fileType: 'json', + template: { name: '{{ name }}' }, + }), + { + initialProps: { + fileContent: JSON.stringify([{ name: 'Ada' }, { name: 'Grace' }, { name: 'Katherine' }]), + }, + } + ); + + act(() => result.current.onRowChange(3)); + expect(result.current.currentRow).toBe(3); + + rerender({ fileContent: JSON.stringify([{ name: 'Ada' }]) }); + + expect(result.current.currentRow).toBe(1); + expect(result.current.totalRows).toBe(1); + }); +}); diff --git a/web/packages/studio/src/components/transform/useTransformPreview.ts b/web/packages/studio/src/components/transform/useTransformPreview.ts new file mode 100644 index 0000000000..bad2e8345a --- /dev/null +++ b/web/packages/studio/src/components/transform/useTransformPreview.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { renderTemplate } from '@studio/components/transform/renderTemplate'; +import { parseFileContent, type Row } from '@studio/util/files'; +import { useMemo, useState } from 'react'; + +interface Props { + fileContent: string | undefined; + fileType: string; + /** The `schema_transform` template applied to the previewed row. */ + template: Record; + /** + * Column the transform generates rather than reads. The preview stands in a + * placeholder for it so a generated identifier does not render as empty. + */ + generatedIdColumn?: string; +} + +/** Stable stand-in for a generated identifier, so the preview does not churn. */ +const placeholderId = (index: number): string => + ((index + 1) * 2654435761).toString(16).slice(-8).padStart(8, '0'); + +/** + * Renders one source row through the current template so the mapping can be + * checked against real data before anything is written. + */ +export const useTransformPreview = ({ + fileContent, + fileType, + template, + generatedIdColumn, +}: Props) => { + const [currentRow, setCurrentRow] = useState(1); + + const rows = useMemo(() => { + if (!fileContent) return []; + try { + return parseFileContent({ content: fileContent, fileType }).rows; + } catch { + return []; + } + }, [fileContent, fileType]); + + const rowIndex = Math.min(currentRow - 1, rows.length - 1); + const sourceRow: Row | null = rows[rowIndex] ?? null; + + const rendered = useMemo(() => { + if (!sourceRow || Object.keys(template).length === 0) return null; + const input = generatedIdColumn + ? { ...sourceRow, [generatedIdColumn]: placeholderId(rowIndex) } + : sourceRow; + return renderTemplate(template, input); + }, [sourceRow, template, generatedIdColumn, rowIndex]); + + const totalRows = rows.length; + + // `currentRow` can outlive the file it was chosen for, so report the row actually shown. + const displayedRow = totalRows === 0 ? currentRow : rowIndex + 1; + + const onRowChange = (row: number) => setCurrentRow(Math.min(Math.max(1, row), totalRows)); + + return { + currentRow: displayedRow, + totalRows, + sourceRow, + afterRow: rendered?.row ?? null, + approximated: rendered?.approximated ?? false, + onRowChange, + }; +}; diff --git a/web/packages/studio/src/routes/DataDesignerJobDetailsRoute/index.tsx b/web/packages/studio/src/routes/DataDesignerJobDetailsRoute/index.tsx index 1358c6a34a..ba7c3c8ae1 100644 --- a/web/packages/studio/src/routes/DataDesignerJobDetailsRoute/index.tsx +++ b/web/packages/studio/src/routes/DataDesignerJobDetailsRoute/index.tsx @@ -17,9 +17,11 @@ import { Text, } from '@nvidia/foundations-react-core'; import { DataDesignerJobActionsMenu } from '@studio/components/DataDesignerJobActionsMenu'; +import { DataDesignerTransformModal } from '@studio/components/DataDesignerTransformModal'; import { CreateFileSplitsModal } from '@studio/components/FilesTable/CreateFileSplitsModal'; import { Loading } from '@studio/components/Layouts/Loading'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { BUILDER_CONFIG_FILENAME } from '@studio/routes/DataDesignerJobDetailsRoute/builderConfig'; import { DataDesignerConfigPanel } from '@studio/routes/DataDesignerJobDetailsRoute/DataDesignerConfigPanel'; import { DatasetProfilerSection } from '@studio/routes/DataDesignerJobDetailsRoute/DatasetProfilerSection'; import { JobDatasetEditorSection } from '@studio/routes/DataDesignerJobDetailsRoute/JobDatasetEditorSection'; @@ -29,7 +31,7 @@ import { useDataDesignerArtifactsFileset } from '@studio/routes/DataDesignerJobD import { useDataDesignerJobFromRoute } from '@studio/routes/DataDesignerJobDetailsRoute/useDataDesignerJobFromRoute'; import { getDataDesignerJobListRoute } from '@studio/routes/utils'; import { formatDateTime } from '@studio/util/date'; -import { ArrowLeft, Split } from 'lucide-react'; +import { ArrowLeft } from 'lucide-react'; import { useRef, useState, type FC } from 'react'; import { Link, useNavigate } from 'react-router'; @@ -48,6 +50,7 @@ export const DataDesignerJobDetailsRoute: FC = () => { const navigate = useNavigate(); const [isConfigPanelOpen, setIsConfigPanelOpen] = useState(false); const [isSplitModalOpen, setIsSplitModalOpen] = useState(false); + const [isTransformModalOpen, setIsTransformModalOpen] = useState(false); const [cancelError, setCancelError] = useState(undefined); const [selectedTab, setSelectedTab] = useState(undefined); @@ -65,6 +68,11 @@ export const DataDesignerJobDetailsRoute: FC = () => { .filter((path) => /\.(json|jsonl|parquet)$/i.test(path)); const canSplit = Boolean(splitDatasetId) && splitFileOptions.length > 0; + const transformFileOptions = splitFileOptions.filter( + (path) => !path.endsWith(BUILDER_CONFIG_FILENAME) + ); + const canTransform = Boolean(filesetWorkspace && filesetName) && transformFileOptions.length > 0; + useBreadcrumbs({ items: [ { @@ -114,17 +122,22 @@ export const DataDesignerJobDetailsRoute: FC = () => { {job.status ? : null} - setIsTransformModalOpen(true), + }, + { + label: 'Split', + disabled: !canSplit, + onSelect: () => setIsSplitModalOpen(true), + divider: {}, + }, + ]} onViewConfig={() => setIsConfigPanelOpen(true)} onDeleted={() => navigate(getDataDesignerJobListRoute(workspace))} onCancelError={setCancelError} @@ -190,6 +203,19 @@ export const DataDesignerJobDetailsRoute: FC = () => { onClose={() => setIsConfigPanelOpen(false)} /> + {isTransformModalOpen && ( + setIsTransformModalOpen(false)} + workspace={workspace} + sourceJobName={job.name} + filesetWorkspace={filesetWorkspace} + filesetName={filesetName} + fileOptions={transformFileOptions} + defaultNumRecords={job.spec?.job_config?.num_records ?? 0} + /> + )} + {isSplitModalOpen && (