feat(studio): Transform through Data Designer Processors

web/packages/studio/src/api/datasets/constants.ts

@@ -39,5 +39,3 @@ export const BINARY_FILE_EXTENSIONS = new Set([
   // Documents
   'pdf',
 ]);
-
-export const COMPLETION_PROMPT_KEY_ORDER = ['prompt', 'instruction', 'question']; // Searches for a prompt in the following keys

web/packages/studio/src/api/datasets/useDatasetFileTransform.ts

@@ -1,188 +1,85 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { useChatCompletions } from '@nemo/common/src/hooks/useChatCompletions';
-import { getEntityReference } from '@nemo/common/src/namedEntity';
 import { useToast } from '@nemo/common/src/providers/toast/useToast';
-import { isDefined } from '@nemo/common/src/utils/isDefined';
 import { filesUploadFile } from '@nemo/sdk/generated/platform/api';
-import type { FilesetFileOutput, ModelEntity } from '@nemo/sdk/generated/platform/schema';
-import { COMPLETION_PROMPT_KEY_ORDER } from '@studio/api/datasets/constants';
+import type { FilesetFileOutput } from '@nemo/sdk/generated/platform/schema';
 import { invalidateDatasetCaches } from '@studio/api/datasets/invalidateDatasetCaches';
-import { TransformFileFormFields } from '@studio/components/FilesTable/TransformFileModal/types';
-import { parseFileContent, Row } from '@studio/util/files';
+import { renderTemplate } from '@studio/components/transform/renderTemplate';
+import { parseFileContent } from '@studio/util/files';
 import { useMutation, UseMutationOptions } from '@tanstack/react-query';
-import Handlebars from 'handlebars';
-import { ChatCompletion, ChatCompletionCreateParams } from 'openai/resources/index.mjs';
-import { useCallback, useState } from 'react';
+import { useCallback } from 'react';
 
-type MutationProps = {
+interface MutationProps {
   workspace: string;
   datasetName: string;
-  filepath: TransformFileFormFields['filepath'];
-  mappings: TransformFileFormFields['mappings'];
+  filepath: string;
+  /** The `schema_transform` template each row is rewritten through. */
+  template: Record<string, unknown>;
   fileContent: string;
-  model?: ModelEntity;
-};
+  /**
+   * Column the template references but the source file does not have. A Data
+   * Designer job would declare it as a UUID sampler; here each row gets its own
+   * short identifier as it is rewritten.
+   */
+  generatedIdColumn?: string;
+}
 
 type Props = Omit<UseMutationOptions<FilesetFileOutput, Error, MutationProps>, 'mutationFn'>;
 
+/**
+ * Rewrites a fileset file in place through a transform template. The mapping is
+ * applied in the browser with the same renderer that drives the preview, so what
+ * the user approved is exactly what is uploaded.
+ */
 export const useDatasetFileTransform = ({ onError, onSuccess }: Props) => {
   const toast = useToast();
-  const { mutateAsync: createChatCompletions } = useChatCompletions();
-  const [progressLabel, setProgressLabel] = useState<string>('');
-  const [progressValue, setProgressValue] = useState<number>(0);
-  const progPreInferValue = 20;
-  const progPostInferValue = 90;
 
   const mutationFn = useCallback(
-    async ({ fileContent, filepath, model, mappings, workspace, datasetName }: MutationProps) => {
-      setProgressValue(10);
-
-      // Parse JSON objects
-      const content = parseFileContent({
+    async ({
+      fileContent,
+      filepath,
+      template,
+      workspace,
+      datasetName,
+      generatedIdColumn,
+    }: MutationProps) => {
+      const { rows, failures } = parseFileContent({
         content: fileContent,
         fileType: filepath.split('.').at(-1),
       });
-      const { failures } = content;
-      let { rows } = content;
       if (failures?.length) {
         toast.error(`${failures.length} Line(s) had parsing errors.`);
       }
 
-      // Re-map each row to the described mappings
-      if (mappings) {
-        rows = rows
-          .map((row) => {
-            const newRow: Record<string, unknown> = {};
-            let skipInvalidRow = false;
-            mappings.forEach(({ key, value }) => {
-              // Pre-process the row to stringify arrays and objects
-              const processedRow = Object.fromEntries(
-                Object.entries(row).map(([k, v]) => [
-                  k,
-                  Array.isArray(v) || (typeof v === 'object' && v !== null) ? JSON.stringify(v) : v,
-                ])
-              );
-
-              const template = Handlebars.compile(value);
-              // Handle nested keys (e.g. "user.profile.name")
-              const keyParts = key.split('.');
-              let current = newRow;
-
-              // Process all parts except the last one
-              for (let i = 0; i < keyParts.length - 1; i++) {
-                const part = keyParts[i];
-                if (!(part in current)) {
-                  current[part] = {};
-                }
-                current = current[part] as Record<string, unknown>;
-              }
-
-              // Handle the last part of the key
-              const lastPart = keyParts[keyParts.length - 1];
-              const compiledValue = template(processedRow);
-
-              // Try to parse as JSON if it looks like an array or object
-              try {
-                if (compiledValue.trim().startsWith('[') || compiledValue.trim().startsWith('{')) {
-                  current[lastPart] = JSON.parse(compiledValue);
-                } else {
-                  current[lastPart] = compiledValue;
-                }
-              } catch {
-                skipInvalidRow = true;
-              }
-            });
-            return skipInvalidRow ? undefined : newRow;
-          })
-          .filter(Boolean) as Row[];
-      }
-      setProgressValue(progPreInferValue);
-
-      // Generate completions if necessary
-      if (model) {
-        const chatCompletionRequests = rows
-          .map((row) => {
-            let userMsg = '';
-            for (const key of COMPLETION_PROMPT_KEY_ORDER) {
-              if (key in row) {
-                userMsg = row[key] as string;
-                break;
-              }
-            }
-            if (!userMsg) {
-              return undefined;
-            }
-            const messages = [
-              {
-                role: 'user',
-                content: userMsg,
-              },
-            ];
-            if (model.prompt?.system_prompt) {
-              messages.unshift({ role: 'system', content: model.prompt.system_prompt });
-            }
-            return {
-              messages,
-              model: getEntityReference(model),
-            };
-          })
-          .filter(isDefined) as ChatCompletionCreateParams[];
-        setProgressLabel(`Inferencing ${chatCompletionRequests.length} rows...`);
-        const completions = (await createChatCompletions({
-          requests: chatCompletionRequests,
-          onTaskComplete: ({ completedTasks }) => {
-            const ratioComplete = completedTasks / chatCompletionRequests.length;
-            const inferProgress = ratioComplete * 70;
-            setProgressLabel(`Inferencing... (${completedTasks}/${chatCompletionRequests.length})`);
-            setProgressValue(progPreInferValue + inferProgress);
-          },
-        })) as ChatCompletion[];
-        rows = completions.map((completion, index) => {
-          return {
-            input: { category: '', ...rows[index] },
-            response: completion.choices[0].message.content,
-            llm_name: model.name,
-          };
-        });
-      }
-      setProgressValue(progPostInferValue);
-
-      // Upload file to fileset
-      setProgressLabel('Uploading...');
-      setProgressValue(95);
-      const fileContent2 = rows.map((row) => JSON.stringify(row)).join('\n');
-      const blob = new Blob([fileContent2], { type: 'application/json' });
+      const transformed = rows.map((row) => {
+        const input = generatedIdColumn
+          ? { ...row, [generatedIdColumn]: crypto.randomUUID().replaceAll('-', '').slice(0, 8) }
+          : row;
+        return renderTemplate(template, input).row;
+      });
+      const blob = new Blob([transformed.map((row) => JSON.stringify(row)).join('\n')], {
+        type: 'application/json',
+      });
 
       return filesUploadFile(workspace, datasetName, filepath, blob);
     },
-    [createChatCompletions, toast]
+    [toast]
   );
 
-  return {
-    ...useMutation({
-      mutationFn,
-      onError: (data, variables, onMutateResult, context) => {
-        onError?.(data, variables, onMutateResult, context);
-      },
-      onSuccess: (data, variables, onMutateResult, context) => {
-        invalidateDatasetCaches(
-          variables.workspace,
-          variables.datasetName,
-          ['files', 'content'],
-          variables.filepath
-        );
-        onSuccess?.(data, variables, onMutateResult, context);
-      },
-      onMutate: () => {
-        setProgressLabel('');
-      },
-      onSettled: () => {
-        setProgressLabel('');
-      },
-    }),
-    progressLabel,
-    progressValue,
-  };
+  return useMutation({
+    mutationFn,
+    onError: (data, variables, onMutateResult, context) => {
+      onError?.(data, variables, onMutateResult, context);
+    },
+    onSuccess: (data, variables, onMutateResult, context) => {
+      invalidateDatasetCaches(
+        variables.workspace,
+        variables.datasetName,
+        ['files', 'content'],
+        variables.filepath
+      );
+      onSuccess?.(data, variables, onMutateResult, context);
+    },
+  });
 };

web/packages/studio/src/components/DataDesignerTransformModal/buildTransformJobRequest.test.ts

@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  buildSeedPath,
+  buildTransformJobRequest,
+} from '@studio/components/DataDesignerTransformModal/buildTransformJobRequest';
+
+const input = {
+  jobName: 'support-evals-agent-eval-tasks',
+  processorName: 'agent_eval_tasks',
+  filesetWorkspace: 'default',
+  filesetName: 'support-evals-artifacts',
+  filePath: 'dataset.parquet',
+  numRecords: 500,
+  template: { id: '{{ task_id }}' },
+};
+
+describe('buildSeedPath', () => {
+  it('addresses a single file inside a workspace-qualified fileset', () => {
+    expect(buildSeedPath('default', 'my-fileset', 'nested/dataset.parquet')).toBe(
+      'default/my-fileset#nested/dataset.parquet'
+    );
+  });
+});
+
+describe('buildTransformJobRequest', () => {
+  it('declares no columns so the job generates nothing', () => {
+    expect(buildTransformJobRequest(input).spec.config.columns).toEqual([]);
+  });
+
+  it('declares a UUID sampler when an id column has to be generated', () => {
+    const request = buildTransformJobRequest({ ...input, generatedIdColumn: 'row_id' });
+    expect(request.spec.config.columns).toEqual([
+      {
+        name: 'row_id',
+        column_type: 'sampler',
+        sampler_type: 'uuid',
+        params: { short_form: true },
+      },
+    ]);
+    // A sampler needs no model, so the job is still inference-free.
+    expect(request.spec.config.model_configs).toBeUndefined();
+  });
+
+  it('seeds from the source file in order', () => {
+    const { seed_config: seedConfig } = buildTransformJobRequest(input).spec.config;
+    expect(seedConfig).toEqual({
+      source: { seed_type: 'nmp', path: 'default/support-evals-artifacts#dataset.parquet' },
+      sampling_strategy: 'ordered',
+    });
+  });
+
+  it('carries the template through on a schema_transform processor', () => {
+    expect(buildTransformJobRequest(input).spec.config.processors).toEqual([
+      {
+        processor_type: 'schema_transform',
+        name: 'agent_eval_tasks',
+        template: { id: '{{ task_id }}' },
+      },
+    ]);
+  });
+
+  it('requests no models, since nothing is generated', () => {
+    expect(buildTransformJobRequest(input).spec.config.model_configs).toBeUndefined();
+  });
+
+  it('passes the row count and job name through', () => {
+    const request = buildTransformJobRequest(input);
+    expect(request.name).toBe('support-evals-agent-eval-tasks');
+    expect(request.spec.num_records).toBe(500);
+  });
+});