feat: data copilot text to sql

.gitignore

@@ -10,4 +10,6 @@ scripts/scaffold.yaml
 node_modules
 .prettierrc
 **/.env*
-!.env.dist
+!.env.dist
+database/Dockerfile.flyway
+database/flyway_migrate.sh

database/migrate.sh

@@ -2,7 +2,14 @@
 set -ex
 set +o history
 
-# Grab all command line arguments to pass them into Docker, or default to "migrate".
+# Check if first argument is --host-network
+DOCKER_NETWORK=""
+if [ "$1" = "--host-network" ]; then
+    DOCKER_NETWORK="--network host"
+    shift  # Remove --host-network from arguments
+fi
+
+# Grab remaining command line arguments to pass them into Docker, or default to "migrate".
 if [ $# -eq 0 ]; then
     FLYWAY_COMMAND=("migrate")
 else
@@ -11,7 +18,7 @@ fi
 
 echo "Running Flyway command: ${FLYWAY_COMMAND[@]} on jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}"
 
-docker run --rm \
+docker run --rm ${DOCKER_NETWORK} \
   -v "$(pwd)/migrations:/tmp/migrations" \
   flyway/flyway:latest-alpine \
   -locations="filesystem:/tmp/migrations" \

database/migrations/V1756973486__addConversationIdToChatResponses.sql

@@ -0,0 +1,8 @@
+ALTER TABLE chat_responses 
+ADD COLUMN conversation_id UUID DEFAULT gen_random_uuid();
+
+-- Create index for efficient conversation queries
+CREATE INDEX idx_chat_responses_conversation_id ON chat_responses(conversation_id);
+
+-- Create index for efficient conversation + timestamp queries  
+CREATE INDEX idx_chat_responses_conversation_created_at ON chat_responses(conversation_id, created_at);

database/migrations/V1756973487__alterChatResponsesRouterResponseCheck.sql

@@ -0,0 +1,9 @@
+-- Drop the existing check constraint
+ALTER TABLE chat_responses DROP CONSTRAINT chat_responses_router_response_check;
+
+UPDATE chat_responses SET router_response = 'create_query' 
+WHERE router_response = 'text-to-sql';
+
+-- Add the new check constraint with 'create_query' instead of 'text-to-sql'
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check 
+    CHECK (router_response IN ('pipes', 'create_query', 'stop'));

database/migrations/V1759392166__addAskClarificationRouterResponseType.sql

@@ -0,0 +1,9 @@
+-- Drop the existing check constraint
+ALTER TABLE chat_responses DROP CONSTRAINT chat_responses_router_response_check;
+
+-- Add the new check constraint with 'ask_clarification'
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check
+    CHECK (router_response IN ('pipes', 'create_query', 'stop', 'ask_clarification'));
+
+-- Add clarification_question column to store the clarification question
+ALTER TABLE chat_responses ADD COLUMN IF NOT EXISTS clarification_question TEXT;

database/migrations/V1759927411__createChatResponseAgentStepsTable.sql

@@ -0,0 +1,27 @@
+-- Create enum for agent types
+CREATE TYPE agent_type AS ENUM ('ROUTER', 'PIPE', 'TEXT_TO_SQL', 'AUDITOR', 'CHART', 'EXECUTE_INSTRUCTIONS');
+
+-- Create table to track individual agent execution steps
+CREATE TABLE IF NOT EXISTS chat_response_agent_steps (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    chat_response_id UUID NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    model TEXT,
+    agent agent_type NOT NULL,
+    response JSONB,
+    input_tokens INTEGER DEFAULT 0,
+    output_tokens INTEGER DEFAULT 0,
+    response_time_seconds NUMERIC NOT NULL DEFAULT 0,
+    instructions TEXT,
+    error_message TEXT,
+
+    CONSTRAINT fk_chat_response
+        FOREIGN KEY (chat_response_id)
+        REFERENCES chat_responses(id)
+        ON DELETE CASCADE
+);
+
+-- Create indexes for efficient querying
+CREATE INDEX idx_agent_steps_chat_response_id ON chat_response_agent_steps(chat_response_id);
+CREATE INDEX idx_agent_steps_created_at ON chat_response_agent_steps(created_at DESC);
+CREATE INDEX idx_agent_steps_agent_type ON chat_response_agent_steps(agent);

database/migrations/V1759927412__makeChatResponsesNullable.sql

@@ -0,0 +1,18 @@
+-- Make router fields nullable to allow early creation of chat_responses
+ALTER TABLE chat_responses ALTER COLUMN router_response DROP NOT NULL;
+ALTER TABLE chat_responses ALTER COLUMN router_reason DROP NOT NULL;
+
+-- Drop existing constraints
+ALTER TABLE chat_responses DROP CONSTRAINT IF EXISTS chat_responses_router_response_check;
+ALTER TABLE chat_responses DROP CONSTRAINT IF EXISTS check_pipe_instructions;
+
+-- Add new constraint that allows NULL or valid enum values
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check
+    CHECK (router_response IS NULL OR router_response IN ('pipes', 'create_query', 'stop', 'ask_clarification'));
+
+-- Recreate pipe_instructions check with NULL handling
+ALTER TABLE chat_responses ADD CONSTRAINT check_pipe_instructions CHECK (
+    router_response IS NULL OR
+    (router_response = 'pipes' AND pipe_instructions IS NOT NULL) OR
+    (router_response != 'pipes' AND pipe_instructions IS NULL)
+);

frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue

@@ -24,33 +24,11 @@ SPDX-License-Identifier: MIT
     >
       {{ reasoning }}
     </div>
-    <div class="my-4">{{ message.content }}</div>
-
-    <span 
-      class="flex items-center p-3 border border-solid border-neutral-200 
-      rounded-xl bg-white justify-between cursor-pointer hover:bg-neutral-50"
-      @click="emit('select')"
-    >
-      <lfx-chat-result-label
-        :version="version"
-        :label="getTitle(message.id)"
-      />
-      <lfx-icon
-        v-if="!isSelected"
-        name="arrow-rotate-left"
-        :size="16"
-        class="text-neutral-400"
-      />
-    </span>
-
   </div>
 </template>
 <script setup lang="ts">
 import { computed, ref } from 'vue';
-import { storeToRefs } from 'pinia';
 import type { AIMessage } from '../../types/copilot.types'
-import LfxChatResultLabel from '../shared/result-label.vue'
-import { useCopilotStore } from '../../store/copilot.store';
 import LfxChatLabel from './chat-label.vue'
 import LfxIcon from '~/components/uikit/icon/icon.vue'
 
@@ -60,23 +38,12 @@ const props = defineProps<{
   isSelected: boolean | undefined
 }>()
 
-const { resultData } = storeToRefs(useCopilotStore());
-
 const isReasonExpanded = ref(false);
-// TODO: Implement feedback backend
-
-const emit = defineEmits<{
-  (e: 'select'): void
-}>()
 
 const reasoning = computed(() => {
   return props.message.explanation || props.message.sql;
 })
 
-const getTitle = (id: string) => {
-  const result = resultData.value.find(r => String(r.id) === String(id));
-  return result?.title || 'Loading...';
-}
 </script>
 
 <script lang="ts">

frontend/app/components/shared/modules/copilot/components/copilot-modal.vue

@@ -78,11 +78,11 @@ const isModalOpen = computed({
   }
 })
 
-const handleDataUpdate = (id: string, data: MessageData[], routerReasoning?: string) => {
+const handleDataUpdate = (id: string, data: MessageData[], conversationId?: string) => {
   resultData.value.push({
     id,
     data,
-    routerReasoning
+    conversationId
   });
 
 

frontend/app/components/shared/modules/copilot/components/copilot-sidebar.vue

@@ -133,7 +133,7 @@ const emit = defineEmits<{
   (e: 'update:selectedResult', value: string): void;
   (e: 'update:isLoading', value: boolean): void;
   (e: 'update:error', value: string): void;
-  (e: 'update:data', id: string, value: MessageData[], routerReasoning?: string): void;
+  (e: 'update:data', id: string, value: MessageData[], conversationId?: string): void;
 }>();
 
 const { copilotDefaults, selectedResultId, selectedWidgetKey } = storeToRefs(useCopilotStore());
@@ -147,6 +147,7 @@ const input = ref('')
 const streamingStatus = ref('')
 const error = ref('')
 const messages = ref<Array<AIMessage>>([]) // tempData as AIMessage
+const conversationId = ref<string | undefined>(undefined)
 const isEmptyMessages = computed(() => messages.value.length === 0)
 
 const isLoading = computed<boolean>({
@@ -183,35 +184,40 @@ const callChatApi = async (userMessage: string) => {
       const response = await copilotApiService.callChatStream(
         messages.value, 
         copilotDefaults.value.project, 
-        selectedWidgetKey.value, 
-        copilotDefaults.value.params)
+        selectedWidgetKey.value,
+        copilotDefaults.value.params,
+        conversationId.value)
 
       // Handle the streaming response
-      await copilotApiService.handleStreamingResponse(response, messages.value, (status) => {
-        streamingStatus.value = status;
-      }, (message, index) => {
-        if (index === -1) {
-          messages.value.push(message);
-        } else {
+      const returnedConversationId = await copilotApiService.handleStreamingResponse(
+        response, messages.value, (status) => {
+          streamingStatus.value = status;
+        }, (message, index) => {
+          if (index === -1) {
+            messages.value.push(message);
+          } else {
           messages.value[index] = message;
         }
 
         if (message.data) {
-          // Find router reasoning from the latest router-status message in the conversation
-          const routerReasoning = messages.value
-            .slice()
-            .reverse()
-            .find(msg => msg.type === 'router-status' && msg.routerReasoning)
-            ?.routerReasoning;
-
-          emit('update:data', message.id, message.data, routerReasoning);
+          // Pass the current conversation ID instead of extracting routerReasoning
+          emit('update:data', message.id, message.data, conversationId.value);
           selectedResultId.value = message.id;
         }
         scrollToEnd();
-      }, () => {
+      }, (receivedConversationId) => {
         isLoading.value = false;
         streamingStatus.value = '';
+        // Store the conversationId for subsequent calls
+        if (receivedConversationId) {
+          conversationId.value = receivedConversationId;
+        }
       });
+
+      // Also capture conversationId from the return value as backup
+      if (returnedConversationId && !conversationId.value) {
+        conversationId.value = returnedConversationId;
+      }
     }
   } catch (err) {
     console.error('Failed to send message:', err)
@@ -241,7 +247,13 @@ const selectResult = (id: string) => {
   selectedResultId.value = id;
 }
 
-watch(copilotDefaults, (newDefaults) => {
+watch(copilotDefaults, (newDefaults, oldDefaults) => {
+  // Clear conversation when widget changes
+  if (oldDefaults && newDefaults.widget !== oldDefaults.widget) {
+    conversationId.value = undefined;
+    messages.value = [];
+  }
+
   if (newDefaults.question) {
     callChatApi(newDefaults.question);
   }

frontend/app/components/shared/modules/copilot/components/results/chart-results.vue

@@ -78,7 +78,7 @@ const props = defineProps<{
   config: Config | null,
   isSnapshotModalOpen: boolean,
   chartErrorType?: ChartErrorType,
-  routerReasoning?: string
+  conversationId?: string
 }>()
 
 const isSnapshotModalOpen = computed({
@@ -120,7 +120,7 @@ const generateChart = async () => {
 
   isLoading.value = true;
 
-  const response = await copilotApiService.callChartApi(props.data, props.routerReasoning);
+  const response = await copilotApiService.callChartApi(props.data, props.conversationId);
   const data = await response.json();
 
   if (data.config && data.success && data.dataMapping) {

frontend/app/components/shared/modules/copilot/components/results/results-section.vue

@@ -47,7 +47,7 @@ SPDX-License-Identifier: MIT
               :config="selectedResultConfig"
               :is-snapshot-modal-open="isSnapshotModalOpen"
               :chart-error-type="selectedResultChartErrorType"
-              :router-reasoning="selectedResultRouterReasoning"
+              :conversation-id="selectedResultConversationId"
               @update:config="handleConfigUpdate"
               @update:is-loading="handleChartLoading"
               @update:is-snapshot-modal-open="isSnapshotModalOpen = $event"
@@ -75,7 +75,7 @@ SPDX-License-Identifier: MIT
 </template>
 
 <script setup lang="ts">
-import { computed, ref, watch } from 'vue';
+import { computed, ref } from 'vue';
 import { storeToRefs } from 'pinia';
 import LfxCopilotLoadingState from '../shared/loading-state.vue';
 import LfxCopilotEmptyResult from '../info/empty-result.vue';
@@ -98,9 +98,9 @@ const props = defineProps<{
 const { resultData, selectedResultId } = storeToRefs(useCopilotStore());
 
 const isChartError = ref(false);
-const selectedTab = ref('chart');
+const selectedTab = ref('data');
 const isSnapshotModalOpen = ref(false);
-const isChartLoading = ref(true);
+const isChartLoading = ref(false);
 const selectedResultConfig = computed<Config | null>(() => {
   return resultData.value.find(result => result.id === selectedResultId.value)?.chartConfig || null;
 });
@@ -117,8 +117,8 @@ const selectedResultChartErrorType = computed(() => {
   return resultData.value.find(result => result.id === selectedResultId.value)?.chartErrorType;
 })
 
-const selectedResultRouterReasoning = computed(() => {
-  return resultData.value.find(result => result.id === selectedResultId.value)?.routerReasoning;
+const selectedResultConversationId = computed(() => {
+  return resultData.value.find(result => result.id === selectedResultId.value)?.conversationId;
 })
 
 const isEmpty = computed(() => {
@@ -147,11 +147,8 @@ const handleChartLoading = (value: boolean) => {
   emit('update:isChartLoading', value);
 }
 
-watch(isChartLoading, (value) => {
-  if (value) {
-    selectedTab.value = 'chart';
-  } 
-})
+// Removed watcher that forced chart tab selection during loading
+// Now users stay on data tab by default and can manually switch to chart if desired
 </script>
 
 <script lang="ts">