datacube: enable saving and loading local file source in DataCube grid

.changeset/little-crabs-exist.md

@@ -0,0 +1,6 @@
+---
+'@finos/legend-application-data-cube': patch
+'@finos/legend-data-cube': patch
+---
+
+datacube: enable saving and loading local file source in DataCube grid

packages/legend-application-data-cube/src/components/builder/LegendDataCubeBuilder.tsx

@@ -38,7 +38,6 @@ import {
 import { useEffect } from 'react';
 import { LegendDataCubeSettingStorageKey } from '../../__lib__/LegendDataCubeSetting.js';
 import type { LegendDataCubeBuilderStore } from '../../stores/builder/LegendDataCubeBuilderStore.js';
-import { LocalFileDataCubeSource } from '../../stores/model/LocalFileDataCubeSource.js';
 
 const LegendDataCubeBuilderHeader = observer(() => {
   const store = useLegendDataCubeBuilderStore();
@@ -58,11 +57,7 @@ const LegendDataCubeBuilderHeader = observer(() => {
       <FormButton
         compact={true}
         className="ml-1.5"
-        disabled={
-          !store.builder?.dataCube ||
-          /* TODO: @gs-gunjan we should allow saving DataCube using CSV file source */
-          store.builder.source instanceof LocalFileDataCubeSource
-        }
+        disabled={!store.builder?.dataCube}
         onClick={() => store.saverDisplay.open()}
       >
         Save DataCube

packages/legend-application-data-cube/src/components/builder/LegendDataCubeSourceLoader.tsx

@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer } from 'mobx-react-lite';
+import { FormButton } from '@finos/legend-data-cube';
+import { useLegendDataCubeBuilderStore } from './LegendDataCubeBuilderStoreProvider.js';
+import { LocalFileDataCubeSourceLoaderBuilderState } from '../../stores/builder/source/loader/LocalFileDataCubeSourceLoaderBuilderState.js';
+import { LocalFileDataCubeSourceLoader } from './source/loader/LocalFileDataCubeSourceLoader.js';
+
+export const LegendDataCubeSourceLoader = observer(() => {
+  const store = useLegendDataCubeBuilderStore();
+  const state = store.sourceLoader;
+  const sourceLoaderBuilder = state.sourceLoaderBuilder;
+
+  return (
+    <>
+      <div className="h-[calc(100%_-_40px)] w-full px-2 pt-2">
+        <div className="h-full w-full border border-neutral-300 bg-white">
+          <div className="h-full w-full select-none">
+            <div className="ml-2 h-[1px] w-[calc(100%_-_16px)] bg-neutral-200" />
+            <div className="h-[calc(100%_-_41px)] w-full overflow-auto">
+              {sourceLoaderBuilder instanceof
+                LocalFileDataCubeSourceLoaderBuilderState && (
+                <LocalFileDataCubeSourceLoader
+                  sourceBuilder={sourceLoaderBuilder}
+                />
+              )}
+            </div>
+          </div>
+        </div>
+      </div>
+      <div className="flex h-10 items-center justify-end px-2">
+        <FormButton onClick={() => state.display.close()}>Cancel</FormButton>
+        <FormButton
+          className="ml-2"
+          disabled={
+            !sourceLoaderBuilder.isValid || state.finalizeState.isInProgress
+          }
+          onClick={() => {
+            state
+              .finalize()
+              .catch((error) => store.alertService.alertUnhandledError(error));
+          }}
+        >
+          OK
+        </FormButton>
+      </div>
+    </>
+  );
+});

packages/legend-application-data-cube/src/components/builder/source/LocalFileDataCubeSourceBuilder.tsx

@@ -31,7 +31,7 @@ export const LocalFileDataCubeSourceBuilder = observer(
           text={`Currently, support for local file comes with the following limitations:
 - Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
 - Data from uploaded file will not be stored nor shared.
-- DataCube created with local file source cannot be saved.`}
+- DataCube from uploaded file can be stored but when loading this, you will have to reupload source data.`}
         />
         <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
           <input

packages/legend-application-data-cube/src/components/builder/source/loader/LocalFileDataCubeSourceLoader.tsx

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { observer } from 'mobx-react-lite';
+import { AlertType, FormAlert, FormCodeEditor } from '@finos/legend-data-cube';
+import { CODE_EDITOR_LANGUAGE } from '@finos/legend-code-editor';
+import type { LocalFileDataCubeSourceLoaderBuilderState } from '../../../../stores/builder/source/loader/LocalFileDataCubeSourceLoaderBuilderState.js';
+
+export const LocalFileDataCubeSourceLoader = observer(
+  (props: { sourceBuilder: LocalFileDataCubeSourceLoaderBuilderState }) => {
+    const { sourceBuilder } = props;
+
+    return (
+      <div className="h-full w-full p-2">
+        <FormAlert
+          message="Local file support is experimental"
+          type={AlertType.WARNING}
+          text={`Currently, support for local file comes with the following limitations:
+- Only CSV files are supported, but not all variants of CSV files are supported (required header row, comma delimiter, single escape quote).
+- Data from uploaded file will not be stored nor shared.`}
+        />
+        <div className="mt-2 flex h-6 w-full items-center text-neutral-500">
+          <input
+            type="file"
+            onChange={(event) => {
+              sourceBuilder.processFile(event.target.files?.[0]);
+            }}
+            className="w-full"
+          />
+        </div>
+        {sourceBuilder.previewText !== undefined && (
+          <div className="mt-2 h-40">
+            <FormCodeEditor
+              value={sourceBuilder.previewText}
+              language={CODE_EDITOR_LANGUAGE.TEXT}
+              isReadOnly={true}
+              hidePadding={true}
+              title="Data Preview"
+            />
+          </div>
+        )}
+      </div>
+    );
+  },
+);

packages/legend-application-data-cube/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -194,12 +194,89 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
           RawLocalFileQueryDataCubeSource.serialization.fromJson(value);
         const source = new LocalFileDataCubeSource();
         source.fileName = rawSource.fileName;
+        source.fileFormat = rawSource.fileFormat;
         source.count = rawSource.count;
-        source.db = rawSource.db;
-        source.model = rawSource.model;
-        source.runtime = rawSource.runtime;
-        source.schema = rawSource.schema;
-        source.table = rawSource.table;
+
+        const { schemaName, tableName, tableSpec } =
+          LegendDataCubeDuckDBEngine.getTableDetailsByReference(
+            rawSource.dbReference,
+          );
+
+        const { model, database, schema, table, runtime } =
+          this._synthesizeMinimalModelContext({
+            schemaName,
+            tableName,
+            tableColumns: tableSpec.map((col) => {
+              const column = new V1_Column();
+              column.name = col[0] as string;
+              // TODO: confirm this is in accordance to engine
+              // check if we have a duckdb enum mapping
+              // See https://duckdb.org/docs/sql/data_types/overview.html
+              switch (col[1] as string) {
+                case 'BIT': {
+                  column.type = new V1_Bit();
+                  break;
+                }
+                case 'BOOLEAN': {
+                  // TODO: understand why boolean is not present in relationalDataType
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                case 'DATE': {
+                  column.type = new V1_Date();
+                  break;
+                }
+                case 'DECIMAL': {
+                  column.type = new V1_Decimal();
+                  break;
+                }
+                case 'DOUBLE': {
+                  column.type = new V1_Double();
+                  break;
+                }
+                case 'FLOAT': {
+                  column.type = new V1_Float();
+                  break;
+                }
+                case 'INTEGER': {
+                  column.type = new V1_Integer();
+                  break;
+                }
+                case 'TININT': {
+                  column.type = new V1_TinyInt();
+                  break;
+                }
+                case 'SMALLINT': {
+                  column.type = new V1_SmallInt();
+                  break;
+                }
+                case 'BIGINT': {
+                  column.type = new V1_BigInt();
+                  break;
+                }
+                case 'TIMESTAMP': {
+                  column.type = new V1_Timestamp();
+                  break;
+                }
+                case 'VARCHAR': {
+                  column.type = new V1_VarChar();
+                  break;
+                }
+                default: {
+                  throw new UnsupportedOperationError(
+                    `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
+                  );
+                }
+              }
+              return column;
+            }),
+          });
+
+        source.db = database.path;
+        source.model = model;
+        source.table = table.name;
+        source.schema = schema.name;
+        source.runtime = runtime.path;
 
         const query = new V1_ClassInstance();
         query.type = V1_ClassInstanceType.RELATION_STORE_ACCESSOR;
@@ -797,93 +874,14 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     }
   }
 
-  async ingestLocalFileData(
-    data: string,
-    format: string,
-  ): Promise<DataCubeSource | undefined> {
-    const {
-      schema: schemaName,
-      table: tableName,
-      tableSpec,
-    } = await this._duckDBEngine.ingestLocalFileData(data, format);
-
-    const { model, database, schema, table, runtime } =
-      this._synthesizeMinimalModelContext({
-        schemaName,
-        tableName,
-        tableColumns: tableSpec.map((col) => {
-          const column = new V1_Column();
-          column.name = col[0] as string;
-          // TODO: confirm this is in accordance to engine
-          // check if we have a duckdb enum mapping
-          // See https://duckdb.org/docs/sql/data_types/overview.html
-          switch (col[1] as string) {
-            case 'BIT': {
-              column.type = new V1_Bit();
-              break;
-            }
-            case 'BOOLEAN': {
-              // TODO: understand why boolean is not present in relationalDataType
-              column.type = new V1_VarChar();
-              break;
-            }
-            case 'DATE': {
-              column.type = new V1_Date();
-              break;
-            }
-            case 'DECIMAL': {
-              column.type = new V1_Decimal();
-              break;
-            }
-            case 'DOUBLE': {
-              column.type = new V1_Double();
-              break;
-            }
-            case 'FLOAT': {
-              column.type = new V1_Float();
-              break;
-            }
-            case 'INTEGER': {
-              column.type = new V1_Integer();
-              break;
-            }
-            case 'TININT': {
-              column.type = new V1_TinyInt();
-              break;
-            }
-            case 'SMALLINT': {
-              column.type = new V1_SmallInt();
-              break;
-            }
-            case 'BIGINT': {
-              column.type = new V1_BigInt();
-              break;
-            }
-            case 'TIMESTAMP': {
-              column.type = new V1_Timestamp();
-              break;
-            }
-            case 'VARCHAR': {
-              column.type = new V1_VarChar();
-              break;
-            }
-            default: {
-              throw new UnsupportedOperationError(
-                `Can't ingest local file data: failed to find matching relational data type for DuckDB type '${col[1]}' when synthesizing table definition`,
-              );
-            }
-          }
-          return column;
-        }),
-      });
+  async ingestLocalFileData(data: string, format: string) {
+    const { dbReference, columnNames } =
+      await this._duckDBEngine.ingestLocalFileData(data, format);
+    return { dbReference, columnNames };
+  }
 
-    const source = new LocalFileDataCubeSource();
-    source.model = model;
-    source.runtime = runtime.path;
-    source.db = database.path;
-    source.schema = schema.name;
-    source.table = table.name;
-    return source;
+  async clearLocalFileIngestData() {
+    await this._duckDBEngine.clearLocalFileDataIngest();
   }
 
   private _synthesizeMinimalModelContext(data: {