Skip to content

Commit

Permalink
Export all matched runs in Aim UI (#37)
Browse files Browse the repository at this point in the history 
Download the whole runs table even if it is not completely displayed.
  • Loading branch information
@fabiovincenzi
fabiovincenzi authored Jan 30, 2024
1 parent 0b38c7b commit 3df493a
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 63 deletions.
23 changes: 21 additions & 2 deletions src/src/components/Table/Table.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import { isEmpty, isEqual, isNil } from 'lodash-es';
import { useResizeObserver } from 'hooks';
import _ from 'lodash-es';

import CircularProgress from '@material-ui/core/CircularProgress';

import { Button, Icon, Text } from 'components/kit';
import ControlPopover from 'components/ControlPopover/ControlPopover';
import IllustrationBlock from 'components/IllustrationBlock/IllustrationBlock';
Expand Down Expand Up @@ -140,6 +142,16 @@ const Table = React.forwardRef(function Table(
width: 0,
availableSpace: 0,
});
const [isExporting, setIsExporting] = React.useState(false);

const handleExport = async () => {
setIsExporting(true);
try {
await onExport();
} finally {
setIsExporting(false);
}
};

let groups = !Array.isArray(rowData);

Expand Down Expand Up @@ -853,8 +865,15 @@ const Table = React.forwardRef(function Table(
fullWidth
variant='outlined'
size='small'
onClick={onExport}
startIcon={<Icon fontSize={14} name='download' />}
onClick={handleExport}
startIcon={
isExporting ? (
<CircularProgress size={14} />
) : (
<Icon fontSize={14} name='download' />
)
}
disabled={isExporting}
>
<Text size={14} color='inherit'>
Export
Expand Down
168 changes: 107 additions & 61 deletions src/src/services/models/explorer/createAppModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2271,6 +2271,46 @@ function createAppModel(appConfig: IAppInitialConfig) {
onRunsTagsChange({ runHash, tags, model, updateModelData });
}

function getAllRunsData(queryString?: string): {
call: (exceptionHandler: (detail: any) => void) => Promise<any>;
abort: () => void;
} {
if (runsRequestRef) {
runsRequestRef.abort();
}
runsRequestRef = runsService.getRunsData(queryString);
return {
call: async () => {
try {
const stream = await runsRequestRef.call((detail) => {
exceptionHandler({ detail, model });
});
let bufferPairs = decodeBufferPairs(
stream as ReadableStream<any>,
);
let decodedPairs = decodePathsVals(bufferPairs);
let objects = iterFoldTree(decodedPairs, 1);
const runsData: IRun<IMetricTrace | IParamTrace>[] = [];

for await (let [keys, val] of objects) {
const data = { ...(val as any), hash: keys[0] };
if (!data.hash.startsWith('progress')) {
const runData: any = val;
runsData.push({ ...runData, hash: keys[0] } as any);
}
}
return runsData;
} catch (ex: Error | any) {
if (ex.name === 'AbortError') {
// eslint-disable-next-line no-console
console.error(`${ex.name}, ${ex.message}`);
}
}
},
abort: runsRequestRef.abort,
};
}

function getRunsData(
shouldUrlUpdate?: boolean,
shouldResetSelectedRows?: boolean,
Expand Down Expand Up @@ -2897,72 +2937,81 @@ function createAppModel(appConfig: IAppInitialConfig) {
}
}

function onExportTableData(): void {
function onExportTableData(): Promise<void> {
// @TODO need to get data and params from state not from processData
const { data, params, metricsColumns } = processData(
model.getState()?.rawData,
);
const tableData = getDataAsTableRows(
data,
metricsColumns,
params,
true,
);
const configData = model.getState()?.config;
const tableColumns: ITableColumn[] = getRunsTableColumns(
metricsColumns,
params,
configData?.table.columnsOrder!,
configData?.table.hiddenColumns!,
);
const excludedFields: string[] = ['#', 'actions'];
const filteredHeader: string[] = tableColumns.reduce(
(acc: string[], column: ITableColumn) =>
acc.concat(
excludedFields.indexOf(column.key) === -1 && !column.isHidden
? column.key
: [],
),
[],
const runsDataToExport = getAllRunsData(
model.getState()?.config?.select?.query,
);
const exceptionHandler = (detail: any) => {
// eslint-disable-next-line no-console
console.error('An error occurred:', detail);
};

let emptyRow: { [key: string]: string } = {};
filteredHeader.forEach((column: string) => {
emptyRow[column] = '--';
});
return runsDataToExport.call(exceptionHandler).then((rawData) => {
const { data, params, metricsColumns } = processData(rawData);
const tableData = getDataAsTableRows(
data,
metricsColumns,
params,
true,
);
const configData = model.getState()?.config;
const tableColumns: ITableColumn[] = getRunsTableColumns(
metricsColumns,
params,
configData?.table.columnsOrder!,
configData?.table.hiddenColumns!,
);
const excludedFields: string[] = ['#', 'actions'];
const filteredHeader: string[] = tableColumns.reduce(
(acc: string[], column: ITableColumn) =>
acc.concat(
excludedFields.indexOf(column.key) === -1 && !column.isHidden
? column.key
: [],
),
[],
);

const groupedRows: IMetricTableRowData[][] =
data.length > 1
? Object.keys(tableData.rows).map(
(groupedRowKey: string) => tableData.rows[groupedRowKey].items,
)
: [
Array.isArray(tableData.rows)
? tableData.rows
: tableData.rows[Object.keys(tableData.rows)[0]].items,
];
let emptyRow: { [key: string]: string } = {};
filteredHeader.forEach((column: string) => {
emptyRow[column] = '--';
});

const dataToExport: { [key: string]: string }[] = [];
const groupedRows: IMetricTableRowData[][] =
data.length > 1
? Object.keys(tableData.rows).map(
(groupedRowKey: string) =>
tableData.rows[groupedRowKey].items,
)
: [
Array.isArray(tableData.rows)
? tableData.rows
: tableData.rows[Object.keys(tableData.rows)[0]].items,
];

groupedRows?.forEach(
(groupedRow: IMetricTableRowData[], groupedRowIndex: number) => {
groupedRow?.forEach((row: IMetricTableRowData) => {
const filteredRow = getFilteredRow({
columnKeys: filteredHeader,
row,
const dataToExport: { [key: string]: string }[] = [];

groupedRows?.forEach(
(groupedRow: IMetricTableRowData[], groupedRowIndex: number) => {
groupedRow?.forEach((row: IMetricTableRowData) => {
const filteredRow = getFilteredRow({
columnKeys: filteredHeader,
row,
});
dataToExport.push(filteredRow);
});
dataToExport.push(filteredRow);
});
if (groupedRows?.length - 1 !== groupedRowIndex) {
dataToExport.push(emptyRow);
}
},
);
const blob = new Blob([JsonToCSV(dataToExport)], {
type: 'text/csv;charset=utf-8;',
if (groupedRows?.length - 1 !== groupedRowIndex) {
dataToExport.push(emptyRow);
}
},
);
const blob = new Blob([JsonToCSV(dataToExport)], {
type: 'text/csv;charset=utf-8;',
});
saveAs(blob, `runs-${moment().format(DATE_EXPORTING_FORMAT)}.csv`);
analytics.trackEvent(ANALYTICS_EVENT_KEYS[appName].table.exports.csv);
});
saveAs(blob, `runs-${moment().format(DATE_EXPORTING_FORMAT)}.csv`);
analytics.trackEvent(ANALYTICS_EVENT_KEYS[appName].table.exports.csv);
}

function onModelNotificationDelete(id: number): void {
Expand Down Expand Up @@ -3375,9 +3424,6 @@ function createAppModel(appConfig: IAppInitialConfig) {
runsRequestRef.abort();
}
const configData = { ...model.getState()?.config };
if (queryString) {
configData.select.query = queryString;
}
runsRequestRef = runsService.getRunsData(configData?.select?.query);
setRequestProgress(model);
return {
Expand Down

0 comments on commit 3df493a

Please sign in to comment.