Skip to content

Commit 2d1cd74

Browse files
praveen5959praveen5959
authored
fix: moved graph query to new counts API (#422)
This PR moves all the count related graph queries to the new counts API implemented here parseablehq/parseable#1103
1 parent 328e830 commit 2d1cd74

File tree

6 files changed

+151
-308
lines changed

6 files changed

+151
-308
lines changed

Diff for: ‎src/@types/parseable/api/query.ts

+7
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ export type LogsQuery = {
55
access: string[] | null;
66
};
77

8+
export type GraphQueryOpts = {
9+
stream: string;
10+
startTime: string;
11+
endTime: string;
12+
numBins: number;
13+
};
14+
815
export enum SortOrder {
916
ASCENDING = 1,
1017
DESCENDING = -1,

Diff for: ‎src/api/constants.ts

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const parseParamsToQueryString = (params: Params) => {
2020
// Streams Management
2121
export const LOG_STREAM_LIST_URL = `${API_V1}/logstream`;
2222
export const LOG_STREAMS_SCHEMA_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/schema`;
23+
export const GRAPH_DATA_URL = `${API_V1}/counts`;
2324
export const LOG_QUERY_URL = (params?: Params, resourcePath = 'query') =>
2425
`${API_V1}/${resourcePath}` + parseParamsToQueryString(params);
2526
export const LOG_STREAMS_ALERTS_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/alert`;

Diff for: ‎src/api/query.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Axios } from './axios';
2-
import { LOG_QUERY_URL } from './constants';
3-
import { Log, LogsQuery, LogsResponseWithHeaders } from '@/@types/parseable/api/query';
2+
import { GRAPH_DATA_URL, LOG_QUERY_URL } from './constants';
3+
import { GraphQueryOpts, Log, LogsQuery, LogsResponseWithHeaders } from '@/@types/parseable/api/query';
44
import timeRangeUtils from '@/utils/timeRangeUtils';
55
import { CorrelationQueryBuilder, QueryBuilder } from '@/utils/queryBuilder';
66

@@ -91,3 +91,7 @@ export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '') => {
9191
const endPoint = LOG_QUERY_URL({ fields: true }, queryBuilder.getResourcePath());
9292
return Axios().post<LogsResponseWithHeaders>(endPoint, makeCustomQueryRequestData(logsQuery, query), {});
9393
};
94+
95+
export const getGraphData = (data: GraphQueryOpts) => {
96+
return Axios().post<LogsResponseWithHeaders>(GRAPH_DATA_URL, data);
97+
};

Diff for: ‎src/hooks/useQueryResult.tsx

+25-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { getQueryResultWithHeaders, getQueryResult } from '@/api/query';
2-
import { LogsQuery } from '@/@types/parseable/api/query';
1+
import { getQueryResultWithHeaders, getQueryResult, getGraphData } from '@/api/query';
2+
import { GraphQueryOpts, LogsQuery } from '@/@types/parseable/api/query';
33
import { notifications } from '@mantine/notifications';
44
import { isAxiosError, AxiosError } from 'axios';
55
import { IconCheck } from '@tabler/icons-react';
@@ -52,6 +52,29 @@ export const useQueryResult = () => {
5252
return { fetchQueryMutation };
5353
};
5454

55+
export const useGraphData = () => {
56+
const fetchGraphDataHandler = async (data: GraphQueryOpts) => {
57+
const response = await getGraphData(data);
58+
if (response.status !== 200) {
59+
throw new Error(response.statusText);
60+
}
61+
return response.data;
62+
};
63+
64+
const fetchGraphDataMutation = useMutation(fetchGraphDataHandler, {
65+
onError: (data: AxiosError) => {
66+
if (isAxiosError(data) && data.response) {
67+
const error = data.response?.data as string;
68+
typeof error === 'string' && notifyError({ message: error });
69+
} else if (data.message && typeof data.message === 'string') {
70+
notifyError({ message: data.message });
71+
}
72+
},
73+
});
74+
75+
return { fetchGraphDataMutation };
76+
};
77+
5578
export const useFetchCount = () => {
5679
const [currentStream] = useAppStore((store) => store.currentStream);
5780
const { setTotalCount } = logsStoreReducers;

Diff for: ‎src/pages/Correlation/components/MultiEventTimeLineGraph.tsx

+49-145
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Paper, Skeleton, Stack, Text } from '@mantine/core';
22
import classes from '../styles/Correlation.module.css';
3-
import { useQueryResult } from '@/hooks/useQueryResult';
3+
import { useGraphData } from '@/hooks/useQueryResult';
44
import { useCallback, useEffect, useMemo, useState } from 'react';
55
import dayjs from 'dayjs';
66
import { AreaChart } from '@mantine/charts';
@@ -9,20 +9,13 @@ import { appStoreReducers, useAppStore } from '@/layouts/MainLayout/providers/Ap
99
import { LogsResponseWithHeaders } from '@/@types/parseable/api/query';
1010
import _ from 'lodash';
1111
import timeRangeUtils from '@/utils/timeRangeUtils';
12-
import { filterStoreReducers, useFilterStore } from '@/pages/Stream/providers/FilterProvider';
1312
import { useCorrelationStore } from '../providers/CorrelationProvider';
1413

15-
const { parseQuery } = filterStoreReducers;
1614
const { makeTimeRangeLabel } = timeRangeUtils;
1715
const { setTimeRange } = appStoreReducers;
1816

1917
type CompactInterval = 'minute' | 'day' | 'hour' | 'quarter-hour' | 'half-hour' | 'month';
2018

21-
function removeOffsetFromQuery(query: string): string {
22-
const offsetRegex = /\sOFFSET\s+\d+/i;
23-
return query.replace(offsetRegex, '');
24-
}
25-
2619
const getCompactType = (interval: number): CompactInterval => {
2720
const totalMinutes = interval / (1000 * 60);
2821
if (totalMinutes <= 60) {
@@ -44,74 +37,6 @@ const getCompactType = (interval: number): CompactInterval => {
4437
}
4538
};
4639

47-
const getStartOfTs = (time: Date, compactType: CompactInterval): Date => {
48-
if (compactType === 'minute') {
49-
return time;
50-
} else if (compactType === 'hour') {
51-
return new Date(time.getFullYear(), time.getMonth(), time.getDate(), time.getHours());
52-
} else if (compactType === 'quarter-hour') {
53-
const roundOff = 1000 * 60 * 15;
54-
return new Date(Math.floor(time.getTime() / roundOff) * roundOff);
55-
} else if (compactType === 'half-hour') {
56-
const roundOff = 1000 * 60 * 30;
57-
return new Date(Math.floor(time.getTime() / roundOff) * roundOff);
58-
} else if (compactType === 'day') {
59-
return new Date(time.getFullYear(), time.getMonth(), time.getDate());
60-
} else {
61-
return new Date(time.getFullYear(), time.getMonth());
62-
}
63-
};
64-
65-
const getEndOfTs = (time: Date, compactType: CompactInterval): Date => {
66-
if (compactType === 'minute') {
67-
return time;
68-
} else if (compactType === 'hour') {
69-
return new Date(time.getFullYear(), time.getMonth(), time.getDate(), time.getHours() + 1);
70-
} else if (compactType === 'quarter-hour') {
71-
const roundOff = 1000 * 60 * 15;
72-
return new Date(Math.round(time.getTime() / roundOff) * roundOff);
73-
} else if (compactType === 'half-hour') {
74-
const roundOff = 1000 * 60 * 30;
75-
return new Date(Math.round(time.getTime() / roundOff) * roundOff);
76-
} else if (compactType === 'day') {
77-
return new Date(time.getFullYear(), time.getMonth(), time.getDate() + 1);
78-
} else {
79-
return new Date(time.getFullYear(), time.getMonth() + 1);
80-
}
81-
};
82-
83-
const getModifiedTimeRange = (
84-
startTime: Date,
85-
endTime: Date,
86-
interval: number,
87-
): { modifiedStartTime: Date; modifiedEndTime: Date; compactType: CompactInterval } => {
88-
const compactType = getCompactType(interval);
89-
const modifiedStartTime = getStartOfTs(startTime, compactType);
90-
const modifiedEndTime = getEndOfTs(endTime, compactType);
91-
return { modifiedEndTime, modifiedStartTime, compactType };
92-
};
93-
94-
const compactTypeIntervalMap = {
95-
minute: '1 minute',
96-
hour: '1 hour',
97-
day: '24 hour',
98-
'quarter-hour': '15 minute',
99-
'half-hour': '30 minute',
100-
month: '1 month',
101-
};
102-
103-
const generateCountQuery = (
104-
streamName: string,
105-
startTime: Date,
106-
endTime: Date,
107-
compactType: CompactInterval,
108-
whereClause: string,
109-
) => {
110-
const range = compactTypeIntervalMap[compactType];
111-
/* eslint-disable no-useless-escape */
112-
return `SELECT DATE_BIN('${range}', p_timestamp, '${startTime.toISOString()}') AS date_bin_timestamp, COUNT(*) AS log_count FROM \"${streamName}\" WHERE p_timestamp BETWEEN '${startTime.toISOString()}' AND '${endTime.toISOString()}' AND ${whereClause} GROUP BY date_bin_timestamp ORDER BY date_bin_timestamp`;
113-
};
114-
11540
const NoDataView = (props: { isError: boolean }) => {
11641
return (
11742
<Stack style={{ width: '100%', height: '100%', alignItems: 'center', justifyContent: 'center' }}>
@@ -136,38 +61,6 @@ const calcAverage = (data: LogsResponseWithHeaders | undefined) => {
13661
return parseInt(Math.abs(total / records.length).toFixed(0));
13762
};
13863

139-
const getAllIntervals = (start: Date, end: Date, compactType: CompactInterval): Date[] => {
140-
const result = [];
141-
let currentDate = new Date(start);
142-
143-
while (currentDate <= end) {
144-
result.push(new Date(currentDate));
145-
currentDate = incrementDateByCompactType(currentDate, compactType);
146-
}
147-
148-
return result;
149-
};
150-
151-
const incrementDateByCompactType = (date: Date, type: CompactInterval): Date => {
152-
const tempDate = new Date(date);
153-
if (type === 'minute') {
154-
tempDate.setMinutes(tempDate.getMinutes() + 1);
155-
} else if (type === 'hour') {
156-
tempDate.setHours(tempDate.getHours() + 1);
157-
} else if (type === 'day') {
158-
tempDate.setDate(tempDate.getDate() + 1);
159-
} else if (type === 'quarter-hour') {
160-
tempDate.setMinutes(tempDate.getMinutes() + 15);
161-
} else if (type === 'half-hour') {
162-
tempDate.setMinutes(tempDate.getMinutes() + 30);
163-
} else if (type === 'month') {
164-
tempDate.setMonth(tempDate.getMonth() + 1);
165-
} else {
166-
tempDate;
167-
}
168-
return new Date(tempDate);
169-
};
170-
17164
type GraphTickItem = {
17265
events: number;
17366
minute: Date;
@@ -243,6 +136,11 @@ function ChartTooltip({ payload, series }: ChartTooltipProps) {
243136
);
244137
}
245138

139+
type LogRecord = {
140+
counts_timestamp: string;
141+
log_count: number;
142+
};
143+
246144
// date_bin removes tz info
247145
// filling data with empty values where there is no rec
248146
const parseGraphData = (
@@ -256,39 +154,53 @@ const parseGraphData = (
256154
const firstResponse = dataSets[0]?.records || [];
257155
const secondResponse = dataSets[1]?.records || [];
258156

259-
const { modifiedEndTime, modifiedStartTime, compactType } = getModifiedTimeRange(startTime, endTime, interval);
260-
const allTimestamps = getAllIntervals(modifiedStartTime, modifiedEndTime, compactType);
157+
const compactType = getCompactType(interval);
158+
const ticksCount = interval < 10 * 60 * 1000 ? interval / (60 * 1000) : interval < 60 * 60 * 1000 ? 10 : 60;
159+
const intervalDuration = (endTime.getTime() - startTime.getTime()) / ticksCount;
160+
161+
const allTimestamps = Array.from(
162+
{ length: ticksCount },
163+
(_, index) => new Date(startTime.getTime() + index * intervalDuration),
164+
);
261165

262166
const hasSecondDataset = dataSets[1] !== undefined;
263167

168+
const isValidRecord = (record: any): record is LogRecord => {
169+
return typeof record.counts_timestamp === 'string' && typeof record.log_count === 'number';
170+
};
171+
264172
const secondResponseMap =
265173
secondResponse.length > 0
266174
? new Map(
267-
secondResponse.map((entry) => [new Date(`${entry.date_bin_timestamp}Z`).toISOString(), entry.log_count]),
175+
secondResponse
176+
.filter((entry) => isValidRecord(entry))
177+
.map((entry) => {
178+
const timestamp = entry.counts_timestamp;
179+
if (timestamp != null) {
180+
return [new Date(timestamp).getTime(), entry.log_count];
181+
}
182+
return null;
183+
})
184+
.filter((entry): entry is [number, number] => entry !== null),
268185
)
269186
: new Map();
270-
const calculateTimeRange = (timestamp: Date | string) => {
271-
const startTime = dayjs(timestamp);
272-
const endTimeByCompactType = incrementDateByCompactType(startTime.toDate(), compactType);
273-
const endTime = dayjs(endTimeByCompactType);
274-
return { startTime, endTime };
275-
};
187+
276188
const combinedData = allTimestamps.map((ts) => {
277189
const firstRecord = firstResponse.find((record) => {
278-
const recordTimestamp = new Date(`${record.date_bin_timestamp}Z`).toISOString();
279-
const tsISO = ts.toISOString();
190+
if (!isValidRecord(record)) return false;
191+
const recordTimestamp = new Date(record.counts_timestamp).getTime();
192+
const tsISO = ts.getTime();
280193
return recordTimestamp === tsISO;
281194
});
282195

283196
const secondCount = secondResponseMap?.get(ts.toISOString()) ?? 0;
284-
const { startTime, endTime } = calculateTimeRange(ts);
285197

286198
const defaultOpts: Record<string, any> = {
287199
stream: firstRecord?.log_count || 0,
288200
minute: ts,
289201
compactType,
290-
startTime,
291-
endTime,
202+
startTime: dayjs(ts),
203+
endTime: dayjs(new Date(ts.getTime() + intervalDuration)),
292204
};
293205

294206
if (hasSecondDataset) {
@@ -302,9 +214,8 @@ const parseGraphData = (
302214
};
303215

304216
const MultiEventTimeLineGraph = () => {
305-
const { fetchQueryMutation } = useQueryResult();
217+
const { fetchGraphDataMutation } = useGraphData();
306218
const [fields] = useCorrelationStore((store) => store.fields);
307-
const [appliedQuery] = useFilterStore((store) => store.appliedQuery);
308219
const [streamData] = useCorrelationStore((store) => store.streamData);
309220
const [timeRange] = useAppStore((store) => store.timeRange);
310221
const [multipleStreamData, setMultipleStreamData] = useState<{ [key: string]: any }>({});
@@ -334,30 +245,23 @@ const MultiEventTimeLineGraph = () => {
334245

335246
const streamNames = Object.keys(fields);
336247
const streamsToFetch = streamNames.filter((streamName) => !Object.keys(streamData).includes(streamName));
337-
const queries = streamsToFetch.map((streamKey) => {
338-
const { modifiedEndTime, modifiedStartTime, compactType } = getModifiedTimeRange(startTime, endTime, interval);
248+
const totalMinutes = interval / (1000 * 60);
249+
const numBins = Math.trunc(totalMinutes < 10 ? totalMinutes : totalMinutes < 60 ? 10 : 60);
250+
const eventTimeLineGraphOpts = streamsToFetch.map((streamKey) => {
339251
const logsQuery = {
340-
startTime: modifiedStartTime,
341-
endTime: modifiedEndTime,
342-
access: [],
343-
};
344-
const whereClause = parseQuery(appliedQuery, streamKey).where;
345-
const query = generateCountQuery(streamKey, modifiedStartTime, modifiedEndTime, compactType, whereClause);
346-
const graphQuery = removeOffsetFromQuery(query);
347-
348-
return {
349-
queryEngine: 'Parseable',
350-
logsQuery,
351-
query: graphQuery,
352-
streamKey,
252+
stream: streamKey,
253+
startTime: dayjs(startTime).toISOString(),
254+
endTime: dayjs(endTime).add(1, 'minute').toISOString(),
255+
numBins,
353256
};
257+
return logsQuery;
354258
});
355-
Promise.all(queries.map((queryData: any) => fetchQueryMutation.mutateAsync(queryData)))
259+
Promise.all(eventTimeLineGraphOpts.map((queryData: any) => fetchGraphDataMutation.mutateAsync(queryData)))
356260
.then((results) => {
357261
setMultipleStreamData((prevData: any) => {
358262
const newData = { ...prevData };
359263
results.forEach((result, index) => {
360-
newData[queries[index].streamKey] = result;
264+
newData[eventTimeLineGraphOpts[index].stream] = result;
361265
});
362266
return newData;
363267
});
@@ -367,8 +271,8 @@ const MultiEventTimeLineGraph = () => {
367271
});
368272
}, [fields, timeRange]);
369273

370-
const isLoading = fetchQueryMutation.isLoading;
371-
const avgEventCount = useMemo(() => calcAverage(fetchQueryMutation?.data), [fetchQueryMutation?.data]);
274+
const isLoading = fetchGraphDataMutation.isLoading;
275+
const avgEventCount = useMemo(() => calcAverage(fetchGraphDataMutation?.data), [fetchGraphDataMutation?.data]);
372276
const graphData = useMemo(() => {
373277
if (!streamGraphData || streamGraphData.length === 0 || streamGraphData.length !== Object.keys(fields).length)
374278
return [];
@@ -394,7 +298,7 @@ const MultiEventTimeLineGraph = () => {
394298
return (
395299
<Stack className={classes.graphContainer}>
396300
<Skeleton
397-
visible={fetchQueryMutation.isLoading}
301+
visible={fetchGraphDataMutation.isLoading}
398302
h="100%"
399303
w={isLoading ? '98%' : '100%'}
400304
style={isLoading ? { marginLeft: '1.8rem', alignSelf: 'center' } : !hasData ? { marginLeft: '1rem' } : {}}>
@@ -433,7 +337,7 @@ const MultiEventTimeLineGraph = () => {
433337
dotProps={{ strokeWidth: 1, r: 2.5 }}
434338
/>
435339
) : (
436-
<NoDataView isError={fetchQueryMutation.isError} />
340+
<NoDataView isError={fetchGraphDataMutation.isError} />
437341
)}
438342
</Skeleton>
439343
</Stack>

0 commit comments

Comments
 (0)