diff --git a/apps/insights/src/app/api/pyth/get-history/route.ts b/apps/insights/src/app/api/pyth/get-history/route.ts new file mode 100644 index 0000000000..ab06debd23 --- /dev/null +++ b/apps/insights/src/app/api/pyth/get-history/route.ts @@ -0,0 +1,64 @@ +import type { NextRequest } from "next/server"; +import { z } from "zod"; + +import { getHistory } from "../../../../services/clickhouse"; + +const queryParamsSchema = z.object({ + symbol: z.string(), + range: z.enum(["1H", "1D", "1W", "1M"]), + cluster: z.enum(["pythnet", "pythtest-conformance"]), + from: z.string().transform(Number), + until: z.string().transform(Number), +}); + +export async function GET(req: NextRequest) { + const searchParams = req.nextUrl.searchParams; + + // Parse and validate query parameters + const symbol = searchParams.get("symbol"); + const range = searchParams.get("range"); + const cluster = searchParams.get("cluster"); + const from = searchParams.get("from"); + const until = searchParams.get("until"); + + if (!symbol || !range || !cluster) { + return new Response( + "Missing required parameters. Must provide `symbol`, `range`, and `cluster`", + { status: 400 } + ); + } + + try { + // Validate parameters using the schema + const validatedParams = queryParamsSchema.parse({ + symbol, + range, + cluster, + from, + until, + }); + + const data = await getHistory({ + symbol: validatedParams.symbol, + range: validatedParams.range, + cluster: validatedParams.cluster, + from: validatedParams.from, + until: validatedParams.until, + }); + + return Response.json(data); + } catch (error) { + if (error instanceof z.ZodError) { + return new Response( + `Invalid parameters: ${error.errors.map(e => e.message).join(", ")}`, + { status: 400 } + ); + } + + console.error("Error fetching history data:", error); + return new Response( + "Internal server error", + { status: 500 } + ); + } +} diff --git a/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts b/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts index a1085ea612..62f0c87e75 100644 --- a/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts +++ b/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts @@ -1 +1 @@ -export { ChartPageLoading as default } from "../../../../components/PriceFeed/chart-page"; +export { ChartPageLoading as default } from "../../../../components/PriceFeed/Chart/chart-page"; diff --git a/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts b/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts index 165ad5abcd..10e4cc3c72 100644 --- a/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts +++ b/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts @@ -1,3 +1,3 @@ -export { ChartPage as default } from "../../../../components/PriceFeed/chart-page"; +export { ChartPage as default } from "../../../../components/PriceFeed/Chart/chart-page"; export const revalidate = 3600; diff --git a/apps/insights/src/components/PriceFeed/chart-page.module.scss b/apps/insights/src/components/PriceFeed/Chart/chart-page.module.scss similarity index 100% rename from apps/insights/src/components/PriceFeed/chart-page.module.scss rename to apps/insights/src/components/PriceFeed/Chart/chart-page.module.scss diff --git a/apps/insights/src/components/PriceFeed/chart-page.tsx b/apps/insights/src/components/PriceFeed/Chart/chart-page.tsx similarity index 93% rename from apps/insights/src/components/PriceFeed/chart-page.tsx rename to apps/insights/src/components/PriceFeed/Chart/chart-page.tsx index aa2797827d..a60cfe9506 100644 --- a/apps/insights/src/components/PriceFeed/chart-page.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart-page.tsx @@ -5,7 +5,8 @@ import { Spinner } from "@pythnetwork/component-library/Spinner"; import { Chart } from "./chart"; import styles from "./chart-page.module.scss"; -import { getFeed } from "./get-feed"; +import { getFeed } from "../get-feed"; +import { ChartToolbar } from "./chart-toolbar"; type Props = { params: Promise<{ @@ -26,7 +27,7 @@ type ChartPageImplProps = }); const ChartPageImpl = (props: ChartPageImplProps) => ( - + }>
{props.isLoading ? (
diff --git a/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx b/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx new file mode 100644 index 0000000000..634774740a --- /dev/null +++ b/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx @@ -0,0 +1,70 @@ +'use client' +import { SingleToggleGroup } from "@pythnetwork/component-library/SingleToggleGroup"; +import { useLogger } from "@pythnetwork/component-library/useLogger"; +import { parseAsStringEnum, useQueryState } from 'nuqs'; +import { useCallback } from 'react'; + +export enum Interval { + Live, + OneHour, + OneDay, + OneWeek, + OneMonth, +} + +export const INTERVAL_NAMES = { + [Interval.Live]: "Live", + [Interval.OneHour]: "1H", + [Interval.OneDay]: "1D", + [Interval.OneWeek]: "1W", + [Interval.OneMonth]: "1M", +} as const; + +export const toInterval = (name: (typeof INTERVAL_NAMES)[keyof typeof INTERVAL_NAMES]): Interval => { + switch (name) { + case "Live": { + return Interval.Live; + } + case "1H": { + return Interval.OneHour; + } + case "1D": { + return Interval.OneDay; + } + case "1W": { + return Interval.OneWeek; + } + case "1M": { + return Interval.OneMonth; + } + } +}; +export const ChartToolbar = () => { + const logger = useLogger(); + const [interval, setInterval] = useQueryState( + "interval", + parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"), + ); + + const handleSelectionChange = useCallback((newValue: Interval) => { + setInterval(INTERVAL_NAMES[newValue]).catch((error: unknown) => { + logger.error("Failed to update interval", error); + }); + }, [logger, setInterval]); + + return ( + + ); +}; diff --git a/apps/insights/src/components/PriceFeed/chart.module.scss b/apps/insights/src/components/PriceFeed/Chart/chart.module.scss similarity index 100% rename from apps/insights/src/components/PriceFeed/chart.module.scss rename to apps/insights/src/components/PriceFeed/Chart/chart.module.scss diff --git a/apps/insights/src/components/PriceFeed/chart.tsx b/apps/insights/src/components/PriceFeed/Chart/chart.tsx similarity index 58% rename from apps/insights/src/components/PriceFeed/chart.tsx rename to apps/insights/src/components/PriceFeed/Chart/chart.tsx index 478d1172cd..d2ebf407c9 100644 --- a/apps/insights/src/components/PriceFeed/chart.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart.tsx @@ -5,20 +5,23 @@ import { useResizeObserver, useMountEffect } from "@react-hookz/web"; import type { IChartApi, ISeriesApi, UTCTimestamp } from "lightweight-charts"; import { LineSeries, LineStyle, createChart } from "lightweight-charts"; import { useTheme } from "next-themes"; +import { parseAsStringEnum, useQueryState } from 'nuqs'; import type { RefObject } from "react"; import { useEffect, useRef, useCallback } from "react"; import { z } from "zod"; +import { INTERVAL_NAMES } from './chart-toolbar'; import styles from "./chart.module.scss"; -import { useLivePriceData } from "../../hooks/use-live-price-data"; -import { usePriceFormatter } from "../../hooks/use-price-formatter"; -import { Cluster } from "../../services/pyth"; +import { useLivePriceData } from "../../../hooks/use-live-price-data"; +import { usePriceFormatter } from "../../../hooks/use-price-formatter"; +import { Cluster } from "../../../services/pyth"; type Props = { symbol: string; feedId: string; }; + export const Chart = ({ symbol, feedId }: Props) => { const chartContainerRef = useChart(symbol, feedId); @@ -37,6 +40,59 @@ const useChart = (symbol: string, feedId: string) => { useChartColors(chartContainerRef, chartRef); return chartContainerRef; }; +const historySchema = z.array( + z.object({ + timestamp: z.number(), + openPrice: z.number(), + lowPrice: z.number(), + closePrice: z.number(), + highPrice: z.number(), + avgPrice: z.number(), + avgConfidence: z.number(), + avgEmaPrice: z.number(), + avgEmaConfidence: z.number(), + startSlot: z.string(), + endSlot: z.string(), + })); +const fetchHistory = async ({ symbol, range, cluster, from, until }: { symbol: string, range: string, cluster: string, from: bigint, until: bigint }) => { + const url = new URL("/api/pyth/get-history", globalThis.location.origin); + url.searchParams.set("symbol", symbol); + url.searchParams.set("range", range); + url.searchParams.set("cluster", cluster); + url.searchParams.set("from", from.toString()); + url.searchParams.set("until", until.toString()); + console.log("fetching history", {from: new Date(Number(from) * 1000), until: new Date(Number(until) * 1000)}, url.toString()); + return fetch(url).then(async (data) => historySchema.parse(await data.json())); +} + +const checkPriceData = (data: {time: UTCTimestamp}[]) => { + const chartData = [...data].sort((a, b) => a.time - b.time); + if(chartData.length < 2) { + return; + } + const firstElement = chartData.at(-2); + const secondElement = chartData.at(-1); + if(!firstElement || !secondElement ) { + return; + } + const detectedInterval = secondElement.time - firstElement.time + for(let i = 0; i < chartData.length - 1; i++) { + const currentElement = chartData[i]; + const nextElement = chartData[i + 1]; + if(!currentElement || !nextElement) { + return; + } + const interval = nextElement.time - currentElement.time + if(interval !== detectedInterval) { + console.warn("Price chartData is not consistent", { + current: currentElement, + next: nextElement, + detectedInterval, + }); + } + } + return detectedInterval; +} const useChartElem = (symbol: string, feedId: string) => { const logger = useLogger(); @@ -46,15 +102,26 @@ const useChartElem = (symbol: string, feedId: string) => { const earliestDateRef = useRef(undefined); const isBackfilling = useRef(false); const priceFormatter = usePriceFormatter(); + const [interval] = useQueryState( + "interval", + parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"), + ); + useEffect(() => { + if(interval !== "Live") { + backfillData(); + } + }, [interval]); const backfillData = useCallback(() => { - if (!isBackfilling.current && earliestDateRef.current) { + console.log(earliestDateRef) + const { from, until } = backfillIntervals({ interval, earliestDate: earliestDateRef.current }); + + if (!isBackfilling.current) { isBackfilling.current = true; - const url = new URL("/historical-prices", globalThis.location.origin); - url.searchParams.set("symbol", symbol); - url.searchParams.set("until", earliestDateRef.current.toString()); - fetch(url) - .then(async (data) => historicalDataSchema.parse(await data.json())) + // seconds to date + const range = interval === "Live" ? "1H" : interval; + console.log("backfilling", {from: new Date(Number(from) * 1000), until: new Date(Number(until) * 1000)}); + fetchHistory({ symbol, range, cluster: "pythnet", from, until }) .then((data) => { const firstPoint = data[0]; if (firstPoint) { @@ -65,33 +132,43 @@ const useChartElem = (symbol: string, feedId: string) => { chartRef.current.resolution === Resolution.Tick ) { const convertedData = data.map( - ({ timestamp, price, confidence }) => ({ - time: getLocalTimestamp(new Date(timestamp * 1000)), - price, - confidence, + ({ timestamp, avgPrice, avgConfidence }) => ({ + time: getLocalTimestamp(new Date(timestamp*1000)), + price: avgPrice, + confidence: avgConfidence, }), ); - chartRef.current.price.setData([ - ...convertedData.map(({ time, price }) => ({ + // console.log("convertedData", + // {current: chartRef?.current?.price.data().map(({ time, value }) => ({ + // time: new Date(time*1000), + // value, + // })), + // converted: convertedData.map(({ time, price }) => ({ + // time: new Date(time*1000), + // value: price, + // })) + // }); + const newPriceData = [...convertedData.map(({ time, price }) => ({ time, value: price, })), - ...chartRef.current.price.data(), - ]); - chartRef.current.confidenceHigh.setData([ - ...convertedData.map(({ time, price, confidence }) => ({ + ...chartRef.current.price.data(),] + const newConfidenceHighData = [...convertedData.map(({ time, price, confidence }) => ({ time, value: price + confidence, })), - ...chartRef.current.confidenceHigh.data(), - ]); - chartRef.current.confidenceLow.setData([ - ...convertedData.map(({ time, price, confidence }) => ({ + ...chartRef.current.confidenceHigh.data(),] + const newConfidenceLowData = [...convertedData.map(({ time, price, confidence }) => ({ + time, + value: price - confidence, + })), ...chartRef.current.confidenceLow.data(),] + checkPriceData(convertedData.map(({ time, price }) => ({ time, - value: price - confidence, - })), - ...chartRef.current.confidenceLow.data(), - ]); + value: price, + }))); + chartRef.current.price.setData(newPriceData); + chartRef.current.confidenceHigh.setData(newConfidenceHighData); + chartRef.current.confidenceLow.setData(newConfidenceLowData); } isBackfilling.current = false; }) @@ -99,7 +176,7 @@ const useChartElem = (symbol: string, feedId: string) => { logger.error("Error fetching historical prices", error); }); } - }, [logger, symbol]); + }, [logger, symbol, interval]); useMountEffect(() => { const chartElem = chartContainerRef.current; @@ -149,6 +226,10 @@ const useChartElem = (symbol: string, feedId: string) => { }); useEffect(() => { + // don't live update if the interval is not set to live + if(interval !== "Live") { + return; + } if (current && chartRef.current) { earliestDateRef.current ??= current.timestamp; const { price, confidence } = current.aggregate; @@ -194,14 +275,6 @@ type ChartRefContents = { } ); -const historicalDataSchema = z.array( - z.strictObject({ - timestamp: z.number(), - price: z.number(), - confidence: z.number(), - }), -); - const priceFormat = { type: "price", precision: 5, @@ -301,3 +374,37 @@ const getLocalTimestamp = (date: Date): UTCTimestamp => date.getSeconds(), date.getMilliseconds(), ) / 1000) as UTCTimestamp; + + const backfillIntervals = ({ interval, earliestDate } : { + interval: typeof INTERVAL_NAMES[keyof typeof INTERVAL_NAMES], + earliestDate: bigint | undefined, + }) => { + const until = earliestDate ?? BigInt(Math.floor(Date.now() / 1000)); + let from; + + switch (interval) { + case '1H': { + from = until - 3600n; + break; + } + case '1D': { + from = until - 86_400n; // seconds in a day + + break; + } + case '1W': { + from = until - 604_800n; // seconds in a week + + break; + } + case '1M': { + from = until - 2_592_000n; + + break; + } + default: { + from = until - 100n; + } + } + return { from, until }; + } \ No newline at end of file diff --git a/apps/insights/src/historicalChart.ts b/apps/insights/src/historicalChart.ts new file mode 100644 index 0000000000..d8fd41aa5f --- /dev/null +++ b/apps/insights/src/historicalChart.ts @@ -0,0 +1,10 @@ +import { INTERVAL_NAMES } from "./components/PriceFeed/Chart/chart-toolbar"; + +export const intervalToResolution = (interval: typeof INTERVAL_NAMES[keyof typeof INTERVAL_NAMES]) => { + switch (interval) { + case '1H': + return Resolution.Hour; + case '1D': + return Resolution.Day; + } +} \ No newline at end of file diff --git a/apps/insights/src/services/clickhouse.ts b/apps/insights/src/services/clickhouse.ts index c0b3b2521a..f66c190070 100644 --- a/apps/insights/src/services/clickhouse.ts +++ b/apps/insights/src/services/clickhouse.ts @@ -183,6 +183,8 @@ const _getYesterdaysPrices = async (symbols: string[]) => }, ); + + const _getPublisherRankingHistory = async ({ cluster, key, @@ -369,13 +371,94 @@ export const getHistoricalPrices = async ({ }, ); + +// Main history function +export const getHistory = async ({ + symbol, + range, + cluster, + from, + until, +}: { + symbol: string; + range: "1H" | "1D" | "1W" | "1M"; + cluster: "pythnet" | "pythtest-conformance"; + from: number; + until: number; +}) => { + + // Calculate interval parameters based on range + const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "HOUR" : "DAY"; + + let additional_cluster_clause = ""; + if (cluster === "pythtest-conformance") { + additional_cluster_clause = " OR (cluster = 'pythtest')"; + } + + const query = ` + SELECT + toUnixTimestamp(toStartOfInterval(publishTime, INTERVAL 1 ${interval_unit})) AS timestamp, + argMin(price, slot) AS openPrice, + min(price) AS lowPrice, + argMax(price, slot) AS closePrice, + max(price) AS highPrice, + avg(price) AS avgPrice, + avg(confidence) AS avgConfidence, + avg(emaPrice) AS avgEmaPrice, + avg(emaConfidence) AS avgEmaConfidence, + min(slot) AS startSlot, + max(slot) AS endSlot + FROM prices + FINAL + WHERE ((symbol = {symbol: String}) OR (symbol = {base_quote_symbol: String})) + AND ((cluster = {cluster: String})${additional_cluster_clause}) + AND (version = 2) + AND (publishTime > fromUnixTimestamp(toInt64({from: String}))) + AND (publishTime <= fromUnixTimestamp(toInt64({until: String}))) + AND (time > (fromUnixTimestamp(toInt64({from: String})))) + AND (time <= (fromUnixTimestamp(toInt64({until: String})))) + AND (publisher = {publisher: String}) + AND (status = 1) + GROUP BY timestamp + ORDER BY timestamp ASC + SETTINGS do_not_merge_across_partitions_select_final=1 + `; +console.log(query) + const data = await safeQuery(z.array( + z.object({ + timestamp: z.number(), + openPrice: z.number(), + lowPrice: z.number(), + closePrice: z.number(), + highPrice: z.number(), + avgPrice: z.number(), + avgConfidence: z.number(), + avgEmaPrice: z.number(), + avgEmaConfidence: z.number(), + startSlot: z.string(), + endSlot: z.string(), + })), { + query, + query_params: { + symbol, + cluster, + publisher: "", + base_quote_symbol: symbol.split(".")[-1], + from, + until, + }, + }); + console.log("from", new Date(from * 1000), from, "until", new Date(until * 1000), until, data[0], data[data.length - 1]); + + return data; +}; + const safeQuery = async ( schema: ZodSchema, query: Omit[0], "format">, ) => { const rows = await client.query({ ...query, format: "JSON" }); const result = await rows.json(); - return schema.parse(result.data); }; diff --git a/apps/insights/src/services/pyth/get-feeds.ts b/apps/insights/src/services/pyth/get-feeds.ts index 93cc9ff486..30caaca800 100644 --- a/apps/insights/src/services/pyth/get-feeds.ts +++ b/apps/insights/src/services/pyth/get-feeds.ts @@ -1,9 +1,9 @@ import { Cluster, priceFeedsSchema } from "."; -import { getPythMetadataCached } from "./get-metadata"; +import { getPythMetadata } from "./get-metadata"; import { redisCache } from "../../cache"; const _getFeeds = async (cluster: Cluster) => { - const unfilteredData = await getPythMetadataCached(cluster); + const unfilteredData = await getPythMetadata(cluster); const filtered = unfilteredData.symbols .filter( (symbol) => @@ -22,8 +22,8 @@ const _getFeeds = async (cluster: Cluster) => { publisher: publisher.toBase58(), })) ?? [], }, - })); - return priceFeedsSchema.parse(filtered); + })) + return priceFeedsSchema.parse(filtered) }; export const getFeeds = redisCache.define("getFeeds", _getFeeds).getFeeds; diff --git a/apps/insights/src/services/pyth/get-metadata.ts b/apps/insights/src/services/pyth/get-metadata.ts index a87dd0733f..b5cd814d89 100644 --- a/apps/insights/src/services/pyth/get-metadata.ts +++ b/apps/insights/src/services/pyth/get-metadata.ts @@ -1,11 +1,11 @@ import { clients, Cluster } from "."; import { memoryOnlyCache } from "../../cache"; -const getPythMetadata = async (cluster: Cluster) => { +const _getPythMetadata = async (cluster: Cluster) => { return clients[cluster].getData(); }; -export const getPythMetadataCached = memoryOnlyCache.define( +export const getPythMetadata = memoryOnlyCache.define( "getPythMetadata", - getPythMetadata, + _getPythMetadata, ).getPythMetadata; diff --git a/apps/insights/src/services/pyth/get-publishers-for-cluster.ts b/apps/insights/src/services/pyth/get-publishers-for-cluster.ts index 452a369156..0c6fa5e854 100644 --- a/apps/insights/src/services/pyth/get-publishers-for-cluster.ts +++ b/apps/insights/src/services/pyth/get-publishers-for-cluster.ts @@ -1,9 +1,9 @@ import { Cluster } from "."; -import { getPythMetadataCached } from "./get-metadata"; +import { getPythMetadata } from "./get-metadata"; import { redisCache } from "../../cache"; const _getPublishersForCluster = async (cluster: Cluster) => { - const data = await getPythMetadataCached(cluster); + const data = await getPythMetadata(cluster); const result: Record = {}; for (const key of data.productPrice.keys()) { const price = data.productPrice.get(key);