Skip to content

Commit cfdc111

Browse files
boudraboudra
authored
Merge pull request #302 from gitcoinco/fix/block-from-timestamp
Fix getBlockFromTimestamp
2 parents 8753b17 + 84fe931 commit cfdc111

10 files changed

+501
-211
lines changed

‎package-lock.json

+28-8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"license": "ISC",
3030
"dependencies": {
3131
"@sentry/node": "^7.51.0",
32+
"better-sqlite3": "^8.6.0",
3233
"chainsauce": "github:gitcoinco/chainsauce#main",
3334
"cors": "^2.8.5",
3435
"csv-parser": "^3.0.0",
@@ -69,6 +70,7 @@
6970
"@types/supertest": "^2.0.12",
7071
"@types/throttle-debounce": "^5.0.0",
7172
"@types/tmp": "^0.2.3",
73+
"@types/better-sqlite3": "^7.6.5",
7274
"@types/write-file-atomic": "^4.0.0",
7375
"@typescript-eslint/eslint-plugin": "^5.55.0",
7476
"@typescript-eslint/parser": "^5.55.0",

‎src/blockCache.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
export type Block = {
2+
chainId: number;
3+
blockNumber: bigint;
4+
timestampInSecs: number;
5+
};
6+
7+
export interface BlockCache {
8+
init(): Promise<void>;
9+
getTimestampByBlockNumber(
10+
chainId: number,
11+
blockNumber: bigint
12+
): Promise<number | null>;
13+
getBlockNumberByTimestamp(
14+
chainId: number,
15+
timestampInSecs: number
16+
): Promise<bigint | null>;
17+
saveBlock(block: Block): Promise<void>;
18+
getClosestBoundsForTimestamp(
19+
chainId: number,
20+
timestampInSecs: number
21+
): Promise<{ before: Block | null; after: Block | null }>;
22+
}

‎src/blockCache/sqlite.test.ts

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import { createSqliteBlockCache } from "./sqlite.js";
2+
import { BlockCache } from "../blockCache.js";
3+
import { it, describe, expect, beforeEach, afterEach } from "vitest";
4+
import Sqlite from "better-sqlite3";
5+
import fs from "fs/promises";
6+
import os from "os";
7+
import path from "path";
8+
9+
describe("createSqliteBlockCache", () => {
10+
let db: Sqlite.Database;
11+
let blockCache: BlockCache;
12+
13+
beforeEach(() => {
14+
db = new Sqlite(":memory:");
15+
blockCache = createSqliteBlockCache({ db });
16+
});
17+
18+
afterEach(() => {
19+
db.close();
20+
});
21+
22+
it("should initialize without errors", async () => {
23+
await expect(blockCache.init()).resolves.not.toThrow();
24+
});
25+
26+
it("should initialize if using invalid table name", () => {
27+
expect(() => {
28+
createSqliteBlockCache({
29+
db,
30+
tableName: "invalid table name",
31+
});
32+
}).toThrow();
33+
34+
expect(() => {
35+
createSqliteBlockCache({
36+
db,
37+
tableName: "table/",
38+
});
39+
}).toThrow();
40+
});
41+
42+
it("should throw if already initialized", async () => {
43+
await blockCache.init();
44+
await expect(blockCache.init()).rejects.toThrow("Already initialized");
45+
});
46+
47+
it("should save and retrieve a block by number", async () => {
48+
await blockCache.init();
49+
const block = {
50+
chainId: 1,
51+
blockNumber: BigInt(1),
52+
timestampInSecs: 12345,
53+
};
54+
await blockCache.saveBlock(block);
55+
56+
const timestampInSecs = await blockCache.getTimestampByBlockNumber(
57+
1,
58+
BigInt(1)
59+
);
60+
expect(timestampInSecs).toEqual(block.timestampInSecs);
61+
});
62+
63+
it("should save and retrieve a block by timestamp", async () => {
64+
await blockCache.init();
65+
const block = {
66+
chainId: 1,
67+
blockNumber: BigInt(1),
68+
timestampInSecs: 12345,
69+
};
70+
await blockCache.saveBlock(block);
71+
72+
const blockNumber = await blockCache.getBlockNumberByTimestamp(1, 12345);
73+
expect(blockNumber).toEqual(block.blockNumber);
74+
});
75+
76+
it("should get closest bounds for timestamp", async () => {
77+
await blockCache.init();
78+
const block1 = { chainId: 1, blockNumber: BigInt(1), timestampInSecs: 10 };
79+
const block2 = { chainId: 1, blockNumber: BigInt(2), timestampInSecs: 20 };
80+
81+
await blockCache.saveBlock(block1);
82+
await blockCache.saveBlock(block2);
83+
84+
const bounds = await blockCache.getClosestBoundsForTimestamp(1, 15);
85+
expect(bounds.before).toEqual(block1);
86+
expect(bounds.after).toEqual(block2);
87+
});
88+
});
89+
90+
describe("createSqliteBlockCache with dbPath", () => {
91+
it("should initialize without errors using dbPath", async () => {
92+
const tmpFilePath = path.join(os.tmpdir(), `tmpdb-${Date.now()}.db`);
93+
const diskBlockCache = createSqliteBlockCache({ dbPath: tmpFilePath });
94+
await expect(diskBlockCache.init()).resolves.not.toThrow();
95+
await fs.rm(tmpFilePath);
96+
});
97+
});

‎src/blockCache/sqlite.ts

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
import Sqlite from "better-sqlite3";
2+
import { BlockCache, Block } from "../blockCache.js";
3+
4+
const defaultTableName = "blocks";
5+
6+
export type Options =
7+
| {
8+
dbPath: string;
9+
tableName?: string;
10+
}
11+
| { db: Sqlite.Database; tableName?: string };
12+
13+
interface Row {
14+
chainId: number;
15+
blockNumber: string;
16+
timestamp: number;
17+
}
18+
19+
type UninitializedState = { state: "uninitialized" };
20+
type InitializedState = {
21+
state: "initialized";
22+
db: Sqlite.Database;
23+
getTimestampByBlockNumberStmt: Sqlite.Statement;
24+
getBlockNumberByTimestampStmt: Sqlite.Statement;
25+
saveBlockStmt: Sqlite.Statement;
26+
getBeforeStmt: Sqlite.Statement;
27+
getAfterStmt: Sqlite.Statement;
28+
};
29+
type State = UninitializedState | InitializedState;
30+
31+
export function createSqliteBlockCache(opts: Options): BlockCache {
32+
let dbState: State = { state: "uninitialized" };
33+
34+
if (opts.tableName !== undefined && /[^a-zA-Z0-9_]/.test(opts.tableName)) {
35+
throw new Error(`Table name ${opts.tableName} has invalid characters.`);
36+
}
37+
38+
const tableName = opts.tableName ?? defaultTableName;
39+
40+
return {
41+
async init(): Promise<void> {
42+
if (dbState.state === "initialized") {
43+
throw new Error("Already initialized");
44+
}
45+
46+
const db = "db" in opts ? opts.db : new Sqlite(opts.dbPath);
47+
48+
db.exec("PRAGMA journal_mode = WAL;");
49+
50+
// TODO: Add proper migrations, with Kysely?
51+
db.exec(
52+
`CREATE TABLE IF NOT EXISTS ${tableName} (
53+
chainId INTEGER,
54+
blockNumber TEXT,
55+
timestamp INTEGER,
56+
PRIMARY KEY (chainId, blockNumber)
57+
)`
58+
);
59+
60+
db.exec(
61+
`CREATE INDEX IF NOT EXISTS idx_chainId_timestamp_blockNumber
62+
ON ${tableName} (chainId, timestamp, blockNumber DESC);`
63+
);
64+
65+
dbState = {
66+
state: "initialized",
67+
db,
68+
getTimestampByBlockNumberStmt: db.prepare(
69+
`SELECT * FROM ${tableName} WHERE chainId = ? AND blockNumber = ?`
70+
),
71+
getBlockNumberByTimestampStmt: db.prepare(
72+
`SELECT * FROM ${tableName} WHERE chainId = ? AND timestamp = ?`
73+
),
74+
saveBlockStmt: db.prepare(
75+
`INSERT OR REPLACE INTO ${tableName} (chainId, blockNumber, timestamp) VALUES (?, ?, ?)`
76+
),
77+
getBeforeStmt: db.prepare(
78+
`SELECT * FROM ${tableName} WHERE chainId = ? AND timestamp < ? ORDER BY timestamp DESC, blockNumber DESC LIMIT 1`
79+
),
80+
getAfterStmt: db.prepare(
81+
`SELECT * FROM ${tableName} WHERE chainId = ? AND timestamp >= ? ORDER BY timestamp ASC, blockNumber ASC LIMIT 1`
82+
),
83+
};
84+
85+
return Promise.resolve();
86+
},
87+
88+
async getTimestampByBlockNumber(
89+
chainId,
90+
blockNumber
91+
): Promise<number | null> {
92+
if (dbState.state === "uninitialized") {
93+
throw new Error("SQLite database not initialized");
94+
}
95+
96+
const row = dbState.getTimestampByBlockNumberStmt.get(
97+
chainId,
98+
blockNumber.toString()
99+
) as Row | undefined;
100+
101+
return Promise.resolve(row ? row.timestamp : null);
102+
},
103+
104+
async getBlockNumberByTimestamp(
105+
chainId,
106+
timestamp
107+
): Promise<bigint | null> {
108+
if (dbState.state === "uninitialized") {
109+
throw new Error("SQLite database not initialized");
110+
}
111+
112+
const row = dbState.getBlockNumberByTimestampStmt.get(
113+
chainId,
114+
timestamp
115+
) as Row | undefined;
116+
117+
return Promise.resolve(row ? BigInt(row.blockNumber) : null);
118+
},
119+
120+
async saveBlock(block: Block): Promise<void> {
121+
if (dbState.state === "uninitialized") {
122+
throw new Error("SQLite database not initialized");
123+
}
124+
125+
dbState.saveBlockStmt.run(
126+
block.chainId,
127+
block.blockNumber.toString(),
128+
block.timestampInSecs
129+
);
130+
131+
return Promise.resolve();
132+
},
133+
134+
async getClosestBoundsForTimestamp(
135+
chainId,
136+
timestamp
137+
): Promise<{ before: Block | null; after: Block | null }> {
138+
if (dbState.state === "uninitialized") {
139+
throw new Error("SQLite database not initialized");
140+
}
141+
142+
const before = dbState.getBeforeStmt.get(chainId, timestamp) as
143+
| Row
144+
| undefined;
145+
146+
const after = dbState.getAfterStmt.get(chainId, timestamp) as
147+
| Row
148+
| undefined;
149+
150+
return Promise.resolve({
151+
before: before
152+
? {
153+
chainId: before.chainId,
154+
timestampInSecs: before.timestamp,
155+
blockNumber: BigInt(before.blockNumber),
156+
}
157+
: null,
158+
after: after
159+
? {
160+
chainId: after.chainId,
161+
timestampInSecs: after.timestamp,
162+
blockNumber: BigInt(after.blockNumber),
163+
}
164+
: null,
165+
});
166+
},
167+
};
168+
}

‎src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ async function catchupAndWatchChain(
223223
rpcProvider,
224224
chain: config.chain,
225225
logger: chainLogger.child({ subsystem: "PriceUpdater" }),
226+
blockCachePath: path.join(config.storageDir, "..", "blockCache.db"),
226227
withCacheFn:
227228
pricesCache === null
228229
? undefined

0 commit comments

Comments
 (0)