Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
5a92750
Bumping drift-common to 5b9a0026783451597eae8d7218a400338928a165
github-actions[bot] Nov 20, 2024
22090f9
Bumping drift-common to 8178bc7515ef09c51290c1463bf7eba3c28de214
github-actions[bot] Nov 21, 2024
8952cc2
Bumping drift-common to c251089a1a4f10ad21814358290dab2541a6eb66
github-actions[bot] Nov 21, 2024
d43d2d2
chore: update trades to helius
jackwaller Nov 21, 2024
4b9b337
chore: update clients to be env vars
jackwaller Nov 21, 2024
3682cbf
Merge pull request #278 from drift-labs/jack/env-clients
jackwaller Nov 21, 2024
e4a78db
chore: null checks
jackwaller Nov 21, 2024
1fbcc74
Merge pull request #281 from drift-labs/jack/env-clients
jackwaller Nov 21, 2024
84000b8
Bumping drift-common to e67d009f244055511f1423b30cd48f18f766afb6
github-actions[bot] Nov 21, 2024
96008aa
Bumping drift-common to 33005401904ed85fd88e07f9a1933e1c663a95d9
github-actions[bot] Nov 22, 2024
5c98fc0
Bumping drift-common to ef16c0c1728ec6d2320a002f346835b091785d7c
github-actions[bot] Nov 25, 2024
791f7a3
Bumping drift-common to 70eac64f8585290a326c61c0bd2f81814e77eaa6
github-actions[bot] Nov 26, 2024
cc4adfa
Bumping drift-common to 79d7d2f2fb10b4c211d1c2ac58484a43cf54404b
github-actions[bot] Nov 27, 2024
fa8fd24
reduce dlob publisher health check grace period, clean up logs
wphan Nov 27, 2024
00e167d
fix logs
wphan Nov 27, 2024
90bcd11
Merge pull request #283 from drift-labs/wphan/improve_dlob_health_checks
wphan Nov 27, 2024
5099774
Merge branch 'mainnet-beta'
wphan Nov 27, 2024
321e3be
Merge branch 'mainnet-beta'
wphan Nov 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion drift-common
9 changes: 4 additions & 5 deletions src/core/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,16 +181,15 @@ export const setHealthStatus = (status: HEALTH_STATUS): void => {
* We may be hit by multiple sources performing health checks on us, so this middleware will latch
* to its health state and only update every `healthCheckInterval`.
*
* A grace period is also used to only report unhealthy if we have been unhealthy for a certain
* amount of time. This prevents reporting unhealthy even if we are just in the middle of a
* bulk account load.
* A grace period is also used to report unhealthy only if we have been unhealthy beyond the grace period.
*/
const handleHealthCheck = (
healthCheckGracePeriod: number,
slotSource: SlotSource
) => {
return async (_req, res, _next) => {
if (healthStatus === HEALTH_STATUS.Restart) {
logger.error(`Health status: Restart`);
res.writeHead(500);
res.end(`NOK`);
return;
Expand Down Expand Up @@ -227,14 +226,14 @@ const handleHealthCheck = (
lastHealthCheckPerformed = Date.now();

if (!lastHealthCheckState && !inGracePeriod) {
healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber;
setHealthStatus(HEALTH_STATUS.UnhealthySlotSubscriber);

res.writeHead(500);
res.end(`NOK`);
return;
}

healthStatus = HEALTH_STATUS.Ok;
setHealthStatus(HEALTH_STATUS.Ok);
res.writeHead(200);
res.end('OK');
};
Expand Down
16 changes: 7 additions & 9 deletions src/dlob-subscriber/DLOBSubscriberIO.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,10 +221,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
!skipSlotCheck &&
!isPerpMarketAndPrelaunchMarket
) {
console.log(`Unhealthy process due to slot diffs for market ${marketName}:
dlobProvider slot: ${slot}
oracle slot: ${l2Formatted['oracleData']['slot']}
`);
console.log(
`Unhealthy process due to slot diffs for market ${marketName}. dlobProviderSlot: ${slot}, oracleSlot: ${l2Formatted['oracleData']['slot']}`
);
setHealthStatus(HEALTH_STATUS.Restart);
}

Expand All @@ -239,16 +238,15 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD &&
!skipSlotCheck
) {
console.log(`Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms:
dlobProvider slot: ${slot}
market slot: ${l2Formatted['marketSlot']}
`);
logger.warn(
`Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms. dlobProviderSlot: ${slot}, marketSlot: ${l2Formatted['marketSlot']}`
);
setHealthStatus(HEALTH_STATUS.Restart);
} else if (
lastMarketSlotAndTime &&
l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot
) {
console.log(
logger.warn(
`Updating market slot for ${marketArgs.marketName} with slot ${l2Formatted['marketSlot']}`
);
this.lastMarketSlotMap
Expand Down
4 changes: 3 additions & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ const clients = process.env.REDIS_CLIENT?.trim()

clients?.forEach((client) => envClients.push(RedisClientPrefix[client]));

const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS];
const REDIS_CLIENTS = envClients.length
? envClients
: [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS];
console.log('Redis Clients:', REDIS_CLIENTS);

const driftEnv = (process.env.ENV || 'devnet') as DriftEnv;
Expand Down
4 changes: 2 additions & 2 deletions src/publishers/dlobPublisher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -455,10 +455,10 @@ const main = async () => {

app.get(
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
);
app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider));
app.get('/', handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider));
const server = app.listen(8080);

// Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients
Expand Down
4 changes: 3 additions & 1 deletion src/serverLite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ const clients = process.env.REDIS_CLIENT?.trim()

clients?.forEach((client) => envClients.push(RedisClientPrefix[client]));

const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS];
const REDIS_CLIENTS = envClients.length
? envClients
: [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS];
console.log('Redis Clients:', REDIS_CLIENTS);

const driftEnv = (process.env.ENV || 'devnet') as DriftEnv;
Expand Down
Loading