Skip to content

Commit 89e0930

Browse files
geroplgeropl
authored
[server] Introduces ReadinessProbe (#20669)
* [server] Introduce ReadinessController and probe at /ready Tool: gitpod/catfood.gitpod.cloud * [server] Move /live and /ready endpoints to a separate express app and port Tool: gitpod/catfood.gitpod.cloud * [memory-bank] task-related learnings Tool: gitpod/catfood.gitpod.cloud * [server] Introduce `server_readiness_probe` feature flag so we can disable the ReadinessProbe if required Tool: gitpod/catfood.gitpod.cloud * docs: formalize Product Requirements Document workflow - Add PRD workflow to systemPatterns.md as a standardized development process - Update .clinerules with instructions to follow the PRD workflow - Update activeContext.md and progress.md to reference the new workflow This formalizes the process we used for implementing the server readiness probe feature. Tool: gitpod/catfood.gitpod.cloud * [server] ReadinessProbe: add redis as dependency Tool: gitpod/catfood.gitpod.cloud * review comments Tool: gitpod/catfood.gitpod.cloud * [dev] Remove outdated gopls config Tool: gitpod/catfood.gitpod.cloud * [server] Fix import Tool: gitpod/catfood.gitpod.cloud
1 parent 5d557f7 commit 89e0930

File tree

14 files changed

+288
-35
lines changed

14 files changed

+288
-35
lines changed

Diff for: ‎components/content-service/content-service.code-workspace

-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
},
4444
"go.lintTool": "golangci-lint",
4545
"gopls": {
46-
"allowModfileModifications": true
4746
}
4847
}
4948
}

Diff for: ‎components/image-builder-mk3/image-builder.code-workspace

-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
},
4646
"go.lintTool": "golangci-lint",
4747
"gopls": {
48-
"allowModfileModifications": true
4948
}
5049
}
5150
}

Diff for: ‎components/server/src/container-module.ts

+5
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ import { WebhookEventGarbageCollector } from "./jobs/webhook-gc";
7171
import { WorkspaceGarbageCollector } from "./jobs/workspace-gc";
7272
import { LinkedInService } from "./linkedin-service";
7373
import { LivenessController } from "./liveness/liveness-controller";
74+
import { ReadinessController } from "./liveness/readiness-controller";
7475
import { RedisSubscriber } from "./messaging/redis-subscriber";
7576
import { MonitoringEndpointsApp } from "./monitoring-endpoints";
7677
import { OAuthController } from "./oauth-server/oauth-controller";
@@ -135,6 +136,7 @@ import { AnalyticsController } from "./analytics-controller";
135136
import { InstallationAdminCleanup } from "./jobs/installation-admin-cleanup";
136137
import { AuditLogService } from "./audit/AuditLogService";
137138
import { AuditLogGarbageCollectorJob } from "./jobs/auditlog-gc";
139+
import { ProbesApp } from "./liveness/probes";
138140

139141
export const productionContainerModule = new ContainerModule(
140142
(bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
@@ -240,7 +242,10 @@ export const productionContainerModule = new ContainerModule(
240242
bind(IWorkspaceManagerClientCallMetrics).toService(IClientCallMetrics);
241243

242244
bind(WorkspaceDownloadService).toSelf().inSingletonScope();
245+
246+
bind(ProbesApp).toSelf().inSingletonScope();
243247
bind(LivenessController).toSelf().inSingletonScope();
248+
bind(ReadinessController).toSelf().inSingletonScope();
244249

245250
bind(OneTimeSecretServer).toSelf().inSingletonScope();
246251

Diff for: ‎components/server/src/liveness/probes.ts

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/**
2+
* Copyright (c) 2025 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import express from "express";
8+
import { inject, injectable } from "inversify";
9+
import { LivenessController } from "./liveness-controller";
10+
import { ReadinessController } from "./readiness-controller";
11+
12+
@injectable()
13+
export class ProbesApp {
14+
constructor(
15+
@inject(LivenessController) protected readonly livenessController: LivenessController,
16+
@inject(ReadinessController) protected readonly readinessController: ReadinessController,
17+
) {}
18+
19+
public create(): express.Application {
20+
const probesApp = express();
21+
probesApp.use("/live", this.livenessController.apiRouter);
22+
probesApp.use("/ready", this.readinessController.apiRouter);
23+
return probesApp;
24+
}
25+
}

Diff for: ‎components/server/src/liveness/readiness-controller.ts

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/**
2+
* Copyright (c) 2025 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { injectable, inject } from "inversify";
8+
import express from "express";
9+
import { TypeORM } from "@gitpod/gitpod-db/lib";
10+
import { SpiceDBClientProvider } from "../authorization/spicedb";
11+
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
12+
import { v1 } from "@authzed/authzed-node";
13+
import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
14+
import { Redis } from "ioredis";
15+
16+
@injectable()
17+
export class ReadinessController {
18+
@inject(TypeORM) protected readonly typeOrm: TypeORM;
19+
@inject(SpiceDBClientProvider) protected readonly spiceDBClientProvider: SpiceDBClientProvider;
20+
@inject(Redis) protected readonly redis: Redis;
21+
22+
get apiRouter(): express.Router {
23+
const router = express.Router();
24+
this.addReadinessHandler(router);
25+
return router;
26+
}
27+
28+
protected addReadinessHandler(router: express.Router) {
29+
router.get("/", async (_, res) => {
30+
try {
31+
// Check feature flag first
32+
const readinessProbeEnabled = await getExperimentsClientForBackend().getValueAsync(
33+
"server_readiness_probe",
34+
true, // Default to readiness probe, skip if false
35+
{},
36+
);
37+
38+
if (!readinessProbeEnabled) {
39+
log.debug("Readiness check skipped due to feature flag");
40+
res.status(200);
41+
return;
42+
}
43+
44+
// Check database connection
45+
const dbConnection = await this.checkDatabaseConnection();
46+
if (!dbConnection) {
47+
log.warn("Readiness check failed: Database connection failed");
48+
res.status(503).send("Database connection failed");
49+
return;
50+
}
51+
52+
// Check SpiceDB connection
53+
const spiceDBConnection = await this.checkSpiceDBConnection();
54+
if (!spiceDBConnection) {
55+
log.warn("Readiness check failed: SpiceDB connection failed");
56+
res.status(503).send("SpiceDB connection failed");
57+
return;
58+
}
59+
60+
// Check Redis connection
61+
const redisConnection = await this.checkRedisConnection();
62+
if (!redisConnection) {
63+
log.warn("Readiness check failed: Redis connection failed");
64+
res.status(503).send("Redis connection failed");
65+
return;
66+
}
67+
68+
// All connections are good
69+
res.status(200).send("Ready");
70+
} catch (error) {
71+
log.error("Readiness check failed", error);
72+
res.status(503).send("Readiness check failed");
73+
}
74+
});
75+
}
76+
77+
private async checkDatabaseConnection(): Promise<boolean> {
78+
try {
79+
const connection = await this.typeOrm.getConnection();
80+
// Simple query to verify connection is working
81+
await connection.query("SELECT 1");
82+
return true;
83+
} catch (error) {
84+
log.error("Database connection check failed", error);
85+
return false;
86+
}
87+
}
88+
89+
private async checkSpiceDBConnection(): Promise<boolean> {
90+
try {
91+
const client = this.spiceDBClientProvider.getClient();
92+
93+
// Send a request, to verify that the connection works
94+
const req = v1.ReadSchemaRequest.create({});
95+
const response = await client.readSchema(req);
96+
log.debug("SpiceDB connection check successful", { schemaLength: response.schemaText.length });
97+
98+
return true;
99+
} catch (error) {
100+
log.error("SpiceDB connection check failed", error);
101+
return false;
102+
}
103+
}
104+
105+
private async checkRedisConnection(): Promise<boolean> {
106+
try {
107+
// Simple PING command to verify connection is working
108+
const result = await this.redis.ping();
109+
log.debug("Redis connection check successful", { result });
110+
return result === "PONG";
111+
} catch (error) {
112+
log.error("Redis connection check failed", error);
113+
return false;
114+
}
115+
}
116+
}

Diff for: ‎components/server/src/server.ts

+19-5
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import { NewsletterSubscriptionController } from "./user/newsletter-subscription
3636
import { Config } from "./config";
3737
import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
3838
import { WsConnectionHandler } from "./express/ws-connection-handler";
39-
import { LivenessController } from "./liveness/liveness-controller";
4039
import { IamSessionApp } from "./iam/iam-session-app";
4140
import { API } from "./api/server";
4241
import { GithubApp } from "./prebuilds/github-app";
@@ -53,6 +52,11 @@ import {
5352
} from "./workspace/headless-log-service";
5453
import { runWithRequestContext } from "./util/request-context";
5554
import { AnalyticsController } from "./analytics-controller";
55+
import { ProbesApp as ProbesAppProvider } from "./liveness/probes";
56+
57+
const MONITORING_PORT = 9500;
58+
const IAM_SESSION_PORT = 9876;
59+
const PROBES_PORT = 9400;
5660

5761
@injectable()
5862
export class Server {
@@ -65,6 +69,8 @@ export class Server {
6569
protected privateApiServer?: http.Server;
6670

6771
protected readonly eventEmitter = new EventEmitter();
72+
protected probesApp: express.Application;
73+
protected probesServer?: http.Server;
6874
protected app?: express.Application;
6975
protected httpServer?: http.Server;
7076
protected monitoringApp?: express.Application;
@@ -79,7 +85,6 @@ export class Server {
7985
@inject(UserController) private readonly userController: UserController,
8086
@inject(WebsocketConnectionManager) private readonly websocketConnectionHandler: WebsocketConnectionManager,
8187
@inject(WorkspaceDownloadService) private readonly workspaceDownloadService: WorkspaceDownloadService,
82-
@inject(LivenessController) private readonly livenessController: LivenessController,
8388
@inject(MonitoringEndpointsApp) private readonly monitoringEndpointsApp: MonitoringEndpointsApp,
8489
@inject(CodeSyncService) private readonly codeSyncService: CodeSyncService,
8590
@inject(HeadlessLogController) private readonly headlessLogController: HeadlessLogController,
@@ -100,6 +105,7 @@ export class Server {
100105
@inject(API) private readonly api: API,
101106
@inject(RedisSubscriber) private readonly redisSubscriber: RedisSubscriber,
102107
@inject(AnalyticsController) private readonly analyticsController: AnalyticsController,
108+
@inject(ProbesAppProvider) private readonly probesAppProvider: ProbesAppProvider,
103109
) {}
104110

105111
public async init(app: express.Application) {
@@ -113,6 +119,9 @@ export class Server {
113119
await this.typeOrm.connect();
114120
log.info("connected to DB");
115121

122+
// probes
123+
this.probesApp = this.probesAppProvider.create();
124+
116125
// metrics
117126
app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
118127
const startTime = Date.now();
@@ -319,7 +328,6 @@ export class Server {
319328
// Authorization: none
320329
app.use(this.oneTimeSecretServer.apiRouter);
321330
app.use(this.newsletterSubscriptionController.apiRouter);
322-
app.use("/live", this.livenessController.apiRouter);
323331
app.use("/version", (req: express.Request, res: express.Response, next: express.NextFunction) => {
324332
res.send(this.config.version);
325333
});
@@ -351,22 +359,27 @@ export class Server {
351359
throw new Error("server cannot start, not initialized");
352360
}
353361

362+
const probeServer = this.probesApp.listen(PROBES_PORT, () => {
363+
log.info(`probes server listening on port: ${(<AddressInfo>probeServer.address()).port}`);
364+
});
365+
this.probesServer = probeServer;
366+
354367
const httpServer = this.app.listen(port, () => {
355368
this.eventEmitter.emit(Server.EVENT_ON_START, httpServer);
356369
log.info(`server listening on port: ${(<AddressInfo>httpServer.address()).port}`);
357370
});
358371
this.httpServer = httpServer;
359372

360373
if (this.monitoringApp) {
361-
this.monitoringHttpServer = this.monitoringApp.listen(9500, "localhost", () => {
374+
this.monitoringHttpServer = this.monitoringApp.listen(MONITORING_PORT, "localhost", () => {
362375
log.info(
363376
`monitoring app listening on port: ${(<AddressInfo>this.monitoringHttpServer!.address()).port}`,
364377
);
365378
});
366379
}
367380

368381
if (this.iamSessionApp) {
369-
this.iamSessionAppServer = this.iamSessionApp.listen(9876, () => {
382+
this.iamSessionAppServer = this.iamSessionApp.listen(IAM_SESSION_PORT, () => {
370383
log.info(
371384
`IAM session server listening on port: ${(<AddressInfo>this.iamSessionAppServer!.address()).port}`,
372385
);
@@ -406,6 +419,7 @@ export class Server {
406419
race(this.stopServer(this.httpServer), "stop httpserver"),
407420
race(this.stopServer(this.privateApiServer), "stop private api server"),
408421
race(this.stopServer(this.publicApiServer), "stop public api server"),
422+
race(this.stopServer(this.probesServer), "stop probe server"),
409423
race((async () => this.disposables.dispose())(), "dispose disposables"),
410424
]);
411425

Diff for: ‎components/ws-manager-mk2/ws-manager-mk2.code-workspace

-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
},
4747
"go.lintTool": "golangci-lint",
4848
"gopls": {
49-
"allowModfileModifications": true
5049
}
5150
}
5251
}

Diff for: ‎gitpod-ws.code-workspace

-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,6 @@
162162
"go.lintTool": "golangci-lint",
163163
"go.lintFlags": ["-disable", "govet,errcheck,staticcheck", "--allow-parallel-runners", "--timeout", "15m"],
164164
"gopls": {
165-
"allowModfileModifications": true
166165
},
167166
"prettier.configPath": "/workspace/gitpod/.prettierrc.json"
168167
}

Diff for: ‎install/installer/pkg/components/server/constants.go

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ const (
2525
DebugNodePortName = "debugnode"
2626
ServicePort = 3000
2727
personalAccessTokenSigningKeyMountPath = "/secrets/personal-access-token-signing-key"
28+
ProbesPort = 9400
29+
ProbesPortName = "probes"
2830

2931
AdminCredentialsSecretName = "admin-credentials"
3032
AdminCredentialsSecretMountPath = "/credentials/admin"

0 commit comments

Comments
 (0)