Skip to content

Commit 6d06398

Browse files
committed
refactor: enhance scheduler and worker pool with context cancellation support
1 parent c91b7ee commit 6d06398

4 files changed

Lines changed: 64 additions & 43 deletions

File tree

cron/scheduler.go

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,11 @@ type Scheduler struct {
7171
parsedMu sync.RWMutex
7272
parsed map[string]cronlib.Schedule
7373

74-
stopCh chan struct{}
75-
stopOnce sync.Once
76-
wg sync.WaitGroup
74+
stopCh chan struct{}
75+
cancelCtx context.Context
76+
cancelFunc context.CancelFunc
77+
stopOnce sync.Once
78+
wg sync.WaitGroup
7779
}
7880

7981
// NewScheduler creates a Scheduler.
@@ -110,6 +112,7 @@ func NewScheduler(
110112

111113
// Start launches the leader election and cron tick goroutines.
112114
func (s *Scheduler) Start(_ context.Context) error {
115+
s.cancelCtx, s.cancelFunc = context.WithCancel(context.Background())
113116
s.wg.Add(2)
114117
go s.leaderLoop()
115118
go s.tickLoop()
@@ -126,6 +129,9 @@ func (s *Scheduler) Start(_ context.Context) error {
126129
func (s *Scheduler) Stop(_ context.Context) error {
127130
s.stopOnce.Do(func() {
128131
close(s.stopCh)
132+
if s.cancelFunc != nil {
133+
s.cancelFunc()
134+
}
129135
s.wg.Wait()
130136
s.logger.Info("cron scheduler stopped")
131137
})
@@ -154,7 +160,7 @@ func (s *Scheduler) leaderLoop() {
154160
}
155161

156162
func (s *Scheduler) tryLeadership() {
157-
ctx := context.Background()
163+
ctx := s.cancelCtx
158164

159165
// Try to renew first (cheap if already leader).
160166
renewed, err := s.clusterStore.RenewLeadership(ctx, s.workerID, s.leaderTTL)
@@ -195,7 +201,7 @@ func (s *Scheduler) tickLoop() {
195201
}
196202

197203
func (s *Scheduler) tick() {
198-
ctx := context.Background()
204+
ctx := s.cancelCtx
199205

200206
// Check if we are the leader.
201207
leader, err := s.clusterStore.GetLeader(ctx)

go.mod

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,16 @@ require (
1414
github.com/uptrace/bun/dialect/pgdialect v1.2.16
1515
github.com/uptrace/bun/driver/pgdriver v1.2.16
1616
github.com/vmihailenco/msgpack/v5 v5.4.1
17-
github.com/xraph/forge v1.4.5
17+
github.com/xraph/forge v1.6.0
1818
github.com/xraph/forgeui v1.4.1
19-
github.com/xraph/grove v1.4.3
20-
github.com/xraph/grove/drivers/mongodriver v1.4.3
21-
github.com/xraph/grove/drivers/pgdriver v1.4.3
22-
github.com/xraph/grove/drivers/sqlitedriver v1.4.3
23-
github.com/xraph/grove/kv v1.4.3
24-
github.com/xraph/grove/kv/drivers/redisdriver v1.4.3
25-
github.com/xraph/relay v1.4.2
26-
github.com/xraph/vessel v1.0.0
19+
github.com/xraph/grove v1.5.1
20+
github.com/xraph/grove/drivers/mongodriver v1.5.1
21+
github.com/xraph/grove/drivers/pgdriver v1.5.1
22+
github.com/xraph/grove/drivers/sqlitedriver v1.5.1
23+
github.com/xraph/grove/kv v1.5.1
24+
github.com/xraph/grove/kv/drivers/redisdriver v1.5.1
25+
github.com/xraph/relay v1.5.0
26+
github.com/xraph/vessel v1.0.2
2727
go.jetify.com/typeid/v2 v2.0.0-alpha.3
2828
go.mongodb.org/mongo-driver/v2 v2.5.0
2929
go.opentelemetry.io/otel v1.40.0
@@ -124,7 +124,7 @@ require (
124124
github.com/uptrace/bunrouter v1.0.23 // indirect
125125
github.com/x448/float16 v0.8.4 // indirect
126126
github.com/xraph/confy v0.5.0 // indirect
127-
github.com/xraph/go-utils v1.1.0
127+
github.com/xraph/go-utils v1.1.1
128128
github.com/yusufpapurcu/wmi v1.2.4 // indirect
129129
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
130130
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect

go.sum

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -423,28 +423,28 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
423423
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
424424
github.com/xraph/confy v0.5.0 h1:7dK3hx3MQKlNPK9mFSm07iyU05kUnx6um8d86/gyajg=
425425
github.com/xraph/confy v0.5.0/go.mod h1:/uhVfKibPR+kn7MI9LkVVekk84NP0sxsKZ9sFQoQ5Kc=
426-
github.com/xraph/forge v1.4.5 h1:WLhdv65woppGE1Yr558IkyiJa/lTmMB9d2rzuwSNJQM=
427-
github.com/xraph/forge v1.4.5/go.mod h1:8LYQ2q1DBXcRFpEXBeFEhMrFiE2jzH1DedD6lnBc1h0=
426+
github.com/xraph/forge v1.6.0 h1:dT0+GRNh5M27IFOn3qAExELjKMYqDG1xlcTYC87C8eY=
427+
github.com/xraph/forge v1.6.0/go.mod h1:xSjL8lpXSXHsOpsU7FB/WZPJ0kynpX7fozojWeJiU5E=
428428
github.com/xraph/forgeui v1.4.1 h1:LHK1t/sZ+9zL+MNUZralO9/rc0f5UCa19dpbWTuRMNg=
429429
github.com/xraph/forgeui v1.4.1/go.mod h1:rH/+wb1tt2pXSHotWAvoP+Lt846xlIjuwPDSpS5K5mw=
430-
github.com/xraph/go-utils v1.1.0 h1:bPO4C/mwm0I5ZZa3RxbwUqKhy3Vnxvm3s98vmX8CrjY=
431-
github.com/xraph/go-utils v1.1.0/go.mod h1:tZN4SuGy9otCo6dETp7Cvkgyiy0BvaJaZbTBv4Euvo4=
432-
github.com/xraph/grove v1.4.3 h1:YZCF7wFFuyaU7crO2i1WJ8BErlsCmxXMKsNhxcN9+U0=
433-
github.com/xraph/grove v1.4.3/go.mod h1:3vrjpafTi3VA3OD05xIHMRsfqlcZW34+/tOJqzlUvv8=
434-
github.com/xraph/grove/drivers/mongodriver v1.4.3 h1:0PNwF7/zxXdRYAYKBIa2zn8Lh3ZDwlFaXVN+oAUzzfY=
435-
github.com/xraph/grove/drivers/mongodriver v1.4.3/go.mod h1:46El0w2/vGeqTTWYTfq9I0sXXxaLBiOJFaXPEEBVwAY=
436-
github.com/xraph/grove/drivers/pgdriver v1.4.3 h1:gRjj3i/f97sM8hNedqMmtI1LDZQOc5EXTeBwjHWdsT0=
437-
github.com/xraph/grove/drivers/pgdriver v1.4.3/go.mod h1:aXris8DoBvJwGTpnF2kMC50aBwpxv/e+4mGMRFpC9B0=
438-
github.com/xraph/grove/drivers/sqlitedriver v1.4.3 h1:J5seKm3WjBThCxmGg5JdNmwDQZDTT70Q4FU7ZLxTIm4=
439-
github.com/xraph/grove/drivers/sqlitedriver v1.4.3/go.mod h1:YwhMU+qclOkK0pRrbfvyuhGRU8eREY5jEfViI2FjqFk=
440-
github.com/xraph/grove/kv v1.4.3 h1:YPbT5kyPWKpTfL9T3noZURxubX/m+AuU0SdHqck3Z0I=
441-
github.com/xraph/grove/kv v1.4.3/go.mod h1:vMM7s8Vl90WwZACttXWqIHRdggATveaVSAcnoqbiwzY=
442-
github.com/xraph/grove/kv/drivers/redisdriver v1.4.3 h1:TwvQEDfiNO9GDJ+vstVXRfW4BozmufZ32m5SF3Cpl5s=
443-
github.com/xraph/grove/kv/drivers/redisdriver v1.4.3/go.mod h1:/TMnYUGubFMMxkRJm/JtgmfMtzLArMyPLlKAxkyfdeo=
444-
github.com/xraph/relay v1.4.2 h1:ytpb/rs3jemfYslRjYvaguP2kHyzFApU2FA1MbVXxtI=
445-
github.com/xraph/relay v1.4.2/go.mod h1:5yyUzMas+xw3POM66EsKaSVlvGy/YnZ/rFB+fhIJHFI=
446-
github.com/xraph/vessel v1.0.0 h1:n2q30d0OGPENpFfmOUgEuS99Y+X6b6WTfzdOHiE4Ds0=
447-
github.com/xraph/vessel v1.0.0/go.mod h1:quT3UWDXZF0RLL34H3ijXP9kVnh2pdfnn0f2s3ezChA=
430+
github.com/xraph/go-utils v1.1.1 h1:k5TOQ1qhXLdEX8W5F60mSuPcFOPc8sBsnhmjx/Kpr5g=
431+
github.com/xraph/go-utils v1.1.1/go.mod h1:tZN4SuGy9otCo6dETp7Cvkgyiy0BvaJaZbTBv4Euvo4=
432+
github.com/xraph/grove v1.5.1 h1:vjbw22dIwzNcJzL84ZtIrza/a7YPaFCgK3oVr8tslbE=
433+
github.com/xraph/grove v1.5.1/go.mod h1:bgjHNhnmyfEyzbdpcppRt+Zf24nNcbGKlo450Mi4giI=
434+
github.com/xraph/grove/drivers/mongodriver v1.5.1 h1:IJvvqIQCkiShA65cE0lKufkzkDU9yZJae3GHZX/nmsw=
435+
github.com/xraph/grove/drivers/mongodriver v1.5.1/go.mod h1:xojoSuw3qSm3NIe3xBmasocRQ7oyju3XbjvdlCXVXIU=
436+
github.com/xraph/grove/drivers/pgdriver v1.5.1 h1:GGYoyVShIFs+iHn8WUYOEV8XM72XpHkLaZisVpn04z4=
437+
github.com/xraph/grove/drivers/pgdriver v1.5.1/go.mod h1:9IZG13Q7iXEl7nGSWoYqzytiEDmI/eoTCb/BoCOfWA0=
438+
github.com/xraph/grove/drivers/sqlitedriver v1.5.1 h1:raVV4w/zxrwr8NMrK8Adl9erK96DU7c74G6kiO7XtlE=
439+
github.com/xraph/grove/drivers/sqlitedriver v1.5.1/go.mod h1:xzHewWROOPVn0Luu8/sWEAhSgmDnw3xmNrRw0xcefnM=
440+
github.com/xraph/grove/kv v1.5.1 h1:0JCo3H8w35iZZPYgg9nByDmmRWivpy9kXd/B5vjq+Os=
441+
github.com/xraph/grove/kv v1.5.1/go.mod h1:rdwh/Ja2WOhZ3YwrIp8vBeDNATAOj+2vrK+XVuRHrJU=
442+
github.com/xraph/grove/kv/drivers/redisdriver v1.5.1 h1:ZbzatevpHiZ259T5QiHQWZUfpUQe2jgYtfGh9ukKGP4=
443+
github.com/xraph/grove/kv/drivers/redisdriver v1.5.1/go.mod h1:15TFWsrEvCTHKiqgn40x06VzQTFEaiWyjpVoA2m3FFU=
444+
github.com/xraph/relay v1.5.0 h1:TYgzISyM/frYEXvxj4NevHGNASQ3GE4OywuKmheOky4=
445+
github.com/xraph/relay v1.5.0/go.mod h1:8jw27gFmzT79G4T6vZKXK3YRDhumLKDzjbFh6sQzq0s=
446+
github.com/xraph/vessel v1.0.2 h1:IeNTwxiFgqH2vW9lh8PNXr1SeGdEc7cxQ6sH7jMokfo=
447+
github.com/xraph/vessel v1.0.2/go.mod h1:5hgrMbuczxu2kRIM3iVJ3wPSb6HOvbMhV4nkRFaPNqs=
448448
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
449449
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
450450
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

worker/pool.go

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ type Pool struct {
4343
queueManager QueueManager
4444

4545
stopCh chan struct{}
46+
cancelCtx context.Context // Cancelled on Stop to interrupt in-flight store operations
47+
cancelFunc context.CancelFunc // Cancels cancelCtx
4648
wg sync.WaitGroup
4749
mu sync.Mutex
4850
running bool
@@ -125,6 +127,7 @@ func (p *Pool) Start(_ context.Context) error {
125127
return nil
126128
}
127129
p.running = true
130+
p.cancelCtx, p.cancelFunc = context.WithCancel(context.Background())
128131

129132
p.logger.Info("worker pool starting",
130133
log.String("worker_id", p.workerID.String()),
@@ -165,8 +168,11 @@ func (p *Pool) Stop(ctx context.Context) error {
165168

166169
p.logger.Info("worker pool stopping", log.String("worker_id", p.workerID.String()))
167170

168-
// Signal all workers to stop.
171+
// Signal all workers to stop and cancel in-flight store operations.
169172
close(p.stopCh)
173+
if p.cancelFunc != nil {
174+
p.cancelFunc()
175+
}
170176

171177
// Wait for completion or context deadline.
172178
done := make(chan struct{})
@@ -195,11 +201,16 @@ func (p *Pool) dequeueLoop() {
195201
select {
196202
case <-p.stopCh:
197203
return
204+
case <-p.cancelCtx.Done():
205+
return
198206
default:
199207
}
200208

201-
jobs, err := p.store.DequeueJobs(context.Background(), p.queues, 1)
209+
jobs, err := p.store.DequeueJobs(p.cancelCtx, p.queues, 1)
202210
if err != nil {
211+
if p.cancelCtx.Err() != nil {
212+
return // Clean exit during shutdown
213+
}
203214
p.logger.Error("dequeue error", log.String("error", err.Error()))
204215
p.sleep()
205216
continue
@@ -217,7 +228,10 @@ func (p *Pool) dequeueLoop() {
217228
// Rate limited — return the job to pending with a small delay.
218229
j.State = job.StatePending
219230
j.RunAt = time.Now().Add(p.pollInterval)
220-
if updateErr := p.store.UpdateJob(context.Background(), j); updateErr != nil {
231+
if updateErr := p.store.UpdateJob(p.cancelCtx, j); updateErr != nil {
232+
if p.cancelCtx.Err() != nil {
233+
return
234+
}
221235
p.logger.Error("failed to re-enqueue rate-limited job",
222236
log.String("job_id", j.ID.String()),
223237
log.String("error", updateErr.Error()),
@@ -227,9 +241,9 @@ func (p *Pool) dequeueLoop() {
227241
continue
228242
}
229243

230-
p.extensions.EmitJobStarted(context.Background(), j)
244+
p.extensions.EmitJobStarted(p.cancelCtx, j)
231245

232-
ctx, cancel := context.WithCancel(context.Background())
246+
ctx, cancel := context.WithCancel(p.cancelCtx)
233247
p.trackJob(j.ID.String(), cancel)
234248

235249
execErr := p.executor.Execute(ctx, j)
@@ -282,7 +296,7 @@ func (p *Pool) sendHeartbeats() {
282296
p.logger.Warn("heartbeat: invalid job id", log.String("job_id", jobIDStr))
283297
continue
284298
}
285-
if err := p.store.HeartbeatJob(context.Background(), parsedID, p.workerID); err != nil {
299+
if err := p.store.HeartbeatJob(p.cancelCtx, parsedID, p.workerID); err != nil {
286300
p.logger.Warn("heartbeat failed",
287301
log.String("job_id", jobIDStr),
288302
log.String("error", err.Error()),
@@ -309,7 +323,7 @@ func (p *Pool) reaperLoop() {
309323
}
310324

311325
func (p *Pool) reapStaleJobs() {
312-
stale, err := p.store.ReapStaleJobs(context.Background(), p.staleJobThreshold)
326+
stale, err := p.store.ReapStaleJobs(p.cancelCtx, p.staleJobThreshold)
313327
if err != nil {
314328
p.logger.Error("reap stale jobs error", log.String("error", err.Error()))
315329
return
@@ -322,7 +336,7 @@ func (p *Pool) reapStaleJobs() {
322336
j.HeartbeatAt = nil
323337
j.StartedAt = nil
324338

325-
if updateErr := p.store.UpdateJob(context.Background(), j); updateErr != nil {
339+
if updateErr := p.store.UpdateJob(p.cancelCtx, j); updateErr != nil {
326340
p.logger.Error("reap: failed to reset stale job",
327341
log.String("job_id", j.ID.String()),
328342
log.String("error", updateErr.Error()),
@@ -341,6 +355,7 @@ func (p *Pool) sleep() {
341355
select {
342356
case <-time.After(p.pollInterval):
343357
case <-p.stopCh:
358+
case <-p.cancelCtx.Done():
344359
}
345360
}
346361

0 commit comments

Comments
 (0)