Skip to content

Commit 052bcf6

Browse files
RebeccaMahanyRebeccaMahany
authored
Handle kv.sqlite database contention/incomplete transactions (#2790)
1 parent 146aa5b commit 052bcf6

4 files changed

Lines changed: 100 additions & 28 deletions

File tree

‎ee/agent/storage/sqlite/keyvalue_store_sqlite.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ type storeName int
2525
const (
2626
StartupSettingsStore storeName = iota
2727
WatchdogLogStore storeName = 1
28+
29+
busyTimeoutMs int = 10000 // 10 seconds
2830
)
2931

3032
var missingMigrationErrFormat = regexp.MustCompile(`no migration found for version \d+`)
@@ -69,7 +71,7 @@ func OpenRO(ctx context.Context, slogger *slog.Logger, rootDirectory string, nam
6971
return nil, fmt.Errorf("unsupported table %d", name)
7072
}
7173

72-
conn, err := sql.Open("sqlite", dbLocation(rootDirectory))
74+
conn, err := open(dbLocation(rootDirectory))
7375
if err != nil {
7476
return nil, fmt.Errorf("opening startup db in %s: %w", rootDirectory, err)
7577
}
@@ -125,7 +127,7 @@ func validatedDbConn(ctx context.Context, rootDirectory string) (*sql.DB, error)
125127
}
126128

127129
// Open and validate connection
128-
conn, err := sql.Open("sqlite", startupDbFilepath)
130+
conn, err := open(startupDbFilepath)
129131
if err != nil {
130132
return nil, fmt.Errorf("opening startup db in %s: %w", rootDirectory, err)
131133
}
@@ -148,7 +150,7 @@ func validateDb(ctx context.Context, dbFilepath string) error {
148150
return fmt.Errorf("determining if db exists: %w", err)
149151
}
150152

151-
conn, err := sql.Open("sqlite", dbFilepath)
153+
conn, err := open(dbFilepath)
152154
if err != nil {
153155
return fmt.Errorf("creating connection: %w", err)
154156
}
@@ -174,6 +176,18 @@ func dbLocation(rootDirectory string) string {
174176
return path.Join(filepath.ToSlash(rootDirectory), "kv.sqlite")
175177
}
176178

179+
// open standardizes the options included in the DSN when creating a connection.
180+
// Notably, it sets _busy_timeout so that we do not immediately get SQLITE_BUSY
181+
// when there's database contention.
182+
// See: https://sqlite.org/c3ref/busy_timeout.html
183+
func open(dbFilepath string) (*sql.DB, error) {
184+
conn, err := sql.Open("sqlite", fmt.Sprintf("file:%s?_busy_timeout=%d", dbFilepath, busyTimeoutMs))
185+
if err != nil {
186+
return nil, fmt.Errorf("opening db: %w", err)
187+
}
188+
return conn, nil
189+
}
190+
177191
// migrate makes sure that the database schema is correct.
178192
func (s *sqliteStore) migrate() error {
179193
d, err := iofs.New(migrations, "migrations")

‎ee/agent/storage/sqlite/keyvalue_store_sqlite_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package agentsqlite
22

33
import (
4+
"context"
45
"fmt"
56
"os"
7+
"strconv"
68
"testing"
9+
"time"
710

811
"github.com/golang-migrate/migrate/v4"
912
"github.com/golang-migrate/migrate/v4/database/sqlite"
@@ -12,6 +15,7 @@ import (
1215
"github.com/kolide/launcher/v2/pkg/log/multislogger"
1316
"github.com/stretchr/testify/require"
1417
"go.uber.org/goleak"
18+
"golang.org/x/sync/semaphore"
1519
)
1620

1721
func TestMain(m *testing.M) {
@@ -130,6 +134,56 @@ func TestGetSet(t *testing.T) {
130134
require.NoError(t, s.Close())
131135
}
132136

137+
func TestGetSet_Simultaneous(t *testing.T) {
138+
t.Parallel()
139+
140+
testRootDir := t.TempDir()
141+
142+
s, err := OpenRW(t.Context(), testRootDir, StartupSettingsStore)
143+
require.NoError(t, err, "creating test store")
144+
145+
flagKey := []byte(keys.UpdateChannel.String())
146+
147+
queryCount := int64(100)
148+
queryWait := semaphore.NewWeighted(queryCount)
149+
queryErrors := make(chan error, queryCount)
150+
for i := range queryCount {
151+
queryWait.Acquire(t.Context(), 1)
152+
go func(idx int) {
153+
defer queryWait.Release(1)
154+
// Alternate between get and set operations so we test both reads and writes
155+
if idx%2 == 0 {
156+
flagVal := []byte(strconv.Itoa(idx))
157+
if err := s.Set(flagKey, flagVal); err != nil {
158+
queryErrors <- err
159+
}
160+
return
161+
}
162+
if _, err := s.Get(flagKey); err != nil {
163+
queryErrors <- err
164+
}
165+
}(int(i))
166+
}
167+
168+
// Wait up to 5x busyTimeout for all queries to complete
169+
queryCtx, queryCancel := context.WithTimeout(t.Context(), time.Duration(busyTimeoutMs*1000000*5))
170+
defer queryCancel()
171+
172+
// Confirm queries completed
173+
require.NoError(t, queryWait.Acquire(queryCtx, queryCount), "timed out waiting for simultaneous queries to complete")
174+
175+
// Confirm no errors for queries
176+
select {
177+
case exampleErr := <-queryErrors:
178+
t.Fatalf("got %d errors out of %d operations; example: %v", len(queryErrors), queryCount, exampleErr)
179+
default:
180+
// No errors
181+
}
182+
close(queryErrors)
183+
184+
require.NoError(t, s.Close())
185+
}
186+
133187
func TestUpdate(t *testing.T) {
134188
t.Parallel()
135189

‎go.mod‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ require (
3131
golang.org/x/image v0.43.0
3232
golang.org/x/net v0.55.0 // indirect
3333
golang.org/x/sync v0.21.0
34-
golang.org/x/sys v0.46.0
34+
golang.org/x/sys v0.47.0
3535
golang.org/x/text v0.39.0
3636
golang.org/x/time v0.8.0
3737
google.golang.org/grpc v1.82.1
@@ -66,7 +66,7 @@ require (
6666
go.opentelemetry.io/otel/sdk/metric v1.43.0
6767
go.uber.org/goleak v1.3.0
6868
go.yaml.in/yaml/v4 v4.0.0-rc.4
69-
modernc.org/sqlite v1.37.1
69+
modernc.org/sqlite v1.56.0
7070
)
7171

7272
require (
@@ -105,7 +105,7 @@ require (
105105
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
106106
github.com/magiconair/properties v1.8.9 // indirect
107107
github.com/mattn/go-colorable v0.1.14 // indirect
108-
github.com/mattn/go-isatty v0.0.20 // indirect
108+
github.com/mattn/go-isatty v0.0.24 // indirect
109109
github.com/mattn/go-runewidth v0.0.14 // indirect
110110
github.com/mholt/archives v0.1.5 // indirect
111111
github.com/mikelolasagasti/xz v1.0.1 // indirect
@@ -115,7 +115,7 @@ require (
115115
github.com/mitchellh/reflectwalk v1.0.2 // indirect
116116
github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect
117117
github.com/muesli/termenv v0.15.1 // indirect
118-
github.com/ncruces/go-strftime v0.1.9 // indirect
118+
github.com/ncruces/go-strftime v1.0.0 // indirect
119119
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
120120
github.com/opencontainers/go-digest v1.0.0 // indirect
121121
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
@@ -149,7 +149,7 @@ require (
149149
golang.org/x/term v0.44.0 // indirect
150150
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
151151
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
152-
modernc.org/libc v1.65.7 // indirect
152+
modernc.org/libc v1.74.4 // indirect
153153
modernc.org/mathutil v1.7.1 // indirect
154154
modernc.org/memory v1.11.0 // indirect
155155
)

‎go.sum‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
197197
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
198198
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
199199
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
200-
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
201-
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
200+
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
201+
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
202202
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
203203
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
204204
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -290,8 +290,8 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg
290290
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
291291
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
292292
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
293-
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
294-
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
293+
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
294+
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
295295
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
296296
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
297297
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -317,8 +317,8 @@ github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ
317317
github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
318318
github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
319319
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
320-
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
321-
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
320+
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
321+
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
322322
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
323323
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
324324
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
@@ -637,8 +637,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
637637
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
638638
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
639639
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
640-
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
641-
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
640+
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
641+
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
642642
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
643643
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
644644
golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -773,26 +773,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
773773
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
774774
howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
775775
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
776-
modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s=
777-
modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
778-
modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
779-
modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
780-
modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8=
781-
modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
776+
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
777+
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
778+
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
779+
modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
780+
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
781+
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
782782
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
783783
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
784-
modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00=
785-
modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU=
784+
modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
785+
modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
786+
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
787+
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
788+
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
789+
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
786790
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
787791
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
788792
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
789793
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
790-
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
791-
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
794+
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
795+
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
792796
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
793797
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
794-
modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs=
795-
modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g=
798+
modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
799+
modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
796800
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
797801
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
798802
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

0 commit comments

Comments
 (0)