From ea20fb5e4ba26633436ac2b785fd6b49278f3675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 4 Jun 2025 19:17:45 +0200 Subject: [PATCH 01/26] mod: add sqldb/v2 dependency to `litd` In upcoming commits, we will introduce support for sqldb/v2 in Lightning Terminal, in order to support code migrations. This is needed, so that we can ensure that the kvdb to sql migration is run exactly after an exact sql migration version, and not after all sql migrations have been executed. The reason why that's important is that sql table definition may change with future sql migrations, while the kvdb to sql migration will remain the same and always expect the same table definition. Therefore, if the kvdb to sql migration is run after all sql migrations, it may fail due to a table definition mismatch. This commit adds the sqldb/v2 dependency to `litd`, so that we can implement it's usage and execution of the kvdb to sql migration correctly in the upcoming commits. NOTE: currently the sqldb/v2 dependency hasn't been shipped in any lnd release, so this dependency currently a forked version of sqldb/v2. --- go.mod | 21 ++++++++++++--------- go.sum | 38 ++++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 06094c72b..710a60783 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/lightningnetwork/lnd/fn/v2 v2.0.8 github.com/lightningnetwork/lnd/kvdb v1.4.16 github.com/lightningnetwork/lnd/sqldb v1.0.11-0.20250623231731-45c15646c68b + github.com/lightningnetwork/lnd/sqldb/v2 v2.0.0-00010101000000-000000000000 github.com/lightningnetwork/lnd/tlv v1.3.2 github.com/lightningnetwork/lnd/tor v1.1.6 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f @@ -50,10 +51,10 @@ require ( github.com/stretchr/testify v1.10.0 github.com/urfave/cli v1.22.14 go.etcd.io/bbolt v1.3.11 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.37.0 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 golang.org/x/net v0.38.0 - golang.org/x/sync v0.12.0 + golang.org/x/sync v0.13.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/macaroon-bakery.v2 v2.3.0 @@ -96,8 +97,8 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/decred/dcrd/lru v1.1.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/docker/cli v28.0.1+incompatible // indirect - github.com/docker/docker v28.0.1+incompatible // indirect + github.com/docker/cli v28.1.1+incompatible // indirect + github.com/docker/docker v28.1.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -127,12 +128,12 @@ require ( github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/pgx/v5 v5.7.4 // indirect github.com/jackc/puddle v1.3.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jackpal/gateway v1.0.5 // indirect github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad // indirect github.com/jedib0t/go-pretty/v6 v6.2.7 // indirect @@ -213,8 +214,8 @@ require ( go.uber.org/zap v1.23.0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/term v0.31.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect @@ -251,3 +252,5 @@ replace github.com/golang-migrate/migrate/v4 => github.com/lightninglabs/migrate // tapd wants v0.19.0-12, but loop can't handle that yet. So we'll just use the // previous version for now. replace github.com/lightninglabs/lndclient => github.com/lightninglabs/lndclient v0.19.0-11 + +replace github.com/lightningnetwork/lnd/sqldb/v2 => github.com/ViktorTigerstrom/lnd/sqldb/v2 v2.0.0-20250826145625-90193015d32e diff --git a/go.sum b/go.sum index d710ed254..1452f7386 100644 --- a/go.sum +++ b/go.sum @@ -616,6 +616,8 @@ github.com/NebulousLabs/go-upnp v0.0.0-20180202185039-29b680b06c82/go.mod h1:Gbu github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ViktorTigerstrom/lnd/sqldb/v2 v2.0.0-20250826145625-90193015d32e h1:K/0okw/xOj2A9azbly/21ZcaFX+1W/uw8zcw2gjXa0s= +github.com/ViktorTigerstrom/lnd/sqldb/v2 v2.0.0-20250826145625-90193015d32e/go.mod h1:cOlSdR9DuBvP3srdUOERecUrJ/mEL+VZmvBfQuN240E= github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344 h1:cDVUiFo+npB0ZASqnw4q90ylaVAbnYyx0JYqK4YcGok= github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344/go.mod h1:9pIqrY6SXNL8vjRQE5Hd/OL5GyK/9MrGUWs87z/eFfk= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= @@ -762,10 +764,10 @@ github.com/dhui/dktest v0.4.5 h1:uUfYBIVREmj/Rw6MvgmqNAYzTiKOHJak+enB5Di73MM= github.com/dhui/dktest v0.4.5/go.mod h1:tmcyeHDKagvlDrz7gDKq4UAJOLIfVZYkfD5OnHDwcCo= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v28.0.1+incompatible h1:g0h5NQNda3/CxIsaZfH4Tyf6vpxFth7PYl3hgCPOKzs= -github.com/docker/cli v28.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v28.0.1+incompatible h1:FCHjSRdXhNRFjlHMTv4jUNlIBbTeRjrWfeFuJp7jpo0= -github.com/docker/docker v28.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v28.1.1+incompatible h1:eyUemzeI45DY7eDPuwUcmDyDj1pM98oD5MdSpiItp8k= +github.com/docker/cli v28.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v28.1.1+incompatible h1:49M11BFLsVO1gxY9UX9p/zwkE/rswggs8AdFmXQw51I= +github.com/docker/docker v28.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -1039,8 +1041,8 @@ github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -1053,15 +1055,15 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg= +github.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad h1:heFfj7z0pGsNCekUlsFhO2jstxO4b5iQ665LjwM5mDc= @@ -1500,8 +1502,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1682,8 +1684,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1796,8 +1798,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1814,8 +1816,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 47ab4f8e70746c9ca7c7d6a1ddb9d1924bb63f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 13:33:48 +0200 Subject: [PATCH 02/26] db: add `LitdMigrationStream` A core component of `sqldb/v2` useage, is that the package allows and requires that the callsite defines a `sqldb.MigrationStream` that will be run during the initialization of the `sqldb/v2 database instance. The `sqldb.MigrationStream` defines the exact sql migrations to run, as well as additional code migrations will be run after each individual migration version. This commit introduces the core definition of the migration stream for `litd`, and will be further exteded in upcoming commits that will introduce kvdb to sql code migration. Note that as of this commit, as no part of the `litd` codebase uses the `sqldb/v2` database instances, this migration stream is not yet used. --- db/sql_migrations.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 db/sql_migrations.go diff --git a/db/sql_migrations.go b/db/sql_migrations.go new file mode 100644 index 000000000..57d283aa3 --- /dev/null +++ b/db/sql_migrations.go @@ -0,0 +1,30 @@ +package db + +import ( + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/lightningnetwork/lnd/sqldb/v2" +) + +var ( + LitdMigrationStream = sqldb.MigrationStream{ + MigrateTableName: pgx.DefaultMigrationsTable, + SQLFileDirectory: "sqlc/migrations", + Schemas: sqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // database. This is used to implement downgrade protection for + // the daemon. + // + // NOTE: This MUST be updated when a new migration is added. + LatestMigrationVersion: LatestMigrationVersion, + + MakePostMigrationChecks: func( + db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, + error) { + + return make(map[uint]migrate.PostStepCallback), nil + }, + } + LitdMigrationStreams = []sqldb.MigrationStream{LitdMigrationStream} +) From 4bffeadca873d4bdfb38c390fe58aea9e4d32d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 13:15:03 +0200 Subject: [PATCH 03/26] db+sqlc: use sqldb/v2 `BackendType` definition The `sqldb/v2` package now provides a definition for the `BackendType` type. As useage of `sqldb/v2` requires useage of that type, we update `litd` to use the `BackendType` definition from `sqldb/v2`, instead of it's own definition. --- db/interfaces.go | 7 ++++--- db/sqlc/db_custom.go | 27 +++++++-------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/db/interfaces.go b/db/interfaces.go index ba64520b4..bb39df9ea 100644 --- a/db/interfaces.go +++ b/db/interfaces.go @@ -8,6 +8,7 @@ import ( "time" "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightningnetwork/lnd/sqldb/v2" ) var ( @@ -56,7 +57,7 @@ type BatchedTx[Q any] interface { txBody func(Q) error) error // Backend returns the type of the database backend used. - Backend() sqlc.BackendType + Backend() sqldb.BackendType } // Tx represents a database transaction that can be committed or rolled back. @@ -277,7 +278,7 @@ func (t *TransactionExecutor[Q]) ExecTx(ctx context.Context, } // Backend returns the type of the database backend used. -func (t *TransactionExecutor[Q]) Backend() sqlc.BackendType { +func (t *TransactionExecutor[Q]) Backend() sqldb.BackendType { return t.BatchedQuerier.Backend() } @@ -301,7 +302,7 @@ func (s *BaseDB) BeginTx(ctx context.Context, opts TxOptions) (*sql.Tx, error) { } // Backend returns the type of the database backend used. -func (s *BaseDB) Backend() sqlc.BackendType { +func (s *BaseDB) Backend() sqldb.BackendType { return s.Queries.Backend() } diff --git a/db/sqlc/db_custom.go b/db/sqlc/db_custom.go index f4bf7f611..43892d050 100644 --- a/db/sqlc/db_custom.go +++ b/db/sqlc/db_custom.go @@ -2,21 +2,8 @@ package sqlc import ( "context" -) - -// BackendType is an enum that represents the type of database backend we're -// using. -type BackendType uint8 - -const ( - // BackendTypeUnknown indicates we're using an unknown backend. - BackendTypeUnknown BackendType = iota - - // BackendTypeSqlite indicates we're using a SQLite backend. - BackendTypeSqlite - // BackendTypePostgres indicates we're using a Postgres backend. - BackendTypePostgres + "github.com/lightningnetwork/lnd/sqldb/v2" ) // wrappedTX is a wrapper around a DBTX that also stores the database backend @@ -24,16 +11,16 @@ const ( type wrappedTX struct { DBTX - backendType BackendType + backendType sqldb.BackendType } // Backend returns the type of database backend we're using. -func (q *Queries) Backend() BackendType { +func (q *Queries) Backend() sqldb.BackendType { wtx, ok := q.db.(*wrappedTX) if !ok { // Shouldn't happen unless a new database backend type is added // but not initialized correctly. - return BackendTypeUnknown + return sqldb.BackendTypeUnknown } return wtx.backendType @@ -41,12 +28,12 @@ func (q *Queries) Backend() BackendType { // NewSqlite creates a new Queries instance for a SQLite database. func NewSqlite(db DBTX) *Queries { - return &Queries{db: &wrappedTX{db, BackendTypeSqlite}} + return &Queries{db: &wrappedTX{db, sqldb.BackendTypeSqlite}} } // NewPostgres creates a new Queries instance for a Postgres database. func NewPostgres(db DBTX) *Queries { - return &Queries{db: &wrappedTX{db, BackendTypePostgres}} + return &Queries{db: &wrappedTX{db, sqldb.BackendTypePostgres}} } // CustomQueries defines a set of custom queries that we define in addition @@ -62,5 +49,5 @@ type CustomQueries interface { arg ListActionsParams) ([]Action, error) // Backend returns the type of the database backend used. - Backend() BackendType + Backend() sqldb.BackendType } From c7e3b9d0f3850a74aa1df96d2afcc07df55f2e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 13:18:55 +0200 Subject: [PATCH 04/26] sqlc: introduce `NewForType` helper method The upcoming implementation of `sqldb/v2` will extensively create a `Queries` object on the fly. To make more intuitive how to create the queries object for specific database types, we introduce a `NewForType` helper method. This also mimics how `tapd` creates the `Queries` object, and in order not let `litd` have it's own definition of how `Queries` object are created on the fly, the upcoming `sqldb/v2` usage will utilize this helper method. --- db/sqlc/db_custom.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db/sqlc/db_custom.go b/db/sqlc/db_custom.go index 43892d050..9037d9614 100644 --- a/db/sqlc/db_custom.go +++ b/db/sqlc/db_custom.go @@ -36,6 +36,11 @@ func NewPostgres(db DBTX) *Queries { return &Queries{db: &wrappedTX{db, sqldb.BackendTypePostgres}} } +// NewForType creates a new Queries instance for the given database type. +func NewForType(db DBTX, typ sqldb.BackendType) *Queries { + return &Queries{db: &wrappedTX{db, typ}} +} + // CustomQueries defines a set of custom queries that we define in addition // to the ones generated by sqlc. type CustomQueries interface { From d8bbce886f10062f5cf509eceb614e4c764f41e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 13:26:50 +0200 Subject: [PATCH 05/26] db: add sqldb/v2 `PostgresStore` creation helper As we will change the `accounts`, `session` & `firewalldb` packages to use the `sqldb/v2` package, we need to make those packages use the `sqldb/v2` `PostgresStore` when setting up their test postgres databases, instead of `litd`'s own `PostgresStore` version. In order to enable that functionality, we add a new helper function that creates a `PostgresStore` using the `sqldb/v2` package, in addition to helper function that creates a `PostgresStore` using the `litd` version. Once we have shifted all of `litd`'s code to use the `sqldb/v2` definition, we will remove the `litd` version. --- db/postgres.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/db/postgres.go b/db/postgres.go index 16e41dc09..e329fac15 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -9,6 +9,7 @@ import ( postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" ) @@ -164,8 +165,26 @@ func (s *PostgresStore) ExecuteMigrations(target MigrationTarget, ) } +// NewTestPostgresV2DB is a helper function that creates a Postgres database for +// testing, using the sqldb v2 package's definition of the PostgresStore. +func NewTestPostgresV2DB(t *testing.T) *sqldb.PostgresStore { + t.Helper() + + t.Logf("Creating new Postgres DB for testing") + + sqlFixture := sqldb.NewTestPgFixture(t, DefaultPostgresFixtureLifetime) + t.Cleanup(func() { + sqlFixture.TearDown(t) + }) + + return sqldb.NewTestPostgresDB(t, sqlFixture, LitdMigrationStreams) +} + // NewTestPostgresDB is a helper function that creates a Postgres database for -// testing. +// testing, using the litd db package's definition of the PostgresStore. +// +// TODO(viktor): remove this once the sqldb v2 package is implemented in +// all of litd's packages. func NewTestPostgresDB(t *testing.T) *PostgresStore { t.Helper() From b00d77d26c6affdb4dbcac7e4ac42b736a4b425c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Tue, 22 Jul 2025 12:20:04 +0200 Subject: [PATCH 06/26] accounts: use `sqldb/v2` in accounts package Update the accounts package to use the `sqldb/v2` package instead of the older version. --- accounts/sql_migration_test.go | 17 +++----- accounts/store_sql.go | 66 +++++++++++++++++++---------- accounts/test_postgres.go | 4 +- accounts/test_sql.go | 11 +++-- accounts/test_sqlite.go | 12 ++++-- config_dev.go | 77 ++++++++++++++++++++++++++++++---- 6 files changed, 137 insertions(+), 50 deletions(-) diff --git a/accounts/sql_migration_test.go b/accounts/sql_migration_test.go index b84485e1d..ef9ff82f2 100644 --- a/accounts/sql_migration_test.go +++ b/accounts/sql_migration_test.go @@ -2,18 +2,17 @@ package accounts import ( "context" - "database/sql" "fmt" "testing" "time" - "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" - "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" "pgregory.net/rapid" @@ -36,7 +35,7 @@ func TestAccountStoreMigration(t *testing.T) { } makeSQLDB := func(t *testing.T) (*SQLStore, - *db.TransactionExecutor[SQLQueries]) { + *SQLQueriesExecutor[SQLQueries]) { testDBStore := NewTestDB(t, clock) @@ -45,13 +44,9 @@ func TestAccountStoreMigration(t *testing.T) { baseDB := store.BaseDB - genericExecutor := db.NewTransactionExecutor( - baseDB, func(tx *sql.Tx) SQLQueries { - return baseDB.WithTx(tx) - }, - ) + queries := sqlc.NewForType(baseDB, baseDB.BackendType) - return store, genericExecutor + return store, NewSQLQueriesExecutor(baseDB, queries) } assertMigrationResults := func(t *testing.T, sqlStore *SQLStore, @@ -343,7 +338,7 @@ func TestAccountStoreMigration(t *testing.T) { return MigrateAccountStoreToSQL( ctx, kvStore.db, tx, ) - }, + }, sqldb.NoOpReset, ) require.NoError(t, err) diff --git a/accounts/store_sql.go b/accounts/store_sql.go index 830f16587..c7e8ab070 100644 --- a/accounts/store_sql.go +++ b/accounts/store_sql.go @@ -16,6 +16,7 @@ import ( "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/sqldb/v2" ) const ( @@ -33,6 +34,8 @@ const ( // //nolint:lll type SQLQueries interface { + sqldb.BaseQuerier + AddAccountInvoice(ctx context.Context, arg sqlc.AddAccountInvoiceParams) error DeleteAccount(ctx context.Context, id int64) error DeleteAccountPayment(ctx context.Context, arg sqlc.DeleteAccountPaymentParams) error @@ -53,12 +56,13 @@ type SQLQueries interface { GetAccountInvoice(ctx context.Context, arg sqlc.GetAccountInvoiceParams) (sqlc.AccountInvoice, error) } -// BatchedSQLQueries is a version of the SQLQueries that's capable -// of batched database operations. +// BatchedSQLQueries combines the SQLQueries interface with the BatchedTx +// interface, allowing for multiple queries to be executed in single SQL +// transaction. type BatchedSQLQueries interface { SQLQueries - db.BatchedTx[SQLQueries] + sqldb.BatchedTx[SQLQueries] } // SQLStore represents a storage backend. @@ -68,19 +72,37 @@ type SQLStore struct { db BatchedSQLQueries // BaseDB represents the underlying database connection. - *db.BaseDB + *sqldb.BaseDB clock clock.Clock } -// NewSQLStore creates a new SQLStore instance given an open BatchedSQLQueries -// storage backend. -func NewSQLStore(sqlDB *db.BaseDB, clock clock.Clock) *SQLStore { - executor := db.NewTransactionExecutor( - sqlDB, func(tx *sql.Tx) SQLQueries { - return sqlDB.WithTx(tx) +type SQLQueriesExecutor[T sqldb.BaseQuerier] struct { + *sqldb.TransactionExecutor[T] + + SQLQueries +} + +func NewSQLQueriesExecutor(baseDB *sqldb.BaseDB, + queries *sqlc.Queries) *SQLQueriesExecutor[SQLQueries] { + + executor := sqldb.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) SQLQueries { + return queries.WithTx(tx) }, ) + return &SQLQueriesExecutor[SQLQueries]{ + TransactionExecutor: executor, + SQLQueries: queries, + } +} + +// NewSQLStore creates a new SQLStore instance given an open BatchedSQLQueries +// storage backend. +func NewSQLStore(sqlDB *sqldb.BaseDB, queries *sqlc.Queries, + clock clock.Clock) *SQLStore { + + executor := NewSQLQueriesExecutor(sqlDB, queries) return &SQLStore{ db: executor, @@ -157,7 +179,7 @@ func (s *SQLStore) NewAccount(ctx context.Context, balance lnwire.MilliSatoshi, } return nil - }) + }, sqldb.NoOpReset) if err != nil { return nil, err } @@ -299,7 +321,7 @@ func (s *SQLStore) AddAccountInvoice(ctx context.Context, alias AccountID, } return s.markAccountUpdated(ctx, db, acctID) - }) + }, sqldb.NoOpReset) } func getAccountIDByAlias(ctx context.Context, db SQLQueries, alias AccountID) ( @@ -377,7 +399,7 @@ func (s *SQLStore) UpdateAccountBalanceAndExpiry(ctx context.Context, } return s.markAccountUpdated(ctx, db, id) - }) + }, sqldb.NoOpReset) } // CreditAccount increases the balance of the account with the given alias by @@ -412,7 +434,7 @@ func (s *SQLStore) CreditAccount(ctx context.Context, alias AccountID, } return s.markAccountUpdated(ctx, db, id) - }) + }, sqldb.NoOpReset) } // DebitAccount decreases the balance of the account with the given alias by the @@ -453,7 +475,7 @@ func (s *SQLStore) DebitAccount(ctx context.Context, alias AccountID, } return s.markAccountUpdated(ctx, db, id) - }) + }, sqldb.NoOpReset) } // Account retrieves an account from the SQL store and un-marshals it. If the @@ -475,7 +497,7 @@ func (s *SQLStore) Account(ctx context.Context, alias AccountID) ( account, err = getAndMarshalAccount(ctx, db, id) return err - }) + }, sqldb.NoOpReset) return account, err } @@ -507,7 +529,7 @@ func (s *SQLStore) Accounts(ctx context.Context) ([]*OffChainBalanceAccount, } return nil - }) + }, sqldb.NoOpReset) return accounts, err } @@ -524,7 +546,7 @@ func (s *SQLStore) RemoveAccount(ctx context.Context, alias AccountID) error { } return db.DeleteAccount(ctx, id) - }) + }, sqldb.NoOpReset) } // UpsertAccountPayment updates or inserts a payment entry for the given @@ -634,7 +656,7 @@ func (s *SQLStore) UpsertAccountPayment(ctx context.Context, alias AccountID, } return s.markAccountUpdated(ctx, db, id) - }) + }, sqldb.NoOpReset) } // DeleteAccountPayment removes a payment entry from the account with the given @@ -677,7 +699,7 @@ func (s *SQLStore) DeleteAccountPayment(ctx context.Context, alias AccountID, } return s.markAccountUpdated(ctx, db, id) - }) + }, sqldb.NoOpReset) } // LastIndexes returns the last invoice add and settle index or @@ -704,7 +726,7 @@ func (s *SQLStore) LastIndexes(ctx context.Context) (uint64, uint64, error) { } return err - }) + }, sqldb.NoOpReset) return uint64(addIndex), uint64(settleIndex), err } @@ -729,7 +751,7 @@ func (s *SQLStore) StoreLastIndexes(ctx context.Context, addIndex, Name: settleIndexName, Value: int64(settleIndex), }) - }) + }, sqldb.NoOpReset) } // Close closes the underlying store. diff --git a/accounts/test_postgres.go b/accounts/test_postgres.go index 16665030d..a88ed3a84 100644 --- a/accounts/test_postgres.go +++ b/accounts/test_postgres.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,5 +24,5 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } diff --git a/accounts/test_sql.go b/accounts/test_sql.go index 3c1ee7f16..ca2f43d6f 100644 --- a/accounts/test_sql.go +++ b/accounts/test_sql.go @@ -5,15 +5,20 @@ package accounts import ( "testing" - "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" ) // createStore is a helper function that creates a new SQLStore and ensure that // it is closed when during the test cleanup. -func createStore(t *testing.T, sqlDB *db.BaseDB, clock clock.Clock) *SQLStore { - store := NewSQLStore(sqlDB, clock) +func createStore(t *testing.T, sqlDB *sqldb.BaseDB, + clock clock.Clock) *SQLStore { + + queries := sqlc.NewForType(sqlDB, sqlDB.BackendType) + + store := NewSQLStore(sqlDB, queries, clock) t.Cleanup(func() { require.NoError(t, store.Close()) }) diff --git a/accounts/test_sqlite.go b/accounts/test_sqlite.go index 9d899b3e2..a31f990a6 100644 --- a/accounts/test_sqlite.go +++ b/accounts/test_sqlite.go @@ -8,6 +8,7 @@ import ( "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // ErrDBClosed is an error that is returned when a database operation is @@ -16,7 +17,10 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestSqliteDB(t).BaseDB, clock) + return createStore( + t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + clock, + ) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,7 +28,7 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore( - t, db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, - ) + tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + + return createStore(t, tDb.BaseDB, clock) } diff --git a/config_dev.go b/config_dev.go index 90b8b290f..42b9d7611 100644 --- a/config_dev.go +++ b/config_dev.go @@ -8,9 +8,11 @@ import ( "github.com/lightninglabs/lightning-terminal/accounts" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightninglabs/lightning-terminal/firewalldb" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" ) const ( @@ -101,14 +103,41 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { return stores, err } - sqlStore, err := db.NewSqliteStore(cfg.Sqlite) + // Until we have fully added support for sqldb/v2 in all of our + // stores, we need to use the db packages definition of the + // SQLite store for the packages that still haven't added + // support for sqldb/v2. This is only temporary and will be + // removed once all stores have been updated to use sqldb/v2. + legacySqlStore, err := db.NewSqliteStore(cfg.Sqlite) + + sqlStore, err := sqldb.NewSqliteStore(&sqldb.SqliteConfig{ + SkipMigrations: cfg.Sqlite.SkipMigrations, + SkipMigrationDbBackup: cfg.Sqlite.SkipMigrationDbBackup, + }, cfg.Sqlite.DatabaseFileName) if err != nil { return stores, err } - acctStore := accounts.NewSQLStore(sqlStore.BaseDB, clock) - sessStore := session.NewSQLStore(sqlStore.BaseDB, clock) - firewallStore := firewalldb.NewSQLDB(sqlStore.BaseDB, clock) + if !cfg.Sqlite.SkipMigrations { + err = sqldb.ApplyAllMigrations( + sqlStore, db.LitdMigrationStreams, + ) + if err != nil { + return stores, fmt.Errorf("error applying "+ + "migrations to SQLite store: %w", err, + ) + } + } + + queries := sqlc.NewForType(sqlStore, sqlStore.BackendType) + + acctStore := accounts.NewSQLStore( + sqlStore.BaseDB, queries, clock, + ) + sessStore := session.NewSQLStore(legacySqlStore.BaseDB, clock) + firewallStore := firewalldb.NewSQLDB( + legacySqlStore.BaseDB, clock, + ) stores.accounts = acctStore stores.sessions = sessStore @@ -116,14 +145,46 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { stores.closeFns["sqlite"] = sqlStore.BaseDB.Close case DatabaseBackendPostgres: - sqlStore, err := db.NewPostgresStore(cfg.Postgres) + // Until we have fully added support for sqldb/v2 in all of our + // stores, we need to use the db packages definition of the + // Postgres store for the packages that still haven't added + // support for sqldb/v2. This is only temporary and will be + // removed once all stores have been updated to use sqldb/v2. + legacySqlStore, err := db.NewPostgresStore(cfg.Postgres) + + sqlStore, err := sqldb.NewPostgresStore(&sqldb.PostgresConfig{ + Dsn: cfg.Postgres.DSN(false), + MaxOpenConnections: cfg.Postgres.MaxOpenConnections, + MaxIdleConnections: cfg.Postgres.MaxIdleConnections, + ConnMaxLifetime: cfg.Postgres.ConnMaxLifetime, + ConnMaxIdleTime: cfg.Postgres.ConnMaxIdleTime, + RequireSSL: cfg.Postgres.RequireSSL, + SkipMigrations: cfg.Postgres.SkipMigrations, + }) if err != nil { return stores, err } - acctStore := accounts.NewSQLStore(sqlStore.BaseDB, clock) - sessStore := session.NewSQLStore(sqlStore.BaseDB, clock) - firewallStore := firewalldb.NewSQLDB(sqlStore.BaseDB, clock) + if !cfg.Postgres.SkipMigrations { + err = sqldb.ApplyAllMigrations( + sqlStore, db.LitdMigrationStreams, + ) + if err != nil { + return stores, fmt.Errorf("error applying "+ + "migrations to Postgres store: %w", err, + ) + } + } + + queries := sqlc.NewForType(sqlStore, sqlStore.BackendType) + + acctStore := accounts.NewSQLStore( + sqlStore.BaseDB, queries, clock, + ) + sessStore := session.NewSQLStore(legacySqlStore.BaseDB, clock) + firewallStore := firewalldb.NewSQLDB( + legacySqlStore.BaseDB, clock, + ) stores.accounts = acctStore stores.sessions = sessStore From d869a916a885c57ca073b6d99b38f2c2b9d6dffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Tue, 22 Jul 2025 12:21:06 +0200 Subject: [PATCH 07/26] multi: use `sqldb/v2` in session package Update the session package to use the `sqldb/v2` package instead of the older version. --- config_dev.go | 8 +++- firewalldb/sql_migration_test.go | 8 +++- session/sql_migration_test.go | 17 +++------ session/sql_store.go | 64 +++++++++++++++++++++----------- session/test_postgres.go | 4 +- session/test_sql.go | 11 ++++-- session/test_sqlite.go | 12 ++++-- 7 files changed, 79 insertions(+), 45 deletions(-) diff --git a/config_dev.go b/config_dev.go index 42b9d7611..1ecf509df 100644 --- a/config_dev.go +++ b/config_dev.go @@ -134,7 +134,9 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { acctStore := accounts.NewSQLStore( sqlStore.BaseDB, queries, clock, ) - sessStore := session.NewSQLStore(legacySqlStore.BaseDB, clock) + sessStore := session.NewSQLStore( + sqlStore.BaseDB, queries, clock, + ) firewallStore := firewalldb.NewSQLDB( legacySqlStore.BaseDB, clock, ) @@ -181,7 +183,9 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { acctStore := accounts.NewSQLStore( sqlStore.BaseDB, queries, clock, ) - sessStore := session.NewSQLStore(legacySqlStore.BaseDB, clock) + sessStore := session.NewSQLStore( + sqlStore.BaseDB, queries, clock, + ) firewallStore := firewalldb.NewSQLDB( legacySqlStore.BaseDB, clock, ) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 4d71f5cc0..9e0a0eb13 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -797,6 +797,8 @@ func createPrivacyPairs(t *testing.T, ctx context.Context, sessSQLStore, ok := sessionStore.(*session.SQLStore) require.True(t, ok) + queries := sqlc.NewForType(sessSQLStore, sessSQLStore.BackendType) + for i := range numSessions { sess, err := sessionStore.NewSession( ctx, fmt.Sprintf("session-%d", i), @@ -806,7 +808,7 @@ func createPrivacyPairs(t *testing.T, ctx context.Context, require.NoError(t, err) groupID := sess.GroupID - sqlGroupID, err := sessSQLStore.GetSessionIDByAlias( + sqlGroupID, err := queries.GetSessionIDByAlias( ctx, groupID[:], ) require.NoError(t, err) @@ -850,6 +852,8 @@ func randomPrivacyPairs(t *testing.T, ctx context.Context, sessSQLStore, ok := sessionStore.(*session.SQLStore) require.True(t, ok) + queries := sqlc.NewForType(sessSQLStore, sessSQLStore.BackendType) + for i := range numSessions { sess, err := sessionStore.NewSession( ctx, fmt.Sprintf("session-%d", i), @@ -859,7 +863,7 @@ func randomPrivacyPairs(t *testing.T, ctx context.Context, require.NoError(t, err) groupID := sess.GroupID - sqlGroupID, err := sessSQLStore.GetSessionIDByAlias( + sqlGroupID, err := queries.GetSessionIDByAlias( ctx, groupID[:], ) require.NoError(t, err) diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go index 3c3b018e7..8fc416f6e 100644 --- a/session/sql_migration_test.go +++ b/session/sql_migration_test.go @@ -2,17 +2,16 @@ package session import ( "context" - "database/sql" "fmt" "testing" "time" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/macaroons" - "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" "golang.org/x/exp/rand" @@ -38,7 +37,7 @@ func TestSessionsStoreMigration(t *testing.T) { } makeSQLDB := func(t *testing.T, acctStore accounts.Store) (*SQLStore, - *db.TransactionExecutor[SQLQueries]) { + *SQLQueriesExecutor[SQLQueries]) { // Create a sql store with a linked account store. testDBStore := NewTestDBWithAccounts(t, clock, acctStore) @@ -48,13 +47,9 @@ func TestSessionsStoreMigration(t *testing.T) { baseDB := store.BaseDB - genericExecutor := db.NewTransactionExecutor( - baseDB, func(tx *sql.Tx) SQLQueries { - return baseDB.WithTx(tx) - }, - ) + queries := sqlc.NewForType(baseDB, baseDB.BackendType) - return store, genericExecutor + return store, NewSQLQueriesExecutor(baseDB, queries) } // assertMigrationResults asserts that the sql store contains the @@ -375,7 +370,7 @@ func TestSessionsStoreMigration(t *testing.T) { return MigrateSessionStoreToSQL( ctx, kvStore.DB, tx, ) - }, + }, sqldb.NoOpReset, ) require.NoError(t, err) diff --git a/session/sql_store.go b/session/sql_store.go index b1d366fe7..26662a574 100644 --- a/session/sql_store.go +++ b/session/sql_store.go @@ -14,6 +14,7 @@ import ( "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/fn" + "github.com/lightningnetwork/lnd/sqldb/v2" "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon.v2" ) @@ -21,6 +22,8 @@ import ( // SQLQueries is a subset of the sqlc.Queries interface that can be used to // interact with session related tables. type SQLQueries interface { + sqldb.BaseQuerier + GetAliasBySessionID(ctx context.Context, id int64) ([]byte, error) GetSessionByID(ctx context.Context, id int64) (sqlc.Session, error) GetSessionsInGroup(ctx context.Context, groupID sql.NullInt64) ([]sqlc.Session, error) @@ -51,12 +54,13 @@ type SQLQueries interface { var _ Store = (*SQLStore)(nil) -// BatchedSQLQueries is a version of the SQLQueries that's capable of batched -// database operations. +// BatchedSQLQueries combines the SQLQueries interface with the BatchedTx +// interface, allowing for multiple queries to be executed in single SQL +// transaction. type BatchedSQLQueries interface { SQLQueries - db.BatchedTx[SQLQueries] + sqldb.BatchedTx[SQLQueries] } // SQLStore represents a storage backend. @@ -66,19 +70,37 @@ type SQLStore struct { db BatchedSQLQueries // BaseDB represents the underlying database connection. - *db.BaseDB + *sqldb.BaseDB clock clock.Clock } -// NewSQLStore creates a new SQLStore instance given an open BatchedSQLQueries -// storage backend. -func NewSQLStore(sqlDB *db.BaseDB, clock clock.Clock) *SQLStore { - executor := db.NewTransactionExecutor( - sqlDB, func(tx *sql.Tx) SQLQueries { - return sqlDB.WithTx(tx) +type SQLQueriesExecutor[T sqldb.BaseQuerier] struct { + *sqldb.TransactionExecutor[T] + + SQLQueries +} + +func NewSQLQueriesExecutor(baseDB *sqldb.BaseDB, + queries *sqlc.Queries) *SQLQueriesExecutor[SQLQueries] { + + executor := sqldb.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) SQLQueries { + return queries.WithTx(tx) }, ) + return &SQLQueriesExecutor[SQLQueries]{ + TransactionExecutor: executor, + SQLQueries: queries, + } +} + +// NewSQLStore creates a new SQLStore instance given an open BatchedSQLQueries +// storage backend. +func NewSQLStore(sqlDB *sqldb.BaseDB, queries *sqlc.Queries, + clock clock.Clock) *SQLStore { + + executor := NewSQLQueriesExecutor(sqlDB, queries) return &SQLStore{ db: executor, @@ -281,7 +303,7 @@ func (s *SQLStore) NewSession(ctx context.Context, label string, typ Type, } return nil - }) + }, sqldb.NoOpReset) if err != nil { mappedSQLErr := db.MapSQLError(err) var uniqueConstraintErr *db.ErrSqlUniqueConstraintViolation @@ -325,7 +347,7 @@ func (s *SQLStore) ListSessionsByType(ctx context.Context, t Type) ([]*Session, } return nil - }) + }, sqldb.NoOpReset) return sessions, err } @@ -358,7 +380,7 @@ func (s *SQLStore) ListSessionsByState(ctx context.Context, state State) ( } return nil - }) + }, sqldb.NoOpReset) return sessions, err } @@ -417,7 +439,7 @@ func (s *SQLStore) ShiftState(ctx context.Context, alias ID, dest State) error { State: int16(dest), }, ) - }) + }, sqldb.NoOpReset) } // DeleteReservedSessions deletes all sessions that are in the StateReserved @@ -428,7 +450,7 @@ func (s *SQLStore) DeleteReservedSessions(ctx context.Context) error { var writeTxOpts db.QueriesTxOptions return s.db.ExecTx(ctx, &writeTxOpts, func(db SQLQueries) error { return db.DeleteSessionsWithState(ctx, int16(StateReserved)) - }) + }, sqldb.NoOpReset) } // GetSessionByLocalPub fetches the session with the given local pub key. @@ -458,7 +480,7 @@ func (s *SQLStore) GetSessionByLocalPub(ctx context.Context, } return nil - }) + }, sqldb.NoOpReset) if err != nil { return nil, err } @@ -491,7 +513,7 @@ func (s *SQLStore) ListAllSessions(ctx context.Context) ([]*Session, error) { } return nil - }) + }, sqldb.NoOpReset) return sessions, err } @@ -521,7 +543,7 @@ func (s *SQLStore) UpdateSessionRemotePubKey(ctx context.Context, alias ID, RemotePublicKey: remoteKey, }, ) - }) + }, sqldb.NoOpReset) } // getSqlUnusedAliasAndKeyPair can be used to generate a new, unused, local @@ -576,7 +598,7 @@ func (s *SQLStore) GetSession(ctx context.Context, alias ID) (*Session, error) { } return nil - }) + }, sqldb.NoOpReset) return sess, err } @@ -617,7 +639,7 @@ func (s *SQLStore) GetGroupID(ctx context.Context, sessionID ID) (ID, error) { legacyGroupID, err = IDFromBytes(legacyGroupIDB) return err - }) + }, sqldb.NoOpReset) if err != nil { return ID{}, err } @@ -666,7 +688,7 @@ func (s *SQLStore) GetSessionIDs(ctx context.Context, legacyGroupID ID) ([]ID, } return nil - }) + }, sqldb.NoOpReset) if err != nil { return nil, err } diff --git a/session/test_postgres.go b/session/test_postgres.go index cb5aa061d..d8d5d2111 100644 --- a/session/test_postgres.go +++ b/session/test_postgres.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,5 +24,5 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } diff --git a/session/test_sql.go b/session/test_sql.go index a83186069..5623c8207 100644 --- a/session/test_sql.go +++ b/session/test_sql.go @@ -6,8 +6,9 @@ import ( "testing" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" ) @@ -22,8 +23,12 @@ func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, // createStore is a helper function that creates a new SQLStore and ensure that // it is closed when during the test cleanup. -func createStore(t *testing.T, sqlDB *db.BaseDB, clock clock.Clock) *SQLStore { - store := NewSQLStore(sqlDB, clock) +func createStore(t *testing.T, sqlDB *sqldb.BaseDB, + clock clock.Clock) *SQLStore { + + queries := sqlc.NewForType(sqlDB, sqlDB.BackendType) + + store := NewSQLStore(sqlDB, queries, clock) t.Cleanup(func() { require.NoError(t, store.Close()) }) diff --git a/session/test_sqlite.go b/session/test_sqlite.go index 0ceb0e046..84d946ce2 100644 --- a/session/test_sqlite.go +++ b/session/test_sqlite.go @@ -8,6 +8,7 @@ import ( "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // ErrDBClosed is an error that is returned when a database operation is @@ -16,7 +17,10 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestSqliteDB(t).BaseDB, clock) + return createStore( + t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + clock, + ) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,7 +28,7 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore( - t, db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, - ) + tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + + return createStore(t, tDb.BaseDB, clock) } From 431c3d6b699e140a5bc19da3250cbc3f5e378995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 01:58:22 +0200 Subject: [PATCH 08/26] firewalldb: add `ListAllKVStoresRecords` to queries Previous commits had forgotten to add the `ListAllKVStoresRecords` query to the `firewalldb.SQLKVStoreQueries` interface. As that is required to make the query useable when defining the `sqldb/v2` `TransactionExecutor` for the `firewalldb` package, this commit adds it to the interface. --- firewalldb/kvstores_sql.go | 1 + 1 file changed, 1 insertion(+) diff --git a/firewalldb/kvstores_sql.go b/firewalldb/kvstores_sql.go index 248892130..852e1ebaf 100644 --- a/firewalldb/kvstores_sql.go +++ b/firewalldb/kvstores_sql.go @@ -30,6 +30,7 @@ type SQLKVStoreQueries interface { UpdateGlobalKVStoreRecord(ctx context.Context, arg sqlc.UpdateGlobalKVStoreRecordParams) error UpdateGroupKVStoreRecord(ctx context.Context, arg sqlc.UpdateGroupKVStoreRecordParams) error InsertKVStoreRecord(ctx context.Context, arg sqlc.InsertKVStoreRecordParams) error + ListAllKVStoresRecords(ctx context.Context) ([]sqlc.Kvstore, error) DeleteAllTempKVStores(ctx context.Context) error GetOrInsertFeatureID(ctx context.Context, name string) (int64, error) GetOrInsertRuleID(ctx context.Context, name string) (int64, error) From 8d6b9d0832f63e07fd34d4f91a8c87ed63deaaca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 12:17:17 +0200 Subject: [PATCH 09/26] firewalldb: rename sqlStore to store in mig test rename `sqlStore` to `store` in the firewalldb sql migration test file, to make the name shorted. This is done in preparation for future commits which will lengthen the lines where `sqlStore` is used, which otherwise would make the lines exceed the 80 character limit. --- firewalldb/sql_migration_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 9e0a0eb13..2e142eedb 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -79,7 +79,7 @@ func TestFirewallDBMigration(t *testing.T) { // The assertKvStoreMigrationResults function will currently assert that // the migrated kv stores entries in the SQLDB match the original kv // stores entries in the BoltDB. - assertKvStoreMigrationResults := func(t *testing.T, sqlStore *SQLDB, + assertKvStoreMigrationResults := func(t *testing.T, store *SQLDB, kvEntries []*kvEntry) { var ( @@ -92,7 +92,7 @@ func TestFirewallDBMigration(t *testing.T) { getRuleID := func(ruleName string) int64 { ruleID, ok := ruleIDs[ruleName] if !ok { - ruleID, err = sqlStore.GetRuleID(ctx, ruleName) + ruleID, err = store.GetRuleID(ctx, ruleName) require.NoError(t, err) ruleIDs[ruleName] = ruleID @@ -104,7 +104,7 @@ func TestFirewallDBMigration(t *testing.T) { getGroupID := func(groupAlias []byte) int64 { groupID, ok := groupIDs[string(groupAlias)] if !ok { - groupID, err = sqlStore.GetSessionIDByAlias( + groupID, err = store.GetSessionIDByAlias( ctx, groupAlias, ) require.NoError(t, err) @@ -118,7 +118,7 @@ func TestFirewallDBMigration(t *testing.T) { getFeatureID := func(featureName string) int64 { featureID, ok := featureIDs[featureName] if !ok { - featureID, err = sqlStore.GetFeatureID( + featureID, err = store.GetFeatureID( ctx, featureName, ) require.NoError(t, err) @@ -132,7 +132,7 @@ func TestFirewallDBMigration(t *testing.T) { // First we extract all migrated kv entries from the SQLDB, // in order to be able to compare them to the original kv // entries, to ensure that the migration was successful. - sqlKvEntries, err := sqlStore.ListAllKVStoresRecords(ctx) + sqlKvEntries, err := store.ListAllKVStoresRecords(ctx) require.NoError(t, err) require.Equal(t, len(kvEntries), len(sqlKvEntries)) @@ -148,7 +148,7 @@ func TestFirewallDBMigration(t *testing.T) { ruleID := getRuleID(entry.ruleName) if entry.groupAlias.IsNone() { - sqlVal, err := sqlStore.GetGlobalKVStoreRecord( + sqlVal, err := store.GetGlobalKVStoreRecord( ctx, sqlc.GetGlobalKVStoreRecordParams{ Key: entry.key, @@ -166,7 +166,7 @@ func TestFirewallDBMigration(t *testing.T) { groupAlias := entry.groupAlias.UnwrapOrFail(t) groupID := getGroupID(groupAlias[:]) - v, err := sqlStore.GetGroupKVStoreRecord( + v, err := store.GetGroupKVStoreRecord( ctx, sqlc.GetGroupKVStoreRecordParams{ Key: entry.key, @@ -191,7 +191,7 @@ func TestFirewallDBMigration(t *testing.T) { entry.featureName.UnwrapOrFail(t), ) - sqlVal, err := sqlStore.GetFeatureKVStoreRecord( + sqlVal, err := store.GetFeatureKVStoreRecord( ctx, sqlc.GetFeatureKVStoreRecordParams{ Key: entry.key, From b5d44386b3b72951e893287849405acbaa2394b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Tue, 22 Jul 2025 12:20:33 +0200 Subject: [PATCH 10/26] firewalldb: use `sqldb/v2` in firewalldb package Update the firewalldb package to use the `sqldb/v2` package instead of the older version. --- config_dev.go | 18 ++---------- firewalldb/actions_sql.go | 8 +++--- firewalldb/kvstores_sql.go | 3 +- firewalldb/sql_migration_test.go | 38 ++++++++++++-------------- firewalldb/sql_store.go | 47 ++++++++++++++++++++++++-------- firewalldb/test_postgres.go | 4 +-- firewalldb/test_sql.go | 10 +++++-- firewalldb/test_sqlite.go | 16 +++++++---- 8 files changed, 81 insertions(+), 63 deletions(-) diff --git a/config_dev.go b/config_dev.go index 1ecf509df..cc8a5b319 100644 --- a/config_dev.go +++ b/config_dev.go @@ -103,13 +103,6 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { return stores, err } - // Until we have fully added support for sqldb/v2 in all of our - // stores, we need to use the db packages definition of the - // SQLite store for the packages that still haven't added - // support for sqldb/v2. This is only temporary and will be - // removed once all stores have been updated to use sqldb/v2. - legacySqlStore, err := db.NewSqliteStore(cfg.Sqlite) - sqlStore, err := sqldb.NewSqliteStore(&sqldb.SqliteConfig{ SkipMigrations: cfg.Sqlite.SkipMigrations, SkipMigrationDbBackup: cfg.Sqlite.SkipMigrationDbBackup, @@ -138,7 +131,7 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { sqlStore.BaseDB, queries, clock, ) firewallStore := firewalldb.NewSQLDB( - legacySqlStore.BaseDB, clock, + sqlStore.BaseDB, queries, clock, ) stores.accounts = acctStore @@ -147,13 +140,6 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { stores.closeFns["sqlite"] = sqlStore.BaseDB.Close case DatabaseBackendPostgres: - // Until we have fully added support for sqldb/v2 in all of our - // stores, we need to use the db packages definition of the - // Postgres store for the packages that still haven't added - // support for sqldb/v2. This is only temporary and will be - // removed once all stores have been updated to use sqldb/v2. - legacySqlStore, err := db.NewPostgresStore(cfg.Postgres) - sqlStore, err := sqldb.NewPostgresStore(&sqldb.PostgresConfig{ Dsn: cfg.Postgres.DSN(false), MaxOpenConnections: cfg.Postgres.MaxOpenConnections, @@ -187,7 +173,7 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { sqlStore.BaseDB, queries, clock, ) firewallStore := firewalldb.NewSQLDB( - legacySqlStore.BaseDB, clock, + sqlStore.BaseDB, queries, clock, ) stores.accounts = acctStore diff --git a/firewalldb/actions_sql.go b/firewalldb/actions_sql.go index 75c9d0a6d..4d5448313 100644 --- a/firewalldb/actions_sql.go +++ b/firewalldb/actions_sql.go @@ -12,7 +12,7 @@ import ( "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/fn" - "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // SQLAccountQueries is a subset of the sqlc.Queries interface that can be used @@ -167,7 +167,7 @@ func (s *SQLDB) AddAction(ctx context.Context, } return nil - }) + }, sqldb.NoOpReset) if err != nil { return nil, err } @@ -202,7 +202,7 @@ func (s *SQLDB) SetActionState(ctx context.Context, al ActionLocator, Valid: errReason != "", }, }) - }) + }, sqldb.NoOpReset) } // ListActions returns a list of Actions. The query IndexOffset and MaxNum @@ -350,7 +350,7 @@ func (s *SQLDB) ListActions(ctx context.Context, } return nil - }) + }, sqldb.NoOpReset) return actions, lastIndex, uint64(totalCount), err } diff --git a/firewalldb/kvstores_sql.go b/firewalldb/kvstores_sql.go index 852e1ebaf..0c1847706 100644 --- a/firewalldb/kvstores_sql.go +++ b/firewalldb/kvstores_sql.go @@ -11,6 +11,7 @@ import ( "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/fn" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // SQLKVStoreQueries is a subset of the sqlc.Queries interface that can be @@ -46,7 +47,7 @@ func (s *SQLDB) DeleteTempKVStores(ctx context.Context) error { return s.db.ExecTx(ctx, &writeTxOpts, func(tx SQLQueries) error { return tx.DeleteAllTempKVStores(ctx) - }) + }, sqldb.NoOpReset) } // GetKVStores constructs a new rules.KVStores in a namespace defined by the diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 2e142eedb..46fae50c9 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -10,12 +10,11 @@ import ( "time" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db" "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/fn" - "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" ) @@ -58,7 +57,7 @@ func TestFirewallDBMigration(t *testing.T) { } makeSQLDB := func(t *testing.T, sessionsStore session.Store) (*SQLDB, - *db.TransactionExecutor[SQLQueries]) { + *SQLQueriesExecutor[SQLQueries]) { testDBStore := NewTestDBWithSessions(t, sessionsStore, clock) @@ -67,13 +66,9 @@ func TestFirewallDBMigration(t *testing.T) { baseDB := store.BaseDB - genericExecutor := db.NewTransactionExecutor( - baseDB, func(tx *sql.Tx) SQLQueries { - return baseDB.WithTx(tx) - }, - ) + queries := sqlc.NewForType(baseDB, baseDB.BackendType) - return store, genericExecutor + return store, NewSQLQueriesExecutor(baseDB, queries) } // The assertKvStoreMigrationResults function will currently assert that @@ -92,7 +87,9 @@ func TestFirewallDBMigration(t *testing.T) { getRuleID := func(ruleName string) int64 { ruleID, ok := ruleIDs[ruleName] if !ok { - ruleID, err = store.GetRuleID(ctx, ruleName) + ruleID, err = store.db.GetRuleID( + ctx, ruleName, + ) require.NoError(t, err) ruleIDs[ruleName] = ruleID @@ -104,7 +101,7 @@ func TestFirewallDBMigration(t *testing.T) { getGroupID := func(groupAlias []byte) int64 { groupID, ok := groupIDs[string(groupAlias)] if !ok { - groupID, err = store.GetSessionIDByAlias( + groupID, err = store.db.GetSessionIDByAlias( ctx, groupAlias, ) require.NoError(t, err) @@ -118,7 +115,7 @@ func TestFirewallDBMigration(t *testing.T) { getFeatureID := func(featureName string) int64 { featureID, ok := featureIDs[featureName] if !ok { - featureID, err = store.GetFeatureID( + featureID, err = store.db.GetFeatureID( ctx, featureName, ) require.NoError(t, err) @@ -132,7 +129,7 @@ func TestFirewallDBMigration(t *testing.T) { // First we extract all migrated kv entries from the SQLDB, // in order to be able to compare them to the original kv // entries, to ensure that the migration was successful. - sqlKvEntries, err := store.ListAllKVStoresRecords(ctx) + sqlKvEntries, err := store.db.ListAllKVStoresRecords(ctx) require.NoError(t, err) require.Equal(t, len(kvEntries), len(sqlKvEntries)) @@ -148,7 +145,7 @@ func TestFirewallDBMigration(t *testing.T) { ruleID := getRuleID(entry.ruleName) if entry.groupAlias.IsNone() { - sqlVal, err := store.GetGlobalKVStoreRecord( + sqlVal, err := store.db.GetGlobalKVStoreRecord( ctx, sqlc.GetGlobalKVStoreRecordParams{ Key: entry.key, @@ -166,7 +163,7 @@ func TestFirewallDBMigration(t *testing.T) { groupAlias := entry.groupAlias.UnwrapOrFail(t) groupID := getGroupID(groupAlias[:]) - v, err := store.GetGroupKVStoreRecord( + v, err := store.db.GetGroupKVStoreRecord( ctx, sqlc.GetGroupKVStoreRecordParams{ Key: entry.key, @@ -191,7 +188,7 @@ func TestFirewallDBMigration(t *testing.T) { entry.featureName.UnwrapOrFail(t), ) - sqlVal, err := store.GetFeatureKVStoreRecord( + sqlVal, err := store.db.GetFeatureKVStoreRecord( ctx, sqlc.GetFeatureKVStoreRecordParams{ Key: entry.key, @@ -229,7 +226,7 @@ func TestFirewallDBMigration(t *testing.T) { // First assert that the SQLDB contains the expected privacy // pairs. for groupID, groupPairs := range privPairs { - storePairs, err := sqlStore.GetAllPrivacyPairs( + storePairs, err := sqlStore.db.GetAllPrivacyPairs( ctx, groupID, ) require.NoError(t, err) @@ -251,11 +248,12 @@ func TestFirewallDBMigration(t *testing.T) { // Then assert that SQLDB doesn't contain any other privacy // pairs than the expected ones. - sessions, err := sqlStore.ListSessions(ctx) + queries := sqlc.NewForType(sqlStore, sqlStore.BackendType) + sessions, err := queries.ListSessions(ctx) require.NoError(t, err) for _, dbSession := range sessions { - sessionPairs, err := sqlStore.GetAllPrivacyPairs( + sessionPairs, err := sqlStore.db.GetAllPrivacyPairs( ctx, dbSession.ID, ) if errors.Is(err, sql.ErrNoRows) { @@ -397,7 +395,7 @@ func TestFirewallDBMigration(t *testing.T) { return MigrateFirewallDBToSQL( ctx, firewallStore.DB, tx, ) - }, + }, sqldb.NoOpReset, ) require.NoError(t, err) diff --git a/firewalldb/sql_store.go b/firewalldb/sql_store.go index f17010f2c..1be887ace 100644 --- a/firewalldb/sql_store.go +++ b/firewalldb/sql_store.go @@ -5,7 +5,9 @@ import ( "database/sql" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // SQLSessionQueries is a subset of the sqlc.Queries interface that can be used @@ -18,17 +20,20 @@ type SQLSessionQueries interface { // SQLQueries is a subset of the sqlc.Queries interface that can be used to // interact with various firewalldb tables. type SQLQueries interface { + sqldb.BaseQuerier + SQLKVStoreQueries SQLPrivacyPairQueries SQLActionQueries } -// BatchedSQLQueries is a version of the SQLQueries that's capable of batched -// database operations. +// BatchedSQLQueries combines the SQLQueries interface with the BatchedTx +// interface, allowing for multiple queries to be executed in single SQL +// transaction. type BatchedSQLQueries interface { SQLQueries - db.BatchedTx[SQLQueries] + sqldb.BatchedTx[SQLQueries] } // SQLDB represents a storage backend. @@ -38,11 +43,31 @@ type SQLDB struct { db BatchedSQLQueries // BaseDB represents the underlying database connection. - *db.BaseDB + *sqldb.BaseDB clock clock.Clock } +type SQLQueriesExecutor[T sqldb.BaseQuerier] struct { + *sqldb.TransactionExecutor[T] + + SQLQueries +} + +func NewSQLQueriesExecutor(baseDB *sqldb.BaseDB, + queries *sqlc.Queries) *SQLQueriesExecutor[SQLQueries] { + + executor := sqldb.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) SQLQueries { + return queries.WithTx(tx) + }, + ) + return &SQLQueriesExecutor[SQLQueries]{ + TransactionExecutor: executor, + SQLQueries: queries, + } +} + // A compile-time assertion to ensure that SQLDB implements the RulesDB // interface. var _ RulesDB = (*SQLDB)(nil) @@ -53,12 +78,10 @@ var _ ActionDB = (*SQLDB)(nil) // NewSQLDB creates a new SQLStore instance given an open SQLQueries // storage backend. -func NewSQLDB(sqlDB *db.BaseDB, clock clock.Clock) *SQLDB { - executor := db.NewTransactionExecutor( - sqlDB, func(tx *sql.Tx) SQLQueries { - return sqlDB.WithTx(tx) - }, - ) +func NewSQLDB(sqlDB *sqldb.BaseDB, queries *sqlc.Queries, + clock clock.Clock) *SQLDB { + + executor := NewSQLQueriesExecutor(sqlDB, queries) return &SQLDB{ db: executor, @@ -88,7 +111,7 @@ func (e *sqlExecutor[T]) Update(ctx context.Context, var txOpts db.QueriesTxOptions return e.db.ExecTx(ctx, &txOpts, func(queries SQLQueries) error { return fn(ctx, e.wrapTx(queries)) - }) + }, sqldb.NoOpReset) } // View opens a database read transaction and executes the function f with the @@ -104,5 +127,5 @@ func (e *sqlExecutor[T]) View(ctx context.Context, return e.db.ExecTx(ctx, &txOpts, func(queries SQLQueries) error { return fn(ctx, e.wrapTx(queries)) - }) + }, sqldb.NoOpReset) } diff --git a/firewalldb/test_postgres.go b/firewalldb/test_postgres.go index 732b19b4a..3a54a81c4 100644 --- a/firewalldb/test_postgres.go +++ b/firewalldb/test_postgres.go @@ -11,11 +11,11 @@ import ( // NewTestDB is a helper function that creates an BBolt database for testing. func NewTestDB(t *testing.T, clock clock.Clock) FirewallDBs { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new BoltStore with a // connection to an existing BBolt database for testing. func NewTestDBFromPath(t *testing.T, _ string, clock clock.Clock) FirewallDBs { - return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) } diff --git a/firewalldb/test_sql.go b/firewalldb/test_sql.go index a412441f8..b7e3d9052 100644 --- a/firewalldb/test_sql.go +++ b/firewalldb/test_sql.go @@ -6,10 +6,12 @@ import ( "testing" "time" + "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" "github.com/stretchr/testify/require" ) @@ -55,8 +57,10 @@ func assertEqualActions(t *testing.T, expected, got *Action) { // createStore is a helper function that creates a new SQLDB and ensure that // it is closed when during the test cleanup. -func createStore(t *testing.T, sqlDB *db.BaseDB, clock clock.Clock) *SQLDB { - store := NewSQLDB(sqlDB, clock) +func createStore(t *testing.T, sqlDB *sqldb.BaseDB, clock clock.Clock) *SQLDB { + queries := sqlc.NewForType(sqlDB, sqlDB.BackendType) + + store := NewSQLDB(sqlDB, queries, clock) t.Cleanup(func() { require.NoError(t, store.Close()) }) diff --git a/firewalldb/test_sqlite.go b/firewalldb/test_sqlite.go index 49b956d7d..ab184b5a6 100644 --- a/firewalldb/test_sqlite.go +++ b/firewalldb/test_sqlite.go @@ -7,17 +7,23 @@ import ( "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" ) // NewTestDB is a helper function that creates an BBolt database for testing. func NewTestDB(t *testing.T, clock clock.Clock) FirewallDBs { - return createStore(t, db.NewTestSqliteDB(t).BaseDB, clock) + return createStore( + t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + clock, + ) } // NewTestDBFromPath is a helper function that creates a new BoltStore with a // connection to an existing BBolt database for testing. -func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) FirewallDBs { - return createStore( - t, db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, - ) +func NewTestDBFromPath(t *testing.T, dbPath string, + clock clock.Clock) FirewallDBs { + + tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + + return createStore(t, tDb.BaseDB, clock) } From 133ac68cbee4021e28c9499b0513ec7ccb76e605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 01:46:27 +0200 Subject: [PATCH 11/26] multi: remove unused db code As we've now switched over to using sqldb v2 for most of the db objects, we can remove a lot of deprecated code that's no longer used in the litd project. This commit removes that code. --- db/interfaces.go | 306 ------------------------------------------- db/migrations.go | 271 -------------------------------------- db/postgres.go | 154 ---------------------- db/sqlc/db_custom.go | 10 -- db/sqlite.go | 285 ---------------------------------------- 5 files changed, 1026 deletions(-) diff --git a/db/interfaces.go b/db/interfaces.go index bb39df9ea..d8fd37ad2 100644 --- a/db/interfaces.go +++ b/db/interfaces.go @@ -1,311 +1,5 @@ package db -import ( - "context" - "database/sql" - "math" - prand "math/rand" - "time" - - "github.com/lightninglabs/lightning-terminal/db/sqlc" - "github.com/lightningnetwork/lnd/sqldb/v2" -) - -var ( - // DefaultStoreTimeout is the default timeout used for any interaction - // with the storage/database. - DefaultStoreTimeout = time.Second * 10 -) - -const ( - // DefaultNumTxRetries is the default number of times we'll retry a - // transaction if it fails with an error that permits transaction - // repetition. - DefaultNumTxRetries = 10 - - // DefaultInitialRetryDelay is the default initial delay between - // retries. This will be used to generate a random delay between -50% - // and +50% of this value, so 20 to 60 milliseconds. The retry will be - // doubled after each attempt until we reach DefaultMaxRetryDelay. We - // start with a random value to avoid multiple goroutines that are - // created at the same time to effectively retry at the same time. - DefaultInitialRetryDelay = time.Millisecond * 40 - - // DefaultMaxRetryDelay is the default maximum delay between retries. - DefaultMaxRetryDelay = time.Second * 3 -) - -// TxOptions represents a set of options one can use to control what type of -// database transaction is created. Transaction can wither be read or write. -type TxOptions interface { - // ReadOnly returns true if the transaction should be read only. - ReadOnly() bool -} - -// BatchedTx is a generic interface that represents the ability to execute -// several operations to a given storage interface in a single atomic -// transaction. Typically, Q here will be some subset of the main sqlc.Querier -// interface allowing it to only depend on the routines it needs to implement -// any additional business logic. -type BatchedTx[Q any] interface { - // ExecTx will execute the passed txBody, operating upon generic - // parameter Q (usually a storage interface) in a single transaction. - // The set of TxOptions are passed in in order to allow the caller to - // specify if a transaction should be read-only and optionally what - // type of concurrency control should be used. - ExecTx(ctx context.Context, txOptions TxOptions, - txBody func(Q) error) error - - // Backend returns the type of the database backend used. - Backend() sqldb.BackendType -} - -// Tx represents a database transaction that can be committed or rolled back. -type Tx interface { - // Commit commits the database transaction, an error should be returned - // if the commit isn't possible. - Commit() error - - // Rollback rolls back an incomplete database transaction. - // Transactions that were able to be committed can still call this as a - // noop. - Rollback() error -} - -// QueryCreator is a generic function that's used to create a Querier, which is -// a type of interface that implements storage related methods from a database -// transaction. This will be used to instantiate an object callers can use to -// apply multiple modifications to an object interface in a single atomic -// transaction. -type QueryCreator[Q any] func(*sql.Tx) Q - -// BatchedQuerier is a generic interface that allows callers to create a new -// database transaction based on an abstract type that implements the TxOptions -// interface. -type BatchedQuerier interface { - // Querier is the underlying query source, this is in place so we can - // pass a BatchedQuerier implementation directly into objects that - // create a batched version of the normal methods they need. - sqlc.Querier - - // CustomQueries is the set of custom queries that we have manually - // defined in addition to the ones generated by sqlc. - sqlc.CustomQueries - - // BeginTx creates a new database transaction given the set of - // transaction options. - BeginTx(ctx context.Context, options TxOptions) (*sql.Tx, error) -} - -// txExecutorOptions is a struct that holds the options for the transaction -// executor. This can be used to do things like retry a transaction due to an -// error a certain amount of times. -type txExecutorOptions struct { - numRetries int - initialRetryDelay time.Duration - maxRetryDelay time.Duration -} - -// defaultTxExecutorOptions returns the default options for the transaction -// executor. -func defaultTxExecutorOptions() *txExecutorOptions { - return &txExecutorOptions{ - numRetries: DefaultNumTxRetries, - initialRetryDelay: DefaultInitialRetryDelay, - maxRetryDelay: DefaultMaxRetryDelay, - } -} - -// randRetryDelay returns a random retry delay between -50% and +50% -// of the configured delay that is doubled for each attempt and capped at a max -// value. -func (t *txExecutorOptions) randRetryDelay(attempt int) time.Duration { - halfDelay := t.initialRetryDelay / 2 - randDelay := prand.Int63n(int64(t.initialRetryDelay)) //nolint:gosec - - // 50% plus 0%-100% gives us the range of 50%-150%. - initialDelay := halfDelay + time.Duration(randDelay) - - // If this is the first attempt, we just return the initial delay. - if attempt == 0 { - return initialDelay - } - - // For each subsequent delay, we double the initial delay. This still - // gives us a somewhat random delay, but it still increases with each - // attempt. If we double something n times, that's the same as - // multiplying the value with 2^n. We limit the power to 32 to avoid - // overflows. - factor := time.Duration(math.Pow(2, math.Min(float64(attempt), 32))) - actualDelay := initialDelay * factor - - // Cap the delay at the maximum configured value. - if actualDelay > t.maxRetryDelay { - return t.maxRetryDelay - } - - return actualDelay -} - -// TxExecutorOption is a functional option that allows us to pass in optional -// argument when creating the executor. -type TxExecutorOption func(*txExecutorOptions) - -// WithTxRetries is a functional option that allows us to specify the number of -// times a transaction should be retried if it fails with a repeatable error. -func WithTxRetries(numRetries int) TxExecutorOption { - return func(o *txExecutorOptions) { - o.numRetries = numRetries - } -} - -// WithTxRetryDelay is a functional option that allows us to specify the delay -// to wait before a transaction is retried. -func WithTxRetryDelay(delay time.Duration) TxExecutorOption { - return func(o *txExecutorOptions) { - o.initialRetryDelay = delay - } -} - -// TransactionExecutor is a generic struct that abstracts away from the type of -// query a type needs to run under a database transaction, and also the set of -// options for that transaction. The QueryCreator is used to create a query -// given a database transaction created by the BatchedQuerier. -type TransactionExecutor[Query any] struct { - BatchedQuerier - - createQuery QueryCreator[Query] - - opts *txExecutorOptions -} - -// NewTransactionExecutor creates a new instance of a TransactionExecutor given -// a Querier query object and a concrete type for the type of transactions the -// Querier understands. -func NewTransactionExecutor[Querier any](db BatchedQuerier, - createQuery QueryCreator[Querier], - opts ...TxExecutorOption) *TransactionExecutor[Querier] { - - txOpts := defaultTxExecutorOptions() - for _, optFunc := range opts { - optFunc(txOpts) - } - - return &TransactionExecutor[Querier]{ - BatchedQuerier: db, - createQuery: createQuery, - opts: txOpts, - } -} - -// ExecTx is a wrapper for txBody to abstract the creation and commit of a db -// transaction. The db transaction is embedded in a `*Queries` that txBody -// needs to use when executing each one of the queries that need to be applied -// atomically. This can be used by other storage interfaces to parameterize the -// type of query and options run, in order to have access to batched operations -// related to a storage object. -func (t *TransactionExecutor[Q]) ExecTx(ctx context.Context, - txOptions TxOptions, txBody func(Q) error) error { - - waitBeforeRetry := func(attemptNumber int) { - retryDelay := t.opts.randRetryDelay(attemptNumber) - - log.Tracef("Retrying transaction due to tx serialization or "+ - "deadlock error, attempt_number=%v, delay=%v", - attemptNumber, retryDelay) - - // Before we try again, we'll wait with a random backoff based - // on the retry delay. - time.Sleep(retryDelay) - } - - for i := 0; i < t.opts.numRetries; i++ { - // Create the db transaction. - tx, err := t.BatchedQuerier.BeginTx(ctx, txOptions) - if err != nil { - dbErr := MapSQLError(err) - if IsSerializationOrDeadlockError(dbErr) { - // Nothing to roll back here, since we didn't - // even get a transaction yet. - waitBeforeRetry(i) - continue - } - - return dbErr - } - - // Rollback is safe to call even if the tx is already closed, - // so if the tx commits successfully, this is a no-op. - defer func() { - _ = tx.Rollback() - }() - - if err := txBody(t.createQuery(tx)); err != nil { - dbErr := MapSQLError(err) - if IsSerializationOrDeadlockError(dbErr) { - // Roll back the transaction, then pop back up - // to try once again. - _ = tx.Rollback() - - waitBeforeRetry(i) - continue - } - - return dbErr - } - - // Commit transaction. - if err = tx.Commit(); err != nil { - dbErr := MapSQLError(err) - if IsSerializationOrDeadlockError(dbErr) { - // Roll back the transaction, then pop back up - // to try once again. - _ = tx.Rollback() - - waitBeforeRetry(i) - continue - } - - return dbErr - } - - return nil - } - - // If we get to this point, then we weren't able to successfully commit - // a tx given the max number of retries. - return ErrRetriesExceeded -} - -// Backend returns the type of the database backend used. -func (t *TransactionExecutor[Q]) Backend() sqldb.BackendType { - return t.BatchedQuerier.Backend() -} - -// BaseDB is the base database struct that each implementation can embed to -// gain some common functionality. -type BaseDB struct { - *sql.DB - - *sqlc.Queries -} - -// BeginTx wraps the normal sql specific BeginTx method with the TxOptions -// interface. This interface is then mapped to the concrete sql tx options -// struct. -func (s *BaseDB) BeginTx(ctx context.Context, opts TxOptions) (*sql.Tx, error) { - sqlOptions := sql.TxOptions{ - ReadOnly: opts.ReadOnly(), - Isolation: sql.LevelSerializable, - } - return s.DB.BeginTx(ctx, &sqlOptions) -} - -// Backend returns the type of the database backend used. -func (s *BaseDB) Backend() sqldb.BackendType { - return s.Queries.Backend() -} - // QueriesTxOptions defines the set of db txn options the SQLQueries // understands. type QueriesTxOptions struct { diff --git a/db/migrations.go b/db/migrations.go index 79d63587e..204bf40f6 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -1,21 +1,5 @@ package db -import ( - "bytes" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "strings" - - "github.com/btcsuite/btclog/v2" - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - "github.com/golang-migrate/migrate/v4/source/httpfs" - "github.com/lightninglabs/taproot-assets/fn" -) - const ( // LatestMigrationVersion is the latest migration version of the // database. This is used to implement downgrade protection for the @@ -24,258 +8,3 @@ const ( // NOTE: This MUST be updated when a new migration is added. LatestMigrationVersion = 5 ) - -// MigrationTarget is a functional option that can be passed to applyMigrations -// to specify a target version to migrate to. `currentDbVersion` is the current -// (migration) version of the database, or None if unknown. -// `maxMigrationVersion` is the maximum migration version known to the driver, -// or None if unknown. -type MigrationTarget func(mig *migrate.Migrate, - currentDbVersion int, maxMigrationVersion uint) error - -var ( - // TargetLatest is a MigrationTarget that migrates to the latest - // version available. - TargetLatest = func(mig *migrate.Migrate, _ int, _ uint) error { - return mig.Up() - } - - // TargetVersion is a MigrationTarget that migrates to the given - // version. - TargetVersion = func(version uint) MigrationTarget { - return func(mig *migrate.Migrate, _ int, _ uint) error { - return mig.Migrate(version) - } - } -) - -var ( - // ErrMigrationDowngrade is returned when a database downgrade is - // detected. - ErrMigrationDowngrade = errors.New("database downgrade detected") -) - -// migrationOption is a functional option that can be passed to migrate related -// methods to modify their behavior. -type migrateOptions struct { - latestVersion fn.Option[uint] -} - -// defaultMigrateOptions returns a new migrateOptions instance with default -// settings. -func defaultMigrateOptions() *migrateOptions { - return &migrateOptions{} -} - -// MigrateOpt is a functional option that can be passed to migrate related -// methods to modify behavior. -type MigrateOpt func(*migrateOptions) - -// WithLatestVersion allows callers to override the default latest version -// setting. -func WithLatestVersion(version uint) MigrateOpt { - return func(o *migrateOptions) { - o.latestVersion = fn.Some(version) - } -} - -// migrationLogger is a logger that wraps the passed btclog.Logger so it can be -// used to log migrations. -type migrationLogger struct { - log btclog.Logger -} - -// Printf is like fmt.Printf. We map this to the target logger based on the -// current log level. -func (m *migrationLogger) Printf(format string, v ...interface{}) { - // Trim trailing newlines from the format. - format = strings.TrimRight(format, "\n") - - switch m.log.Level() { - case btclog.LevelTrace: - m.log.Tracef(format, v...) - case btclog.LevelDebug: - m.log.Debugf(format, v...) - case btclog.LevelInfo: - m.log.Infof(format, v...) - case btclog.LevelWarn: - m.log.Warnf(format, v...) - case btclog.LevelError: - m.log.Errorf(format, v...) - case btclog.LevelCritical: - m.log.Criticalf(format, v...) - case btclog.LevelOff: - } -} - -// Verbose should return true when verbose logging output is wanted -func (m *migrationLogger) Verbose() bool { - return m.log.Level() <= btclog.LevelDebug -} - -// applyMigrations executes database migration files found in the given file -// system under the given path, using the passed database driver and database -// name, up to or down to the given target version. -func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string, - targetVersion MigrationTarget, opts *migrateOptions) error { - - // With the migrate instance open, we'll create a new migration source - // using the embedded file system stored in sqlSchemas. The library - // we're using can't handle a raw file system interface, so we wrap it - // in this intermediate layer. - migrateFileServer, err := httpfs.New(http.FS(fs), path) - if err != nil { - return err - } - - // Finally, we'll run the migration with our driver above based on the - // open DB, and also the migration source stored in the file system - // above. - sqlMigrate, err := migrate.NewWithInstance( - "migrations", migrateFileServer, dbName, driver, - ) - if err != nil { - return err - } - - migrationVersion, _, _ := sqlMigrate.Version() - - // As the down migrations may end up *dropping* data, we want to - // prevent that without explicit accounting. - latestVersion := opts.latestVersion.UnwrapOr(LatestMigrationVersion) - if migrationVersion > latestVersion { - return fmt.Errorf("%w: database version is newer than the "+ - "latest migration version, preventing downgrade: "+ - "db_version=%v, latest_migration_version=%v", - ErrMigrationDowngrade, migrationVersion, latestVersion) - } - - // Report the current version of the database before the migration. - currentDbVersion, _, err := driver.Version() - if err != nil { - return fmt.Errorf("unable to get current db version: %w", err) - } - log.Infof("Attempting to apply migration(s) "+ - "(current_db_version=%v, latest_migration_version=%v)", - currentDbVersion, latestVersion) - - // Apply our local logger to the migration instance. - sqlMigrate.Log = &migrationLogger{log} - - // Execute the migration based on the target given. - err = targetVersion(sqlMigrate, currentDbVersion, latestVersion) - if err != nil && !errors.Is(err, migrate.ErrNoChange) { - return err - } - - // Report the current version of the database after the migration. - currentDbVersion, _, err = driver.Version() - if err != nil { - return fmt.Errorf("unable to get current db version: %w", err) - } - log.Infof("Database version after migration: %v", currentDbVersion) - - return nil -} - -// replacerFS is an implementation of a fs.FS virtual file system that wraps an -// existing file system but does a search-and-replace operation on each file -// when it is opened. -type replacerFS struct { - parentFS fs.FS - replaces map[string]string -} - -// A compile-time assertion to make sure replacerFS implements the fs.FS -// interface. -var _ fs.FS = (*replacerFS)(nil) - -// newReplacerFS creates a new replacer file system, wrapping the given parent -// virtual file system. Each file within the file system is undergoing a -// search-and-replace operation when it is opened, using the given map where the -// key denotes the search term and the value the term to replace each occurrence -// with. -func newReplacerFS(parent fs.FS, replaces map[string]string) *replacerFS { - return &replacerFS{ - parentFS: parent, - replaces: replaces, - } -} - -// Open opens a file in the virtual file system. -// -// NOTE: This is part of the fs.FS interface. -func (t *replacerFS) Open(name string) (fs.File, error) { - f, err := t.parentFS.Open(name) - if err != nil { - return nil, err - } - - stat, err := f.Stat() - if err != nil { - return nil, err - } - - if stat.IsDir() { - return f, err - } - - return newReplacerFile(f, t.replaces) -} - -type replacerFile struct { - parentFile fs.File - buf bytes.Buffer -} - -// A compile-time assertion to make sure replacerFile implements the fs.File -// interface. -var _ fs.File = (*replacerFile)(nil) - -func newReplacerFile(parent fs.File, replaces map[string]string) (*replacerFile, - error) { - - content, err := io.ReadAll(parent) - if err != nil { - return nil, err - } - - contentStr := string(content) - for from, to := range replaces { - contentStr = strings.ReplaceAll(contentStr, from, to) - } - - var buf bytes.Buffer - _, err = buf.WriteString(contentStr) - if err != nil { - return nil, err - } - - return &replacerFile{ - parentFile: parent, - buf: buf, - }, nil -} - -// Stat returns statistics/info about the file. -// -// NOTE: This is part of the fs.File interface. -func (t *replacerFile) Stat() (fs.FileInfo, error) { - return t.parentFile.Stat() -} - -// Read reads as many bytes as possible from the file into the given slice. -// -// NOTE: This is part of the fs.File interface. -func (t *replacerFile) Read(bytes []byte) (int, error) { - return t.buf.Read(bytes) -} - -// Close closes the underlying file. -// -// NOTE: This is part of the fs.File interface. -func (t *replacerFile) Close() error { - // We already fully read and then closed the file when creating this - // instance, so there's nothing to do for us here. - return nil -} diff --git a/db/postgres.go b/db/postgres.go index e329fac15..a418f58e3 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -1,27 +1,16 @@ package db import ( - "database/sql" "fmt" "testing" "time" - postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" - "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightningnetwork/lnd/sqldb/v2" - "github.com/stretchr/testify/require" ) const ( dsnTemplate = "postgres://%v:%v@%v:%d/%v?sslmode=%v" - - // defaultMaxIdleConns is the number of permitted idle connections. - defaultMaxIdleConns = 6 - - // defaultConnMaxIdleTime is the amount of time a connection can be - // idle before it is closed. - defaultConnMaxIdleTime = 5 * time.Minute ) var ( @@ -31,16 +20,6 @@ var ( // fully executed yet. So this time needs to be chosen correctly to be // longer than the longest expected individual test run time. DefaultPostgresFixtureLifetime = 60 * time.Minute - - // postgresSchemaReplacements is a map of schema strings that need to be - // replaced for postgres. This is needed because we write the schemas - // to work with sqlite primarily, and postgres has some differences. - postgresSchemaReplacements = map[string]string{ - "BLOB": "BYTEA", - "INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY", - "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", - "UNHEX": "DECODE", - } ) // PostgresConfig holds the postgres database configuration. @@ -77,94 +56,6 @@ func (s *PostgresConfig) DSN(hidePassword bool) string { s.DBName, sslMode) } -// PostgresStore is a database store implementation that uses a Postgres -// backend. -type PostgresStore struct { - cfg *PostgresConfig - - *BaseDB -} - -// NewPostgresStore creates a new store that is backed by a Postgres database -// backend. -func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { - log.Infof("Using SQL database '%s'", cfg.DSN(true)) - - rawDb, err := sql.Open("pgx", cfg.DSN(false)) - if err != nil { - return nil, err - } - - maxConns := defaultMaxConns - if cfg.MaxOpenConnections > 0 { - maxConns = cfg.MaxOpenConnections - } - - maxIdleConns := defaultMaxIdleConns - if cfg.MaxIdleConnections > 0 { - maxIdleConns = cfg.MaxIdleConnections - } - - connMaxLifetime := defaultConnMaxLifetime - if cfg.ConnMaxLifetime > 0 { - connMaxLifetime = cfg.ConnMaxLifetime - } - - connMaxIdleTime := defaultConnMaxIdleTime - if cfg.ConnMaxIdleTime > 0 { - connMaxIdleTime = cfg.ConnMaxIdleTime - } - - rawDb.SetMaxOpenConns(maxConns) - rawDb.SetMaxIdleConns(maxIdleConns) - rawDb.SetConnMaxLifetime(connMaxLifetime) - rawDb.SetConnMaxIdleTime(connMaxIdleTime) - - queries := sqlc.NewPostgres(rawDb) - s := &PostgresStore{ - cfg: cfg, - BaseDB: &BaseDB{ - DB: rawDb, - Queries: queries, - }, - } - - // Now that the database is open, populate the database with our set of - // schemas based on our embedded in-memory file system. - if !cfg.SkipMigrations { - if err := s.ExecuteMigrations(TargetLatest); err != nil { - return nil, fmt.Errorf("error executing migrations: "+ - "%w", err) - } - } - - return s, nil -} - -// ExecuteMigrations runs migrations for the Postgres database, depending on the -// target given, either all migrations or up to a given version. -func (s *PostgresStore) ExecuteMigrations(target MigrationTarget, - optFuncs ...MigrateOpt) error { - - opts := defaultMigrateOptions() - for _, optFunc := range optFuncs { - optFunc(opts) - } - - driver, err := postgres_migrate.WithInstance( - s.DB, &postgres_migrate.Config{}, - ) - if err != nil { - return fmt.Errorf("error creating postgres migration: %w", err) - } - - postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements) - return applyMigrations( - postgresFS, driver, "sqlc/migrations", s.cfg.DBName, target, - opts, - ) -} - // NewTestPostgresV2DB is a helper function that creates a Postgres database for // testing, using the sqldb v2 package's definition of the PostgresStore. func NewTestPostgresV2DB(t *testing.T) *sqldb.PostgresStore { @@ -179,48 +70,3 @@ func NewTestPostgresV2DB(t *testing.T) *sqldb.PostgresStore { return sqldb.NewTestPostgresDB(t, sqlFixture, LitdMigrationStreams) } - -// NewTestPostgresDB is a helper function that creates a Postgres database for -// testing, using the litd db package's definition of the PostgresStore. -// -// TODO(viktor): remove this once the sqldb v2 package is implemented in -// all of litd's packages. -func NewTestPostgresDB(t *testing.T) *PostgresStore { - t.Helper() - - t.Logf("Creating new Postgres DB for testing") - - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) - store, err := NewPostgresStore(sqlFixture.GetConfig()) - require.NoError(t, err) - - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - - return store -} - -// NewTestPostgresDBWithVersion is a helper function that creates a Postgres -// database for testing and migrates it to the given version. -func NewTestPostgresDBWithVersion(t *testing.T, version uint) *PostgresStore { - t.Helper() - - t.Logf("Creating new Postgres DB for testing, migrating to version %d", - version) - - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) - storeCfg := sqlFixture.GetConfig() - storeCfg.SkipMigrations = true - store, err := NewPostgresStore(storeCfg) - require.NoError(t, err) - - err = store.ExecuteMigrations(TargetVersion(version)) - require.NoError(t, err) - - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - - return store -} diff --git a/db/sqlc/db_custom.go b/db/sqlc/db_custom.go index 9037d9614..af556eae7 100644 --- a/db/sqlc/db_custom.go +++ b/db/sqlc/db_custom.go @@ -26,16 +26,6 @@ func (q *Queries) Backend() sqldb.BackendType { return wtx.backendType } -// NewSqlite creates a new Queries instance for a SQLite database. -func NewSqlite(db DBTX) *Queries { - return &Queries{db: &wrappedTX{db, sqldb.BackendTypeSqlite}} -} - -// NewPostgres creates a new Queries instance for a Postgres database. -func NewPostgres(db DBTX) *Queries { - return &Queries{db: &wrappedTX{db, sqldb.BackendTypePostgres}} -} - // NewForType creates a new Queries instance for the given database type. func NewForType(db DBTX, typ sqldb.BackendType) *Queries { return &Queries{db: &wrappedTX{db, typ}} diff --git a/db/sqlite.go b/db/sqlite.go index 803362fa8..4e831a074 100644 --- a/db/sqlite.go +++ b/db/sqlite.go @@ -1,49 +1,9 @@ package db import ( - "database/sql" - "fmt" - "net/url" - "path/filepath" - "testing" - "time" - - "github.com/golang-migrate/migrate/v4" - sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite" - "github.com/lightninglabs/lightning-terminal/db/sqlc" - "github.com/stretchr/testify/require" _ "modernc.org/sqlite" // Register relevant drivers. ) -const ( - // sqliteOptionPrefix is the string prefix sqlite uses to set various - // options. This is used in the following format: - // * sqliteOptionPrefix || option_name = option_value. - sqliteOptionPrefix = "_pragma" - - // sqliteTxLockImmediate is a dsn option used to ensure that write - // transactions are started immediately. - sqliteTxLockImmediate = "_txlock=immediate" - - // defaultMaxConns is the number of permitted active and idle - // connections. We want to limit this so it isn't unlimited. We use the - // same value for the number of idle connections as, this can speed up - // queries given a new connection doesn't need to be established each - // time. - defaultMaxConns = 25 - - // defaultConnMaxLifetime is the maximum amount of time a connection can - // be reused for before it is closed. - defaultConnMaxLifetime = 10 * time.Minute -) - -var ( - // sqliteSchemaReplacements is a map of schema strings that need to be - // replaced for sqlite. There currently aren't any replacements, because - // the SQL files are written with SQLite compatibility in mind. - sqliteSchemaReplacements = map[string]string{} -) - // SqliteConfig holds all the config arguments needed to interact with our // sqlite DB. // @@ -61,248 +21,3 @@ type SqliteConfig struct { // found. DatabaseFileName string `long:"dbfile" description:"The full path to the database."` } - -// SqliteStore is a sqlite3 based database for the Taproot Asset daemon. -type SqliteStore struct { - cfg *SqliteConfig - - *BaseDB -} - -// NewSqliteStore attempts to open a new sqlite database based on the passed -// config. -func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { - // The set of pragma options are accepted using query options. For now - // we only want to ensure that foreign key constraints are properly - // enforced. - pragmaOptions := []struct { - name string - value string - }{ - { - name: "foreign_keys", - value: "on", - }, - { - name: "journal_mode", - value: "WAL", - }, - { - name: "busy_timeout", - value: "5000", - }, - { - // With the WAL mode, this ensures that we also do an - // extra WAL sync after each transaction. The normal - // sync mode skips this and gives better performance, - // but risks durability. - name: "synchronous", - value: "full", - }, - { - // This is used to ensure proper durability for users - // running on Mac OS. It uses the correct fsync system - // call to ensure items are fully flushed to disk. - name: "fullfsync", - value: "true", - }, - } - sqliteOptions := make(url.Values) - for _, option := range pragmaOptions { - sqliteOptions.Add( - sqliteOptionPrefix, - fmt.Sprintf("%v=%v", option.name, option.value), - ) - } - - // Construct the DSN which is just the database file name, appended - // with the series of pragma options as a query URL string. For more - // details on the formatting here, see the modernc.org/sqlite docs: - // https://pkg.go.dev/modernc.org/sqlite#Driver.Open. - dsn := fmt.Sprintf( - "%v?%v&%v", cfg.DatabaseFileName, sqliteOptions.Encode(), - sqliteTxLockImmediate, - ) - db, err := sql.Open("sqlite", dsn) - if err != nil { - return nil, err - } - - db.SetMaxOpenConns(defaultMaxConns) - db.SetMaxIdleConns(defaultMaxConns) - db.SetConnMaxLifetime(defaultConnMaxLifetime) - - queries := sqlc.NewSqlite(db) - s := &SqliteStore{ - cfg: cfg, - BaseDB: &BaseDB{ - DB: db, - Queries: queries, - }, - } - - // Now that the database is open, populate the database with our set of - // schemas based on our embedded in-memory file system. - if !cfg.SkipMigrations { - if err := s.ExecuteMigrations(s.backupAndMigrate); err != nil { - return nil, fmt.Errorf("error executing migrations: "+ - "%w", err) - } - } - - return s, nil -} - -// backupSqliteDatabase creates a backup of the given SQLite database. -func backupSqliteDatabase(srcDB *sql.DB, dbFullFilePath string) error { - if srcDB == nil { - return fmt.Errorf("backup source database is nil") - } - - // Create a database backup file full path from the given source - // database full file path. - // - // Get the current time and format it as a Unix timestamp in - // nanoseconds. - timestamp := time.Now().UnixNano() - - // Add the timestamp to the backup name. - backupFullFilePath := fmt.Sprintf( - "%s.%d.backup", dbFullFilePath, timestamp, - ) - - log.Infof("Creating backup of database file: %v -> %v", - dbFullFilePath, backupFullFilePath) - - // Create the database backup. - vacuumIntoQuery := "VACUUM INTO ?;" - stmt, err := srcDB.Prepare(vacuumIntoQuery) - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec(backupFullFilePath) - if err != nil { - return err - } - - return nil -} - -// backupAndMigrate is a helper function that creates a database backup before -// initiating the migration, and then migrates the database to the latest -// version. -func (s *SqliteStore) backupAndMigrate(mig *migrate.Migrate, - currentDbVersion int, maxMigrationVersion uint) error { - - // Determine if a database migration is necessary given the current - // database version and the maximum migration version. - versionUpgradePending := currentDbVersion < int(maxMigrationVersion) - if !versionUpgradePending { - log.Infof("Current database version is up-to-date, skipping "+ - "migration attempt and backup creation "+ - "(current_db_version=%v, max_migration_version=%v)", - currentDbVersion, maxMigrationVersion) - return nil - } - - // At this point, we know that a database migration is necessary. - // Create a backup of the database before starting the migration. - if !s.cfg.SkipMigrationDbBackup { - log.Infof("Creating database backup (before applying " + - "migration(s))") - - err := backupSqliteDatabase(s.DB, s.cfg.DatabaseFileName) - if err != nil { - return err - } - } else { - log.Infof("Skipping database backup creation before applying " + - "migration(s)") - } - - log.Infof("Applying migrations to database") - return mig.Up() -} - -// ExecuteMigrations runs migrations for the sqlite database, depending on the -// target given, either all migrations or up to a given version. -func (s *SqliteStore) ExecuteMigrations(target MigrationTarget, - optFuncs ...MigrateOpt) error { - - opts := defaultMigrateOptions() - for _, optFunc := range optFuncs { - optFunc(opts) - } - - driver, err := sqlite_migrate.WithInstance( - s.DB, &sqlite_migrate.Config{}, - ) - if err != nil { - return fmt.Errorf("error creating sqlite migration: %w", err) - } - - sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements) - return applyMigrations( - sqliteFS, driver, "sqlc/migrations", "sqlite", target, opts, - ) -} - -// NewTestSqliteDB is a helper function that creates an SQLite database for -// testing. -func NewTestSqliteDB(t *testing.T) *SqliteStore { - t.Helper() - - // TODO(roasbeef): if we pass :memory: for the file name, then we get - // an in mem version to speed up tests - dbPath := filepath.Join(t.TempDir(), "tmp.db") - t.Logf("Creating new SQLite DB handle for testing: %s", dbPath) - - return NewTestSqliteDbHandleFromPath(t, dbPath) -} - -// NewTestSqliteDbHandleFromPath is a helper function that creates a SQLite -// database handle given a database file path. -func NewTestSqliteDbHandleFromPath(t *testing.T, dbPath string) *SqliteStore { - t.Helper() - - sqlDB, err := NewSqliteStore(&SqliteConfig{ - DatabaseFileName: dbPath, - SkipMigrations: false, - }) - require.NoError(t, err) - - t.Cleanup(func() { - require.NoError(t, sqlDB.DB.Close()) - }) - - return sqlDB -} - -// NewTestSqliteDBWithVersion is a helper function that creates an SQLite -// database for testing and migrates it to the given version. -func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore { - t.Helper() - - t.Logf("Creating new SQLite DB for testing, migrating to version %d", - version) - - // TODO(roasbeef): if we pass :memory: for the file name, then we get - // an in mem version to speed up tests - dbFileName := filepath.Join(t.TempDir(), "tmp.db") - sqlDB, err := NewSqliteStore(&SqliteConfig{ - DatabaseFileName: dbFileName, - SkipMigrations: true, - }) - require.NoError(t, err) - - err = sqlDB.ExecuteMigrations(TargetVersion(version)) - require.NoError(t, err) - - t.Cleanup(func() { - require.NoError(t, sqlDB.DB.Close()) - }) - - return sqlDB -} From 779284859c849a2830f8cd55faaf2a21e87e1071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 01:47:28 +0200 Subject: [PATCH 12/26] mutli: rename `db.NewTestPostgresV2DB` function As the legacy `NewTestPostgresDB` function is no longer used and has been removed, it no longer makes sense to have a `V2` suffix on the `NewTestPostgresV2DB` function. This commit renames it to `NewTestPostgresDB`, to indicate that this function now replaces the legacy function. --- accounts/test_postgres.go | 4 ++-- db/postgres.go | 6 +++--- firewalldb/test_postgres.go | 4 ++-- session/test_postgres.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/accounts/test_postgres.go b/accounts/test_postgres.go index a88ed3a84..16665030d 100644 --- a/accounts/test_postgres.go +++ b/accounts/test_postgres.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,5 +24,5 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } diff --git a/db/postgres.go b/db/postgres.go index a418f58e3..780a7a13d 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -56,9 +56,9 @@ func (s *PostgresConfig) DSN(hidePassword bool) string { s.DBName, sslMode) } -// NewTestPostgresV2DB is a helper function that creates a Postgres database for -// testing, using the sqldb v2 package's definition of the PostgresStore. -func NewTestPostgresV2DB(t *testing.T) *sqldb.PostgresStore { +// NewTestPostgresDB is a helper function that creates a Postgres database for +// testing. +func NewTestPostgresDB(t *testing.T) *sqldb.PostgresStore { t.Helper() t.Logf("Creating new Postgres DB for testing") diff --git a/firewalldb/test_postgres.go b/firewalldb/test_postgres.go index 3a54a81c4..732b19b4a 100644 --- a/firewalldb/test_postgres.go +++ b/firewalldb/test_postgres.go @@ -11,11 +11,11 @@ import ( // NewTestDB is a helper function that creates an BBolt database for testing. func NewTestDB(t *testing.T, clock clock.Clock) FirewallDBs { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new BoltStore with a // connection to an existing BBolt database for testing. func NewTestDBFromPath(t *testing.T, _ string, clock clock.Clock) FirewallDBs { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } diff --git a/session/test_postgres.go b/session/test_postgres.go index d8d5d2111..cb5aa061d 100644 --- a/session/test_postgres.go +++ b/session/test_postgres.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,5 +24,5 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return createStore(t, db.NewTestPostgresV2DB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } From d8d23310c93602d15032601484fcc4a783ea5b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 01:16:52 +0200 Subject: [PATCH 13/26] sqlcmig6: add `sqlcmig6` package This commit introduces the `sqlcmig6` package, which at the time of this commit contains the same queries and models as `sqlc` package. Importantly though, once the kvdb to sql migration is made available in production, the `sqlcmig6` package will not change, as it is intended to represent the sql db as it was at the time of the migration. The sqlcmig6 package is therefore intended to be used in the kvdb to sql migration code, as it is will always be compatible with the sql database when all sql migrations prior to the kvdb to sql migration are applied. When additional sql migrations are added in the future, they may effect the `sqlc` package in such a way that the standard `sqlc` queries and models aren't compatible with kvdb to sql migration code any longer. By preserving the `sqlcmig6` package, we ensure that the kvdb to sql migration code can always use the same queries and models that were available at the time of the migration, even if the `sqlc` package changes in the future. Note that the `sqlcmig6` package have not been generated by `sqlc` (the queries and models are copied from the `sqlc` package), as it is not intended to be changed in the future. --- .golangci.yml | 6 + db/sqlcmig6/accounts.sql.go | 390 ++++++++++++++++++ db/sqlcmig6/actions.sql.go | 73 ++++ db/sqlcmig6/actions_custom.go | 210 ++++++++++ db/sqlcmig6/db.go | 27 ++ db/sqlcmig6/db_custom.go | 48 +++ db/sqlcmig6/kvstores.sql.go | 376 +++++++++++++++++ db/sqlcmig6/models.go | 124 ++++++ db/sqlcmig6/privacy_paris.sql.go | 91 +++++ db/sqlcmig6/querier.go | 75 ++++ db/sqlcmig6/sessions.sql.go | 675 +++++++++++++++++++++++++++++++ 11 files changed, 2095 insertions(+) create mode 100644 db/sqlcmig6/accounts.sql.go create mode 100644 db/sqlcmig6/actions.sql.go create mode 100644 db/sqlcmig6/actions_custom.go create mode 100644 db/sqlcmig6/db.go create mode 100644 db/sqlcmig6/db_custom.go create mode 100644 db/sqlcmig6/kvstores.sql.go create mode 100644 db/sqlcmig6/models.go create mode 100644 db/sqlcmig6/privacy_paris.sql.go create mode 100644 db/sqlcmig6/querier.go create mode 100644 db/sqlcmig6/sessions.sql.go diff --git a/.golangci.yml b/.golangci.yml index a748699f0..e6fc648bd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -61,3 +61,9 @@ issues: - unused - deadcode - varcheck + # As the db/sqlcmig6 package has been copied from sqlc generated code, + # but isn't marked as code generated by sqlc, the code line length in the + # package will often exceed the lll limit. + - path: db/sqlcmig6/.* + linters: + - lll diff --git a/db/sqlcmig6/accounts.sql.go b/db/sqlcmig6/accounts.sql.go new file mode 100644 index 000000000..479c82b36 --- /dev/null +++ b/db/sqlcmig6/accounts.sql.go @@ -0,0 +1,390 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" + "time" +) + +const addAccountInvoice = `-- name: AddAccountInvoice :exec +INSERT INTO account_invoices (account_id, hash) +VALUES ($1, $2) +` + +type AddAccountInvoiceParams struct { + AccountID int64 + Hash []byte +} + +func (q *Queries) AddAccountInvoice(ctx context.Context, arg AddAccountInvoiceParams) error { + _, err := q.db.ExecContext(ctx, addAccountInvoice, arg.AccountID, arg.Hash) + return err +} + +const deleteAccount = `-- name: DeleteAccount :exec +DELETE FROM accounts +WHERE id = $1 +` + +func (q *Queries) DeleteAccount(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, deleteAccount, id) + return err +} + +const deleteAccountPayment = `-- name: DeleteAccountPayment :exec +DELETE FROM account_payments +WHERE hash = $1 +AND account_id = $2 +` + +type DeleteAccountPaymentParams struct { + Hash []byte + AccountID int64 +} + +func (q *Queries) DeleteAccountPayment(ctx context.Context, arg DeleteAccountPaymentParams) error { + _, err := q.db.ExecContext(ctx, deleteAccountPayment, arg.Hash, arg.AccountID) + return err +} + +const getAccount = `-- name: GetAccount :one +SELECT id, alias, label, type, initial_balance_msat, current_balance_msat, last_updated, expiration +FROM accounts +WHERE id = $1 +` + +func (q *Queries) GetAccount(ctx context.Context, id int64) (Account, error) { + row := q.db.QueryRowContext(ctx, getAccount, id) + var i Account + err := row.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.Type, + &i.InitialBalanceMsat, + &i.CurrentBalanceMsat, + &i.LastUpdated, + &i.Expiration, + ) + return i, err +} + +const getAccountByLabel = `-- name: GetAccountByLabel :one +SELECT id, alias, label, type, initial_balance_msat, current_balance_msat, last_updated, expiration +FROM accounts +WHERE label = $1 +` + +func (q *Queries) GetAccountByLabel(ctx context.Context, label sql.NullString) (Account, error) { + row := q.db.QueryRowContext(ctx, getAccountByLabel, label) + var i Account + err := row.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.Type, + &i.InitialBalanceMsat, + &i.CurrentBalanceMsat, + &i.LastUpdated, + &i.Expiration, + ) + return i, err +} + +const getAccountIDByAlias = `-- name: GetAccountIDByAlias :one +SELECT id +FROM accounts +WHERE alias = $1 +` + +func (q *Queries) GetAccountIDByAlias(ctx context.Context, alias int64) (int64, error) { + row := q.db.QueryRowContext(ctx, getAccountIDByAlias, alias) + var id int64 + err := row.Scan(&id) + return id, err +} + +const getAccountIndex = `-- name: GetAccountIndex :one +SELECT value +FROM account_indices +WHERE name = $1 +` + +func (q *Queries) GetAccountIndex(ctx context.Context, name string) (int64, error) { + row := q.db.QueryRowContext(ctx, getAccountIndex, name) + var value int64 + err := row.Scan(&value) + return value, err +} + +const getAccountInvoice = `-- name: GetAccountInvoice :one +SELECT account_id, hash +FROM account_invoices +WHERE account_id = $1 + AND hash = $2 +` + +type GetAccountInvoiceParams struct { + AccountID int64 + Hash []byte +} + +func (q *Queries) GetAccountInvoice(ctx context.Context, arg GetAccountInvoiceParams) (AccountInvoice, error) { + row := q.db.QueryRowContext(ctx, getAccountInvoice, arg.AccountID, arg.Hash) + var i AccountInvoice + err := row.Scan(&i.AccountID, &i.Hash) + return i, err +} + +const getAccountPayment = `-- name: GetAccountPayment :one +SELECT account_id, hash, status, full_amount_msat FROM account_payments +WHERE hash = $1 +AND account_id = $2 +` + +type GetAccountPaymentParams struct { + Hash []byte + AccountID int64 +} + +func (q *Queries) GetAccountPayment(ctx context.Context, arg GetAccountPaymentParams) (AccountPayment, error) { + row := q.db.QueryRowContext(ctx, getAccountPayment, arg.Hash, arg.AccountID) + var i AccountPayment + err := row.Scan( + &i.AccountID, + &i.Hash, + &i.Status, + &i.FullAmountMsat, + ) + return i, err +} + +const insertAccount = `-- name: InsertAccount :one +INSERT INTO accounts (type, initial_balance_msat, current_balance_msat, last_updated, label, alias, expiration) +VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id +` + +type InsertAccountParams struct { + Type int16 + InitialBalanceMsat int64 + CurrentBalanceMsat int64 + LastUpdated time.Time + Label sql.NullString + Alias int64 + Expiration time.Time +} + +func (q *Queries) InsertAccount(ctx context.Context, arg InsertAccountParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertAccount, + arg.Type, + arg.InitialBalanceMsat, + arg.CurrentBalanceMsat, + arg.LastUpdated, + arg.Label, + arg.Alias, + arg.Expiration, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const listAccountInvoices = `-- name: ListAccountInvoices :many +SELECT account_id, hash +FROM account_invoices +WHERE account_id = $1 +` + +func (q *Queries) ListAccountInvoices(ctx context.Context, accountID int64) ([]AccountInvoice, error) { + rows, err := q.db.QueryContext(ctx, listAccountInvoices, accountID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []AccountInvoice + for rows.Next() { + var i AccountInvoice + if err := rows.Scan(&i.AccountID, &i.Hash); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listAccountPayments = `-- name: ListAccountPayments :many +SELECT account_id, hash, status, full_amount_msat +FROM account_payments +WHERE account_id = $1 +` + +func (q *Queries) ListAccountPayments(ctx context.Context, accountID int64) ([]AccountPayment, error) { + rows, err := q.db.QueryContext(ctx, listAccountPayments, accountID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []AccountPayment + for rows.Next() { + var i AccountPayment + if err := rows.Scan( + &i.AccountID, + &i.Hash, + &i.Status, + &i.FullAmountMsat, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listAllAccounts = `-- name: ListAllAccounts :many +SELECT id, alias, label, type, initial_balance_msat, current_balance_msat, last_updated, expiration +FROM accounts +ORDER BY id +` + +func (q *Queries) ListAllAccounts(ctx context.Context) ([]Account, error) { + rows, err := q.db.QueryContext(ctx, listAllAccounts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Account + for rows.Next() { + var i Account + if err := rows.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.Type, + &i.InitialBalanceMsat, + &i.CurrentBalanceMsat, + &i.LastUpdated, + &i.Expiration, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setAccountIndex = `-- name: SetAccountIndex :exec +INSERT INTO account_indices (name, value) +VALUES ($1, $2) + ON CONFLICT (name) +DO UPDATE SET value = $2 +` + +type SetAccountIndexParams struct { + Name string + Value int64 +} + +func (q *Queries) SetAccountIndex(ctx context.Context, arg SetAccountIndexParams) error { + _, err := q.db.ExecContext(ctx, setAccountIndex, arg.Name, arg.Value) + return err +} + +const updateAccountBalance = `-- name: UpdateAccountBalance :one +UPDATE accounts +SET current_balance_msat = $1 +WHERE id = $2 +RETURNING id +` + +type UpdateAccountBalanceParams struct { + CurrentBalanceMsat int64 + ID int64 +} + +func (q *Queries) UpdateAccountBalance(ctx context.Context, arg UpdateAccountBalanceParams) (int64, error) { + row := q.db.QueryRowContext(ctx, updateAccountBalance, arg.CurrentBalanceMsat, arg.ID) + var id int64 + err := row.Scan(&id) + return id, err +} + +const updateAccountExpiry = `-- name: UpdateAccountExpiry :one +UPDATE accounts +SET expiration = $1 +WHERE id = $2 +RETURNING id +` + +type UpdateAccountExpiryParams struct { + Expiration time.Time + ID int64 +} + +func (q *Queries) UpdateAccountExpiry(ctx context.Context, arg UpdateAccountExpiryParams) (int64, error) { + row := q.db.QueryRowContext(ctx, updateAccountExpiry, arg.Expiration, arg.ID) + var id int64 + err := row.Scan(&id) + return id, err +} + +const updateAccountLastUpdate = `-- name: UpdateAccountLastUpdate :one +UPDATE accounts +SET last_updated = $1 +WHERE id = $2 +RETURNING id +` + +type UpdateAccountLastUpdateParams struct { + LastUpdated time.Time + ID int64 +} + +func (q *Queries) UpdateAccountLastUpdate(ctx context.Context, arg UpdateAccountLastUpdateParams) (int64, error) { + row := q.db.QueryRowContext(ctx, updateAccountLastUpdate, arg.LastUpdated, arg.ID) + var id int64 + err := row.Scan(&id) + return id, err +} + +const upsertAccountPayment = `-- name: UpsertAccountPayment :exec +INSERT INTO account_payments (account_id, hash, status, full_amount_msat) +VALUES ($1, $2, $3, $4) +ON CONFLICT (account_id, hash) +DO UPDATE SET status = $3, full_amount_msat = $4 +` + +type UpsertAccountPaymentParams struct { + AccountID int64 + Hash []byte + Status int16 + FullAmountMsat int64 +} + +func (q *Queries) UpsertAccountPayment(ctx context.Context, arg UpsertAccountPaymentParams) error { + _, err := q.db.ExecContext(ctx, upsertAccountPayment, + arg.AccountID, + arg.Hash, + arg.Status, + arg.FullAmountMsat, + ) + return err +} diff --git a/db/sqlcmig6/actions.sql.go b/db/sqlcmig6/actions.sql.go new file mode 100644 index 000000000..a39d51e5d --- /dev/null +++ b/db/sqlcmig6/actions.sql.go @@ -0,0 +1,73 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" + "time" +) + +const insertAction = `-- name: InsertAction :one +INSERT INTO actions ( + session_id, account_id, macaroon_identifier, actor_name, feature_name, action_trigger, + intent, structured_json_data, rpc_method, rpc_params_json, created_at, + action_state, error_reason +) VALUES ( + $1, $2, $3, $4, $5, $6, + $7, $8, $9, $10, $11, $12, $13 +) RETURNING id +` + +type InsertActionParams struct { + SessionID sql.NullInt64 + AccountID sql.NullInt64 + MacaroonIdentifier []byte + ActorName sql.NullString + FeatureName sql.NullString + ActionTrigger sql.NullString + Intent sql.NullString + StructuredJsonData []byte + RpcMethod string + RpcParamsJson []byte + CreatedAt time.Time + ActionState int16 + ErrorReason sql.NullString +} + +func (q *Queries) InsertAction(ctx context.Context, arg InsertActionParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertAction, + arg.SessionID, + arg.AccountID, + arg.MacaroonIdentifier, + arg.ActorName, + arg.FeatureName, + arg.ActionTrigger, + arg.Intent, + arg.StructuredJsonData, + arg.RpcMethod, + arg.RpcParamsJson, + arg.CreatedAt, + arg.ActionState, + arg.ErrorReason, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const setActionState = `-- name: SetActionState :exec +UPDATE actions +SET action_state = $1, + error_reason = $2 +WHERE id = $3 +` + +type SetActionStateParams struct { + ActionState int16 + ErrorReason sql.NullString + ID int64 +} + +func (q *Queries) SetActionState(ctx context.Context, arg SetActionStateParams) error { + _, err := q.db.ExecContext(ctx, setActionState, arg.ActionState, arg.ErrorReason, arg.ID) + return err +} diff --git a/db/sqlcmig6/actions_custom.go b/db/sqlcmig6/actions_custom.go new file mode 100644 index 000000000..f01772d51 --- /dev/null +++ b/db/sqlcmig6/actions_custom.go @@ -0,0 +1,210 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" + "strconv" + "strings" +) + +// ActionQueryParams defines the parameters for querying actions. +type ActionQueryParams struct { + SessionID sql.NullInt64 + AccountID sql.NullInt64 + FeatureName sql.NullString + ActorName sql.NullString + RpcMethod sql.NullString + State sql.NullInt16 + EndTime sql.NullTime + StartTime sql.NullTime + GroupID sql.NullInt64 +} + +// ListActionsParams defines the parameters for listing actions, including +// the ActionQueryParams for filtering and a Pagination struct for +// pagination. The Reversed field indicates whether the results should be +// returned in reverse order based on the created_at timestamp. +type ListActionsParams struct { + ActionQueryParams + Reversed bool + *Pagination +} + +// Pagination defines the pagination parameters for listing actions. +type Pagination struct { + NumOffset int32 + NumLimit int32 +} + +// ListActions retrieves a list of actions based on the provided +// ListActionsParams. +func (q *Queries) ListActions(ctx context.Context, + arg ListActionsParams) ([]Action, error) { + + query, args := buildListActionsQuery(arg) + rows, err := q.db.QueryContext(ctx, fillPlaceHolders(query), args...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Action + for rows.Next() { + var i Action + if err := rows.Scan( + &i.ID, + &i.SessionID, + &i.AccountID, + &i.MacaroonIdentifier, + &i.ActorName, + &i.FeatureName, + &i.ActionTrigger, + &i.Intent, + &i.StructuredJsonData, + &i.RpcMethod, + &i.RpcParamsJson, + &i.CreatedAt, + &i.ActionState, + &i.ErrorReason, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +// CountActions returns the number of actions that match the provided +// ActionQueryParams. +func (q *Queries) CountActions(ctx context.Context, + arg ActionQueryParams) (int64, error) { + + query, args := buildActionsQuery(arg, true) + row := q.db.QueryRowContext(ctx, query, args...) + + var count int64 + err := row.Scan(&count) + + return count, err +} + +// buildActionsQuery constructs a SQL query to retrieve actions based on the +// provided parameters. We do this manually so that if, for example, we have +// a sessionID we are filtering by, then this appears in the query as: +// `WHERE a.session_id = ?` which will properly make use of the underlying +// index. If we were instead to use a single SQLC query, it would include many +// WHERE clauses like: +// "WHERE a.session_id = COALESCE(sqlc.narg('session_id'), a.session_id)". +// This would use the index if run against postres but not when run against +// sqlite. +// +// The 'count' param indicates whether the query should return a count of +// actions that match the criteria or the actions themselves. +func buildActionsQuery(params ActionQueryParams, count bool) (string, []any) { + var ( + conditions []string + args []any + ) + + if params.SessionID.Valid { + conditions = append(conditions, "a.session_id = ?") + args = append(args, params.SessionID.Int64) + } + if params.AccountID.Valid { + conditions = append(conditions, "a.account_id = ?") + args = append(args, params.AccountID.Int64) + } + if params.FeatureName.Valid { + conditions = append(conditions, "a.feature_name = ?") + args = append(args, params.FeatureName.String) + } + if params.ActorName.Valid { + conditions = append(conditions, "a.actor_name = ?") + args = append(args, params.ActorName.String) + } + if params.RpcMethod.Valid { + conditions = append(conditions, "a.rpc_method = ?") + args = append(args, params.RpcMethod.String) + } + if params.State.Valid { + conditions = append(conditions, "a.action_state = ?") + args = append(args, params.State.Int16) + } + if params.EndTime.Valid { + conditions = append(conditions, "a.created_at <= ?") + args = append(args, params.EndTime.Time) + } + if params.StartTime.Valid { + conditions = append(conditions, "a.created_at >= ?") + args = append(args, params.StartTime.Time) + } + if params.GroupID.Valid { + conditions = append(conditions, ` + EXISTS ( + SELECT 1 + FROM sessions s + WHERE s.id = a.session_id AND s.group_id = ? + )`) + args = append(args, params.GroupID.Int64) + } + + query := "SELECT a.* FROM actions a" + if count { + query = "SELECT COUNT(*) FROM actions a" + } + if len(conditions) > 0 { + query += " WHERE " + strings.Join(conditions, " AND ") + } + + return query, args +} + +// buildListActionsQuery constructs a SQL query to retrieve a list of actions +// based on the provided parameters. It builds upon the `buildActionsQuery` +// function, adding pagination and ordering based on the reversed parameter. +func buildListActionsQuery(params ListActionsParams) (string, []interface{}) { + query, args := buildActionsQuery(params.ActionQueryParams, false) + + // Determine order direction. + order := "ASC" + if params.Reversed { + order = "DESC" + } + query += " ORDER BY a.created_at " + order + + // Maybe paginate. + if params.Pagination != nil { + query += " LIMIT ? OFFSET ?" + args = append(args, params.NumLimit, params.NumOffset) + } + + return query, args +} + +// fillPlaceHolders replaces all '?' placeholders in the SQL query with +// positional placeholders like $1, $2, etc. This is necessary for +// compatibility with Postgres. +func fillPlaceHolders(query string) string { + var ( + sb strings.Builder + argNum = 1 + ) + + for i := range len(query) { + if query[i] != '?' { + sb.WriteByte(query[i]) + continue + } + + sb.WriteString("$") + sb.WriteString(strconv.Itoa(argNum)) + argNum++ + } + + return sb.String() +} diff --git a/db/sqlcmig6/db.go b/db/sqlcmig6/db.go new file mode 100644 index 000000000..82ff72dd8 --- /dev/null +++ b/db/sqlcmig6/db.go @@ -0,0 +1,27 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/db/sqlcmig6/db_custom.go b/db/sqlcmig6/db_custom.go new file mode 100644 index 000000000..e128cb290 --- /dev/null +++ b/db/sqlcmig6/db_custom.go @@ -0,0 +1,48 @@ +package sqlcmig6 + +import ( + "context" + + "github.com/lightningnetwork/lnd/sqldb/v2" +) + +// wrappedTX is a wrapper around a DBTX that also stores the database backend +// type. +type wrappedTX struct { + DBTX + + backendType sqldb.BackendType +} + +// Backend returns the type of database backend we're using. +func (q *Queries) Backend() sqldb.BackendType { + wtx, ok := q.db.(*wrappedTX) + if !ok { + // Shouldn't happen unless a new database backend type is added + // but not initialized correctly. + return sqldb.BackendTypeUnknown + } + + return wtx.backendType +} + +// NewForType creates a new Queries instance for the given database type. +func NewForType(db DBTX, typ sqldb.BackendType) *Queries { + return &Queries{db: &wrappedTX{db, typ}} +} + +// CustomQueries defines a set of custom queries that we define in addition +// to the ones generated by sqlc. +type CustomQueries interface { + // CountActions returns the number of actions that match the provided + // ActionQueryParams. + CountActions(ctx context.Context, arg ActionQueryParams) (int64, error) + + // ListActions retrieves a list of actions based on the provided + // ListActionsParams. + ListActions(ctx context.Context, + arg ListActionsParams) ([]Action, error) + + // Backend returns the type of the database backend used. + Backend() sqldb.BackendType +} diff --git a/db/sqlcmig6/kvstores.sql.go b/db/sqlcmig6/kvstores.sql.go new file mode 100644 index 000000000..3c076c5a4 --- /dev/null +++ b/db/sqlcmig6/kvstores.sql.go @@ -0,0 +1,376 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" +) + +const deleteAllTempKVStores = `-- name: DeleteAllTempKVStores :exec +DELETE FROM kvstores +WHERE perm = false +` + +func (q *Queries) DeleteAllTempKVStores(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, deleteAllTempKVStores) + return err +} + +const deleteFeatureKVStoreRecord = `-- name: DeleteFeatureKVStoreRecord :exec +DELETE FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id = $4 + AND feature_id = $5 +` + +type DeleteFeatureKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 + FeatureID sql.NullInt64 +} + +func (q *Queries) DeleteFeatureKVStoreRecord(ctx context.Context, arg DeleteFeatureKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, deleteFeatureKVStoreRecord, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + arg.FeatureID, + ) + return err +} + +const deleteGlobalKVStoreRecord = `-- name: DeleteGlobalKVStoreRecord :exec +DELETE FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id IS NULL + AND feature_id IS NULL +` + +type DeleteGlobalKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool +} + +func (q *Queries) DeleteGlobalKVStoreRecord(ctx context.Context, arg DeleteGlobalKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, deleteGlobalKVStoreRecord, arg.Key, arg.RuleID, arg.Perm) + return err +} + +const deleteGroupKVStoreRecord = `-- name: DeleteGroupKVStoreRecord :exec +DELETE FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id = $4 + AND feature_id IS NULL +` + +type DeleteGroupKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 +} + +func (q *Queries) DeleteGroupKVStoreRecord(ctx context.Context, arg DeleteGroupKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, deleteGroupKVStoreRecord, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + ) + return err +} + +const getFeatureID = `-- name: GetFeatureID :one +SELECT id +FROM features +WHERE name = $1 +` + +func (q *Queries) GetFeatureID(ctx context.Context, name string) (int64, error) { + row := q.db.QueryRowContext(ctx, getFeatureID, name) + var id int64 + err := row.Scan(&id) + return id, err +} + +const getFeatureKVStoreRecord = `-- name: GetFeatureKVStoreRecord :one +SELECT value +FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id = $4 + AND feature_id = $5 +` + +type GetFeatureKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 + FeatureID sql.NullInt64 +} + +func (q *Queries) GetFeatureKVStoreRecord(ctx context.Context, arg GetFeatureKVStoreRecordParams) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getFeatureKVStoreRecord, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + arg.FeatureID, + ) + var value []byte + err := row.Scan(&value) + return value, err +} + +const getGlobalKVStoreRecord = `-- name: GetGlobalKVStoreRecord :one +SELECT value +FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id IS NULL + AND feature_id IS NULL +` + +type GetGlobalKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool +} + +func (q *Queries) GetGlobalKVStoreRecord(ctx context.Context, arg GetGlobalKVStoreRecordParams) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getGlobalKVStoreRecord, arg.Key, arg.RuleID, arg.Perm) + var value []byte + err := row.Scan(&value) + return value, err +} + +const getGroupKVStoreRecord = `-- name: GetGroupKVStoreRecord :one +SELECT value +FROM kvstores +WHERE entry_key = $1 + AND rule_id = $2 + AND perm = $3 + AND group_id = $4 + AND feature_id IS NULL +` + +type GetGroupKVStoreRecordParams struct { + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 +} + +func (q *Queries) GetGroupKVStoreRecord(ctx context.Context, arg GetGroupKVStoreRecordParams) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getGroupKVStoreRecord, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + ) + var value []byte + err := row.Scan(&value) + return value, err +} + +const getOrInsertFeatureID = `-- name: GetOrInsertFeatureID :one +INSERT INTO features (name) +VALUES ($1) +ON CONFLICT(name) DO UPDATE SET name = excluded.name +RETURNING id +` + +func (q *Queries) GetOrInsertFeatureID(ctx context.Context, name string) (int64, error) { + row := q.db.QueryRowContext(ctx, getOrInsertFeatureID, name) + var id int64 + err := row.Scan(&id) + return id, err +} + +const getOrInsertRuleID = `-- name: GetOrInsertRuleID :one +INSERT INTO rules (name) +VALUES ($1) +ON CONFLICT(name) DO UPDATE SET name = excluded.name +RETURNING id +` + +func (q *Queries) GetOrInsertRuleID(ctx context.Context, name string) (int64, error) { + row := q.db.QueryRowContext(ctx, getOrInsertRuleID, name) + var id int64 + err := row.Scan(&id) + return id, err +} + +const getRuleID = `-- name: GetRuleID :one +SELECT id +FROM rules +WHERE name = $1 +` + +func (q *Queries) GetRuleID(ctx context.Context, name string) (int64, error) { + row := q.db.QueryRowContext(ctx, getRuleID, name) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertKVStoreRecord = `-- name: InsertKVStoreRecord :exec +INSERT INTO kvstores (perm, rule_id, group_id, feature_id, entry_key, value) +VALUES ($1, $2, $3, $4, $5, $6) +` + +type InsertKVStoreRecordParams struct { + Perm bool + RuleID int64 + GroupID sql.NullInt64 + FeatureID sql.NullInt64 + EntryKey string + Value []byte +} + +func (q *Queries) InsertKVStoreRecord(ctx context.Context, arg InsertKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, insertKVStoreRecord, + arg.Perm, + arg.RuleID, + arg.GroupID, + arg.FeatureID, + arg.EntryKey, + arg.Value, + ) + return err +} + +const listAllKVStoresRecords = `-- name: ListAllKVStoresRecords :many +SELECT id, perm, rule_id, group_id, feature_id, entry_key, value +FROM kvstores +` + +func (q *Queries) ListAllKVStoresRecords(ctx context.Context) ([]Kvstore, error) { + rows, err := q.db.QueryContext(ctx, listAllKVStoresRecords) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Kvstore + for rows.Next() { + var i Kvstore + if err := rows.Scan( + &i.ID, + &i.Perm, + &i.RuleID, + &i.GroupID, + &i.FeatureID, + &i.EntryKey, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateFeatureKVStoreRecord = `-- name: UpdateFeatureKVStoreRecord :exec +UPDATE kvstores +SET value = $1 +WHERE entry_key = $2 + AND rule_id = $3 + AND perm = $4 + AND group_id = $5 + AND feature_id = $6 +` + +type UpdateFeatureKVStoreRecordParams struct { + Value []byte + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 + FeatureID sql.NullInt64 +} + +func (q *Queries) UpdateFeatureKVStoreRecord(ctx context.Context, arg UpdateFeatureKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, updateFeatureKVStoreRecord, + arg.Value, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + arg.FeatureID, + ) + return err +} + +const updateGlobalKVStoreRecord = `-- name: UpdateGlobalKVStoreRecord :exec +UPDATE kvstores +SET value = $1 +WHERE entry_key = $2 + AND rule_id = $3 + AND perm = $4 + AND group_id IS NULL + AND feature_id IS NULL +` + +type UpdateGlobalKVStoreRecordParams struct { + Value []byte + Key string + RuleID int64 + Perm bool +} + +func (q *Queries) UpdateGlobalKVStoreRecord(ctx context.Context, arg UpdateGlobalKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, updateGlobalKVStoreRecord, + arg.Value, + arg.Key, + arg.RuleID, + arg.Perm, + ) + return err +} + +const updateGroupKVStoreRecord = `-- name: UpdateGroupKVStoreRecord :exec +UPDATE kvstores +SET value = $1 +WHERE entry_key = $2 + AND rule_id = $3 + AND perm = $4 + AND group_id = $5 + AND feature_id IS NULL +` + +type UpdateGroupKVStoreRecordParams struct { + Value []byte + Key string + RuleID int64 + Perm bool + GroupID sql.NullInt64 +} + +func (q *Queries) UpdateGroupKVStoreRecord(ctx context.Context, arg UpdateGroupKVStoreRecordParams) error { + _, err := q.db.ExecContext(ctx, updateGroupKVStoreRecord, + arg.Value, + arg.Key, + arg.RuleID, + arg.Perm, + arg.GroupID, + ) + return err +} diff --git a/db/sqlcmig6/models.go b/db/sqlcmig6/models.go new file mode 100644 index 000000000..9e57c28eb --- /dev/null +++ b/db/sqlcmig6/models.go @@ -0,0 +1,124 @@ +package sqlcmig6 + +import ( + "database/sql" + "time" +) + +type Account struct { + ID int64 + Alias int64 + Label sql.NullString + Type int16 + InitialBalanceMsat int64 + CurrentBalanceMsat int64 + LastUpdated time.Time + Expiration time.Time +} + +type AccountIndex struct { + Name string + Value int64 +} + +type AccountInvoice struct { + AccountID int64 + Hash []byte +} + +type AccountPayment struct { + AccountID int64 + Hash []byte + Status int16 + FullAmountMsat int64 +} + +type Action struct { + ID int64 + SessionID sql.NullInt64 + AccountID sql.NullInt64 + MacaroonIdentifier []byte + ActorName sql.NullString + FeatureName sql.NullString + ActionTrigger sql.NullString + Intent sql.NullString + StructuredJsonData []byte + RpcMethod string + RpcParamsJson []byte + CreatedAt time.Time + ActionState int16 + ErrorReason sql.NullString +} + +type Feature struct { + ID int64 + Name string +} + +type Kvstore struct { + ID int64 + Perm bool + RuleID int64 + GroupID sql.NullInt64 + FeatureID sql.NullInt64 + EntryKey string + Value []byte +} + +type PrivacyPair struct { + GroupID int64 + RealVal string + PseudoVal string +} + +type Rule struct { + ID int64 + Name string +} + +type Session struct { + ID int64 + Alias []byte + Label string + State int16 + Type int16 + Expiry time.Time + CreatedAt time.Time + RevokedAt sql.NullTime + ServerAddress string + DevServer bool + MacaroonRootKey int64 + PairingSecret []byte + LocalPrivateKey []byte + LocalPublicKey []byte + RemotePublicKey []byte + Privacy bool + AccountID sql.NullInt64 + GroupID sql.NullInt64 +} + +type SessionFeatureConfig struct { + SessionID int64 + FeatureName string + Config []byte +} + +type SessionMacaroonCaveat struct { + ID int64 + SessionID int64 + CaveatID []byte + VerificationID []byte + Location sql.NullString +} + +type SessionMacaroonPermission struct { + ID int64 + SessionID int64 + Entity string + Action string +} + +type SessionPrivacyFlag struct { + SessionID int64 + Flag int32 +} diff --git a/db/sqlcmig6/privacy_paris.sql.go b/db/sqlcmig6/privacy_paris.sql.go new file mode 100644 index 000000000..20af09a2f --- /dev/null +++ b/db/sqlcmig6/privacy_paris.sql.go @@ -0,0 +1,91 @@ +package sqlcmig6 + +import ( + "context" +) + +const getAllPrivacyPairs = `-- name: GetAllPrivacyPairs :many +SELECT real_val, pseudo_val +FROM privacy_pairs +WHERE group_id = $1 +` + +type GetAllPrivacyPairsRow struct { + RealVal string + PseudoVal string +} + +func (q *Queries) GetAllPrivacyPairs(ctx context.Context, groupID int64) ([]GetAllPrivacyPairsRow, error) { + rows, err := q.db.QueryContext(ctx, getAllPrivacyPairs, groupID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetAllPrivacyPairsRow + for rows.Next() { + var i GetAllPrivacyPairsRow + if err := rows.Scan(&i.RealVal, &i.PseudoVal); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getPseudoForReal = `-- name: GetPseudoForReal :one +SELECT pseudo_val +FROM privacy_pairs +WHERE group_id = $1 AND real_val = $2 +` + +type GetPseudoForRealParams struct { + GroupID int64 + RealVal string +} + +func (q *Queries) GetPseudoForReal(ctx context.Context, arg GetPseudoForRealParams) (string, error) { + row := q.db.QueryRowContext(ctx, getPseudoForReal, arg.GroupID, arg.RealVal) + var pseudo_val string + err := row.Scan(&pseudo_val) + return pseudo_val, err +} + +const getRealForPseudo = `-- name: GetRealForPseudo :one +SELECT real_val +FROM privacy_pairs +WHERE group_id = $1 AND pseudo_val = $2 +` + +type GetRealForPseudoParams struct { + GroupID int64 + PseudoVal string +} + +func (q *Queries) GetRealForPseudo(ctx context.Context, arg GetRealForPseudoParams) (string, error) { + row := q.db.QueryRowContext(ctx, getRealForPseudo, arg.GroupID, arg.PseudoVal) + var real_val string + err := row.Scan(&real_val) + return real_val, err +} + +const insertPrivacyPair = `-- name: InsertPrivacyPair :exec +INSERT INTO privacy_pairs (group_id, real_val, pseudo_val) +VALUES ($1, $2, $3) +` + +type InsertPrivacyPairParams struct { + GroupID int64 + RealVal string + PseudoVal string +} + +func (q *Queries) InsertPrivacyPair(ctx context.Context, arg InsertPrivacyPairParams) error { + _, err := q.db.ExecContext(ctx, insertPrivacyPair, arg.GroupID, arg.RealVal, arg.PseudoVal) + return err +} diff --git a/db/sqlcmig6/querier.go b/db/sqlcmig6/querier.go new file mode 100644 index 000000000..57e229b5f --- /dev/null +++ b/db/sqlcmig6/querier.go @@ -0,0 +1,75 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" +) + +type Querier interface { + AddAccountInvoice(ctx context.Context, arg AddAccountInvoiceParams) error + DeleteAccount(ctx context.Context, id int64) error + DeleteAccountPayment(ctx context.Context, arg DeleteAccountPaymentParams) error + DeleteAllTempKVStores(ctx context.Context) error + DeleteFeatureKVStoreRecord(ctx context.Context, arg DeleteFeatureKVStoreRecordParams) error + DeleteGlobalKVStoreRecord(ctx context.Context, arg DeleteGlobalKVStoreRecordParams) error + DeleteGroupKVStoreRecord(ctx context.Context, arg DeleteGroupKVStoreRecordParams) error + DeleteSessionsWithState(ctx context.Context, state int16) error + GetAccount(ctx context.Context, id int64) (Account, error) + GetAccountByLabel(ctx context.Context, label sql.NullString) (Account, error) + GetAccountIDByAlias(ctx context.Context, alias int64) (int64, error) + GetAccountIndex(ctx context.Context, name string) (int64, error) + GetAccountInvoice(ctx context.Context, arg GetAccountInvoiceParams) (AccountInvoice, error) + GetAccountPayment(ctx context.Context, arg GetAccountPaymentParams) (AccountPayment, error) + GetAliasBySessionID(ctx context.Context, id int64) ([]byte, error) + GetAllPrivacyPairs(ctx context.Context, groupID int64) ([]GetAllPrivacyPairsRow, error) + GetFeatureID(ctx context.Context, name string) (int64, error) + GetFeatureKVStoreRecord(ctx context.Context, arg GetFeatureKVStoreRecordParams) ([]byte, error) + GetGlobalKVStoreRecord(ctx context.Context, arg GetGlobalKVStoreRecordParams) ([]byte, error) + GetGroupKVStoreRecord(ctx context.Context, arg GetGroupKVStoreRecordParams) ([]byte, error) + GetOrInsertFeatureID(ctx context.Context, name string) (int64, error) + GetOrInsertRuleID(ctx context.Context, name string) (int64, error) + GetPseudoForReal(ctx context.Context, arg GetPseudoForRealParams) (string, error) + GetRealForPseudo(ctx context.Context, arg GetRealForPseudoParams) (string, error) + GetRuleID(ctx context.Context, name string) (int64, error) + GetSessionAliasesInGroup(ctx context.Context, groupID sql.NullInt64) ([][]byte, error) + GetSessionByAlias(ctx context.Context, alias []byte) (Session, error) + GetSessionByID(ctx context.Context, id int64) (Session, error) + GetSessionByLocalPublicKey(ctx context.Context, localPublicKey []byte) (Session, error) + GetSessionFeatureConfigs(ctx context.Context, sessionID int64) ([]SessionFeatureConfig, error) + GetSessionIDByAlias(ctx context.Context, alias []byte) (int64, error) + GetSessionMacaroonCaveats(ctx context.Context, sessionID int64) ([]SessionMacaroonCaveat, error) + GetSessionMacaroonPermissions(ctx context.Context, sessionID int64) ([]SessionMacaroonPermission, error) + GetSessionPrivacyFlags(ctx context.Context, sessionID int64) ([]SessionPrivacyFlag, error) + GetSessionsInGroup(ctx context.Context, groupID sql.NullInt64) ([]Session, error) + InsertAccount(ctx context.Context, arg InsertAccountParams) (int64, error) + InsertAction(ctx context.Context, arg InsertActionParams) (int64, error) + InsertKVStoreRecord(ctx context.Context, arg InsertKVStoreRecordParams) error + InsertPrivacyPair(ctx context.Context, arg InsertPrivacyPairParams) error + InsertSession(ctx context.Context, arg InsertSessionParams) (int64, error) + InsertSessionFeatureConfig(ctx context.Context, arg InsertSessionFeatureConfigParams) error + InsertSessionMacaroonCaveat(ctx context.Context, arg InsertSessionMacaroonCaveatParams) error + InsertSessionMacaroonPermission(ctx context.Context, arg InsertSessionMacaroonPermissionParams) error + InsertSessionPrivacyFlag(ctx context.Context, arg InsertSessionPrivacyFlagParams) error + ListAccountInvoices(ctx context.Context, accountID int64) ([]AccountInvoice, error) + ListAccountPayments(ctx context.Context, accountID int64) ([]AccountPayment, error) + ListAllAccounts(ctx context.Context) ([]Account, error) + ListAllKVStoresRecords(ctx context.Context) ([]Kvstore, error) + ListSessions(ctx context.Context) ([]Session, error) + ListSessionsByState(ctx context.Context, state int16) ([]Session, error) + ListSessionsByType(ctx context.Context, type_ int16) ([]Session, error) + SetAccountIndex(ctx context.Context, arg SetAccountIndexParams) error + SetActionState(ctx context.Context, arg SetActionStateParams) error + SetSessionGroupID(ctx context.Context, arg SetSessionGroupIDParams) error + SetSessionRemotePublicKey(ctx context.Context, arg SetSessionRemotePublicKeyParams) error + SetSessionRevokedAt(ctx context.Context, arg SetSessionRevokedAtParams) error + UpdateAccountBalance(ctx context.Context, arg UpdateAccountBalanceParams) (int64, error) + UpdateAccountExpiry(ctx context.Context, arg UpdateAccountExpiryParams) (int64, error) + UpdateAccountLastUpdate(ctx context.Context, arg UpdateAccountLastUpdateParams) (int64, error) + UpdateFeatureKVStoreRecord(ctx context.Context, arg UpdateFeatureKVStoreRecordParams) error + UpdateGlobalKVStoreRecord(ctx context.Context, arg UpdateGlobalKVStoreRecordParams) error + UpdateGroupKVStoreRecord(ctx context.Context, arg UpdateGroupKVStoreRecordParams) error + UpdateSessionState(ctx context.Context, arg UpdateSessionStateParams) error + UpsertAccountPayment(ctx context.Context, arg UpsertAccountPaymentParams) error +} + +var _ Querier = (*Queries)(nil) diff --git a/db/sqlcmig6/sessions.sql.go b/db/sqlcmig6/sessions.sql.go new file mode 100644 index 000000000..bc492043e --- /dev/null +++ b/db/sqlcmig6/sessions.sql.go @@ -0,0 +1,675 @@ +package sqlcmig6 + +import ( + "context" + "database/sql" + "time" +) + +const deleteSessionsWithState = `-- name: DeleteSessionsWithState :exec +DELETE FROM sessions +WHERE state = $1 +` + +func (q *Queries) DeleteSessionsWithState(ctx context.Context, state int16) error { + _, err := q.db.ExecContext(ctx, deleteSessionsWithState, state) + return err +} + +const getAliasBySessionID = `-- name: GetAliasBySessionID :one +SELECT alias FROM sessions +WHERE id = $1 +` + +func (q *Queries) GetAliasBySessionID(ctx context.Context, id int64) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getAliasBySessionID, id) + var alias []byte + err := row.Scan(&alias) + return alias, err +} + +const getSessionAliasesInGroup = `-- name: GetSessionAliasesInGroup :many +SELECT alias FROM sessions +WHERE group_id = $1 +` + +func (q *Queries) GetSessionAliasesInGroup(ctx context.Context, groupID sql.NullInt64) ([][]byte, error) { + rows, err := q.db.QueryContext(ctx, getSessionAliasesInGroup, groupID) + if err != nil { + return nil, err + } + defer rows.Close() + var items [][]byte + for rows.Next() { + var alias []byte + if err := rows.Scan(&alias); err != nil { + return nil, err + } + items = append(items, alias) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getSessionByAlias = `-- name: GetSessionByAlias :one +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE alias = $1 +` + +func (q *Queries) GetSessionByAlias(ctx context.Context, alias []byte) (Session, error) { + row := q.db.QueryRowContext(ctx, getSessionByAlias, alias) + var i Session + err := row.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ) + return i, err +} + +const getSessionByID = `-- name: GetSessionByID :one +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE id = $1 +` + +func (q *Queries) GetSessionByID(ctx context.Context, id int64) (Session, error) { + row := q.db.QueryRowContext(ctx, getSessionByID, id) + var i Session + err := row.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ) + return i, err +} + +const getSessionByLocalPublicKey = `-- name: GetSessionByLocalPublicKey :one +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE local_public_key = $1 +` + +func (q *Queries) GetSessionByLocalPublicKey(ctx context.Context, localPublicKey []byte) (Session, error) { + row := q.db.QueryRowContext(ctx, getSessionByLocalPublicKey, localPublicKey) + var i Session + err := row.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ) + return i, err +} + +const getSessionFeatureConfigs = `-- name: GetSessionFeatureConfigs :many +SELECT session_id, feature_name, config FROM session_feature_configs +WHERE session_id = $1 +` + +func (q *Queries) GetSessionFeatureConfigs(ctx context.Context, sessionID int64) ([]SessionFeatureConfig, error) { + rows, err := q.db.QueryContext(ctx, getSessionFeatureConfigs, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SessionFeatureConfig + for rows.Next() { + var i SessionFeatureConfig + if err := rows.Scan(&i.SessionID, &i.FeatureName, &i.Config); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getSessionIDByAlias = `-- name: GetSessionIDByAlias :one +SELECT id FROM sessions +WHERE alias = $1 +` + +func (q *Queries) GetSessionIDByAlias(ctx context.Context, alias []byte) (int64, error) { + row := q.db.QueryRowContext(ctx, getSessionIDByAlias, alias) + var id int64 + err := row.Scan(&id) + return id, err +} + +const getSessionMacaroonCaveats = `-- name: GetSessionMacaroonCaveats :many +SELECT id, session_id, caveat_id, verification_id, location FROM session_macaroon_caveats +WHERE session_id = $1 +` + +func (q *Queries) GetSessionMacaroonCaveats(ctx context.Context, sessionID int64) ([]SessionMacaroonCaveat, error) { + rows, err := q.db.QueryContext(ctx, getSessionMacaroonCaveats, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SessionMacaroonCaveat + for rows.Next() { + var i SessionMacaroonCaveat + if err := rows.Scan( + &i.ID, + &i.SessionID, + &i.CaveatID, + &i.VerificationID, + &i.Location, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getSessionMacaroonPermissions = `-- name: GetSessionMacaroonPermissions :many +SELECT id, session_id, entity, action FROM session_macaroon_permissions +WHERE session_id = $1 +` + +func (q *Queries) GetSessionMacaroonPermissions(ctx context.Context, sessionID int64) ([]SessionMacaroonPermission, error) { + rows, err := q.db.QueryContext(ctx, getSessionMacaroonPermissions, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SessionMacaroonPermission + for rows.Next() { + var i SessionMacaroonPermission + if err := rows.Scan( + &i.ID, + &i.SessionID, + &i.Entity, + &i.Action, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getSessionPrivacyFlags = `-- name: GetSessionPrivacyFlags :many +SELECT session_id, flag FROM session_privacy_flags +WHERE session_id = $1 +` + +func (q *Queries) GetSessionPrivacyFlags(ctx context.Context, sessionID int64) ([]SessionPrivacyFlag, error) { + rows, err := q.db.QueryContext(ctx, getSessionPrivacyFlags, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SessionPrivacyFlag + for rows.Next() { + var i SessionPrivacyFlag + if err := rows.Scan(&i.SessionID, &i.Flag); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getSessionsInGroup = `-- name: GetSessionsInGroup :many +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE group_id = $1 +` + +func (q *Queries) GetSessionsInGroup(ctx context.Context, groupID sql.NullInt64) ([]Session, error) { + rows, err := q.db.QueryContext(ctx, getSessionsInGroup, groupID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Session + for rows.Next() { + var i Session + if err := rows.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertSession = `-- name: InsertSession :one +INSERT INTO sessions ( + alias, label, state, type, expiry, created_at, + server_address, dev_server, macaroon_root_key, pairing_secret, + local_private_key, local_public_key, remote_public_key, privacy, group_id, account_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, + $8, $9, $10, $11, $12, + $13, $14, $15, $16 +) RETURNING id +` + +type InsertSessionParams struct { + Alias []byte + Label string + State int16 + Type int16 + Expiry time.Time + CreatedAt time.Time + ServerAddress string + DevServer bool + MacaroonRootKey int64 + PairingSecret []byte + LocalPrivateKey []byte + LocalPublicKey []byte + RemotePublicKey []byte + Privacy bool + GroupID sql.NullInt64 + AccountID sql.NullInt64 +} + +func (q *Queries) InsertSession(ctx context.Context, arg InsertSessionParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertSession, + arg.Alias, + arg.Label, + arg.State, + arg.Type, + arg.Expiry, + arg.CreatedAt, + arg.ServerAddress, + arg.DevServer, + arg.MacaroonRootKey, + arg.PairingSecret, + arg.LocalPrivateKey, + arg.LocalPublicKey, + arg.RemotePublicKey, + arg.Privacy, + arg.GroupID, + arg.AccountID, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertSessionFeatureConfig = `-- name: InsertSessionFeatureConfig :exec +INSERT INTO session_feature_configs ( + session_id, feature_name, config +) VALUES ( + $1, $2, $3 +) +` + +type InsertSessionFeatureConfigParams struct { + SessionID int64 + FeatureName string + Config []byte +} + +func (q *Queries) InsertSessionFeatureConfig(ctx context.Context, arg InsertSessionFeatureConfigParams) error { + _, err := q.db.ExecContext(ctx, insertSessionFeatureConfig, arg.SessionID, arg.FeatureName, arg.Config) + return err +} + +const insertSessionMacaroonCaveat = `-- name: InsertSessionMacaroonCaveat :exec +INSERT INTO session_macaroon_caveats ( + session_id, caveat_id, verification_id, location +) VALUES ( + $1, $2, $3, $4 +) +` + +type InsertSessionMacaroonCaveatParams struct { + SessionID int64 + CaveatID []byte + VerificationID []byte + Location sql.NullString +} + +func (q *Queries) InsertSessionMacaroonCaveat(ctx context.Context, arg InsertSessionMacaroonCaveatParams) error { + _, err := q.db.ExecContext(ctx, insertSessionMacaroonCaveat, + arg.SessionID, + arg.CaveatID, + arg.VerificationID, + arg.Location, + ) + return err +} + +const insertSessionMacaroonPermission = `-- name: InsertSessionMacaroonPermission :exec +INSERT INTO session_macaroon_permissions ( + session_id, entity, action +) VALUES ( + $1, $2, $3 +) +` + +type InsertSessionMacaroonPermissionParams struct { + SessionID int64 + Entity string + Action string +} + +func (q *Queries) InsertSessionMacaroonPermission(ctx context.Context, arg InsertSessionMacaroonPermissionParams) error { + _, err := q.db.ExecContext(ctx, insertSessionMacaroonPermission, arg.SessionID, arg.Entity, arg.Action) + return err +} + +const insertSessionPrivacyFlag = `-- name: InsertSessionPrivacyFlag :exec +INSERT INTO session_privacy_flags ( + session_id, flag +) VALUES ( + $1, $2 +) +` + +type InsertSessionPrivacyFlagParams struct { + SessionID int64 + Flag int32 +} + +func (q *Queries) InsertSessionPrivacyFlag(ctx context.Context, arg InsertSessionPrivacyFlagParams) error { + _, err := q.db.ExecContext(ctx, insertSessionPrivacyFlag, arg.SessionID, arg.Flag) + return err +} + +const listSessions = `-- name: ListSessions :many +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +ORDER BY created_at +` + +func (q *Queries) ListSessions(ctx context.Context) ([]Session, error) { + rows, err := q.db.QueryContext(ctx, listSessions) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Session + for rows.Next() { + var i Session + if err := rows.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSessionsByState = `-- name: ListSessionsByState :many +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE state = $1 +ORDER BY created_at +` + +func (q *Queries) ListSessionsByState(ctx context.Context, state int16) ([]Session, error) { + rows, err := q.db.QueryContext(ctx, listSessionsByState, state) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Session + for rows.Next() { + var i Session + if err := rows.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSessionsByType = `-- name: ListSessionsByType :many +SELECT id, alias, label, state, type, expiry, created_at, revoked_at, server_address, dev_server, macaroon_root_key, pairing_secret, local_private_key, local_public_key, remote_public_key, privacy, account_id, group_id FROM sessions +WHERE type = $1 +ORDER BY created_at +` + +func (q *Queries) ListSessionsByType(ctx context.Context, type_ int16) ([]Session, error) { + rows, err := q.db.QueryContext(ctx, listSessionsByType, type_) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Session + for rows.Next() { + var i Session + if err := rows.Scan( + &i.ID, + &i.Alias, + &i.Label, + &i.State, + &i.Type, + &i.Expiry, + &i.CreatedAt, + &i.RevokedAt, + &i.ServerAddress, + &i.DevServer, + &i.MacaroonRootKey, + &i.PairingSecret, + &i.LocalPrivateKey, + &i.LocalPublicKey, + &i.RemotePublicKey, + &i.Privacy, + &i.AccountID, + &i.GroupID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setSessionGroupID = `-- name: SetSessionGroupID :exec +UPDATE sessions +SET group_id = $1 +WHERE id = $2 +` + +type SetSessionGroupIDParams struct { + GroupID sql.NullInt64 + ID int64 +} + +func (q *Queries) SetSessionGroupID(ctx context.Context, arg SetSessionGroupIDParams) error { + _, err := q.db.ExecContext(ctx, setSessionGroupID, arg.GroupID, arg.ID) + return err +} + +const setSessionRemotePublicKey = `-- name: SetSessionRemotePublicKey :exec +UPDATE sessions +SET remote_public_key = $1 +WHERE id = $2 +` + +type SetSessionRemotePublicKeyParams struct { + RemotePublicKey []byte + ID int64 +} + +func (q *Queries) SetSessionRemotePublicKey(ctx context.Context, arg SetSessionRemotePublicKeyParams) error { + _, err := q.db.ExecContext(ctx, setSessionRemotePublicKey, arg.RemotePublicKey, arg.ID) + return err +} + +const setSessionRevokedAt = `-- name: SetSessionRevokedAt :exec +UPDATE sessions +SET revoked_at = $1 +WHERE id = $2 +` + +type SetSessionRevokedAtParams struct { + RevokedAt sql.NullTime + ID int64 +} + +func (q *Queries) SetSessionRevokedAt(ctx context.Context, arg SetSessionRevokedAtParams) error { + _, err := q.db.ExecContext(ctx, setSessionRevokedAt, arg.RevokedAt, arg.ID) + return err +} + +const updateSessionState = `-- name: UpdateSessionState :exec +UPDATE sessions +SET state = $1 +WHERE id = $2 +` + +type UpdateSessionStateParams struct { + State int16 + ID int64 +} + +func (q *Queries) UpdateSessionState(ctx context.Context, arg UpdateSessionStateParams) error { + _, err := q.db.ExecContext(ctx, updateSessionState, arg.State, arg.ID) + return err +} From 0355557ddee3e99f470b2a2545b158673ffd0077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 27 Aug 2025 15:29:55 +0200 Subject: [PATCH 14/26] sqlcmig6: add `CustomQueries` to `Queries` In order to be able to use the custom queries with the `sqlcmig6.Queries`, we assert at compile time that `sqlcmig6.Queries` implements the `CustomQueries` interface. --- db/sqlcmig6/db_custom.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/sqlcmig6/db_custom.go b/db/sqlcmig6/db_custom.go index e128cb290..e331e48fd 100644 --- a/db/sqlcmig6/db_custom.go +++ b/db/sqlcmig6/db_custom.go @@ -46,3 +46,5 @@ type CustomQueries interface { // Backend returns the type of the database backend used. Backend() sqldb.BackendType } + +var _ CustomQueries = (*Queries)(nil) From 0f3b78b62ab86e6afec52e861497a3a7464ea570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 27 Aug 2025 15:29:16 +0200 Subject: [PATCH 15/26] sqlcmig6: add transaction executor for Queries This commit adds a helper struct that creates a `sqldb/v2` transaction executor that wraps the `sqlcmig6.Queries` type. --- db/sqlcmig6/db.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/db/sqlcmig6/db.go b/db/sqlcmig6/db.go index 82ff72dd8..01c374d05 100644 --- a/db/sqlcmig6/db.go +++ b/db/sqlcmig6/db.go @@ -3,6 +3,7 @@ package sqlcmig6 import ( "context" "database/sql" + "github.com/lightningnetwork/lnd/sqldb/v2" ) type DBTX interface { @@ -25,3 +26,23 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { db: tx, } } + +type TxExecutor[T sqldb.BaseQuerier] struct { + *sqldb.TransactionExecutor[T] + + *Queries +} + +func NewTxExecutor(baseDB *sqldb.BaseDB, + queries *Queries) *TxExecutor[*Queries] { + + executor := sqldb.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) *Queries { + return queries.WithTx(tx) + }, + ) + return &TxExecutor[*Queries]{ + TransactionExecutor: executor, + Queries: queries, + } +} From 310a1b50ebb00b72fae9db2b777a59f0ba6807e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 01:49:37 +0200 Subject: [PATCH 16/26] accounts: use `sqlcmig6` for kvdb to sql migration This commit updates the accounts package to use the new `sqlcmig6` package for kvdb to SQL migration. --- accounts/sql_migration.go | 87 +++++++++++++++++++++++++++++----- accounts/sql_migration_test.go | 10 ++-- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/accounts/sql_migration.go b/accounts/sql_migration.go index c36b51c6f..0b8747c0f 100644 --- a/accounts/sql_migration.go +++ b/accounts/sql_migration.go @@ -11,8 +11,11 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" "github.com/pmezard/go-difflib/difflib" ) @@ -27,7 +30,7 @@ var ( // the KV database to the SQL database. The migration is done in a single // transaction to ensure that all accounts are migrated or none at all. func MigrateAccountStoreToSQL(ctx context.Context, kvStore kvdb.Backend, - tx SQLQueries) error { + tx *sqlcmig6.Queries) error { log.Infof("Starting migration of the KV accounts store to SQL") @@ -50,7 +53,7 @@ func MigrateAccountStoreToSQL(ctx context.Context, kvStore kvdb.Backend, // to the SQL database. The migration is done in a single transaction to ensure // that all accounts are migrated or none at all. func migrateAccountsToSQL(ctx context.Context, kvStore kvdb.Backend, - tx SQLQueries) error { + tx *sqlcmig6.Queries) error { log.Infof("Starting migration of accounts from KV to SQL") @@ -68,7 +71,7 @@ func migrateAccountsToSQL(ctx context.Context, kvStore kvdb.Backend, kvAccount.ID, err) } - migratedAccount, err := getAndMarshalAccount( + migratedAccount, err := getAndMarshalMig6Account( ctx, tx, migratedAccountID, ) if err != nil { @@ -151,17 +154,79 @@ func getBBoltAccounts(db kvdb.Backend) ([]*OffChainBalanceAccount, error) { return accounts, nil } +// getAndMarshalAccount retrieves the account with the given ID. If the account +// cannot be found, then ErrAccNotFound is returned. +func getAndMarshalMig6Account(ctx context.Context, db *sqlcmig6.Queries, + id int64) (*OffChainBalanceAccount, error) { + + dbAcct, err := db.GetAccount(ctx, id) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrAccNotFound + } else if err != nil { + return nil, err + } + + return marshalDBMig6Account(ctx, db, dbAcct) +} + +func marshalDBMig6Account(ctx context.Context, db *sqlcmig6.Queries, + dbAcct sqlcmig6.Account) (*OffChainBalanceAccount, error) { + + alias, err := AccountIDFromInt64(dbAcct.Alias) + if err != nil { + return nil, err + } + + account := &OffChainBalanceAccount{ + ID: alias, + Type: AccountType(dbAcct.Type), + InitialBalance: lnwire.MilliSatoshi(dbAcct.InitialBalanceMsat), + CurrentBalance: dbAcct.CurrentBalanceMsat, + LastUpdate: dbAcct.LastUpdated.UTC(), + ExpirationDate: dbAcct.Expiration.UTC(), + Invoices: make(AccountInvoices), + Payments: make(AccountPayments), + Label: dbAcct.Label.String, + } + + invoices, err := db.ListAccountInvoices(ctx, dbAcct.ID) + if err != nil { + return nil, err + } + for _, invoice := range invoices { + var hash lntypes.Hash + copy(hash[:], invoice.Hash) + account.Invoices[hash] = struct{}{} + } + + payments, err := db.ListAccountPayments(ctx, dbAcct.ID) + if err != nil { + return nil, err + } + + for _, payment := range payments { + var hash lntypes.Hash + copy(hash[:], payment.Hash) + account.Payments[hash] = &PaymentEntry{ + Status: lnrpc.Payment_PaymentStatus(payment.Status), + FullAmount: lnwire.MilliSatoshi(payment.FullAmountMsat), + } + } + + return account, nil +} + // migrateSingleAccountToSQL runs the migration for a single account from the // KV database to the SQL database. func migrateSingleAccountToSQL(ctx context.Context, - tx SQLQueries, account *OffChainBalanceAccount) (int64, error) { + tx *sqlcmig6.Queries, account *OffChainBalanceAccount) (int64, error) { accountAlias, err := account.ID.ToInt64() if err != nil { return 0, err } - insertAccountParams := sqlc.InsertAccountParams{ + insertAccountParams := sqlcmig6.InsertAccountParams{ Type: int16(account.Type), InitialBalanceMsat: int64(account.InitialBalance), CurrentBalanceMsat: account.CurrentBalance, @@ -180,7 +245,7 @@ func migrateSingleAccountToSQL(ctx context.Context, } for hash := range account.Invoices { - addInvoiceParams := sqlc.AddAccountInvoiceParams{ + addInvoiceParams := sqlcmig6.AddAccountInvoiceParams{ AccountID: sqlId, Hash: hash[:], } @@ -192,7 +257,7 @@ func migrateSingleAccountToSQL(ctx context.Context, } for hash, paymentEntry := range account.Payments { - upsertPaymentParams := sqlc.UpsertAccountPaymentParams{ + upsertPaymentParams := sqlcmig6.UpsertAccountPaymentParams{ AccountID: sqlId, Hash: hash[:], Status: int16(paymentEntry.Status), @@ -211,7 +276,7 @@ func migrateSingleAccountToSQL(ctx context.Context, // migrateAccountsIndicesToSQL runs the migration for the account indices from // the KV database to the SQL database. func migrateAccountsIndicesToSQL(ctx context.Context, kvStore kvdb.Backend, - tx SQLQueries) error { + tx *sqlcmig6.Queries) error { log.Infof("Starting migration of accounts indices from KV to SQL") @@ -233,7 +298,7 @@ func migrateAccountsIndicesToSQL(ctx context.Context, kvStore kvdb.Backend, settleIndexName, settleIndex) } - setAddIndexParams := sqlc.SetAccountIndexParams{ + setAddIndexParams := sqlcmig6.SetAccountIndexParams{ Name: addIndexName, Value: int64(addIndex), } @@ -243,7 +308,7 @@ func migrateAccountsIndicesToSQL(ctx context.Context, kvStore kvdb.Backend, return err } - setSettleIndexParams := sqlc.SetAccountIndexParams{ + setSettleIndexParams := sqlcmig6.SetAccountIndexParams{ Name: settleIndexName, Value: int64(settleIndex), } diff --git a/accounts/sql_migration_test.go b/accounts/sql_migration_test.go index ef9ff82f2..4f3ae1741 100644 --- a/accounts/sql_migration_test.go +++ b/accounts/sql_migration_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lnrpc" @@ -35,7 +35,7 @@ func TestAccountStoreMigration(t *testing.T) { } makeSQLDB := func(t *testing.T) (*SQLStore, - *SQLQueriesExecutor[SQLQueries]) { + *sqlcmig6.TxExecutor[*sqlcmig6.Queries]) { testDBStore := NewTestDB(t, clock) @@ -44,9 +44,9 @@ func TestAccountStoreMigration(t *testing.T) { baseDB := store.BaseDB - queries := sqlc.NewForType(baseDB, baseDB.BackendType) + queries := sqlcmig6.NewForType(baseDB, baseDB.BackendType) - return store, NewSQLQueriesExecutor(baseDB, queries) + return store, sqlcmig6.NewTxExecutor(baseDB, queries) } assertMigrationResults := func(t *testing.T, sqlStore *SQLStore, @@ -334,7 +334,7 @@ func TestAccountStoreMigration(t *testing.T) { // Perform the migration. err = txEx.ExecTx( ctx, sqldb.WriteTxOpt(), - func(tx SQLQueries) error { + func(tx *sqlcmig6.Queries) error { return MigrateAccountStoreToSQL( ctx, kvStore.db, tx, ) From 91432b0cd1a725e9c26eb1ad972e6eae13466efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 01:56:25 +0200 Subject: [PATCH 17/26] session: use `sqlcmig6` for kvdb to sql migration This commit updates the session package to use the new `sqlcmig6` package for kvdb to SQL migration. --- session/sql_migration.go | 235 ++++++++++++++++++++++++++++++---- session/sql_migration_test.go | 10 +- 2 files changed, 217 insertions(+), 28 deletions(-) diff --git a/session/sql_migration.go b/session/sql_migration.go index 0288a03cb..4cae41edb 100644 --- a/session/sql_migration.go +++ b/session/sql_migration.go @@ -10,12 +10,17 @@ import ( "sort" "time" + "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" + "github.com/lightninglabs/lightning-node-connect/mailbox" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" + "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/sqldb" "github.com/pmezard/go-difflib/difflib" "go.etcd.io/bbolt" + "gopkg.in/macaroon-bakery.v2/bakery" + "gopkg.in/macaroon.v2" ) var ( @@ -32,7 +37,7 @@ var ( // NOTE: As sessions may contain linked accounts, the accounts sql migration // MUST be run prior to this migration. func MigrateSessionStoreToSQL(ctx context.Context, kvStore *bbolt.DB, - tx SQLQueries) error { + tx *sqlcmig6.Queries) error { log.Infof("Starting migration of the KV sessions store to SQL") @@ -119,7 +124,7 @@ func getBBoltSessions(db *bbolt.DB) ([]*Session, error) { // from the KV database to the SQL database, and validates that the migrated // sessions match the original sessions. func migrateSessionsToSQLAndValidate(ctx context.Context, - tx SQLQueries, kvSessions []*Session) error { + tx *sqlcmig6.Queries, kvSessions []*Session) error { for _, kvSession := range kvSessions { err := migrateSingleSessionToSQL(ctx, tx, kvSession) @@ -128,18 +133,9 @@ func migrateSessionsToSQLAndValidate(ctx context.Context, kvSession.ID, err) } - // Validate that the session was correctly migrated and matches - // the original session in the kv store. - sqlSess, err := tx.GetSessionByAlias(ctx, kvSession.ID[:]) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - err = ErrSessionNotFound - } - return fmt.Errorf("unable to get migrated session "+ - "from sql store: %w", err) - } - - migratedSession, err := unmarshalSession(ctx, tx, sqlSess) + migratedSession, err := getAndUnmarshalSession( + ctx, tx, kvSession.ID[:], + ) if err != nil { return fmt.Errorf("unable to unmarshal migrated "+ "session: %w", err) @@ -173,12 +169,205 @@ func migrateSessionsToSQLAndValidate(ctx context.Context, return nil } +func getAndUnmarshalSession(ctx context.Context, + tx *sqlcmig6.Queries, legacyID []byte) (*Session, error) { + + // Validate that the session was correctly migrated and matches + // the original session in the kv store. + sqlSess, err := tx.GetSessionByAlias(ctx, legacyID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + err = ErrSessionNotFound + } + + return nil, fmt.Errorf("unable to get migrated session "+ + "from sql store: %w", err) + } + + migratedSession, err := unmarshalMig6Session(ctx, tx, sqlSess) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal migrated "+ + "session: %w", err) + } + + return migratedSession, nil +} + +func unmarshalMig6Session(ctx context.Context, db *sqlcmig6.Queries, + dbSess sqlcmig6.Session) (*Session, error) { + + var legacyGroupID ID + if dbSess.GroupID.Valid { + groupID, err := db.GetAliasBySessionID( + ctx, dbSess.GroupID.Int64, + ) + if err != nil { + return nil, fmt.Errorf("unable to get legacy group "+ + "Alias: %v", err) + } + + legacyGroupID, err = IDFromBytes(groupID) + if err != nil { + return nil, fmt.Errorf("unable to get legacy Alias: %v", + err) + } + } + + var acctAlias fn.Option[accounts.AccountID] + if dbSess.AccountID.Valid { + account, err := db.GetAccount(ctx, dbSess.AccountID.Int64) + if err != nil { + return nil, fmt.Errorf("unable to get account: %v", err) + } + + accountAlias, err := accounts.AccountIDFromInt64(account.Alias) + if err != nil { + return nil, fmt.Errorf("unable to get account ID: %v", err) + } + acctAlias = fn.Some(accountAlias) + } + + legacyID, err := IDFromBytes(dbSess.Alias) + if err != nil { + return nil, fmt.Errorf("unable to get legacy Alias: %v", err) + } + + var revokedAt time.Time + if dbSess.RevokedAt.Valid { + revokedAt = dbSess.RevokedAt.Time + } + + localPriv, localPub := btcec.PrivKeyFromBytes(dbSess.LocalPrivateKey) + + var remotePub *btcec.PublicKey + if len(dbSess.RemotePublicKey) != 0 { + remotePub, err = btcec.ParsePubKey(dbSess.RemotePublicKey) + if err != nil { + return nil, fmt.Errorf("unable to parse remote "+ + "public key: %v", err) + } + } + + // Get the macaroon permissions if they exist. + perms, err := db.GetSessionMacaroonPermissions(ctx, dbSess.ID) + if err != nil { + return nil, fmt.Errorf("unable to get macaroon "+ + "permissions: %v", err) + } + + // Get the macaroon caveats if they exist. + caveats, err := db.GetSessionMacaroonCaveats(ctx, dbSess.ID) + if err != nil { + return nil, fmt.Errorf("unable to get macaroon "+ + "caveats: %v", err) + } + + var macRecipe *MacaroonRecipe + if perms != nil || caveats != nil { + macRecipe = &MacaroonRecipe{ + Permissions: unmarshalMig6MacPerms(perms), + Caveats: unmarshalMig6MacCaveats(caveats), + } + } + + // Get the feature configs if they exist. + featureConfigs, err := db.GetSessionFeatureConfigs(ctx, dbSess.ID) + if err != nil { + return nil, fmt.Errorf("unable to get feature configs: %v", err) + } + + var featureCfgs *FeaturesConfig + if featureConfigs != nil { + featureCfgs = unmarshalMig6FeatureConfigs(featureConfigs) + } + + // Get the privacy flags if they exist. + privacyFlags, err := db.GetSessionPrivacyFlags(ctx, dbSess.ID) + if err != nil { + return nil, fmt.Errorf("unable to get privacy flags: %v", err) + } + + var privFlags PrivacyFlags + if privacyFlags != nil { + privFlags = unmarshalMig6PrivacyFlags(privacyFlags) + } + + var pairingSecret [mailbox.NumPassphraseEntropyBytes]byte + copy(pairingSecret[:], dbSess.PairingSecret) + + return &Session{ + ID: legacyID, + Label: dbSess.Label, + State: State(dbSess.State), + Type: Type(dbSess.Type), + Expiry: dbSess.Expiry, + CreatedAt: dbSess.CreatedAt, + RevokedAt: revokedAt, + ServerAddr: dbSess.ServerAddress, + DevServer: dbSess.DevServer, + MacaroonRootKey: uint64(dbSess.MacaroonRootKey), + PairingSecret: pairingSecret, + LocalPrivateKey: localPriv, + LocalPublicKey: localPub, + RemotePublicKey: remotePub, + WithPrivacyMapper: dbSess.Privacy, + GroupID: legacyGroupID, + PrivacyFlags: privFlags, + MacaroonRecipe: macRecipe, + FeatureConfig: featureCfgs, + AccountID: acctAlias, + }, nil +} + +func unmarshalMig6MacPerms(dbPerms []sqlcmig6.SessionMacaroonPermission) []bakery.Op { + ops := make([]bakery.Op, len(dbPerms)) + for i, dbPerm := range dbPerms { + ops[i] = bakery.Op{ + Entity: dbPerm.Entity, + Action: dbPerm.Action, + } + } + + return ops +} + +func unmarshalMig6MacCaveats(dbCaveats []sqlcmig6.SessionMacaroonCaveat) []macaroon.Caveat { + caveats := make([]macaroon.Caveat, len(dbCaveats)) + for i, dbCaveat := range dbCaveats { + caveats[i] = macaroon.Caveat{ + Id: dbCaveat.CaveatID, + VerificationId: dbCaveat.VerificationID, + Location: dbCaveat.Location.String, + } + } + + return caveats +} + +func unmarshalMig6FeatureConfigs(dbConfigs []sqlcmig6.SessionFeatureConfig) *FeaturesConfig { + configs := make(FeaturesConfig, len(dbConfigs)) + for _, dbConfig := range dbConfigs { + configs[dbConfig.FeatureName] = dbConfig.Config + } + + return &configs +} + +func unmarshalMig6PrivacyFlags(dbFlags []sqlcmig6.SessionPrivacyFlag) PrivacyFlags { + flags := make(PrivacyFlags, len(dbFlags)) + for i, dbFlag := range dbFlags { + flags[i] = PrivacyFlag(dbFlag.Flag) + } + + return flags +} + // migrateSingleSessionToSQL runs the migration for a single session from the // KV database to the SQL database. Note that if the session links to an // account, the linked accounts store MUST have been migrated before that // session is migrated. func migrateSingleSessionToSQL(ctx context.Context, - tx SQLQueries, session *Session) error { + tx *sqlcmig6.Queries, session *Session) error { var ( acctID sql.NullInt64 @@ -214,7 +403,7 @@ func migrateSingleSessionToSQL(ctx context.Context, } // Proceed to insert the session into the sql db. - sqlId, err := tx.InsertSession(ctx, sqlc.InsertSessionParams{ + sqlId, err := tx.InsertSession(ctx, sqlcmig6.InsertSessionParams{ Alias: session.ID[:], Label: session.Label, State: int16(session.State), @@ -240,7 +429,7 @@ func migrateSingleSessionToSQL(ctx context.Context, // has been created. if !session.RevokedAt.IsZero() { err = tx.SetSessionRevokedAt( - ctx, sqlc.SetSessionRevokedAtParams{ + ctx, sqlcmig6.SetSessionRevokedAtParams{ ID: sqlId, RevokedAt: sqldb.SQLTime( session.RevokedAt.UTC(), @@ -266,7 +455,7 @@ func migrateSingleSessionToSQL(ctx context.Context, } // Now lets set the group ID for the session. - err = tx.SetSessionGroupID(ctx, sqlc.SetSessionGroupIDParams{ + err = tx.SetSessionGroupID(ctx, sqlcmig6.SetSessionGroupIDParams{ ID: sqlId, GroupID: sqldb.SQLInt64(groupID), }) @@ -280,7 +469,7 @@ func migrateSingleSessionToSQL(ctx context.Context, // We start by inserting the macaroon permissions. for _, sessionPerm := range session.MacaroonRecipe.Permissions { err = tx.InsertSessionMacaroonPermission( - ctx, sqlc.InsertSessionMacaroonPermissionParams{ + ctx, sqlcmig6.InsertSessionMacaroonPermissionParams{ SessionID: sqlId, Entity: sessionPerm.Entity, Action: sessionPerm.Action, @@ -294,7 +483,7 @@ func migrateSingleSessionToSQL(ctx context.Context, // Next we insert the macaroon caveats. for _, caveat := range session.MacaroonRecipe.Caveats { err = tx.InsertSessionMacaroonCaveat( - ctx, sqlc.InsertSessionMacaroonCaveatParams{ + ctx, sqlcmig6.InsertSessionMacaroonCaveatParams{ SessionID: sqlId, CaveatID: caveat.Id, VerificationID: caveat.VerificationId, @@ -313,7 +502,7 @@ func migrateSingleSessionToSQL(ctx context.Context, if session.FeatureConfig != nil { for featureName, config := range *session.FeatureConfig { err = tx.InsertSessionFeatureConfig( - ctx, sqlc.InsertSessionFeatureConfigParams{ + ctx, sqlcmig6.InsertSessionFeatureConfigParams{ SessionID: sqlId, FeatureName: featureName, Config: config, @@ -328,7 +517,7 @@ func migrateSingleSessionToSQL(ctx context.Context, // Finally we insert the privacy flags. for _, privacyFlag := range session.PrivacyFlags { err = tx.InsertSessionPrivacyFlag( - ctx, sqlc.InsertSessionPrivacyFlagParams{ + ctx, sqlcmig6.InsertSessionPrivacyFlagParams{ SessionID: sqlId, Flag: int32(privacyFlag), }, diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go index 8fc416f6e..624713dec 100644 --- a/session/sql_migration_test.go +++ b/session/sql_migration_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/macaroons" @@ -37,7 +37,7 @@ func TestSessionsStoreMigration(t *testing.T) { } makeSQLDB := func(t *testing.T, acctStore accounts.Store) (*SQLStore, - *SQLQueriesExecutor[SQLQueries]) { + *sqlcmig6.TxExecutor[*sqlcmig6.Queries]) { // Create a sql store with a linked account store. testDBStore := NewTestDBWithAccounts(t, clock, acctStore) @@ -47,9 +47,9 @@ func TestSessionsStoreMigration(t *testing.T) { baseDB := store.BaseDB - queries := sqlc.NewForType(baseDB, baseDB.BackendType) + queries := sqlcmig6.NewForType(baseDB, baseDB.BackendType) - return store, NewSQLQueriesExecutor(baseDB, queries) + return store, sqlcmig6.NewTxExecutor(baseDB, queries) } // assertMigrationResults asserts that the sql store contains the @@ -366,7 +366,7 @@ func TestSessionsStoreMigration(t *testing.T) { err = txEx.ExecTx( ctx, sqldb.WriteTxOpt(), - func(tx SQLQueries) error { + func(tx *sqlcmig6.Queries) error { return MigrateSessionStoreToSQL( ctx, kvStore.DB, tx, ) From 6693fbceebe38d3b339d0fd936bbf645e1832cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 28 Aug 2025 02:00:27 +0200 Subject: [PATCH 18/26] firewalldb: use queries to assert migration results As the firewalldb package kvdb to sql migration tests creates `sqlc` models to assert the migration results, we will need to update those call sites to instead use the `sqlcmig6` models instead, in order to be compatible with the `sqlcmig6.Queries` queries. However, since we can't update the `SQLDB` methods to use `sqlcmig6` models as params, we need to update the test code assertion to instead use the `sqlc.Queries` object directly instead of the `SQLDB` object. This makes it easy to swap that `sqlc.Queries` object to a `sqlcmig6.Queries` object in the commit that updates the firewalldb package to use the `sqlcmig6` package for the kvdb to sql migration. --- firewalldb/sql_migration_test.go | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 46fae50c9..08d04fc02 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -74,7 +74,7 @@ func TestFirewallDBMigration(t *testing.T) { // The assertKvStoreMigrationResults function will currently assert that // the migrated kv stores entries in the SQLDB match the original kv // stores entries in the BoltDB. - assertKvStoreMigrationResults := func(t *testing.T, store *SQLDB, + assertKvStoreMigrationResults := func(t *testing.T, store *sqlc.Queries, kvEntries []*kvEntry) { var ( @@ -87,9 +87,7 @@ func TestFirewallDBMigration(t *testing.T) { getRuleID := func(ruleName string) int64 { ruleID, ok := ruleIDs[ruleName] if !ok { - ruleID, err = store.db.GetRuleID( - ctx, ruleName, - ) + ruleID, err = store.GetRuleID(ctx, ruleName) require.NoError(t, err) ruleIDs[ruleName] = ruleID @@ -101,7 +99,7 @@ func TestFirewallDBMigration(t *testing.T) { getGroupID := func(groupAlias []byte) int64 { groupID, ok := groupIDs[string(groupAlias)] if !ok { - groupID, err = store.db.GetSessionIDByAlias( + groupID, err = store.GetSessionIDByAlias( ctx, groupAlias, ) require.NoError(t, err) @@ -115,7 +113,7 @@ func TestFirewallDBMigration(t *testing.T) { getFeatureID := func(featureName string) int64 { featureID, ok := featureIDs[featureName] if !ok { - featureID, err = store.db.GetFeatureID( + featureID, err = store.GetFeatureID( ctx, featureName, ) require.NoError(t, err) @@ -126,10 +124,10 @@ func TestFirewallDBMigration(t *testing.T) { return featureID } - // First we extract all migrated kv entries from the SQLDB, + // First we extract all migrated kv entries from the store, // in order to be able to compare them to the original kv // entries, to ensure that the migration was successful. - sqlKvEntries, err := store.db.ListAllKVStoresRecords(ctx) + sqlKvEntries, err := store.ListAllKVStoresRecords(ctx) require.NoError(t, err) require.Equal(t, len(kvEntries), len(sqlKvEntries)) @@ -145,7 +143,7 @@ func TestFirewallDBMigration(t *testing.T) { ruleID := getRuleID(entry.ruleName) if entry.groupAlias.IsNone() { - sqlVal, err := store.db.GetGlobalKVStoreRecord( + sqlVal, err := store.GetGlobalKVStoreRecord( ctx, sqlc.GetGlobalKVStoreRecordParams{ Key: entry.key, @@ -163,7 +161,7 @@ func TestFirewallDBMigration(t *testing.T) { groupAlias := entry.groupAlias.UnwrapOrFail(t) groupID := getGroupID(groupAlias[:]) - v, err := store.db.GetGroupKVStoreRecord( + v, err := store.GetGroupKVStoreRecord( ctx, sqlc.GetGroupKVStoreRecordParams{ Key: entry.key, @@ -188,7 +186,7 @@ func TestFirewallDBMigration(t *testing.T) { entry.featureName.UnwrapOrFail(t), ) - sqlVal, err := store.db.GetFeatureKVStoreRecord( + sqlVal, err := store.GetFeatureKVStoreRecord( ctx, sqlc.GetFeatureKVStoreRecordParams{ Key: entry.key, @@ -219,14 +217,14 @@ func TestFirewallDBMigration(t *testing.T) { // BoltDB. It also asserts that the SQL DB does not contain any other // privacy pairs than the expected ones. assertPrivacyMapperMigrationResults := func(t *testing.T, - sqlStore *SQLDB, privPairs privacyPairs) { + sqlStore *sqlc.Queries, privPairs privacyPairs) { var totalExpectedPairs, totalPairs int // First assert that the SQLDB contains the expected privacy // pairs. for groupID, groupPairs := range privPairs { - storePairs, err := sqlStore.db.GetAllPrivacyPairs( + storePairs, err := sqlStore.GetAllPrivacyPairs( ctx, groupID, ) require.NoError(t, err) @@ -246,14 +244,13 @@ func TestFirewallDBMigration(t *testing.T) { } } - // Then assert that SQLDB doesn't contain any other privacy + // Then assert that store doesn't contain any other privacy // pairs than the expected ones. - queries := sqlc.NewForType(sqlStore, sqlStore.BackendType) - sessions, err := queries.ListSessions(ctx) + sessions, err := sqlStore.ListSessions(ctx) require.NoError(t, err) for _, dbSession := range sessions { - sessionPairs, err := sqlStore.db.GetAllPrivacyPairs( + sessionPairs, err := sqlStore.GetAllPrivacyPairs( ctx, dbSession.ID, ) if errors.Is(err, sql.ErrNoRows) { @@ -272,7 +269,7 @@ func TestFirewallDBMigration(t *testing.T) { // The assertMigrationResults asserts that the migrated entries in the // firewall SQLDB match the expected results which should represent the // original entries in the BoltDB. - assertMigrationResults := func(t *testing.T, sqlStore *SQLDB, + assertMigrationResults := func(t *testing.T, sqlStore *sqlc.Queries, expRes *expectedResult) { // Assert that the kv store migration results match the expected @@ -400,7 +397,10 @@ func TestFirewallDBMigration(t *testing.T) { require.NoError(t, err) // Assert migration results. - assertMigrationResults(t, sqlStore, entries) + queries := sqlc.NewForType( + sqlStore, sqlStore.BackendType, + ) + assertMigrationResults(t, queries, entries) }) } } From 2c246cecc328cd399a4a44409846f9a7b044570e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 02:16:16 +0200 Subject: [PATCH 19/26] firewalldb: use `sqlcmig6` for kvdb to sql migration This commit updates the firewalldb package to use the new `sqlcmig6` package for kvdb to SQL migration. --- firewalldb/sql_migration.go | 39 ++++++++++++++++---------------- firewalldb/sql_migration_test.go | 34 ++++++++++++++-------------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/firewalldb/sql_migration.go b/firewalldb/sql_migration.go index 9d870d842..378e44e16 100644 --- a/firewalldb/sql_migration.go +++ b/firewalldb/sql_migration.go @@ -6,8 +6,7 @@ import ( "database/sql" "errors" "fmt" - - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/sqldb" "go.etcd.io/bbolt" @@ -80,7 +79,7 @@ type privacyPairs = map[int64]map[string]string // NOTE: As sessions may contain linked sessions and accounts, the sessions and // accounts sql migration MUST be run prior to this migration. func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB, - sqlTx SQLQueries) error { + sqlTx *sqlcmig6.Queries) error { log.Infof("Starting migration of the rules DB to SQL") @@ -105,7 +104,7 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB, // database to the SQL database. The function also asserts that the // migrated values match the original values in the KV store. func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB, - sqlTx SQLQueries) error { + sqlTx *sqlcmig6.Queries) error { log.Infof("Starting migration of the KV stores to SQL") @@ -367,7 +366,7 @@ func collectKVPairs(bkt *bbolt.Bucket, errorOnBuckets, perm bool, } // insertPair inserts a single key-value pair into the SQL database. -func insertPair(ctx context.Context, tx SQLQueries, +func insertPair(ctx context.Context, tx *sqlcmig6.Queries, entry *kvEntry) (*sqlKvEntry, error) { ruleID, err := tx.GetOrInsertRuleID(ctx, entry.ruleName) @@ -375,7 +374,7 @@ func insertPair(ctx context.Context, tx SQLQueries, return nil, err } - p := sqlc.InsertKVStoreRecordParams{ + p := sqlcmig6.InsertKVStoreRecordParams{ Perm: entry.perm, RuleID: ruleID, EntryKey: entry.key, @@ -427,13 +426,13 @@ func insertPair(ctx context.Context, tx SQLQueries, // getSQLValue retrieves the key value for the given kvEntry from the SQL // database. -func getSQLValue(ctx context.Context, tx SQLQueries, +func getSQLValue(ctx context.Context, tx *sqlcmig6.Queries, entry *sqlKvEntry) ([]byte, error) { switch { case entry.featureID.Valid && entry.groupID.Valid: return tx.GetFeatureKVStoreRecord( - ctx, sqlc.GetFeatureKVStoreRecordParams{ + ctx, sqlcmig6.GetFeatureKVStoreRecordParams{ Perm: entry.perm, RuleID: entry.ruleID, GroupID: entry.groupID, @@ -443,7 +442,7 @@ func getSQLValue(ctx context.Context, tx SQLQueries, ) case entry.groupID.Valid: return tx.GetGroupKVStoreRecord( - ctx, sqlc.GetGroupKVStoreRecordParams{ + ctx, sqlcmig6.GetGroupKVStoreRecordParams{ Perm: entry.perm, RuleID: entry.ruleID, GroupID: entry.groupID, @@ -452,7 +451,7 @@ func getSQLValue(ctx context.Context, tx SQLQueries, ) case !entry.featureID.Valid && !entry.groupID.Valid: return tx.GetGlobalKVStoreRecord( - ctx, sqlc.GetGlobalKVStoreRecordParams{ + ctx, sqlcmig6.GetGlobalKVStoreRecordParams{ Perm: entry.perm, RuleID: entry.ruleID, Key: entry.key, @@ -501,7 +500,7 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool, // from the KV database to the SQL database. The function also asserts that the // migrated values match the original values in the privacy mapper store. func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB, - sqlTx SQLQueries) error { + sqlTx *sqlcmig6.Queries) error { log.Infof("Starting migration of the privacy mapper store to SQL") @@ -536,7 +535,7 @@ func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB, // collectPrivacyPairs collects all privacy pairs from the KV store. func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB, - sqlTx SQLQueries) (privacyPairs, error) { + sqlTx *sqlcmig6.Queries) (privacyPairs, error) { groupPairs := make(privacyPairs) @@ -665,7 +664,7 @@ func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) { } // insertPrivacyPairs inserts the collected privacy pairs into the SQL database. -func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries, +func insertPrivacyPairs(ctx context.Context, sqlTx *sqlcmig6.Queries, pairs privacyPairs) error { for groupId, groupPairs := range pairs { @@ -684,12 +683,12 @@ func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries, // an error if a duplicate pair is found. The function takes a map of real // to pseudo values, where the key is the real value and the value is the // corresponding pseudo value. -func insertGroupPairs(ctx context.Context, sqlTx SQLQueries, groupID int64, - pairs map[string]string) error { +func insertGroupPairs(ctx context.Context, sqlTx *sqlcmig6.Queries, + groupID int64, pairs map[string]string) error { for realVal, pseudoVal := range pairs { err := sqlTx.InsertPrivacyPair( - ctx, sqlc.InsertPrivacyPairParams{ + ctx, sqlcmig6.InsertPrivacyPairParams{ GroupID: groupID, RealVal: realVal, PseudoVal: pseudoVal, @@ -706,7 +705,7 @@ func insertGroupPairs(ctx context.Context, sqlTx SQLQueries, groupID int64, // validatePrivacyPairsMigration validates that the migrated privacy pairs // match the original values in the KV store. -func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries, +func validatePrivacyPairsMigration(ctx context.Context, sqlTx *sqlcmig6.Queries, pairs privacyPairs) error { for groupId, groupPairs := range pairs { @@ -727,12 +726,12 @@ func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries, // for each real value, the pseudo value in the SQL database matches the // original pseudo value, and vice versa. If any mismatch is found, it returns // an error indicating the mismatch. -func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries, +func validateGroupPairsMigration(ctx context.Context, sqlTx *sqlcmig6.Queries, groupID int64, pairs map[string]string) error { for realVal, pseudoVal := range pairs { resPseudoVal, err := sqlTx.GetPseudoForReal( - ctx, sqlc.GetPseudoForRealParams{ + ctx, sqlcmig6.GetPseudoForRealParams{ GroupID: groupID, RealVal: realVal, }, @@ -752,7 +751,7 @@ func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries, } resRealVal, err := sqlTx.GetRealForPseudo( - ctx, sqlc.GetRealForPseudoParams{ + ctx, sqlcmig6.GetRealForPseudoParams{ GroupID: groupID, PseudoVal: pseudoVal, }, diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 08d04fc02..3171bd715 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -10,7 +10,7 @@ import ( "time" "github.com/lightninglabs/lightning-terminal/accounts" - "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" "github.com/lightninglabs/lightning-terminal/session" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/fn" @@ -56,26 +56,26 @@ func TestFirewallDBMigration(t *testing.T) { t.Skipf("Skipping Firewall DB migration test for kvdb build") } - makeSQLDB := func(t *testing.T, sessionsStore session.Store) (*SQLDB, - *SQLQueriesExecutor[SQLQueries]) { + makeSQLDB := func(t *testing.T, sStore session.Store) (*SQLDB, + *sqlcmig6.TxExecutor[*sqlcmig6.Queries]) { - testDBStore := NewTestDBWithSessions(t, sessionsStore, clock) + testDBStore := NewTestDBWithSessions(t, sStore, clock) store, ok := testDBStore.(*SQLDB) require.True(t, ok) baseDB := store.BaseDB - queries := sqlc.NewForType(baseDB, baseDB.BackendType) + queries := sqlcmig6.NewForType(baseDB, baseDB.BackendType) - return store, NewSQLQueriesExecutor(baseDB, queries) + return store, sqlcmig6.NewTxExecutor(baseDB, queries) } // The assertKvStoreMigrationResults function will currently assert that // the migrated kv stores entries in the SQLDB match the original kv // stores entries in the BoltDB. - assertKvStoreMigrationResults := func(t *testing.T, store *sqlc.Queries, - kvEntries []*kvEntry) { + assertKvStoreMigrationResults := func(t *testing.T, + store *sqlcmig6.Queries, kvEntries []*kvEntry) { var ( ruleIDs = make(map[string]int64) @@ -145,7 +145,7 @@ func TestFirewallDBMigration(t *testing.T) { if entry.groupAlias.IsNone() { sqlVal, err := store.GetGlobalKVStoreRecord( ctx, - sqlc.GetGlobalKVStoreRecordParams{ + sqlcmig6.GetGlobalKVStoreRecordParams{ Key: entry.key, Perm: entry.perm, RuleID: ruleID, @@ -163,7 +163,7 @@ func TestFirewallDBMigration(t *testing.T) { v, err := store.GetGroupKVStoreRecord( ctx, - sqlc.GetGroupKVStoreRecordParams{ + sqlcmig6.GetGroupKVStoreRecordParams{ Key: entry.key, Perm: entry.perm, RuleID: ruleID, @@ -188,7 +188,7 @@ func TestFirewallDBMigration(t *testing.T) { sqlVal, err := store.GetFeatureKVStoreRecord( ctx, - sqlc.GetFeatureKVStoreRecordParams{ + sqlcmig6.GetFeatureKVStoreRecordParams{ Key: entry.key, Perm: entry.perm, RuleID: ruleID, @@ -217,7 +217,7 @@ func TestFirewallDBMigration(t *testing.T) { // BoltDB. It also asserts that the SQL DB does not contain any other // privacy pairs than the expected ones. assertPrivacyMapperMigrationResults := func(t *testing.T, - sqlStore *sqlc.Queries, privPairs privacyPairs) { + sqlStore *sqlcmig6.Queries, privPairs privacyPairs) { var totalExpectedPairs, totalPairs int @@ -269,7 +269,7 @@ func TestFirewallDBMigration(t *testing.T) { // The assertMigrationResults asserts that the migrated entries in the // firewall SQLDB match the expected results which should represent the // original entries in the BoltDB. - assertMigrationResults := func(t *testing.T, sqlStore *sqlc.Queries, + assertMigrationResults := func(t *testing.T, sqlStore *sqlcmig6.Queries, expRes *expectedResult) { // Assert that the kv store migration results match the expected @@ -388,7 +388,7 @@ func TestFirewallDBMigration(t *testing.T) { // Perform the migration. err = txEx.ExecTx(ctx, sqldb.WriteTxOpt(), - func(tx SQLQueries) error { + func(tx *sqlcmig6.Queries) error { return MigrateFirewallDBToSQL( ctx, firewallStore.DB, tx, ) @@ -397,7 +397,7 @@ func TestFirewallDBMigration(t *testing.T) { require.NoError(t, err) // Assert migration results. - queries := sqlc.NewForType( + queries := sqlcmig6.NewForType( sqlStore, sqlStore.BackendType, ) assertMigrationResults(t, queries, entries) @@ -795,7 +795,7 @@ func createPrivacyPairs(t *testing.T, ctx context.Context, sessSQLStore, ok := sessionStore.(*session.SQLStore) require.True(t, ok) - queries := sqlc.NewForType(sessSQLStore, sessSQLStore.BackendType) + queries := sqlcmig6.NewForType(sessSQLStore, sessSQLStore.BackendType) for i := range numSessions { sess, err := sessionStore.NewSession( @@ -850,7 +850,7 @@ func randomPrivacyPairs(t *testing.T, ctx context.Context, sessSQLStore, ok := sessionStore.(*session.SQLStore) require.True(t, ok) - queries := sqlc.NewForType(sessSQLStore, sessSQLStore.BackendType) + queries := sqlcmig6.NewForType(sessSQLStore, sessSQLStore.BackendType) for i := range numSessions { sess, err := sessionStore.NewSession( From abb4781df339c04f44741cea1d0ed2a36ca4944a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 28 Aug 2025 02:07:08 +0200 Subject: [PATCH 20/26] firewalldb: refactor execution of mig test assert Since the `assertMigrationResults` now expects a `*sqlcmig6.Queries` instance to be passed directly into the function, there is no need to create a separate *sqlcmig6.Queries instance than the one created when the `*sqlcmig6.TxExecutor` transaction is created. We therefore refactor the execution of the `assertMigrationResults` function to be called in the scope of that transaction. Due to this change, there's also no longer a need to return the `*SQLDB` instance from the `makeSQLDB` helper function. --- firewalldb/sql_migration_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/firewalldb/sql_migration_test.go b/firewalldb/sql_migration_test.go index 3171bd715..680cdd4a9 100644 --- a/firewalldb/sql_migration_test.go +++ b/firewalldb/sql_migration_test.go @@ -56,8 +56,8 @@ func TestFirewallDBMigration(t *testing.T) { t.Skipf("Skipping Firewall DB migration test for kvdb build") } - makeSQLDB := func(t *testing.T, sStore session.Store) (*SQLDB, - *sqlcmig6.TxExecutor[*sqlcmig6.Queries]) { + makeSQLDB := func(t *testing.T, + sStore session.Store) *sqlcmig6.TxExecutor[*sqlcmig6.Queries] { testDBStore := NewTestDBWithSessions(t, sStore, clock) @@ -68,7 +68,7 @@ func TestFirewallDBMigration(t *testing.T) { queries := sqlcmig6.NewForType(baseDB, baseDB.BackendType) - return store, sqlcmig6.NewTxExecutor(baseDB, queries) + return sqlcmig6.NewTxExecutor(baseDB, queries) } // The assertKvStoreMigrationResults function will currently assert that @@ -384,23 +384,25 @@ func TestFirewallDBMigration(t *testing.T) { // Create the SQL store that we will migrate the data // to. - sqlStore, txEx := makeSQLDB(t, sessionsStore) + txEx := makeSQLDB(t, sessionsStore) // Perform the migration. err = txEx.ExecTx(ctx, sqldb.WriteTxOpt(), func(tx *sqlcmig6.Queries) error { - return MigrateFirewallDBToSQL( + err = MigrateFirewallDBToSQL( ctx, firewallStore.DB, tx, ) + if err != nil { + return err + } + + // Assert migration results. + assertMigrationResults(t, tx, entries) + + return nil }, sqldb.NoOpReset, ) require.NoError(t, err) - - // Assert migration results. - queries := sqlcmig6.NewForType( - sqlStore, sqlStore.BackendType, - ) - assertMigrationResults(t, queries, entries) }) } } From 20fd026bc141504ae799b3a897cc92d2e0b98c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 02:59:28 +0200 Subject: [PATCH 21/26] db: export the embed.FS `SqlSchemas` In upcoming commits, other packages than the `db` package will need to be able to access the `SqlSchemas`. This commit exports it in preparation for those changes --- db/schemas.go | 2 +- db/sql_migrations.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/schemas.go b/db/schemas.go index 1a7a2096f..dce7fa84c 100644 --- a/db/schemas.go +++ b/db/schemas.go @@ -6,4 +6,4 @@ import ( ) //go:embed sqlc/migrations/*.*.sql -var sqlSchemas embed.FS +var SqlSchemas embed.FS diff --git a/db/sql_migrations.go b/db/sql_migrations.go index 57d283aa3..c47ca0758 100644 --- a/db/sql_migrations.go +++ b/db/sql_migrations.go @@ -10,7 +10,7 @@ var ( LitdMigrationStream = sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, SQLFileDirectory: "sqlc/migrations", - Schemas: sqlSchemas, + Schemas: SqlSchemas, // LatestMigrationVersion is the latest migration version of the // database. This is used to implement downgrade protection for From 671b68ec6bdd1b34de150cfd4449b56541d12a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 02:39:36 +0200 Subject: [PATCH 22/26] multi: introduce migration stream for unit tests In upcoming commits, we will introduce a new migration stream package that will need to reference the db package, as well as the accounts, session and firewalldb package in future commits. To avoid circular dependencies, we therefore introduce a new migration stream that unit tests can use, in order to avoid having to import the new migration stream package. --- accounts/test_sqlite.go | 7 +++++-- db/migrations.go | 32 ++++++++++++++++++++++++++++++++ db/postgres.go | 2 +- firewalldb/test_sqlite.go | 7 +++++-- session/test_sqlite.go | 7 +++++-- 5 files changed, 48 insertions(+), 7 deletions(-) diff --git a/accounts/test_sqlite.go b/accounts/test_sqlite.go index a31f990a6..b1e8be871 100644 --- a/accounts/test_sqlite.go +++ b/accounts/test_sqlite.go @@ -18,7 +18,8 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { return createStore( - t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + t, + sqldb.NewTestSqliteDB(t, db.MakeTestMigrationStreams()).BaseDB, clock, ) } @@ -28,7 +29,9 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + tDb := sqldb.NewTestSqliteDBFromPath( + t, dbPath, db.MakeTestMigrationStreams(), + ) return createStore(t, tDb.BaseDB, clock) } diff --git a/db/migrations.go b/db/migrations.go index 204bf40f6..2284470b0 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -1,5 +1,11 @@ package db +import ( + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/lightningnetwork/lnd/sqldb/v2" +) + const ( // LatestMigrationVersion is the latest migration version of the // database. This is used to implement downgrade protection for the @@ -8,3 +14,29 @@ const ( // NOTE: This MUST be updated when a new migration is added. LatestMigrationVersion = 5 ) + +// MakeTestMigrationStreams creates the migration streams for the unit test +// environment. +func MakeTestMigrationStreams() []sqldb.MigrationStream { + migStream := sqldb.MigrationStream{ + MigrateTableName: pgx.DefaultMigrationsTable, + SQLFileDirectory: "sqlc/migrations", + Schemas: SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // database. This is used to implement downgrade protection for + // the daemon. + // + // NOTE: This MUST be updated when a new migration is added. + LatestMigrationVersion: LatestMigrationVersion, + + MakePostMigrationChecks: func( + db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, + error) { + + return make(map[uint]migrate.PostStepCallback), nil + }, + } + + return []sqldb.MigrationStream{migStream} +} diff --git a/db/postgres.go b/db/postgres.go index 780a7a13d..21fc0edb1 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -68,5 +68,5 @@ func NewTestPostgresDB(t *testing.T) *sqldb.PostgresStore { sqlFixture.TearDown(t) }) - return sqldb.NewTestPostgresDB(t, sqlFixture, LitdMigrationStreams) + return sqldb.NewTestPostgresDB(t, sqlFixture, MakeTestMigrationStreams()) } diff --git a/firewalldb/test_sqlite.go b/firewalldb/test_sqlite.go index ab184b5a6..3f91546af 100644 --- a/firewalldb/test_sqlite.go +++ b/firewalldb/test_sqlite.go @@ -13,7 +13,8 @@ import ( // NewTestDB is a helper function that creates an BBolt database for testing. func NewTestDB(t *testing.T, clock clock.Clock) FirewallDBs { return createStore( - t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + t, + sqldb.NewTestSqliteDB(t, db.MakeTestMigrationStreams()).BaseDB, clock, ) } @@ -23,7 +24,9 @@ func NewTestDB(t *testing.T, clock clock.Clock) FirewallDBs { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) FirewallDBs { - tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + tDb := sqldb.NewTestSqliteDBFromPath( + t, dbPath, db.MakeTestMigrationStreams(), + ) return createStore(t, tDb.BaseDB, clock) } diff --git a/session/test_sqlite.go b/session/test_sqlite.go index 84d946ce2..c9dbc5934 100644 --- a/session/test_sqlite.go +++ b/session/test_sqlite.go @@ -18,7 +18,8 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { return createStore( - t, sqldb.NewTestSqliteDB(t, db.LitdMigrationStreams).BaseDB, + t, + sqldb.NewTestSqliteDB(t, db.MakeTestMigrationStreams()).BaseDB, clock, ) } @@ -28,7 +29,9 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - tDb := sqldb.NewTestSqliteDBFromPath(t, dbPath, db.LitdMigrationStreams) + tDb := sqldb.NewTestSqliteDBFromPath( + t, dbPath, db.MakeTestMigrationStreams(), + ) return createStore(t, tDb.BaseDB, clock) } From c647d34223c37e7ce0179fee16a4d877988abc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 20:05:45 +0200 Subject: [PATCH 23/26] migrationstreams: introduce `migrationstreams` pkg The upcoming kvdb to sql code migration will be added to as part of the `sqldb/v2` migration stream. However, since the kvdb to sql migration will need to use the migration functions present in the `accounts`, `firewalldb`, and `session` packages, the migration will need to referernce those packages. That would lead to a circular dependency though if the migration stream was defined in the `db` package, as those packages need to import the `db` package. To avoid this, we introduce a new `migrationstreams` package that contains the migration streams, and ensure that the `db` package doesn't import the `migrationstreams` package. --- config_dev.go | 5 +++-- db/migrations.go | 3 +++ db/migrationstreams/log.go | 25 +++++++++++++++++++++ db/{ => migrationstreams}/sql_migrations.go | 7 +++--- log.go | 5 +++++ 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 db/migrationstreams/log.go rename db/{ => migrationstreams}/sql_migrations.go (82%) diff --git a/config_dev.go b/config_dev.go index cc8a5b319..d3beb2105 100644 --- a/config_dev.go +++ b/config_dev.go @@ -8,6 +8,7 @@ import ( "github.com/lightninglabs/lightning-terminal/accounts" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/migrationstreams" "github.com/lightninglabs/lightning-terminal/db/sqlc" "github.com/lightninglabs/lightning-terminal/firewalldb" "github.com/lightninglabs/lightning-terminal/session" @@ -113,7 +114,7 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { if !cfg.Sqlite.SkipMigrations { err = sqldb.ApplyAllMigrations( - sqlStore, db.LitdMigrationStreams, + sqlStore, migrationstreams.LitdMigrationStreams, ) if err != nil { return stores, fmt.Errorf("error applying "+ @@ -155,7 +156,7 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { if !cfg.Postgres.SkipMigrations { err = sqldb.ApplyAllMigrations( - sqlStore, db.LitdMigrationStreams, + sqlStore, migrationstreams.LitdMigrationStreams, ) if err != nil { return stores, fmt.Errorf("error applying "+ diff --git a/db/migrations.go b/db/migrations.go index 2284470b0..af499b397 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -17,6 +17,9 @@ const ( // MakeTestMigrationStreams creates the migration streams for the unit test // environment. +// +// NOTE: This function is not located in the migrationstreams package to avoid +// cyclic dependencies. func MakeTestMigrationStreams() []sqldb.MigrationStream { migStream := sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, diff --git a/db/migrationstreams/log.go b/db/migrationstreams/log.go new file mode 100644 index 000000000..0a3e80731 --- /dev/null +++ b/db/migrationstreams/log.go @@ -0,0 +1,25 @@ +package migrationstreams + +import ( + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/build" +) + +const Subsystem = "MIGS" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log btclog.Logger + +// The default amount of logging is none. +func init() { + UseLogger(build.NewSubLogger(Subsystem, nil)) +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/db/sql_migrations.go b/db/migrationstreams/sql_migrations.go similarity index 82% rename from db/sql_migrations.go rename to db/migrationstreams/sql_migrations.go index c47ca0758..ce9602698 100644 --- a/db/sql_migrations.go +++ b/db/migrationstreams/sql_migrations.go @@ -1,8 +1,9 @@ -package db +package migrationstreams import ( "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/sqldb/v2" ) @@ -10,14 +11,14 @@ var ( LitdMigrationStream = sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, SQLFileDirectory: "sqlc/migrations", - Schemas: SqlSchemas, + Schemas: db.SqlSchemas, // LatestMigrationVersion is the latest migration version of the // database. This is used to implement downgrade protection for // the daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion: LatestMigrationVersion, + LatestMigrationVersion: db.LatestMigrationVersion, MakePostMigrationChecks: func( db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, diff --git a/log.go b/log.go index b803f72d2..0535a819a 100644 --- a/log.go +++ b/log.go @@ -8,6 +8,7 @@ import ( "github.com/lightninglabs/lightning-terminal/accounts" "github.com/lightninglabs/lightning-terminal/autopilotserver" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightninglabs/lightning-terminal/db/migrationstreams" "github.com/lightninglabs/lightning-terminal/firewall" "github.com/lightninglabs/lightning-terminal/firewalldb" mid "github.com/lightninglabs/lightning-terminal/rpcmiddleware" @@ -91,6 +92,10 @@ func SetupLoggers(root *build.SubLoggerManager, intercept signal.Interceptor) { root, subservers.Subsystem, intercept, subservers.UseLogger, ) lnd.AddSubLogger(root, db.Subsystem, intercept, db.UseLogger) + lnd.AddSubLogger( + root, migrationstreams.Subsystem, intercept, + migrationstreams.UseLogger, + ) // Add daemon loggers to lnd's root logger. faraday.SetupLoggers(root, intercept) From 6627275e356917d81bfab44f8ff885457d388cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Fri, 11 Jul 2025 21:38:29 +0200 Subject: [PATCH 24/26] multi: introduce dev migrations When the kvdb to sql migration is initially introduced, we will want to ensure that it is only run under dev builds during the testing phase. We therefore introduce the functionality to have separate dev migrations, which are only included in a separate migration stream that is used only in dev builds. Note that these dev migrations are currently not included in the `sqlc.yaml` file, to ensure that the main sqlc models doesn't include the dev migrations. --- db/migrations.go | 31 ++++++++++- db/migrationstreams/sql_migrations.go | 2 + db/migrationstreams/sql_migrations_dev.go | 55 +++++++++++++++++++ db/schemas.go | 2 +- .../000001_dev_test_migration.down.sql | 1 + .../000001_dev_test_migration.up.sql | 1 + scripts/gen_sqlc_docker.sh | 4 +- 7 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 db/migrationstreams/sql_migrations_dev.go create mode 100644 db/sqlc/migrations_dev/000001_dev_test_migration.down.sql create mode 100644 db/sqlc/migrations_dev/000001_dev_test_migration.up.sql diff --git a/db/migrations.go b/db/migrations.go index af499b397..84f3909b9 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -13,6 +13,15 @@ const ( // // NOTE: This MUST be updated when a new migration is added. LatestMigrationVersion = 5 + + // LatestDevMigrationVersion is the latest dev migration version of the + // database. This is used to implement downgrade protection for the + // daemon. This represents the latest number used in the migrations_dev + // directory. + // + // NOTE: This MUST be updated when a migration is added or removed, from + // the migrations_dev directory. + LatestDevMigrationVersion = 1 ) // MakeTestMigrationStreams creates the migration streams for the unit test @@ -41,5 +50,25 @@ func MakeTestMigrationStreams() []sqldb.MigrationStream { }, } - return []sqldb.MigrationStream{migStream} + migStreamDev := sqldb.MigrationStream{ + MigrateTableName: pgx.DefaultMigrationsTable + "_dev", + SQLFileDirectory: "sqlc/migrations_dev", + Schemas: SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // dev migrations database. This is used to implement downgrade + // protection for the daemon. + // + // NOTE: This MUST be updated when a new dev migration is added. + LatestMigrationVersion: LatestDevMigrationVersion, + + MakePostMigrationChecks: func( + db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, + error) { + + return make(map[uint]migrate.PostStepCallback), nil + }, + } + + return []sqldb.MigrationStream{migStream, migStreamDev} } diff --git a/db/migrationstreams/sql_migrations.go b/db/migrationstreams/sql_migrations.go index ce9602698..5bcf8ba76 100644 --- a/db/migrationstreams/sql_migrations.go +++ b/db/migrationstreams/sql_migrations.go @@ -1,3 +1,5 @@ +//go:build !dev + package migrationstreams import ( diff --git a/db/migrationstreams/sql_migrations_dev.go b/db/migrationstreams/sql_migrations_dev.go new file mode 100644 index 000000000..bc32b39f0 --- /dev/null +++ b/db/migrationstreams/sql_migrations_dev.go @@ -0,0 +1,55 @@ +//go:build dev + +package migrationstreams + +import ( + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/sqldb/v2" +) + +var ( + // Create the prod migration stream. + migStream = sqldb.MigrationStream{ + MigrateTableName: pgx.DefaultMigrationsTable, + SQLFileDirectory: "sqlc/migrations", + Schemas: db.SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // database. This is used to implement downgrade protection for + // the daemon. + // + // NOTE: This MUST be updated when a new migration is added. + LatestMigrationVersion: db.LatestMigrationVersion, + + MakePostMigrationChecks: func( + db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, + error) { + + return make(map[uint]migrate.PostStepCallback), nil + }, + } + + // Create the dev migration stream. + migStreamDev = sqldb.MigrationStream{ + MigrateTableName: pgx.DefaultMigrationsTable + "_dev", + SQLFileDirectory: "sqlc/migrations_dev", + Schemas: db.SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // dev migrations database. This is used to implement downgrade + // protection for the daemon. + // + // NOTE: This MUST be updated when a new dev migration is added. + LatestMigrationVersion: db.LatestDevMigrationVersion, + + MakePostMigrationChecks: func( + db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, + error) { + + return make(map[uint]migrate.PostStepCallback), nil + }, + } + LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev} +) diff --git a/db/schemas.go b/db/schemas.go index dce7fa84c..565fb5615 100644 --- a/db/schemas.go +++ b/db/schemas.go @@ -5,5 +5,5 @@ import ( _ "embed" ) -//go:embed sqlc/migrations/*.*.sql +//go:embed sqlc/migration*/*.*.sql var SqlSchemas embed.FS diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql b/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql new file mode 100644 index 000000000..0d246b2dd --- /dev/null +++ b/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql @@ -0,0 +1 @@ +-- Comment to ensure the file created and picked up in the migration stream. \ No newline at end of file diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql b/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql new file mode 100644 index 000000000..0d246b2dd --- /dev/null +++ b/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql @@ -0,0 +1 @@ +-- Comment to ensure the file created and picked up in the migration stream. \ No newline at end of file diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index 16db97f2c..3d93f37ff 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -5,7 +5,7 @@ set -e # restore_files is a function to restore original schema files. restore_files() { echo "Restoring SQLite bigint patch..." - for file in db/sqlc/migrations/*.up.sql.bak; do + for file in db/sqlc/{migrations,migrations_dev}/*.up.sql.bak; do mv "$file" "${file%.bak}" done } @@ -30,7 +30,7 @@ GOMODCACHE=$(go env GOMODCACHE) # source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER # PRIMARY KEY". echo "Applying SQLite bigint patch..." -for file in db/sqlc/migrations/*.up.sql; do +for file in db/sqlc/{migrations,migrations_dev}/*.up.sql; do echo "Patching $file" sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file" done From 0225d408eff0479fb757da4bfb232c0897135c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Mon, 9 Jun 2025 14:39:51 +0200 Subject: [PATCH 25/26] accounts: export kvdb DB The next commit that adds the kvdb to sql code migration will need to initialize the accounts kvdb DB. This commit exports that struct so that it can be referenced outside of the accounts package. --- accounts/service.go | 4 ++-- accounts/service_test.go | 2 +- accounts/sql_migration_test.go | 6 +++--- accounts/store_kvdb.go | 22 +++++++++++----------- accounts/test_kvdb.go | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/accounts/service.go b/accounts/service.go index 102b9ea84..697cf7518 100644 --- a/accounts/service.go +++ b/accounts/service.go @@ -361,7 +361,7 @@ func (s *InterceptorService) CreditAccount(ctx context.Context, return nil, ErrAccountServiceDisabled } - // Credit the account in the db. + // Credit the account in the DB. err := s.store.CreditAccount(ctx, accountID, amount) if err != nil { return nil, fmt.Errorf("unable to credit account: %w", err) @@ -386,7 +386,7 @@ func (s *InterceptorService) DebitAccount(ctx context.Context, return nil, ErrAccountServiceDisabled } - // Debit the account in the db. + // Debit the account in the DB. err := s.store.DebitAccount(ctx, accountID, amount) if err != nil { return nil, fmt.Errorf("unable to debit account: %w", err) diff --git a/accounts/service_test.go b/accounts/service_test.go index 1d4388664..f69d0fe93 100644 --- a/accounts/service_test.go +++ b/accounts/service_test.go @@ -246,7 +246,7 @@ func TestAccountService(t *testing.T) { // Ensure that the service was started successfully and // still running though, despite the closing of the - // db store. + // DB store. require.True(t, s.IsRunning()) // Now let's send the invoice update, which should fail. diff --git a/accounts/sql_migration_test.go b/accounts/sql_migration_test.go index 4f3ae1741..ecec67eb5 100644 --- a/accounts/sql_migration_test.go +++ b/accounts/sql_migration_test.go @@ -301,7 +301,7 @@ func TestAccountStoreMigration(t *testing.T) { ) require.NoError(t, err) t.Cleanup(func() { - require.NoError(t, kvStore.db.Close()) + require.NoError(t, kvStore.DB.Close()) }) // Populate the kv store. @@ -336,7 +336,7 @@ func TestAccountStoreMigration(t *testing.T) { ctx, sqldb.WriteTxOpt(), func(tx *sqlcmig6.Queries) error { return MigrateAccountStoreToSQL( - ctx, kvStore.db, tx, + ctx, kvStore.DB, tx, ) }, sqldb.NoOpReset, ) @@ -440,7 +440,7 @@ func rapidRandomizeAccounts(t *testing.T, kvStore *BoltStore) { acct := makeAccountGen().Draw(t, "account") // Then proceed to insert the account with its invoices and - // payments into the db + // payments into the DB newAcct, err := kvStore.NewAccount( ctx, acct.balance, acct.expiry, acct.label, ) diff --git a/accounts/store_kvdb.go b/accounts/store_kvdb.go index a419017a8..c8f0282ee 100644 --- a/accounts/store_kvdb.go +++ b/accounts/store_kvdb.go @@ -24,7 +24,7 @@ import ( const ( // DBFilename is the filename within the data directory which contains // the macaroon stores. - DBFilename = "accounts.db" + DBFilename = "accounts.DB" // dbPathPermission is the default permission the account database // directory is created with (if it does not exist). @@ -60,7 +60,7 @@ var ( // BoltStore wraps the bolt DB that stores all accounts and their balances. type BoltStore struct { - db kvdb.Backend + DB kvdb.Backend clock clock.Clock } @@ -101,7 +101,7 @@ func NewBoltStore(dir, fileName string, clock clock.Clock) (*BoltStore, error) { // Return the DB wrapped in a BoltStore object. return &BoltStore{ - db: db, + DB: db, clock: clock, }, nil } @@ -110,7 +110,7 @@ func NewBoltStore(dir, fileName string, clock clock.Clock) (*BoltStore, error) { // // NOTE: This is part of the Store interface. func (s *BoltStore) Close() error { - return s.db.Close() + return s.DB.Close() } // NewAccount creates a new OffChainBalanceAccount with the given balance and a @@ -162,7 +162,7 @@ func (s *BoltStore) NewAccount(ctx context.Context, balance lnwire.MilliSatoshi, // Try storing the account in the account database, so we can keep track // of its balance. - err := s.db.Update(func(tx walletdb.ReadWriteTx) error { + err := s.DB.Update(func(tx walletdb.ReadWriteTx) error { bucket := tx.ReadWriteBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound @@ -364,7 +364,7 @@ func (s *BoltStore) DeleteAccountPayment(_ context.Context, id AccountID, func (s *BoltStore) updateAccount(id AccountID, updateFn func(*OffChainBalanceAccount) error) error { - return s.db.Update(func(tx kvdb.RwTx) error { + return s.DB.Update(func(tx kvdb.RwTx) error { bucket := tx.ReadWriteBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound @@ -451,7 +451,7 @@ func (s *BoltStore) Account(_ context.Context, id AccountID) ( // Try looking up and reading the account by its ID from the local // bolt DB. var accountBinary []byte - err := s.db.View(func(tx kvdb.RTx) error { + err := s.DB.View(func(tx kvdb.RTx) error { bucket := tx.ReadBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound @@ -487,7 +487,7 @@ func (s *BoltStore) Accounts(_ context.Context) ([]*OffChainBalanceAccount, error) { var accounts []*OffChainBalanceAccount - err := s.db.View(func(tx kvdb.RTx) error { + err := s.DB.View(func(tx kvdb.RTx) error { // This function will be called in the ForEach and receive // the key and value of each account in the DB. The key, which // is also the ID is not used because it is also marshaled into @@ -531,7 +531,7 @@ func (s *BoltStore) Accounts(_ context.Context) ([]*OffChainBalanceAccount, // // NOTE: This is part of the Store interface. func (s *BoltStore) RemoveAccount(_ context.Context, id AccountID) error { - return s.db.Update(func(tx kvdb.RwTx) error { + return s.DB.Update(func(tx kvdb.RwTx) error { bucket := tx.ReadWriteBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound @@ -554,7 +554,7 @@ func (s *BoltStore) LastIndexes(_ context.Context) (uint64, uint64, error) { var ( addValue, settleValue []byte ) - err := s.db.View(func(tx kvdb.RTx) error { + err := s.DB.View(func(tx kvdb.RTx) error { bucket := tx.ReadBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound @@ -592,7 +592,7 @@ func (s *BoltStore) StoreLastIndexes(_ context.Context, addIndex, byteOrder.PutUint64(addValue, addIndex) byteOrder.PutUint64(settleValue, settleIndex) - return s.db.Update(func(tx kvdb.RwTx) error { + return s.DB.Update(func(tx kvdb.RwTx) error { bucket := tx.ReadWriteBucket(accountBucketName) if bucket == nil { return ErrAccountBucketNotFound diff --git a/accounts/test_kvdb.go b/accounts/test_kvdb.go index 546c1eee7..1d181928c 100644 --- a/accounts/test_kvdb.go +++ b/accounts/test_kvdb.go @@ -28,7 +28,7 @@ func NewTestDBFromPath(t *testing.T, dbPath string, require.NoError(t, err) t.Cleanup(func() { - require.NoError(t, store.db.Close()) + require.NoError(t, store.DB.Close()) }) return store From f8bfb97f73c59c77b0eac02903f488545e3fd47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Thu, 24 Jul 2025 02:43:17 +0200 Subject: [PATCH 26/26] multi: add dev kvdb to sql code migration Add the necessary code to trigger the kvdb to sql code migration in dev builds. --- config_dev.go | 15 ++- config_prod.go | 5 +- db/migrations.go | 3 +- .../post_migration_callbacks_dev.go | 107 ++++++++++++++++++ db/migrationstreams/sql_migrations.go | 15 ++- db/migrationstreams/sql_migrations_dev.go | 42 ++++++- ...00001_code_migration_kvdb_to_sql.down.sql} | 0 ... 000001_code_migration_kvdb_to_sql.up.sql} | 0 terminal.go | 2 +- 9 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 db/migrationstreams/post_migration_callbacks_dev.go rename db/sqlc/migrations_dev/{000001_dev_test_migration.down.sql => 000001_code_migration_kvdb_to_sql.down.sql} (100%) rename db/sqlc/migrations_dev/{000001_dev_test_migration.up.sql => 000001_code_migration_kvdb_to_sql.up.sql} (100%) diff --git a/config_dev.go b/config_dev.go index d3beb2105..30a1c54d7 100644 --- a/config_dev.go +++ b/config_dev.go @@ -3,6 +3,7 @@ package terminal import ( + "context" "fmt" "path/filepath" @@ -87,7 +88,9 @@ func defaultDevConfig() *DevConfig { } // NewStores creates a new stores instance based on the chosen database backend. -func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { +func NewStores(ctx context.Context, cfg *Config, + clock clock.Clock) (*stores, error) { + var ( networkDir = filepath.Join(cfg.LitDir, cfg.Network) stores = &stores{ @@ -114,7 +117,10 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { if !cfg.Sqlite.SkipMigrations { err = sqldb.ApplyAllMigrations( - sqlStore, migrationstreams.LitdMigrationStreams, + sqlStore, + migrationstreams.MakeMigrationStreams( + ctx, cfg.MacaroonPath, clock, + ), ) if err != nil { return stores, fmt.Errorf("error applying "+ @@ -156,7 +162,10 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { if !cfg.Postgres.SkipMigrations { err = sqldb.ApplyAllMigrations( - sqlStore, migrationstreams.LitdMigrationStreams, + sqlStore, + migrationstreams.MakeMigrationStreams( + ctx, cfg.MacaroonPath, clock, + ), ) if err != nil { return stores, fmt.Errorf("error applying "+ diff --git a/config_prod.go b/config_prod.go index ac6e6d996..1f11507a6 100644 --- a/config_prod.go +++ b/config_prod.go @@ -3,6 +3,7 @@ package terminal import ( + "context" "fmt" "path/filepath" @@ -29,7 +30,9 @@ func (c *DevConfig) Validate(_, _ string) error { // NewStores creates a new instance of the stores struct using the default Bolt // backend since in production, this is currently the only backend supported. -func NewStores(cfg *Config, clock clock.Clock) (*stores, error) { +func NewStores(_ context.Context, cfg *Config, + clock clock.Clock) (*stores, error) { + networkDir := filepath.Join(cfg.LitDir, cfg.Network) stores := &stores{ diff --git a/db/migrations.go b/db/migrations.go index 84f3909b9..bbf35642f 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -28,7 +28,8 @@ const ( // environment. // // NOTE: This function is not located in the migrationstreams package to avoid -// cyclic dependencies. +// cyclic dependencies. This test migration stream does not run the kvdb to sql +// migration, as we already have separate unit tests which tests the migration. func MakeTestMigrationStreams() []sqldb.MigrationStream { migStream := sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, diff --git a/db/migrationstreams/post_migration_callbacks_dev.go b/db/migrationstreams/post_migration_callbacks_dev.go new file mode 100644 index 000000000..45c73783a --- /dev/null +++ b/db/migrationstreams/post_migration_callbacks_dev.go @@ -0,0 +1,107 @@ +//go:build dev + +package migrationstreams + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "time" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db/sqlcmig6" + "github.com/lightninglabs/lightning-terminal/firewalldb" + "github.com/lightninglabs/lightning-terminal/session" + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/sqldb/v2" +) + +// MakePostStepCallbacksMig6 turns the post migration checks into a map of post +// step callbacks that can be used with the migrate package. The keys of the map +// are the migration versions, and the values are the callbacks that will be +// executed after the migration with the corresponding version is applied. +func MakePostStepCallbacksMig6(ctx context.Context, db *sqldb.BaseDB, + macPath string, clock clock.Clock, + migVersion uint) migrate.PostStepCallback { + + mig6queries := sqlcmig6.NewForType(db, db.BackendType) + mig6executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) *sqlcmig6.Queries { + return mig6queries.WithTx(tx) + }, + ) + + return func(_ *migrate.Migration, _ database.Driver) error { + // We ignore the actual driver that's being returned here, since + // we use migrate.NewWithInstance() to create the migration + // instance from our already instantiated database backend that + // is also passed into this function. + return mig6executor.ExecTx( + ctx, sqldb.WriteTxOpt(), + func(q6 *sqlcmig6.Queries) error { + log.Infof("Running post migration callback "+ + "for migration version %d", migVersion) + + return kvdbToSqlMigrationCallback( + ctx, macPath, db, clock, q6, + ) + }, sqldb.NoOpReset, + ) + } +} + +func kvdbToSqlMigrationCallback(ctx context.Context, macPath string, + _ *sqldb.BaseDB, clock clock.Clock, q *sqlcmig6.Queries) error { + + start := time.Now() + log.Infof("Starting KVDB to SQL migration for all stores") + + accountStore, err := accounts.NewBoltStore( + filepath.Dir(macPath), accounts.DBFilename, clock, + ) + if err != nil { + return err + } + + err = accounts.MigrateAccountStoreToSQL(ctx, accountStore.DB, q) + if err != nil { + return fmt.Errorf("error migrating account store to "+ + "SQL: %v", err) + } + + sessionStore, err := session.NewDB( + filepath.Dir(macPath), session.DBFilename, + clock, accountStore, + ) + if err != nil { + return err + } + + err = session.MigrateSessionStoreToSQL(ctx, sessionStore.DB, q) + if err != nil { + return fmt.Errorf("error migrating session store to "+ + "SQL: %v", err) + } + + firewallStore, err := firewalldb.NewBoltDB( + filepath.Dir(macPath), firewalldb.DBFilename, + sessionStore, accountStore, clock, + ) + if err != nil { + return err + } + + err = firewalldb.MigrateFirewallDBToSQL(ctx, firewallStore.DB, q) + if err != nil { + return fmt.Errorf("error migrating firewalldb store "+ + "to SQL: %v", err) + } + + log.Infof("Succesfully migrated all KVDB stores to SQL in: %v", + time.Since(start)) + + return nil +} diff --git a/db/migrationstreams/sql_migrations.go b/db/migrationstreams/sql_migrations.go index 5bcf8ba76..41d6ccb88 100644 --- a/db/migrationstreams/sql_migrations.go +++ b/db/migrationstreams/sql_migrations.go @@ -3,14 +3,20 @@ package migrationstreams import ( + "context" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/pgx/v5" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/sqldb/v2" ) -var ( - LitdMigrationStream = sqldb.MigrationStream{ +// MakeMigrationStreams creates the migration streams for production +// environments. +func MakeMigrationStreams(_ context.Context, _ string, + _ clock.Clock) []sqldb.MigrationStream { + + migStream := sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, SQLFileDirectory: "sqlc/migrations", Schemas: db.SqlSchemas, @@ -29,5 +35,6 @@ var ( return make(map[uint]migrate.PostStepCallback), nil }, } - LitdMigrationStreams = []sqldb.MigrationStream{LitdMigrationStream} -) + + return []sqldb.MigrationStream{migStream} +} diff --git a/db/migrationstreams/sql_migrations_dev.go b/db/migrationstreams/sql_migrations_dev.go index bc32b39f0..76c9dd096 100644 --- a/db/migrationstreams/sql_migrations_dev.go +++ b/db/migrationstreams/sql_migrations_dev.go @@ -3,15 +3,30 @@ package migrationstreams import ( + "context" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/pgx/v5" "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/sqldb/v2" ) -var ( +const ( + // KVDBtoSQLMigVersion is the version of the migration that migrates the + // kvdb to the sql database. + // + // TODO: When this the kvdb to sql migration goes live into prod, this + // should be moved to non dev db/migrations.go file, and this constant + // value should be updated to reflect the real migration number. + KVDBtoSQLMigVersion = 1 +) + +// MakeMigrationStreams creates the migration streams for the dev environments. +func MakeMigrationStreams(ctx context.Context, macPath string, + clock clock.Clock) []sqldb.MigrationStream { + // Create the prod migration stream. - migStream = sqldb.MigrationStream{ + migStream := sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable, SQLFileDirectory: "sqlc/migrations", Schemas: db.SqlSchemas, @@ -32,7 +47,7 @@ var ( } // Create the dev migration stream. - migStreamDev = sqldb.MigrationStream{ + migStreamDev := sqldb.MigrationStream{ MigrateTableName: pgx.DefaultMigrationsTable + "_dev", SQLFileDirectory: "sqlc/migrations_dev", Schemas: db.SqlSchemas, @@ -48,8 +63,23 @@ var ( db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback, error) { - return make(map[uint]migrate.PostStepCallback), nil + // Any Callbacks added to this map will be executed when + // after the dev migration number for the uint key in + // the map has been applied. If no entry exists for a + // given uint, then no callback will be executed for + // that migration number. This is useful for adding a + // code migration step as a callback to be run + // after a specific migration of a given number has been + // applied. + res := make(map[uint]migrate.PostStepCallback) + + res[KVDBtoSQLMigVersion] = MakePostStepCallbacksMig6( + ctx, db, macPath, clock, KVDBtoSQLMigVersion, + ) + + return res, nil }, } - LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev} -) + + return []sqldb.MigrationStream{migStream, migStreamDev} +} diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql b/db/sqlc/migrations_dev/000001_code_migration_kvdb_to_sql.down.sql similarity index 100% rename from db/sqlc/migrations_dev/000001_dev_test_migration.down.sql rename to db/sqlc/migrations_dev/000001_code_migration_kvdb_to_sql.down.sql diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql b/db/sqlc/migrations_dev/000001_code_migration_kvdb_to_sql.up.sql similarity index 100% rename from db/sqlc/migrations_dev/000001_dev_test_migration.up.sql rename to db/sqlc/migrations_dev/000001_code_migration_kvdb_to_sql.up.sql diff --git a/terminal.go b/terminal.go index 7e4d552c7..5f8fa2efb 100644 --- a/terminal.go +++ b/terminal.go @@ -447,7 +447,7 @@ func (g *LightningTerminal) start(ctx context.Context) error { return fmt.Errorf("could not create network directory: %v", err) } - g.stores, err = NewStores(g.cfg, clock.NewDefaultClock()) + g.stores, err = NewStores(ctx, g.cfg, clock.NewDefaultClock()) if err != nil { return fmt.Errorf("could not create stores: %v", err) }