Skip to content

Implement https://sqlite.org/c3ref/vtab_in.html #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions func.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package sqlite
import (
"errors"
"fmt"
"iter"
"math"
"math/bits"
"strconv"
Expand Down Expand Up @@ -225,6 +226,32 @@ func (v Value) Type() ColumnType {
return ColumnType(lib.Xsqlite3_value_type(v.tls, v.ptrOrType))
}

// All iterates all values of an IN operator
// https://www.sqlite.org/c3ref/vtab_in.html
// https://www.sqlite.org/c3ref/vtab_in_first.html
func (v Value) All() iter.Seq[Value] {
Copy link
Owner

@zombiezen zombiezen Jul 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While I appreciate the simplicity of the iterator interface, I'm not quite sold on this API for a few reasons:

  1. It obscures errors.
  2. It's not a common enough operation to justify being a method IMO.
  3. The lifetime of the Value objects from these functions is somewhat tricky and that complexity is not being surfaced to the user.
  4. I'm not fond of giving the user ability to call this function outside of the context of best index, since SQLite docs say the behavior is undefined, which is worse than giving an error.

I don't have a great counter-proposal for this design, but I didn't want to block the feedback on me coming up with something.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah good points. What if this were a method on the VTCursor interface instead? Then it would be much harder to use outside the Filter function (the only place sqlite_vtab_in_{first,next} are valid to use).

Also do you think the golang iter interface is ok or passing in a callback function is preferred?

return func(yield func(Value) bool) {
if v.tls == nil ||
v.ptrOrType == 0 ||
ColumnType(lib.Xsqlite3_value_type(v.tls, v.ptrOrType)) != TypeNull {
return
}
ppVal := lib.Xsqlite3_malloc(v.tls, int32(unsafe.Sizeof(uintptr(0))))
if ppVal != 0 {
defer lib.Xsqlite3_free(v.tls, ppVal)
}
for rc := ResultCode(lib.Xsqlite3_vtab_in_first(v.tls, v.ptrOrType, ppVal)); rc == ResultOK && *(*uintptr)(unsafe.Pointer(ppVal)) != 0; rc = ResultCode(lib.Xsqlite3_vtab_in_next(v.tls, v.ptrOrType, ppVal)) {
// do something with pVal
if !yield(Value{
tls: v.tls,
ptrOrType: *(*uintptr)(unsafe.Pointer(ppVal)),
}) {
return
}
}
}
}

// Conversions follow the table in https://sqlite.org/c3ref/column_blob.html

// Int returns the value as an integer.
Expand Down
7 changes: 3 additions & 4 deletions go.work.sum
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
Expand All @@ -24,7 +24,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -37,6 +36,7 @@ golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFK
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
Expand All @@ -49,7 +49,6 @@ golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y=
modernc.org/ccgo/v3 v3.17.0/go.mod h1:Sg3fwVpmLvCUTaqEUjiBDAvshIaKDB0RXaf+zgqFu8I=
Expand Down
9 changes: 9 additions & 0 deletions index_constraint.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ type IndexConstraint struct {
RValue Value
// RValueKnown indicates whether RValue is set.
RValueKnown bool
// InOpAllAtOnce is true if the constraint is an IN operator
// that can be processed all at once
InOpAllAtOnce bool
Comment on lines +35 to +37
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: We abbreviate operator for common usages, but this is less common, so let's spell it out here.

Suggested change
// InOpAllAtOnce is true if the constraint is an IN operator
// that can be processed all at once
InOpAllAtOnce bool
// InOperatorAllAtOnce is true if the constraint is an IN operator
// that can be processed all at once
InOperatorAllAtOnce bool

}

func (c *IndexConstraint) copyFromC(tls *libc.TLS, infoPtr uintptr, i int32, ppVal uintptr) {
Expand All @@ -43,6 +46,12 @@ func (c *IndexConstraint) copyFromC(tls *libc.TLS, infoPtr uintptr, i int32, ppV
Usable: src.Fusable != 0,
}

if c.Op == IndexConstraintEq {
if lib.Xsqlite3_vtab_in(tls, infoPtr, int32(i), -1) != 0 {
c.InOpAllAtOnce = true
}
}

const binaryCollation = "BINARY"
cCollation := lib.Xsqlite3_vtab_collation(tls, infoPtr, int32(i))
if isCStringEqual(cCollation, binaryCollation) {
Expand Down
8 changes: 7 additions & 1 deletion vtable.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,14 +252,17 @@ func (outputs *IndexOutputs) copyToC(tls *libc.TLS, infoPtr uintptr) error {
info := (*lib.Sqlite3_index_info)(unsafe.Pointer(infoPtr))

aConstraintUsage := info.FaConstraintUsage
for _, u := range outputs.ConstraintUsage {
for i, u := range outputs.ConstraintUsage {
ptr := (*lib.Sqlite3_index_constraint_usage)(unsafe.Pointer(aConstraintUsage))
ptr.FargvIndex = int32(u.ArgvIndex)
if u.Omit {
ptr.Fomit = 1
} else {
ptr.Fomit = 0
}
if u.InOpAllAtOnce {
lib.Xsqlite3_vtab_in(tls, infoPtr, int32(i), 1)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps check the return value of this? IIUC if it's zero, that indicates that the library user has improperly used this call.

}
aConstraintUsage += unsafe.Sizeof(lib.Sqlite3_index_constraint_usage{})
}
info.FidxNum = outputs.ID.Num
Expand Down Expand Up @@ -306,6 +309,9 @@ type IndexConstraintUsage struct {
// SQLite will always double-check that rows satisfy the constraint if Omit is false,
// but may skip this check if Omit is true.
Omit bool
// Set InOpAllAtOnce to true if the constraint is an IN operator and the
// Filter method should receive all values at once.
InOpAllAtOnce bool
Comment on lines +312 to +314
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: We abbreviate operator for common usages, but this is less common, so let's spell it out here.

Suggested change
// Set InOpAllAtOnce to true if the constraint is an IN operator and the
// Filter method should receive all values at once.
InOpAllAtOnce bool
// If InOperatorAllAtOnce is true, then the Filter method
// will receive all values of an IN operator at once.
InOperatorAllAtOnce bool

The doc comment should also include instructions on how to access the values.

}

// IndexID is a virtual table index identifier.
Expand Down
54 changes: 50 additions & 4 deletions vtable_example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,19 @@ func ExampleVTable() {
log.Fatal(err)
}

err = sqlitex.ExecuteTransient(
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't feel like something that needs to be included in an introductory example. IIUC SQLite will fall back to a simpler set of indices and still operate without this.

It definitely still needs testing, but shouldn't be in this example.

conn,
`SELECT a, b FROM templatevtab WHERE a IN (1001, 1003, 1005, 1007, 1009) ORDER BY rowid;`,
&sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
fmt.Printf("%4d, %4d\n", stmt.ColumnInt(0), stmt.ColumnInt(1))
return nil
},
},
)
if err != nil {
log.Fatal(err)
}
// Output:
// 1001, 2001
// 1002, 2002
Expand All @@ -49,6 +62,11 @@ func ExampleVTable() {
// 1008, 2008
// 1009, 2009
// 1010, 2010
// 1001, 2001
// 1003, 2003
// 1005, 2005
// 1007, 2007
// 1009, 2009
}

type templatevtab struct{}
Expand All @@ -66,10 +84,22 @@ func templatevtabConnect(c *sqlite.Conn, opts *sqlite.VTableConnectOptions) (sql
return vtab, cfg, nil
}

func (vt *templatevtab) BestIndex(*sqlite.IndexInputs) (*sqlite.IndexOutputs, error) {
func (vt *templatevtab) BestIndex(in *sqlite.IndexInputs) (*sqlite.IndexOutputs, error) {

constraintUsage := make([]sqlite.IndexConstraintUsage, len(in.Constraints))

argvIndex := 1
for i, c := range in.Constraints {
if c.InOpAllAtOnce {
constraintUsage[i].ArgvIndex = argvIndex
constraintUsage[i].InOpAllAtOnce = true
argvIndex++
}
}
return &sqlite.IndexOutputs{
EstimatedCost: 10,
EstimatedRows: 10,
ConstraintUsage: constraintUsage,
EstimatedCost: 10,
EstimatedRows: 10,
}, nil
}

Expand All @@ -86,10 +116,17 @@ func (vt *templatevtab) Destroy() error {
}

type templatevtabCursor struct {
rowid int64
rowid int64
inVals []int64
}

func (cur *templatevtabCursor) Filter(id sqlite.IndexID, argv []sqlite.Value) error {
for _, v := range argv {
for vv := range v.All() {
cur.inVals = append(cur.inVals, vv.Int64())
}
}

cur.rowid = 1
return nil
}
Expand All @@ -102,8 +139,14 @@ func (cur *templatevtabCursor) Next() error {
func (cur *templatevtabCursor) Column(i int, noChange bool) (sqlite.Value, error) {
switch i {
case templatevarColumnA:
if len(cur.inVals) > 0 {
return sqlite.IntegerValue(cur.inVals[cur.rowid-1]), nil
}
return sqlite.IntegerValue(1000 + cur.rowid), nil
case templatevarColumnB:
if len(cur.inVals) > 0 {
return sqlite.IntegerValue(1000 + cur.inVals[cur.rowid-1]), nil
}
return sqlite.IntegerValue(2000 + cur.rowid), nil
default:
panic("unreachable")
Expand All @@ -115,6 +158,9 @@ func (cur *templatevtabCursor) RowID() (int64, error) {
}

func (cur *templatevtabCursor) EOF() bool {
if len(cur.inVals) > 0 {
return int(cur.rowid) > len(cur.inVals)
}
return cur.rowid > 10
}

Expand Down