diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index d2a225e9569..8c347aa4ebd 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -16,6 +16,11 @@ package engine import ( "context" + "os" + "strconv" + "strings" + "time" + gms "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/eventscheduler" "github.com/dolthub/go-mysql-server/sql" @@ -26,10 +31,6 @@ import ( _ "github.com/dolthub/go-mysql-server/sql/variables" "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/sirupsen/logrus" - "os" - "strconv" - "strings" - "time" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 5844926d0f8..3ae8cb70e45 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -19,7 +19,6 @@ import ( "crypto/tls" "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro" "net" "net/http" "os" @@ -56,6 +55,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster" _ "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro" "github.com/dolthub/dolt/go/libraries/doltcore/sqlserver" "github.com/dolthub/dolt/go/libraries/events" "github.com/dolthub/dolt/go/libraries/utils/config" diff --git a/go/libraries/doltcore/sqle/dprocedures/stats_funcs.go b/go/libraries/doltcore/sqle/dprocedures/stats_funcs.go index a7884bbc4fb..c6937a74efe 100644 --- a/go/libraries/doltcore/sqle/dprocedures/stats_funcs.go +++ b/go/libraries/doltcore/sqle/dprocedures/stats_funcs.go @@ -18,9 +18,10 @@ import ( "context" "encoding/json" "fmt" + "strconv" + "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" - "strconv" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" ) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 96a40ce5d45..1aff1cdbd9e 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -17,7 +17,6 @@ package enginetest import ( "context" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/ref" "os" "runtime" "sync" @@ -35,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro" @@ -1453,11 +1453,6 @@ func TestStatBranchTests(t *testing.T) { RunStatBranchTests(t, harness) } -func TestStatsFunctions(t *testing.T) { - harness := newDoltEnginetestHarness(t) - RunStatsFunctionsTest(t, harness) -} - func TestDiffTableFunction(t *testing.T) { harness := newDoltEnginetestHarness(t) RunDiffTableFunctionTests(t, harness) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index d53dc74921a..0747f743b1b 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -1164,21 +1164,6 @@ func mustNewEngine(t *testing.T, h enginetest.Harness) enginetest.QueryEngine { return e } -func RunStatsFunctionsTest(t *testing.T, harness DoltEnginetestHarness) { - defer harness.Close() - for _, test := range StatProcTests { - t.Run(test.Name, func(t *testing.T) { - // reset engine so provider statistics are clean - harness = harness.NewHarness(t).WithConfigureStats(true) - harness.Setup(setup.MydbData) - harness.SkipSetupCommit() - e := mustNewEngine(t, harness) - defer e.Close() - enginetest.TestScriptWithEngine(t, e, harness, test) - }) - } -} - func RunDiffTableFunctionTests(t *testing.T, harness DoltEnginetestHarness) { for _, test := range DiffTableFunctionScriptTests { t.Run(test.Name, func(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index da10cb19cc2..f5b46cd4cea 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -17,6 +17,20 @@ package enginetest import ( "context" "fmt" + "runtime" + "strings" + "testing" + "time" + + gms "github.com/dolthub/go-mysql-server" + "github.com/dolthub/go-mysql-server/enginetest" + "github.com/dolthub/go-mysql-server/enginetest/scriptgen/setup" + "github.com/dolthub/go-mysql-server/memory" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/mysql_db" + "github.com/dolthub/go-mysql-server/sql/rowexec" + "github.com/stretchr/testify/require" + "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -28,18 +42,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/types" - gms "github.com/dolthub/go-mysql-server" - "github.com/dolthub/go-mysql-server/enginetest" - "github.com/dolthub/go-mysql-server/enginetest/scriptgen/setup" - "github.com/dolthub/go-mysql-server/memory" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/mysql_db" - "github.com/dolthub/go-mysql-server/sql/rowexec" - "github.com/stretchr/testify/require" - "runtime" - "strings" - "testing" - "time" ) type DoltHarness struct { diff --git a/go/libraries/doltcore/sqle/enginetest/stats_queries.go b/go/libraries/doltcore/sqle/enginetest/stats_queries.go index d3c737619cb..8cfe99e6478 100644 --- a/go/libraries/doltcore/sqle/enginetest/stats_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/stats_queries.go @@ -16,11 +16,13 @@ package enginetest import ( "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "strings" + "github.com/dolthub/go-mysql-server/enginetest/queries" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/types" - "strings" + + "github.com/dolthub/dolt/go/libraries/doltcore/schema" ) // fillerVarchar pushes the tree into level 3 @@ -588,8 +590,6 @@ var StatBranchTests = []queries.ScriptTest{ { Name: "multi branch stats", SetUpScript: []string{ - "set @@PERSIST.dolt_stats_auto_refresh_interval = 0;", - "set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;", "set @@PERSIST.dolt_stats_branches = 'main,feat';", "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", "insert into xy values (0,0,'a'), (1,0,'a'), (2,0,'a'), (3,0,'a'), (4,1,'a'), (5,2,'a')", @@ -701,214 +701,3 @@ var StatBranchTests = []queries.ScriptTest{ }, }, } - -var StatProcTests = []queries.ScriptTest{ - { - Name: "deleting stats removes information_schema access point", - SetUpScript: []string{ - "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", - "insert into xy values (0,0,0)", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "analyze table xy", - }, - { - Query: "select count(*) from information_schema.column_statistics", - Expected: []sql.Row{{2}}, - }, - { - Query: "call dolt_stats_drop()", - }, - { - Query: "select count(*) from information_schema.column_statistics", - Expected: []sql.Row{{0}}, - }, - }, - }, - { - Name: "restart empty stats panic", - SetUpScript: []string{ - "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "analyze table xy", - }, - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{0}}, - }, - { - Query: "set @@GLOBAL.dolt_stats_auto_refresh_threshold = 0", - Expected: []sql.Row{{}}, - }, - { - Query: "set @@GLOBAL.dolt_stats_auto_refresh_interval = 0", - Expected: []sql.Row{{}}, - }, - { - // don't panic - Query: "call dolt_stats_restart()", - }, - { - Query: "select sleep(.1)", - }, - { - Query: "insert into xy values (0,0,0)", - }, - { - Query: "select sleep(.1)", - }, - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{2}}, - }, - }, - }, - { - Name: "basic start, status, stop loop", - SetUpScript: []string{ - "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", - "insert into xy values (0,0,'a'), (2,0,'a'), (4,1,'a'), (6,2,'a')", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{0}}, - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"no active stats thread"}}, - }, - // set refresh interval arbitrarily high to avoid updating when we restart - { - Query: "set @@PERSIST.dolt_stats_auto_refresh_interval = 100000;", - Expected: []sql.Row{{}}, - }, - { - Query: "set @@PERSIST.dolt_stats_auto_refresh_threshold = 0", - Expected: []sql.Row{{}}, - }, - { - Query: "call dolt_stats_restart()", - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"restarted thread: mydb"}}, - }, - { - Query: "set @@PERSIST.dolt_stats_auto_refresh_interval = 0;", - Expected: []sql.Row{{}}, - }, - // new restart picks up 0-interval, will start refreshing immediately - { - Query: "call dolt_stats_restart()", - }, - { - Query: "select sleep(.1)", - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"refreshed mydb"}}, - }, - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{2}}, - }, - // kill refresh thread - { - Query: "call dolt_stats_stop()", - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"cancelled thread: mydb"}}, - }, - // insert without refresh thread will not update stats - { - Query: "insert into xy values (1,0,'a'), (3,0,'a'), (5,2,'a'), (7,1,'a')", - }, - { - Query: "select sleep(.1)", - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"cancelled thread: mydb"}}, - }, - // manual analyze will update stats - { - Query: "analyze table xy", - Expected: []sql.Row{{"xy", "analyze", "status", "OK"}}, - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"refreshed mydb"}}, - }, - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{2}}, - }, - // kill refresh thread and delete stats ref - { - Query: "call dolt_stats_drop()", - }, - { - Query: "call dolt_stats_status()", - Expected: []sql.Row{{"dropped"}}, - }, - { - Query: "select count(*) from dolt_statistics", - Expected: []sql.Row{{0}}, - }, - }, - }, - { - Name: "test purge", - SetUpScript: []string{ - "set @@PERSIST.dolt_stats_auto_refresh_enabled = 0;", - "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", - "insert into xy values (1, 1, 'a'), (2,1,'a'), (3,1,'a'), (4,2,'b'), (5,2,'b'), (6,3,'c');", - "analyze table xy", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select count(*) as cnt from dolt_statistics group by table_name, index_name order by cnt", - Expected: []sql.Row{{1}, {1}}, - }, - { - Query: "call dolt_stats_purge()", - }, - { - Query: "select count(*) from dolt_statistics;", - Expected: []sql.Row{{0}}, - }, - }, - }, - { - Name: "test prune", - SetUpScript: []string{ - "set @@PERSIST.dolt_stats_auto_refresh_enabled = 0;", - "CREATE table xy (x bigint primary key, y int, z varchar(500), key(y,z));", - "insert into xy values (1, 1, 'a'), (2,1,'a'), (3,1,'a'), (4,2,'b'), (5,2,'b'), (6,3,'c');", - "analyze table xy", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select count(*) as cnt from dolt_statistics group by table_name, index_name order by cnt", - Expected: []sql.Row{{1}, {1}}, - }, - { - Query: "call dolt_stats_prune()", - }, - { - Query: "select count(*) from dolt_statistics;", - Expected: []sql.Row{{2}}, - }, - }, - }, -} - -func mustNewStatQual(s string) sql.StatQualifier { - qual, _ := sql.NewQualifierFromString(s) - return qual -} diff --git a/go/libraries/doltcore/sqle/statspro/bucket_builder.go b/go/libraries/doltcore/sqle/statspro/bucket_builder.go index f521ebe83bd..2c974223f84 100644 --- a/go/libraries/doltcore/sqle/statspro/bucket_builder.go +++ b/go/libraries/doltcore/sqle/statspro/bucket_builder.go @@ -17,9 +17,10 @@ package statspro import ( "container/heap" "context" + "sort" + "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/stats" - "sort" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" diff --git a/go/libraries/doltcore/sqle/statspro/gc.go b/go/libraries/doltcore/sqle/statspro/gc.go index e8ecd9371ec..fbfd14783e1 100644 --- a/go/libraries/doltcore/sqle/statspro/gc.go +++ b/go/libraries/doltcore/sqle/statspro/gc.go @@ -17,16 +17,18 @@ package statspro import ( "context" "errors" + "log" + "strconv" + "strings" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "log" - "strconv" - "strings" ) type GcMarkJob struct { diff --git a/go/libraries/doltcore/sqle/statspro/noop_provider.go b/go/libraries/doltcore/sqle/statspro/noop_provider.go index f54e84d51b3..c17dae10f41 100644 --- a/go/libraries/doltcore/sqle/statspro/noop_provider.go +++ b/go/libraries/doltcore/sqle/statspro/noop_provider.go @@ -1,9 +1,10 @@ package statspro import ( + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/go-mysql-server/sql" ) type StatsNoop struct{} diff --git a/go/libraries/doltcore/sqle/statspro/provider.go b/go/libraries/doltcore/sqle/statspro/provider.go index f58920287e5..4884a6644e9 100644 --- a/go/libraries/doltcore/sqle/statspro/provider.go +++ b/go/libraries/doltcore/sqle/statspro/provider.go @@ -17,6 +17,15 @@ package statspro import ( "context" "fmt" + "log" + "path" + "path/filepath" + "strings" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" + "golang.org/x/sync/errgroup" + "github.com/dolthub/dolt/go/cmd/dolt/doltversion" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -28,13 +37,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/stats" - "golang.org/x/sync/errgroup" - "log" - "path" - "path/filepath" - "strings" ) var _ sql.StatsProvider = (*StatsCoord)(nil) diff --git a/go/libraries/doltcore/sqle/statspro/scheduler.go b/go/libraries/doltcore/sqle/statspro/scheduler.go index a68ac3c07fa..658936073bf 100644 --- a/go/libraries/doltcore/sqle/statspro/scheduler.go +++ b/go/libraries/doltcore/sqle/statspro/scheduler.go @@ -18,6 +18,18 @@ import ( "context" "errors" "fmt" + "io" + "log" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" + "github.com/sirupsen/logrus" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/ref" @@ -29,16 +41,6 @@ import ( "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/stats" - "github.com/sirupsen/logrus" - "io" - "log" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" ) type StatsJob interface { @@ -362,7 +364,6 @@ func (sc *StatsCoord) lockedStop(ctx context.Context) error { case <-j.done: return nil } - return nil } func (sc *StatsCoord) Restart(ctx context.Context) error { diff --git a/go/libraries/doltcore/sqle/statspro/scheduler_test.go b/go/libraries/doltcore/sqle/statspro/scheduler_test.go index 8ee659ab0d0..1dce6e7a943 100644 --- a/go/libraries/doltcore/sqle/statspro/scheduler_test.go +++ b/go/libraries/doltcore/sqle/statspro/scheduler_test.go @@ -17,20 +17,6 @@ package statspro import ( "context" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" - "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" - "github.com/dolthub/dolt/go/libraries/doltcore/env" - "github.com/dolthub/dolt/go/libraries/doltcore/ref" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" - "github.com/dolthub/dolt/go/store/prolly/tree" - gms "github.com/dolthub/go-mysql-server" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/analyzer" - "github.com/dolthub/go-mysql-server/sql/stats" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" "io" "log" "os" @@ -39,6 +25,22 @@ import ( "sync" "testing" "time" + + gms "github.com/dolthub/go-mysql-server" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/analyzer" + "github.com/dolthub/go-mysql-server/sql/stats" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" + "github.com/dolthub/dolt/go/store/prolly/tree" ) func TestScheduleLoop(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/statspro/script_test.go b/go/libraries/doltcore/sqle/statspro/script_test.go index cd61fe9acc0..c866b8c85f1 100644 --- a/go/libraries/doltcore/sqle/statspro/script_test.go +++ b/go/libraries/doltcore/sqle/statspro/script_test.go @@ -1,12 +1,14 @@ package statspro import ( - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures" - "github.com/dolthub/go-mysql-server/sql" - "github.com/stretchr/testify/require" "log" "strconv" "testing" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures" ) type scriptTest struct { @@ -687,7 +689,7 @@ func TestStatScripts(t *testing.T) { }, }, } - + for _, tt := range scripts { t.Run(tt.name, func(t *testing.T) { ctx, sqlEng, sc, _ := emptySetup(t, threads, false) diff --git a/go/libraries/doltcore/sqle/statspro/seed_job.go b/go/libraries/doltcore/sqle/statspro/seed_job.go index 6b58c9604f5..3704c546b4f 100644 --- a/go/libraries/doltcore/sqle/statspro/seed_job.go +++ b/go/libraries/doltcore/sqle/statspro/seed_job.go @@ -18,6 +18,11 @@ import ( "context" "errors" "fmt" + "strings" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/sqle" @@ -26,9 +31,6 @@ import ( "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/stats" - "strings" ) func (sc *StatsCoord) seedDbTables(ctx context.Context, j SeedDbTablesJob) (ret []StatsJob, err error) { diff --git a/go/libraries/doltcore/sqle/statspro/stats_kv.go b/go/libraries/doltcore/sqle/statspro/stats_kv.go index 7f3a0121bc1..b24492597d3 100644 --- a/go/libraries/doltcore/sqle/statspro/stats_kv.go +++ b/go/libraries/doltcore/sqle/statspro/stats_kv.go @@ -19,18 +19,20 @@ import ( "encoding/binary" "errors" "fmt" + "strconv" + "strings" + "sync" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" + "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/stats" - "github.com/dolthub/go-mysql-server/sql/types" - "strconv" - "strings" - "sync" ) var ErrIncompatibleVersion = errors.New("client stats version mismatch") diff --git a/go/libraries/doltcore/sqle/statspro/stats_kv_test.go b/go/libraries/doltcore/sqle/statspro/stats_kv_test.go index b1e111a1a11..94907998137 100644 --- a/go/libraries/doltcore/sqle/statspro/stats_kv_test.go +++ b/go/libraries/doltcore/sqle/statspro/stats_kv_test.go @@ -16,16 +16,18 @@ package statspro import ( "context" + "strconv" + "strings" + "testing" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" + "github.com/stretchr/testify/require" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/stats" - "github.com/stretchr/testify/require" - "strconv" - "strings" - "testing" ) func TestProllyKv(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/statspro/validate.go b/go/libraries/doltcore/sqle/statspro/validate.go index 63279bdbede..f47f92f1580 100644 --- a/go/libraries/doltcore/sqle/statspro/validate.go +++ b/go/libraries/doltcore/sqle/statspro/validate.go @@ -17,14 +17,16 @@ package statspro import ( "context" "fmt" + "strings" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - "strings" ) func generateDeps( diff --git a/go/performance/scripts/dg_sysbench.sh b/go/performance/scripts/dg_sysbench.sh new file mode 100755 index 00000000000..0ce8ca1927a --- /dev/null +++ b/go/performance/scripts/dg_sysbench.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -e +set -o pipefail + +SYSBENCH_TEST="oltp_insert_only" +WORKING_DIR=`mktemp -d` +PPROF=0 +PORT=5433 + +# parse options +# superuser.com/questions/186272/ +while test $# -gt 0 +do + case "$1" in + + --new-new) export DOLT_DEFAULT_BIN_FORMAT="__DOLT__" && + export ENABLE_ROW_ITER_2=true + ;; + + --no-exchange) export SINGLE_THREAD_FEATURE_FLAG=true + ;; + + # benchmark with pprof profiling + --pprof) PPROF=1 + ;; + + # run dolt single threaded + --single) export GOMAXPROCS=1 + ;; + + --row2) export ENABLE_ROW_ITER_2=true + ;; + + --journal) export DOLT_ENABLE_CHUNK_JOURNAL=true + ;; + + # specify sysbench benchmark + *) SYSBENCH_TEST="$1" + ;; + + esac + shift +done + +if [ ! -d "./sysbench-lua-scripts" ]; then + git clone https://github.com/dolthub/sysbench-lua-scripts.git +fi + +# collect custom sysbench scripts +cp ./sysbench-lua-scripts/*.lua "$WORKING_DIR" +cd "$WORKING_DIR" + +# make a sql-server config file +cat < dolt-config.yaml +log_level: "info" + +behavior: + read_only: false + +user: + name: "user" + password: "pass" + +listener: + host: "0.0.0.0" + port: $PORT + read_timeout_millis: 28800000 + write_timeout_millis: 28800000 + +data_dir: . +YAML + +# start a server +mkdir sbtest +cd sbtest +doltgres -config="../dolt-config.yaml" 2> prepare.log & +SERVER_PID="$!" + +set -x + +sleep 1 + +ps aux | grep "doltgres" +lsof -iTCP -sTCP:LISTEN +echo $SERVER_PID +psql --port $PORT --host=0.0.0.0 --db=doltgres -c "create database sbtest" + + +# stop it if it crashes +cleanup() { + kill -15 "$SERVER_PID" +} +trap cleanup EXIT + +# setup benchmark +echo "benchmark $SYSBENCH_TEST bootstrapping at $WORKING_DIR" + + +sysbench \ + --db-driver="pgsql" \ + --pgsql-host="0.0.0.0" \ + --pgsql-port="$PORT" \ + --pgsql-user="user" \ + --pgsql-password="pass" \ + "$SYSBENCH_TEST" prepare + +# restart server to isolate bench run +kill -15 "$SERVER_PID" + +# maybe run with pprof +if [ "$PPROF" -eq 1 ]; then + doltgres --prof cpu -config="../dolt-config.yaml" 2> run.log & +else + doltgres -config="../dolt-config.yaml" 2> run.log & +fi +SERVER_PID="$!" +sleep 1 + + +# run benchmark +echo "benchmark $SYSBENCH_TEST starting at $WORKING_DIR" + +sysbench \ + --db-driver="pgsql" \ + --pgsql-host="0.0.0.0" \ + --pgsql-port="$PORT" \ + --pgsql-user="user" \ + --pgsql-password="pass" \ + --db-ps-mode=disable \ + --time=30 \ + --db-ps-mode=disable \ + "$SYSBENCH_TEST" run + +unset DOLT_ENABLE_CHUNK_JOURNAL +unset DOLT_DEFAULT_BIN_FORMAT +unset ENABLE_ROW_ITER_2 +unset SINGLE_THREAD_FEATURE_FLAG +unset GOMAXPROCS + +echo "benchmark $SYSBENCH_TEST complete at $WORKING_DIR" +if [ "$PPROF" -eq 1 ]; then + # parse run.log to output the profile location + head -n1 "$WORKING_DIR/run.log" | cut -d ":" -f 4 +fi +echo "" diff --git a/go/performance/utils/benchmark_runner/sysbench.go b/go/performance/utils/benchmark_runner/sysbench.go index 5953368b5b2..e6c594e2ce7 100644 --- a/go/performance/utils/benchmark_runner/sysbench.go +++ b/go/performance/utils/benchmark_runner/sysbench.go @@ -17,15 +17,11 @@ package benchmark_runner import ( "context" "fmt" + "github.com/google/uuid" "os" "os/exec" "path/filepath" "strings" - "time" - - "github.com/jmoiron/sqlx" - - "github.com/google/uuid" ) type sysbenchTesterImpl struct { @@ -149,10 +145,6 @@ func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) { return nil, err } - if err := t.collectStats(ctx); err != nil { - return nil, err - } - fmt.Println("Running test", t.test.GetName()) rs, err := t.run(ctx) @@ -162,76 +154,3 @@ func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) { return rs, nil } - -func (t *sysbenchTesterImpl) collectStats(ctx context.Context) error { - if strings.Contains(t.serverConfig.GetServerExec(), "dolt") && !strings.Contains(t.serverConfig.GetServerExec(), "doltgres") { - db, err := sqlx.Open("mysql", fmt.Sprintf("root:@tcp(%s:%d)/test", t.serverConfig.GetHost(), t.serverConfig.GetPort())) - if err != nil { - return err - } - return collectStats(ctx, db) - } - return nil -} - -func collectStats(ctx context.Context, db *sqlx.DB) error { - c, err := db.Connx(ctx) - if err != nil { - return err - } - - { - // configuration, restart, and check needs to be in the same session - tx, err := c.BeginTxx(ctx, nil) - if err != nil { - return err - } - - if _, err := tx.Exec("set @@GLOBAL.dolt_stats_auto_refresh_enabled = 1;"); err != nil { - return err - } - if _, err := tx.Exec("set @@GLOBAL.dolt_stats_auto_refresh_interval = 0;"); err != nil { - return err - } - if _, err := tx.Exec("set @@PERSIST.dolt_stats_auto_refresh_interval = 0;"); err != nil { - return err - } - if _, err := tx.Exec("set @@PERSIST.dolt_stats_auto_refresh_enabled = 1;"); err != nil { - return err - } - if _, err := tx.Exec("call dolt_stats_restart();"); err != nil { - return err - } - - rows := map[string]interface{}{"cnt": 0} - tick := time.NewTicker(5 * time.Second) - for { - if rows["cnt"] != 0 { - fmt.Printf("collected %d histogram buckets\n", rows["cnt"]) - break - } - select { - case <-tick.C: - res, err := tx.Queryx("select count(*) as cnt from dolt_statistics;") - if err != nil { - return err - } - if !res.Next() { - return fmt.Errorf("failed to set statistics") - } - if err := res.MapScan(rows); err != nil { - return err - } - if err := res.Close(); err != nil { - return err - } - } - } - } - - if _, err := c.QueryContext(ctx, "call dolt_stats_stop();"); err != nil { - return err - } - - return nil -} diff --git a/go/performance/utils/benchmark_runner/tpcc.go b/go/performance/utils/benchmark_runner/tpcc.go index 4c7f01a2444..be265e6b568 100644 --- a/go/performance/utils/benchmark_runner/tpcc.go +++ b/go/performance/utils/benchmark_runner/tpcc.go @@ -20,9 +20,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" - - "github.com/jmoiron/sqlx" ) type tpccTesterImpl struct { @@ -54,17 +51,6 @@ func (t *tpccTesterImpl) outputToResult(output []byte) (*Result, error) { return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, false) } -func (t *tpccTesterImpl) collectStats(ctx context.Context) error { - if strings.Contains(t.serverConfig.GetServerExec(), "dolt") && !strings.Contains(t.serverConfig.GetServerExec(), "doltgres") { - db, err := sqlx.Open("mysql", fmt.Sprintf("root:@tcp(%s:%d)/sbt", t.serverConfig.GetHost(), t.serverConfig.GetPort())) - if err != nil { - return err - } - return collectStats(ctx, db) - } - return nil -} - func (t *tpccTesterImpl) prepare(ctx context.Context) error { args := t.test.GetPrepareArgs(t.serverConfig) cmd := exec.CommandContext(ctx, t.tpccCommand, args...) @@ -119,10 +105,6 @@ func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) { return nil, err } - if err := t.collectStats(ctx); err != nil { - return nil, err - } - fmt.Println("Running test", t.test.GetName()) rs, err := t.run(ctx) diff --git a/go/store/prolly/tree/mutator.go b/go/store/prolly/tree/mutator.go index b65fdf8f101..a03d042a4a0 100644 --- a/go/store/prolly/tree/mutator.go +++ b/go/store/prolly/tree/mutator.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "github.com/dolthub/dolt/go/store/prolly/message" )