sql-server: Add behavior: auto_gc_behavior: enable.

When Auto GC is enabled, the running sql-server will periodically
collect a Dolt database that is growing in size. This behavior
is currently experimental. Tuning the behavior around how often to collect is
ongoing work.

go/cmd/dolt/commands/engine/sqlengine.go

@@ -40,6 +40,7 @@ import (
 	dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	dblr "github.com/dolthub/dolt/go/libraries/doltcore/sqle/binlogreplication"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/kvexec"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/mysql_file_handler"
@@ -80,6 +81,7 @@ type SqlEngineConfig struct {
 	JwksConfig                 []servercfg.JwksConfig
 	SystemVariables            SystemVariables
 	ClusterController          *cluster.Controller
+	AutoGCController           *dsqle.AutoGCController
 	BinlogReplicaController    binlogreplication.BinlogReplicaController
 	EventSchedulerStatus       eventscheduler.SchedulerStatus
 }
@@ -113,7 +115,8 @@ func NewSqlEngine(
 		return nil, err
 	}
 
-	all := dbs[:]
+	all := make([]dsess.SqlDatabase, len(dbs))
+	copy(all, dbs)
 
 	// this is overwritten only for server sessions
 	for _, db := range dbs {
@@ -192,6 +195,17 @@ func NewSqlEngine(
 	statsPro := statspro.NewProvider(pro, statsnoms.NewNomsStatsFactory(mrEnv.RemoteDialProvider()))
 	engine.Analyzer.Catalog.StatsProvider = statsPro
 
+	if config.AutoGCController != nil {
+		err = config.AutoGCController.RunBackgroundThread(bThreads, sqlEngine.NewDefaultContext)
+		if err != nil {
+			return nil, err
+		}
+		config.AutoGCController.ApplyCommitHooks(ctx, mrEnv, dbs...)
+		pro.InitDatabaseHooks = append(pro.InitDatabaseHooks, config.AutoGCController.InitDatabaseHook())
+		// XXX: We force session aware safepoint controller if auto_gc is on.
+		dprocedures.UseSessionAwareSafepointController = true
+	}
+
 	engine.Analyzer.ExecBuilder = rowexec.NewOverrideBuilder(kvexec.Builder{})
 	sessFactory := doltSessionFactory(pro, statsPro, mrEnv.Config(), bcController, gcSafepointController, config.Autocommit)
 	sqlEngine.provider = pro

go/cmd/dolt/commands/sqlserver/command_line_config.go

@@ -489,6 +489,10 @@ func (cfg *commandLineServerConfig) ValueSet(value string) bool {
 	return ok
 }
 
+func (cfg *commandLineServerConfig) AutoGCBehavior() servercfg.AutoGCBehavior {
+	return stubAutoGCBehavior{}
+}
+
 // DoltServerConfigReader is the default implementation of ServerConfigReader suitable for parsing Dolt config files
 // and command line options.
 type DoltServerConfigReader struct{}
@@ -510,3 +514,10 @@ func (d DoltServerConfigReader) ReadConfigFile(cwdFS filesys.Filesys, file strin
 func (d DoltServerConfigReader) ReadConfigArgs(args *argparser.ArgParseResults, dataDirOverride string) (servercfg.ServerConfig, error) {
 	return NewCommandLineConfig(nil, args, dataDirOverride)
 }
+
+type stubAutoGCBehavior struct {
+}
+
+func (stubAutoGCBehavior) Enable() bool {
+	return false
+}

go/cmd/dolt/commands/sqlserver/server.go

@@ -257,6 +257,16 @@ func ConfigureServices(
 	}
 	controller.Register(InitEventSchedulerStatus)
 
+	InitAutoGCController := &svcs.AnonService{
+		InitF: func(context.Context) error {
+			if serverConfig.AutoGCBehavior().Enable() {
+				config.AutoGCController = sqle.NewAutoGCController(lgr)
+			}
+			return nil
+		},
+	}
+	controller.Register(InitAutoGCController)
+
 	var sqlEngine *engine.SqlEngine
 	InitSqlEngine := &svcs.AnonService{
 		InitF: func(ctx context.Context) (err error) {

go/libraries/doltcore/doltdb/doltdb.go

@@ -1917,23 +1917,21 @@ func (ddb *DoltDB) IsTableFileStore() bool {
 
 // ChunkJournal returns the ChunkJournal for this DoltDB, if one is in use.
 func (ddb *DoltDB) ChunkJournal() *nbs.ChunkJournal {
-	tableFileStore, ok := datas.ChunkStoreFromDatabase(ddb.db).(chunks.TableFileStore)
-	if !ok {
-		return nil
-	}
+	cs := datas.ChunkStoreFromDatabase(ddb.db)
 
-	generationalNbs, ok := tableFileStore.(*nbs.GenerationalNBS)
-	if !ok {
-		return nil
+	var store *nbs.NomsBlockStore
+	generationalNBS, ok := cs.(*nbs.GenerationalNBS)
+	if ok {
+		store = generationalNBS.NewGen().(*nbs.NomsBlockStore)
+	} else {
+		store = cs.(*nbs.NomsBlockStore)
 	}
 
-	newGen := generationalNbs.NewGen()
-	nbs, ok := newGen.(*nbs.NomsBlockStore)
-	if !ok {
+	if store != nil {
+		return store.ChunkJournal()
+	} else {
 		return nil
 	}
-
-	return nbs.ChunkJournal()
 }
 
 func (ddb *DoltDB) TableFileStoreHasJournal(ctx context.Context) (bool, error) {

go/libraries/doltcore/servercfg/serverconfig.go

@@ -48,6 +48,7 @@ const (
 	DefaultReadOnly                = false
 	DefaultLogLevel                = LogLevel_Info
 	DefaultAutoCommit              = true
+	DefaultAutoGCBehaviorEnable    = false
 	DefaultDoltTransactionCommit   = false
 	DefaultMaxConnections          = 100
 	DefaultDataDir                 = "."
@@ -198,6 +199,8 @@ type ServerConfig interface {
 	EventSchedulerStatus() string
 	// ValueSet returns whether the value string provided was explicitly set in the config
 	ValueSet(value string) bool
+	// AutoGCBehavior defines parameters around how auto-GC works for the running server.
+	AutoGCBehavior() AutoGCBehavior
 }
 
 // DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values.
@@ -214,6 +217,9 @@ func defaultServerConfigYAML() *YAMLConfig {
 			ReadOnly:              ptr(DefaultReadOnly),
 			AutoCommit:            ptr(DefaultAutoCommit),
 			DoltTransactionCommit: ptr(DefaultDoltTransactionCommit),
+			AutoGCBehavior: &AutoGCBehaviorYAMLConfig{
+				Enable_: ptr(DefaultAutoGCBehaviorEnable),
+			},
 		},
 		UserConfig: UserYAMLConfig{
 			Name:     ptr(""),
@@ -445,3 +451,7 @@ func CheckForUnixSocket(config ServerConfig) (string, bool, error) {
 
 	return "", false, nil
 }
+
+type AutoGCBehavior interface {
+	Enable() bool
+}

go/libraries/doltcore/servercfg/yaml_config.go

@@ -65,6 +65,8 @@ type BehaviorYAMLConfig struct {
 	DoltTransactionCommit *bool `yaml:"dolt_transaction_commit,omitempty"`
 
 	EventSchedulerStatus *string `yaml:"event_scheduler,omitempty" minver:"1.17.0"`
+
+	AutoGCBehavior *AutoGCBehaviorYAMLConfig `yaml:"auto_gc_behavior,omitempty" minver:"TBD"`
 }
 
 // UserYAMLConfig contains server configuration regarding the user account clients must use to connect
@@ -176,6 +178,7 @@ func YamlConfigFromFile(fs filesys.Filesys, path string) (ServerConfig, error) {
 
 func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig {
 	systemVars := cfg.SystemVars()
+	autoGCBehavior := toAutoGCBehaviorYAML(cfg.AutoGCBehavior())
 	return &YAMLConfig{
 		LogLevelStr:       ptr(string(cfg.LogLevel())),
 		MaxQueryLenInLogs: nillableIntPtr(cfg.MaxLoggedQueryLen()),
@@ -186,6 +189,7 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig {
 			DisableClientMultiStatements: ptr(cfg.DisableClientMultiStatements()),
 			DoltTransactionCommit:        ptr(cfg.DoltTransactionCommit()),
 			EventSchedulerStatus:         ptr(cfg.EventSchedulerStatus()),
+			AutoGCBehavior:               autoGCBehavior,
 		},
 		ListenerConfig: ListenerYAMLConfig{
 			HostStr:                 ptr(cfg.Host()),
@@ -817,6 +821,13 @@ func (cfg YAMLConfig) ClusterConfig() ClusterConfig {
 	return cfg.ClusterCfg
 }
 
+func (cfg YAMLConfig) AutoGCBehavior() AutoGCBehavior {
+	if cfg.BehaviorConfig.AutoGCBehavior == nil {
+		return nil
+	}
+	return cfg.BehaviorConfig.AutoGCBehavior
+}
+
 func (cfg YAMLConfig) EventSchedulerStatus() string {
 	if cfg.BehaviorConfig.EventSchedulerStatus == nil {
 		return "ON"
@@ -922,3 +933,20 @@ func (cfg YAMLConfig) ValueSet(value string) bool {
 	}
 	return false
 }
+
+type AutoGCBehaviorYAMLConfig struct {
+	Enable_ *bool `yaml:"enable,omitempty" minver:"TBD"`
+}
+
+func (a *AutoGCBehaviorYAMLConfig) Enable() bool {
+	if a.Enable_ == nil {
+		return false
+	}
+	return *a.Enable_
+}
+
+func toAutoGCBehaviorYAML(a AutoGCBehavior) *AutoGCBehaviorYAMLConfig {
+	return &AutoGCBehaviorYAMLConfig{
+		Enable_: ptr(a.Enable()),
+	}
+}

go/libraries/doltcore/servercfg/yaml_config_test.go

@@ -34,6 +34,8 @@ behavior:
     dolt_transaction_commit: true
     disable_client_multi_statements: false
     event_scheduler: ON
+    auto_gc_behavior:
+        enable: false
 
 listener:
     host: localhost