Skip to content

Commit

Permalink
Merge pull request #8482 from dolthub/taylor/dolt-schema
Browse files Browse the repository at this point in the history 
Move dolt system tables to dolt schema for doltgres
  • Loading branch information
@tbantle22
tbantle22 authored Oct 28, 2024
2 parents 1d97bd2 + ac54d05 commit 746ebc6
Show file tree
Hide file tree
Showing 11 changed files with 225 additions and 120 deletions.
75 changes: 48 additions & 27 deletions go/libraries/doltcore/doltdb/system_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func IsFullTextTable(name string) bool {
// IsReadOnlySystemTable returns whether the table name given is a system table that should not be included in command line
// output (e.g. dolt status) by default.
func IsReadOnlySystemTable(name string) bool {
return HasDoltPrefix(name) && !set.NewStrSet(writeableSystemTables).Contains(name) && !IsFullTextTable(name)
return HasDoltPrefix(name) && !set.NewStrSet(getWriteableSystemTables()).Contains(name) && !IsFullTextTable(name)
}

// IsNonAlterableSystemTable returns whether the table name given is a system table that cannot be dropped or altered
Expand Down Expand Up @@ -120,7 +120,7 @@ func GetPersistedSystemTables(ctx context.Context, root RootValue) ([]string, er

// GetGeneratedSystemTables returns table names of all generated system tables.
func GetGeneratedSystemTables(ctx context.Context, root RootValue) ([]string, error) {
s := set.NewStrSet(generatedSystemTables)
s := set.NewStrSet(getGeneratedSystemTables())

tn, err := root.GetTableNames(ctx, DefaultSchemaName)
if err != nil {
Expand All @@ -137,33 +137,29 @@ func GetGeneratedSystemTables(ctx context.Context, root RootValue) ([]string, er
// The set of reserved dolt_ tables that should be considered part of user space, like any other user-created table,
// for the purposes of the dolt command line. These tables cannot be created or altered explicitly, but can be updated
// like normal SQL tables.
var writeableSystemTables = []string{
DocTableName,
DoltQueryCatalogTableName,
SchemasTableName,
ProceduresTableName,
IgnoreTableName,
RebaseTableName,
}

var persistedSystemTables = []string{
DocTableName,
DoltQueryCatalogTableName,
SchemasTableName,
ProceduresTableName,
IgnoreTableName,
var getWriteableSystemTables = func() []string {
return []string{
GetDocTableName(),
DoltQueryCatalogTableName,
SchemasTableName,
ProceduresTableName,
IgnoreTableName,
RebaseTableName,
}
}

var generatedSystemTables = []string{
BranchesTableName,
RemoteBranchesTableName,
LogTableName,
TableOfTablesInConflictName,
TableOfTablesWithViolationsName,
CommitsTableName,
CommitAncestorsTableName,
StatusTableName,
RemotesTableName,
var getGeneratedSystemTables = func() []string {
return []string{
GetBranchesTableName(),
RemoteBranchesTableName,
GetLogTableName(),
TableOfTablesInConflictName,
TableOfTablesWithViolationsName,
CommitsTableName,
CommitAncestorsTableName,
GetStatusTableName(),
RemotesTableName,
}
}

var generatedSystemTablePrefixes = []string{
Expand All @@ -182,6 +178,11 @@ const (
ReadmeDoc = "README.md"
)

// GetDocTableName returns the name of the dolt table containing documents such as the license and readme
var GetDocTableName = func() string {
return DocTableName
}

const (
// DocTableName is the name of the dolt table containing documents such as the license and readme
DocTableName = "dolt_docs"
Expand Down Expand Up @@ -246,6 +247,26 @@ const (
DoltWorkspaceTablePrefix = "dolt_workspace_"
)

// GetBranchesTableName returns the branches system table name
var GetBranchesTableName = func() string {
return BranchesTableName
}

// GetLogTableName returns the log system table name
var GetLogTableName = func() string {
return LogTableName
}

// GetStatusTableName returns the status system table name.
var GetStatusTableName = func() string {
return StatusTableName
}

// GetTagsTableName returns the tags table name
var GetTagsTableName = func() string {
return TagsTableName
}

const (
// LogTableName is the log system table name
LogTableName = "dolt_log"
Expand Down
2 changes: 1 addition & 1 deletion go/libraries/doltcore/env/actions/table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ func validateTablesExist(ctx context.Context, currRoot doltdb.RootValue, unknown
func RemoveDocsTable(tbls []string) []string {
var result []string
for _, tblName := range tbls {
if tblName != doltdb.DocTableName {
if tblName != doltdb.GetDocTableName() {
result = append(result, tblName)
}
}
Expand Down
144 changes: 91 additions & 53 deletions go/libraries/doltcore/sqle/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,17 +419,24 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds

var dt sql.Table
found := false
tname := doltdb.TableName{Name: lwrName, Schema: db.schemaName}
switch lwrName {
case doltdb.LogTableName:
if head == nil {
var err error
head, err = ds.GetHeadCommit(ctx, db.RevisionQualifiedName())
if err != nil {
return nil, false, err
}
case doltdb.GetLogTableName(), doltdb.LogTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
if head == nil {
var err error
head, err = ds.GetHeadCommit(ctx, db.RevisionQualifiedName())
if err != nil {
return nil, false, err
}
}

dt, found = dtables.NewLogTable(ctx, db.Name(), db.ddb, head), true
dt, found = dtables.NewLogTable(ctx, db.Name(), db.ddb, head), true
}
case doltdb.DiffTableName:
if head == nil {
var err error
Expand All @@ -456,8 +463,14 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
dt, found = dtables.NewTableOfTablesConstraintViolations(ctx, root), true
case doltdb.SchemaConflictsTableName:
dt, found = dtables.NewSchemaConflictsTable(ctx, db.RevisionQualifiedName(), db.ddb), true
case doltdb.BranchesTableName:
dt, found = dtables.NewBranchesTable(ctx, db), true
case doltdb.GetBranchesTableName(), doltdb.BranchesTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
dt, found = dtables.NewBranchesTable(ctx, db), true
}
case doltdb.RemoteBranchesTableName:
dt, found = dtables.NewRemoteBranchesTable(ctx, db), true
case doltdb.RemotesTableName:
Expand All @@ -466,23 +479,35 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
dt, found = dtables.NewCommitsTable(ctx, db.Name(), db.ddb), true
case doltdb.CommitAncestorsTableName:
dt, found = dtables.NewCommitAncestorsTable(ctx, db.Name(), db.ddb), true
case doltdb.StatusTableName:
sess := dsess.DSessFromSess(ctx.Session)
adapter := dsess.NewSessionStateAdapter(
sess, db.RevisionQualifiedName(),
concurrentmap.New[string, env.Remote](),
concurrentmap.New[string, env.BranchConfig](),
concurrentmap.New[string, env.Remote]())
ws, err := sess.WorkingSet(ctx, db.RevisionQualifiedName())
case doltdb.GetStatusTableName(), doltdb.StatusTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
sess := dsess.DSessFromSess(ctx.Session)
adapter := dsess.NewSessionStateAdapter(
sess, db.RevisionQualifiedName(),
concurrentmap.New[string, env.Remote](),
concurrentmap.New[string, env.BranchConfig](),
concurrentmap.New[string, env.Remote]())
ws, err := sess.WorkingSet(ctx, db.RevisionQualifiedName())
if err != nil {
return nil, false, err
}

dt, found = dtables.NewStatusTable(ctx, db.ddb, ws, adapter), true
dt, found = dtables.NewStatusTable(ctx, db.ddb, ws, adapter), true
}
case doltdb.MergeStatusTableName:
dt, found = dtables.NewMergeStatusTable(db.RevisionQualifiedName()), true
case doltdb.TagsTableName:
dt, found = dtables.NewTagsTable(ctx, db.ddb), true
case doltdb.GetTagsTableName(), doltdb.TagsTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
dt, found = dtables.NewTagsTable(ctx, db.ddb), true
}
case dtables.AccessTableName:
basCtx := branch_control.GetBranchAwareSession(ctx)
if basCtx != nil {
Expand All @@ -508,16 +533,25 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
versionableTable := backingTable.(dtables.VersionableTable)
dt, found = dtables.NewIgnoreTable(ctx, versionableTable), true
}
case doltdb.DocTableName:
backingTable, _, err := db.getTable(ctx, root, doltdb.DocTableName)
case doltdb.GetDocTableName(), doltdb.DocTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if backingTable == nil {
dt, found = dtables.NewEmptyDocsTable(ctx), true
} else {
versionableTable := backingTable.(dtables.VersionableTable)
dt, found = dtables.NewDocsTable(ctx, versionableTable), true
if !resolve.UseSearchPath || isDoltgresSystemTable {
if resolve.UseSearchPath && lwrName == doltdb.DocTableName {
db.schemaName = "dolt"
}
backingTable, _, err := db.getTable(ctx, root, doltdb.GetDocTableName())
if err != nil {
return nil, false, err
}
if backingTable == nil {
dt, found = dtables.NewEmptyDocsTable(ctx), true
} else {
versionableTable := backingTable.(dtables.VersionableTable)
dt, found = dtables.NewDocsTable(ctx, versionableTable), true
}
}
case doltdb.StatisticsTableName:
dt, found = dtables.NewStatisticsTable(ctx, db.Name(), db.ddb, asOf), true
Expand Down Expand Up @@ -691,11 +725,12 @@ func (db Database) GetTableNamesAsOf(ctx *sql.Context, time interface{}) ([]stri
return nil, nil
}

tblNames, err := root.GetTableNames(ctx, doltdb.DefaultSchemaName)
tblNames, err := db.getAllTableNames(ctx, root, false)
if err != nil {
return nil, err
}
return filterDoltInternalTables(tblNames), nil

return filterDoltInternalTables(tblNames, db.schemaName), nil
}

// getTable returns the user table with the given baseName from the root given
Expand Down Expand Up @@ -799,9 +834,7 @@ func (db Database) checkForPgCatalogTable(ctx *sql.Context, tableName string) (s
// case-insensitive manner. The table is returned along with its case-sensitive matched name. An error is returned if
// no such table exists.
func (db Database) resolveUserTable(ctx *sql.Context, root doltdb.RootValue, tableName string) (doltdb.TableName, *doltdb.Table, bool, error) {
// dolt_docs table is branch-specific, not schema-specific for doltgres
isDoltDocs := tableName == doltdb.DocTableName
if resolve.UseSearchPath && db.schemaName == "" && !isDoltDocs {
if resolve.UseSearchPath && db.schemaName == "" {
return resolve.TableWithSearchPath(ctx, root, tableName)
} else {
return db.tableInsensitive(ctx, root, tableName)
Expand Down Expand Up @@ -838,7 +871,7 @@ func (db Database) tableInsensitive(ctx *sql.Context, root doltdb.RootValue, tab
}
}

tableNames, err := db.getAllTableNames(ctx, root)
tableNames, err := db.getAllTableNames(ctx, root, true)
if err != nil {
return doltdb.TableName{}, nil, false, err
}
Expand Down Expand Up @@ -891,20 +924,23 @@ func (db Database) newDoltTable(tableName string, sch schema.Schema, tbl *doltdb
// name resolution in queries is handled by GetTableInsensitive. Use GetAllTableNames for an unfiltered list of all
// tables in user space.
func (db Database) GetTableNames(ctx *sql.Context) ([]string, error) {
tblNames, err := db.GetAllTableNames(ctx)
showSystemTablesVar, err := ctx.GetSessionVariable(ctx, dsess.ShowSystemTables)
if err != nil {
return nil, err
}
// TODO: should just elide system table gathering if we don't need them
showSystemTables, err := ctx.GetSessionVariable(ctx, dsess.ShowSystemTables)

showSystemTables := showSystemTablesVar.(int8) == 1
tblNames, err := db.GetAllTableNames(ctx, showSystemTables)
if err != nil {
return nil, err
}
if showSystemTables.(int8) == 1 {

if showSystemTables {
return tblNames, nil
} else {
return filterDoltInternalTables(tblNames), nil
}

// TODO: Figure out way to remove filterDoltInternalTables
return filterDoltInternalTables(tblNames, db.schemaName), nil
}

func (db Database) SchemaName() string {
Expand All @@ -913,22 +949,17 @@ func (db Database) SchemaName() string {

// GetAllTableNames returns all user-space tables, including system tables in user space
// (e.g. dolt_docs, dolt_query_catalog).
func (db Database) GetAllTableNames(ctx *sql.Context) ([]string, error) {
func (db Database) GetAllTableNames(ctx *sql.Context, showSystemTables bool) ([]string, error) {
root, err := db.GetRoot(ctx)

if err != nil {
return nil, err
}

return db.getAllTableNames(ctx, root)
return db.getAllTableNames(ctx, root, showSystemTables)
}

func (db Database) getAllTableNames(ctx *sql.Context, root doltdb.RootValue) ([]string, error) {
systemTables, err := doltdb.GetGeneratedSystemTables(ctx, root)
if err != nil {
return nil, err
}

func (db Database) getAllTableNames(ctx *sql.Context, root doltdb.RootValue, showSystemTables bool) ([]string, error) {
var err error
var result []string
// If we are in a schema-enabled session and the schema name is not set, we need to union all table names in all
// schemas in the search_path
Expand All @@ -947,14 +978,21 @@ func (db Database) getAllTableNames(ctx *sql.Context, root doltdb.RootValue) ([]
}
}

result = append(result, systemTables...)
if showSystemTables {
systemTables, err := doltdb.GetGeneratedSystemTables(ctx, root)
if err != nil {
return nil, err
}
result = append(result, systemTables...)
}

return result, nil
}

func filterDoltInternalTables(tblNames []string) []string {
func filterDoltInternalTables(tblNames []string, schemaName string) []string {
result := []string{}
for _, tbl := range tblNames {
if !doltdb.HasDoltPrefix(tbl) {
if !doltdb.HasDoltPrefix(tbl) && schemaName != "dolt" {
result = append(result, tbl)
}
}
Expand Down Expand Up @@ -1174,7 +1212,7 @@ func (db Database) CreateIndexedTable(ctx *sql.Context, tableName string, sch sq

// CreateFulltextTableNames returns a set of names that will be used to create Full-Text pseudo-index tables.
func (db Database) CreateFulltextTableNames(ctx *sql.Context, parentTableName string, parentIndexName string) (fulltext.IndexTableNames, error) {
allTableNames, err := db.GetAllTableNames(ctx)
allTableNames, err := db.GetAllTableNames(ctx, true)
if err != nil {
return fulltext.IndexTableNames{}, err
}
Expand Down
6 changes: 6 additions & 0 deletions go/libraries/doltcore/sqle/database_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,12 @@ func (p *DoltDatabaseProvider) CreateCollatedDatabase(ctx *sql.Context, name str
if err != nil {
return err
}
workingRoot, err = workingRoot.CreateDatabaseSchema(ctx, schema.DatabaseSchema{
Name: "dolt",
})
if err != nil {
return err
}

if err = newEnv.UpdateWorkingRoot(ctx, workingRoot); err != nil {
return err
Expand Down
Loading

0 comments on commit 746ebc6

Please sign in to comment.