Merge pull request #8636 from dolthub/taylor/more-dg-types

Schema fixes for `dolt_status`, `dolt_merge_status`, and `dolt_constraint_violations_*` tables for doltgres

go/libraries/doltcore/doltdb/table.go

@@ -20,13 +20,9 @@ import (
 	"fmt"
 	"unicode"
 
-	"github.com/dolthub/go-mysql-server/sql"
-	gmstypes "github.com/dolthub/go-mysql-server/sql/types"
-
 	"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
-	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/prolly/tree"
 	"github.com/dolthub/dolt/go/store/types"
@@ -35,8 +31,6 @@ import (
 
 var ErrNoConflictsResolved = errors.New("no conflicts resolved")
 
-const dolt_row_hash_tag = 0
-
 // IsValidTableName checks if name is a valid identifier, and doesn't end with space characters
 func IsValidTableName(name string) bool {
 	if len(name) == 0 || unicode.IsSpace(rune(name[len(name)-1])) {
@@ -351,46 +345,6 @@ func (t *Table) getNomsConflictSchemas(ctx context.Context) (base, sch, mergeSch
 	return cs.Base, cs.Schema, cs.MergeSchema, nil
 }
 
-// GetConstraintViolationsSchema returns this table's dolt_constraint_violations system table schema.
-func (t *Table) GetConstraintViolationsSchema(ctx context.Context) (schema.Schema, error) {
-	sch, err := t.GetSchema(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	typeType, err := typeinfo.FromSqlType(
-		gmstypes.MustCreateEnumType([]string{"foreign key", "unique index", "check constraint", "not null"}, sql.Collation_Default))
-	if err != nil {
-		return nil, err
-	}
-	typeCol, err := schema.NewColumnWithTypeInfo("violation_type", schema.DoltConstraintViolationsTypeTag, typeType, true, "", false, "")
-	if err != nil {
-		return nil, err
-	}
-	infoCol, err := schema.NewColumnWithTypeInfo("violation_info", schema.DoltConstraintViolationsInfoTag, typeinfo.JSONType, false, "", false, "")
-	if err != nil {
-		return nil, err
-	}
-
-	colColl := schema.NewColCollection()
-
-	// the commit hash or working set hash of the right side during merge
-	colColl = colColl.Append(schema.NewColumn("from_root_ish", 0, types.StringKind, false))
-	colColl = colColl.Append(typeCol)
-	if schema.IsKeyless(sch) {
-		// If this is a keyless table, we need to add a new column for the keyless table's generated row hash.
-		// We need to add this internal row hash value, in order to guarantee a unique primary key in the
-		// constraint violations table.
-		colColl = colColl.Append(schema.NewColumn("dolt_row_hash", dolt_row_hash_tag, types.BlobKind, true))
-	} else {
-		colColl = colColl.Append(sch.GetPKCols().GetColumns()...)
-	}
-	colColl = colColl.Append(sch.GetNonPKCols().GetColumns()...)
-	colColl = colColl.Append(infoCol)
-
-	return schema.SchemaFromCols(colColl)
-}
-
 // GetConstraintViolations returns a map of all constraint violations for this table, along with a bool indicating
 // whether the table has any violations.
 func (t *Table) GetConstraintViolations(ctx context.Context) (types.Map, error) {

go/libraries/doltcore/merge/violations_prolly.go

@@ -25,7 +25,7 @@ import (
 	"github.com/dolthub/dolt/go/store/val"
 )
 
-func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, vd val.TupleDesc, ns tree.NodeStore) (violationType uint64, key sql.Row, value sql.Row, err error) {
+func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, vd val.TupleDesc, ns tree.NodeStore) (violationType any, key sql.Row, value sql.Row, err error) {
 	art, err := itr.Next(ctx)
 	if err != nil {
 		return
@@ -56,7 +56,13 @@ func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, v
 	return MapCVType(art.ArtType), key, value, nil
 }
 
-func MapCVType(artifactType prolly.ArtifactType) (outType uint64) {
+// MapCVType maps an ArtifactType to value for a sql.Row in the dolt_constraint_violations_* table.
+// This is used by Doltgres to convert the ArtifactType to the correct type.
+var MapCVType = func(artifactType prolly.ArtifactType) any {
+	return mapCVType(artifactType)
+}
+
+func mapCVType(artifactType prolly.ArtifactType) (outType uint64) {
 	switch artifactType {
 	case prolly.ArtifactTypeForeignKeyViol:
 		outType = uint64(CvType_ForeignKey)
@@ -72,7 +78,16 @@ func MapCVType(artifactType prolly.ArtifactType) (outType uint64) {
 	return
 }
 
-func UnmapCVType(in CvType) (out prolly.ArtifactType) {
+// UnmapCVType unmaps a sql.Row value from the dolt_constraint_violations_* table to an ArtifactType.
+// This is used by Doltgres to convert a value of a different type to an ArtifactType.
+var UnmapCVType = func(in any) (out prolly.ArtifactType) {
+	if cv, ok := in.(uint64); ok {
+		return unmapCVType(CvType(cv))
+	}
+	panic("invalid type")
+}
+
+func unmapCVType(in CvType) (out prolly.ArtifactType) {
 	switch in {
 	case CvType_ForeignKey:
 		out = prolly.ArtifactTypeForeignKeyViol

go/libraries/doltcore/sqle/dtables/constraint_violations.go

@@ -42,7 +42,7 @@ func newNomsCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.Root
 	if err != nil {
 		return nil, err
 	}
-	cvSch, err := tbl.GetConstraintViolationsSchema(ctx)
+	cvSch, err := getConstraintViolationsSchema(ctx, tbl, tblName, root)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go

@@ -15,31 +15,89 @@
 package dtables
 
 import (
+	"context"
 	"encoding/json"
 
 	"github.com/dolthub/go-mysql-server/sql"
+	gmstypes "github.com/dolthub/go-mysql-server/sql/types"
+	"github.com/dolthub/vitess/go/sqltypes"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
 	"github.com/dolthub/dolt/go/libraries/doltcore/merge"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
+	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/pool"
 	"github.com/dolthub/dolt/go/store/prolly"
 	"github.com/dolthub/dolt/go/store/prolly/tree"
+	"github.com/dolthub/dolt/go/store/types"
 	"github.com/dolthub/dolt/go/store/val"
 )
 
+func getDoltConstraintViolationsBaseSqlSchema() sql.Schema {
+	return []*sql.Column{
+		{Name: "from_root_ish", Type: gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 1023), PrimaryKey: false, Nullable: true},
+		{Name: "violation_type", Type: gmstypes.MustCreateEnumType([]string{"foreign key", "unique index", "check constraint", "not null"}, sql.Collation_Default), PrimaryKey: true},
+	}
+}
+
+// GetDoltConstraintViolationsBaseSqlSchema returns the base schema for the dolt_constraint_violations_* system table.
+// This is used by Doltgres to update the dolt_constraint_violations_* schema using Doltgres types.
+var GetDoltConstraintViolationsBaseSqlSchema = getDoltConstraintViolationsBaseSqlSchema
+
+// getConstraintViolationsSchema returns a table's dolt_constraint_violations system table schema.
+func getConstraintViolationsSchema(ctx context.Context, t *doltdb.Table, tn doltdb.TableName, root doltdb.RootValue) (schema.Schema, error) {
+	sch, err := t.GetSchema(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	baseSch := sql.NewPrimaryKeySchema(GetDoltConstraintViolationsBaseSqlSchema())
+	baseDoltSch, err := sqlutil.ToDoltSchema(ctx, root, tn, baseSch, root, sql.Collation_Default)
+	if err != nil {
+		return nil, err
+	}
+	baseColColl := baseDoltSch.GetAllCols()
+	baseCols := baseColColl.GetColumns()
+
+	schSize := sch.GetAllCols().Size()
+	if schema.IsKeyless(sch) {
+		// Keyless tables have an additional dolt_row_hash column
+		schSize += 1
+	}
+
+	cols := make([]schema.Column, 0, baseColColl.Size()+schSize)
+	cols = append(cols, baseCols[0:2]...)
+	infoCol, err := schema.NewColumnWithTypeInfo("violation_info", schema.DoltConstraintViolationsInfoTag, typeinfo.JSONType, false, "", false, "")
+	if err != nil {
+		return nil, err
+	}
+
+	if schema.IsKeyless(sch) {
+		// If this is a keyless table, we need to add a new column for the keyless table's generated row hash.
+		// We need to add this internal row hash value, in order to guarantee a unique primary key in the
+		// constraint violations table.
+		cols = append(cols, schema.NewColumn("dolt_row_hash", 0, types.BlobKind, true))
+	} else {
+		cols = append(cols, sch.GetPKCols().GetColumns()...)
+	}
+	cols = append(cols, sch.GetNonPKCols().GetColumns()...)
+	cols = append(cols, infoCol)
+
+	return schema.NewSchema(schema.NewColCollection(cols...), nil, schema.Collation_Default, nil, nil)
+}
+
 func newProllyCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.RootValue, rs RootSetter) (sql.Table, error) {
 	var tbl *doltdb.Table
 	var err error
 	tbl, tblName, err = getTableInsensitiveOrError(ctx, root, tblName)
 	if err != nil {
 		return nil, err
 	}
-	cvSch, err := tbl.GetConstraintViolationsSchema(ctx)
+	cvSch, err := getConstraintViolationsSchema(ctx, tbl, tblName, root)
 	if err != nil {
 		return nil, err
 	}
@@ -278,7 +336,7 @@ func (d *prollyCVDeleter) Delete(ctx *sql.Context, r sql.Row) error {
 	d.kb.PutCommitAddr(d.kd.Count()-2, h)
 
 	// Finally the artifact type
-	artType := merge.UnmapCVType(merge.CvType(r[1].(uint64)))
+	artType := merge.UnmapCVType(r[1])
 	d.kb.PutUint8(d.kd.Count()-1, uint8(artType))
 
 	key := d.kb.Build(d.pool)

go/libraries/doltcore/sqle/dtables/merge_status_table.go

@@ -41,16 +41,24 @@ func (mst MergeStatusTable) String() string {
 	return mst.tableName
 }
 
-func (mst MergeStatusTable) Schema() sql.Schema {
+func getDoltMergeStatusSchema(dbName, tableName string) sql.Schema {
 	return []*sql.Column{
-		{Name: "is_merging", Type: types.Boolean, Source: mst.tableName, PrimaryKey: false, Nullable: false, DatabaseSource: mst.dbName},
-		{Name: "source", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName},
-		{Name: "source_commit", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName},
-		{Name: "target", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName},
-		{Name: "unmerged_tables", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName},
+		{Name: "is_merging", Type: types.Boolean, Source: tableName, PrimaryKey: false, Nullable: false, DatabaseSource: dbName},
+		{Name: "source", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName},
+		{Name: "source_commit", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName},
+		{Name: "target", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName},
+		{Name: "unmerged_tables", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName},
 	}
 }
 
+// GetDoltMergeStatusSchema returns the schema of the dolt_merge_status system table. This is used
+// by Doltgres to update the dolt_merge_status schema using Doltgres types.
+var GetDoltMergeStatusSchema = getDoltMergeStatusSchema
+
+func (mst MergeStatusTable) Schema() sql.Schema {
+	return GetDoltMergeStatusSchema(mst.dbName, mst.tableName)
+}
+
 func (mst MergeStatusTable) Collation() sql.CollationID {
 	return sql.Collation_Default
 }

go/libraries/doltcore/sqle/dtables/status_table.go

@@ -61,14 +61,22 @@ func (st StatusTable) String() string {
 	return st.tableName
 }
 
-func (st StatusTable) Schema() sql.Schema {
+func getDoltStatusSchema(tableName string) sql.Schema {
 	return []*sql.Column{
-		{Name: "table_name", Type: types.Text, Source: st.tableName, PrimaryKey: true, Nullable: false},
-		{Name: "staged", Type: types.Boolean, Source: st.tableName, PrimaryKey: true, Nullable: false},
-		{Name: "status", Type: types.Text, Source: st.tableName, PrimaryKey: true, Nullable: false},
+		{Name: "table_name", Type: types.Text, Source: tableName, PrimaryKey: true, Nullable: false},
+		{Name: "staged", Type: types.Boolean, Source: tableName, PrimaryKey: true, Nullable: false},
+		{Name: "status", Type: types.Text, Source: tableName, PrimaryKey: true, Nullable: false},
 	}
 }
 
+// GetDoltStatusSchema returns the schema of the dolt_status system table. This is used
+// by Doltgres to update the dolt_status schema using Doltgres types.
+var GetDoltStatusSchema = getDoltStatusSchema
+
+func (st StatusTable) Schema() sql.Schema {
+	return GetDoltStatusSchema(st.tableName)
+}
+
 func (st StatusTable) Collation() sql.CollationID {
 	return sql.Collation_Default
 }

go/libraries/doltcore/sqle/dtables/workspace_table.go

@@ -409,6 +409,9 @@ func NewWorkspaceTable(ctx *sql.Context, workspaceTableName string, tableName do
 
 	sch := sql.NewPrimaryKeySchema(GetDoltWorkspaceBaseSqlSchema())
 	baseDoltSch, err := sqlutil.ToDoltSchema(ctx, head, tableName, sch, head, sql.Collation_Default)
+	if err != nil {
+		return nil, err
+	}
 
 	totalSch, err := workspaceSchema(fromSch, toSch, baseDoltSch)
 	if err != nil {