Skip to content

Commit

Permalink
Support multiline proto format (#120)
Browse files Browse the repository at this point in the history 
* Implement CLI_PROTOTEXT_MULTILINE

* Update README.md
  • Loading branch information
@apstndb
apstndb authored Jan 25, 2025
1 parent fa17868 commit 6f2b02b
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 26 deletions.
51 changes: 38 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,7 @@ They have almost same semantics with [Spanner JDBC properties](https://cloud.goo
| CLI_AUTOWRAP | READ_WRITE | `"TRUE"` |
| CLI_DATABASE_DIALECT | READ_WRITE | `"TRUE"` |
| CLI_ENABLE_HIGHLIGHT | READ_WRITE | `"TRUE"` |
| CLI_PROTOTEXT_MULTILINE | READ_WRITE | `"TRUE"` |

### Batch statements

Expand Down Expand Up @@ -885,7 +886,7 @@ spanner> SYNC PROTO BUNDLE DELETE (`examples.shipping.OrderHistory`);
Query OK, 0 rows affected (8.24 sec)
```

### `PROTO`/`ENUM` value formatting
#### `PROTO`/`ENUM` value formatting

Loaded proto descriptors are used for formatting `PROTO` and `ENUM` column values.

Expand All @@ -894,12 +895,13 @@ spanner> SELECT p, p.*
FROM (
SELECT AS VALUE NEW examples.spanner.music.SingerInfo{singer_id: 1, birth_date: "1970-01-01", nationality: "Japan", genre: "POP"}
) AS p;
+----------------------------------+-----------+------------+-------------+-------+
| p | singer_id | birth_date | nationality | genre |
+----------------------------------+-----------+------------+-------------+-------+
| CAESCjE5NzAtMDEtMDEaBUphcGFuIAA= | 1 | 1970-01-01 | Japan | 0 |
+----------------------------------+-----------+------------+-------------+-------+
1 rows in set (2.22 msecs)
+----------------------------------+-----------+------------+-------------+-------------+
| p | singer_id | birth_date | nationality | genre |
| PROTO<SingerInfo> | INT64 | STRING | STRING | ENUM<Genre> |
+----------------------------------+-----------+------------+-------------+-------------+
| CAESCjE5NzAtMDEtMDEaBUphcGFuIAA= | 1 | 1970-01-01 | Japan | 0 |
+----------------------------------+-----------+------------+-------------+-------------+
1 rows in set (0.33 msecs)
spanner> SET CLI_PROTO_DESCRIPTOR_FILE += "testdata/protos/singer.proto";
Empty set (0.00 sec)
Expand All @@ -919,12 +921,35 @@ spanner> SELECT p, p.*
FROM (
SELECT AS VALUE NEW examples.spanner.music.SingerInfo{singer_id: 1, birth_date: "1970-01-01", nationality: "Japan", genre: "POP"}
) AS p;
+-------------------------------------------------------------------+-----------+------------+-------------+-------+
| p | singer_id | birth_date | nationality | genre |
+-------------------------------------------------------------------+-----------+------------+-------------+-------+
| singer_id:1 birth_date:"1970-01-01" nationality:"Japan" genre:POP | 1 | 1970-01-01 | Japan | POP |
+-------------------------------------------------------------------+-----------+------------+-------------+-------+
1 rows in set (1.43 msecs)
+-------------------------------------------------------------------+-----------+------------+-------------+-------------+
| p | singer_id | birth_date | nationality | genre |
| PROTO<SingerInfo> | INT64 | STRING | STRING | ENUM<Genre> |
+-------------------------------------------------------------------+-----------+------------+-------------+-------------+
| singer_id:1 birth_date:"1970-01-01" nationality:"Japan" genre:POP | 1 | 1970-01-01 | Japan | POP |
+-------------------------------------------------------------------+-----------+------------+-------------+-------------+
1 rows in set (0.87 msecs)
```

You can enable multiline format.

```
spanner> SET CLI_PROTOTEXT_MULTILINE=TRUE;
Empty set (0.00 sec)
spanner> SELECT p, p.*
FROM (
SELECT AS VALUE NEW examples.spanner.music.SingerInfo{singer_id: 1, birth_date: "1970-01-01", nationality: "Japan", genre: "POP"}
) AS p;
+--------------------------+-----------+------------+-------------+-------------+
| p | singer_id | birth_date | nationality | genre |
| PROTO<SingerInfo> | INT64 | STRING | STRING | ENUM<Genre> |
+--------------------------+-----------+------------+-------------+-------------+
| singer_id: 1 | 1 | 1970-01-01 | Japan | POP |
| birth_date: "1970-01-01" | | | | |
| nationality: "Japan" | | | | |
| genre: POP | | | | |
+--------------------------+-----------+------------+-------------+-------------+
1 rows in set (3.37 msecs)
```
### memefish integration

Expand Down
9 changes: 5 additions & 4 deletions decoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"encoding/base64"
"errors"
"strconv"
"strings"

"cloud.google.com/go/spanner"
sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
Expand All @@ -43,7 +44,7 @@ func DecodeColumn(column spanner.GenericColumnValue) (string, error) {
return spanvalue.FormatColumnSpannerCLICompatible(column)
}

func formatConfigWithProto(fds *descriptorpb.FileDescriptorSet) (*spanvalue.FormatConfig, error) {
func formatConfigWithProto(fds *descriptorpb.FileDescriptorSet, multiline bool) (*spanvalue.FormatConfig, error) {
types, err := dynamicTypesByFDS(fds)
if err != nil {
return nil, err
Expand All @@ -57,7 +58,7 @@ func formatConfigWithProto(fds *descriptorpb.FileDescriptorSet) (*spanvalue.Form
FormatStructParen: spanvalue.FormatBracketStruct,
},
FormatComplexPlugins: []spanvalue.FormatComplexFunc{
formatProto(types),
formatProto(types, multiline),
formatEnum(types),
},
FormatNullable: spanvalue.FormatNullableSpannerCLICompatible,
Expand Down Expand Up @@ -85,7 +86,7 @@ type protoEnumResolver interface {
var _ protoEnumResolver = (*dynamicpb.Types)(nil)
var _ protoEnumResolver = (*protoregistry.Types)(nil)

func formatProto(types protoEnumResolver) func(formatter spanvalue.Formatter, value spanner.GenericColumnValue, toplevel bool) (string, error) {
func formatProto(types protoEnumResolver, multiline bool) func(formatter spanvalue.Formatter, value spanner.GenericColumnValue, toplevel bool) (string, error) {
return func(formatter spanvalue.Formatter, value spanner.GenericColumnValue, toplevel bool) (string, error) {
if value.Type.GetCode() != sppb.TypeCode_PROTO {
return "", spanvalue.ErrFallthrough
Expand All @@ -107,7 +108,7 @@ func formatProto(types protoEnumResolver) func(formatter spanvalue.Formatter, va
if err = proto.Unmarshal(b, m.Interface()); err != nil {
return "", err
}
return prototext.MarshalOptions{Multiline: false}.Format(m.Interface()), nil
return strings.TrimSpace(prototext.MarshalOptions{Multiline: multiline}.Format(m.Interface())), nil
}
}

Expand Down
110 changes: 104 additions & 6 deletions decoder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,18 @@ import (
"testing"
"time"

"cloud.google.com/go/spanner/testdata/protos"
"github.com/apstndb/spantype/typector"
"github.com/apstndb/spanvalue/gcvctor"
"github.com/google/go-cmp/cmp"
"github.com/samber/lo"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/descriptorpb"
"google.golang.org/protobuf/types/dynamicpb"

sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
"google.golang.org/protobuf/types/known/structpb"
Expand Down Expand Up @@ -82,9 +91,12 @@ type jsonMessage struct {

func TestDecodeColumn(t *testing.T) {
tests := []struct {
desc string
value interface{}
want string
desc string
value interface{}
fds *descriptorpb.FileDescriptorSet
multiline bool
want string
wantMessage proto.Message
}{
// non-nullable
{
Expand Down Expand Up @@ -378,12 +390,98 @@ func TestDecodeColumn(t *testing.T) {

for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
got, err := DecodeColumn(createColumnValue(t, test.value))
got, err := lo.Must(formatConfigWithProto(test.fds, test.multiline)).FormatToplevelColumn(createColumnValue(t, test.value))
if err != nil {
t.Error(err)
}
if test.wantMessage != nil {
nm := dynamicpb.NewMessageType(test.wantMessage.ProtoReflect().Descriptor()).New()
err := prototext.Unmarshal([]byte(got), nm.Interface())
if err != nil {
t.Error(err)
}
if diff := cmp.Diff(nm.Interface(), test.wantMessage, protocmp.Transform()); diff != "" {
t.Errorf("formatConfigWithProto(%v) mismatch (-got +want):\n%s", test.value, diff)
}
} else {
if got != test.want {
t.Errorf("DecodeColumn(%v) = %v, want = %v", test.value, got, test.want)
}
}
})
}
}

func TestDecodeColumnRoundTripEnum(t *testing.T) {
tests := []struct {
desc string
value interface{}
fds *descriptorpb.FileDescriptorSet
want interface {
Type() protoreflect.EnumType
Number() protoreflect.EnumNumber
}
}{
{
desc: "proto with FileDescriptorProto",
value: gcvctor.EnumValue("examples.spanner.music.Genre", int64(protos.Genre_POP)),
fds: &descriptorpb.FileDescriptorSet{File: []*descriptorpb.FileDescriptorProto{protodesc.ToFileDescriptorProto(protos.File_singer_proto)}},
want: protos.Genre_POP.Enum(),
},
}

for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
got, err := lo.Must(formatConfigWithProto(test.fds, false)).FormatToplevelColumn(createColumnValue(t, test.value))
if err != nil {
t.Error(err)
}
if got != test.want {
t.Errorf("DecodeColumn(%v) = %v, want = %v", test.value, got, test.want)

gotEnumValue := test.want.Type().Descriptor().Values().ByName(protoreflect.Name(got))
gotNumber := gotEnumValue.Number()
if gotNumber != test.want.Number() {
t.Errorf("formatConfigWithProto(%v): %v(%v), want: %v(%v)", test.value, gotEnumValue.Name(), gotNumber, test.want, test.want.Number())
}
})
}
}

func TestDecodeColumnRoundTripProto(t *testing.T) {
tests := []struct {
desc string
value interface{}
fds *descriptorpb.FileDescriptorSet
multiline bool
want proto.Message
}{
{
desc: "proto with FileDescriptorProto",
value: gcvctor.ProtoValue("examples.spanner.music.SingerInfo", lo.Must(proto.Marshal(&protos.SingerInfo{SingerId: proto.Int64(1), Genre: protos.Genre_POP.Enum()}))),
fds: &descriptorpb.FileDescriptorSet{File: []*descriptorpb.FileDescriptorProto{protodesc.ToFileDescriptorProto(protos.File_singer_proto)}},
want: &protos.SingerInfo{SingerId: proto.Int64(1), Genre: protos.Genre_POP.Enum()},
},
{
desc: "proto with FileDescriptorProto",
value: gcvctor.ProtoValue("examples.spanner.music.SingerInfo", lo.Must(proto.Marshal(&protos.SingerInfo{SingerId: proto.Int64(1), Genre: protos.Genre_POP.Enum()}))),
fds: &descriptorpb.FileDescriptorSet{File: []*descriptorpb.FileDescriptorProto{protodesc.ToFileDescriptorProto(protos.File_singer_proto)}},
multiline: true,
want: &protos.SingerInfo{SingerId: proto.Int64(1), Genre: protos.Genre_POP.Enum()},
},
}

for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
got, err := lo.Must(formatConfigWithProto(test.fds, test.multiline)).FormatToplevelColumn(createColumnValue(t, test.value))
if err != nil {
t.Error(err)
}

nm := dynamicpb.NewMessageType(test.want.ProtoReflect().Descriptor()).New()
if err := prototext.Unmarshal([]byte(got), nm.Interface()); err != nil {
t.Error(err)
}
if diff := cmp.Diff(nm.Interface(), test.want, protocmp.Transform()); diff != "" {
t.Errorf("formatConfigWithProto(%v) mismatch (-got +want):\n%s", test.value, diff)
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions execute_sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
)

func executeSQL(ctx context.Context, session *Session, sql string) (*Result, error) {
fc, err := formatConfigWithProto(session.systemVariables.ProtoDescriptor)
fc, err := formatConfigWithProto(session.systemVariables.ProtoDescriptor, session.systemVariables.MultilineProtoText)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -378,7 +378,7 @@ func executeInformationSchemaBasedStatement(ctx context.Context, session *Sessio
return nil, fmt.Errorf(`%q can not be used in a read-write transaction`, stmtName)
}

fc, err := formatConfigWithProto(session.systemVariables.ProtoDescriptor)
fc, err := formatConfigWithProto(session.systemVariables.ProtoDescriptor, session.systemVariables.MultilineProtoText)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion session.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ func (s *Session) runQueryWithOptions(ctx context.Context, stmt spanner.Statemen
// RunUpdate executes a DML statement on the running read-write transaction.
// It returns error if there is no running read-write transaction.
func (s *Session) RunUpdate(ctx context.Context, stmt spanner.Statement) ([]Row, []string, int64, *sppb.ResultSetMetadata, error) {
fc, err := formatConfigWithProto(s.systemVariables.ProtoDescriptor)
fc, err := formatConfigWithProto(s.systemVariables.ProtoDescriptor, s.systemVariables.MultilineProtoText)
if err != nil {
return nil, nil, 0, nil, err
}
Expand Down
4 changes: 4 additions & 0 deletions system_variables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type systemVariables struct {
EnableProgressBar bool
ImpersonateServiceAccount string
EnableADCPlus bool
MultilineProtoText bool
}

var errIgnored = errors.New("ignored")
Expand Down Expand Up @@ -506,6 +507,9 @@ var accessorMap = map[string]accessor{
"CLI_ENABLE_HIGHLIGHT": boolAccessor(func(variables *systemVariables) *bool {
return &variables.EnableHighlight
}),
"CLI_PROTOTEXT_MULTILINE": boolAccessor(func(variables *systemVariables) *bool {
return &variables.MultilineProtoText
}),
"CLI_QUERY_MODE": {
Getter: func(this *systemVariables, name string) (map[string]string, error) {
if this.QueryMode == nil {
Expand Down

0 comments on commit 6f2b02b

Please sign in to comment.