System variables phase 2 (#17)

* Transform most of spanner-cli flags to system variables * CLI_PROJECT * CLI_INSTANCE * CLI_DATABASE * CLI_PROMPT * CLI_HISTORY_FILE * CLI_ROLE * CLI_ENDPOINT * CLI_DIRECTED_READ * Implement prompt system variable expansions * Bump go-flags to latest, and utilize it. * Migrate integration test to use testcontainers/testcontainers-go/modules/gcloud
apstndb · Oct 31, 2024 · b71c7fe · b71c7fe
1 parent c11998b
commit b71c7fe
Show file tree

Hide file tree

Showing 16 changed files with 708 additions and 241 deletions.
diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml
@@ -23,34 +23,11 @@ name: CI
 jobs:
   test:
     runs-on: ubuntu-latest
-    services:
-      emulator:
-        image: gcr.io/cloud-spanner-emulator/emulator:1.5.24
-        ports:
-          - 9010:9010
-          - 9020:9020
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.23'
       - run: go version
-      - name: Wait on
-        uses: iFaxity/[email protected]
-        with:
-          resource: http-get://localhost:9020/v1/projects/fake-project/instances
-      - run: make setup-emulator
-        env:
-          SPANNER_EMULATOR_HOST: localhost:9010
-          SPANNER_EMULATOR_HOST_REST: localhost:9020
-          PROJECT: fake-project
-          INSTANCE: fake-instance
-          DATABASE: fake-database
-      - name: make test with Cloud Spanner Emulator
-        run: make test
-        env:
-          SPANNER_EMULATOR_HOST: localhost:9010
-          SPANNER_EMULATOR_HOST_REST: localhost:9020
-          PROJECT: fake-project
-          INSTANCE: fake-instance
-          DATABASE: fake-database
+      - name: make test
+        run: make test
diff --git a/Makefile b/Makefile
@@ -14,16 +14,15 @@ release: clean cross-compile package
 	@echo "released as draft"
 
 clean:
-	rm -f spanner-cli
+	rm -f spanner-mycli
 	rm -rf dist/
 	go clean -testcache
 
 run:
-	./spanner-cli -p ${PROJECT} -i ${INSTANCE} -d ${DATABASE}
+	./spanner-mycli -p ${PROJECT} -i ${INSTANCE} -d ${DATABASE}
 
 test:
-	@SPANNER_CLI_INTEGRATION_TEST_PROJECT_ID=${PROJECT} SPANNER_CLI_INTEGRATION_TEST_INSTANCE_ID=${INSTANCE} SPANNER_CLI_INTEGRATION_TEST_DATABASE_ID=${DATABASE} SPANNER_CLI_INTEGRATION_TEST_CREDENTIAL=${CREDENTIAL} go test -v ./...
+	go test -v ./...
 
-setup-emulator:
-	curl -s "${SPANNER_EMULATOR_HOST_REST}/v1/projects/${PROJECT}/instances" --data '{"instanceId": "'${INSTANCE}'"}'
-	curl -s "${SPANNER_EMULATOR_HOST_REST}/v1/projects/${PROJECT}/instances/${INSTANCE}/databases" --data '{"createStatement": "CREATE DATABASE `'${DATABASE}'`"}'
+fasttest:
+	go test --tags skip_slow_test -v ./...
diff --git a/README.md b/README.md
@@ -13,19 +13,20 @@ You can control your Spanner databases with idiomatic SQL commands.
 * Respects my minor use cases
 * Respects batch use cases as well as interactive use cases
 * More `gcloud spanner databases execute-sql` compatibilities
-  * Support `--sql` flag
-* Minimize to use own syntax
-  * Generalized system variables concept inspired by Spanner JDBC
+  * Support compatible flags (`--sql`)
+* Generalized concepts to extend without a lot of original syntax
+  * Generalized system variables concept inspired by Spanner JDBC properties
     * `SET <name> = <value>` statement
     * `SHOW VARIABLES` statement
-    * `SHOW VARIABLE <name> statement`
+    * `SHOW VARIABLE <name>` statement
     * `--set <name>=<value>` flag
 * Improved interactive experience
   * Use [`reeflective/readline`](https://github.com/reeflective/readline) instead of [`chzyer/readline"`](https://github.com/chzyer/readline)
     * Native multi-line editing and histories
   * Improved prompt
     * Use `%` for prompt expansion, instead of `\` to avoid escaping
     * Allow newlines in prompt using `%n`
+    * System variables expansion
 * Utilize other libraries
   * Dogfooding [`cloudspannerecosystem/memefish`](https://github.com/cloudspannerecosystem/memefish)
 
@@ -260,19 +261,49 @@ and `{}` for a mutually exclusive keyword.
 | Set variable | `SET <name> = <value>;` | |
 | Show variables | `SHOW VARIABLES;` | |
 
+## System Variables
+
+### Spanner JDBC inspired variables
+
+They have almost same semantics with [Spanner JDBC properties](https://cloud.google.com/spanner/docs/jdbc-session-mgmt-commands?hl=en)
+
+| Name                         | Type       | Example                              |
+|------------------------------|------------|--------------------------------------|
+| READ_ONLY_STALENESS          | READ_WRITE | `"analyze_20241017_15_59_17UTC"`     |
+| OPTIMIZER_VERSION            | READ_WRITE | `"7"`                                |
+| OPTIMIZER_STATISTICS_PACKAGE | READ_WRITE | `"7"`                                |
+| RPC_PRIORITY                 | READ_WRITE | `"MEDIUM"`                           |
+| READ_TIMESTAMP               | READ_ONLY  | `"2024-11-01T05:28:58.943332+09:00"` |
+| COMMIT_RESPONSE              | READ_ONLY  | `"2024-11-01T05:31:11.311894+09:00"` |
+
+### spanner-mycli original variables
+
+| Name             | READ/WRITE | Example                                        |
+|------------------|------------|------------------------------------------------|
+| CLI_PROJECT      | READ_ONLY  | `"myproject"`                                  |
+| CLI_INSTANCE     | READ_ONLY  | `"myinstance"`                                 |
+| CLI_DATABASE     | READ_ONLY  | `"mydb"`                                       |
+| CLI_DIRECT_READ  | READ_ONLY  | `"asia-northeast:READ_ONLY"`                   |
+| CLI_ENDPOINT     | READ_ONLY  | `"spanner.me-central2.rep.googleapis.com:443"` |
+| CLI_FORMAT       | READ_WRITE | `"TABLE"`                                      |
+| CLI_HISTORY_FILE | READ_ONLY  | `"/tmp/spanner_mycli_readline.tmp"`            |
+| CLI_PROMPT       | READ_WRITE | `"spanner%t> "`                                |
+| CLI_ROLE         | READ_ONLY  | `"spanner_info_reader"`                        |
+| CLI_VERBOSE      | READ_WRITE | `TRUE`                                         |
 ## Customize prompt
 
-You can customize the prompt by `--prompt` option.  
-There are some defined variables for being used in prompt.
+You can customize the prompt by `--prompt` option or `CLI_PROMPT` system variable.  
+There are some escape sequences for being used in prompt.
 
-Variables:
+Escape sequences:
 
 * `%p` : GCP Project ID
 * `%i` : Cloud Spanner Instance ID
 * `%d` : Cloud Spanner Database ID
-* `%t` : In transaction
+* `%t` : In transaction mode `(ro txn)` or `(rw txn)`
 * `%n` : Newline
 * `%%` : Character `%`
+* `%{VAR_NAME}` : System variable expansion
 
 Example:
 
@@ -304,7 +335,7 @@ The default prompt is `spanner%t> `.
 ## Config file
 
 This tool supports a configuration file called `spanner_mycli.cnf`, similar to `my.cnf`.  
-The config file path must be `~/.spanner_mycli.cnf`.  
+The config file path must be `~/.spanner_mycli.cnf`.
 In the config file, you can set default option values for command line options.
 
 Example:
@@ -328,7 +359,7 @@ prompt = "[%p:%i:%d]%t> "
 You can set [request priority](https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions#Priority) for command level or transaction level.
 By default `MEDIUM` priority is used for every request.
 
-To set a priority for command line level, you can use `--priority={HIGH|MEDIUM|LOW}` command line option.
+To set a priority for command line level, you can use `--priority={HIGH|MEDIUM|LOW}` command line option or `CLI_PRIORITY` system variable.
 
 To set a priority for transaction level, you can use `PRIORITY {HIGH|MEDIUM|LOW}` keyword.
 
@@ -410,10 +441,12 @@ Run unit tests.
 $ make test
 ```
 
-Run integration tests, which connects to real Cloud Spanner database.
+Note: It requires Docker because integration tests using [testcontainers](https://testcontainers.com/).
+
+Or run test except integration tests.
 
 ```
-$ PROJECT=${PROJECT_ID} INSTANCE=${INSTANCE_ID} DATABASE=${DATABASE_ID} CREDENTIAL=${CREDENTIAL} make test
+$ make fasttest
 ```
 
 ## TODO

diff --git a/cli.go b/cli.go
@@ -24,6 +24,7 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"regexp"
 	"strings"
 	"time"
 
@@ -48,56 +49,35 @@ const (
 )
 
 const (
-	defaultPrompt      = `spanner%t> `
-	defaultHistoryFile = `/tmp/spanner_mycli_readline.tmp`
-
 	exitCodeSuccess = 0
 	exitCodeError   = 1
 )
 
 type Cli struct {
 	Session         *Session
-	Prompt          string
-	HistoryFile     string
 	Credential      []byte
 	InStream        io.ReadCloser
 	OutStream       io.Writer
 	ErrStream       io.Writer
-	Verbose         bool
-	Priority        sppb.RequestOptions_Priority
-	Endpoint        string
 	SystemVariables *systemVariables
 }
 
 type command struct {
-	Stmt     Statement
-	Vertical bool
+	Stmt Statement
 }
 
-func NewCli(projectId, instanceId, databaseId, prompt, historyFile string, credential []byte, inStream io.ReadCloser, outStream, errStream io.Writer, verbose bool, role, endpoint string, directedRead *sppb.DirectedReadOptions, sysVars *systemVariables) (*Cli, error) {
-	session, err := createSession(projectId, instanceId, databaseId, credential, role, endpoint, directedRead, sysVars)
+func NewCli(credential []byte, inStream io.ReadCloser, outStream, errStream io.Writer, sysVars *systemVariables) (*Cli, error) {
+	session, err := createSession(credential, sysVars)
 	if err != nil {
 		return nil, err
 	}
 
-	if prompt == "" {
-		prompt = defaultPrompt
-	}
-
-	if historyFile == "" {
-		historyFile = defaultHistoryFile
-	}
-
 	return &Cli{
 		Session:         session,
-		Prompt:          prompt,
-		HistoryFile:     historyFile,
 		Credential:      credential,
 		InStream:        inStream,
 		OutStream:       outStream,
 		ErrStream:       errStream,
-		Verbose:         verbose,
-		Endpoint:        endpoint,
 		SystemVariables: sysVars,
 	}, nil
 }
@@ -145,7 +125,7 @@ func (c *Cli) RunInteractive() int {
 		return false
 	}
 
-	shell.History.AddFromFile("history name", c.HistoryFile)
+	shell.History.AddFromFile("history name", c.SystemVariables.HistoryFile)
 
 	exists, err := c.Session.DatabaseExists()
 	if err != nil {
@@ -154,7 +134,7 @@ func (c *Cli) RunInteractive() int {
 	if exists {
 		fmt.Fprintf(c.OutStream, "Connected.\n")
 	} else {
-		return c.ExitOnError(fmt.Errorf("unknown database %q", c.Session.databaseId))
+		return c.ExitOnError(fmt.Errorf("unknown database %q", c.SystemVariables.Database))
 	}
 
 	for {
@@ -192,7 +172,12 @@ func (c *Cli) RunInteractive() int {
 		}
 
 		if s, ok := stmt.(*UseStatement); ok {
-			newSession, err := createSession(c.Session.projectId, c.Session.instanceId, s.Database, c.Credential, s.Role, c.Endpoint, c.Session.directedRead, c.SystemVariables)
+			newSystemVariables := *c.SystemVariables
+
+			newSystemVariables.Database = s.Database
+			newSystemVariables.Role = s.Role
+
+			newSession, err := createSession(c.Credential, &newSystemVariables)
 			if err != nil {
 				c.PrintInteractiveError(err)
 				continue
@@ -212,13 +197,18 @@ func (c *Cli) RunInteractive() int {
 
 			c.Session.Close()
 			c.Session = newSession
+
+			c.SystemVariables = &newSystemVariables
+
 			fmt.Fprintf(c.OutStream, "Database changed")
 			continue
 		}
 
 		if s, ok := stmt.(*DropDatabaseStatement); ok {
-			if c.Session.databaseId == s.DatabaseId {
-				c.PrintInteractiveError(fmt.Errorf("database %q is currently used, it can not be dropped", s.DatabaseId))
+			if c.SystemVariables.Database == s.DatabaseId {
+				c.PrintInteractiveError(
+					fmt.Errorf("database %q is currently used, it can not be dropped", s.DatabaseId),
+				)
 				continue
 			}
 
@@ -331,7 +321,7 @@ func (c *Cli) PrintBatchError(err error) {
 }
 
 func (c *Cli) PrintResult(result *Result, mode DisplayMode, interactive bool) {
-	printResult(c.OutStream, result, mode, interactive, c.Verbose)
+	printResult(c.OutStream, result, mode, interactive, c.SystemVariables.Verbose)
 }
 
 func (c *Cli) PrintProgressingMark() func() {
@@ -357,29 +347,43 @@ func (c *Cli) PrintProgressingMark() func() {
 	return stop
 }
 
+var promptRe = regexp.MustCompile(`(%[^{])|%\{[^}]+}`)
+var promptSystemVariableRe = regexp.MustCompile(`%\{([^}]+)}`)
+
 func (c *Cli) getInterpolatedPrompt() string {
-	return strings.NewReplacer(
-		"%%", "%",
-		"%n", "\n",
-		"%p", c.Session.projectId,
-		"%i", c.Session.projectId,
-		"%d", c.Session.databaseId,
-		"%t", lo.
-			If(c.Session.InReadWriteTransaction(), "(rw txn)").
-			ElseIf(c.Session.InReadOnlyTransaction(), "(ro txn)").
-			Else(""),
-	).Replace(c.Prompt)
+	return promptRe.ReplaceAllStringFunc(c.SystemVariables.Prompt, func(s string) string {
+		return lo.Switch[string, string](s).
+			Case("%%", "%").
+			Case("%n", "\n").
+			Case("%p", c.SystemVariables.Project).
+			Case("%i", c.SystemVariables.Instance).
+			Case("%d", c.SystemVariables.Database).
+			Case("%t", lo.
+				If(c.Session.InReadWriteTransaction(), "(rw txn)").
+				ElseIf(c.Session.InReadOnlyTransaction(), "(ro txn)").
+				Else("")).
+			DefaultF(
+				func() string {
+					varName := promptSystemVariableRe.FindStringSubmatch(s)[1]
+					value, err := c.SystemVariables.Get(varName)
+					if err != nil {
+						return fmt.Sprintf("INVALID_VAR{%v}", varName)
+					}
+					return value[varName]
+				},
+			)
+	})
 }
 
-func createSession(projectId string, instanceId string, databaseId string, credential []byte, role string, endpoint string, directedRead *sppb.DirectedReadOptions, sysVars *systemVariables) (*Session, error) {
+func createSession(credential []byte, sysVars *systemVariables) (*Session, error) {
 	var opts []option.ClientOption
 	if credential != nil {
 		opts = append(opts, option.WithCredentialsJSON(credential))
 	}
-	if endpoint != "" {
-		opts = append(opts, option.WithEndpoint(endpoint))
+	if sysVars.Endpoint != "" {
+		opts = append(opts, option.WithEndpoint(sysVars.Endpoint))
 	}
-	return NewSession(projectId, instanceId, databaseId, role, directedRead, sysVars, opts...)
+	return NewSession(sysVars, opts...)
 }
 
 func readInteractiveInput(rl *readline.Shell, prompt string) (*inputStatement, error) {
@@ -567,16 +571,16 @@ func buildCommands(input string) ([]*command, error) {
 
 		// Flush pending DDLs
 		if len(pendingDdls) > 0 {
-			cmds = append(cmds, &command{&BulkDdlStatement{pendingDdls}, false})
+			cmds = append(cmds, &command{&BulkDdlStatement{pendingDdls}})
 			pendingDdls = nil
 		}
 
-		cmds = append(cmds, &command{stmt, separated.delim == delimiterVertical})
+		cmds = append(cmds, &command{stmt})
 	}
 
 	// Flush pending DDLs
 	if len(pendingDdls) > 0 {
-		cmds = append(cmds, &command{&BulkDdlStatement{pendingDdls}, false})
+		cmds = append(cmds, &command{&BulkDdlStatement{pendingDdls}})
 	}
 
 	return cmds, nil