Add support for index expressions

Author: ajroetker

lex/dialect_sql.go

@@ -159,7 +159,7 @@ var (
 		{Token: TokenChange, Lexer: LexDdlAlterColumn},
 		{Token: TokenWith, Lexer: LexJsonOrKeyValue, Optional: true},
 	}
-	// SqlCreate CREATE {SCHEMA | DATABASE | SOURCE | TABLE | VIEW | CONTINUOUSVIEW}
+	// SqlCreate CREATE {SCHEMA | INDEX | DATABASE | SOURCE | TABLE | VIEW | CONTINUOUSVIEW}
 	SqlCreate = []*Clause{
 		{Token: TokenCreate, Lexer: LexCreate},
 		{Token: TokenEngine, Lexer: LexDdlTableStorage, Optional: true},
@@ -397,6 +397,7 @@ func LexCreate(l *Lexer) StateFn {
 	/*
 		CREATE TABLE [IF NOT EXISTS] <identity> [WITH]
 		CREATE SOURCE [IF NOT EXISTS] <identity> [WITH]
+		CREATE INDEX [IF NOT EXISTS] <identity> [WITH]
 		CREATE [OR REPLACE] VIEW <identity> AS <select_statement> [WITH]
 	*/
 
@@ -413,6 +414,11 @@ func LexCreate(l *Lexer) StateFn {
 		l.Emit(TokenTable)
 		l.Push("LexDdlTable", LexDdlTable)
 		return nil
+	case "index":
+		l.ConsumeWord(keyWord)
+		l.Emit(TokenIndex)
+		l.Push("LexDdlIndex", LexDdlIndex)
+		return nil
 	case "source":
 		l.ConsumeWord(keyWord)
 		l.Emit(TokenSource)
@@ -550,6 +556,72 @@ func LexDrop(l *Lexer) StateFn {
 	return lexNotExists
 }
 
+// LexDdlIndex data definition language index
+func LexDdlIndex(l *Lexer) StateFn {
+
+	/*
+		CREATE  INDEX [IF NOT EXISTS] index_name (column_name,...)
+	*/
+	l.SkipWhiteSpaces()
+	r := l.Next()
+
+	// Cover the logic and grouping
+	switch r {
+	case '(':
+		// Start of columns
+		l.Emit(TokenLeftParenthesis)
+		l.Push("LexDdlTableColumn", LexDdlIndex)
+		return LexColumnNames
+	case ')':
+		// end of columns
+		l.Emit(TokenRightParenthesis)
+		return LexDdlTableStorage
+	case '-', '/': // comment?
+		p := l.Peek()
+		if p == '-' {
+			l.backup()
+			l.Push("LexDdlIndex", LexDdlIndex)
+			return LexInlineComment
+		}
+		u.Warnf("unhandled comment non inline ")
+	case ';':
+		l.backup()
+		return nil
+	}
+
+	l.backup()
+	word := strings.ToLower(l.PeekWord())
+	switch word {
+	case "if":
+		l.Push("LexDdlIndex", LexDdlIndex)
+		return lexNotExists
+	case "with":
+		l.ConsumeWord("with")
+		l.Emit(TokenWith)
+		return LexJsonOrKeyValue
+	case "on":
+		l.ConsumeWord("on")
+		l.Emit(TokenOn)
+		return LexDdlIndex
+	}
+	r = l.Peek()
+	if r == ',' {
+		l.Emit(TokenComma)
+		l.Push("LexDdlIndex", l.clauseState())
+		return LexExpressionOrIdentity
+	}
+	if l.isNextKeyword(word) {
+		return nil
+	}
+	// ensure we don't get into a recursive death spiral here?
+	if len(l.stack) < 10 {
+		l.Push("LexDdlIndex", LexDdlIndex)
+	} else {
+		u.Errorf("Gracefully refusing to add more LexDdlIndex: ")
+	}
+	return LexExpressionOrIdentity
+}
+
 // LexDdlTable data definition language table
 func LexDdlTable(l *Lexer) StateFn {
 

lex/dialect_sql_parse_test.go

@@ -3,14 +3,12 @@ package lex_test
 import (
 	"testing"
 
-	u "github.com/araddon/gou"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/lytics/qlbridge/rel"
 )
 
 func parseSqlTest(t *testing.T, sql string) {
-	u.Debugf("parsing sql: %s", sql)
 	sqlRequest, err := rel.ParseSql(sql)
 	assert.Equal(t, nil, err)
 	assert.NotEqual(t, nil, sqlRequest, "Must parse: %s  \n\t%v", sql, err)
@@ -20,7 +18,6 @@ func parseSqlTest(t *testing.T, sql string) {
 	}
 }
 func parseSqlError(t *testing.T, sql string) {
-	u.Debugf("parse looking for error sql: %s", sql)
 	_, err := rel.ParseSql(sql)
 	assert.NotEqual(t, nil, err, "Must error on parse: %s", sql)
 }
@@ -35,7 +32,7 @@ func TestSqlParser(t *testing.T) {
 	parseSqlError(t, `SELECT a, tolower(b) AS b INTO newtable FROM FROM WHERE a != "hello";`)
 	parseSqlTest(t, `
 		SELECT a.language, a.template, Count(*) AS count
-		FROM 
+		FROM
 			(Select Distinct language, template FROM content WHERE language != "en" OFFSET 1) AS a
 			Left Join users AS b
 				On b.language = a.language AND b.template = b.template
@@ -47,10 +44,10 @@ func TestSqlParser(t *testing.T) {
 	// CREATE
 	parseSqlTest(t, `CREATE CONTINUOUSVIEW viewx AS SELECT a FROM tbl;`)
 	parseSqlError(t, `CREATE FAKEITEM viewx;`)
-	parseSqlTest(t, `CREATE OR REPLACE VIEW viewx 
-		AS SELECT a, b FROM mydb.tbl 
+	parseSqlTest(t, `CREATE OR REPLACE VIEW viewx
+		AS SELECT a, b FROM mydb.tbl
 		WITH stuff = "hello";`)
-	parseSqlTest(t, `CREATE TABLE articles 
+	parseSqlTest(t, `CREATE TABLE articles
 			--comment-here
 			(
 			 ID int(11) NOT NULL AUTO_INCREMENT,

lex/dialect_sql_test.go

@@ -96,7 +96,7 @@ func TestLexSqlCreate(t *testing.T) {
 
 	// CREATE {DATABASE | SCHEMA | SOURCE | VIEW | CONTINUOUSVIEW} [IF NOT EXISTS] db_name
 	// [create_specification] ...
-	verifyTokens(t, `CREATE SCHEMA IF NOT EXISTS mysource 
+	verifyTokens(t, `CREATE SCHEMA IF NOT EXISTS mysource
 		WITH stuff = "hello";
 		`,
 		[]Token{
@@ -145,8 +145,8 @@ func TestLexSqlCreate(t *testing.T) {
 			tv(TokenValue, "hello"),
 		})
 
-	verifyTokens(t, `CREATE OR REPLACE VIEW viewx 
-			AS SELECT a, b FROM mydb.tbl 
+	verifyTokens(t, `CREATE OR REPLACE VIEW viewx
+			AS SELECT a, b FROM mydb.tbl
 			WITH stuff = "hello";`,
 		[]Token{
 			tv(TokenCreate, "CREATE"),
@@ -167,7 +167,7 @@ func TestLexSqlCreate(t *testing.T) {
 			tv(TokenValue, "hello"),
 		})
 
-	verifyTokens(t, `CREATE TABLE articles 
+	verifyTokens(t, `CREATE TABLE articles
 		 (
 		  ID int(11) NOT NULL AUTO_INCREMENT,
 		  Email char(150) NOT NULL DEFAULT '',
@@ -240,7 +240,7 @@ func TestLexSqlCreate(t *testing.T) {
 	verifyTokens(t, `CREATE INDEX IF NOT EXISTS my_index ON my_table (col1, col2) WITH { "key": "value" };`,
 		[]Token{
 			tv(TokenCreate, "CREATE"),
-			tv(TokenIdentity, "INDEX"), // Assuming INDEX is treated as an Identity for now
+			tv(TokenIndex, "INDEX"), // Assuming INDEX is treated as an Identity for now
 			tv(TokenIf, "IF"),
 			tv(TokenNegate, "NOT"),
 			tv(TokenExists, "EXISTS"),
@@ -253,11 +253,11 @@ func TestLexSqlCreate(t *testing.T) {
 			tv(TokenIdentity, "col2"),
 			tv(TokenRightParenthesis, ")"),
 			tv(TokenWith, "WITH"),
-			tv(TokenLeftBracket, "{"),
-			tv(TokenValue, "key"),
+			tv(TokenLeftBrace, "{"),
+			tv(TokenIdentity, "key"),
 			tv(TokenColon, ":"),
 			tv(TokenValue, "value"),
-			tv(TokenRightBracket, "}"),
+			tv(TokenRightBrace, "}"),
 		})
 }
 func TestLexSqlDrop(t *testing.T) {

lex/lexer_test.go

@@ -276,8 +276,8 @@ func verifyTokens(t *testing.T, sql string, tokens []Token) {
 	for _, goodToken := range tokens {
 		tok := l.NextToken()
 		//u.Debugf("got:%v  want:%v", tok, goodToken)
-		assert.Equal(t, tok.T, goodToken.T, "want='%v' has %v ", goodToken.T, tok)
-		assert.Equal(t, tok.V, goodToken.V, "want='%v' has %v ", goodToken.V, tok)
+		assert.Equal(t, tok.T, goodToken.T, "want='%q' has %q ", goodToken.T, tok)
+		assert.Equal(t, tok.V, goodToken.V, "want='%q' has %q ", goodToken.V, tok)
 	}
 }
 

lex/token.go

@@ -178,6 +178,7 @@ const (
 	TokenView           TokenType = 404 // VIEW
 	TokenContinuousView TokenType = 405 // CONTINUOUSVIEW
 	TokenTemp           TokenType = 406 // TEMP or TEMPORARY
+	TokenIndex          TokenType = 407 // INDEX
 
 	// ddl other
 	TokenChange       TokenType = 410 // change

rel/parse_sql.go

@@ -716,9 +716,9 @@ func (m *Sqlbridge) parseCreate() (*SqlCreate, error) {
 		}
 		req.OrReplace = true
 	}
-	// CREATE {DATABASE|SCHEMA|TABLE|VIEW|SOURCE|CONTINUOUSVIEW} <identity>
+	// CREATE {INDEX|DATABASE|SCHEMA|TABLE|VIEW|SOURCE|CONTINUOUSVIEW} <identity>
 	switch m.Cur().T {
-	case lex.TokenTable, lex.TokenSource, lex.TokenDatabase, lex.TokenSchema:
+	case lex.TokenTable, lex.TokenSource, lex.TokenIndex, lex.TokenDatabase, lex.TokenSchema:
 		req.Tok = m.Next()
 	case lex.TokenView, lex.TokenContinuousView:
 		req.Tok = m.Next()
@@ -744,7 +744,7 @@ func (m *Sqlbridge) parseCreate() (*SqlCreate, error) {
 		req.Select = sel
 		return req, nil
 	default:
-		return nil, m.ErrMsg("Expected view, table, source, schema, database, continuousview for CREATE got")
+		return nil, m.ErrMsg("Expected index, view, table, source, schema, database, continuousview for CREATE got")
 	}
 
 	// [IF NOT EXISTS]
@@ -803,6 +803,7 @@ func (m *Sqlbridge) parseCreate() (*SqlCreate, error) {
 		// just with
 	case lex.TokenSchema:
 		// just with for now
+	case lex.TokenIndex:
 	default:
 		return nil, fmt.Errorf("not implemented %v", req.Tok.V)
 	}

rel/sql.go

@@ -205,7 +205,7 @@ type (
 	SqlCreate struct {
 		Raw         string       // full original raw statement
 		Identity    string       // identity of table, view, etc
-		Tok         lex.Token    // CREATE [TABLE,VIEW,CONTINUOUSVIEW,TRIGGER] etc
+		Tok         lex.Token    // CREATE [INDEX|TABLE,VIEW,CONTINUOUSVIEW,TRIGGER] etc
 		OrReplace   bool         // OR REPLACE
 		IfNotExists bool         // IF NOT EXISTS
 		Cols        []*DdlColumn // columns