Merge pull request #121 from skx/120-subtraction

Handle subtraction vs. negative numbers correctly.
skx · Nov 30, 2022 · 99d8a64 · 99d8a64
2 parents 3966855 + 7367461
commit 99d8a64
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 2 deletions.
diff --git a/tokenizer/tokenizer.go b/tokenizer/tokenizer.go
@@ -5,7 +5,6 @@
 //
 // Our interpeter is intentionally naive, and executes tokens directly, without
 // any intermediary representation.
-//
 package tokenizer
 
 import (
@@ -79,7 +78,17 @@ func (l *Tokenizer) NextToken() token.Token {
 		tok = newToken(token.PLUS, l.ch)
 	case rune('-'):
 		// -3 is "-3".  "3 - 4" is -1.
-		if isDigit(l.peekChar()) {
+		//
+		// However we have to add a couple of special cases such that:
+		//
+		//     "... X-3" is "X - 3"
+		// and "3-3" is "3 - 3" not "3 -3"
+		//
+		// We do that by ensuring we look at the previous token and force
+		// a "minus" rather than a negative number if the prev. token was
+		// an identifier, or a number.
+		//
+		if isDigit(l.peekChar()) && l.prevToken.Type != token.IDENT && l.prevToken.Type != token.INT {
 			// swallow the -
 			l.readChar()
 

diff --git a/tokenizer/tokenizer_test.go b/tokenizer/tokenizer_test.go
@@ -305,3 +305,71 @@ func TestNullString(t *testing.T) {
 		}
 	}
 }
+
+// TestIssue120 tests that we parse subtraction vs. negative numbers
+// as expected.
+func TestIssue120(t *testing.T) {
+	input := `10 LET A=3
+20 A=A-3
+30 LET B=20-3
+40 LET C=-3
+50 LET D= 3 - -3
+`
+
+	tests := []struct {
+		expectedType    token.Type
+		expectedLiteral string
+	}{
+		{token.LINENO, "10"},
+		{token.LET, "LET"},
+		{token.IDENT, "A"},
+		{token.ASSIGN, "="},
+		{token.INT, "3"},
+		{token.NEWLINE, "\\n"},
+
+		{token.LINENO, "20"},
+		{token.IDENT, "A"},
+		{token.ASSIGN, "="},
+		{token.IDENT, "A"},
+		{token.MINUS, "-"},
+		{token.INT, "3"},
+		{token.NEWLINE, "\\n"},
+
+		{token.LINENO, "30"},
+		{token.LET, "LET"},
+		{token.IDENT, "B"},
+		{token.ASSIGN, "="},
+		{token.INT, "20"},
+		{token.MINUS, "-"},
+		{token.INT, "3"},
+		{token.NEWLINE, "\\n"},
+
+		{token.LINENO, "40"},
+		{token.LET, "LET"},
+		{token.IDENT, "C"},
+		{token.ASSIGN, "="},
+		{token.INT, "-3"},
+		{token.NEWLINE, "\\n"},
+
+		{token.LINENO, "50"},
+		{token.LET, "LET"},
+		{token.IDENT, "D"},
+		{token.ASSIGN, "="},
+		{token.INT, "3"},
+		{token.MINUS, "-"},
+		{token.INT, "-3"},
+		{token.NEWLINE, "\\n"},
+
+		{token.EOF, ""},
+	}
+	l := New(input)
+	for i, tt := range tests {
+		tok := l.NextToken()
+		if tok.Type != tt.expectedType {
+			t.Fatalf("tests[%d] - tokentype wrong, expected=%q, got=%v", i, tt.expectedType, tok)
+		}
+		if tok.Literal != tt.expectedLiteral {
+			t.Fatalf("tests[%d] - Literal wrong, expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal)
+		}
+	}
+}