From acf0c0375236982b95c15722ba932c2cd6e58855 Mon Sep 17 00:00:00 2001
From: Jelle De Loecker <jelle@elevenways.be>
Date: Fri, 16 Feb 2024 17:38:46 +0100
Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fix=20some=20number=20parsing=20?=
 =?UTF-8?q?issues=20in=20the=20tokenizer?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/function.js  |  17 +++++++-
 test/function.js | 110 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/lib/function.js b/lib/function.js
index 22b94ab..3b54837 100644
--- a/lib/function.js
+++ b/lib/function.js
@@ -237,6 +237,7 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 	    prev_token,
 	    is_digit = false,
 	    end_char,
+		has_dot = false,
 	    escaped = false,
 	    result = [],
 	    length = source_code.length,
@@ -271,6 +272,7 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 		string_state = false;
 		current_state = null;
 		end_char = null;
+		has_dot = false;
 
 		if (buffer) {
 			pushChar(last_char);
@@ -485,7 +487,14 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 				pushChar(char);
 			} else if (current_state == TOKEN_NUMBER) {
 
-				if (is_digit || char == '.' || char == 'e' || char == 'E') {
+				if (char == '.') {
+					if (has_dot) {
+						check_next_state = true;
+					} else {
+						pushChar(char);
+						has_dot = true;
+					}
+				} else if (is_digit || char == 'e' || char == 'E') {
 					pushChar(char);
 				} else if (char == '_' && prev != '_') {
 					// Ignore
@@ -533,6 +542,12 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 			createBuffer(TOKEN_SQUARE, char);
 		} else if (is_digit) {
 			createBuffer(TOKEN_NUMBER, char);
+
+			// It technically isn't valid JavaScript either,
+			// But it is even more wrong when 'a.0.1.b' is considered a number
+			if (prev_usable_token?.value === '.' || prev_usable_token?.value === '?.') {
+				endState();
+			}
 		} else {
 
 			if (char == '/') {
diff --git a/test/function.js b/test/function.js
index 91c78cc..db80c1f 100644
--- a/test/function.js
+++ b/test/function.js
@@ -719,6 +719,116 @@ string` + `another
 				{ type: 'number', value: '1000000.01', line_start: 0, line_end: 0 }
 			]);
 		});
+
+		it('should not detect numbers in property chains', () => {
+
+			let source = `this.0.test`;
+			let tokens = Function.tokenize(source, true);
+
+			deepAlike(tokens, [
+				{
+					type: 'keyword',
+					value: 'this',
+					line_start: 0,
+					line_end: 0,
+					name: 'this'
+				},
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '0', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+
+			tokens = Function.tokenize(`b = this.0.1.test`, true);
+
+			deepAlike(tokens, [
+				{ type: 'name', value: 'b', line_start: 0, line_end: 0 },
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '=',
+					line_start: 0,
+					line_end: 0,
+					name: 'assign'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{
+					type: 'keyword',
+					value: 'this',
+					line_start: 0,
+					line_end: 0,
+					name: 'this'
+				},
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '0', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '1', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+
+			tokens = Function.tokenize(`let a  = 0.1.test`, true);
+
+			deepAlike(tokens, [
+				{
+					type: 'keyword',
+					value: 'let',
+					line_start: 0,
+					line_end: 0,
+					name: 'let'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{ type: 'name', value: 'a', line_start: 0, line_end: 0 },
+				{ type: 'whitespace', value: '  ', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '=',
+					line_start: 0,
+					line_end: 0,
+					name: 'assign'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{ type: 'number', value: '0.1', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+		});
 	});
 
 	describe('.getArgumentNames(fnc)', function() {