🐛 Fix some number parsing issues in the tokenizer

lib/function.js

@@ -237,6 +237,7 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 	    prev_token,
 	    is_digit = false,
 	    end_char,
+		has_dot = false,
 	    escaped = false,
 	    result = [],
 	    length = source_code.length,
@@ -271,6 +272,7 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 		string_state = false;
 		current_state = null;
 		end_char = null;
+		has_dot = false;
 
 		if (buffer) {
 			pushChar(last_char);
@@ -485,7 +487,14 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 				pushChar(char);
 			} else if (current_state == TOKEN_NUMBER) {
 
-				if (is_digit || char == '.' || char == 'e' || char == 'E') {
+				if (char == '.') {
+					if (has_dot) {
+						check_next_state = true;
+					} else {
+						pushChar(char);
+						has_dot = true;
+					}
+				} else if (is_digit || char == 'e' || char == 'E') {
 					pushChar(char);
 				} else if (char == '_' && prev != '_') {
 					// Ignore
@@ -533,6 +542,12 @@ defStat(function tokenize(source_code, add_type, throw_errors) {
 			createBuffer(TOKEN_SQUARE, char);
 		} else if (is_digit) {
 			createBuffer(TOKEN_NUMBER, char);
+
+			// It technically isn't valid JavaScript either,
+			// But it is even more wrong when 'a.0.1.b' is considered a number
+			if (prev_usable_token?.value === '.' || prev_usable_token?.value === '?.') {
+				endState();
+			}
 		} else {
 
 			if (char == '/') {

test/function.js

@@ -719,6 +719,116 @@ string` + `another
 				{ type: 'number', value: '1000000.01', line_start: 0, line_end: 0 }
 			]);
 		});
+
+		it('should not detect numbers in property chains', () => {
+
+			let source = `this.0.test`;
+			let tokens = Function.tokenize(source, true);
+
+			deepAlike(tokens, [
+				{
+					type: 'keyword',
+					value: 'this',
+					line_start: 0,
+					line_end: 0,
+					name: 'this'
+				},
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '0', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+
+			tokens = Function.tokenize(`b = this.0.1.test`, true);
+
+			deepAlike(tokens, [
+				{ type: 'name', value: 'b', line_start: 0, line_end: 0 },
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '=',
+					line_start: 0,
+					line_end: 0,
+					name: 'assign'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{
+					type: 'keyword',
+					value: 'this',
+					line_start: 0,
+					line_end: 0,
+					name: 'this'
+				},
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '0', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'number', value: '1', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+
+			tokens = Function.tokenize(`let a  = 0.1.test`, true);
+
+			deepAlike(tokens, [
+				{
+					type: 'keyword',
+					value: 'let',
+					line_start: 0,
+					line_end: 0,
+					name: 'let'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{ type: 'name', value: 'a', line_start: 0, line_end: 0 },
+				{ type: 'whitespace', value: '  ', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '=',
+					line_start: 0,
+					line_end: 0,
+					name: 'assign'
+				},
+				{ type: 'whitespace', value: ' ', line_start: 0, line_end: 0 },
+				{ type: 'number', value: '0.1', line_start: 0, line_end: 0 },
+				{
+					type: 'punct',
+					value: '.',
+					line_start: 0,
+					line_end: 0,
+					name: 'dot'
+				},
+				{ type: 'name', value: 'test', line_start: 0, line_end: 0 }
+			]);
+		});
 	});
 
 	describe('.getArgumentNames(fnc)', function() {