Added block comments tokens to tokenizer (#479)

Author: AndreasArvidsson

src/core/tokenizer.ts

@@ -34,6 +34,10 @@ const FIXED_TOKENS = [
   "\\r",
   "\\n",
   "\\t",
+  "/*",
+  "*/",
+  "<!--",
+  "-->",
 ];
 const REPEATABLE_SYMBOLS_REGEX = REPEATABLE_SYMBOLS.map(escapeRegExp)
   .map((s) => `${s}+`)

src/test/suite/tokenizer.test.ts

@@ -48,6 +48,8 @@ const tests: TestCase[] = [
   ["``", ["`", "`"]],
   // Comments
   ["// Hello world", ["//", "Hello", "world"]],
+  ["/* Hello world */", ["/*", "Hello", "world", "*/"]],
+  ["<!-- Hello world -->", ["<!--", "Hello", "world", "-->"]],
   // Hex colors
   ["#aaaaaa", ["#", "aaaaaa"]],
   ["#11aaaa", ["#", "11aaaa"]],