make sure reserved top level/new line keywords are always reserved

Author: mvorisek

src/Tokenizer.php

@@ -29,6 +29,7 @@ final class Tokenizer
     private array $reserved = [
         'ACCESSIBLE',
         'ACTION',
+        'ADD',
         'AFTER',
         'AGAINST',
         'AGGREGATE',
@@ -37,6 +38,7 @@ final class Tokenizer
         'ALTER',
         'ANALYSE',
         'ANALYZE',
+        'AND',
         'AS',
         'ASC',
         'AUTO_INCREMENT',
@@ -46,6 +48,7 @@ final class Tokenizer
         'BETWEEN',
         'BINLOG',
         'BOTH',
+        'BY',
         'CASCADE',
         'CASE',
         'CHANGE',
@@ -87,6 +90,7 @@ final class Tokenizer
         'DISTINCTROW',
         'DIV',
         'DO',
+        'DROP',
         'DUMPFILE',
         'DUPLICATE',
         'DYNAMIC',
@@ -99,6 +103,8 @@ final class Tokenizer
         'ESCAPE',
         'ESCAPED',
         'EVENTS',
+        'EXCEPT',
+        'EXCLUDE',
         'EXEC',
         'EXECUTE',
         'EXISTS',
@@ -115,6 +121,7 @@ final class Tokenizer
         'FOR',
         'FORCE',
         'FOREIGN',
+        'FROM',
         'FULL',
         'FULLTEXT',
         'FUNCTION',
@@ -123,6 +130,7 @@ final class Tokenizer
         'GRANTS',
         'GROUP',
         'GROUPS',
+        'HAVING',
         'HEAP',
         'HIGH_PRIORITY',
         'HOSTS',
@@ -137,21 +145,26 @@ final class Tokenizer
         'INDEX',
         'INDEXES',
         'INFILE',
+        'INNER',
         'INSERT',
         'INSERT_ID',
         'INSERT_METHOD',
+        'INTERSECT',
         'INTERVAL',
         'INTO',
         'INVOKER',
         'IS',
         'ISOLATION',
+        'JOIN',
         'KEY',
         'KEYS',
         'KILL',
         'LAST_INSERT_ID',
         'LEADING',
+        'LEFT',
         'LEVEL',
         'LIKE',
+        'LIMIT',
         'LINEAR',
         'LINES',
         'LOAD',
@@ -177,6 +190,7 @@ final class Tokenizer
         'MINUTE',
         'MINUTE_SECOND',
         'MODE',
+        'MODIFY',
         'MONTH',
         'MRG_MYISAM',
         'MYISAM',
@@ -193,6 +207,9 @@ final class Tokenizer
         'OPTIMIZE',
         'OPTION',
         'OPTIONALLY',
+        'OR',
+        'ORDER',
+        'OUTER',
         'OUTFILE',
         'OVER',
         'PACK_KEYS',
@@ -232,16 +249,19 @@ final class Tokenizer
         'RETURN',
         'RETURNS',
         'REVOKE',
+        'RIGHT',
         'RLIKE',
         'ROLLBACK',
         'ROW',
         'ROW_FORMAT',
         'ROWS',
         'SECOND',
         'SECURITY',
+        'SELECT',
         'SEPARATOR',
         'SERIALIZABLE',
         'SESSION',
+        'SET',
         'SHARE',
         'SHOW',
         'SHUTDOWN',
@@ -292,18 +312,24 @@ final class Tokenizer
         'TYPES',
         'UNBOUNDED',
         'UNCOMMITTED',
+        'UNION',
         'UNIQUE',
         'UNLOCK',
         'UNSIGNED',
+        'UPDATE',
         'USAGE',
         'USE',
         'USING',
+        'VALUES',
         'VARIABLES',
         'VIEW',
         'WHEN',
+        'WHERE',
+        'WINDOW',
         'WITH',
         'WORK',
         'WRITE',
+        'XOR',
         'YEAR_MONTH',
     ];
 

tests/TokenizerTest.php

@@ -7,9 +7,36 @@
 use Doctrine\SqlFormatter\Tokenizer;
 use PHPUnit\Framework\Attributes\DoesNotPerformAssertions;
 use PHPUnit\Framework\TestCase;
+use ReflectionClass;
+
+use function array_diff;
+use function explode;
 
 final class TokenizerTest extends TestCase
 {
+    public function testKeywordsReservedToplevelAndNewlineAreReserved(): void
+    {
+        $tokenizerReflClass = new ReflectionClass(Tokenizer::class);
+        /** @var list<string> $tokenizerReserved */
+        $tokenizerReserved = $tokenizerReflClass->getProperty('reserved')->getDefaultValue();
+        /** @var list<string> $tokenizerReservedToplevel */
+        $tokenizerReservedToplevel = $tokenizerReflClass->getProperty('reservedToplevel')->getDefaultValue();
+        /** @var list<string> $tokenizerReservedNewline */
+        $tokenizerReservedNewline = $tokenizerReflClass->getProperty('reservedNewline')->getDefaultValue();
+
+        $kws = [];
+
+        foreach ([...$tokenizerReservedToplevel, ...$tokenizerReservedNewline] as $v) {
+            foreach (explode(' ', $v) as $kw) {
+                $kws[] = $kw;
+            }
+        }
+
+        $kwsDiff = array_diff($kws, $tokenizerReserved);
+
+        self::assertSame([], $kwsDiff);
+    }
+
     #[DoesNotPerformAssertions]
     public function testThereAreNoRegressions(): void
     {