From 55e44062372287590aab490b18d9cd3df4fb83a3 Mon Sep 17 00:00:00 2001 From: Jakub Vrana Date: Fri, 19 Sep 2025 18:02:53 +0200 Subject: [PATCH 1/2] Treat #> as operator --- src/Tokenizer.php | 2 +- tests/TokenizerTest.php | 14 ++++++++++++++ tests/clihighlight.txt | 6 ++++++ tests/compress.txt | 4 ++++ tests/format-highlight.html | 6 ++++++ tests/format.txt | 6 ++++++ tests/highlight.html | 4 ++++ tests/sql.sql | 4 ++++ 8 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/Tokenizer.php b/src/Tokenizer.php index f8f35a6..90fff08 100644 --- a/src/Tokenizer.php +++ b/src/Tokenizer.php @@ -842,7 +842,7 @@ private function makeTokenizeRegexes(): array return [ Token::TOKEN_TYPE_WHITESPACE => '\s+', - Token::TOKEN_TYPE_COMMENT => '(?:--|#)[^\n]*+', + Token::TOKEN_TYPE_COMMENT => '(?:--|#(?!>))[^\n]*+', // #>, #>> and <#> are PostgreSQL operators Token::TOKEN_TYPE_BLOCK_COMMENT => '/\*(?:[^*]+|\*(?!/))*+(?:\*|$)(?:/|$)', // 1. backtick quoted string using `` to escape // 2. square bracket quoted string (SQL Server) using ]] to escape diff --git a/tests/TokenizerTest.php b/tests/TokenizerTest.php index 1e8af7c..7668542 100644 --- a/tests/TokenizerTest.php +++ b/tests/TokenizerTest.php @@ -1658,6 +1658,20 @@ public static function tokenizeData(): Generator ], '/* foo...', ]; + + yield 'PostgreSQL operator' => [ + [ + new Token(Token::TOKEN_TYPE_RESERVED_TOPLEVEL, 'select'), + new Token(Token::TOKEN_TYPE_WHITESPACE, ' '), + new Token(Token::TOKEN_TYPE_WORD, 'json'), + new Token(Token::TOKEN_TYPE_WHITESPACE, ' '), + new Token(Token::TOKEN_TYPE_BOUNDARY, '#'), + new Token(Token::TOKEN_TYPE_BOUNDARY, '>'), + new Token(Token::TOKEN_TYPE_WHITESPACE, ' '), + new Token(Token::TOKEN_TYPE_RESERVED, 'null'), + ], + 'select json #> null', + ]; } public function testTokenizeLongConcat(): void diff --git a/tests/clihighlight.txt b/tests/clihighlight.txt index 01fcff0..75c8429 100644 --- a/tests/clihighlight.txt +++ b/tests/clihighlight.txt @@ -1205,3 +1205,9 @@ MY_NON_TOP_LEVEL_KEYWORD_FX_3(); CREATE TABLE t ( c VARCHAR(20) ) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB +--- +SELECT + '{}' :: json #> '{}' +--- +SELECT + vector1 <#> vector2 diff --git a/tests/compress.txt b/tests/compress.txt index 58a0391..f7fcba4 100644 --- a/tests/compress.txt +++ b/tests/compress.txt @@ -111,3 +111,7 @@ SELECT a FROM test STRAIGHT_JOIN test2 ON test.id = test2.id SELECT t.id, t.start, t.end, t.end AS e2, t.limit, t.begin, t.case, t.when, t.then, t.else FROM t WHERE t.start = t.end --- CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB +--- +SELECT '{}'::json #> '{}' +--- +SELECT vector1 <#> vector2 diff --git a/tests/format-highlight.html b/tests/format-highlight.html index 5286c90..a059de3 100644 --- a/tests/format-highlight.html +++ b/tests/format-highlight.html @@ -1205,3 +1205,9 @@ 
CREATE TABLE t (
   c VARCHAR(20)
 ) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+--- +
SELECT
+  '{}' :: json #> '{}'
+--- +
SELECT
+  vector1 <#> vector2
diff --git a/tests/format.txt b/tests/format.txt index 9216132..868db4b 100644 --- a/tests/format.txt +++ b/tests/format.txt @@ -1203,3 +1203,9 @@ WHERE CREATE TABLE t ( c VARCHAR(20) ) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB +--- +SELECT + '{}' :: json #> '{}' +--- +SELECT + vector1 <#> vector2 diff --git a/tests/highlight.html b/tests/highlight.html index fa79610..e988db0 100644 --- a/tests/highlight.html +++ b/tests/highlight.html @@ -425,3 +425,7 @@ WHERE t.start = t.end --- 
CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
+--- +
SELECT '{}'::json #> '{}'
+--- +
SELECT vector1 <#> vector2
diff --git a/tests/sql.sql b/tests/sql.sql index d595f76..c0811e5 100644 --- a/tests/sql.sql +++ b/tests/sql.sql @@ -425,3 +425,7 @@ FROM t WHERE t.start = t.end --- CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB +--- +SELECT '{}'::json #> '{}' +--- +SELECT vector1 <#> vector2 From a5aa636593996f1df34d011bd365b0a14172ccd7 Mon Sep 17 00:00:00 2001 From: Jakub Vrana Date: Fri, 26 Sep 2025 11:08:56 +0200 Subject: [PATCH 2/2] Allow regenerating expected formatter outputs --- CONTRIBUTING.md | 4 ++++ bin/regenerate-expected-output | 29 +++++++++++++++++++++++++++++ tests/SqlFormatterTest.php | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100755 bin/regenerate-expected-output diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7a6ce7..363a849 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,3 +20,7 @@ vendor/bin/phpunit --testdox echo '' | vendor/bin/phpcs vendor/bin/phpstan analyze ``` + +## Regenerating expected output + +To regenerate expected tests output, run `bin/regenerate-expected-output`. diff --git a/bin/regenerate-expected-output b/bin/regenerate-expected-output new file mode 100755 index 0000000..824236d --- /dev/null +++ b/bin/regenerate-expected-output @@ -0,0 +1,29 @@ +#!/usr/bin/env php +