From 197669ff54a06403411252d6207e27ccd337d346 Mon Sep 17 00:00:00 2001 From: Isak Date: Thu, 30 Oct 2025 10:51:11 +0100 Subject: [PATCH] tries to switch to antlr4ng and antlr-ng parser WIP --- packages/language-support/package.json | 9 +- .../completionCoreCompletions.ts | 48 ++++++----- .../autocompletion/schemaBasedCompletions.ts | 8 +- .../src/formatting/formatting.ts | 68 +++++++-------- .../src/formatting/formattingHelpers.ts | 37 +++++---- .../src/formatting/standardizer.ts | 4 +- packages/language-support/src/helpers.ts | 53 ++++++------ packages/language-support/src/index.ts | 14 ++-- packages/language-support/src/lexerSymbols.ts | 4 +- .../language-support/src/parserWrapper.ts | 82 ++++++++++--------- .../language-support/src/signatureHelp.ts | 12 +-- .../src/syntaxColouring/syntaxColouring.ts | 10 +-- .../syntaxColouring/syntaxColouringHelpers.ts | 4 +- .../syntaxValidation/completionCoreErrors.ts | 8 +- .../syntaxValidationHelpers.ts | 21 ++--- .../language-support/src/tests/lexer.test.ts | 14 ++-- .../src/treeUtil.ts | 40 ++++----- pnpm-lock.yaml | 9 +- 18 files changed, 234 insertions(+), 211 deletions(-) diff --git a/packages/language-support/package.json b/packages/language-support/package.json index b58beffb5..ab1f588ba 100644 --- a/packages/language-support/package.json +++ b/packages/language-support/package.json @@ -51,14 +51,15 @@ "node": ">=22.15.0" }, "dependencies": { - "antlr4": "4.13.2", + "antlr4ng": "3.0.16", + "antlr4-c3": "3.4.4", "fastest-levenshtein": "^1.0.16", "vscode-languageserver-types": "^3.17.3" }, "scripts": { - "gen-parser": "cross-env ANTLR4_TOOLS_ANTLR_VERSION=4.13.2 antlr4 -Dlanguage=TypeScript -visitor src/antlr-grammar/CypherCmdLexer.g4 src/antlr-grammar/CypherCmdParser.g4 -o src/generated-parser/ -Xexact-output-dir", - "build": "cd ../../vendor/antlr4-c3/ && pnpm build && cd - && pnpm gen-parser && concurrently 'npm:build-esm-and-types' 'npm:build-commonjs' 'npm:build-cypherfmt-cli'", - "build-esm-and-types": "tsc --module esnext --outDir dist/esm/project/language-support && cp src/syntaxValidation/semanticAnalysis.js dist/esm/project/language-support/src/syntaxValidation/semanticAnalysis.js && mkdir -p dist/esm/vendor/antlr4-c3/dist/ && cp -r ../../vendor/antlr4-c3/dist/esm dist/esm/vendor/antlr4-c3/dist/", + "gen-parser": "cross-env antlr-ng -Dlanguage=TypeScript --generate-visitor true src/antlr-grammar/CypherCmdLexer.g4 src/antlr-grammar/CypherCmdParser.g4 -o src/generated-parser/ --exact-output-dir", + "build": "pnpm gen-parser && concurrently 'npm:build-esm-and-types' 'npm:build-commonjs' 'npm:build-cypherfmt-cli'", + "build-esm-and-types": "tsc --module esnext --outDir dist/esm/project/language-support && cp src/syntaxValidation/semanticAnalysis.js dist/esm/project/language-support/src/syntaxValidation/semanticAnalysis.js", "build-commonjs": "esbuild ./src/index.ts --bundle --format=cjs --sourcemap --outfile=dist/cjs/index.cjs", "build-cypherfmt-cli": "esbuild ./src/formatting/cli.ts --bundle --format=esm --sourcemap --outfile=dist/esm/formatting/cli.mjs --external:fs --external:path", "dev": "pnpm build && concurrently 'tsc --module esnext --outDir dist/esm/project/language-support --watch' 'npm:build-commonjs -- --watch'", diff --git a/packages/language-support/src/autocompletion/completionCoreCompletions.ts b/packages/language-support/src/autocompletion/completionCoreCompletions.ts index 481a7160e..81ede5f88 100644 --- a/packages/language-support/src/autocompletion/completionCoreCompletions.ts +++ b/packages/language-support/src/autocompletion/completionCoreCompletions.ts @@ -4,8 +4,9 @@ import { InsertTextFormat, } from 'vscode-languageserver-types'; import { DbSchema } from '../dbSchema'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; -import CypherParser, { +import { CypherCmdLexer as CypherLexer } from '../generated-parser/CypherCmdLexer'; +import { + CypherCmdParser as CypherParser, CallClauseContext, Expression2Context, } from '../generated-parser/CypherCmdParser'; @@ -24,12 +25,13 @@ import { import { getMethodName, ParsedStatement } from '../parserWrapper'; -import type { CandidateRule } from '../../../../vendor/antlr4-c3/dist/esm/index.js'; +import type { ICandidateRule } from 'antlr4-c3'; import { CandidatesCollection, CodeCompletionCore, - Token, -} from '../../../../vendor/antlr4-c3/dist/esm/index.js'; +} from 'antlr4-c3'; +import { Token } from 'antlr4ng'; + import { _internalFeatureFlags } from '../featureFlags'; import { CompletionItem, @@ -80,7 +82,7 @@ const procedureReturnCompletions = ( }; const functionNameCompletions = ( - candidateRule: CandidateRule, + candidateRule: ICandidateRule, tokens: Token[], dbSchema: DbSchema, cypherVersion: CypherVersion, @@ -93,7 +95,7 @@ const functionNameCompletions = ( ); const procedureNameCompletions = ( - candidateRule: CandidateRule, + candidateRule: ICandidateRule, tokens: Token[], dbSchema: DbSchema, cypherVersion: CypherVersion, @@ -143,7 +145,7 @@ function getMethodCompletionItem( } const namespacedCompletion = ( - candidateRule: CandidateRule, + candidateRule: ICandidateRule, tokens: Token[], signatures: Record | Record, type: 'procedure' | 'function', @@ -246,8 +248,10 @@ function getTokenCompletions( const firstToken = isConsoleCommand ? tokenNames[tokenNumber].toLowerCase() : tokenNames[tokenNumber]; + + - const followUpIndexes = followUpList.indexes; + const followUpIndexes = followUpList; const firstIgnoredToken = followUpIndexes.findIndex((t) => ignoredTokens.has(t), ); @@ -269,12 +273,12 @@ function getTokenCompletions( ' ' + (isConsoleCommand ? followUpString.toLowerCase() : followUpString); - if (followUpList.optional) { - return [ - { label: firstToken, kind }, - { label: followUp, kind }, - ]; - } + // if (followUpList.optional) { + // return [ + // { label: firstToken, kind }, + // { label: followUp, kind }, + // ]; + // } return [{ label: followUp, kind }]; } @@ -345,7 +349,7 @@ enum ExpectedParameterType { Any = 'ANY', } -const inferExpectedParameterTypeFromContext = (context: CandidateRule) => { +const inferExpectedParameterTypeFromContext = (context: ICandidateRule) => { const parentRule = context.ruleList.at(-1); if ( @@ -421,7 +425,7 @@ function couldBeNodeOrRel( } function calculateNamespacePrefix( - candidateRule: CandidateRule, + candidateRule: ICandidateRule, tokens: Token[], ): string | null { const ruleTokens = tokens.slice(candidateRule.startTokenIndex); @@ -429,7 +433,7 @@ function calculateNamespacePrefix( const nonSpaceTokens = ruleTokens.filter( (token) => - token.type !== CypherLexer.SPACE && token.type !== CypherLexer.EOF, + token.type !== CypherLexer.SPACE && token.type !== CypherLexer.cEOF, ); const lastNonSpaceIsDot = nonSpaceTokens.at(-1)?.type === CypherLexer.DOT; @@ -478,7 +482,7 @@ export function completionCoreCompletion( // When we have EOF with a different text in the token, it means the parser has failed to parse it. // We give empty completions in that case because the query is severely broken at the // point of completion (e.g. an unclosed string) - if (eof.type === CypherLexer.EOF && eof.text !== '') { + if (eof.type === CypherLexer.cEOF && eof.text !== '') { return []; } @@ -563,13 +567,13 @@ export function completionCoreCompletion( if (ruleNumber === CypherParser.RULE_procedureResultItem) { const callContext = findParent( - parsingResult.lastRule.parentCtx, + parsingResult.lastRule.parent, (x) => x instanceof CallClauseContext, ); if (callContext instanceof CallClauseContext) { const procedureNameCtx = callContext.procedureName(); const existingYieldItems = new Set( - callContext.procedureResultItem_list().map((a) => a.getText()), + callContext.procedureResultItem().map((a) => a.getText()), ); const name = getMethodName(procedureNameCtx); return procedureReturnCompletions( @@ -794,7 +798,7 @@ type CompletionHelperArgs = { dbSchema: DbSchema; previousToken?: Token; tokens: Token[]; - candidateRule: CandidateRule; + candidateRule: ICandidateRule; }; function completeAliasName({ candidateRule, diff --git a/packages/language-support/src/autocompletion/schemaBasedCompletions.ts b/packages/language-support/src/autocompletion/schemaBasedCompletions.ts index a58782fdd..96fea3e4f 100644 --- a/packages/language-support/src/autocompletion/schemaBasedCompletions.ts +++ b/packages/language-support/src/autocompletion/schemaBasedCompletions.ts @@ -11,7 +11,7 @@ import { PatternElementContext, QuantifierContext, } from '../generated-parser/CypherCmdParser'; -import { ParserRuleContext } from 'antlr4'; +import { ParserRuleContext } from 'antlr4ng'; import { backtickIfNeeded } from './autocompletionHelpers'; import { _internalFeatureFlags } from '../featureFlags'; @@ -128,7 +128,7 @@ export function completeRelationshipType( // limitation: not checking PathPatternNonEmptyContext // limitation: not handling parenthesized paths const patternContext = findParent( - parsingResult.lastRule.parentCtx, + parsingResult.lastRule.parent, (x) => x instanceof PatternElementContext, ); @@ -137,9 +137,7 @@ export function completeRelationshipType( .toReversed() .find((child) => { if (child instanceof ParserRuleContext) { - if (child.exception === null) { - return true; - } + return true; } }); diff --git a/packages/language-support/src/formatting/formatting.ts b/packages/language-support/src/formatting/formatting.ts index 51f85b021..51536c8df 100644 --- a/packages/language-support/src/formatting/formatting.ts +++ b/packages/language-support/src/formatting/formatting.ts @@ -4,8 +4,8 @@ import { ParserRuleContext, TerminalNode, Token, -} from 'antlr4'; -import { default as CypherCmdLexer } from '../generated-parser/CypherCmdLexer'; +} from 'antlr4ng'; +import { CypherCmdLexer as CypherCmdLexer } from '../generated-parser/CypherCmdLexer'; import { AddPropContext, ArrowLineContext, @@ -112,7 +112,7 @@ import { WhereClauseContext, WithClauseContext, } from '../generated-parser/CypherCmdParser'; -import CypherCmdParserVisitor from '../generated-parser/CypherCmdParserVisitor'; +import { CypherCmdParserVisitor } from '../generated-parser/CypherCmdParserVisitor'; import { Chunk, CommentChunk, @@ -577,7 +577,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { let gapText = ''; if (this.previousTokenIndex < errorTokenIndex - 1) { - const skippedTokens = this.tokenStream.tokens.slice( + const skippedTokens = this.tokenStream.getTokens().slice( this.previousTokenIndex + 1, errorTokenIndex, ); @@ -607,7 +607,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { }; visitStatementsOrCommands = (ctx: StatementsOrCommandsContext) => { - const n = ctx.statementOrCommand_list().length; + const n = ctx.statementOrCommand().length; for (let i = 0; i < n; i++) { this._visit(ctx.statementOrCommand(i)); if (i < n - 1 || ctx.SEMICOLON(i)) { @@ -746,7 +746,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this.avoidSpaceBetween(); this.visit(ctx.COLON()); this.avoidSpaceBetween(); - const n = ctx.symbolicNameString_list().length; + const n = ctx.symbolicNameString().length; for (let i = 0; i < n; i++) { this.visit(ctx.symbolicNameString(i)); if (i < n - 1) { @@ -802,7 +802,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { }; visitWhen = (ctx: WhenContext) => { - const n = ctx.whenBranch_list().length; + const n = ctx.whenBranch().length; for (let i = 0; i < n; i++) { this.breakLine(); this._visit(ctx.whenBranch(i)); @@ -868,7 +868,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this._visit(ctx.patternList()); this.endGroup(patternListGrp); this.endGroup(matchClauseGrp); - const n = ctx.hint_list().length; + const n = ctx.hint().length; for (let i = 0; i < n; i++) { this._visit(ctx.hint(i)); } @@ -901,7 +901,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { } this._visit(ctx.DELETE()); const deleteIndent = this.addIndentation(); - const n = ctx.expression_list().length; + const n = ctx.expression().length; for (let i = 0; i < n; i++) { this._visit(ctx.expression(i)); if (i < n - 1) { @@ -1002,7 +1002,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { const letGrp = this.startGroup(); this._visit(ctx.LET()); const letIndent = this.addIndentation(); - const n = ctx.letItem_list().length; + const n = ctx.letItem().length; for (let i = 0; i < n; i++) { this._visit(ctx.letItem(i)); if (i < n - 1) { @@ -1049,7 +1049,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { if (ctx.TIMES()) { this._visit(ctx.TIMES()); } - const n = ctx.returnItem_list().length; + const n = ctx.returnItem().length; if (n === 1) { this.setOneItemProperty(); } @@ -1166,7 +1166,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { }; visitLabelExpression2 = (ctx: LabelExpression2Context) => { - const n = ctx.EXCLAMATION_MARK_list().length; + const n = ctx.EXCLAMATION_MARK().length; for (let i = 0; i < n; i++) { this._visitTerminalRaw(ctx.EXCLAMATION_MARK(i)); this.concatenate(); @@ -1184,7 +1184,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { if (this.chunkList.length === 0) { this.addCommentsBefore(node); } - if (node.symbol.type === CypherCmdLexer.EOF) { + if (node.symbol.type === CypherCmdLexer.cEOF) { return; } let text = node.getText(); @@ -1375,7 +1375,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this.avoidSpaceBetween(); this._visit(ctx.leftArrow()); } - const arrowLineList = ctx.arrowLine_list(); + const arrowLineList = ctx.arrowLine(); // Concatenations are to ensure the (left) arrow remains // on the previous line (styleguide rule) if we need to break within the pattern this._visit(arrowLineList[0]); @@ -1532,7 +1532,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { }; visitPatternList = (ctx: PatternListContext) => { - const n = ctx.pattern_list().length; + const n = ctx.pattern().length; const wholePatternListGrp = this.startGroup(); for (let i = 0; i < n; i++) { const patternListItemGrp = this.startGroup(); @@ -1621,7 +1621,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { // Handled separately because the dots aren't operators visitNamespace = (ctx: NamespaceContext) => { - const n = ctx.DOT_list().length; + const n = ctx.DOT().length; for (let i = 0; i < n; i++) { this._visit(ctx.symbolicNameString(i)); this.avoidSpaceBetween(); @@ -1632,7 +1632,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { // Handled separately because the dot is not an operator visitProperty = (ctx: PropertyContext) => { this._visitTerminalRaw(ctx.DOT()); - if (!(ctx.parentCtx instanceof MapProjectionElementContext)) { + if (!(ctx.parent instanceof MapProjectionElementContext)) { this.concatenate(); } this._visit(ctx.propertyKeyName()); @@ -1696,7 +1696,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { }; visitExpression9 = (ctx: Expression9Context) => { - const n = ctx.NOT_list().length; + const n = ctx.NOT().length; if (n === 0) { this.visitChildren(ctx); return; @@ -1736,7 +1736,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { visitExpression2 = (ctx: Expression2Context) => { this._visit(ctx.expression1()); - const n = ctx.postFix_list().length; + const n = ctx.postFix().length; for (let i = 0; i < n; i++) { this.avoidSpaceBetween(); this._visit(ctx.postFix(i)); @@ -1862,7 +1862,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { const wholeCaseGrp = this.startGroup(); this.mustBreakBetween(); this.visit(ctx.CASE()); - const n = ctx.caseAlternative_list().length; + const n = ctx.caseAlternative().length; for (let i = 0; i < n; i++) { this.mustBreakBetween(); this.visit(ctx.caseAlternative(i)); @@ -1888,7 +1888,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { const wholeAlternativeGrp = this.startGroup(); this._visit(ctx.WHEN()); const whenIndent = this.addIndentation(); - const n = ctx.extendedWhen_list().length; + const n = ctx.extendedWhen().length; for (let i = 0; i < n; i++) { this.visit(ctx.extendedWhen(i)); if (i < n - 1) { @@ -1913,7 +1913,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this.avoidBreakBetween(); this._visit(ctx.expression(0)); this.mustBreakBetween(); - const n = ctx.extendedCaseAlternative_list().length; + const n = ctx.extendedCaseAlternative().length; for (let i = 0; i < n; i++) { this.mustBreakBetween(); this.visit(ctx.extendedCaseAlternative(i)); @@ -1958,7 +1958,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { if (ctx.TIMES()) { this._visit(ctx.TIMES()); } else { - const n = ctx.variable_list().length; + const n = ctx.variable().length; for (let i = 0; i < n; i++) { const functionArgGrp = this.startGroup(); this._visit(ctx.variable(i)); @@ -1980,7 +1980,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { // Handled separately because UNION wants to look a certain way visitUnion = (ctx: UnionContext) => { this._visit(ctx.singleQuery(0)); - const n = ctx.singleQuery_list().length - 1; + const n = ctx.singleQuery().length - 1; for (let i = 0; i < n; i++) { const unionIndent = this.addIndentation(); this.breakLine(); @@ -2065,7 +2065,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { } this._visit(ctx.ALL()); this._visit(ctx.DISTINCT()); - const n = ctx.functionArgument_list().length; + const n = ctx.functionArgument().length; if (n === 1) { this.setOneItemProperty(); } @@ -2103,7 +2103,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this._visit(ctx.pattern()); this.removeIndentation(mergeClauseIndent); this.endGroup(mergeGrp); - const n = ctx.mergeAction_list().length; + const n = ctx.mergeAction().length; for (let i = 0; i < n; i++) { this._visit(ctx.mergeAction(i)); } @@ -2180,7 +2180,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { const setClauseGrp = this.startGroup(); this._visit(ctx.SET()); const setIndent = this.addIndentation(); - const n = ctx.setItem_list().length; + const n = ctx.setItem().length; for (let i = 0; i < n; i++) { const setItemGrp = this.startGroup(); this._visit(ctx.setItem(i)); @@ -2201,7 +2201,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this.setSpecialSplitProperty(); const mapIndent = this.addIndentation(); this.avoidSpaceBetween(); - const n = ctx.expression_list().length; + const n = ctx.expression().length; for (let i = 0; i < n; i++) { const keyValueGrp = this.startGroup(); this._visit(ctx.propertyKeyName(i)); @@ -2250,7 +2250,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this._visit(ctx.LCURLY()); const mapProjectionIndent = this.addIndentation(); this.avoidSpaceBetween(); - const n = ctx.mapProjectionElement_list().length; + const n = ctx.mapProjectionElement().length; for (let i = 0; i < n; i++) { this._visit(ctx.mapProjectionElement(i)); if (i < n - 1) { @@ -2306,7 +2306,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this._visit(ctx.LBRACKET()); this.setSpecialSplitProperty(); const listIndent = this.addIndentation(); - const n = ctx.expression_list().length; + const n = ctx.expression().length; for (let i = 0; i < n; i++) { const listElemGrp = this.startGroup(); this._visit(ctx.expression(i)); @@ -2367,7 +2367,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { this._visit(ctx.expression()); this._visit(ctx.BAR()); - const n = ctx.clause_list().length; + const n = ctx.clause().length; const clauseIndent = this.addIndentation(); for (let i = 0; i < n; i++) { this._visit(ctx.clause(i)); @@ -2395,7 +2395,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { const procedureNameGrp = this.startGroup(); this._visit(ctx.procedureName()); this.endGroup(procedureNameGrp); - const n = ctx.procedureArgument_list().length; + const n = ctx.procedureArgument().length; if (ctx.LPAREN()) { this.avoidSpaceBetween(); this.avoidBreakBetween(); @@ -2426,7 +2426,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor { } if (ctx.YIELD()) { const yieldGrp = this.startGroup(); - const m = ctx.procedureResultItem_list().length; + const m = ctx.procedureResultItem().length; this._visit(ctx.YIELD()); if (ctx.TIMES()) { this._visit(ctx.TIMES()); @@ -2525,7 +2525,7 @@ export function formatQuery( }; } - const targetToken = findTargetToken(tokens.tokens, cursorPosition); + const targetToken = findTargetToken(tokens.getTokens(), cursorPosition); if (!targetToken) { return { formattedQuery: visitor.format(), diff --git a/packages/language-support/src/formatting/formattingHelpers.ts b/packages/language-support/src/formatting/formattingHelpers.ts index 50b62f354..619b51248 100644 --- a/packages/language-support/src/formatting/formattingHelpers.ts +++ b/packages/language-support/src/formatting/formattingHelpers.ts @@ -1,6 +1,6 @@ -import { CharStreams, CommonTokenStream, TerminalNode, Token } from 'antlr4'; -import { default as CypherCmdLexer } from '../generated-parser/CypherCmdLexer'; -import CypherCmdParser, { +import { CharStream, CommonTokenStream, TerminalNode, Token } from 'antlr4ng'; +import { CypherCmdLexer } from '../generated-parser/CypherCmdLexer'; +import { CypherCmdParser, EscapedSymbolicNameStringContext, UnescapedSymbolicNameStringContext, } from '../generated-parser/CypherCmdParser'; @@ -122,7 +122,7 @@ function isSymbolicName(node: TerminalNode): boolean { } export function getParseTreeAndTokens(query: string) { - const inputStream = CharStreams.fromString(query); + const inputStream = CharStream.fromString(query); const lexer = new CypherCmdLexer(inputStream); const tokens = new CommonTokenStream(lexer); const parser = new CypherCmdParser(tokens); @@ -131,20 +131,21 @@ export function getParseTreeAndTokens(query: string) { const tree = parser.statementsOrCommands(); let unParseable: string | undefined; let firstUnParseableToken: Token | undefined; - if (tree.exception) { - const idx = tree.exception.offendingToken.tokenIndex; - const errorTokens = tokens.tokens.slice(idx); - const hiddenBefore = (tokens.getHiddenTokensToLeft(idx) || []) - .map((t) => t.text) - .join(''); - unParseable = - hiddenBefore + - errorTokens - .slice(0, -1) - .map((t) => t.text) - .join(''); - firstUnParseableToken = tree.exception.offendingToken; - } + //TODO Undo + // if (tree.exception) { + // const idx = tree.exception.offendingToken.tokenIndex; + // const errorTokens = tokens.getTokens().slice(idx); + // const hiddenBefore = (tokens.getHiddenTokensToLeft(idx) || []) + // .map((t) => t.text) + // .join(''); + // unParseable = + // hiddenBefore + + // errorTokens + // .slice(0, -1) + // .map((t) => t.text) + // .join(''); + // firstUnParseableToken = tree.exception.offendingToken; + // } return { tree, tokens, unParseable, firstUnParseableToken }; } diff --git a/packages/language-support/src/formatting/standardizer.ts b/packages/language-support/src/formatting/standardizer.ts index 06e6682ab..18eb802e2 100644 --- a/packages/language-support/src/formatting/standardizer.ts +++ b/packages/language-support/src/formatting/standardizer.ts @@ -1,6 +1,6 @@ -import { TerminalNode } from 'antlr4'; +import { TerminalNode } from 'antlr4ng'; import { StatementsOrCommandsContext } from '../generated-parser/CypherCmdParser'; -import CypherCmdParserVisitor from '../generated-parser/CypherCmdParserVisitor'; +import {CypherCmdParserVisitor} from '../generated-parser/CypherCmdParserVisitor'; import { getParseTreeAndTokens } from './formattingHelpers'; class StandardizingVisitor extends CypherCmdParserVisitor { diff --git a/packages/language-support/src/helpers.ts b/packages/language-support/src/helpers.ts index 1288615bc..a68dfb40a 100644 --- a/packages/language-support/src/helpers.ts +++ b/packages/language-support/src/helpers.ts @@ -1,20 +1,24 @@ // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore There is a default export but not in the types -import antlrDefaultExport, { +//TODO check how we can import this from -> antlrDefaultExport, + +import { CommonTokenStream, ParserRuleContext, ParseTree, Token, -} from 'antlr4'; +} from 'antlr4ng'; import { DbSchema } from './dbSchema'; -import CypherLexer from './generated-parser/CypherCmdLexer'; -import CypherParser, { +import { CypherCmdLexer as CypherLexer } from './generated-parser/CypherCmdLexer'; +import { CypherCmdParser as CypherParser, NodePatternContext, RelationshipPatternContext, } from './generated-parser/CypherCmdParser'; import { ParsedStatement, ParsingResult } from './parserWrapper'; import { CypherVersion } from './types'; + +const x = Token.EOF; /* In antlr we have ParseTree @@ -28,18 +32,18 @@ Both TerminalNode and RuleContext have parentCtx, but ParseTree doesn't This type fixes that because it's what we need to traverse the tree most of the time */ -export type EnrichedParseTree = ParseTree & { - parentCtx: ParserRuleContext | undefined; -}; +// export type EnrichedParseTree = ParseTree & { +// parentCtx: ParserRuleContext | undefined; +// }; export function findParent( - leaf: EnrichedParseTree | undefined, - condition: (node: EnrichedParseTree) => boolean, -): EnrichedParseTree { - let current: EnrichedParseTree | undefined = leaf; - + leaf: ParseTree | undefined, + condition: (node: ParseTree) => boolean, +): ParseTree { + let current: ParseTree | undefined = leaf; + while (current && !condition(current)) { - current = current.parentCtx; + current = current.parent; } return current; @@ -61,7 +65,7 @@ type AntlrDefaultExport = { }; }; }; -export const antlrUtils = antlrDefaultExport as unknown as AntlrDefaultExport; +// export const antlrUtils = antlrDefaultExport as unknown as AntlrDefaultExport; export function inNodeLabel(lastNode: ParserRuleContext) { const nodePattern = findParent( @@ -118,29 +122,32 @@ export function findCaret( export function splitIntoStatements( tokenStream: CommonTokenStream, lexer: CypherLexer, -): CommonTokenStream[] { +): Token[][] { tokenStream.fill(); - const tokens = tokenStream.tokens; - + const tokens = tokenStream.getTokens(); + let i = 0; - const result: CommonTokenStream[] = []; + const result: Token[][] = []; let chunk: Token[] = []; let offset = 0; while (i < tokens.length) { - const current = tokens[i].clone(); + const current = tokens[i]//.clone(); current.tokenIndex -= offset; chunk.push(current); if ( current.type === CypherLexer.SEMICOLON || - current.type === CypherLexer.EOF + current.type === CypherLexer.cEOF ) { + //TODO check if this works // This does not relex since we are not calling fill on the token stream - const tokenStream = new CommonTokenStream(lexer); - tokenStream.tokens = chunk; - result.push(tokenStream); + //const tokenStream = new CommonTokenStream(chunk)// lexer); + //tokenStream........ + //tokenStream.tokens = chunk; + result.push(chunk); + // const newStream = new CommonTokenStream() offset = i + 1; chunk = []; } diff --git a/packages/language-support/src/index.ts b/packages/language-support/src/index.ts index 224c52553..3202dcf98 100644 --- a/packages/language-support/src/index.ts +++ b/packages/language-support/src/index.ts @@ -1,11 +1,11 @@ -export type { ParserRuleContext } from 'antlr4'; +export type { ParserRuleContext } from 'antlr4ng'; export { autocomplete } from './autocompletion/autocompletion'; export { shouldAutoCompleteYield } from './autocompletion/autocompletionHelpers'; export { backtickIfNeeded } from './autocompletion/autocompletionHelpers'; export type { DbSchema } from './dbSchema'; export { _internalFeatureFlags } from './featureFlags'; export { formatQuery } from './formatting/formatting'; -export { antlrUtils } from './helpers'; +// export { antlrUtils } from './helpers'; export { CypherTokenType, lexerSymbols } from './lexerSymbols'; export { parse, @@ -35,12 +35,12 @@ export type { Neo4jProcedure, SymbolTable, } from './types'; -export { CypherLexer, CypherParser, CypherParserListener, CypherParserVisitor }; +export { CypherLexer, CypherParser, CypherParserListener, CypherCmdParserVisitor }; -import CypherLexer from './generated-parser/CypherCmdLexer'; -import CypherParser from './generated-parser/CypherCmdParser'; -import CypherParserListener from './generated-parser/CypherCmdParserListener'; -import CypherParserVisitor from './generated-parser/CypherCmdParserVisitor'; +import { CypherCmdLexer as CypherLexer } from './generated-parser/CypherCmdLexer'; +import { CypherCmdParser as CypherParser } from './generated-parser/CypherCmdParser'; +import { CypherCmdParserListener as CypherParserListener } from './generated-parser/CypherCmdParserListener'; +import { CypherCmdParserVisitor } from './generated-parser/CypherCmdParserVisitor'; export * from './generated-parser/CypherCmdLexer'; export * from './generated-parser/CypherCmdParser'; diff --git a/packages/language-support/src/lexerSymbols.ts b/packages/language-support/src/lexerSymbols.ts index b2d0597a3..20ab71aa8 100644 --- a/packages/language-support/src/lexerSymbols.ts +++ b/packages/language-support/src/lexerSymbols.ts @@ -1,4 +1,4 @@ -import CypherLexer from './generated-parser/CypherCmdLexer'; +import {CypherCmdLexer as CypherLexer} from './generated-parser/CypherCmdLexer'; export enum CypherTokenType { comment = 'comment', @@ -84,7 +84,7 @@ export const lexerStringLiteral = [ export const lexerGarbage = [ CypherLexer.ErrorChar, - CypherLexer.EOF, + CypherLexer.cEOF, CypherLexer.SPACE, ]; diff --git a/packages/language-support/src/parserWrapper.ts b/packages/language-support/src/parserWrapper.ts index c917dd170..ad58c494d 100644 --- a/packages/language-support/src/parserWrapper.ts +++ b/packages/language-support/src/parserWrapper.ts @@ -1,19 +1,19 @@ -import type { ParserRuleContext, Token } from 'antlr4'; +import { ParserRuleContext, Token } from 'antlr4ng'; import { ParseTreeWalker, - CharStreams, + CharStream, CommonTokenStream, ParseTreeListener, -} from 'antlr4'; +} from 'antlr4ng'; -import CypherLexer from './generated-parser/CypherCmdLexer'; +import { CypherCmdLexer as CypherLexer } from './generated-parser/CypherCmdLexer'; import { DiagnosticSeverity, Position } from 'vscode-languageserver-types'; import { _internalFeatureFlags } from './featureFlags'; import { ClauseContext, CypherVersionContext, - default as CypherParser, + CypherCmdParser as CypherParser, FunctionNameContext, LabelNameContext, LabelOrRelTypeContext, @@ -127,15 +127,16 @@ export type ParsedFunction = HasPosition & { export type ParsedProcedure = ParsedFunction; export function createParsingScaffolding(query: string): ParsingScaffolding { - const inputStream = CharStreams.fromString(query); + const inputStream = CharStream.fromString(query); const lexer = new CypherLexer(inputStream); const tokenStream = new CommonTokenStream(lexer); const stmTokenStreams = splitIntoStatements(tokenStream, lexer); const statementsScaffolding: StatementParsingScaffolding[] = stmTokenStreams.map((t) => { - const tokens = [...t.tokens]; - const parser = new CypherParser(t); + const tokens = [...t]; + //TODO check if this works (now passing full stream) + const parser = new CypherParser(tokenStream); parser.removeErrorListeners(); return { @@ -155,12 +156,21 @@ export function parseStatementsStrs(query: string): string[] { const result: string[] = []; for (const statement of statements) { - const tokenStream = statement.parser?.getTokenStream() ?? []; - const tokens = (tokenStream as CommonTokenStream).tokens; - const statementStr = tokens - .filter((token) => token.type !== CypherLexer.EOF) - .map((token) => token.text) - .join(''); + const fullStatement = statement.getText(); + //Suppose this is what we were doing before... but constant ";" is not nice + let statementStr: string; + if(fullStatement.at(-1) == ";") { + statementStr = fullStatement.substring(0, fullStatement.length-1) + } else { + statementStr = fullStatement; + } + + // const tokenStream = statement.parser?.getTokenStream() ?? []; + // const tokens = (tokenStream as CommonTokenStream).getTokens(); + // const statementStr = tokens + // .filter((token) => token.type !== CypherLexer.EOF) + // .map((token) => token.text) + // .join(''); // Do not return empty statements if (statementStr.trimLeft().length != 0) { @@ -205,7 +215,9 @@ export function createParsingResult( consoleCommandsEnabled, ); const lastRuleListener = new LastRuleListener(caretPosition); - parser._parseListeners = [ + + + const listeners = [ labelsCollector, parameterFinder, variableFinder, @@ -213,6 +225,7 @@ export function createParsingResult( cypherVersionCollector, lastRuleListener, ]; + listeners.forEach(x => parser.addParseListener(x)) parser.addErrorListener(errorListener); const ctx = parser.statementsOrCommands(); // The statement is empty if we cannot find anything that is not EOF or a space @@ -287,7 +300,7 @@ export function parseParameters( } // This listener collects all labels and relationship types -class LabelAndRelTypesCollector extends ParseTreeListener { +class LabelAndRelTypesCollector implements ParseTreeListener { labelOrRelTypes: LabelOrRelType[] = []; enterEveryRule() { @@ -306,7 +319,7 @@ class LabelAndRelTypesCollector extends ParseTreeListener { // like in the case MATCH (n:) RETURN n // RETURN would be incorrectly idenfified as the label // If this is the case, the context containing the label would have an error node - if (ctx.parentCtx && !hasErrorNodesUnder(ctx.parentCtx)) { + if (ctx.parent && !hasErrorNodesUnder(ctx.parent)) { this.labelOrRelTypes.push({ labelType: getLabelType(ctx), labelText: ctx.getText(), @@ -324,8 +337,8 @@ class LabelAndRelTypesCollector extends ParseTreeListener { // Read comment for the label name case if ( isDefined(symbolicName) && - ctx.parentCtx && - !hasErrorNodesUnder(ctx.parentCtx) + ctx.parent && + !hasErrorNodesUnder(ctx.parent) ) { this.labelOrRelTypes.push({ labelType: getLabelType(ctx), @@ -434,10 +447,10 @@ class VariableCollector implements ParseTreeListener { // We check if the variable is in the end of the statement const nextTokenIndex = ctx.stop?.tokenIndex; - const nextTokenIsEOF = - nextTokenIndex !== undefined && - ctx.parser?.getTokenStream().get(nextTokenIndex + 1)?.type === - CypherParser.EOF; + const nextTokenIsEOF = false + // nextTokenIndex !== undefined && + // ctx.parser?.getTokenStream().get(nextTokenIndex + 1)?.type === + // CypherParser.cEOF; const definesVariable = rulesDefiningOrUsingVariables.includes( // @ts-expect-error the antlr4 types don't include ruleIndex but it is there, fix as the official types are improved @@ -460,7 +473,7 @@ class VariableCollector implements ParseTreeListener { export function getMethodName( ctx: ProcedureNameContext | FunctionNameContext, ): string { - const namespaces = ctx.namespace().symbolicNameString_list(); + const namespaces = ctx.namespace().symbolicNameString(); const methodName = ctx.symbolicNameString(); const normalizedName = [...namespaces, methodName] @@ -485,13 +498,12 @@ function getNamespaceString(ctx: SymbolicNameStringContext): string { } // This listener collects all functions and procedures -class MethodsCollector extends ParseTreeListener { +class MethodsCollector implements ParseTreeListener { public procedures: ParsedProcedure[] = []; public functions: ParsedFunction[] = []; private tokens: Token[]; constructor(tokens: Token[]) { - super(); this.tokens = tokens; } @@ -542,14 +554,10 @@ class MethodsCollector extends ParseTreeListener { } } -class CypherVersionCollector extends ParseTreeListener { +class CypherVersionCollector implements ParseTreeListener { public cypherVersion: CypherVersion; public invalidVersionError: SyntaxDiagnostic; - constructor() { - super(); - } - enterEveryRule() { /* no-op */ } @@ -617,7 +625,7 @@ function parseToCommand( tokens: Token[], isEmptyStatement: boolean, ): ParsedCommand { - const stmt = stmts.statementOrCommand_list().at(0); + const stmt = stmts.statementOrCommand().at(0); if (stmt) { const start = stmts.start; @@ -626,13 +634,13 @@ function parseToCommand( if (stop && stop.type === CypherLexer.SEMICOLON) { stop = tokens[stop.tokenIndex - 1]; } - const inputstream = start.getInputStream(); + const inputstream = start.inputStream; const cypherStmt = stmt.preparsedStatement()?.statement(); if (cypherStmt) { // we get the original text input to preserve whitespace // stripping the preparser part of it const stmtStart = cypherStmt.start; - const statement = inputstream.getText(stmtStart.start, stop.stop); + const statement = inputstream.getTextFromRange(stmtStart.start, stop.stop); return { type: 'cypher', statement, start: stmtStart, stop: stop }; } @@ -676,10 +684,10 @@ function parseToCommand( const cypherMap = paramArgs.map(); if (cypherMap) { const names = cypherMap - ?.propertyKeyName_list() + ?.propertyKeyName() .map((name) => name.getText()); const expressions = cypherMap - ?.expression_list() + ?.expression() .map((expr) => expr.getText()); if (names && expressions && names.length === expressions.length) { @@ -804,7 +812,7 @@ function parseToCommand( return { type: 'parse-error', start, stop }; } const stopToken = stop ?? tokens.at(-1); - const statement = inputstream.getText(start.start, stopToken.stop); + const statement = inputstream.getTextFromRange(start.start, stopToken.stop); return { type: 'cypher', statement, start: start, stop: stopToken }; } return { type: 'parse-error', start: stmts.start, stop: stmts.stop }; diff --git a/packages/language-support/src/signatureHelp.ts b/packages/language-support/src/signatureHelp.ts index ba4919ad2..699e1bd3e 100644 --- a/packages/language-support/src/signatureHelp.ts +++ b/packages/language-support/src/signatureHelp.ts @@ -3,16 +3,16 @@ import { SignatureInformation, } from 'vscode-languageserver-types'; -import { ParseTreeWalker } from 'antlr4'; -import CypherParser, { +import { ParseTreeWalker } from 'antlr4ng'; +import {CypherCmdParser as CypherParser, CallClauseContext, ExpressionContext, FunctionInvocationContext, } from './generated-parser/CypherCmdParser'; -import { Token } from '../../../vendor/antlr4-c3/dist/esm/index.js'; +import { Token } from 'antlr4ng'; import { DbSchema } from './dbSchema'; -import CypherCmdParserListener from './generated-parser/CypherCmdParserListener'; +import {CypherCmdParserListener} from './generated-parser/CypherCmdParserListener'; import { findCaret, isDefined, resolveCypherVersion } from './helpers'; import { parserWrapper } from './parserWrapper'; import { Neo4jFunction, Neo4jProcedure } from './types'; @@ -135,7 +135,7 @@ class SignatureHelper extends CypherCmdParserListener { isDefined(ctx.LPAREN()) ) { const methodName = ctx.functionName().getText(); - const previousArguments = ctx.COMMA_list().filter((arg) => { + const previousArguments = ctx.COMMA().filter((arg) => { return arg.symbol.stop <= this.caretToken.start; }); @@ -156,7 +156,7 @@ class SignatureHelper extends CypherCmdParserListener { isDefined(ctx.LPAREN()) ) { const methodName = ctx.procedureName().getText(); - const previousArguments = ctx.COMMA_list().filter((arg) => { + const previousArguments = ctx.COMMA().filter((arg) => { return arg.symbol.stop <= this.caretToken.start; }); diff --git a/packages/language-support/src/syntaxColouring/syntaxColouring.ts b/packages/language-support/src/syntaxColouring/syntaxColouring.ts index cf60a9be1..e9bcca4e9 100644 --- a/packages/language-support/src/syntaxColouring/syntaxColouring.ts +++ b/packages/language-support/src/syntaxColouring/syntaxColouring.ts @@ -1,4 +1,4 @@ -import { ParseTreeWalker, TerminalNode, Token } from 'antlr4'; +import { ParseTreeWalker, TerminalNode, Token } from 'antlr4ng'; import { AccessModeArgsContext, @@ -35,8 +35,8 @@ import { SemanticTokensLegend, SemanticTokenTypes, } from 'vscode-languageserver-types'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; -import CypherParserListener from '../generated-parser/CypherCmdParserListener'; +import {CypherCmdLexer as CypherLexer} from '../generated-parser/CypherCmdLexer'; +import {CypherCmdParserListener as CypherParserListener} from '../generated-parser/CypherCmdParserListener'; import { CypherTokenType } from '../lexerSymbols'; import { parserWrapper } from '../parserWrapper'; import { @@ -169,7 +169,7 @@ class SyntaxHighlighter extends CypherParserListener { ) { const namespace = ctx.namespace(); - namespace.symbolicNameString_list().forEach((namespaceName) => { + namespace.symbolicNameString().forEach((namespaceName) => { this.addToken(namespaceName.start, tokenType, namespaceName.getText()); }); @@ -338,7 +338,7 @@ function colourLexerTokens(tokens: Token[]) { if (!tokenFinished) { tokens.slice(i + 1, tokens.length).forEach((nextToken) => { - if (nextToken.type !== CypherLexer.EOF) { + if (nextToken.type !== CypherLexer.cEOF) { tokenStr += nextToken.text; } }); diff --git a/packages/language-support/src/syntaxColouring/syntaxColouringHelpers.ts b/packages/language-support/src/syntaxColouring/syntaxColouringHelpers.ts index ed6e8bb09..0da84734d 100644 --- a/packages/language-support/src/syntaxColouring/syntaxColouringHelpers.ts +++ b/packages/language-support/src/syntaxColouring/syntaxColouringHelpers.ts @@ -1,8 +1,8 @@ import { SemanticTokenTypes } from 'vscode-languageserver-types'; -import { Token } from 'antlr4'; +import { Token } from 'antlr4ng'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; +import {CypherCmdLexer as CypherLexer} from '../generated-parser/CypherCmdLexer'; import { isCommentOpener } from '../helpers'; import { CypherTokenType, lexerSymbols } from '../lexerSymbols'; diff --git a/packages/language-support/src/syntaxValidation/completionCoreErrors.ts b/packages/language-support/src/syntaxValidation/completionCoreErrors.ts index 3d5b19863..ece0568bb 100644 --- a/packages/language-support/src/syntaxValidation/completionCoreErrors.ts +++ b/packages/language-support/src/syntaxValidation/completionCoreErrors.ts @@ -1,8 +1,8 @@ -import { Token } from 'antlr4'; +import { Token } from 'antlr4ng'; import { distance } from 'fastest-levenshtein'; -import { CodeCompletionCore } from '../../../../vendor/antlr4-c3/dist/esm/index.js'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; -import CypherParser from '../generated-parser/CypherCmdParser'; +import { CodeCompletionCore } from 'antlr4-c3'; +import {CypherCmdLexer as CypherLexer} from '../generated-parser/CypherCmdLexer'; +import {CypherCmdParser as CypherParser} from '../generated-parser/CypherCmdParser'; import { CypherTokenType, keywordNames, diff --git a/packages/language-support/src/syntaxValidation/syntaxValidationHelpers.ts b/packages/language-support/src/syntaxValidation/syntaxValidationHelpers.ts index 44cdb73bc..7f664214b 100644 --- a/packages/language-support/src/syntaxValidation/syntaxValidationHelpers.ts +++ b/packages/language-support/src/syntaxValidation/syntaxValidationHelpers.ts @@ -1,13 +1,14 @@ import { CommonToken, - ErrorListener as ANTLRErrorListener, + ANTLRErrorListener, ParserRuleContext, Recognizer, Token, -} from 'antlr4'; + ATNSimulator, +} from 'antlr4ng'; import { DiagnosticSeverity, Position } from 'vscode-languageserver-types'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; -import CypherParser, { +import {CypherCmdLexer as CypherLexer} from '../generated-parser/CypherCmdLexer'; +import {CypherCmdParser as CypherParser, ConsoleCommandContext, PreparserOptionContext, } from '../generated-parser/CypherCmdParser'; @@ -15,7 +16,7 @@ import { findParent, isCommentOpener } from '../helpers'; import { completionCoreErrormessage } from './completionCoreErrors'; import { SyntaxDiagnostic } from './syntaxValidation'; -export class SyntaxErrorsListener implements ANTLRErrorListener { +export class SyntaxErrorsListener implements ANTLRErrorListener { errors: SyntaxDiagnostic[]; unfinishedToken: boolean; tokens: Token[]; @@ -28,9 +29,9 @@ export class SyntaxErrorsListener implements ANTLRErrorListener { this.consoleCommandsEnabled = consoleCommandsEnabled; } - public syntaxError( + public syntaxError( recognizer: Recognizer, - offendingSymbol: T, + offendingSymbol: S, line: number, charPositionInLine: number, ): void { @@ -40,8 +41,8 @@ export class SyntaxErrorsListener implements ANTLRErrorListener { if (!this.unfinishedToken) { const startLine = line - 1; const startColumn = charPositionInLine; - const parser = recognizer as CypherParser; - const ctx: ParserRuleContext = parser._ctx; + const parser = recognizer as unknown as CypherParser; + const ctx: ParserRuleContext = parser.context; const tokenIndex = offendingSymbol.tokenIndex; const nextTokenIndex = tokenIndex + 1; const nextToken = this.tokens.at(nextTokenIndex); @@ -105,7 +106,7 @@ export class SyntaxErrorsListener implements ANTLRErrorListener { if (errorMessage) { const endColumn = - offendingSymbol.type === CypherLexer.EOF + offendingSymbol.type === CypherLexer.cEOF ? startColumn : startColumn + offendingSymbol.text.length; diff --git a/packages/language-support/src/tests/lexer.test.ts b/packages/language-support/src/tests/lexer.test.ts index bc77edc95..e066624e9 100644 --- a/packages/language-support/src/tests/lexer.test.ts +++ b/packages/language-support/src/tests/lexer.test.ts @@ -1,6 +1,6 @@ -import { CharStreams, CommonTokenStream } from 'antlr4'; -import CypherLexer from '../generated-parser/CypherCmdLexer'; -import CypherParser from '../generated-parser/CypherCmdParser'; +import { CharStream, CommonTokenStream } from 'antlr4ng'; +import { CypherCmdLexer as CypherLexer} from '../generated-parser/CypherCmdLexer'; +import { CypherCmdParser as CypherParser} from '../generated-parser/CypherCmdParser'; import { CypherTokenType, lexerSymbols, tokenNames } from '../lexerSymbols'; describe('Lexer tokens', () => { @@ -14,7 +14,7 @@ describe('Lexer tokens', () => { const symbolNumber = parseInt(k, 10); // There is no "-1" position in the symbolic name list so we handle EOF separately - if (symbolNumber === CypherParser.EOF) { + if (symbolNumber === CypherParser.cEOF) { return 'EOF'; } return symbolicNames[symbolNumber]; @@ -41,13 +41,13 @@ describe('Lexer tokens', () => { keywordTokens.forEach((token) => { const tokenName = tokenNames[token]; - const inputStream = CharStreams.fromString(tokenName); + const inputStream = CharStream.fromString(tokenName); const lexer = new CypherLexer(inputStream); const tokenStream = new CommonTokenStream(lexer); tokenStream.fill(); - const tokens = tokenStream.tokens; + const tokens = tokenStream.getTokens(); expect(tokens.length).toBe(2); // If the test fails, it is useful to see the token that was parsed @@ -68,7 +68,7 @@ describe('Lexer tokens', () => { ); } expect(tokens[0].type).toBe(token); - expect(tokens[1].type).toBe(CypherLexer.EOF); + expect(tokens[1].type).toBe(CypherLexer.cEOF); }); }); }); diff --git a/packages/react-codemirror-playground/src/treeUtil.ts b/packages/react-codemirror-playground/src/treeUtil.ts index 35f174867..3f1066a08 100644 --- a/packages/react-codemirror-playground/src/treeUtil.ts +++ b/packages/react-codemirror-playground/src/treeUtil.ts @@ -4,32 +4,32 @@ interface SimpleTree { } import { - antlrUtils, + // antlrUtils, CypherParser, parse, ParserRuleContext, } from '@neo4j-cypher/language-support'; -export function getDebugTree(cypher: string): SimpleTree { - const statements = parse(cypher); +// export function getDebugTree(cypher: string): SimpleTree { +// const statements = parse(cypher); - function walk(node: ParserRuleContext): SimpleTree { - const name = antlrUtils.tree.Trees.getNodeText( - node, - CypherParser.ruleNames, - CypherParser, - ); +// function walk(node: ParserRuleContext): SimpleTree { +// const name = antlrUtils.tree.Trees.getNodeText( +// node, +// CypherParser.ruleNames, +// CypherParser, +// ); - return { - name: name, - children: antlrUtils.tree.Trees.getChildren(node).map(walk), - }; - } +// return { +// name: name, +// children: antlrUtils.tree.Trees.getChildren(node).map(walk), +// }; +// } - const children = statements.map((statement) => walk(statement)); +// const children = statements.map((statement) => walk(statement)); - return { - name: 'topNode', - children: children, - }; -} +// return { +// name: 'topNode', +// children: children, +// }; +// } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c2959460a..eaf199a04 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,9 +111,12 @@ importers: packages/language-support: dependencies: - antlr4: - specifier: 4.13.2 - version: 4.13.2 + antlr4-c3: + specifier: 3.4.4 + version: 3.4.4 + antlr4ng: + specifier: 3.0.16 + version: 3.0.16 fastest-levenshtein: specifier: ^1.0.16 version: 1.0.16