diff --git a/ydb/public/lib/ydb_cli/commands/interactive/complete/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/complete/ya.make new file mode 100644 index 000000000000..e2d8c56e93eb --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/complete/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + yql_completer.cpp +) + +PEERDIR( + contrib/restricted/patched/replxx + yql/essentials/sql/v1/complete +) + +END() + diff --git a/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.cpp b/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.cpp new file mode 100644 index 000000000000..e2b48fa5ea7d --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.cpp @@ -0,0 +1,45 @@ +#include "yql_completer.h" + +#include + +namespace NYdb::NConsoleClient { + + class TYQLCompleter: public IYQLCompleter { + public: + using TPtr = THolder; + + explicit TYQLCompleter(NSQLComplete::ISqlCompletionEngine::TPtr engine) + : Engine(std::move(engine)) + { + } + + TCompletions Apply(const std::string& prefix, int& /* contextLen */) override { + auto completion = Engine->Complete({ + .Text = prefix, + .CursorPosition = prefix.length(), + }); + + replxx::Replxx::completions_t entries; + for (auto& candidate : completion.Candidates) { + candidate.Content += ' '; + entries.emplace_back( + std::move(candidate.Content), + ReplxxColorOf(candidate.Kind)); + } + return entries; + } + + private: + static replxx::Replxx::Color ReplxxColorOf(NSQLComplete::ECandidateKind /* kind */) { + return replxx::Replxx::Color::DEFAULT; + } + + NSQLComplete::ISqlCompletionEngine::TPtr Engine; + }; + + IYQLCompleter::TPtr MakeYQLCompleter() { + return IYQLCompleter::TPtr( + new TYQLCompleter(NSQLComplete::MakeSqlCompletionEngine())); + } + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.h b/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.h new file mode 100644 index 000000000000..083a248b0ff8 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/complete/yql_completer.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include + +namespace NYdb::NConsoleClient { + + using TCompletions = replxx::Replxx::completions_t; + + class IYQLCompleter { + public: + using TPtr = THolder; + + virtual TCompletions Apply(const std::string& prefix, int& contextLen) = 0; + virtual ~IYQLCompleter() = default; + }; + + IYQLCompleter::TPtr MakeYQLCompleter(); + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/ut/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ut/ya.make new file mode 100644 index 000000000000..abff63532c23 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ut/ya.make @@ -0,0 +1,7 @@ +UNITTEST_FOR(ydb/public/lib/ydb_cli/commands/interactive/highlight) + +SRCS( + yql_highlighter_ut.cpp +) + +END() \ No newline at end of file diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make new file mode 100644 index 000000000000..26a7b0e16156 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +SRCS( + yql_highlighter.cpp + yql_position.cpp +) + +PEERDIR( + contrib/restricted/patched/replxx + yql/essentials/sql/v1/lexer + yql/essentials/sql/v1/lexer/antlr4 + yql/essentials/sql/v1/lexer/antlr4_ansi + yql/essentials/sql/settings +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp new file mode 100644 index 000000000000..ddb08f6974db --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp @@ -0,0 +1,313 @@ +#include "yql_highlighter.h" + +#include "yql_position.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace NYdb::NConsoleClient { + using NSQLTranslation::ILexer; + using NSQLTranslation::ParseTranslationSettings; + using NSQLTranslation::SQL_MAX_PARSER_ERRORS; + using NSQLTranslation::TParsedToken; + using NSQLTranslation::TParsedTokenList; + using NSQLTranslation::TTranslationSettings; + using NSQLTranslationV1::IsProbablyKeyword; + using NSQLTranslationV1::MakeLexer; + using NYql::TIssues; + + using std::regex_constants::ECMAScript; + using std::regex_constants::icase; + + NSQLTranslationV1::TLexers MakeLexers() { + NSQLTranslationV1::TLexers lexers; + lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); + lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); + return lexers; + } + + constexpr const char* builtinFunctionPattern = ( // + "^(" + "abs|aggregate_by|aggregate_list|aggregate_list_distinct|agg_list|agg_list_distinct|" + "as_table|avg|avg_if|adaptivedistancehistogram|adaptivewardhistogram|adaptiveweighthistogram|" + "addmember|addtimezone|aggregateflatten|aggregatetransforminput|aggregatetransformoutput|" + "aggregationfactory|asatom|asdict|asdictstrict|asenum|aslist|asliststrict|asset|assetstrict|" + "asstruct|astagged|astuple|asvariant|atomcode|bitcast|bit_and|bit_or|bit_xor|bool_and|" + "bool_or|bool_xor|bottom|bottom_by|blockwardhistogram|blockweighthistogram|cast|coalesce|" + "concat|concat_strict|correlation|count|count_if|covariance|covariance_population|" + "covariance_sample|callableargument|callableargumenttype|callableresulttype|callabletype|" + "callabletypecomponents|callabletypehandle|choosemembers|combinemembers|countdistinctestimate|" + "currentauthenticateduser|currentoperationid|currentoperationsharedid|currenttzdate|" + "currenttzdatetime|currenttztimestamp|currentutcdate|currentutcdatetime|currentutctimestamp|" + "dense_rank|datatype|datatypecomponents|datatypehandle|dictaggregate|dictcontains|dictcreate|" + "dicthasitems|dictitems|dictkeytype|dictkeys|dictlength|dictlookup|dictpayloadtype|dictpayloads|" + "dicttype|dicttypecomponents|dicttypehandle|each|each_strict|emptydicttype|emptydicttypehandle|" + "emptylisttype|emptylisttypehandle|endswith|ensure|ensureconvertibleto|ensuretype|enum|" + "evaluateatom|evaluatecode|evaluateexpr|evaluatetype|expandstruct|filter|filter_strict|find|" + "first_value|folder|filecontent|filepath|flattenmembers|forceremovemember|forceremovemembers|" + "forcerenamemembers|forcespreadmembers|formatcode|formattype|frombytes|frompg|funccode|" + "greatest|grouping|gathermembers|generictype|histogram|hll|hoppingwindowpgcast|hyperloglog|" + "if|if_strict|instanceof|json_exists|json_query|json_value|jointablerow|just|lag|last_value|" + "lead|least|len|length|like|likely|like_strict|lambdaargumentscount|lambdacode|" + "lambdaoptionalargumentscount|linearhistogram|listaggregate|listall|listany|listavg|listcode|" + "listcollect|listconcat|listcreate|listdistinct|listenumerate|listextend|listextendstrict|" + "listextract|listfilter|listflatmap|listflatten|listfold|listfold1|listfold1map|listfoldmap|" + "listfromrange|listfromtuple|listhas|listhasitems|listhead|listindexof|listitemtype|listlast|" + "listlength|listmap|listmax|listmin|listnotnull|listreplicate|listreverse|listskip|" + "listskipwhile|listskipwhileinclusive|listsort|listsortasc|listsortdesc|listsum|listtake|" + "listtakewhile|listtakewhileinclusive|listtotuple|listtype|listtypehandle|listunionall|" + "listuniq|listzip|listzipall|loghistogram|logarithmichistogram|max|max_by|max_of|median|" + "min|min_by|min_of|mode|multi_aggregate_by|nanvl|nvl|nothing|nulltype|nulltypehandle|" + "optionalitemtype|optionaltype|optionaltypehandle|percentile|parsefile|parsetype|" + "parsetypehandle|pgand|pgarray|pgcall|pgconst|pgnot|pgop|pgor|pickle|quotecode|range|" + "range_strict|rank|regexp|regexp_strict|rfind|row_number|random|randomnumber|randomuuid|" + "removemember|removemembers|removetimezone|renamemembers|replacemember|reprcode|resourcetype|" + "resourcetypehandle|resourcetypetag|some|stddev|stddev_population|stddev_sample|substring|" + "sum|sum_if|sessionstart|sessionwindow|setcreate|setdifference|setincludes|setintersection|" + "setisdisjoint|setsymmetricdifference|setunion|spreadmembers|stablepickle|startswith|staticmap|" + "staticzip|streamitemtype|streamtype|streamtypehandle|structmembertype|structmembers|" + "structtypecomponents|structtypehandle|subqueryextend|subqueryextendfor|subquerymerge|" + "subquerymergefor|subqueryunionall|subqueryunionallfor|subqueryunionmerge|subqueryunionmergefor" + "|top|topfreq|top_by|tablename|tablepath|tablerecordindex|tablerow|tablerows|taggedtype|" + "taggedtypecomponents|taggedtypehandle|tobytes|todict|tomultidict|topg|toset|tosorteddict|" + "tosortedmultidict|trymember|tupleelementtype|tupletype|tupletypecomponents|tupletypehandle|" + "typehandle|typekind|typeof|udaf|udf|unittype|unpickle|untag|unwrap|variance|variance_population|" + "variance_sample|variant|varianttype|varianttypehandle|variantunderlyingtype|voidtype|" + "voidtypehandle|way|worldcode|weakfield" + ")$"); + + constexpr const char* typePattern = ( // + "^(" + "bool|date|datetime|decimal|double|float|int16|int32|int64|int8|interval|json|jsondocument|" + "string|timestamp|tzdate|tzdatetime|tztimestamp|uint16|uint32|uint64|uint8|utf8|uuid|yson|" + "text|bytes" + ")$"); + + bool IsAnsiQuery(const TString& queryUtf8) { + TTranslationSettings settings; + TIssues issues; + ParseTranslationSettings(queryUtf8, settings, issues); + return settings.AnsiLexer; + } + + TColorSchema TColorSchema::Monaco() { + using replxx::color::rgb666; + + return { + .keyword = TColor::BLUE, + .operation = rgb666(3, 3, 3), + .identifier = { + .function = rgb666(4, 1, 5), + .type = rgb666(2, 3, 2), + .variable = TColor::DEFAULT, + .quoted = rgb666(1, 3, 3), + }, + .string = rgb666(3, 0, 0), + .number = TColor::BRIGHTGREEN, + .comment = rgb666(2, 2, 2), + .unknown = TColor::DEFAULT, + }; + } + + TColorSchema TColorSchema::Debug() { + using replxx::color::rgb666; + + return { + .keyword = TColor::BLUE, + .operation = TColor::GRAY, + .identifier = { + .function = TColor::MAGENTA, + .type = TColor::YELLOW, + .variable = TColor::RED, + .quoted = TColor::CYAN, + }, + .string = TColor::GREEN, + .number = TColor::BRIGHTGREEN, + .comment = rgb666(2, 2, 2), + .unknown = TColor::DEFAULT, + }; + } + + class TYQLHighlighter: public IYQLHighlighter { + public: + explicit TYQLHighlighter(TColorSchema color) + : Coloring(color) + , BuiltinFunctionRegex(builtinFunctionPattern, ECMAScript | icase) + , TypeRegex(typePattern, ECMAScript | icase) + , CppLexer(MakeLexer(MakeLexers(), /* ansi = */ false, /* antlr4 = */ true)) + , ANSILexer(MakeLexer(MakeLexers(), /* ansi = */ true, /* antlr4 = */ true)) + { + } + + public: + void Apply(TStringBuf queryUtf8, TColors& colors) { + Tokens = Tokenize(TString(queryUtf8)); + auto mapping = TYQLPositionMapping::Build(TString(queryUtf8)); + + for (std::size_t i = 0; i < Tokens.size() - 1; ++i) { + const auto& token = Tokens.at(i); + const auto color = ColorOf(token, i); + + const std::ptrdiff_t start = mapping.RawPos(token); + const std::ptrdiff_t length = GetNumberOfUTF8Chars(token.Content); + const std::ptrdiff_t stop = start + length; + + std::fill(std::next(std::begin(colors), start), + std::next(std::begin(colors), stop), color); + } + } + + private: + TParsedTokenList Tokenize(const TString& queryUtf8) { + ILexer& lexer = *SuitableLexer(queryUtf8); + + TParsedTokenList tokens; + TIssues issues; + NSQLTranslation::Tokenize(lexer, queryUtf8, "Query", tokens, issues, SQL_MAX_PARSER_ERRORS); + return tokens; + } + + ILexer::TPtr& SuitableLexer(const TString& queryUtf8) { + if (IsAnsiQuery(queryUtf8)) { + return ANSILexer; + } + return CppLexer; + } + + TColor ColorOf(const TParsedToken& token, size_t index) { + if (IsString(token)) { + return Coloring.string; + } + if (IsFunctionIdentifier(token, index)) { + return Coloring.identifier.function; + } + if (IsTypeIdentifier(token)) { + return Coloring.identifier.type; + } + if (IsVariableIdentifier(token)) { + return Coloring.identifier.variable; + } + if (IsQuotedIdentifier(token)) { + return Coloring.identifier.quoted; + } + if (IsNumber(token)) { + return Coloring.number; + } + if (IsOperation(token)) { + return Coloring.operation; + } + if (IsKeyword(token)) { + return Coloring.keyword; + } + if (IsComment(token)) { + return Coloring.comment; + } + return Coloring.unknown; + } + + bool IsKeyword(const TParsedToken& token) const { + return IsProbablyKeyword(token); + } + + bool IsOperation(const TParsedToken& token) const { + return ( + token.Name == "EQUALS" || + token.Name == "EQUALS2" || + token.Name == "NOT_EQUALS" || + token.Name == "NOT_EQUALS2" || + token.Name == "LESS" || + token.Name == "LESS_OR_EQ" || + token.Name == "GREATER" || + token.Name == "GREATER_OR_EQ" || + token.Name == "SHIFT_LEFT" || + token.Name == "ROT_LEFT" || + token.Name == "AMPERSAND" || + token.Name == "PIPE" || + token.Name == "DOUBLE_PIPE" || + token.Name == "STRUCT_OPEN" || + token.Name == "STRUCT_CLOSE" || + token.Name == "PLUS" || + token.Name == "MINUS" || + token.Name == "TILDA" || + token.Name == "ASTERISK" || + token.Name == "SLASH" || + token.Name == "PERCENT" || + token.Name == "SEMICOLON" || + token.Name == "DOT" || + token.Name == "COMMA" || + token.Name == "LPAREN" || + token.Name == "RPAREN" || + token.Name == "QUESTION" || + token.Name == "COLON" || + token.Name == "COMMAT" || + token.Name == "DOUBLE_COMMAT" || + token.Name == "DOLLAR" || + token.Name == "LBRACE_CURLY" || + token.Name == "RBRACE_CURLY" || + token.Name == "CARET" || + token.Name == "NAMESPACE" || + token.Name == "ARROW" || + token.Name == "RBRACE_SQUARE" || + token.Name == "LBRACE_SQUARE"); + } + + bool IsFunctionIdentifier(const TParsedToken& token, size_t index) { + if (token.Name != "ID_PLAIN") { + return false; + } + return std::regex_search(token.Content.begin(), token.Content.end(), BuiltinFunctionRegex) || + (2 <= index && Tokens.at(index - 1).Name == "NAMESPACE" && Tokens.at(index - 2).Name == "ID_PLAIN") || + (index < Tokens.size() - 1 && Tokens.at(index + 1).Name == "NAMESPACE"); + } + + bool IsTypeIdentifier(const TParsedToken& token) const { + return token.Name == "ID_PLAIN" && std::regex_search(token.Content.begin(), token.Content.end(), TypeRegex); + } + + bool IsVariableIdentifier(const TParsedToken& token) const { + return token.Name == "ID_PLAIN"; + } + + bool IsQuotedIdentifier(const TParsedToken& token) const { + return token.Name == "ID_QUOTED"; + } + + bool IsString(const TParsedToken& token) const { + return token.Name == "STRING_VALUE"; + } + + bool IsNumber(const TParsedToken& token) const { + return token.Name == "DIGITS" || + token.Name == "INTEGER_VALUE" || + token.Name == "REAL" || + token.Name == "BLOB"; + } + + bool IsComment(const TParsedToken& token) const { + return token.Name == "COMMENT"; + } + + private: + TColorSchema Coloring; + std::regex BuiltinFunctionRegex; + std::regex TypeRegex; + + ILexer::TPtr CppLexer; + ILexer::TPtr ANSILexer; + + TParsedTokenList Tokens; + }; + + IYQLHighlighter::TPtr MakeYQLHighlighter(TColorSchema color) { + return IYQLHighlighter::TPtr(new TYQLHighlighter(std::move(color))); + } + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h new file mode 100644 index 000000000000..700bc5b18bf2 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include + +namespace NYdb::NConsoleClient { + + using TColor = replxx::Replxx::Color; + + // Colors are provided as for a UTF32 string + using TColors = replxx::Replxx::colors_t; + + struct TColorSchema { + TColor keyword; + TColor operation; + struct { + TColor function; + TColor type; + TColor variable; + TColor quoted; + } identifier; + TColor string; + TColor number; + TColor comment; + TColor unknown; + + static TColorSchema Monaco(); + static TColorSchema Debug(); + }; + + class IYQLHighlighter { + public: + using TPtr = THolder; + + virtual void Apply(TStringBuf queryUtf8, TColors& colors) = 0; + virtual ~IYQLHighlighter() = default; + }; + + IYQLHighlighter::TPtr MakeYQLHighlighter(TColorSchema color); + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight_ut.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp similarity index 84% rename from ydb/public/lib/ydb_cli/commands/interactive/yql_highlight_ut.cpp rename to ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp index 037556754d58..1afe38f56542 100644 --- a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight_ut.cpp +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp @@ -1,4 +1,4 @@ -#include "yql_highlight.h" +#include "yql_highlighter.h" #include @@ -9,9 +9,9 @@ using namespace NYdb::NConsoleClient; Y_UNIT_TEST_SUITE(YqlHighlightTests) { - auto Coloring = YQLHighlight::ColorSchema::Debug(); + auto Coloring = TColorSchema::Debug(); - std::unordered_map colors = { + std::unordered_map colors = { {'k', Coloring.keyword}, {'o', Coloring.operation}, {'f', Coloring.identifier.function}, @@ -22,35 +22,35 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { {'n', Coloring.number}, {'u', Coloring.unknown}, {'c', Coloring.comment}, - {' ', YQLHighlight::Color::DEFAULT}, + {' ', TColor::DEFAULT}, }; - std::unordered_map symbols = [] { - std::unordered_map symbols; + std::unordered_map symbols = [] { + std::unordered_map symbols; for (const auto& [symbol, color] : colors) { symbols[color] = symbol; } return symbols; }(); - TVector ColorsFromPattern(const TStringBuf& symbols) { - TVector result(symbols.size()); + TVector ColorsFromPattern(const TStringBuf& symbols) { + TVector result(symbols.size()); for (std::size_t i = 0; i < symbols.size(); ++i) { result[i] = colors.at(symbols[i]); } return result; } - TVector Apply(YQLHighlight& highlight, - const TStringBuf& queryUtf8) { + TVector Apply(IYQLHighlighter& highlight, + const TStringBuf& queryUtf8) { const auto queryUtf32 = UTF8ToUTF32(queryUtf8); - TVector colors(queryUtf32.size(), - YQLHighlight::Color::DEFAULT); + TVector colors(queryUtf32.size(), + TColor::DEFAULT); highlight.Apply(queryUtf8, colors); return colors; } - TString SymbolsFromColors(const TVector& colors) { + TString SymbolsFromColors(const TVector& colors) { TString result(colors.size(), '-'); for (std::size_t i = 0; i < colors.size(); ++i) { result[i] = symbols.at(colors[i]); @@ -58,15 +58,15 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { return result; } - void Check(YQLHighlight& highlight, const TStringBuf& query, + void Check(IYQLHighlighter::TPtr& highlight, const TStringBuf& query, const TStringBuf& pattern) { - auto actual = Apply(highlight, query); - UNIT_ASSERT_EQUAL_C(Apply(highlight, query), ColorsFromPattern(pattern), + auto actual = Apply(*highlight, query); + UNIT_ASSERT_EQUAL_C(Apply(*highlight, query), ColorsFromPattern(pattern), SymbolsFromColors(actual)); } Y_UNIT_TEST(Blank) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "", ""); Check(highlight, " ", " "); Check(highlight, " ", " "); @@ -79,7 +79,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Invalid) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "!", "u"); Check(highlight, "й", "u"); Check(highlight, "编", "u"); @@ -89,7 +89,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Keyword) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "SELECT", "kkkkkk"); Check(highlight, "select", "kkkkkk"); Check(highlight, "ALTER", "kkkkk"); @@ -98,13 +98,13 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Operation) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "(1 + 21 / 4)", "on o nn o no"); Check(highlight, "(1+21/4)", "ononnono"); } Y_UNIT_TEST(FunctionIdentifier) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "MIN", "fff"); Check(highlight, "min", "fff"); @@ -121,18 +121,18 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(TypeIdentifier) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "Bool", "tttt"); Check(highlight, "Bool(value)", "ttttovvvvvo"); } Y_UNIT_TEST(VariableIdentifier) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "test", "vvvv"); } Y_UNIT_TEST(QuotedIdentifier) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "`/cluster/database`", "qqqqqqqqqqqqqqqqqqq"); Check(highlight, "`test`select", "qqqqqqkkkkkk"); Check(highlight, "`/cluster", "uuuuuuuuu"); @@ -140,7 +140,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(String) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "\"\"", "ss"); Check(highlight, "\"test\"", "ssssss"); Check(highlight, "\"", "u"); @@ -154,7 +154,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(MultilineString) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "@@", "oo"); Check(highlight, "@@@", "ooo"); @@ -181,7 +181,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(TypedString) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check( highlight, "SELECT \"foo\"u, '[1;2]'y, @@{\"a\":null}@@j;", @@ -189,7 +189,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Number) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "1234", "nnnn"); Check(highlight, "-123", "onnn"); @@ -213,7 +213,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(SQL) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "SELECT id, alias from users", "kkkkkk vvo vvvvv kkkk vvvvv"); Check(highlight, "INSERT INTO users (id, alias) VALUES (12, \"tester\")", @@ -229,7 +229,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Emoji) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "☺", "u"); Check(highlight, "\"☺\"", "sss"); Check(highlight, "`☺`", "qqq"); @@ -254,11 +254,11 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { " ttttovvvvvoo ffffoofffovvvo " "kkkk qqqqqqqqqqqqqqqqqqqqqqqq kkkk vvvvo"; - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); for (std::size_t size = 0; size <= query.size(); ++size) { const TStringBuf prefix(query, 0, size); - auto colors = Apply(highlight, prefix); + auto colors = Apply(*highlight, prefix); Y_DO_NOT_OPTIMIZE_AWAY(colors); if (size == query.size() || IsSpace(pattern[size])) { @@ -269,7 +269,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Comment) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check(highlight, "- select", "o kkkkkk"); Check(highlight, "select -- select", "kkkkkk ccccccccc"); Check(highlight, "-- select\nselect", "cccccccccckkkkkk"); @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(Multiline) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check( highlight, "SELECT *\n" @@ -306,7 +306,7 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) { } Y_UNIT_TEST(ANSI) { - YQLHighlight highlight(Coloring); + auto highlight = MakeYQLHighlighter(Coloring); Check( highlight, diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp new file mode 100644 index 000000000000..15fada3f46cf --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp @@ -0,0 +1,40 @@ +#include "yql_position.h" + +#include + +#include +#include + +namespace NYdb::NConsoleClient { + + ui32 TYQLPositionMapping::RawPos(const TParsedToken& token) const { + return SymbolsCountBeforeLine.at(token.Line) + token.LinePos; + } + + TYQLPositionMapping TYQLPositionMapping::Build(const TString& queryUtf8) { + TVector symbolsCountBeforeLine = {0, 0}; + + TStringStream stream(queryUtf8); + TLineSplitter lines(stream); + + size_t read; + for (TString line; (read = lines.Next(line)) != 0;) { + const auto index = symbolsCountBeforeLine.size(); + const auto previous = symbolsCountBeforeLine.at(index - 1); + + const auto newlineWidth = read - line.size(); + Y_ASSERT(0 <= newlineWidth && newlineWidth <= 2); + + const auto current = GetNumberOfUTF8Chars(line) + newlineWidth; + symbolsCountBeforeLine.emplace_back(previous + current); + } + + return TYQLPositionMapping(std::move(symbolsCountBeforeLine)); + } + + TYQLPositionMapping::TYQLPositionMapping(TVector SymbolsCountBeforeLine) + : SymbolsCountBeforeLine(std::move(SymbolsCountBeforeLine)) + { + } + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h new file mode 100644 index 000000000000..4a9dddec82c6 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include +#include +#include + +namespace NYdb::NConsoleClient { + + using NSQLTranslation::TParsedToken; + + class TYQLPositionMapping final { + public: + // Translates (Line, LinePos) position into RawPos that + // is an absolute symbol position in utf8 symbols array + ui32 RawPos(const TParsedToken& token) const; + + public: + static TYQLPositionMapping Build(const TString& queryUtf8); + + private: + explicit TYQLPositionMapping(TVector SymbolsCountBeforeLine); + + private: + TVector SymbolsCountBeforeLine; + }; + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/interactive/line_reader.cpp b/ydb/public/lib/ydb_cli/commands/interactive/line_reader.cpp index 20044e16c9a7..722ab03f7d66 100644 --- a/ydb/public/lib/ydb_cli/commands/interactive/line_reader.cpp +++ b/ydb/public/lib/ydb_cli/commands/interactive/line_reader.cpp @@ -1,6 +1,7 @@ #include "line_reader.h" -#include "yql_highlight.h" +#include +#include #include #include @@ -37,10 +38,6 @@ std::optional LockFile(TFileHandle& fileHandle) { return FileHandlerLockGuard(&fileHandle); } -replxx::Replxx::Color ReplxxColorOf(NSQLComplete::ECandidateKind /* kind */) { - return replxx::Replxx::Color::DEFAULT; -} - class TLineReader: public ILineReader { public: TLineReader(std::string prompt, std::string historyFilePath); @@ -53,7 +50,8 @@ class TLineReader: public ILineReader { std::string Prompt; std::string HistoryFilePath; TFileHandle HistoryFileHandle; - NSQLComplete::ISqlCompletionEngine::TPtr CompletionEngine; + IYQLCompleter::TPtr YQLCompleter; + IYQLHighlighter::TPtr YQLHighlighter; replxx::Replxx Rx; }; @@ -61,32 +59,17 @@ TLineReader::TLineReader(std::string prompt, std::string historyFilePath) : Prompt(std::move(prompt)) , HistoryFilePath(std::move(historyFilePath)) , HistoryFileHandle(HistoryFilePath.c_str(), EOpenModeFlag::OpenAlways | EOpenModeFlag::RdWr | EOpenModeFlag::AW | EOpenModeFlag::ARUser | EOpenModeFlag::ARGroup) - , CompletionEngine(NSQLComplete::MakeSqlCompletionEngine()) + , YQLCompleter(MakeYQLCompleter()) + , YQLHighlighter(MakeYQLHighlighter(TColorSchema::Monaco())) { Rx.install_window_change_handler(); - auto completion_callback = [this](const std::string& prefix, size_t contextLen) { - auto completion = CompletionEngine->Complete({ - .Text = prefix, - .CursorPosition = prefix.length(), - }); - - contextLen = completion.CompletedToken.SourcePosition; - - replxx::Replxx::completions_t entries; - for (auto& candidate : completion.Candidates) { - candidate.Content += ' '; - entries.emplace_back(std::move(candidate.Content), ReplxxColorOf(candidate.Kind)); - } - return entries; - }; - - auto highlighter_callback = [](const auto& text, auto& colors) { - return YQLHighlight(YQLHighlight::ColorSchema::Monaco()).Apply(text, colors); - }; - - Rx.set_completion_callback(completion_callback); - Rx.set_highlighter_callback(highlighter_callback); + Rx.set_completion_callback([this](const std::string& prefix, int& contextLen) { + return YQLCompleter->Apply(prefix, contextLen); + }); + Rx.set_highlighter_callback([this](const auto& text, auto& colors) { + YQLHighlighter->Apply(text, colors); + }); Rx.enable_bracketed_paste(); Rx.set_unique_history(true); Rx.set_complete_on_empty(true); diff --git a/ydb/public/lib/ydb_cli/commands/interactive/ut/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/ut/ya.make deleted file mode 100644 index 99b0b00d3682..000000000000 --- a/ydb/public/lib/ydb_cli/commands/interactive/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(ydb/public/lib/ydb_cli/commands/interactive) - -SRCS( - yql_highlight_ut.cpp -) - -END() \ No newline at end of file diff --git a/ydb/public/lib/ydb_cli/commands/interactive/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/ya.make index bf30593f0b0f..81b513ae1de3 100644 --- a/ydb/public/lib/ydb_cli/commands/interactive/ya.make +++ b/ydb/public/lib/ydb_cli/commands/interactive/ya.make @@ -3,26 +3,14 @@ LIBRARY() SRCS( interactive_cli.cpp line_reader.cpp - yql_highlight.cpp - yql_position.cpp ) PEERDIR( contrib/restricted/patched/replxx - contrib/libs/antlr4_cpp_runtime - yql/essentials/parser/lexer_common - yql/essentials/parser/proto_ast/gen/v1_antlr4 - yql/essentials/sql/v1/complete - yql/essentials/sql/v1/lexer - yql/essentials/sql/v1/lexer/antlr4 - yql/essentials/sql/v1/lexer/antlr4_ansi - yql/essentials/sql/settings - yql/essentials/utils ydb/public/lib/ydb_cli/common + ydb/public/lib/ydb_cli/commands/interactive/highlight + ydb/public/lib/ydb_cli/commands/interactive/complete ) END() -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp b/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp deleted file mode 100644 index eea4691611a7..000000000000 --- a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp +++ /dev/null @@ -1,296 +0,0 @@ -#include "yql_highlight.h" - -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#define TOKEN(NAME) SQLv1Antlr4Lexer::TOKEN_##NAME - -namespace NYdb { - namespace NConsoleClient { - using NSQLTranslation::ParseTranslationSettings; - using NSQLTranslation::SQL_MAX_PARSER_ERRORS; - using NSQLTranslation::TTranslationSettings; - using NSQLTranslationV1::IsProbablyKeyword; - using NSQLTranslationV1::MakeLexer; - using NYql::TIssues; - - using std::regex_constants::ECMAScript; - using std::regex_constants::icase; - - NSQLTranslationV1::TLexers MakeLexers() { - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - return lexers; - } - - constexpr const char* builtinFunctionPattern = ( // - "^(" - "abs|aggregate_by|aggregate_list|aggregate_list_distinct|agg_list|agg_list_distinct|" - "as_table|avg|avg_if|adaptivedistancehistogram|adaptivewardhistogram|adaptiveweighthistogram|" - "addmember|addtimezone|aggregateflatten|aggregatetransforminput|aggregatetransformoutput|" - "aggregationfactory|asatom|asdict|asdictstrict|asenum|aslist|asliststrict|asset|assetstrict|" - "asstruct|astagged|astuple|asvariant|atomcode|bitcast|bit_and|bit_or|bit_xor|bool_and|" - "bool_or|bool_xor|bottom|bottom_by|blockwardhistogram|blockweighthistogram|cast|coalesce|" - "concat|concat_strict|correlation|count|count_if|covariance|covariance_population|" - "covariance_sample|callableargument|callableargumenttype|callableresulttype|callabletype|" - "callabletypecomponents|callabletypehandle|choosemembers|combinemembers|countdistinctestimate|" - "currentauthenticateduser|currentoperationid|currentoperationsharedid|currenttzdate|" - "currenttzdatetime|currenttztimestamp|currentutcdate|currentutcdatetime|currentutctimestamp|" - "dense_rank|datatype|datatypecomponents|datatypehandle|dictaggregate|dictcontains|dictcreate|" - "dicthasitems|dictitems|dictkeytype|dictkeys|dictlength|dictlookup|dictpayloadtype|dictpayloads|" - "dicttype|dicttypecomponents|dicttypehandle|each|each_strict|emptydicttype|emptydicttypehandle|" - "emptylisttype|emptylisttypehandle|endswith|ensure|ensureconvertibleto|ensuretype|enum|" - "evaluateatom|evaluatecode|evaluateexpr|evaluatetype|expandstruct|filter|filter_strict|find|" - "first_value|folder|filecontent|filepath|flattenmembers|forceremovemember|forceremovemembers|" - "forcerenamemembers|forcespreadmembers|formatcode|formattype|frombytes|frompg|funccode|" - "greatest|grouping|gathermembers|generictype|histogram|hll|hoppingwindowpgcast|hyperloglog|" - "if|if_strict|instanceof|json_exists|json_query|json_value|jointablerow|just|lag|last_value|" - "lead|least|len|length|like|likely|like_strict|lambdaargumentscount|lambdacode|" - "lambdaoptionalargumentscount|linearhistogram|listaggregate|listall|listany|listavg|listcode|" - "listcollect|listconcat|listcreate|listdistinct|listenumerate|listextend|listextendstrict|" - "listextract|listfilter|listflatmap|listflatten|listfold|listfold1|listfold1map|listfoldmap|" - "listfromrange|listfromtuple|listhas|listhasitems|listhead|listindexof|listitemtype|listlast|" - "listlength|listmap|listmax|listmin|listnotnull|listreplicate|listreverse|listskip|" - "listskipwhile|listskipwhileinclusive|listsort|listsortasc|listsortdesc|listsum|listtake|" - "listtakewhile|listtakewhileinclusive|listtotuple|listtype|listtypehandle|listunionall|" - "listuniq|listzip|listzipall|loghistogram|logarithmichistogram|max|max_by|max_of|median|" - "min|min_by|min_of|mode|multi_aggregate_by|nanvl|nvl|nothing|nulltype|nulltypehandle|" - "optionalitemtype|optionaltype|optionaltypehandle|percentile|parsefile|parsetype|" - "parsetypehandle|pgand|pgarray|pgcall|pgconst|pgnot|pgop|pgor|pickle|quotecode|range|" - "range_strict|rank|regexp|regexp_strict|rfind|row_number|random|randomnumber|randomuuid|" - "removemember|removemembers|removetimezone|renamemembers|replacemember|reprcode|resourcetype|" - "resourcetypehandle|resourcetypetag|some|stddev|stddev_population|stddev_sample|substring|" - "sum|sum_if|sessionstart|sessionwindow|setcreate|setdifference|setincludes|setintersection|" - "setisdisjoint|setsymmetricdifference|setunion|spreadmembers|stablepickle|startswith|staticmap|" - "staticzip|streamitemtype|streamtype|streamtypehandle|structmembertype|structmembers|" - "structtypecomponents|structtypehandle|subqueryextend|subqueryextendfor|subquerymerge|" - "subquerymergefor|subqueryunionall|subqueryunionallfor|subqueryunionmerge|subqueryunionmergefor" - "|top|topfreq|top_by|tablename|tablepath|tablerecordindex|tablerow|tablerows|taggedtype|" - "taggedtypecomponents|taggedtypehandle|tobytes|todict|tomultidict|topg|toset|tosorteddict|" - "tosortedmultidict|trymember|tupleelementtype|tupletype|tupletypecomponents|tupletypehandle|" - "typehandle|typekind|typeof|udaf|udf|unittype|unpickle|untag|unwrap|variance|variance_population|" - "variance_sample|variant|varianttype|varianttypehandle|variantunderlyingtype|voidtype|" - "voidtypehandle|way|worldcode|weakfield" - ")$"); - - constexpr const char* typePattern = ( // - "^(" - "bool|date|datetime|decimal|double|float|int16|int32|int64|int8|interval|json|jsondocument|" - "string|timestamp|tzdate|tzdatetime|tztimestamp|uint16|uint32|uint64|uint8|utf8|uuid|yson|" - "text|bytes" - ")$"); - - bool IsAnsiQuery(const TString& queryUtf8) { - TTranslationSettings settings; - TIssues issues; - ParseTranslationSettings(queryUtf8, settings, issues); - return settings.AnsiLexer; - } - - YQLHighlight::ColorSchema YQLHighlight::ColorSchema::Monaco() { - using replxx::color::rgb666; - - return { - .keyword = Color::BLUE, - .operation = rgb666(3, 3, 3), - .identifier = { - .function = rgb666(4, 1, 5), - .type = rgb666(2, 3, 2), - .variable = Color::DEFAULT, - .quoted = rgb666(1, 3, 3), - }, - .string = rgb666(3, 0, 0), - .number = Color::BRIGHTGREEN, - .comment = rgb666(2, 2, 2), - .unknown = Color::DEFAULT, - }; - } - - YQLHighlight::ColorSchema YQLHighlight::ColorSchema::Debug() { - using replxx::color::rgb666; - - return { - .keyword = Color::BLUE, - .operation = Color::GRAY, - .identifier = { - .function = Color::MAGENTA, - .type = Color::YELLOW, - .variable = Color::RED, - .quoted = Color::CYAN, - }, - .string = Color::GREEN, - .number = Color::BRIGHTGREEN, - .comment = rgb666(2, 2, 2), - .unknown = Color::DEFAULT, - }; - } - - YQLHighlight::YQLHighlight(ColorSchema color) - : Coloring(color) - , BuiltinFunctionRegex(builtinFunctionPattern, ECMAScript | icase) - , TypeRegex(typePattern, ECMAScript | icase) - , CppLexer(MakeLexer(MakeLexers(), /* ansi = */ false, /* antlr4 = */ true)) - , ANSILexer(MakeLexer(MakeLexers(), /* ansi = */ true, /* antlr4 = */ true)) - { - } - - void YQLHighlight::Apply(TStringBuf queryUtf8, Colors& colors) { - Tokens = Tokenize(TString(queryUtf8)); - auto mapping = YQLPositionMapping::Build(TString(queryUtf8)); - - for (std::size_t i = 0; i < Tokens.size() - 1; ++i) { - const auto& token = Tokens.at(i); - const auto color = ColorOf(token, i); - - const std::ptrdiff_t start = mapping.RawPos(token); - const std::ptrdiff_t length = GetNumberOfUTF8Chars(token.Content); - const std::ptrdiff_t stop = start + length; - - std::fill(std::next(std::begin(colors), start), - std::next(std::begin(colors), stop), color); - } - } - - TParsedTokenList YQLHighlight::Tokenize(const TString& queryUtf8) { - ILexer& lexer = *SuitableLexer(queryUtf8); - - TParsedTokenList tokens; - TIssues issues; - NSQLTranslation::Tokenize(lexer, queryUtf8, "Query", tokens, issues, SQL_MAX_PARSER_ERRORS); - return tokens; - } - - ILexer::TPtr& YQLHighlight::SuitableLexer(const TString& queryUtf8) { - if (IsAnsiQuery(queryUtf8)) { - return ANSILexer; - } - return CppLexer; - } - - YQLHighlight::Color YQLHighlight::ColorOf(const TParsedToken& token, size_t index) { - if (IsString(token)) { - return Coloring.string; - } - if (IsFunctionIdentifier(token, index)) { - return Coloring.identifier.function; - } - if (IsTypeIdentifier(token)) { - return Coloring.identifier.type; - } - if (IsVariableIdentifier(token)) { - return Coloring.identifier.variable; - } - if (IsQuotedIdentifier(token)) { - return Coloring.identifier.quoted; - } - if (IsNumber(token)) { - return Coloring.number; - } - if (IsOperation(token)) { - return Coloring.operation; - } - if (IsKeyword(token)) { - return Coloring.keyword; - } - if (IsComment(token)) { - return Coloring.comment; - } - return Coloring.unknown; - } - - bool YQLHighlight::IsKeyword(const TParsedToken& token) const { - return IsProbablyKeyword(token); - } - - bool YQLHighlight::IsOperation(const TParsedToken& token) const { - return ( - token.Name == "EQUALS" || - token.Name == "EQUALS2" || - token.Name == "NOT_EQUALS" || - token.Name == "NOT_EQUALS2" || - token.Name == "LESS" || - token.Name == "LESS_OR_EQ" || - token.Name == "GREATER" || - token.Name == "GREATER_OR_EQ" || - token.Name == "SHIFT_LEFT" || - token.Name == "ROT_LEFT" || - token.Name == "AMPERSAND" || - token.Name == "PIPE" || - token.Name == "DOUBLE_PIPE" || - token.Name == "STRUCT_OPEN" || - token.Name == "STRUCT_CLOSE" || - token.Name == "PLUS" || - token.Name == "MINUS" || - token.Name == "TILDA" || - token.Name == "ASTERISK" || - token.Name == "SLASH" || - token.Name == "PERCENT" || - token.Name == "SEMICOLON" || - token.Name == "DOT" || - token.Name == "COMMA" || - token.Name == "LPAREN" || - token.Name == "RPAREN" || - token.Name == "QUESTION" || - token.Name == "COLON" || - token.Name == "COMMAT" || - token.Name == "DOUBLE_COMMAT" || - token.Name == "DOLLAR" || - token.Name == "LBRACE_CURLY" || - token.Name == "RBRACE_CURLY" || - token.Name == "CARET" || - token.Name == "NAMESPACE" || - token.Name == "ARROW" || - token.Name == "RBRACE_SQUARE" || - token.Name == "LBRACE_SQUARE"); - } - - bool YQLHighlight::IsFunctionIdentifier(const TParsedToken& token, size_t index) { - if (token.Name != "ID_PLAIN") { - return false; - } - return std::regex_search(token.Content.begin(), token.Content.end(), BuiltinFunctionRegex) || - (2 <= index && Tokens.at(index - 1).Name == "NAMESPACE" && Tokens.at(index - 2).Name == "ID_PLAIN") || - (index < Tokens.size() - 1 && Tokens.at(index + 1).Name == "NAMESPACE"); - } - - bool YQLHighlight::IsTypeIdentifier(const TParsedToken& token) const { - return token.Name == "ID_PLAIN" && std::regex_search(token.Content.begin(), token.Content.end(), TypeRegex); - } - - bool YQLHighlight::IsVariableIdentifier(const TParsedToken& token) const { - return token.Name == "ID_PLAIN"; - } - - bool YQLHighlight::IsQuotedIdentifier(const TParsedToken& token) const { - return token.Name == "ID_QUOTED"; - } - - bool YQLHighlight::IsString(const TParsedToken& token) const { - return token.Name == "STRING_VALUE"; - } - - bool YQLHighlight::IsNumber(const TParsedToken& token) const { - return token.Name == "DIGITS" || - token.Name == "INTEGER_VALUE" || - token.Name == "REAL" || - token.Name == "BLOB"; - } - - bool YQLHighlight::IsComment(const TParsedToken& token) const { - return token.Name == "COMMENT"; - } - - } // namespace NConsoleClient -} // namespace NYdb diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.h b/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.h deleted file mode 100644 index 14fa76a5bb76..000000000000 --- a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.h +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - -namespace NYdb { - namespace NConsoleClient { - using NSQLTranslation::ILexer; - using NSQLTranslation::TParsedToken; - using NSQLTranslation::TParsedTokenList; - - class YQLHighlight final { - public: - using Color = replxx::Replxx::Color; - - // Colors are provided as for a UTF32 string - using Colors = replxx::Replxx::colors_t; - - struct ColorSchema { - Color keyword; - Color operation; - struct { - Color function; - Color type; - Color variable; - Color quoted; - } identifier; - Color string; - Color number; - Color comment; - Color unknown; - - static ColorSchema Monaco(); - static ColorSchema Debug(); - }; - - public: - explicit YQLHighlight(ColorSchema color); - - public: - void Apply(TStringBuf queryUtf8, Colors& colors); - - private: - TParsedTokenList Tokenize(const TString& queryUtf8); - ILexer::TPtr& SuitableLexer(const TString& queryUtf8); - YQLHighlight::Color ColorOf(const TParsedToken& token, size_t index); - bool IsKeyword(const TParsedToken& token) const; - bool IsOperation(const TParsedToken& token) const; - bool IsFunctionIdentifier(const TParsedToken& token, size_t index); - bool IsTypeIdentifier(const TParsedToken& token) const; - bool IsVariableIdentifier(const TParsedToken& token) const; - bool IsQuotedIdentifier(const TParsedToken& token) const; - bool IsString(const TParsedToken& token) const; - bool IsNumber(const TParsedToken& token) const; - bool IsComment(const TParsedToken& token) const; - - private: - ColorSchema Coloring; - std::regex BuiltinFunctionRegex; - std::regex TypeRegex; - - ILexer::TPtr CppLexer; - ILexer::TPtr ANSILexer; - - TParsedTokenList Tokens; - }; - - } // namespace NConsoleClient -} // namespace NYdb diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_position.cpp b/ydb/public/lib/ydb_cli/commands/interactive/yql_position.cpp deleted file mode 100644 index 79f9bd674a2f..000000000000 --- a/ydb/public/lib/ydb_cli/commands/interactive/yql_position.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "yql_position.h" - -#include - -#include -#include - -namespace NYdb { - namespace NConsoleClient { - - ui32 YQLPositionMapping::RawPos(const TParsedToken& token) const { - return SymbolsCountBeforeLine.at(token.Line) + token.LinePos; - } - - YQLPositionMapping YQLPositionMapping::Build(const TString& queryUtf8) { - TVector symbolsCountBeforeLine = {0, 0}; - - TStringStream stream(queryUtf8); - TLineSplitter lines(stream); - - size_t read; - for (TString line; (read = lines.Next(line)) != 0;) { - const auto index = symbolsCountBeforeLine.size(); - const auto previous = symbolsCountBeforeLine.at(index - 1); - - const auto newlineWidth = read - line.size(); - Y_ASSERT(0 <= newlineWidth && newlineWidth <= 2); - - const auto current = GetNumberOfUTF8Chars(line) + newlineWidth; - symbolsCountBeforeLine.emplace_back(previous + current); - } - - return YQLPositionMapping(std::move(symbolsCountBeforeLine)); - } - - YQLPositionMapping::YQLPositionMapping(TVector SymbolsCountBeforeLine) - : SymbolsCountBeforeLine(std::move(SymbolsCountBeforeLine)) - { - } - - } // namespace NConsoleClient -} // namespace NYdb diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_position.h b/ydb/public/lib/ydb_cli/commands/interactive/yql_position.h deleted file mode 100644 index 5f6781b9c190..000000000000 --- a/ydb/public/lib/ydb_cli/commands/interactive/yql_position.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -namespace NYdb { - namespace NConsoleClient { - - using NSQLTranslation::TParsedToken; - - class YQLPositionMapping final { - public: - // Translates (Line, LinePos) position into RawPos that - // is an absolute symbol position in utf8 symbols array - ui32 RawPos(const TParsedToken& token) const; - - public: - static YQLPositionMapping Build(const TString& queryUtf8); - - private: - explicit YQLPositionMapping(TVector SymbolsCountBeforeLine); - - private: - TVector SymbolsCountBeforeLine; - }; - - } // namespace NConsoleClient -} // namespace NYdb