diff --git a/src/networkprotocoldsl/lexer/token/identifier.hpp b/src/networkprotocoldsl/lexer/token/identifier.hpp index 19b86af..520f178 100644 --- a/src/networkprotocoldsl/lexer/token/identifier.hpp +++ b/src/networkprotocoldsl/lexer/token/identifier.hpp @@ -7,6 +7,7 @@ namespace networkprotocoldsl::lexer::token { struct Identifier { std::string name; + std::string stringify() const { return name; } }; } // namespace networkprotocoldsl::lexer::token diff --git a/src/networkprotocoldsl/lexer/token/keyword.hpp b/src/networkprotocoldsl/lexer/token/keyword.hpp index 314c326..66ed799 100644 --- a/src/networkprotocoldsl/lexer/token/keyword.hpp +++ b/src/networkprotocoldsl/lexer/token/keyword.hpp @@ -1,14 +1,28 @@ #ifndef INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_KEYWORD_HPP #define INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_KEYWORD_HPP +#include + namespace networkprotocoldsl::lexer::token::keyword { -struct For {}; -struct In {}; -struct Message {}; -struct Parts {}; -struct Terminator {}; -struct Tokens {}; +struct For { + std::string stringify() const { return "for"; } +}; +struct In { + std::string stringify() const { return "in"; } +}; +struct Message { + std::string stringify() const { return "message"; } +}; +struct Parts { + std::string stringify() const { return "parts"; } +}; +struct Terminator { + std::string stringify() const { return "terminator"; } +}; +struct Tokens { + std::string stringify() const { return "tokens"; } +}; } // namespace networkprotocoldsl::lexer::token::keyword diff --git a/src/networkprotocoldsl/lexer/token/literal.hpp b/src/networkprotocoldsl/lexer/token/literal.hpp index 7e4fe36..d09d164 100644 --- a/src/networkprotocoldsl/lexer/token/literal.hpp +++ b/src/networkprotocoldsl/lexer/token/literal.hpp @@ -1,18 +1,34 @@ #ifndef INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_LITERAL_HPP #define INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_LITERAL_HPP +#include #include namespace networkprotocoldsl::lexer::token::literal { struct Integer { int value; + std::string stringify() const { + std::stringstream ss = std::stringstream(); + ss << value; + return ss.str(); + } }; struct String { std::string value; + std::string stringify() const { + std::stringstream ss = std::stringstream(); + ss << "\"" << value << "\""; + return ss.str(); + } }; struct Boolean { bool value; + std::string stringify() const { + std::stringstream ss = std::stringstream(); + ss << value; + return ss.str(); + } }; } // namespace networkprotocoldsl::lexer::token::literal diff --git a/src/networkprotocoldsl/lexer/token/punctuation.hpp b/src/networkprotocoldsl/lexer/token/punctuation.hpp index b80a9e5..801ba8d 100644 --- a/src/networkprotocoldsl/lexer/token/punctuation.hpp +++ b/src/networkprotocoldsl/lexer/token/punctuation.hpp @@ -1,17 +1,37 @@ #ifndef INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_PUNCTUATION_HPP #define INCLUDED_NETWORKPROTOCOLDSL_LEXER_TOKEN_PUNCTUATION_HPP +#include + namespace networkprotocoldsl::lexer::token::punctuation { -struct AngleBracketClose {}; -struct AngleBracketOpen {}; -struct Comma {}; -struct CurlyBraceClose {}; -struct CurlyBraceOpen {}; -struct Equal {}; -struct KeyValueSeparator {}; -struct StatementEnd {}; -struct Dot {}; +struct AngleBracketClose { + std::string stringify() const { return ">"; } +}; +struct AngleBracketOpen { + std::string stringify() const { return "<"; } +}; +struct Comma { + std::string stringify() const { return ","; } +}; +struct CurlyBraceClose { + std::string stringify() const { return "}"; } +}; +struct CurlyBraceOpen { + std::string stringify() const { return "{"; } +}; +struct Equal { + std::string stringify() const { return "="; } +}; +struct KeyValueSeparator { + std::string stringify() const { return ":"; } +}; +struct StatementEnd { + std::string stringify() const { return ";"; } +}; +struct Dot { + std::string stringify() const { return "."; } +}; } // namespace networkprotocoldsl::lexer::token::punctuation diff --git a/src/networkprotocoldsl/parser/grammar/identifier.hpp b/src/networkprotocoldsl/parser/grammar/identifier.hpp index cd2064c..ce8f1d9 100644 --- a/src/networkprotocoldsl/parser/grammar/identifier.hpp +++ b/src/networkprotocoldsl/parser/grammar/identifier.hpp @@ -13,8 +13,10 @@ namespace networkprotocoldsl::parser::grammar { class IdentifierReference; // Forward declaration class IdentifierMemberAccess - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "IdentiferMemberAccess"; static void partial_match() {} static IdentifierReference *recurse_one(lexer::token::punctuation::Dot) { return nullptr; @@ -27,8 +29,10 @@ class IdentifierMemberAccess }; class IdentifierReference - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "IdentiferReference"; static void partial_match() {} static IdentifierMemberAccess *recurse_maybe(lexer::token::Identifier) { return nullptr; diff --git a/src/networkprotocoldsl/parser/grammar/literals.hpp b/src/networkprotocoldsl/parser/grammar/literals.hpp index 07532ad..43b33bd 100644 --- a/src/networkprotocoldsl/parser/grammar/literals.hpp +++ b/src/networkprotocoldsl/parser/grammar/literals.hpp @@ -10,8 +10,10 @@ namespace networkprotocoldsl::parser::grammar { class BooleanLiteral - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "BooleanLiteral"; static void partial_match() {} static ParseStateReturn match(TokenIterator begin, TokenIterator end, lexer::token::literal::Boolean b) { @@ -20,8 +22,10 @@ class BooleanLiteral }; class IntegerLiteral - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "IntegerLiteral"; static void partial_match() {} static ParseStateReturn match(TokenIterator begin, TokenIterator end, lexer::token::literal::Integer i) { @@ -30,8 +34,10 @@ class IntegerLiteral }; class StringLiteral - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "StringLiteral"; static void partial_match() {} static ParseStateReturn match(TokenIterator begin, TokenIterator end, lexer::token::literal::String s) { diff --git a/src/networkprotocoldsl/parser/grammar/messagedata.hpp b/src/networkprotocoldsl/parser/grammar/messagedata.hpp index 30ccd6c..ae17b2e 100644 --- a/src/networkprotocoldsl/parser/grammar/messagedata.hpp +++ b/src/networkprotocoldsl/parser/grammar/messagedata.hpp @@ -13,8 +13,10 @@ namespace networkprotocoldsl::parser::grammar { class MessageDataPair - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "MessageDataPair"; static void partial_match() {} static void partial_match(lexer::token::Identifier) {} static Type *recurse_one(lexer::token::Identifier, @@ -35,8 +37,10 @@ class MessageDataPair } }; -class MessageData : public support::RecursiveParser { +class MessageData : public support::RecursiveParser> { public: + static constexpr const char *name = "MessageData"; static void partial_match() {} static bool conditional_partial_match(lexer::token::Identifier id) { if (id.name == "data") { diff --git a/src/networkprotocoldsl/parser/grammar/tokenparts.hpp b/src/networkprotocoldsl/parser/grammar/tokenparts.hpp index 25c3ac9..3ba7ef6 100644 --- a/src/networkprotocoldsl/parser/grammar/tokenparts.hpp +++ b/src/networkprotocoldsl/parser/grammar/tokenparts.hpp @@ -13,8 +13,10 @@ namespace networkprotocoldsl::parser::grammar { -class TokenPart : public support::RecursiveParser { +class TokenPart : public support::RecursiveParser> { public: + static constexpr const char *name = "TokenPart"; static std::tuple *recurse_any() { return nullptr; } @@ -25,8 +27,10 @@ class TokenPart : public support::RecursiveParser { }; class TokenSequence - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "TokenSequence"; static void partial_match() {} static void partial_match(lexer::token::keyword::Tokens) {} static TokenPart *recurse_many(lexer::token::keyword::Tokens, @@ -46,8 +50,10 @@ class TokenSequence } }; -class Terminator : public support::RecursiveParser { +class Terminator : public support::RecursiveParser> { public: + static constexpr const char *name = "Terminator"; static void partial_match() {} static void partial_match(lexer::token::keyword::Terminator) {} static StringLiteral *recurse_one(lexer::token::keyword::Terminator, diff --git a/src/networkprotocoldsl/parser/grammar/traits.hpp b/src/networkprotocoldsl/parser/grammar/traits.hpp index c77fd36..e3f8c36 100644 --- a/src/networkprotocoldsl/parser/grammar/traits.hpp +++ b/src/networkprotocoldsl/parser/grammar/traits.hpp @@ -50,8 +50,73 @@ using NodeVariant = using TokenIterator = std::vector::const_iterator; +static int indent_level(int modify) { + static int indent = 0; + indent += modify; + for (int i = 0; i < indent; ++i) { + std::cerr << ' '; + } + std::cerr << '(' << indent << ") "; + return indent; +} + +static bool trace_enabled() { + static bool r = getenv("GRAMMAR_TRACER") != nullptr; + return r; +} + } // namespace +template class Tracer { +public: + static void output_tokens(const TokenIterator begin, + const TokenIterator end) { + int max = 10; + TokenIterator b = begin; + while (max > 0 && b != end) { + std::cerr << " " << std::visit([&](auto t) { return t.stringify(); }, *b); + b++; + max--; + } + if (b != end) { + std::cerr << "..."; + } + } + + using TokenIterator = ParserContext::TokenIterator; + template + static void trace_start(const char *attempt_type, const TokenIterator &begin, + const TokenIterator &end, Args... args) { + if (!trace_enabled()) + return; + indent_level(1); + std::cerr << "> " << attempt_type << " " << ParserContext::name << " (" + << sizeof...(args) << "):"; + output_tokens(begin, end); + std::cerr << std::endl; + } + template + static void trace_success(const TokenIterator &begin, + const TokenIterator &end, Args... args) { + if (!trace_enabled()) + return; + indent_level(-1); + std::cerr << "< " << ParserContext::name << " [SUCCESS]"; + output_tokens(begin, end); + std::cerr << std::endl; + } + template + static void trace_fail(const TokenIterator &begin, const TokenIterator &end, + Args... args) { + if (!trace_enabled()) + return; + indent_level(-1); + std::cerr << "< " << ParserContext::name << " [FAIL]"; + output_tokens(begin, end); + std::cerr << std::endl; + } +}; + using ParseTraits = support::ParseStateTraits; } // namespace networkprotocoldsl::parser::grammar diff --git a/src/networkprotocoldsl/parser/grammar/typeparameter.hpp b/src/networkprotocoldsl/parser/grammar/typeparameter.hpp index ce0f397..df815fc 100644 --- a/src/networkprotocoldsl/parser/grammar/typeparameter.hpp +++ b/src/networkprotocoldsl/parser/grammar/typeparameter.hpp @@ -16,8 +16,10 @@ namespace networkprotocoldsl::parser::grammar { class Type; // Forward declaration class TypeParameterValue - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "TypeParameterValue"; static std::tuple * recurse_any() { return nullptr; @@ -33,8 +35,10 @@ class TypeParameterValue }; class TypeParameter - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "TypeParameter"; static void partial_match() {} static void partial_match(lexer::token::Identifier) {} static TypeParameterValue *recurse_one(lexer::token::Identifier, @@ -56,8 +60,10 @@ class TypeParameter }; class TypeParameters - : public support::RecursiveParser { + : public support::RecursiveParser> { public: + static constexpr const char *name = "TypeParameters"; static void partial_match() {} static TypeParameter * recurse_many(lexer::token::punctuation::AngleBracketOpen) { @@ -84,8 +90,9 @@ class TypeParameters } }; -class Type : public support::RecursiveParser { +class Type : public support::RecursiveParser> { public: + static constexpr const char *name = "Type"; static void partial_match() {} static TypeParameters *recurse_maybe(lexer::token::Identifier) { return nullptr; diff --git a/src/networkprotocoldsl/parser/support/recursiveparser.hpp b/src/networkprotocoldsl/parser/support/recursiveparser.hpp index 461b27b..c4bb5c3 100644 --- a/src/networkprotocoldsl/parser/support/recursiveparser.hpp +++ b/src/networkprotocoldsl/parser/support/recursiveparser.hpp @@ -2,8 +2,10 @@ #define INCLUDED_NETWORKPROTOCOLDSL_PARSER_SUPPORT_RECURSIVEPARSER_HPP #include +#include #include #include +#include #include #include #include @@ -197,80 +199,123 @@ get_common_variant_index(const std::vector &variants) { return common_index; } -template +template class NoOpTracer { +public: + using TokenIterator = ParseStateTraits::TokenIterator; + template + static void trace_start(const char *attempt_type, const TokenIterator &begin, + const TokenIterator &end, Args... args) {} + template + static void trace_success(const TokenIterator &begin, + const TokenIterator &end, Args... args) {} + template + static void trace_fail(const TokenIterator &begin, const TokenIterator &end, + Args... args) {} +}; + +template > class RecursiveParser { public: using ParseStateReturn = typename ParseStateTraits::ParseStateReturn; using TokenIterator = typename ParseStateTraits::TokenIterator; using ParseNode = typename ParseStateTraits::ParseNode; + template + static ParseStateReturn wrap_parse_with_trace(const TokenIterator begin, + const TokenIterator end, + Args... args) { + ParseStateReturn r = ParserContext::parse(begin, end, args...); + if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); + } else { + Tracer::trace_success(begin, end, args...); + } + return r; + } + template static ParseStateReturn parse(TokenIterator begin, TokenIterator end, Args... args) { if constexpr (is_partial_match::value) { - if (begin == end) + Tracer::trace_start("partial_match", begin, end, args...); + if (begin == end) { + Tracer::trace_fail(begin, end, args...); return {std::nullopt, begin, end}; + } auto token = *begin++; return std::visit( - [=](auto t) { return ParserContext::parse(begin, end, args..., t); }, + [=](auto t) { return wrap_parse_with_trace(begin, end, args..., t); }, token); } else if constexpr (is_conditional_partial_match< ParseStateTraits, ParserContext, Args...>::value) { - if (begin == end) + Tracer::trace_start("conditional_match", begin, end, args...); + if (begin == end) { + Tracer::trace_fail(begin, end, args...); return {std::nullopt, begin, end}; + } if (ParserContext::conditional_partial_match(args...)) { auto token = *begin++; return std::visit( [=](auto t) { - return ParserContext::parse(begin, end, args..., t); + return wrap_parse_with_trace(begin, end, args..., t); }, token); } else { + Tracer::trace_fail(begin, end, args...); return {std::nullopt, begin, end}; } } else if constexpr (is_recurse_one::value) { + Tracer::trace_start("recurse_one", begin, end, args...); ParseStateReturn r = recurse_one::type::parse( begin, end); - if (!r.node.has_value()) + if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); return r; + } return std::visit( [=](auto t) { - return ParserContext::parse(r.begin, end, args..., t); + return wrap_parse_with_trace(r.begin, end, args..., t); }, r.node.value()); } else if constexpr (is_recurse_maybe::value) { + Tracer::trace_start("recurse_maybe", begin, end, args...); ParseStateReturn r = recurse_maybe::type::parse( begin, end); if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); return ParserContext::parse(begin, end, args..., std::nullopt); } else { return std::visit( [=](auto t) { - return ParserContext::parse(r.begin, end, args..., t); + return wrap_parse_with_trace(r.begin, end, args..., t); }, r.node.value()); } } else if constexpr (is_recurse_any::value) { + Tracer::trace_start("recurse_any", begin, end, args...); ParseStateReturn r = recurse_any::parse_any( begin, end); if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); return r; } else { return std::visit( [=](auto t) { - return ParserContext::parse(r.begin, end, args..., t); + return wrap_parse_with_trace(r.begin, end, args..., t); }, r.node.value()); } } else if constexpr (is_recurse_many::value) { + Tracer::trace_start("recurse_many", begin, end, args...); std::vector nodes; while (true) { ParseStateReturn r = @@ -312,16 +357,37 @@ class RecursiveParser { for (auto v : nodes) { output.push_back(std::get(v)); } - return ParserContext::parse(begin, end, args..., output); + auto r = ParserContext::parse(begin, end, args..., output); + if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args..., output); + } else { + Tracer::trace_success(begin, end, args..., output); + } + return r; }, nodes.at(0)); } } - return ParserContext::parse(begin, end, args..., nodes); + auto r = ParserContext::parse(begin, end, args..., nodes); + if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); + } else { + Tracer::trace_success(begin, end, args...); + } + return r; } else if constexpr (is_match::value) { - return ParserContext::match(begin, end, args...); + Tracer::trace_start("match", begin, end, args...); + auto r = ParserContext::match(begin, end, args...); + if (!r.node.has_value()) { + Tracer::trace_fail(begin, end, args...); + } else { + Tracer::trace_success(begin, end, args...); + } + return r; } else { + Tracer::trace_start("unknown", begin, end, args...); + Tracer::trace_fail(begin, end, args...); return {std::nullopt, begin, end}; } } diff --git a/src/networkprotocoldsl/parser/tree/messagepart.hpp b/src/networkprotocoldsl/parser/tree/messagepart.hpp index 420cead..a5e5c4d 100644 --- a/src/networkprotocoldsl/parser/tree/messagepart.hpp +++ b/src/networkprotocoldsl/parser/tree/messagepart.hpp @@ -10,7 +10,9 @@ namespace networkprotocoldsl::parser::tree { struct MessageForLoop; -using MessagePart = std::variant; +using MessagePart = std::variant, + std::shared_ptr, + std::shared_ptr>; } // namespace networkprotocoldsl::parser::tree diff --git a/tests/017-grammar-identifier.cpp b/tests/017-grammar-identifier.cpp index 26e1dda..b558d17 100644 --- a/tests/017-grammar-identifier.cpp +++ b/tests/017-grammar-identifier.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -16,4 +17,18 @@ TEST(IdentifierReferenceTest, IdentifierReferenceMatch) { result.node.value()) ->name, "myIdentifier"); -} \ No newline at end of file +} + +TEST(IdentifierReferenceTest, IdentifierReferenceWithMemberMatch) { + auto maybe_tokens = lexer::tokenize("myIdentifier.member"); + ASSERT_TRUE(maybe_tokens.has_value()); + std::vector tokens = maybe_tokens.value(); + auto result = parser::grammar::IdentifierReference::parse(tokens.cbegin(), + tokens.cend()); + ASSERT_TRUE(result.node.has_value()); + auto id = std::get>( + result.node.value()); + ASSERT_EQ(id->name, "myIdentifier"); + ASSERT_TRUE(id->member.has_value()); + ASSERT_EQ(id->member.value()->name, "member"); +}