Reland "[HLSL][RootSignature] Implement parsing of a DescriptorTable with empty clauses" (#133958)

This pr relands #133302.

It resolves two issues:
- Linking error during build,
[here](#133302 (comment)).
There was a missing dependency for `clangLex` for the
`ParseHLSLRootSignatureTest.cpp` unit testing. This library was added to
the dependencies to resolve the error. It wasn't caught previously as
the library was transitively linked in most build environments
- Warning of unused declaration,
[here](#133302 (comment)).
There was a usability line in `LexHLSLRootSignature.h` of the form
`using TokenKind = enum RootSignatureToken::Kind` which causes this
error. The declaration is removed from the header file to be used
locally in the `.cpp` files that use it.
Notably, the original pr would also exposed `clang::hlsl::TokenKind` to
everywhere it was included, which had a name clash with
`tok::TokenKind`. This is another motivation to change to the proposed
resolution.

---------

Co-authored-by: Finn Plummer <[email protected]>

Author: inbelic

Diff for: clang/include/clang/Basic/DiagnosticParseKinds.td

@@ -1830,4 +1830,8 @@ def err_hlsl_virtual_function
 def err_hlsl_virtual_inheritance
     : Error<"virtual inheritance is unsupported in HLSL">;
 
+// HLSL Root Siganture diagnostic messages
+def err_hlsl_unexpected_end_of_params
+    : Error<"expected %0 to denote end of parameters, or, another valid parameter of %1">;
+
 } // end of Parser diagnostics

Diff for: clang/include/clang/Lex/HLSLRootSignatureTokenKinds.def

@@ -14,16 +14,16 @@
 //===----------------------------------------------------------------------===//
 
 #ifndef TOK
-#define TOK(X)
+#define TOK(X, SPELLING)
 #endif
 #ifndef PUNCTUATOR
-#define PUNCTUATOR(X,Y) TOK(pu_ ## X)
+#define PUNCTUATOR(X,Y) TOK(pu_ ## X, Y)
 #endif
 #ifndef KEYWORD
-#define KEYWORD(X) TOK(kw_ ## X)
+#define KEYWORD(X) TOK(kw_ ## X, #X)
 #endif
 #ifndef ENUM
-#define ENUM(NAME, LIT) TOK(en_ ## NAME)
+#define ENUM(NAME, LIT) TOK(en_ ## NAME, LIT)
 #endif
 
 // Defines the various types of enum
@@ -49,15 +49,15 @@
 #endif
 
 // General Tokens:
-TOK(invalid)
-TOK(end_of_stream)
-TOK(int_literal)
+TOK(invalid, "invalid identifier")
+TOK(end_of_stream, "end of stream")
+TOK(int_literal, "integer literal")
 
 // Register Tokens:
-TOK(bReg)
-TOK(tReg)
-TOK(uReg)
-TOK(sReg)
+TOK(bReg, "b register")
+TOK(tReg, "t register")
+TOK(uReg, "u register")
+TOK(sReg, "s register")
 
 // Punctuators:
 PUNCTUATOR(l_paren, '(')
@@ -69,6 +69,7 @@ PUNCTUATOR(plus,    '+')
 PUNCTUATOR(minus,   '-')
 
 // RootElement Keywords:
+KEYWORD(RootSignature) // used only for diagnostic messaging
 KEYWORD(DescriptorTable)
 
 // DescriptorTable Keywords:

Diff for: clang/include/clang/Lex/LexHLSLRootSignature.h

@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_LEX_LEXHLSLROOTSIGNATURE_H
 #define LLVM_CLANG_LEX_LEXHLSLROOTSIGNATURE_H
 
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/SourceLocation.h"
 
 #include "llvm/ADT/SmallVector.h"
@@ -24,11 +25,11 @@ namespace hlsl {
 
 struct RootSignatureToken {
   enum Kind {
-#define TOK(X) X,
+#define TOK(X, SPELLING) X,
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   };
 
-  Kind Kind = Kind::invalid;
+  Kind TokKind = Kind::invalid;
 
   // Retain the SouceLocation of the token for diagnostics
   clang::SourceLocation TokLoc;
@@ -38,10 +39,21 @@ struct RootSignatureToken {
 
   // Constructors
   RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {}
-  RootSignatureToken(enum Kind Kind, clang::SourceLocation TokLoc)
-      : Kind(Kind), TokLoc(TokLoc) {}
+  RootSignatureToken(Kind TokKind, clang::SourceLocation TokLoc)
+      : TokKind(TokKind), TokLoc(TokLoc) {}
 };
-using TokenKind = enum RootSignatureToken::Kind;
+
+inline const DiagnosticBuilder &
+operator<<(const DiagnosticBuilder &DB, const RootSignatureToken::Kind Kind) {
+  switch (Kind) {
+#define TOK(X, SPELLING)                                                       \
+  case RootSignatureToken::Kind::X:                                            \
+    DB << SPELLING;                                                            \
+    break;
+#include "clang/Lex/HLSLRootSignatureTokenKinds.def"
+  }
+  return DB;
+}
 
 class RootSignatureLexer {
 public:

Diff for: clang/include/clang/Parse/ParseHLSLRootSignature.h

@@ -0,0 +1,107 @@
+//===--- ParseHLSLRootSignature.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the RootSignatureParser interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
+#define LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
+
+#include "clang/Basic/DiagnosticParse.h"
+#include "clang/Lex/LexHLSLRootSignature.h"
+#include "clang/Lex/Preprocessor.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+
+namespace clang {
+namespace hlsl {
+
+class RootSignatureParser {
+public:
+  RootSignatureParser(SmallVector<llvm::hlsl::rootsig::RootElement> &Elements,
+                      RootSignatureLexer &Lexer, clang::Preprocessor &PP);
+
+  /// Consumes tokens from the Lexer and constructs the in-memory
+  /// representations of the RootElements. Tokens are consumed until an
+  /// error is encountered or the end of the buffer.
+  ///
+  /// Returns true if a parsing error is encountered.
+  bool parse();
+
+private:
+  DiagnosticsEngine &getDiags() { return PP.getDiagnostics(); }
+
+  // All private Parse.* methods follow a similar pattern:
+  //   - Each method will start with an assert to denote what the CurToken is
+  // expected to be and will parse from that token forward
+  //
+  //   - Therefore, it is the callers responsibility to ensure that you are
+  // at the correct CurToken. This should be done with the pattern of:
+  //
+  //  if (TryConsumeExpectedToken(RootSignatureToken::Kind))
+  //    if (Parse.*())
+  //      return true;
+  //
+  // or,
+  //
+  //  if (ConsumeExpectedToken(RootSignatureToken::Kind, ...))
+  //    return true;
+  //  if (Parse.*())
+  //    return true;
+  //
+  //   - All methods return true if a parsing error is encountered. It is the
+  // callers responsibility to propogate this error up, or deal with it
+  // otherwise
+  //
+  //   - An error will be raised if the proceeding tokens are not what is
+  // expected, or, there is a lexing error
+
+  /// Root Element parse methods:
+  bool parseDescriptorTable();
+  bool parseDescriptorTableClause();
+
+  /// Invoke the Lexer to consume a token and update CurToken with the result
+  void consumeNextToken() { CurToken = Lexer.ConsumeToken(); }
+
+  /// Return true if the next token one of the expected kinds
+  bool peekExpectedToken(RootSignatureToken::Kind Expected);
+  bool peekExpectedToken(ArrayRef<RootSignatureToken::Kind> AnyExpected);
+
+  /// Consumes the next token and report an error if it is not of the expected
+  /// kind.
+  ///
+  /// Returns true if there was an error reported.
+  bool consumeExpectedToken(
+      RootSignatureToken::Kind Expected, unsigned DiagID = diag::err_expected,
+      RootSignatureToken::Kind Context = RootSignatureToken::Kind::invalid);
+
+  /// Peek if the next token is of the expected kind and if it is then consume
+  /// it.
+  ///
+  /// Returns true if it successfully matches the expected kind and the token
+  /// was consumed.
+  bool tryConsumeExpectedToken(RootSignatureToken::Kind Expected);
+  bool tryConsumeExpectedToken(ArrayRef<RootSignatureToken::Kind> Expected);
+
+private:
+  SmallVector<llvm::hlsl::rootsig::RootElement> &Elements;
+  RootSignatureLexer &Lexer;
+
+  clang::Preprocessor &PP;
+
+  RootSignatureToken CurToken;
+};
+
+} // namespace hlsl
+} // namespace clang
+
+#endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H

Diff for: clang/lib/Lex/LexHLSLRootSignature.cpp

@@ -11,6 +11,8 @@
 namespace clang {
 namespace hlsl {
 
+using TokenKind = RootSignatureToken::Kind;
+
 // Lexer Definitions
 
 static bool IsNumberChar(char C) {
@@ -34,7 +36,7 @@ RootSignatureToken RootSignatureLexer::LexToken() {
   switch (C) {
 #define PUNCTUATOR(X, Y)                                                       \
   case Y: {                                                                    \
-    Result.Kind = TokenKind::pu_##X;                                           \
+    Result.TokKind = TokenKind::pu_##X;                                        \
     AdvanceBuffer();                                                           \
     return Result;                                                             \
   }
@@ -45,7 +47,7 @@ RootSignatureToken RootSignatureLexer::LexToken() {
 
   // Integer literal
   if (isdigit(C)) {
-    Result.Kind = TokenKind::int_literal;
+    Result.TokKind = TokenKind::int_literal;
     Result.NumSpelling = Buffer.take_while(IsNumberChar);
     AdvanceBuffer(Result.NumSpelling.size());
     return Result;
@@ -65,16 +67,16 @@ RootSignatureToken RootSignatureLexer::LexToken() {
     // Convert character to the register type.
     switch (C) {
     case 'b':
-      Result.Kind = TokenKind::bReg;
+      Result.TokKind = TokenKind::bReg;
       break;
     case 't':
-      Result.Kind = TokenKind::tReg;
+      Result.TokKind = TokenKind::tReg;
       break;
     case 'u':
-      Result.Kind = TokenKind::uReg;
+      Result.TokKind = TokenKind::uReg;
       break;
     case 's':
-      Result.Kind = TokenKind::sReg;
+      Result.TokKind = TokenKind::sReg;
       break;
     default:
       llvm_unreachable("Switch for an expected token was not provided");
@@ -100,14 +102,14 @@ RootSignatureToken RootSignatureLexer::LexToken() {
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
 
   // Then attempt to retreive a string from it
-  Result.Kind = Switch.Default(TokenKind::invalid);
+  Result.TokKind = Switch.Default(TokenKind::invalid);
   AdvanceBuffer(TokSpelling.size());
   return Result;
 }
 
 RootSignatureToken RootSignatureLexer::ConsumeToken() {
   // If we previously peeked then just return the previous value over
-  if (NextToken && NextToken->Kind != TokenKind::end_of_stream) {
+  if (NextToken && NextToken->TokKind != TokenKind::end_of_stream) {
     RootSignatureToken Result = *NextToken;
     NextToken = std::nullopt;
     return Result;

Diff for: clang/lib/Parse/CMakeLists.txt

@@ -14,6 +14,7 @@ add_clang_library(clangParse
   ParseExpr.cpp
   ParseExprCXX.cpp
   ParseHLSL.cpp
+  ParseHLSLRootSignature.cpp
   ParseInit.cpp
   ParseObjc.cpp
   ParseOpenMP.cpp