refactor(eventsub): generate entire files

.github/workflows/build.yml

@@ -71,6 +71,12 @@ jobs:
         run: |
           git config --global --add safe.directory '*'
 
+      - name: Install Python
+        run: |
+          sudo apt update
+          sudo DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install \
+            python3 python3-venv
+
       - name: Build
         run: |
           mkdir build
@@ -85,6 +91,7 @@ jobs:
             -DCHATTERINO_PLUGINS="$C2_PLUGINS" \
             -DCMAKE_PREFIX_PATH="$Qt6_DIR/lib/cmake" \
             -DCHATTERINO_STATIC_QT_BUILD=On \
+            -DFORCE_JSON_GENERATION=On \
             ..
           make -j"$(nproc)"
 
@@ -174,6 +181,11 @@ jobs:
           modules: qt5compat qtimageformats
           version: ${{ matrix.qt-version }}
 
+      - name: Setup Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
       # WINDOWS
       - name: Enable Developer Command Prompt (Windows)
         if: startsWith(matrix.os, 'windows')
@@ -239,6 +251,7 @@ jobs:
               -DCHATTERINO_LTO="$Env:C2_ENABLE_LTO" `
               -DCHATTERINO_PLUGINS="$Env:C2_PLUGINS" `
               -DBUILD_WITH_QT6="$Env:C2_BUILD_WITH_QT6" `
+              -DFORCE_JSON_GENERATION=On `
               ..
           set cl=/MP
           nmake /S /NOLOGO
@@ -308,6 +321,7 @@ jobs:
               -DCHATTERINO_LTO="$C2_ENABLE_LTO" \
               -DCHATTERINO_PLUGINS="$C2_PLUGINS" \
               -DBUILD_WITH_QT6="$C2_BUILD_WITH_QT6" \
+              -DFORCE_JSON_GENERATION=On \
               ..
           make -j"$(sysctl -n hw.logicalcpu)"
         shell: bash

CHANGELOG.md

@@ -21,7 +21,7 @@
 - Bugfix: Fixed the reply button showing for inline whispers and announcements. (#5863)
 - Bugfix: Fixed suspicious user treatment update messages not being searchable. (#5865)
 - Bugfix: Ensure miniaudio backend exits even if it doesn't exit cleanly. (#5896)
-- Dev: Add initial experimental EventSub support. (#5837, #5895)
+- Dev: Add initial experimental EventSub support. (#5837, #5895, #5897)
 - Dev: Highlight checks now use non-capturing groups for the boundaries. (#5784)
 - Dev: Updated Conan dependencies. (#5776)
 - Dev: Disable QT keywords (i.e. `emit`, `slots`, and `signals`). (#5882)

lib/twitch-eventsub-ws/CMakeLists.txt

@@ -16,20 +16,22 @@ find_package(Boost REQUIRED OPTIONAL_COMPONENTS json)
 find_package(OpenSSL REQUIRED)
 
 option(BUILD_WITH_QT6 "Build with Qt6" On)
+
+option(FORCE_JSON_GENERATION "Make sure JSON implementations are generated at build time" OFF)
+
 if (BUILD_WITH_QT6)
     set(MAJOR_QT_VERSION "6")
 else()
     set(MAJOR_QT_VERSION "5")
 endif()
 find_package(Qt${MAJOR_QT_VERSION} REQUIRED COMPONENTS Core)
+find_package(Python3 COMPONENTS Interpreter)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-add_subdirectory(src)
-
 include(cmake/GenerateJson.cmake)
 
-generate_json_impls(SRC_TARGET twitch-eventsub-ws GEN_TARGET twitch-eventsub-ws-gen)
+add_subdirectory(src)
 
 feature_summary(WHAT ALL)

lib/twitch-eventsub-ws/ast/check-clang.py

@@ -0,0 +1,3 @@
+from lib import init_clang_cindex
+
+init_clang_cindex()

lib/twitch-eventsub-ws/ast/generate.py

@@ -1,35 +1,71 @@
 #!/usr/bin/env python3
 
 import logging
+import os
+import re
 import sys
+from pathlib import Path
 
-from lib import generate, init_clang_cindex, init_logging, temporary_file
+from lib import (
+    generate,
+    init_clang_cindex,
+    init_logging,
+)
 
 log = logging.getLogger("generate")
 
 
+def _logging_level():
+    def to_bool(v: str | None) -> bool:
+        return not v or v.lower() in ("false", "0", "off", "no")
+
+    return (
+        logging.DEBUG
+        if to_bool(os.environ.get("CI")) or to_bool(os.environ.get("DEBUG_EVENTSUB_GEN"))
+        else logging.WARNING
+    )
+
+
 def main():
-    init_logging()
+    init_logging(_logging_level())
 
     init_clang_cindex()
 
     if len(sys.argv) < 2:
-        log.error(f"Missing header file argument. Usage: {sys.argv[0]} <path-to-header-file>")
+        log.error(f"usage: {sys.argv[0]} <path-to-header>")
+        sys.exit(1)
+
+    additional_includes = []
+    if len(sys.argv) >= 4 and sys.argv[2] == "--includes":
+        additional_includes = sys.argv[3].split(";")
+        log.debug(f"additional includes: {additional_includes}")
+
+    path = Path(sys.argv[1]).resolve()
+
+    if not path.is_file():
+        log.error(f"{path} does not exist")
         sys.exit(1)
 
-    (definitions, implementations) = generate(sys.argv[1])
+    if not path.name.endswith(".hpp"):
+        log.error(f"{path} is not a header file")
+
+    header_path = str(path)
+    def_path = re.sub(r"\.hpp$", ".inc", header_path)
+    source_path = re.sub(r"\.hpp$", ".cpp", header_path)
+    source_path = re.sub(r"include[/\\]twitch-eventsub-ws[/\\]", "src/generated/", source_path)
+
+    log.debug(f"{header_path}: definition={def_path}, implementation={source_path}")
+
+    (definition, implementation) = generate(header_path, additional_includes)
 
-    with temporary_file() as (f, path):
-        log.debug(f"Write definitions to {path}")
-        f.write(definitions)
-        f.write("\n")
-        print(path)
+    # ensure directories are created
+    Path(def_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(source_path).parent.mkdir(parents=True, exist_ok=True)
 
-    with temporary_file() as (f, path):
-        log.debug(f"Write implementations to {path}")
-        f.write(implementations)
-        f.write("\n")
-        print(path)
+    with open(def_path, "w") as f:
+        f.write(definition)
+    with open(source_path, "w") as f:
+        f.write(implementation)
 
 
 if __name__ == "__main__":

lib/twitch-eventsub-ws/ast/lib/__init__.py

@@ -1,15 +1,11 @@
 from .generate import generate
 from .helpers import get_clang_builtin_include_dirs, init_clang_cindex, temporary_file
 from .logging import init_logging
-from .replace import definition_markers, implementation_markers, replace_in_file
 
 __all__ = [
     "generate",
     "get_clang_builtin_include_dirs",
     "init_clang_cindex",
     "temporary_file",
     "init_logging",
-    "definition_markers",
-    "implementation_markers",
-    "replace_in_file",
 ]

lib/twitch-eventsub-ws/ast/lib/build.py

@@ -19,6 +19,7 @@ def build_structs(filename: str, additional_includes: list[str] = []) -> List[St
 
     parse_args = [
         "-std=c++17",
+        "-Werror=implicit-int",
         # Uncomment this if you need to debug where it tries to find headers
         # "-H",
     ]

lib/twitch-eventsub-ws/ast/lib/enum.py

@@ -13,12 +13,13 @@
 
 
 class Enum:
-    def __init__(self, name: str) -> None:
+    def __init__(self, name: str, namespace: tuple[str, ...]) -> None:
         self.name = name
         self.constants: List[EnumConstant] = []
         self.parent: str = ""
         self.comment_commands: CommentCommands = []
         self.inner_root: str = ""
+        self.namespace = namespace
 
     @property
     def full_name(self) -> str:

lib/twitch-eventsub-ws/ast/lib/generate.py

@@ -1,22 +1,57 @@
 from typing import Tuple
 
 import logging
+from pathlib import Path
 
+from .struct import Struct
+from .enum import Enum
 from .build import build_structs
 from .format import format_code
-from .helpers import init_clang_cindex, temporary_file
 from .jinja_env import env
-from .logging import init_logging
 
 log = logging.getLogger(__name__)
 
 
+def _generate_implementation(header_path: str, items: list[Struct | Enum]) -> str:
+    current_ns = ""
+    doc = f"""
+// WARNING: This file is automatically generated. Any changes will be lost.
+#include "{header_path}"
+#include "twitch-eventsub-ws/chrono.hpp" // IWYU pragma: keep
+#include "twitch-eventsub-ws/errors.hpp"
+#include <boost/json.hpp>
+
+"""
+
+    for item in items:
+        next_ns = "::".join(item.namespace)
+        if next_ns != current_ns:
+            if current_ns:
+                doc += "} // namespace " + current_ns + "\n"
+            if next_ns:
+                doc += "namespace " + next_ns + "{\n\n"
+            current_ns = next_ns
+        doc += item.try_value_to_implementation(env) + "\n\n"
+
+    if current_ns:
+        doc += "\n} // namespace " + current_ns + "\n\n"
+
+    return doc
+
+
+def _get_relative_header_path(header_path: str) -> str:
+    root_path = Path(__file__).parent.parent.parent / "include"
+    return Path(header_path).relative_to(root_path).as_posix()
+
+
 def generate(header_path: str, additional_includes: list[str] = []) -> Tuple[str, str]:
     structs = build_structs(header_path, additional_includes)
 
+    rel_header_path = _get_relative_header_path(header_path)
+
     log.debug("Generate & format definitions")
     definitions = format_code("\n\n".join([struct.try_value_to_definition(env) for struct in structs]))
     log.debug("Generate & format implementations")
-    implementations = format_code("\n\n".join([struct.try_value_to_implementation(env) for struct in structs]))
+    implementations = format_code(_generate_implementation(rel_header_path, structs))
 
     return (definitions, implementations)

lib/twitch-eventsub-ws/ast/lib/logging.py

@@ -4,12 +4,12 @@
 from colorama import Fore, Style
 
 
-def init_logging() -> None:
+def init_logging(level=logging.DEBUG) -> None:
     root = logging.getLogger()
-    root.setLevel(logging.DEBUG)
+    root.setLevel(level)
 
     handler = logging.StreamHandler(sys.stderr)
-    handler.setLevel(logging.DEBUG)
+    handler.setLevel(level)
 
     colors = {
         "WARNING": Fore.YELLOW,

lib/twitch-eventsub-ws/ast/lib/member.py

@@ -13,7 +13,7 @@
 log = logging.getLogger(__name__)
 
 
-def get_type_name(type: clang.cindex.Type, namespace: List[str]) -> str:
+def get_type_name(type: clang.cindex.Type, namespace: tuple[str, ...]) -> str:
     if namespace:
         namespace_str = f"{'::'.join(namespace)}::"
     else:
@@ -68,7 +68,7 @@ def apply_comment_commands(self, comment_commands: CommentCommands) -> None:
                     log.warning(f"Unknown comment command found: {other} with value {value}")
 
     @staticmethod
-    def from_field(node: clang.cindex.Cursor, namespace: List[str]) -> Member:
+    def from_field(node: clang.cindex.Cursor, namespace: tuple[str, ...]) -> Member:
         assert node.type is not None
 
         name = node.spelling