refactor(eventsub): generate entire files

.github/workflows/build.yml

@@ -71,6 +71,16 @@ jobs:
         run: |
           git config --global --add safe.directory '*'
 
+      - name: Install Python and libclang (24.04)
+        if: matrix.os == 'ubuntu-24.04'
+        run: |
+          sudo apt update
+          sudo DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install \
+            python3 python3-venv clang-18 clang-format-18 libclang-18-dev
+          echo "LIBCLANG_LIBRARY_FILE=/usr/lib/x86_64-linux-gnu/libclang-18.so" >> "$GITHUB_ENV"
+          sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 42
+          sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-18 42
+
       - name: Build
         run: |
           mkdir build
@@ -85,9 +95,16 @@ jobs:
             -DCHATTERINO_PLUGINS="$C2_PLUGINS" \
             -DCMAKE_PREFIX_PATH="$Qt6_DIR/lib/cmake" \
             -DCHATTERINO_STATIC_QT_BUILD=On \
+            -DFORCE_JSON_GENERATION=${{matrix.os == 'ubuntu-24.04' && 'On' || 'Off'}} \
             ..
           make -j"$(nproc)"
 
+      - name: Check generated sources
+        if: matrix.os == 'ubuntu-24.04'
+        run: |
+          git add -N lib/twitch-eventsub-ws/include lib/twitch-eventsub-ws/src
+          git --no-pager diff --exit-code lib/twitch-eventsub-ws/include lib/twitch-eventsub-ws/src
+
       - name: Package - AppImage (Ubuntu)
         if: matrix.build-appimage
         run: |
@@ -239,6 +256,7 @@ jobs:
               -DCHATTERINO_LTO="$Env:C2_ENABLE_LTO" `
               -DCHATTERINO_PLUGINS="$Env:C2_PLUGINS" `
               -DBUILD_WITH_QT6="$Env:C2_BUILD_WITH_QT6" `
+              -DFORCE_JSON_GENERATION=On `
               ..
           set cl=/MP
           nmake /S /NOLOGO
@@ -308,6 +326,7 @@ jobs:
               -DCHATTERINO_LTO="$C2_ENABLE_LTO" \
               -DCHATTERINO_PLUGINS="$C2_PLUGINS" \
               -DBUILD_WITH_QT6="$C2_BUILD_WITH_QT6" \
+              -DFORCE_JSON_GENERATION=Off \
               ..
           make -j"$(sysctl -n hw.logicalcpu)"
         shell: bash

CHANGELOG.md

@@ -21,7 +21,7 @@
 - Bugfix: Fixed the reply button showing for inline whispers and announcements. (#5863)
 - Bugfix: Fixed suspicious user treatment update messages not being searchable. (#5865)
 - Bugfix: Ensure miniaudio backend exits even if it doesn't exit cleanly. (#5896)
-- Dev: Add initial experimental EventSub support. (#5837, #5895)
+- Dev: Add initial experimental EventSub support. (#5837, #5895, #5897)
 - Dev: Highlight checks now use non-capturing groups for the boundaries. (#5784)
 - Dev: Removed unused PubSub whisper code. (#5898)
 - Dev: Updated Conan dependencies. (#5776)

lib/twitch-eventsub-ws/CMakeLists.txt

@@ -16,20 +16,21 @@ find_package(Boost REQUIRED OPTIONAL_COMPONENTS json)
 find_package(OpenSSL REQUIRED)
 
 option(BUILD_WITH_QT6 "Build with Qt6" On)
+option(FORCE_JSON_GENERATION "Make sure JSON implementations are generated at build time" Off)
+
 if (BUILD_WITH_QT6)
     set(MAJOR_QT_VERSION "6")
 else()
     set(MAJOR_QT_VERSION "5")
 endif()
 find_package(Qt${MAJOR_QT_VERSION} REQUIRED COMPONENTS Core)
+find_package(Python3 COMPONENTS Interpreter)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-add_subdirectory(src)
-
 include(cmake/GenerateJson.cmake)
 
-generate_json_impls(SRC_TARGET twitch-eventsub-ws GEN_TARGET twitch-eventsub-ws-gen)
+add_subdirectory(src)
 
 feature_summary(WHAT ALL)

lib/twitch-eventsub-ws/ast/README.md

@@ -1,21 +1,12 @@
 ## Available scripts
 
-- `generate-and-replace-dir.py`  
-  Usage: `./generate-and-replace-dir.py <path-to-dir-containing-header-files>`  
-  Will find all header files in that directory (it won't search recursively), and if they also have a matching source file, generate json deserialize definition & implementations for them, and replacing the markers in the given file.
-
-- `get-builtin-include-dirs.py`  
-  Usage: `./get-builtin-include-dirs.py`  
+- `get-builtin-include-dirs.py`
+  Usage: `./get-builtin-include-dirs.py`
   Prints what builtin include dirs will be used for any of the other scripts.
 
 - `generate.py`  
-  Usage: `./generate.py <path-to-header-file>`  
-  Generates definitions & implementations for the given header file and write them to temporary files.
-
-- `replace-in-file.py`  
-  Usage: `./replace-in-file.py <path-to-header-file> <path-to-source-file>`
-  Reads sdin for two file paths containing the definition & implementations.  
-  Can be used in tandem with `generate.py`, but realistically it's only there because I used it at some point.
+  Usage: `./generate.py [--includes INCLUDES] [--timestamp path] header_path`
+  Generates definitions & implementations for the given header file and writes them to the appropriate files.
 
 ## Environment variables
 

lib/twitch-eventsub-ws/ast/check-clang.hpp

@@ -0,0 +1,12 @@
+// This is a header used for testing if the library can parse a file.
+
+#include <boost/json.hpp>
+#include <QString>
+
+#include <chrono>
+
+boost::json::value doSomething(QString s,
+                               std::chrono::system_clock::time_point tp);
+
+static_assert(sizeof(QString) > sizeof(int));
+static_assert(sizeof(boost::json::value) > sizeof(int));

lib/twitch-eventsub-ws/ast/check-clang.py

@@ -0,0 +1,20 @@
+import clang.cindex
+import sys
+from pathlib import Path
+
+from lib import init_clang_cindex, parse
+
+init_clang_cindex()
+
+assert len(sys.argv) == 2
+additional_includes = sys.argv[1].split(";")
+print("includes:", additional_includes)
+
+tu = parse(Path(__file__).parent / "check-clang.hpp", additional_includes)
+
+if len(tu.diagnostics) > 0:
+    for diag in tu.diagnostics:
+        print(diag.location)
+        print(diag.spelling)
+        print(diag.option)
+    assert False, "TU had warnings/errors when parsing"

lib/twitch-eventsub-ws/ast/generate.py

@@ -1,35 +1,95 @@
 #!/usr/bin/env python3
 
 import logging
+import os
+import re
 import sys
+import argparse
+from datetime import datetime
+from pathlib import Path
 
-from lib import generate, init_clang_cindex, init_logging, temporary_file
+from lib import (
+    generate,
+    init_clang_cindex,
+    init_logging,
+)
 
 log = logging.getLogger("generate")
 
 
+def _logging_level():
+    def to_bool(v: str | None) -> bool:
+        return bool(v) and v.lower() not in ("false", "0", "off", "no")
+
+    return (
+        logging.DEBUG
+        if to_bool(os.environ.get("CI")) or to_bool(os.environ.get("DEBUG_EVENTSUB_GEN"))
+        else logging.WARNING
+    )
+
+
+def _validate_header_path(value: str) -> str:
+    if not value.endswith(".hpp"):
+        raise argparse.ArgumentTypeError("Header path must end with '.hpp'")
+    if not os.path.isfile(value):
+        raise argparse.ArgumentTypeError(f"File '{value}' does not exist")
+    return value
+
+
 def main():
-    init_logging()
+    init_logging(_logging_level())
 
     init_clang_cindex()
 
     if len(sys.argv) < 2:
-        log.error(f"Missing header file argument. Usage: {sys.argv[0]} <path-to-header-file>")
+        log.error(f"usage: {sys.argv[0]} <path-to-header>")
+        sys.exit(1)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "header_path", type=_validate_header_path, help="Path to an existing header file ending with .hpp"
+    )
+    parser.add_argument(
+        "--includes",
+        type=lambda value: value.split(";") if value else [],
+        default=[],
+        help="Semicolon-separated list of include paths",
+    )
+    parser.add_argument("--timestamp", metavar="path", help="Path to the timestamp file to be generated")
+
+    args = parser.parse_args()
+
+    path = Path(args.header_path).resolve()
+
+    if not path.is_file():
+        log.error(f"{path} does not exist")
         sys.exit(1)
 
-    (definitions, implementations) = generate(sys.argv[1])
+    if not path.name.endswith(".hpp"):
+        log.error(f"{path} is not a header file")
+
+    header_path = str(path)
+    def_path = re.sub(r"\.hpp$", ".inc", header_path)
+    source_path = re.sub(r"\.hpp$", ".cpp", header_path)
+    source_path = re.sub(r"include[/\\]twitch-eventsub-ws[/\\]", "src/generated/", source_path)
+
+    log.debug(f"{header_path}: definition={def_path}, implementation={source_path}")
+
+    (definition, implementation) = generate(header_path, args.includes)
+
+    # ensure directories are created
+    Path(def_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(source_path).parent.mkdir(parents=True, exist_ok=True)
 
-    with temporary_file() as (f, path):
-        log.debug(f"Write definitions to {path}")
-        f.write(definitions)
-        f.write("\n")
-        print(path)
+    with open(def_path, "w") as f:
+        f.write(definition)
+    with open(source_path, "w") as f:
+        f.write(implementation)
 
-    with temporary_file() as (f, path):
-        log.debug(f"Write implementations to {path}")
-        f.write(implementations)
-        f.write("\n")
-        print(path)
+    if args.timestamp:
+        path = Path(args.timestamp)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(str(datetime.now().timestamp()))
 
 
 if __name__ == "__main__":

lib/twitch-eventsub-ws/ast/lib/__init__.py

@@ -1,15 +1,13 @@
 from .generate import generate
 from .helpers import get_clang_builtin_include_dirs, init_clang_cindex, temporary_file
 from .logging import init_logging
-from .replace import definition_markers, implementation_markers, replace_in_file
+from .parse import parse
 
 __all__ = [
     "generate",
     "get_clang_builtin_include_dirs",
     "init_clang_cindex",
-    "temporary_file",
     "init_logging",
-    "definition_markers",
-    "implementation_markers",
-    "replace_in_file",
+    "parse",
+    "temporary_file",
 ]

lib/twitch-eventsub-ws/ast/lib/build.py

@@ -9,50 +9,14 @@
 from .struct import Struct
 from .enum import Enum
 from .walker import Walker
+from .parse import parse
 
 log = logging.getLogger(__name__)
 
 
 def build_structs(filename: str, additional_includes: list[str] = []) -> List[Struct | Enum]:
-    if not os.path.isfile(filename):
-        raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
-
-    parse_args = [
-        "-std=c++17",
-        # Uncomment this if you need to debug where it tries to find headers
-        # "-H",
-    ]
-
-    parse_options = (
-        clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
-    )
-
-    extra_includes, system_includes = get_clang_builtin_include_dirs()
-
-    for include_dir in system_includes:
-        parse_args.append(f"-isystem{include_dir}")
-
-    for include_dir in extra_includes:
-        parse_args.append(f"-I{include_dir}")
-
-    for dir in additional_includes:
-        parse_args.append("-I")
-        parse_args.append(dir)
-
-    # Append default dirs
-    # - Append dir of file
-    file_dir = os.path.dirname(os.path.realpath(filename))
-    parse_args.append(f"-I{file_dir}")
-    # - Append project include dir
-    file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
-    parse_args.append(f"-I{file_subdir}")
-
-    # TODO: Use build_commands if available
-
-    index = clang.cindex.Index.create()
-
     log.debug("Parsing translation unit")
-    tu = index.parse(filename, args=parse_args, options=parse_options)
+    tu = parse(filename, additional_includes)
     root = tu.cursor
 
     for diag in tu.diagnostics: