[eudsl] factor out eudsl-nbgen

.github/actions/setup_base/action.yml

@@ -150,6 +150,8 @@ runs:
         echo "CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros" >> $GITHUB_ENV
         echo "CCACHE_CPP2=true" >> $GITHUB_ENV
         echo "CCACHE_UMASK=002" >> $GITHUB_ENV
+
+        ccache -z
 
         echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
         echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV

.github/workflows/build_test_release_eudsl.yml

@@ -108,11 +108,12 @@ jobs:
             echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
             echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
             echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
-            echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV
+            echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV
           else
             echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
             echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
             echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
+            echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV
           fi
 
           # since linux builds execute in the cibuildwheel almalinux container
@@ -129,24 +130,46 @@ jobs:
             echo MACOSX_DEPLOYMENT_TARGET=13.7 >> $GITHUB_ENV
           fi
 
+          # prevent OOM on free GHA
+          echo "EUDSLPY_DISABLE_COMPILE_OPT=${{ inputs.eudslpy_disable_compile_opt == 'true' && 'ON' || 'OFF' }}" >> $GITHUB_ENV
+
           pip install cibuildwheel
 
       - name: "Build eudsl-tblgen"
         run: |
 
-          ccache -z
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
-          ccache -s
+
+      - name: "Build eudsl-nbgen"
+        run: |
+
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
+          $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse
 
       - name: "Build eudsl-py"
         if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
 
-          # prevent OOM on free GHA
-          export DISABLE_COMPILE_OPT="${{ inputs.debug_with_tmate }}"
-          ccache -z
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
-          ccache -s
+
+      # just to/make sure total build continues to work
+      - name: "Build all of eudsl"
+        run: |
+
+          pip install -r requirements.txt
+          cmake -B $PWD/eudsl-build -S $PWD \
+            -DCMAKE_PREFIX_PATH=$PWD/llvm-install \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-install
+          cmake --build "$PWD/eudsl-build" --target install
 
       - name: "Save cache"
         uses: actions/cache/save@v3
@@ -187,7 +210,7 @@ jobs:
 
     runs-on: ${{ matrix.runs-on }}
 
-    name: "Test eudsl ${{ matrix.name }}"
+    name: "Test eudsl-tblgen ${{ matrix.name }}"
 
     defaults:
       run:
@@ -291,10 +314,22 @@ jobs:
       id-token: write
       contents: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: ["ubuntu-22.04"]
+        include: [
+          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64"},
+          {runs-on: "ubuntu-22.04", name: "macos_arm64"},
+          {runs-on: "ubuntu-22.04", name: "macos_x86_64"},
+          {runs-on: "ubuntu-22.04", name: "windows_x86_64"},
+        ]
+
     steps:
 
       - uses: actions/download-artifact@v4
         with:
+          name: eudsl_${{ matrix.name }}_artifact
           path: wheelhouse
 
       - name: Release current commit

CMakeLists.txt

@@ -0,0 +1,75 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Copyright (c) 2024.
+
+cmake_minimum_required(VERSION 3.29)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(LLVM_SUBPROJECT_TITLE "EUDSL")
+set(EUDSL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  message("Building ${LLVM_SUBPROJECT_TITLE} as a standalone project.")
+  project(${LLVM_SUBPROJECT_TITLE} CXX C)
+  set(EUDSL_STANDALONE_BUILD ON)
+else()
+  enable_language(CXX C)
+  set(EUDSL_STANDALONE_BUILD OFF)
+endif()
+
+if(EUDSL_STANDALONE_BUILD)
+  find_package(LLVM REQUIRED CONFIG)
+  find_package(MLIR REQUIRED CONFIG)
+  find_package(Clang REQUIRED CONFIG PATHS "${LLVM_BINARY_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH)
+
+  message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+  message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
+  message(STATUS "Using ClangConfig.cmake in: ${Clang_DIR}")
+
+  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
+  set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
+  set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR})
+
+  list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
+  list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+  list(APPEND CMAKE_MODULE_PATH "${CLANG_CMAKE_DIR}")
+
+  include(TableGen)
+  include(AddLLVM)
+  include(AddMLIR)
+  include(AddClang)
+
+  set(MLIR_INCLUDE_DIR ${MLIR_INCLUDE_DIRS})
+else()
+  # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir
+  # but not everything so LLVM_BINARY_DIR isn't correct
+  string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}")
+  # Build via external projects mechanism
+  set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include)
+  set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include)
+  set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}")
+
+  set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir)
+  set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include)
+  set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include)
+  set(MLIR_INCLUDE_DIRS "${MLIR_INCLUDE_DIR};${MLIR_GENERATED_INCLUDE_DIR}")
+
+  set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang)
+  set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include)
+  set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include)
+  set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}")
+endif()
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${MLIR_INCLUDE_DIRS})
+include_directories(${CLANG_INCLUDE_DIRS})
+link_directories(${LLVM_BUILD_LIBRARY_DIR})
+add_definitions(${LLVM_DEFINITIONS})
+
+if(NOT TARGET LLVMSupport)
+  message(FATAL_ERROR "LLVMSupport not found")
+endif()
+
+add_subdirectory(projects)

README.md

@@ -7,6 +7,68 @@
 This repository contains the source code for `EUDSL`, a toolkit for the construction of 
 embedded DSLs, in arbitrary languages, for targeting [MLIR](https://mlir.llvm.org).
 
+FYI: this project is currently "alpha" quality.
+
+Currently, there are three components:
+
+1. [eudsl-tblgen](./projects/eudsl-tblgen): Python bindings to [LLVM's TableGen library](https://github.com/llvm/llvm-project/tree/659192b1843c4af180700783caca4cdc7afa3eab/llvm/lib/TableGen);
+2. [eudsl-nbgen](./projects/eudsl-nbgen): A source-to-source translator that translates MLIR headers[^1] into direct `nanobind` bindings;
+3. [eudsl-py](./projects/eudsl-py): Direct Python bindings to MLIR, generated using `eudsl-nbgen`;
+   * Currently only TableGen outputs (various `*.h.inc` files generated by `mlir-tblgen`) are automatically translated but the `eudsl-nbgen` tool can be manually run to translate any MLIR header (to varying degrees of success);
+   * See [projects/eudsl-py/tests](./projects/eudsl-py/tests) for a few examples of what these bindings look like.
+
 ## Getting started
 
-Coming soon...
+Python wheels of all the tools are available at the [`latest` release page](https://github.com/llvm/eudsl/releases/tag/latest).
+They are also `pip install`-able with .e.g
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ pip install eudsl-py -f https://github.com/llvm/eudsl/releases/expanded_assets/latest
+```
+
+`eudsl-py` has a slowly growing set of tests @ [projects/eudsl-py/tests](./projects/eudsl-py/tests).
+
+If you don't want to install locally, here is a [colab notebook minimal working example](https://colab.research.google.com/drive/1l-6rVnsUM3ypn7rKcaF_V6XVdopEM4Df?usp=sharing).
+
+## Developing/building
+
+**Strong recommendation**: check the CI scripts @ [.github/workflows](.github/workflows) - they do a fresh checkout and build on every commit and are written to be read by a non-CI expert.
+
+Firstly, you need a distribution of LLVM. You can build LLVM from source using our submodule by doing (on Mac/Linux or mingw):
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ ./build_tools/build_llvm.sh
+```
+
+Alternatively you can download a distribution for your platform from the [`llvm` release page](https://github.com/llvm/eudsl/releases/tag/llvm).
+
+Then each of the above tools can both be built as a conventional, standalone, CMake project and as a Python wheel.
+The wheel build looks something like:
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ export CMAKE_PREFIX_PATH=$PWD/llvm-install 
+$ pip wheel projects/eudsl-nbgen -w wheelhouse -v
+$ pip wheel projects/eudsl-py -w wheelhouse -v --find-links $PWD/wheelhouse
+```
+
+Note, the trailing `--find-links $PWD/wheelhouse` on `pip wheel projects/eudsl-py` is because `eudsl-nbgen` is a dependency of `eudsl-py` (that can be satisfied using the `eudsl-nbgen` wheel).
+
+If you want to build an individual tool via CMake you can do something like:
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ pip install -r requirements.txt
+$ export CMAKE_PREFIX_PATH=$PWD/llvm-install 
+$ cmake -B $PWD/eudsl-nbgen-build -S $PWD/projects/eudsl-nbgen -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-nbgen-install
+$ cmake --build "$PWD/eudsl-build" --target install
+```
+
+If you want to build all the tools at once using CMake you can use the root [`CMakeLists.txt`](./CMakeLists.txt).
+Note, in this case, `eudsl-nbgen` will automatically be built prior to `eudsl-py`.
+
+## Footnotes
+
+[^1]: Yes C++ headers...

projects/CMakeLists.txt

@@ -0,0 +1,10 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Copyright (c) 2024.
+
+if(NOT WIN32)
+  add_subdirectory(eudsl-py)
+endif()
+add_subdirectory(eudsl-nbgen)
+add_subdirectory(eudsl-tblgen)

projects/eudsl-nbgen/CMakeLists.txt

@@ -0,0 +1,85 @@
+cmake_minimum_required(VERSION 3.29)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(LLVM_SUBPROJECT_TITLE "EUDSL_NBGEN")
+set(EUDSL_NBGEN_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  message("Building ${LLVM_SUBPROJECT_TITLE} as a standalone project.")
+  project(${LLVM_SUBPROJECT_TITLE} CXX C)
+  set(EUDSL_NBGEN_STANDALONE_BUILD ON)
+else()
+  enable_language(CXX C)
+  set(EUDSL_NBGEN_STANDALONE_BUILD OFF)
+endif()
+
+if(EUDSL_NBGEN_STANDALONE_BUILD)
+  find_package(LLVM REQUIRED CONFIG)
+  find_package(Clang REQUIRED CONFIG PATHS "${LLVM_BINARY_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH)
+
+  message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+  message(STATUS "Using ClangConfig.cmake in: ${Clang_DIR}")
+
+  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
+  set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
+
+  list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+  list(APPEND CMAKE_MODULE_PATH "${CLANG_CMAKE_DIR}")
+
+  include(TableGen)
+  include(AddLLVM)
+  include(AddClang)
+  include(HandleLLVMOptions)
+endif()
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${CLANG_INCLUDE_DIRS})
+link_directories(${LLVM_BUILD_LIBRARY_DIR})
+add_definitions(${LLVM_DEFINITIONS})
+
+if(NOT TARGET LLVMSupport)
+  message(FATAL_ERROR "LLVMSupport not found")
+endif()
+
+# this will set the rpath of the exe to be `../lib`, where is where we deposit libLLVM.so below
+add_llvm_executable(eudsl-nbgen src/eudsl-nbgen.cpp)
+target_link_libraries(eudsl-nbgen
+  PRIVATE
+  clangAST
+  clangBasic
+  clangFrontend
+  clangSerialization
+  clangTooling
+)
+
+string(TOLOWER ${LLVM_SUBPROJECT_TITLE} EUDSL_NBGEN_INSTALL_DATADIR)
+if (NOT "$ENV{PIP_BUILD_TRACKER}" STREQUAL "")
+  # pip install
+  # actually installs to venv/bin
+  # https://github.com/scikit-build/scikit-build-core/blob/a887a9b6c057b4ce9d3cfd53ae24e73caf1395a2/docs/build.md?plain=1#L139-L148
+  install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/scripts")
+  if (NOT WIN32)
+    # this actually installs into venv/lib
+    install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/data/lib")
+  endif()
+else()
+  # pip cmake install
+  install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/bin")
+  if (NOT WIN32)
+    install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/lib")
+  endif()
+endif()
+
+install(
+  # the slash is load-bearing...
+  DIRECTORY src/
+  DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}"
+  FILES_MATCHING PATTERN "*\.py"
+)
+
+install(
+  FILES
+  cmake/eudsl_nbgen-config.cmake
+  DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/cmake"
+)