root CMakeLists.txt

.github/workflows/build_test_release_eudsl.yml

@@ -108,7 +108,6 @@ jobs:
             echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
             echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
             echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
-            echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV
             echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV
           else
             echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
@@ -131,27 +130,47 @@ jobs:
             echo MACOSX_DEPLOYMENT_TARGET=13.7 >> $GITHUB_ENV
           fi
           
+          # prevent OOM on free GHA
+          echo "EUDSLPY_DISABLE_COMPILE_OPT=${{ inputs.eudslpy_disable_compile_opt == 'true' && 'ON' || 'OFF' }}" >> $GITHUB_ENV
+          
           pip install cibuildwheel
 
       - name: "Build eudsl-tblgen"
         run: |
           
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
 
       - name: "Build eudsl-nbgen"
-        if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse
 
       - name: "Build eudsl-py"
         if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           
-          # prevent OOM on free GHA
-          export EUDSLPY_DISABLE_COMPILE_OPT="${{ inputs.eudslpy_disable_compile_opt == 'true' && 'ON' || 'OFF' }}"
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            export CCACHE_DIR=/host/$CCACHE_DIR
+          fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
 
+      # just to/make sure total build continues to work
+      - name: "Build all of eudsl"
+        run: |
+          
+          pip install -r requirements.txt
+          cmake -B $PWD/eudsl-build -S $PWD \
+            -DCMAKE_PREFIX_PATH=$PWD/llvm-install \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-install
+          cmake --build "$PWD/eudsl-build" --target install
+
       - name: "Save cache"
         uses: actions/cache/save@v3
         if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}

CMakeLists.txt

@@ -0,0 +1,75 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Copyright (c) 2024.
+
+cmake_minimum_required(VERSION 3.29)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(LLVM_SUBPROJECT_TITLE "EUDSL")
+set(EUDSL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  message("Building ${LLVM_SUBPROJECT_TITLE} as a standalone project.")
+  project(${LLVM_SUBPROJECT_TITLE} CXX C)
+  set(EUDSL_STANDALONE_BUILD ON)
+else()
+  enable_language(CXX C)
+  set(EUDSL_STANDALONE_BUILD OFF)
+endif()
+
+if(EUDSL_STANDALONE_BUILD)
+  find_package(LLVM REQUIRED CONFIG)
+  find_package(MLIR REQUIRED CONFIG)
+  find_package(Clang REQUIRED CONFIG PATHS "${LLVM_BINARY_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH)
+
+  message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+  message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
+  message(STATUS "Using ClangConfig.cmake in: ${Clang_DIR}")
+
+  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
+  set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
+  set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR})
+
+  list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
+  list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+  list(APPEND CMAKE_MODULE_PATH "${CLANG_CMAKE_DIR}")
+
+  include(TableGen)
+  include(AddLLVM)
+  include(AddMLIR)
+  include(AddClang)
+
+  set(MLIR_INCLUDE_DIR ${MLIR_INCLUDE_DIRS})
+else()
+  # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir
+  # but not everything so LLVM_BINARY_DIR isn't correct
+  string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}")
+  # Build via external projects mechanism
+  set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include)
+  set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include)
+  set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}")
+
+  set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir)
+  set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include)
+  set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include)
+  set(MLIR_INCLUDE_DIRS "${MLIR_INCLUDE_DIR};${MLIR_GENERATED_INCLUDE_DIR}")
+
+  set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang)
+  set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include)
+  set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include)
+  set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}")
+endif()
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${MLIR_INCLUDE_DIRS})
+include_directories(${CLANG_INCLUDE_DIRS})
+link_directories(${LLVM_BUILD_LIBRARY_DIR})
+add_definitions(${LLVM_DEFINITIONS})
+
+if(NOT TARGET LLVMSupport)
+  message(FATAL_ERROR "LLVMSupport not found")
+endif()
+
+add_subdirectory(projects)

README.md

@@ -7,6 +7,68 @@
 This repository contains the source code for `EUDSL`, a toolkit for the construction of 
 embedded DSLs, in arbitrary languages, for targeting [MLIR](https://mlir.llvm.org).
 
+FYI: this project is currently "alpha" quality.
+
+Currently there are three components:
+
+1. [eudsl-tblgen](./projects/eudsl-tblgen): Python bindings to [MLIR's TableGen library](https://github.com/llvm/llvm-project/tree/main/mlir/lib/TableGen);
+2. [eudsl-nbgen](./projects/eudsl-nbgen): A source-to-source translator that translates MLIR headers[^1] into direct `nanobind` bindings;
+3. [eudsl-py](./projects/eudsl-py): Direct Python bindings to MLIR, generated using `eudsl-nbgen`;
+   * Currently only TableGen outputs (various `*.h.inc` files generated by `mlir-tblgen`) are automatically translated but the `eudsl-nbgen` tool can be manually run to translate any MLIR header (to varying degrees of success);
+   * See [projects/eudsl-py/tests](./projects/eudsl-py/tests) for a few examples of what these bindings look like.
+
 ## Getting started
 
-Coming soon...
+Python wheels of all the tools are available at the [`latest` release page](https://github.com/llvm/eudsl/releases/tag/latest).
+They are also `pip install`-able with .e.g
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ pip install eudsl-py -f https://github.com/llvm/eudsl/releases/expanded_assets/latest
+```
+
+`eudsl-py` has a slowly growing set of tests @ [projects/eudsl-py/tests](./projects/eudsl-py/tests).
+
+If you don't want to install locally, here is a [colab notebook minimal working example](https://colab.research.google.com/drive/1l-6rVnsUM3ypn7rKcaF_V6XVdopEM4Df?usp=sharing).
+
+## Developing/building
+
+**Strong recommendation**: check the CI scripts @ [.github/workflows](.github/workflows) - they do a fresh checkout and build on every commit and are written to be read by a non-CI expert.
+
+Firstly, you need a distribution of LLVM. You can build LLVM from source using our submodule by doing (on Mac/Linux or mingw):
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ ./build_tools/build_llvm.sh
+```
+
+Alternatively you can download a distribution for your platform from the [`llvm` release page](https://github.com/llvm/eudsl/releases/tag/llvm).
+
+Then each of the above tools can both be built as a conventional, standalone, CMake project and as a Python wheel.
+The wheel build looks something like:
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ export CMAKE_PREFIX_PATH=$PWD/llvm-install 
+$ pip wheel projects/eudsl-nbgen -w wheelhouse -v
+$ pip wheel projects/eudsl-py -w wheelhouse -v --find-links $PWD/wheelhouse
+```
+
+Note, the trailing `--find-links $PWD/wheelhouse` on `pip wheel projects/eudsl-py` is because `eudsl-nbgen` is a dependency of `eudsl-py` (that can be satisfied using the `eudsl-nbgen` wheel).
+
+If you want to build an individual tool via CMake you can do something like:
+
+```shell
+$ cd <EUDSL_CHECKOUT_DIR>
+$ pip install -r requirements.txt
+$ export CMAKE_PREFIX_PATH=$PWD/llvm-install 
+$ cmake -B $PWD/eudsl-nbgen-build -S $PWD/projects/eudsl-nbgen -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-nbgen-install
+$ cmake --build "$PWD/eudsl-build" --target install
+```
+
+If you want to build all the tools at once using CMake you can use the root [`CMakeLists.txt`](./CMakeLists.txt).
+Note, in this case, `eudsl-nbgen` will automatically be built prior to `eudsl-py`.
+
+## Footnotes
+
+[^1]: Yes C++ headers...

projects/CMakeLists.txt

@@ -0,0 +1,10 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Copyright (c) 2024.
+
+if(NOT WIN32)
+  add_subdirectory(eudsl-py)
+endif()
+add_subdirectory(eudsl-nbgen)
+add_subdirectory(eudsl-tblgen)

projects/eudsl-nbgen/CMakeLists.txt

@@ -30,25 +30,7 @@ if(EUDSL_NBGEN_STANDALONE_BUILD)
   include(TableGen)
   include(AddLLVM)
   include(AddClang)
-  # TODO(max): probably don't need this anymore after landing the nanobind fix?
-  # technically we need this on windows too but our LLVM is compiled without exception handling
-  # and that breaks windows
-  if(NOT WIN32)
-    include(HandleLLVMOptions)
-  endif()
-else()
-  # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir
-  # but not everything so LLVM_BINARY_DIR isn't correct
-  string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}")
-  # Build via external projects mechanism
-  set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include)
-  set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include)
-  set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}")
-
-  set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang)
-  set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include)
-  set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include)
-  set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}")
+  include(HandleLLVMOptions)
 endif()
 
 include_directories(${LLVM_INCLUDE_DIRS})
@@ -72,13 +54,25 @@ target_link_libraries(eudsl-nbgen
 )
 
 string(TOLOWER ${LLVM_SUBPROJECT_TITLE} EUDSL_NBGEN_INSTALL_DATADIR)
-# https://github.com/scikit-build/scikit-build-core/blob/a887a9b6c057b4ce9d3cfd53ae24e73caf1395a2/docs/build.md?plain=1#L139-L148
-# actually installs to venv/bin
-install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/scripts")
-# this actually installs into venv/lib
-install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/data/lib")
+if (NOT "$ENV{PIP_BUILD_TRACKER}" STREQUAL "")
+  # pip install
+  # actually installs to venv/bin
+  # https://github.com/scikit-build/scikit-build-core/blob/a887a9b6c057b4ce9d3cfd53ae24e73caf1395a2/docs/build.md?plain=1#L139-L148
+  install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/scripts")
+  if (NOT WIN32)
+    # this actually installs into venv/lib
+    install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/data/lib")
+  endif()
+else()
+  # pip cmake install
+  install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/bin")
+  if (NOT WIN32)
+    install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/lib")
+  endif()
+endif()
 
 install(
+  # the slash is load-bearing...
   DIRECTORY src/
   DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}"
   FILES_MATCHING PATTERN "*\.py"