Libcpp compilation integration

.gitignore

@@ -18,3 +18,7 @@ reports/
 
 #Ignore python cache
 __pycache__/
+
+#Ignore llvm external dependency needed for libcpp headers
+llvm-project/
+libcxx-*/

Toolchain-WASI.cmake

@@ -0,0 +1,27 @@
+# Toolchain-WASI.cmake
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR wasm32)
+
+set(CLANG_BIN "/home/lind/lind-wasm/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04/bin")
+set(CMAKE_C_COMPILER "${CLANG_BIN}/clang")
+set(CMAKE_CXX_COMPILER "${CLANG_BIN}/clang++")
+set(CMAKE_LINKER "${CLANG_BIN}/bin/wasm-ld")
+set(CMAKE_SYSROOT "/home/lind/lind-wasm/build/sysroot")
+
+set(CMAKE_C_COMPILER_TARGET wasm32-unknown-wasi)
+set(CMAKE_CXX_COMPILER_TARGET wasm32-unknown-wasi)
+
+set(CMAKE_C_FLAGS_INIT "-pthread -matomics -mbulk-memory -static -nostdlib -nodefaultlibs -fno-exceptions -fno-unwind-tables")
+set(CMAKE_CXX_FLAGS_INIT "-frtti -pthread -matomics -mbulk-memory -static -nostdlib -nodefaultlibs -fno-exceptions -stdlib=libc++ -fno-unwind-tables")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -nostdlib -nodefaultlibs")
+
+# Optional: disable rpath injection
+set(CMAKE_SKIP_RPATH ON)
+
+# These fix platform error
+set(LLVM_HOST_TRIPLE "wasm32-wasip1")
+set(LLVM_DEFAULT_TARGET_TRIPLE "wasm32-wasip1")
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

build-llvm.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -e
+
+export ROOT_DIR=$(pwd)
+export LLVM_SRC="$ROOT_DIR/llvm-project"
+export INSTALL_PREFIX="$ROOT_DIR/libcxx-wasi-install"
+
+mkdir -p libcxx-build
+
+cmake -B libcxx-build -S "$LLVM_SRC/runtimes" \
+  -DCMAKE_TOOLCHAIN_FILE="$ROOT_DIR/Toolchain-WASI.cmake" \
+  -DLLVM_PATH="$LLVM_SRC/llvm" \
+  -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+  -DLLVM_TARGETS_TO_BUILD="X86" \
+  -DLLVM_DEFAULT_TARGET_TRIPLE="x86_64-unknown-linux-gnu" \
+  -DLLVM_HOST_TRIPLE="wasm32-unknown-wasi" \
+  -DLIBCXX_ENABLE_SHARED=OFF \
+  -DLIBCXX_ENABLE_STATIC=ON \
+  -DLIBCXX_ENABLE_EXCEPTIONS=OFF \
+  -DLIBCXX_USE_COMPILER_RT=ON \
+  -DLIBCXX_ENABLE_RTTI=ON \
+  -DLIBCXXABI_ENABLE_SHARED=OFF \
+  -DLIBCXXABI_ENABLE_STATIC=ON \
+  -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF \
+  -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
+  -DLIBCXXABI_ENABLE_STATIC_UNWINDER=OFF \
+  -DLIBCXX_ENABLE_UNWIND_TABLES=OFF \
+  -DLIBCXX_ENABLE_TIME_ZONE_DATABASE=OFF \
+  -DLIBCXXABI_ENABLE_UNWIND_TABLES=OFF \
+  -DLIBCXXABI_USE_COMPILER_RT=ON \
+  -DLIBCXXABI_ENABLE_RTTI=ON \
+  -DLIBCXXABI_LIBCXX_PATH="$LLVM_SRC/libcxx" \
+  -DLIBCXX_HAS_MUSL_LIBC:BOOL=OFF \
+  -DLIBCXXABI_ENABLE_EXCEPTIONS:BOOL=OFF \
+  -DLIBCXXABI_USE_LLVM_UNWINDER:BOOL=OFF \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" \
+  -DCMAKE_CXX_COMPILER_WORKS=1 \
+  -DCMAKE_C_COMPILER_WORKS=1
+
+cmake --build libcxx-build --target install

cpp_compile.md

@@ -0,0 +1,163 @@
+# Start from commit [commit 337a15a](https://github.com/Lind-Project/lind-wasm/commit/337a15abc0d5a97a5050a46e50d1b69550181842) of libcpp-alice branch
+
+Usually I would document what I do in a linear, chronological manner, without a clear overarching narrative in mind, because I document as I explore each step in a project. But progress is non-linear, so sometimes this style of documenting is not… ideal. 
+
+For this documentation, I will take a structured narrative and do my best to explain everything. I wrote this for someone who hop on to the project with very little prior knowledge as to how or why things work in the way they do.
+
+## Overarching Goal
+To be able to compile .wasm binary (and the .cwasm ELF files as a by-product) with clang++ compiler inside the Lind sandbox. 
+Where we start with
+Alice's issue [#245](https://github.com/Lind-Project/lind-wasm/issues/245) describes a slightly outdated solution to the above problem, Alice later started a new branch (libcpp-alice) and basically formalized that solution. This documentation by Ren will attempt to explain her solution (grossly) and document what is exactly done to achieve the overarching goal. 
+
+## Existing problem
+Because we are really mixing the compilation environment for LLVM, Clang, and the binary developers will eventually want to compile from their own source code, we must introduce external dependencies, specifically LLVM ver 18 and its compiled binary. Such external dependencies, and other run-time compiled dependency files, are not tracked on git in any manner, thus the need of this documentation. What's worse is that, for LLVM, we need to modify its source code slightly. Which is 1. playing with fire and 2. totally not tracked by git.
+
+## What i did
+### Replicate the error
+Of course, the first step of dealing with a problem is to make sure we do have a problem. We start with what Nick points out in issue [#740](https://github.com/Lind-Project/lind-wasm/issues/740#issuecomment-3910086697): that some dummy .cpp file that has # include <algorithm> will fail to compile as the header libraries are just entirely missing, if a user attempts to compile .wasm binary. Thus, I simply created a new directory /trial and put a super simple dummy hello.cpp file there, which uses algorithm header:
+
+```trial/hello.cpp
+#include <algorithm>
+#include <vector>
+#include <iostream>
+
+int main() {
+    std::vector<int> v = {3, 1, 2};
+    std::sort(v.begin(), v.end());
+
+    for (int each : v){
+        std::cout<< each<<' ' ;
+    }
+    std::cout<<std::endl;
+    return 0;
+}
+```
+It also has a simple cout so that if we eventually execute a compiled binary, we would know it totally works.
+
+### Side note
+Compiling .wasm binary involves setting some pretty length and boring flags for clang++ command, the lind-wasm repo already provides a pretty convenient compile script in ``scripts/lind_compile`` which basically bundles up those flags for you. Here are a couple caveats:
+The script can only be used inside docker container
+The script call signature is iffy, you must call it as: ``./scripts/lind_compile main.cpp`` and cannot assume its global availability: ``./lind_compile main.cpp`` **will not do what you want**.
+The script is intended to be used on .c files only. Alice modded the script to make it work on .cpp files only in this branch. A simple future fix is needed to make it work for both .c and .cpp. It won't be hard, but will be a bit tedious. what alice did can be seen here
+
+
+So, the necessary steps here are: creating and stepping into a docker container, creating the dummy hello.cpp file (you can do this before stepping into the docker container step; it really doesn't matter), and attempt to compile with the script and just watch it fail:
+```bash
+lind@e9a40a72b750:~/lind-wasm$ scripts/lind_compile trial/hello.cpp -- -fno-exceptions
+/home/lind/lind-wasm/trial/hello.cpp:1:10: fatal error: 'algorithm' file not found
+    1 | #include <algorithm>
+      |          ^~~~~~~~~~~
+1 error generated.
+```
+
+This error is also recorded in issue [#740](https://github.com/Lind-Project/lind-wasm/issues/740#issuecomment-3910086697) by Nick, as mentioned above.
+
+## Docker Container 
+Here I must assume you have your SSH and IDE set up already. If not, refer to the first tab of this documentation file. And really just ask for help from other PHD students. We start with you already ssh connected to the server, and already git-cloned the project repo on libcpp-alice branch.
+
+```bash
+docker run --privileged --ipc=host --cap-add=SYS_PTRACE --name WHATEVER -it securesystemslab/lind-wasm-dev /bin/bash
+```
+Just remember to name your container whatever unique name you want. In the future, you can find it by simply running docker ps and whenever you exit and want to come back to it, simply:
+
+```bash
+docker exec -it WHATEVER /bin/bash
+```
+Here, obviously, we never really pull down a docker container – one of the many perks of working on a remote SSH. 
+
+Creating the dummy hello.cpp, and try to compile
+This step is pretty self-explanatory. Just remember that we want to compile while assuming everything requires absolute path, so:
+
+```bash
+/home/lind/lind-wasm/scripts/lind_compile /home/lind/lind-wasm/trial/hello.cpp
+```
+ Then you will quickly see the error described in issue [#740](https://github.com/Lind-Project/lind-wasm/issues/740#issuecomment-3910086697).
+
+## Retrieving external dependencies
+If you look at the .gitignore, here and here we see clang+llvm ignored; I retrieved this via curl command from llvm; and you have llvm-project ignored, which is just outright the entire llvm repo, so git clone. We really don have a good reason behind the specific versions we picked for both of them, it is just a hassle must-do. The other suspicious looking gitignored items are runtime generated and I will go over them soon. For now, make sure you are in ``/lind-wasm`` directory. Run:
+
+```bash
+git clone --branch release/18.x --single-branch https://github.com/llvm/llvm-project.git
+```
+The above step is also mentioned in issue [#245](https://github.com/Lind-Project/lind-wasm/issues/245); and it will create ``/llvm-project`` dir for us, resolving one of the two missing dependencies issue. Again, specifically ver 18 is used. (contrary to the issue [#245](https://github.com/Lind-Project/lind-wasm/issues/245) which claims ver 16)
+
+Next, clang+llvm compiled library, this is trickier:
+
+```bash
+mkdir -p ~/tools/llvm18 && cd ~/tools/llvm18
+curl -L -o llvm18.tar.xz "https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz"
+tar -xf llvm18.tar.xz
+```
+Where specifically the compiled 18.1.8 version is used. Again, don ask me why. I have no answer to that.
+
+Now, both external dependencies are retrieved.
+
+## Modification of LLVM source code
+
+This modification, according to Alice, is developed retroactively after encountering compilation errors regarding return type of a certain util function. I am not sure how much "move fast and break things" we have wreaked by doing it. Modify llvm-project/libcxx/src/filesystem/time_utils.h
+and somewhere near line 266 (exactly this line at the moment) you should see the definition of convert_to_timespec() function, its return statement should be modified by adding reinterpret_cast<long*>  and you should end up with something like this:
+
+```/home/lind/lind-wasm/llvm-project/libcxx/src/filesystem/time_utils.h
+
+return set_times_checked(reinterpret_cast<long*>(&dest.tv_sec),
+                         reinterpret_cast<long*>(&dest.tv_nsec),
+                         tp);
+```
+
+And, contrary to issue #245, we need no more modifications on other external library source code. (Such as on clang+llvm's xlocale.h)
+
+## Call build script and migrate .a archives
+Now, with both our external dependencies in good shape, we can finally use ./build-llvm.sh to compile the libcxx-wasi library. Once you call this script, /libcxx-wasi-install dir is created. Next, we need to manually copy over the generated archives into sysroot:
+
+```bash
+cp -r /home/lind/lind-wasm/libcxx-wasi-install/include/c++ \
+      /home/lind/lind-wasm/build/sysroot/include/wasm32-wasi/
+
+cp /home/lind/lind-wasm/libcxx-wasi-install/lib/libc++.a \
+   /home/lind/lind-wasm/libcxx-wasi-install/lib/libc++abi.a \
+   /home/lind/lind-wasm/build/sysroot/lib/wasm32-wasi/
+
+```
+
+### Side note
+We are missing a libunwind.a archive; it is, for native cpp binary, needed to handle throw-except syntax. I tried to modify our CMake script and the build script to have this archive generated and then linked against the compilation process, and discovered that for .wasm binary, libunwind is not the correct dependency used to provide that syntax support. Currently I have no solution to it.
+
+## Last step: test compile and it should work now
+At this point, we have all we need to make the .wasm compilation work. Manually setting the correct clang++ flags is too much work, and luckily we have a scripts/lind_compile script which, originally designed for .c compilation, is almost entirely reusable directly for our .cpp compilation. Again, Alice already made the necessary changes to it in her commit to repurpose it for .cpp compilation only, and so we only need to use it. One more thing: remember we cannot support throw-exception? We do need some manual flag-setting to suppress that part. Luckily our simple dummy program does not need the throw-except syntax anyways. now make sure your are in lind-wasm dir (or just use absolute path for bash script below if you are not – by this point you should be really familiar with the project file hierarchy already.)
+
+```bash
+scripts/lind_compile trial/hello.cpp -- -fno-exceptions
+```
+note you can also additionally add -fno-rtti flag to save memory and speed up the compilation a bit more, but the compilation is quite slow regardless (takes ~1 minute)
+
+And you should now have compiled result file:
+
+trial/hello.cpp.wasm
+
+### side note
+
+the vanilla version of compiling will still fail:
+
+```bash
+lind@e9a40a72b750:~/lind-wasm$ scripts/lind_compile trial/hello.cpp 
+wasm-ld: warning: function signature mismatch: main
+>>> defined as (i32, i32, i32) -> i32 in /home/lind/lind-wasm/build/sysroot/lib/wasm32-wasi/crt1.o
+>>> defined as (i32, i32) -> i32 in /tmp/hello-d53fbc.o
+
+wasm-ld: error: /tmp/hello-d53fbc.o: undefined symbol: __cxa_allocate_exception
+wasm-ld: error: /tmp/hello-d53fbc.o: undefined symbol: __cxa_throw
+wasm-ld: error: /tmp/hello-d53fbc.o: undefined symbol: __cxa_allocate_exception
+wasm-ld: error: /tmp/hello-d53fbc.o: undefined symbol: __cxa_throw
+clang++: error: linker command failed with exit code 1 (use -v to see invocation)
+```
+
+which is a result of missing libunwind.a archive support. I looked into it and it seems it cannot be simply fixed by tweaking the compile flags in the Toolchain-WASI.cmake. This is something to be worked on in the future
+
+**And that is the entire workflow to get .wasm binary compiled!**
+
+–TODO for ren: document the exception error and explain overall why things are done the way they are under issue [#795](https://github.com/Lind-Project/lind-wasm/issues/795)
+
+
+
+
+

scripts/lind_compile

@@ -197,7 +197,7 @@ OUT_CWASM=""
 
 case "${MODE}" in
   full|"compile-only")
-    if [[ "${EXT}" != "c" ]]; then
+    if [[ "${EXT}" != "cpp" ]]; then
       echo "error: ${MODE} mode expects a .c file, got: ${SRC}" >&2
       usage
       exit 2
@@ -229,7 +229,7 @@ case "${MODE}" in
 esac
 
 # --- tool paths & quick checks (anchored to repo) ---
-CLANG_BIN="clang"  # must be on PATH
+CLANG_BIN="clang++"  # must be on PATH
 WASM_OPT_BIN="${REPO_ROOT}/tools/binaryen/bin/wasm-opt"
 LINDBOOT_BIN="${REPO_ROOT}/build/lind-boot"
 LINDFS_ROOT="${REPO_ROOT}/lindfs/"
@@ -261,7 +261,7 @@ if [[ "${PRINT_ARGS}" == "true" && ( "${MODE}" == "full" || "${MODE}" == "compil
   debug_mandatory_clang_flags=(
     "--sysroot=${SYSROOT}"
     --target=wasm32-unknown-wasi
-    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low,--export=__tls_base
+    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low
   )
   debug_default_clang_flags=(
     -pthread
@@ -290,7 +290,8 @@ if [[ "${PRINT_ARGS}" == "true" && ( "${MODE}" == "full" || "${MODE}" == "compil
   debug_mandatory_clang_flags=(
     "--sysroot=${SYSROOT}"
     --target=wasm32-unknown-wasi
-    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low,--export=__tls_base
+    -isystem $SYSROOT/include/wasm32-wasi/c++/v1
+    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low
   )
   debug_default_clang_flags=(
     -pthread
@@ -319,7 +320,8 @@ do_compile() {
   local -a mandatory_clang_flags=(
     "--sysroot=${SYSROOT}"
     --target=wasm32-unknown-wasi
-    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low,--export=__tls_base
+    -isystem $SYSROOT/include/wasm32-wasi/c++/v1
+    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export="__stack_pointer",--export=__stack_low
   )
   local -a default_clang_flags=(
     -pthread
@@ -387,4 +389,4 @@ case "${MODE}" in
   "precompile-only")
     echo "OK: ${OUT_CWASM}"
     ;;
-esac
+esac

trial/hello.cpp

@@ -0,0 +1,14 @@
+#include <algorithm>
+#include <vector>
+#include <iostream>
+
+int main() {
+    std::vector<int> v = {3, 1, 2};
+    std::sort(v.begin(), v.end());
+
+    for (int each : v){
+        std::cout<< each<<' ' ;
+    }
+    std::cout<<std::endl;
+    return 0;
+}