feat: enable mtmd for emscripten

Author: abetlen

.github/workflows/build-and-release.yaml

@@ -159,8 +159,8 @@ jobs:
           CIBW_BUILD_VERBOSITY: "1"
           CIBW_REPAIR_WHEEL_COMMAND: ""
           CIBW_BEFORE_TEST: "curl -L --fail --retry 3 -o /tmp/stories260K.gguf https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf"
-          CIBW_TEST_COMMAND: "python -c \"from llama_cpp import Llama; llm = Llama(model_path='/tmp/stories260K.gguf', n_ctx=64, n_batch=8, n_threads=1, verbose=False); print('loaded', llm.n_vocab(), llm.n_ctx()); print('generated', llm('Once upon a', max_tokens=1, temperature=0)['choices'][0]['text'])\""
-          CMAKE_ARGS: "-DLLAVA_BUILD=OFF -DLLAMA_WASM_MEM64=OFF -DEMSCRIPTEN_SYSTEM_PROCESSOR=wasm32 -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DGGML_METAL=OFF -DGGML_BLAS=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_VULKAN=OFF -DGGML_OPENCL=OFF -DGGML_RPC=OFF -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_BUILD_SERVER=OFF"
+          CIBW_TEST_COMMAND: "python -c \"import llama_cpp.mtmd_cpp as mtmd; from llama_cpp import Llama; print('mtmd marker', mtmd.mtmd_default_marker().decode()); llm = Llama(model_path='/tmp/stories260K.gguf', n_ctx=64, n_batch=8, n_threads=1, verbose=False); print('loaded', llm.n_vocab(), llm.n_ctx()); print('generated', llm('Once upon a', max_tokens=1, temperature=0)['choices'][0]['text'])\""
+          CMAKE_ARGS: "-DLLAMA_WASM_MEM64=OFF -DEMSCRIPTEN_SYSTEM_PROCESSOR=wasm32 -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DGGML_METAL=OFF -DGGML_BLAS=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_VULKAN=OFF -DGGML_OPENCL=OFF -DGGML_RPC=OFF -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_BUILD_SERVER=OFF"
         with:
           output-dir: wheelhouse
 

CMakeLists.txt

@@ -80,7 +80,6 @@ if (LLAMA_BUILD)
             set(CMAKE_SYSTEM_PROCESSOR wasm32 CACHE STRING "Target processor" FORCE)
         endif()
 
-        set(LLAVA_BUILD OFF CACHE BOOL "Build llava shared library and install alongside python package" FORCE)
         set(LLAMA_WASM_MEM64 OFF CACHE BOOL "llama.cpp: enable wasm64 memory" FORCE)
         set(GGML_NATIVE OFF CACHE BOOL "ggml: enable -march=native" FORCE)
         set(GGML_OPENMP OFF CACHE BOOL "ggml: use OpenMP" FORCE)

llama_cpp/_ctypes_extensions.py

@@ -79,18 +79,21 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
                 else f"{lib_dir}{os.pathsep}{ld_library_path}"
             )
 
-        if lib_base_name == "llama":
-            for dependency in ("ggml-base", "ggml-cpu", "ggml"):
-                dependency_path = (
-                    base_path / f"lib{dependency}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}"
-                )
-                if dependency_path.exists():
-                    try:
-                        ctypes.CDLL(str(dependency_path), **cdll_args)  # type: ignore
-                    except Exception as e:
-                        raise RuntimeError(
-                            f"Failed to load shared library '{dependency_path}': {e}"
-                        )
+        emscripten_dependencies = {
+            "llama": ("ggml-base", "ggml-cpu", "ggml"),
+            "mtmd": ("ggml-base", "ggml-cpu", "ggml", "llama"),
+        }
+        for dependency in emscripten_dependencies.get(lib_base_name, ()):
+            dependency_path = (
+                base_path / f"lib{dependency}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}"
+            )
+            if dependency_path.exists():
+                try:
+                    ctypes.CDLL(str(dependency_path), **cdll_args)  # type: ignore
+                except Exception as e:
+                    raise RuntimeError(
+                        f"Failed to load shared library '{dependency_path}': {e}"
+                    )
 
     # Try to load the shared library, handling potential errors
     for lib_path in lib_paths: