feat: update llama.cpp to d2a818231

Author: abetlen

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- feat: update llama.cpp to ggml-org/llama.cpp@d2a818231
+
 ## [0.3.34]
 
 - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794

examples/low_level_api/low_level_api_chat_cpp.py

@@ -76,8 +76,12 @@ def __init__(self, params: GptParams) -> None:
         self.lparams.n_parts = self.params.n_parts
         self.lparams.seed = self.params.seed
         self.lparams.memory_f16 = self.params.memory_f16
-        self.lparams.use_mlock = self.params.use_mlock
-        self.lparams.use_mmap = self.params.use_mmap
+        if self.params.use_mlock:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif self.params.use_mmap:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP
+        else:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
 
         self.model = llama_cpp.llama_model_load_from_file(
             self.params.model.encode("utf8"), self.lparams

examples/server/server.py

@@ -11030,7 +11030,9 @@ def _build_prompt_plan_locked(
                     "multiple videos require MTMD to report frame counts"
                 )
             input_text = mtmd_cpp.mtmd_input_text()
-            input_text.text = prompt.encode("utf-8")
+            input_text_bytes = prompt.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = False
             input_text.parse_special = True
             chunks = mtmd_cpp.mtmd_input_chunks_init()
@@ -11609,10 +11611,14 @@ def build_model_params(
             model_params.tensor_split = tensor_split_ref
         if vocab_only is not None:
             model_params.vocab_only = vocab_only
-        if use_mmap is not None:
-            model_params.use_mmap = use_mmap
-        if use_mlock is not None:
-            model_params.use_mlock = use_mlock
+        if use_mlock:
+            model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif use_mmap is not None:
+            model_params.load_mode = (
+                llama_cpp.LLAMA_LOAD_MODE_MMAP
+                if use_mmap
+                else llama_cpp.LLAMA_LOAD_MODE_NONE
+            )
 
         kv_overrides_ref = None
         if kv_overrides is not None:

llama_cpp/llama.py

@@ -244,8 +244,14 @@ def __init__(
             )  # keep a reference to the array so it is not gc'd
             self.model_params.tensor_split = self._c_tensor_split
         self.model_params.vocab_only = vocab_only
-        self.model_params.use_mmap = use_mmap if lora_path is None else False
-        self.model_params.use_mlock = use_mlock
+        if lora_path is not None:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
+        elif use_mlock:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif use_mmap:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP
+        else:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
 
         # kv_overrides is the original python dict
         self.kv_overrides = kv_overrides
@@ -2142,8 +2148,9 @@ def __getstate__(self):
             main_gpu=self.model_params.main_gpu,
             tensor_split=self.tensor_split,
             vocab_only=self.model_params.vocab_only,
-            use_mmap=self.model_params.use_mmap,
-            use_mlock=self.model_params.use_mlock,
+            use_mmap=self.model_params.load_mode
+            in (llama_cpp.LLAMA_LOAD_MODE_MMAP, llama_cpp.LLAMA_LOAD_MODE_MLOCK),
+            use_mlock=self.model_params.load_mode == llama_cpp.LLAMA_LOAD_MODE_MLOCK,
             kv_overrides=self.kv_overrides,
             # Context Params
             seed=self._seed,

llama_cpp/llama_chat_format.py

@@ -2936,7 +2936,9 @@ def __call__(
 
             # Create input text structure
             input_text = self._mtmd_cpp.mtmd_input_text()
-            input_text.text = text.encode("utf-8")
+            input_text_bytes = text.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = True
             input_text.parse_special = True
 
@@ -3485,7 +3487,9 @@ def raise_exception(message: str):
                 bitmap_cleanup.append(bitmap)
 
             input_text = self._mtmd_cpp.mtmd_input_text()
-            input_text.text = text.encode("utf-8")
+            input_text_bytes = text.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = True
             input_text.parse_special = True
 

llama_cpp/llama_cpp.py

@@ -520,6 +520,18 @@ def _warn_deprecated(symbol: str, hint: str) -> None:
 LLAMA_SPLIT_MODE_TENSOR = 3
 
 
+# enum llama_load_mode {
+#     LLAMA_LOAD_MODE_NONE      = 0, // no special loading mode
+#     LLAMA_LOAD_MODE_MMAP      = 1, // memory map the model
+#     LLAMA_LOAD_MODE_MLOCK     = 2, // mmap + force system to keep model in RAM rather than swapping or compressing
+#     LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available
+# };
+LLAMA_LOAD_MODE_NONE = 0
+LLAMA_LOAD_MODE_MMAP = 1
+LLAMA_LOAD_MODE_MLOCK = 2
+LLAMA_LOAD_MODE_DIRECT_IO = 3
+
+
 # enum llama_context_type {
 #     LLAMA_CONTEXT_TYPE_DEFAULT = 0,
 #     LLAMA_CONTEXT_TYPE_MTP     = 1,
@@ -789,8 +801,9 @@ class llama_model_imatrix_data(ctypes.Structure):
 #     // NULL-terminated list of buffer types to use for tensors that match a pattern
 #     const struct llama_model_tensor_buft_override * tensor_buft_overrides;
 
-#     int32_t n_gpu_layers; // number of layers to store in VRAM
+#     int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers
 #     enum llama_split_mode split_mode; // how to split the model across multiple GPUs
+#     enum llama_load_mode  load_mode;  // how to load the model
 
 #     // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
 #     int32_t main_gpu;
@@ -812,9 +825,6 @@ class llama_model_imatrix_data(ctypes.Structure):
 
 #     // Keep the booleans together to avoid misalignment during copy-by-value.
 #     bool vocab_only;      // only load the vocabulary, no weights
-#     bool use_mmap;        // use mmap if possible
-#     bool use_direct_io;   // use direct io, takes precedence over use_mmap when supported
-#     bool use_mlock;       // force system to keep model in RAM
 #     bool check_tensors;   // validate model tensor data
 #     bool use_extra_bufts; // use extra buffer types (used for weight repacking)
 #     bool no_host;         // bypass host buffer allowing extra buffers to be used
@@ -826,17 +836,15 @@ class llama_model_params(ctypes.Structure):
     Attributes:
         devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
         tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern
-        n_gpu_layers (int): number of layers to store in VRAM
+        n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers
         split_mode (int): how to split the model across multiple GPUs
+        load_mode (int): how to load the model
         main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
         tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
         progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.
         progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback
         kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data
         vocab_only (bool): only load the vocabulary, no weights
-        use_mmap (bool): use mmap if possible
-        use_direct_io (bool): use direct io, takes precedence over use_mmap when supported
-        use_mlock (bool): force system to keep model in RAM
         check_tensors (bool): validate model tensor data
         use_extra_bufts (bool): use extra buffer types (used for weight repacking)
         no_host (bool): bypass host buffer allowing extra buffers to be used
@@ -849,15 +857,13 @@ class llama_model_params(ctypes.Structure):
         ]  # NOTE: unused
         n_gpu_layers: int
         split_mode: int
+        load_mode: int
         main_gpu: int
         tensor_split: CtypesArray[ctypes.c_float]
         progress_callback: Callable[[float, ctypes.c_void_p], bool]
         progress_callback_user_data: ctypes.c_void_p
         kv_overrides: CtypesArray[llama_model_kv_override]
         vocab_only: bool
-        use_mmap: bool
-        use_direct_io: bool
-        use_mlock: bool
         check_tensors: bool
         use_extra_bufts: bool
         no_host: bool
@@ -868,15 +874,13 @@ class llama_model_params(ctypes.Structure):
         ("tensor_buft_overrides", ctypes.c_void_p),  # NOTE: unused
         ("n_gpu_layers", ctypes.c_int32),
         ("split_mode", ctypes.c_int),
+        ("load_mode", ctypes.c_int),
         ("main_gpu", ctypes.c_int32),
         ("tensor_split", ctypes.POINTER(ctypes.c_float)),
         ("progress_callback", llama_progress_callback),
         ("progress_callback_user_data", ctypes.c_void_p),
         ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
         ("vocab_only", ctypes.c_bool),
-        ("use_mmap", ctypes.c_bool),
-        ("use_direct_io", ctypes.c_bool),
-        ("use_mlock", ctypes.c_bool),
         ("check_tensors", ctypes.c_bool),
         ("use_extra_bufts", ctypes.c_bool),
         ("no_host", ctypes.c_bool),
@@ -1276,6 +1280,20 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]:
     ...
 
 
+# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
+@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p)
+def llama_load_mode_name(load_mode: int, /) -> Optional[bytes]:
+    """Get the model load mode name."""
+    ...
+
+
+# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
+@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int)
+def llama_load_mode_from_str(value: bytes, /) -> int:
+    """Get the model load mode from a string."""
+    ...
+
+
 # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
 # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);
 @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p)

llama_cpp/mtmd_cpp.py

@@ -133,8 +133,15 @@ class mtmd_context_params(Structure):
 class mtmd_input_text(Structure):
     """Text input passed to `mtmd_tokenize`."""
 
+    if TYPE_CHECKING:
+        text: Optional[bytes]
+        text_len: int
+        add_special: bool
+        parse_special: bool
+
     _fields_ = [
         ("text", c_char_p),
+        ("text_len", c_size_t),
         ("add_special", c_bool),
         ("parse_special", c_bool),
     ]

tests/test_llama.py

@@ -103,8 +103,12 @@ def test_real_model(llama_cpp_model_path):
     assert os.path.exists(llama_cpp_model_path)
 
     params = llama_cpp.llama_model_default_params()
-    params.use_mmap = llama_cpp.llama_supports_mmap()
-    params.use_mlock = llama_cpp.llama_supports_mlock()
+    if llama_cpp.llama_supports_mlock():
+        params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+    elif llama_cpp.llama_supports_mmap():
+        params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP
+    else:
+        params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
     params.check_tensors = False
 
     model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params)