feat: update llama.cpp to 42e98813e

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- feat: update llama.cpp to ggml-org/llama.cpp@42e98813e
+
 ## [0.3.34]
 
 - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794

examples/low_level_api/low_level_api_chat_cpp.py

@@ -76,8 +76,14 @@ def __init__(self, params: GptParams) -> None:
         self.lparams.n_parts = self.params.n_parts
         self.lparams.seed = self.params.seed
         self.lparams.memory_f16 = self.params.memory_f16
-        self.lparams.use_mlock = self.params.use_mlock
-        self.lparams.use_mmap = self.params.use_mmap
+        if self.params.use_mmap and self.params.use_mlock:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+        elif self.params.use_mlock:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif self.params.use_mmap:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP
+        else:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
 
         self.model = llama_cpp.llama_model_load_from_file(
             self.params.model.encode("utf8"), self.lparams

examples/server/server.py

@@ -10299,6 +10299,7 @@ def __init__(
             llama_cpp.llama_sampler_chain_add(
                 self._sampler,
                 llama_cpp.llama_sampler_init_penalties(
+                    n_vocab,
                     64,
                     1.0,
                     frequency_penalty,
@@ -11030,7 +11031,9 @@ def _build_prompt_plan_locked(
                     "multiple videos require MTMD to report frame counts"
                 )
             input_text = mtmd_cpp.mtmd_input_text()
-            input_text.text = prompt.encode("utf-8")
+            input_text_bytes = prompt.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = False
             input_text.parse_special = True
             chunks = mtmd_cpp.mtmd_input_chunks_init()
@@ -11310,6 +11313,7 @@ def __init__(
                 vocab_only=vocab_only,
                 use_mmap=use_mmap,
                 use_mlock=use_mlock,
+                load_mtp=draft_model == "draft-mtp",
                 kv_overrides=kv_overrides,
             )
         )
@@ -11588,6 +11592,7 @@ def build_model_params(
         vocab_only: Optional[bool],
         use_mmap: Optional[bool],
         use_mlock: Optional[bool],
+        load_mtp: bool,
         kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]],
     ) -> Tuple[Any, Optional[Any], Optional[Any]]:
         model_params = llama_cpp.llama_model_default_params()
@@ -11609,10 +11614,17 @@ def build_model_params(
             model_params.tensor_split = tensor_split_ref
         if vocab_only is not None:
             model_params.vocab_only = vocab_only
-        if use_mmap is not None:
-            model_params.use_mmap = use_mmap
-        if use_mlock is not None:
-            model_params.use_mlock = use_mlock
+        model_params.load_mtp = load_mtp
+        if use_mlock and use_mmap is not False:
+            model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+        elif use_mlock:
+            model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif use_mmap is not None:
+            model_params.load_mode = (
+                llama_cpp.LLAMA_LOAD_MODE_MMAP
+                if use_mmap
+                else llama_cpp.LLAMA_LOAD_MODE_NONE
+            )
 
         kv_overrides_ref = None
         if kv_overrides is not None:

llama_cpp/_internals.py

@@ -276,6 +276,8 @@ def free_ctx():
             self.ctx = None
 
         self._exit_stack.callback(free_ctx)
+        # The native context must be freed before its model.
+        self.model._exit_stack.callback(self.close)
 
     def close(self):
         self._exit_stack.close()
@@ -784,12 +786,14 @@ def add_grammar_lazy_patterns(
 
     def add_penalties(
         self,
+        n_vocab: int,
         penalty_last_n: int,
         penalty_repeat: float,
         penalty_freq: float,
         penalty_present: float,
     ):
         sampler = llama_cpp.llama_sampler_init_penalties(
+            n_vocab,
             penalty_last_n,
             penalty_repeat,
             penalty_freq,
@@ -800,7 +804,6 @@ def add_penalties(
     def add_dry(
         self,
         model: LlamaModel,
-        n_ctx_train: int,
         dry_multiplier: float,
         dry_base: float,
         dry_allowed_length: int,
@@ -814,7 +817,6 @@ def add_dry(
 
         sampler = llama_cpp.llama_sampler_init_dry(
             model.vocab,
-            n_ctx_train,
             dry_multiplier,
             dry_base,
             dry_allowed_length,

llama_cpp/llama.py

@@ -244,8 +244,15 @@ def __init__(
             )  # keep a reference to the array so it is not gc'd
             self.model_params.tensor_split = self._c_tensor_split
         self.model_params.vocab_only = vocab_only
-        self.model_params.use_mmap = use_mmap if lora_path is None else False
-        self.model_params.use_mlock = use_mlock
+        use_mmap = use_mmap and lora_path is None
+        if use_mmap and use_mlock:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+        elif use_mlock:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
+        elif use_mmap:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP
+        else:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
 
         # kv_overrides is the original python dict
         self.kv_overrides = kv_overrides
@@ -737,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p):
             sampler.add_custom(apply_func)
 
         sampler.add_penalties(
-            # n_vocab=self._n_vocab,
+            n_vocab=self._n_vocab,
             # special_eos_id=self._token_eos,
             # linefeed_id=self._token_nl,
             penalty_last_n=self.last_n_tokens_size,
@@ -2142,8 +2149,16 @@ def __getstate__(self):
             main_gpu=self.model_params.main_gpu,
             tensor_split=self.tensor_split,
             vocab_only=self.model_params.vocab_only,
-            use_mmap=self.model_params.use_mmap,
-            use_mlock=self.model_params.use_mlock,
+            use_mmap=self.model_params.load_mode
+            in (
+                llama_cpp.LLAMA_LOAD_MODE_MMAP,
+                llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK,
+            ),
+            use_mlock=self.model_params.load_mode
+            in (
+                llama_cpp.LLAMA_LOAD_MODE_MLOCK,
+                llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK,
+            ),
             kv_overrides=self.kv_overrides,
             # Context Params
             seed=self._seed,

llama_cpp/llama_chat_format.py

@@ -2936,7 +2936,9 @@ def __call__(
 
             # Create input text structure
             input_text = self._mtmd_cpp.mtmd_input_text()
-            input_text.text = text.encode("utf-8")
+            input_text_bytes = text.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = True
             input_text.parse_special = True
 
@@ -3485,7 +3487,9 @@ def raise_exception(message: str):
                 bitmap_cleanup.append(bitmap)
 
             input_text = self._mtmd_cpp.mtmd_input_text()
-            input_text.text = text.encode("utf-8")
+            input_text_bytes = text.encode("utf-8")
+            input_text.text = input_text_bytes
+            input_text.text_len = len(input_text_bytes)
             input_text.add_special = True
             input_text.parse_special = True