diff --git a/CHANGELOG.md b/CHANGELOG.md index 629b645a5a..76c1ac272f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@936918514 + ## [0.3.34] - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 20f7a158ac..43b0b31bb8 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,8 +76,14 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap + if self.params.use_mmap and self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif self.params.use_mmap: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE self.model = llama_cpp.llama_model_load_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/server/server.py b/examples/server/server.py index 72adc79059..452654ed19 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -10299,6 +10299,7 @@ def __init__( llama_cpp.llama_sampler_chain_add( self._sampler, llama_cpp.llama_sampler_init_penalties( + n_vocab, 64, 1.0, frequency_penalty, @@ -11030,7 +11031,9 @@ def _build_prompt_plan_locked( "multiple videos require MTMD to report frame counts" ) input_text = mtmd_cpp.mtmd_input_text() - input_text.text = prompt.encode("utf-8") + input_text_bytes = prompt.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = False input_text.parse_special = True chunks = mtmd_cpp.mtmd_input_chunks_init() @@ -11310,6 +11313,7 @@ def __init__( vocab_only=vocab_only, use_mmap=use_mmap, use_mlock=use_mlock, + load_mtp=draft_model == "draft-mtp", kv_overrides=kv_overrides, ) ) @@ -11588,6 +11592,7 @@ def build_model_params( vocab_only: Optional[bool], use_mmap: Optional[bool], use_mlock: Optional[bool], + load_mtp: bool, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]], ) -> Tuple[Any, Optional[Any], Optional[Any]]: model_params = llama_cpp.llama_model_default_params() @@ -11609,10 +11614,17 @@ def build_model_params( model_params.tensor_split = tensor_split_ref if vocab_only is not None: model_params.vocab_only = vocab_only - if use_mmap is not None: - model_params.use_mmap = use_mmap - if use_mlock is not None: - model_params.use_mlock = use_mlock + model_params.load_mtp = load_mtp + if use_mlock and use_mmap is not False: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap is not None: + model_params.load_mode = ( + llama_cpp.LLAMA_LOAD_MODE_MMAP + if use_mmap + else llama_cpp.LLAMA_LOAD_MODE_NONE + ) kv_overrides_ref = None if kv_overrides is not None: diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b0fe94d01f..b45d34b2c4 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -276,6 +276,8 @@ def free_ctx(): self.ctx = None self._exit_stack.callback(free_ctx) + # The native context must be freed before its model. + self.model._exit_stack.callback(self.close) def close(self): self._exit_stack.close() @@ -784,12 +786,14 @@ def add_grammar_lazy_patterns( def add_penalties( self, + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, ): sampler = llama_cpp.llama_sampler_init_penalties( + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, @@ -800,7 +804,6 @@ def add_penalties( def add_dry( self, model: LlamaModel, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -814,7 +817,6 @@ def add_dry( sampler = llama_cpp.llama_sampler_init_dry( model.vocab, - n_ctx_train, dry_multiplier, dry_base, dry_allowed_length, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b5bffd46b5..14e2f8500f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -244,8 +244,15 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap if lora_path is None else False - self.model_params.use_mlock = use_mlock + use_mmap = use_mmap and lora_path is None + if use_mmap and use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE # kv_overrides is the original python dict self.kv_overrides = kv_overrides @@ -737,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_custom(apply_func) sampler.add_penalties( - # n_vocab=self._n_vocab, + n_vocab=self._n_vocab, # special_eos_id=self._token_eos, # linefeed_id=self._token_nl, penalty_last_n=self.last_n_tokens_size, @@ -2142,8 +2149,16 @@ def __getstate__(self): main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_mlock=self.model_params.use_mlock, + use_mmap=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MMAP, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), + use_mlock=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MLOCK, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), kv_overrides=self.kv_overrides, # Context Params seed=self._seed, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0034bdae98..4f41c2eb75 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2936,7 +2936,9 @@ def __call__( # Create input text structure input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True @@ -3485,7 +3487,9 @@ def raise_exception(message: str): bitmap_cleanup.append(bitmap) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 64399fe317..362516b4d2 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -520,6 +520,20 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +LLAMA_LOAD_MODE_NONE = 0 +LLAMA_LOAD_MODE_MMAP = 1 +LLAMA_LOAD_MODE_MLOCK = 2 +LLAMA_LOAD_MODE_MMAP_MLOCK = 3 +LLAMA_LOAD_MODE_DIRECT_IO = 4 + + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -789,8 +803,9 @@ class llama_model_imatrix_data(ctypes.Structure): # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# int32_t n_gpu_layers; // number of layers to store in VRAM +# int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model # // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; @@ -812,13 +827,11 @@ class llama_model_imatrix_data(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -826,21 +839,20 @@ class llama_model_params(ctypes.Structure): Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern - n_gpu_layers (int): number of layers to store in VRAM + n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io (bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -849,38 +861,36 @@ class llama_model_params(ctypes.Structure): ] # NOTE: unused n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] @@ -1276,6 +1286,20 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int, /) -> Optional[bytes]: + """Get the model load mode name.""" + ... + + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(value: bytes, /) -> int: + """Get the model load mode from a string.""" + ... + + # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) @@ -3531,6 +3555,20 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [llama_vocab_p_ctypes, ctypes.POINTER(ctypes.c_int32)], + ctypes.POINTER(llama_token), +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: CtypesPointer[ctypes.c_int32], + /, +) -> Optional[CtypesPointer[llama_token]]: ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -4686,16 +4724,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# int32_t n_vocab, +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, @@ -4707,18 +4753,16 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 # LLAMA_API struct llama_sampler * llama_sampler_init_dry( # const struct llama_vocab * vocab, -# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, -# int32_t dry_penalty_last_n, +# int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) # const char ** seq_breakers, # size_t num_breakers); @ctypes_function( "llama_sampler_init_dry", [ llama_vocab_p_ctypes, - ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -4730,7 +4774,6 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_dry( vocab: llama_vocab_p, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 35357a3279..101a7db2bb 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -5,8 +5,10 @@ from ctypes import ( CFUNCTYPE, c_bool, + c_char, c_char_p, c_int, + c_int32, c_int64, c_uint8, c_uint32, @@ -68,6 +70,9 @@ mtmd_helper_video_p = NewType("mtmd_helper_video_p", int) mtmd_helper_video_p_ctypes = c_void_p +mtmd_helper_gen_audio_p = NewType("mtmd_helper_gen_audio_p", int) +mtmd_helper_gen_audio_p_ctypes = c_void_p + mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) mtmd_image_tokens_p_ctypes = c_void_p @@ -84,6 +89,16 @@ MTMD_INPUT_CHUNK_TYPE_TEXT = 0 MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 +MTMD_INPUT_CHUNK_TYPE_COUNT = 3 + +MTMD_GEN_AUDIO_TYPE_NONE = 0 +MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + +MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 +MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 + +MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM = 0 +MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV = 1 mtmd_progress_callback = CFUNCTYPE(c_bool, c_float, c_void_p) @@ -133,8 +148,15 @@ class mtmd_context_params(Structure): class mtmd_input_text(Structure): """Text input passed to `mtmd_tokenize`.""" + if TYPE_CHECKING: + text: Optional[bytes] + text_len: int + add_special: bool + parse_special: bool + _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] @@ -168,6 +190,97 @@ class mtmd_caps(Structure): ] +# struct mtmd_gen_audio_info { +# enum mtmd_gen_audio_type type; +# int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# }; +class mtmd_gen_audio_info(Structure): + if TYPE_CHECKING: + type: int + sample_rate: int + + _fields_ = [ + ("type", c_int), + ("sample_rate", c_int32), + ] + + +# struct mtmd_gen_inp { +# enum mtmd_gen_process_type type; +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# int32_t code0; // the sampled codebook 0 entry from backbone +# float * embd; // the hidden state from backbone, must have n_text_embd elements +# int32_t top_k; +# float top_p; +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# int32_t * codes; +# size_t n_codes; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_inp(Structure): + if TYPE_CHECKING: + type: int + code0: int + embd: Optional["_Pointer[c_float]"] + top_k: int + top_p: float + codes: Optional["_Pointer[c_int32]"] + n_codes: int + state_data: Optional["_Pointer[c_char]"] + state_size: int + + _fields_ = [ + ("type", c_int), + ("code0", c_int32), + ("embd", POINTER(c_float)), + ("top_k", c_int32), + ("top_p", c_float), + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("state_data", POINTER(c_char)), + ("state_size", c_size_t), + ] + + +# struct mtmd_gen_out { +# // note: output memory is allocated by the context, valid until next process() call +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# const int32_t * codes; +# size_t n_codes; +# const float * embd; // the generated hidden state, to be fed back to backbone +# // it must have n_text_embd elements +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# const float * audio; +# size_t n_samples; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_out(Structure): + if TYPE_CHECKING: + codes: Optional["_Pointer[c_int32]"] + n_codes: int + embd: Optional["_Pointer[c_float]"] + audio: Optional["_Pointer[c_float]"] + n_samples: int + state_data: Optional["_Pointer[c_char]"] + state_size: int + + _fields_ = [ + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("embd", POINTER(c_float)), + ("audio", POINTER(c_float)), + ("n_samples", c_size_t), + ("state_data", POINTER(c_char)), + ("state_size", c_size_t), + ] + + mtmd_bitmap_lazy_callback = CFUNCTYPE( c_int, c_size_t, @@ -228,6 +341,43 @@ class mtmd_helper_video_init_params(Structure): ] +# struct mtmd_helper_gen_audio_inp { +# llama_seq_id seq_id; +# +# const char * prompt; +# size_t prompt_len; +# +# mtmd_bitmap * speaker_ref; // optional, can be NULL +# const char * lang; // optional, can be NULL +# +# int32_t top_k; +# float top_p; +# +# enum mtmd_helper_gen_audio_outtype out_type; +# }; +class mtmd_helper_gen_audio_inp(Structure): + if TYPE_CHECKING: + seq_id: int + prompt: Optional[bytes] + prompt_len: int + speaker_ref: Optional[mtmd_bitmap_p] + lang: Optional[bytes] + top_k: int + top_p: float + out_type: int + + _fields_ = [ + ("seq_id", llama_cpp.llama_seq_id), + ("prompt", c_char_p), + ("prompt_len", c_size_t), + ("speaker_ref", mtmd_bitmap_p_ctypes), + ("lang", c_char_p), + ("top_k", c_int32), + ("top_p", c_float), + ("out_type", c_int), + ] + + ################################################ # mtmd.h functions ################################################ @@ -547,6 +697,44 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p, /): ... +# // save/load an input chunk to/from a buffer (useful for KV save/load) +# // important: only chunk's metadata will be saved, the actual image/audio data will not be saved +# // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() +# // out_buf can be nullptr (to query expected_out_len) +# // returns 0 on success, non-zero on failure +# MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len); +@ctypes_function( + "mtmd_input_chunk_save", + [mtmd_input_chunk_p_ctypes, POINTER(c_char), c_size_t, POINTER(c_size_t)], + c_int32, +) +def mtmd_input_chunk_save( + chunk: mtmd_input_chunk_p, + out_buf: Optional[CtypesArray[c_char]], + out_len: Union[c_size_t, int], + expected_out_len: "_Pointer[c_size_t]", + /, +) -> int: + """Save an input chunk's metadata to a buffer.""" + ... + + +# // returns nullptr on failure +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len); +@ctypes_function( + "mtmd_input_chunk_load", + [c_char_p, c_size_t], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_load( + buf: bytes, + length: Union[c_size_t, int], + /, +) -> Optional[mtmd_input_chunk_p]: + """Load an input chunk placeholder from saved metadata.""" + ... + + # MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); @ctypes_function( "mtmd_image_tokens_get_n_tokens", [mtmd_image_tokens_p_ctypes], c_size_t @@ -692,6 +880,36 @@ def mtmd_get_cap_from_file(mmproj_fname: bytes, /) -> mtmd_caps: ... +# MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +@ctypes_function( + "mtmd_gen_audio_get_info", + [mtmd_context_p_ctypes], + mtmd_gen_audio_info, +) +def mtmd_gen_audio_get_info(ctx: mtmd_context_p, /) -> mtmd_gen_audio_info: + """Get audio generation information for an MTMD context.""" + ... + + +# // note: this API is stateless, caller must handle state management and audio frame accumulation +# MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, +# const struct mtmd_gen_inp * inp, +# struct mtmd_gen_out * out); +@ctypes_function( + "mtmd_gen_audio_process", + [mtmd_context_p_ctypes, POINTER(mtmd_gen_inp), POINTER(mtmd_gen_out)], + c_int32, +) +def mtmd_gen_audio_process( + ctx: mtmd_context_p, + inp: "_Pointer[mtmd_gen_inp]", + out: "_Pointer[mtmd_gen_out]", + /, +) -> int: + """Process one audio generation step.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function("mtmd_test_create_input_chunks", [], mtmd_input_chunks_p_ctypes) def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: @@ -997,6 +1215,152 @@ def mtmd_helper_video_read_next( ... +# // return true if model can be used for chat +# MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_model_can_chat", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + c_bool, +) +def mtmd_helper_model_can_chat( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> bool: + """Return whether the model can be used for chat.""" + ... + + +# MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( +# struct llama_context * lctx, +# struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_gen_audio_init", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + mtmd_helper_gen_audio_p_ctypes, +) +def mtmd_helper_gen_audio_init( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> Optional[mtmd_helper_gen_audio_p]: + """Initialize an audio generation helper context.""" + ... + + +# MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_free", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_free(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_reset", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_reset(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API int32_t mtmd_helper_gen_audio_set_input( +# mtmd_helper_gen_audio * ctx, +# const struct mtmd_helper_gen_audio_inp * inp); +@ctypes_function( + "mtmd_helper_gen_audio_set_input", + [mtmd_helper_gen_audio_p_ctypes, POINTER(mtmd_helper_gen_audio_inp)], + c_int32, +) +def mtmd_helper_gen_audio_set_input( + ctx: mtmd_helper_gen_audio_p, + inp: "_Pointer[mtmd_helper_gen_audio_inp]", + /, +) -> int: + """Set the audio generation helper input.""" + ... + + +# // processes at most n_batch prompt tokens per call +# // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +# MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( +# mtmd_helper_gen_audio * ctx, +# int32_t n_batch); +@ctypes_function( + "mtmd_helper_gen_audio_step_prompt", + [mtmd_helper_gen_audio_p_ctypes, c_int32], + c_int32, +) +def mtmd_helper_gen_audio_step_prompt( + ctx: mtmd_helper_gen_audio_p, + n_batch: int, + /, +) -> int: + """Process up to n_batch prompt tokens.""" + ... + + +# // generates one frame; must only be called after step_prompt() has returned 0 +# // h_state_out is valid until next step_gen() or reset() call +# MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +# mtmd_helper_gen_audio * ctx, +# llama_token sampled, +# const float * h_state_in, +# const float ** h_state_out); +@ctypes_function( + "mtmd_helper_gen_audio_step_gen", + [ + mtmd_helper_gen_audio_p_ctypes, + llama_cpp.llama_token, + POINTER(c_float), + POINTER(POINTER(c_float)), + ], + c_int32, +) +def mtmd_helper_gen_audio_step_gen( + ctx: mtmd_helper_gen_audio_p, + sampled: llama_cpp.llama_token, + h_state_in: Optional["_Pointer[c_float]"], + h_state_out: "_Pointer[_Pointer[c_float]]", + /, +) -> int: + """Generate one audio frame.""" + ... + + +# // out_data valid until next get_output() or reset() call +# // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +# MTMD_API int32_t mtmd_helper_gen_audio_get_output( +# mtmd_helper_gen_audio * ctx, +# int32_t * out_sample_rate, +# const char ** out_data, +# size_t * out_data_len, +# int64_t * out_n_samples); +@ctypes_function( + "mtmd_helper_gen_audio_get_output", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(c_int32), + POINTER(POINTER(c_char)), + POINTER(c_size_t), + POINTER(c_int64), + ], + c_int32, +) +def mtmd_helper_gen_audio_get_output( + ctx: mtmd_helper_gen_audio_p, + out_sample_rate: "_Pointer[c_int32]", + out_data: "_Pointer[_Pointer[c_char]]", + out_data_len: "_Pointer[c_size_t]", + out_n_samples: Optional["_Pointer[c_int64]"], + /, +) -> int: + """Get accumulated PCM or WAV audio output.""" + ... + + # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "mtmd_log_set", diff --git a/tests/test_llama.py b/tests/test_llama.py index 70fce12d8e..c1c16e30ca 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -103,8 +103,14 @@ def test_real_model(llama_cpp_model_path): assert os.path.exists(llama_cpp_model_path) params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_mlock = llama_cpp.llama_supports_mlock() + if llama_cpp.llama_supports_mmap() and llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif llama_cpp.llama_supports_mmap(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE params.check_tensors = False model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) @@ -155,6 +161,9 @@ def test_real_model(llama_cpp_model_path): assert len(output) == 4 assert output_text + model.close() + assert context.ctx is None + def test_real_llama(llama_cpp_model_path): model = llama_cpp.Llama( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3546c7948..936918514c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 +Subproject commit 936918514ce522b553c0fd80b169a6440e6096c6