llama.cpp/llama_cpp/llama_cpp.py

2159 lines
73 KiB
Python
Raw Normal View History

import sys
import os
2023-03-23 09:33:06 +00:00
import ctypes
2023-04-11 15:59:03 +00:00
from ctypes import (
2023-09-29 02:42:03 +00:00
c_bool,
2023-04-11 15:59:03 +00:00
c_char_p,
2023-09-29 02:42:03 +00:00
c_int,
c_int8,
2023-07-05 05:00:14 +00:00
c_int32,
2023-09-29 02:42:03 +00:00
c_uint8,
2023-07-05 05:00:14 +00:00
c_uint32,
2023-09-29 02:42:03 +00:00
c_size_t,
c_float,
c_double,
2023-04-11 15:59:03 +00:00
c_void_p,
POINTER,
2023-05-05 17:54:22 +00:00
_Pointer, # type: ignore
2023-04-11 15:59:03 +00:00
Structure,
Array,
)
2023-03-23 09:33:06 +00:00
import pathlib
2023-06-26 12:50:38 +00:00
from typing import List, Union
2023-03-23 09:33:06 +00:00
2023-04-11 15:59:03 +00:00
2023-03-23 09:33:06 +00:00
# Load the library
def _load_shared_library(lib_base_name: str):
# Construct the paths to the possible shared library names
_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
# Searching for the library in the current directory under the name "libllama" (default name
# for llamacpp) and "llama" (default name for this repo)
_lib_paths: List[pathlib.Path] = []
# Determine the file extension based on the platform
if sys.platform.startswith("linux"):
_lib_paths += [
_base_path / f"lib{lib_base_name}.so",
]
elif sys.platform == "darwin":
_lib_paths += [
_base_path / f"lib{lib_base_name}.so",
_base_path / f"lib{lib_base_name}.dylib",
]
elif sys.platform == "win32":
_lib_paths += [
_base_path / f"{lib_base_name}.dll",
_base_path / f"lib{lib_base_name}.dll",
]
else:
raise RuntimeError("Unsupported platform")
2023-04-11 15:59:03 +00:00
if "LLAMA_CPP_LIB" in os.environ:
lib_base_name = os.environ["LLAMA_CPP_LIB"]
2023-04-10 15:12:25 +00:00
_lib = pathlib.Path(lib_base_name)
_base_path = _lib.parent.resolve()
_lib_paths = [_lib.resolve()]
cdll_args = dict() # type: ignore
# Add the library directory to the DLL search path on Windows (if needed)
if sys.platform == "win32" and sys.version_info >= (3, 8):
os.add_dll_directory(str(_base_path))
2023-05-17 19:26:38 +00:00
if "CUDA_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths:
if _lib_path.exists():
try:
return ctypes.CDLL(str(_lib_path), **cdll_args)
except Exception as e:
raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}")
2023-04-11 15:59:03 +00:00
raise FileNotFoundError(
f"Shared library with base name '{lib_base_name}' not found"
)
# Specify the base name of the shared library to load
_lib_base_name = "llama"
# Load the library
_lib = _load_shared_library(_lib_base_name)
2023-03-23 09:33:06 +00:00
2023-05-21 21:47:21 +00:00
# Misc
c_float_p = POINTER(c_float)
c_uint8_p = POINTER(c_uint8)
c_size_t_p = POINTER(c_size_t)
# llama.h bindings
2023-06-06 20:23:55 +00:00
GGML_USE_CUBLAS = hasattr(_lib, "ggml_init_cublas")
GGML_CUDA_MAX_DEVICES = 16
LLAMA_MAX_DEVICES = GGML_CUDA_MAX_DEVICES if GGML_USE_CUBLAS else 1
2023-06-06 20:23:55 +00:00
2023-08-24 04:17:00 +00:00
# define LLAMA_DEFAULT_SEED 0xFFFFFFFF
LLAMA_DEFAULT_SEED = 0xFFFFFFFF
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# define LLAMA_MAX_RNG_STATE (64*1024)
LLAMA_MAX_RNG_STATE = 64 * 1024
2023-08-24 04:17:00 +00:00
# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
LLAMA_FILE_MAGIC_GGSN = 0x6767736E
2023-05-21 21:47:21 +00:00
2023-08-24 04:17:00 +00:00
# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
2023-05-21 21:47:21 +00:00
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
2023-10-03 19:23:35 +00:00
# define LLAMA_SESSION_VERSION 2
LLAMA_SESSION_VERSION = 2
2023-05-01 14:44:28 +00:00
2023-07-05 05:00:14 +00:00
2023-06-26 12:50:38 +00:00
# struct llama_model;
llama_model_p = c_void_p
2023-05-21 21:47:21 +00:00
# struct llama_context;
llama_context_p = c_void_p
2023-09-29 02:42:03 +00:00
# typedef int32_t llama_pos;
llama_pos = c_int32
# typedef int32_t llama_token;
llama_token = c_int32
2023-03-23 09:33:06 +00:00
llama_token_p = POINTER(llama_token)
2023-09-29 02:42:03 +00:00
# typedef int32_t llama_seq_id;
llama_seq_id = c_int32
2023-03-23 09:33:06 +00:00
2023-08-24 04:17:00 +00:00
# enum llama_vocab_type {
# LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece
# LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding
# };
LLAMA_VOCAB_TYPE_SPM = 0
LLAMA_VOCAB_TYPE_BPE = 1
2023-08-24 04:17:00 +00:00
# enum llama_token_type {
# LLAMA_TOKEN_TYPE_UNDEFINED = 0,
# LLAMA_TOKEN_TYPE_NORMAL = 1,
# LLAMA_TOKEN_TYPE_UNKNOWN = 2,
# LLAMA_TOKEN_TYPE_CONTROL = 3,
# LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
# LLAMA_TOKEN_TYPE_UNUSED = 5,
# LLAMA_TOKEN_TYPE_BYTE = 6,
# };
LLAMA_TOKEN_TYPE_UNDEFINED = 0
LLAMA_TOKEN_TYPE_NORMAL = 1
LLAMA_TOKEN_TYPE_UNKNOWN = 2
LLAMA_TOKEN_TYPE_CONTROL = 3
LLAMA_TOKEN_TYPE_USER_DEFINED = 4
LLAMA_TOKEN_TYPE_UNUSED = 5
LLAMA_TOKEN_TYPE_BYTE = 6
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // model file types
2023-08-24 04:17:00 +00:00
# enum llama_ftype {
# LLAMA_FTYPE_ALL_F32 = 0,
2023-09-29 02:42:03 +00:00
# LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
# // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
# // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
# LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors
2023-08-24 04:17:00 +00:00
# LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
# };
LLAMA_FTYPE_ALL_F32 = 0
LLAMA_FTYPE_MOSTLY_F16 = 1
LLAMA_FTYPE_MOSTLY_Q4_0 = 2
LLAMA_FTYPE_MOSTLY_Q4_1 = 3
LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4
LLAMA_FTYPE_MOSTLY_Q8_0 = 7
LLAMA_FTYPE_MOSTLY_Q5_0 = 8
LLAMA_FTYPE_MOSTLY_Q5_1 = 9
LLAMA_FTYPE_MOSTLY_Q2_K = 10
LLAMA_FTYPE_MOSTLY_Q3_K_S = 11
LLAMA_FTYPE_MOSTLY_Q3_K_M = 12
LLAMA_FTYPE_MOSTLY_Q3_K_L = 13
LLAMA_FTYPE_MOSTLY_Q4_K_S = 14
LLAMA_FTYPE_MOSTLY_Q4_K_M = 15
LLAMA_FTYPE_MOSTLY_Q5_K_S = 16
LLAMA_FTYPE_MOSTLY_Q5_K_M = 17
LLAMA_FTYPE_MOSTLY_Q6_K = 18
LLAMA_FTYPE_GUESSED = 1024
2023-08-24 04:17:00 +00:00
# enum llama_rope_scaling_type {
# LLAMA_ROPE_SCALING_UNSPECIFIED = -1,
# LLAMA_ROPE_SCALING_NONE = 0,
# LLAMA_ROPE_SCALING_LINEAR = 1,
# LLAMA_ROPE_SCALING_YARN = 2,
# LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN,
# };
LLAMA_ROPE_SCALING_UNSPECIFIED = -1
LLAMA_ROPE_SCALING_NONE = 0
LLAMA_ROPE_SCALING_LINEAR = 1
LLAMA_ROPE_SCALING_YARN = 2
LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN
2023-03-24 18:35:41 +00:00
2023-11-20 19:11:33 +00:00
2023-05-21 21:47:21 +00:00
# typedef struct llama_token_data {
# llama_token id; // token id
# float logit; // log-odds of the token
# float p; // probability of the token
# } llama_token_data;
2023-03-23 09:33:06 +00:00
class llama_token_data(Structure):
_fields_ = [
2023-05-21 21:47:21 +00:00
("id", llama_token),
("logit", c_float),
("p", c_float),
2023-03-23 09:33:06 +00:00
]
2023-03-24 18:35:41 +00:00
2023-03-23 09:33:06 +00:00
llama_token_data_p = POINTER(llama_token_data)
2023-05-01 14:44:28 +00:00
2023-05-21 21:47:21 +00:00
# typedef struct llama_token_data_array {
# llama_token_data * data;
# size_t size;
# bool sorted;
# } llama_token_data_array;
2023-05-01 14:44:28 +00:00
class llama_token_data_array(Structure):
_fields_ = [
("data", llama_token_data_p),
("size", c_size_t),
("sorted", c_bool),
]
llama_token_data_array_p = POINTER(llama_token_data_array)
2023-05-21 21:47:21 +00:00
# typedef void (*llama_progress_callback)(float progress, void *ctx);
2023-03-29 01:10:23 +00:00
llama_progress_callback = ctypes.CFUNCTYPE(None, c_float, c_void_p)
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Input data for llama_decode
# // A llama_batch object can contain input about one or many sequences
# // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
# //
# // - token : the token ids of the input (used when embd is NULL)
# // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
# // - pos : the positions of the respective token in the sequence
# // - seq_id : the sequence to which the respective token belongs
# // - logits : if zero, the logits for the respective token will not be output
# //
# typedef struct llama_batch {
# int32_t n_tokens;
2023-10-19 06:55:08 +00:00
# llama_token * token;
# float * embd;
# llama_pos * pos;
2023-11-01 01:29:35 +00:00
# int32_t * n_seq_id;
2023-10-19 06:55:08 +00:00
# llama_seq_id ** seq_id;
# int8_t * logits;
2023-09-29 02:42:03 +00:00
# // NOTE: helpers for smooth API transition - can be deprecated in the future
# // for future-proof code, use the above fields instead and ignore everything below
# //
# // pos[i] = all_pos_0 + i*all_pos_1
# //
# llama_pos all_pos_0; // used if pos == NULL
# llama_pos all_pos_1; // used if pos == NULL
# llama_seq_id all_seq_id; // used if seq_id == NULL
# } llama_batch;
class llama_batch(Structure):
2023-11-23 05:26:26 +00:00
"""Input data for llama_decode
A llama_batch object can contain input about one or many sequences
The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
Attributes:
token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL)
embd (ctypes.Array[ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence
seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs
"""
2023-09-29 02:42:03 +00:00
_fields_ = [
("n_tokens", c_int32),
("token", POINTER(llama_token)),
("embd", c_float_p),
("pos", POINTER(llama_pos)),
2023-11-01 01:29:35 +00:00
("n_seq_id", POINTER(c_int32)),
2023-10-19 06:55:08 +00:00
("seq_id", POINTER(POINTER(llama_seq_id))),
2023-09-29 02:42:03 +00:00
("logits", POINTER(c_int8)),
("all_pos_0", llama_pos),
("all_pos_1", llama_pos),
("all_seq_id", llama_seq_id),
]
2023-07-15 19:11:01 +00:00
2023-09-29 02:42:03 +00:00
# struct llama_model_params {
# int32_t n_gpu_layers; // number of layers to store in VRAM
# int32_t main_gpu; // the GPU that is used for scratch and small tensors
# const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES)
2023-07-15 19:11:01 +00:00
2023-06-20 15:25:10 +00:00
# // called with a progress value between 0 and 1, pass NULL to disable
# llama_progress_callback progress_callback;
# // context pointer passed to the progress callback
# void * progress_callback_user_data;
2023-05-21 21:47:21 +00:00
2023-07-20 22:51:53 +00:00
2023-06-20 15:25:10 +00:00
# // Keep the booleans together to avoid misalignment during copy-by-value.
2023-05-21 21:47:21 +00:00
# bool vocab_only; // only load the vocabulary, no weights
# bool use_mmap; // use mmap if possible
# bool use_mlock; // force system to keep model in RAM
# };
2023-09-29 02:42:03 +00:00
class llama_model_params(Structure):
2023-03-23 09:33:06 +00:00
_fields_ = [
2023-07-05 05:00:14 +00:00
("n_gpu_layers", c_int32),
("main_gpu", c_int32),
2023-08-24 04:17:00 +00:00
("tensor_split", c_float_p),
2023-06-20 15:25:10 +00:00
("progress_callback", llama_progress_callback),
("progress_callback_user_data", c_void_p),
2023-05-21 21:47:21 +00:00
("vocab_only", c_bool),
("use_mmap", c_bool),
("use_mlock", c_bool),
2023-03-23 09:33:06 +00:00
]
2023-03-24 18:35:41 +00:00
2023-09-29 02:42:03 +00:00
# struct llama_context_params {
2023-11-03 15:34:50 +00:00
# uint32_t seed; // RNG seed, -1 for random
# uint32_t n_ctx; // text context, 0 = from model
# uint32_t n_batch; // prompt processing maximum batch size
# uint32_t n_threads; // number of threads to use for generation
# uint32_t n_threads_batch; // number of threads to use for batch processing
# int8_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
2023-09-29 02:42:03 +00:00
# // ref: https://github.com/ggerganov/llama.cpp/pull/2054
# float rope_freq_base; // RoPE base frequency, 0 = from model
# float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
# float yarn_ext_factor; // YaRN extrapolation mix factor, NaN = from model
# float yarn_attn_factor; // YaRN magnitude scaling factor
# float yarn_beta_fast; // YaRN low correction dim
# float yarn_beta_slow; // YaRN high correction dim
# uint32_t yarn_orig_ctx; // YaRN original context size
2023-09-29 02:42:03 +00:00
# // Keep the booleans together to avoid misalignment during copy-by-value.
2023-11-01 01:29:35 +00:00
# bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)
2023-09-29 23:58:21 +00:00
# bool f16_kv; // use fp16 for KV cache, fp32 otherwise
2023-09-29 02:42:03 +00:00
# bool logits_all; // the llama_eval() call computes all logits, not just the last one
# bool embedding; // embedding mode only
# };
class llama_context_params(Structure):
_fields_ = [
("seed", c_uint32),
("n_ctx", c_uint32),
("n_batch", c_uint32),
("n_threads", c_uint32),
("n_threads_batch", c_uint32),
("rope_scaling_type", c_int8),
2023-09-29 02:42:03 +00:00
("rope_freq_base", c_float),
("rope_freq_scale", c_float),
("yarn_ext_factor", c_float),
("yarn_attn_factor", c_float),
("yarn_beta_fast", c_float),
("yarn_beta_slow", c_float),
("yarn_orig_ctx", c_uint32),
2023-09-29 02:42:03 +00:00
("mul_mat_q", c_bool),
("f16_kv", c_bool),
("logits_all", c_bool),
("embedding", c_bool),
]
2023-03-23 09:33:06 +00:00
2023-08-24 04:17:00 +00:00
# // Signature for logging events
# // Note that text includes the new line character at the end for most events.
# // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
# // if it exists.
# // It might not exist for progress report where '.' is output repeatedly.
# typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data);
llama_log_callback = ctypes.CFUNCTYPE(None, c_int, c_char_p, c_void_p)
2023-11-23 05:26:26 +00:00
"""Signature for logging events
Note that text includes the new line character at the end for most events.
If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
if it exists.
It might not exist for progress report where '.' is output repeatedly."""
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-06-10 16:17:38 +00:00
# // model quantization parameters
# typedef struct llama_model_quantize_params {
# int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
2023-08-24 04:17:00 +00:00
# enum llama_ftype ftype; // quantize to this llama_ftype
2023-06-10 16:17:38 +00:00
# bool allow_requantize; // allow quantizing non-f32/f16 tensors
# bool quantize_output_tensor; // quantize output.weight
2023-09-01 18:26:13 +00:00
# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
2023-11-01 01:29:35 +00:00
# bool pure; // disable k-quant mixtures and quantize all tensors to the same type
2023-06-10 16:17:38 +00:00
# } llama_model_quantize_params;
class llama_model_quantize_params(Structure):
_fields_ = [
("nthread", c_int),
("ftype", c_int),
("allow_requantize", c_bool),
("quantize_output_tensor", c_bool),
2023-09-01 18:26:13 +00:00
("only_copy", c_bool),
2023-06-10 16:17:38 +00:00
]
2023-07-24 17:04:34 +00:00
# // grammar types
# struct llama_grammar;
llama_grammar_p = c_void_p
# // grammar element type
# enum llama_gretype {
# // end of rule definition
# LLAMA_GRETYPE_END = 0,
# // start of alternate definition for rule
# LLAMA_GRETYPE_ALT = 1,
# // non-terminal element: reference to rule
# LLAMA_GRETYPE_RULE_REF = 2,
# // terminal element: character (code point)
# LLAMA_GRETYPE_CHAR = 3,
# // inverse char(s) ([^a], [^a-b] [^abc])
# LLAMA_GRETYPE_CHAR_NOT = 4,
# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to
# // be an inclusive range ([a-z])
# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,
# // modifies a preceding LLAMA_GRETYPE_CHAR or
# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
# LLAMA_GRETYPE_CHAR_ALT = 6,
# };
LLAMA_GRETYPE_END = 0
LLAMA_GRETYPE_ALT = 1
LLAMA_GRETYPE_RULE_REF = 2
LLAMA_GRETYPE_CHAR = 3
LLAMA_GRETYPE_CHAR_NOT = 4
LLAMA_GRETYPE_CHAR_RNG_UPPER = 5
LLAMA_GRETYPE_CHAR_ALT = 6
2023-07-24 17:04:34 +00:00
# typedef struct llama_grammar_element {
# enum llama_gretype type;
# uint32_t value; // Unicode code point or rule ID
# } llama_grammar_element;
class llama_grammar_element(Structure):
_fields_ = [
("type", c_int),
("value", c_uint32),
]
llama_grammar_element_p = POINTER(llama_grammar_element)
2023-07-06 21:57:56 +00:00
# // performance timing information
# struct llama_timings {
# double t_start_ms;
# double t_end_ms;
# double t_load_ms;
# double t_sample_ms;
# double t_p_eval_ms;
# double t_eval_ms;
# int32_t n_sample;
# int32_t n_p_eval;
# int32_t n_eval;
# };
class llama_timings(Structure):
_fields_ = [
("t_start_ms", c_double),
("t_end_ms", c_double),
("t_load_ms", c_double),
("t_sample_ms", c_double),
("t_p_eval_ms", c_double),
("t_eval_ms", c_double),
("n_sample", c_int32),
("n_p_eval", c_int32),
("n_eval", c_int32),
]
2023-09-29 02:42:03 +00:00
# // Helpers for getting default parameters
# LLAMA_API struct llama_model_params llama_model_default_params(void);
def llama_model_default_params() -> llama_model_params:
return _lib.llama_model_default_params()
_lib.llama_model_default_params.argtypes = []
_lib.llama_model_default_params.restype = llama_model_params
2023-08-24 04:17:00 +00:00
# LLAMA_API struct llama_context_params llama_context_default_params(void);
2023-03-23 09:33:06 +00:00
def llama_context_default_params() -> llama_context_params:
2023-03-29 01:10:23 +00:00
return _lib.llama_context_default_params()
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
_lib.llama_context_default_params.argtypes = []
_lib.llama_context_default_params.restype = llama_context_params
2023-04-11 15:59:03 +00:00
2023-08-24 04:17:00 +00:00
# LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
2023-06-10 16:17:38 +00:00
def llama_model_quantize_default_params() -> llama_model_quantize_params:
return _lib.llama_model_quantize_default_params()
_lib.llama_model_quantize_default_params.argtypes = []
_lib.llama_model_quantize_default_params.restype = llama_model_quantize_params
2023-05-21 21:47:21 +00:00
# // Initialize the llama + ggml backend
2023-06-29 05:08:15 +00:00
# // If numa is true, use NUMA optimizations
2023-05-21 21:47:21 +00:00
# // Call once at the start of the program
2023-07-15 19:11:01 +00:00
# LLAMA_API void llama_backend_init(bool numa);
def llama_backend_init(numa: Union[c_bool, bool]):
2023-11-23 05:26:26 +00:00
"""Initialize the llama + ggml backend
If numa is true, use NUMA optimizations
Call once at the start of the program"""
2023-07-15 19:11:01 +00:00
return _lib.llama_backend_init(numa)
_lib.llama_backend_init.argtypes = [c_bool]
_lib.llama_backend_init.restype = None
2023-05-21 21:47:21 +00:00
2023-07-15 19:11:01 +00:00
# // Call once at the end of the program - currently only used for MPI
2023-08-24 04:17:00 +00:00
# LLAMA_API void llama_backend_free(void);
2023-07-15 19:11:01 +00:00
def llama_backend_free():
2023-11-23 05:26:26 +00:00
"""Call once at the end of the program - currently only used for MPI"""
2023-07-15 19:11:01 +00:00
return _lib.llama_backend_free()
2023-05-21 21:47:21 +00:00
2023-07-15 19:11:01 +00:00
_lib.llama_backend_free.argtypes = []
_lib.llama_backend_free.restype = None
2023-05-21 21:47:21 +00:00
2023-06-26 12:50:38 +00:00
# LLAMA_API struct llama_model * llama_load_model_from_file(
2023-08-24 04:17:00 +00:00
# const char * path_model,
2023-10-05 20:07:49 +00:00
# struct llama_model_params params);
2023-06-26 12:50:38 +00:00
def llama_load_model_from_file(
2023-09-29 02:42:03 +00:00
path_model: bytes, params: llama_model_params
2023-06-26 12:50:38 +00:00
) -> llama_model_p:
return _lib.llama_load_model_from_file(path_model, params)
2023-06-26 12:50:38 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_load_model_from_file.argtypes = [c_char_p, llama_model_params]
2023-06-26 12:50:38 +00:00
_lib.llama_load_model_from_file.restype = llama_model_p
# LLAMA_API void llama_free_model(struct llama_model * model);
def llama_free_model(model: llama_model_p):
return _lib.llama_free_model(model)
_lib.llama_free_model.argtypes = [llama_model_p]
_lib.llama_free_model.restype = None
# LLAMA_API struct llama_context * llama_new_context_with_model(
2023-08-24 04:17:00 +00:00
# struct llama_model * model,
2023-06-26 12:50:38 +00:00
# struct llama_context_params params);
def llama_new_context_with_model(
model: llama_model_p, params: llama_context_params
) -> llama_context_p:
return _lib.llama_new_context_with_model(model, params)
_lib.llama_new_context_with_model.argtypes = [llama_model_p, llama_context_params]
_lib.llama_new_context_with_model.restype = llama_context_p
2023-08-24 04:17:00 +00:00
# // Frees all allocated memory
# LLAMA_API void llama_free(struct llama_context * ctx);
def llama_free(ctx: llama_context_p):
2023-11-23 05:26:26 +00:00
"""Frees all allocated memory"""
2023-08-24 04:17:00 +00:00
return _lib.llama_free(ctx)
_lib.llama_free.argtypes = [llama_context_p]
_lib.llama_free.restype = None
# LLAMA_API int64_t llama_time_us(void);
2023-05-21 21:47:21 +00:00
def llama_time_us() -> int:
return _lib.llama_time_us()
_lib.llama_time_us.argtypes = []
_lib.llama_time_us.restype = ctypes.c_int64
2023-08-24 04:17:00 +00:00
# LLAMA_API int llama_max_devices (void);
def llama_max_devices() -> int:
return _lib.llama_max_devices()
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib.llama_max_devices.argtypes = []
_lib.llama_max_devices.restype = c_int
2023-03-24 18:59:29 +00:00
2023-08-24 04:17:00 +00:00
# LLAMA_API bool llama_mmap_supported (void);
def llama_mmap_supported() -> bool:
return _lib.llama_mmap_supported()
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib.llama_mmap_supported.argtypes = []
_lib.llama_mmap_supported.restype = c_bool
# LLAMA_API bool llama_mlock_supported(void);
def llama_mlock_supported() -> bool:
return _lib.llama_mlock_supported()
_lib.llama_mlock_supported.argtypes = []
_lib.llama_mlock_supported.restype = c_bool
2023-09-29 02:42:03 +00:00
# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);
def llama_get_model(ctx: llama_context_p) -> llama_model_p:
return _lib.llama_get_model(ctx)
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_get_model.argtypes = [llama_context_p]
_lib.llama_get_model.restype = llama_model_p
2023-08-24 04:17:00 +00:00
2023-09-09 16:12:32 +00:00
# LLAMA_API int llama_n_ctx (const struct llama_context * ctx);
2023-08-24 04:17:00 +00:00
def llama_n_ctx(ctx: llama_context_p) -> int:
return _lib.llama_n_ctx(ctx)
_lib.llama_n_ctx.argtypes = [llama_context_p]
_lib.llama_n_ctx.restype = c_int
2023-09-29 02:42:03 +00:00
# LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model);
def llama_vocab_type(model: llama_model_p) -> int:
return _lib.llama_vocab_type(model)
2023-09-09 16:12:32 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_vocab_type.argtypes = [llama_model_p]
2023-08-24 22:01:42 +00:00
_lib.llama_vocab_type.restype = c_int
2023-09-29 02:42:03 +00:00
# LLAMA_API int llama_n_vocab (const struct llama_model * model);
def llama_n_vocab(model: llama_model_p) -> int:
return _lib.llama_n_vocab(model)
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_n_vocab.argtypes = [llama_model_p]
_lib.llama_n_vocab.restype = c_int
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# LLAMA_API int llama_n_ctx_train(const struct llama_model * model);
def llama_n_ctx_train(model: llama_model_p) -> int:
return _lib.llama_n_ctx_train(model)
2023-09-09 16:12:32 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_n_ctx_train.argtypes = [llama_model_p]
_lib.llama_n_ctx_train.restype = c_int
2023-09-09 16:12:32 +00:00
2023-09-29 02:42:03 +00:00
# LLAMA_API int llama_n_embd (const struct llama_model * model);
def llama_n_embd(model: llama_model_p) -> int:
return _lib.llama_n_embd(model)
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_n_embd.argtypes = [llama_model_p]
_lib.llama_n_embd.restype = c_int
2023-08-24 04:17:00 +00:00
2023-10-03 19:23:35 +00:00
# // Get the model's RoPE frequency scaling factor
# LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);
def llama_rope_freq_scale_train(model: llama_model_p) -> float:
2023-11-23 05:26:26 +00:00
"""Get the model's RoPE frequency scaling factor"""
2023-10-03 19:23:35 +00:00
return _lib.llama_rope_freq_scale_train(model)
_lib.llama_rope_freq_scale_train.argtypes = [llama_model_p]
_lib.llama_rope_freq_scale_train.restype = c_float
2023-11-20 19:11:33 +00:00
# // Functions to access the model's GGUF metadata scalar values
# // - The functions return the length of the string on success, or -1 on failure
# // - The output string is always null-terminated and cleared on failure
# // - GGUF array values are not supported by these functions
# // Get metadata value as a string by key name
# LLAMA_API int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
def llama_model_meta_val_str(
model: llama_model_p, key: Union[c_char_p, bytes], buf: bytes, buf_size: int
) -> int:
2023-11-23 05:26:26 +00:00
"""Get metadata value as a string by key name"""
2023-11-20 19:11:33 +00:00
return _lib.llama_model_meta_val_str(model, key, buf, buf_size)
_lib.llama_model_meta_val_str.argtypes = [llama_model_p, c_char_p, c_char_p, c_size_t]
_lib.llama_model_meta_val_str.restype = c_int
# // Get the number of metadata key/value pairs
# LLAMA_API int llama_model_meta_count(const struct llama_model * model);
def llama_model_meta_count(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""Get the number of metadata key/value pairs"""
2023-11-20 19:11:33 +00:00
return _lib.llama_model_meta_count(model)
_lib.llama_model_meta_count.argtypes = [llama_model_p]
_lib.llama_model_meta_count.restype = c_int
# // Get metadata key name by index
# LLAMA_API int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);
def llama_model_meta_key_by_index(
model: llama_model_p, i: Union[c_int, int], buf: bytes, buf_size: int
) -> int:
2023-11-23 05:26:26 +00:00
"""Get metadata key name by index"""
2023-11-20 19:11:33 +00:00
return _lib.llama_model_meta_key_by_index(model, i, buf, buf_size)
_lib.llama_model_meta_key_by_index.argtypes = [llama_model_p, c_int, c_char_p, c_size_t]
_lib.llama_model_meta_key_by_index.restype = c_int
# // Get metadata value as a string by index
# LLAMA_API int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);
def llama_model_meta_val_str_by_index(
model: llama_model_p, i: Union[c_int, int], buf: bytes, buf_size: int
) -> int:
2023-11-23 05:26:26 +00:00
"""Get metadata value as a string by index"""
2023-11-20 19:11:33 +00:00
return _lib.llama_model_meta_val_str_by_index(model, i, buf, buf_size)
_lib.llama_model_meta_val_str_by_index.argtypes = [
llama_model_p,
c_int,
c_char_p,
c_size_t,
]
_lib.llama_model_meta_val_str_by_index.restype = c_int
2023-10-03 19:23:35 +00:00
2023-08-24 04:17:00 +00:00
# // Get a string describing the model type
2023-08-25 18:35:53 +00:00
# LLAMA_API int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
2023-09-29 02:42:03 +00:00
def llama_model_desc(
model: llama_model_p, buf: bytes, buf_size: Union[c_size_t, int]
) -> int:
2023-11-23 05:26:26 +00:00
"""Get a string describing the model type"""
2023-08-25 18:35:53 +00:00
return _lib.llama_model_desc(model, buf, buf_size)
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
_lib.llama_model_desc.argtypes = [llama_model_p, c_char_p, c_size_t]
_lib.llama_model_desc.restype = c_int
# // Returns the total size of all the tensors in the model in bytes
# LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
def llama_model_size(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns the total size of all the tensors in the model in bytes"""
2023-08-25 18:35:53 +00:00
return _lib.llama_model_size(model)
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
_lib.llama_model_size.argtypes = [llama_model_p]
_lib.llama_model_size.restype = ctypes.c_uint64
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
# // Returns the total number of parameters in the model
# LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
def llama_model_n_params(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns the total number of parameters in the model"""
2023-08-25 18:35:53 +00:00
return _lib.llama_model_n_params(model)
_lib.llama_model_n_params.argtypes = [llama_model_p]
_lib.llama_model_n_params.restype = ctypes.c_uint64
2023-03-24 18:59:29 +00:00
2023-09-29 02:42:03 +00:00
# // Get a llama model tensor
# LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name);
def llama_get_model_tensor(
model: llama_model_p, name: Union[c_char_p, bytes]
) -> c_void_p:
2023-11-23 05:26:26 +00:00
"""Get a llama model tensor"""
2023-09-29 02:42:03 +00:00
return _lib.llama_get_model_tensor(model, name)
_lib.llama_get_model_tensor.argtypes = [llama_model_p, c_char_p]
_lib.llama_get_model_tensor.restype = c_void_p
2023-06-10 16:17:38 +00:00
# // Returns 0 on success
2023-05-21 21:47:21 +00:00
# LLAMA_API int llama_model_quantize(
# const char * fname_inp,
# const char * fname_out,
2023-06-10 16:17:38 +00:00
# const llama_model_quantize_params * params);
2023-04-22 23:50:28 +00:00
def llama_model_quantize(
2023-06-10 16:17:38 +00:00
fname_inp: bytes,
fname_out: bytes,
params, # type: POINTER(llama_model_quantize_params) # type: ignore
) -> int:
2023-11-23 05:26:26 +00:00
"""Returns 0 on success"""
2023-06-10 16:17:38 +00:00
return _lib.llama_model_quantize(fname_inp, fname_out, params)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-06-10 16:17:38 +00:00
_lib.llama_model_quantize.argtypes = [
c_char_p,
c_char_p,
POINTER(llama_model_quantize_params),
]
_lib.llama_model_quantize.restype = c_int
2023-04-11 15:59:03 +00:00
2023-09-29 02:42:03 +00:00
# // Apply a LoRA adapter to a loaded model
# // path_base_model is the path to a higher quality model to use as a base for
# // the layers modified by the adapter. Can be NULL to use the current loaded model.
# // The model needs to be reloaded before applying a new adapter, otherwise the adapter
# // will be applied on top of the previous one
# // Returns 0 on success
# LLAMA_API DEPRECATED(int llama_apply_lora_from_file(
2023-05-21 21:47:21 +00:00
# struct llama_context * ctx,
# const char * path_lora,
2023-09-29 02:42:03 +00:00
# float scale,
2023-05-21 21:47:21 +00:00
# const char * path_base_model,
2023-09-29 02:42:03 +00:00
# int n_threads),
# "use llama_model_apply_lora_from_file instead");
def llama_apply_lora_from_file(
2023-04-22 23:50:28 +00:00
ctx: llama_context_p,
path_lora: Union[c_char_p, bytes],
2023-09-29 02:42:03 +00:00
scale: Union[c_float, float],
path_base_model: Union[c_char_p, bytes],
n_threads: Union[c_int, int],
) -> int:
2023-11-23 05:26:26 +00:00
"""Apply a LoRA adapter to a loaded model
path_base_model is the path to a higher quality model to use as a base for
the layers modified by the adapter. Can be NULL to use the current loaded model.
The model needs to be reloaded before applying a new adapter, otherwise the adapter
will be applied on top of the previous one
Returns 0 on success"""
2023-09-29 02:42:03 +00:00
return _lib.llama_apply_lora_from_file(
ctx, path_lora, scale, path_base_model, n_threads
)
2023-09-29 02:42:03 +00:00
_lib.llama_apply_lora_from_file.argtypes = [
llama_context_p,
c_char_p,
c_float,
c_char_p,
c_int,
]
_lib.llama_apply_lora_from_file.restype = c_int
2023-06-26 12:50:38 +00:00
# LLAMA_API int llama_model_apply_lora_from_file(
# const struct llama_model * model,
2023-09-29 02:42:03 +00:00
# const char * path_lora,
# float scale,
# const char * path_base_model,
# int n_threads);
2023-06-26 12:50:38 +00:00
def llama_model_apply_lora_from_file(
model: llama_model_p,
path_lora: Union[c_char_p, bytes],
2023-09-29 02:42:03 +00:00
scale: Union[c_float, float],
2023-06-26 12:50:38 +00:00
path_base_model: Union[c_char_p, bytes],
n_threads: Union[c_int, int],
2023-06-26 12:50:38 +00:00
) -> int:
return _lib.llama_model_apply_lora_from_file(
2023-09-29 02:42:03 +00:00
model, path_lora, scale, path_base_model, n_threads
2023-06-26 12:50:38 +00:00
)
_lib.llama_model_apply_lora_from_file.argtypes = [
llama_model_p,
c_char_p,
2023-09-29 02:42:03 +00:00
c_float,
2023-06-26 12:50:38 +00:00
c_char_p,
c_int,
]
_lib.llama_model_apply_lora_from_file.restype = c_int
2023-09-29 02:42:03 +00:00
# //
# // KV cache
# //
2023-09-29 02:42:03 +00:00
# // Returns the number of tokens in the KV cache
# LLAMA_API DEPRECATED(int llama_get_kv_cache_token_count(const struct llama_context * ctx),
# "avoid using this, it will be removed in the future, instead - count the tokens in user code");
def llama_get_kv_cache_token_count(ctx: llama_context_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns the number of tokens in the KV cache"""
return _lib.llama_get_kv_cache_token_count(ctx)
2023-04-11 15:59:03 +00:00
_lib.llama_get_kv_cache_token_count.argtypes = [llama_context_p]
_lib.llama_get_kv_cache_token_count.restype = c_int
2023-04-28 19:32:43 +00:00
2023-11-01 01:29:35 +00:00
# // Clear the KV cache
# LLAMA_API void llama_kv_cache_clear(
# struct llama_context * ctx);
def llama_kv_cache_clear(ctx: llama_context_p):
2023-11-23 05:26:26 +00:00
"""Clear the KV cache"""
2023-11-01 01:29:35 +00:00
return _lib.llama_kv_cache_clear(ctx)
2023-04-27 00:00:54 +00:00
2023-04-28 19:32:43 +00:00
2023-11-01 01:29:35 +00:00
_lib.llama_kv_cache_clear.argtypes = [llama_context_p]
_lib.llama_kv_cache_clear.restype = None
2023-09-29 02:42:03 +00:00
# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
2023-11-01 01:29:35 +00:00
# // seq_id < 0 : match any sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_rm(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1);
def llama_kv_cache_seq_rm(
ctx: llama_context_p,
2023-11-05 21:57:10 +00:00
seq_id: Union[llama_seq_id, int],
2023-09-29 02:42:03 +00:00
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
):
2023-11-23 05:26:26 +00:00
"""Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
seq_id < 0 : match any sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
2023-09-29 02:42:03 +00:00
return _lib.llama_kv_cache_seq_rm(ctx, seq_id, p0, p1)
_lib.llama_kv_cache_seq_rm.argtypes = [
llama_context_p,
llama_seq_id,
llama_pos,
llama_pos,
]
_lib.llama_kv_cache_seq_rm.restype = None
# // Copy all tokens that belong to the specified sequence to another sequence
# // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
2023-10-03 19:23:35 +00:00
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_cp(
# struct llama_context * ctx,
# llama_seq_id seq_id_src,
# llama_seq_id seq_id_dst,
# llama_pos p0,
# llama_pos p1);
def llama_kv_cache_seq_cp(
ctx: llama_context_p,
2023-11-05 21:57:10 +00:00
seq_id_src: Union[llama_seq_id, int],
seq_id_dst: Union[llama_seq_id, int],
2023-09-29 02:42:03 +00:00
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
):
2023-11-23 05:26:26 +00:00
"""Copy all tokens that belong to the specified sequence to another sequence
Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
2023-09-29 02:42:03 +00:00
return _lib.llama_kv_cache_seq_cp(ctx, seq_id_src, seq_id_dst, p0, p1)
_lib.llama_kv_cache_seq_cp.argtypes = [
llama_context_p,
llama_seq_id,
llama_seq_id,
llama_pos,
llama_pos,
]
_lib.llama_kv_cache_seq_cp.restype = None
# // Removes all tokens that do not belong to the specified sequence
# LLAMA_API void llama_kv_cache_seq_keep(
# struct llama_context * ctx,
# llama_seq_id seq_id);
def llama_kv_cache_seq_keep(
ctx: llama_context_p,
2023-11-05 21:57:10 +00:00
seq_id: Union[llama_seq_id, int],
2023-09-29 02:42:03 +00:00
):
2023-11-23 05:26:26 +00:00
"""Removes all tokens that do not belong to the specified sequence"""
2023-09-29 02:42:03 +00:00
return _lib.llama_kv_cache_seq_keep(ctx, seq_id)
_lib.llama_kv_cache_seq_keep.argtypes = [llama_context_p, llama_seq_id]
_lib.llama_kv_cache_seq_keep.restype = None
# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
# // If the KV cache is RoPEd, the KV data is updated accordingly
2023-10-03 19:23:35 +00:00
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_shift(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# llama_pos delta);
def llama_kv_cache_seq_shift(
ctx: llama_context_p,
2023-11-05 21:57:10 +00:00
seq_id: Union[llama_seq_id, int],
2023-09-29 02:42:03 +00:00
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
delta: Union[llama_pos, int],
):
2023-11-23 05:26:26 +00:00
"""Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
If the KV cache is RoPEd, the KV data is updated accordingly
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
2023-09-29 02:42:03 +00:00
return _lib.llama_kv_cache_seq_shift(ctx, seq_id, p0, p1, delta)
_lib.llama_kv_cache_seq_shift.argtypes = [
llama_context_p,
llama_seq_id,
llama_pos,
llama_pos,
llama_pos,
]
_lib.llama_kv_cache_seq_shift.restype = None
# //
# // State / sessions
# //
2023-04-28 19:32:43 +00:00
2023-05-03 13:33:30 +00:00
# Returns the maximum size in bytes of the state (rng, logits, embedding
# and kv_cache) - will often be smaller after compacting tokens
2023-05-21 21:47:21 +00:00
# LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx);
def llama_get_state_size(ctx: llama_context_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns the maximum size in bytes of the state (rng, logits, embedding
and kv_cache) - will often be smaller after compacting tokens"""
2023-04-22 23:50:28 +00:00
return _lib.llama_get_state_size(ctx)
_lib.llama_get_state_size.argtypes = [llama_context_p]
_lib.llama_get_state_size.restype = c_size_t
# Copies the state to the specified destination address.
# Destination needs to have allocated enough memory.
# Returns the number of bytes copied
2023-09-29 02:42:03 +00:00
# LLAMA_API size_t llama_copy_state_data(
# struct llama_context * ctx,
# uint8_t * dst);
2023-05-05 18:12:26 +00:00
def llama_copy_state_data(
2023-05-14 04:04:22 +00:00
ctx: llama_context_p, dst # type: Array[c_uint8]
2023-05-07 23:30:14 +00:00
) -> int:
2023-11-23 05:26:26 +00:00
"""Copies the state to the specified destination address.
Destination needs to have allocated enough memory.
Returns the number of bytes copied"""
2023-05-14 04:04:22 +00:00
return _lib.llama_copy_state_data(ctx, dst)
2023-04-22 23:50:28 +00:00
2023-05-05 18:04:12 +00:00
_lib.llama_copy_state_data.argtypes = [llama_context_p, c_uint8_p]
2023-04-22 23:50:28 +00:00
_lib.llama_copy_state_data.restype = c_size_t
# Set the state reading from the specified address
# Returns the number of bytes read
2023-09-29 02:42:03 +00:00
# LLAMA_API size_t llama_set_state_data(
# struct llama_context * ctx,
# uint8_t * src);
def llama_set_state_data(
ctx: llama_context_p, src # type: Array[c_uint8]
2023-05-07 23:30:14 +00:00
) -> int:
2023-11-23 05:26:26 +00:00
"""Set the state reading from the specified address"""
2023-04-22 23:50:28 +00:00
return _lib.llama_set_state_data(ctx, src)
2023-05-05 18:04:12 +00:00
_lib.llama_set_state_data.argtypes = [llama_context_p, c_uint8_p]
2023-04-22 23:50:28 +00:00
_lib.llama_set_state_data.restype = c_size_t
2023-04-28 19:32:43 +00:00
# Save/load session file
2023-09-29 02:42:03 +00:00
# LLAMA_API bool llama_load_session_file(
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
2023-04-28 19:32:43 +00:00
def llama_load_session_file(
ctx: llama_context_p,
path_session: bytes,
2023-05-05 18:12:26 +00:00
tokens_out, # type: Array[llama_token]
n_token_capacity: Union[c_size_t, int],
2023-05-05 18:12:26 +00:00
n_token_count_out, # type: _Pointer[c_size_t]
) -> int:
2023-04-28 19:32:43 +00:00
return _lib.llama_load_session_file(
ctx, path_session, tokens_out, n_token_capacity, n_token_count_out
)
_lib.llama_load_session_file.argtypes = [
llama_context_p,
c_char_p,
llama_token_p,
c_size_t,
2023-05-05 18:04:12 +00:00
c_size_t_p,
2023-04-28 19:32:43 +00:00
]
_lib.llama_load_session_file.restype = c_size_t
2023-09-29 02:42:03 +00:00
# LLAMA_API bool llama_save_session_file(
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count);
2023-04-28 19:32:43 +00:00
def llama_save_session_file(
ctx: llama_context_p,
path_session: bytes,
2023-05-05 18:12:26 +00:00
tokens, # type: Array[llama_token]
n_token_count: Union[c_size_t, int],
) -> int:
2023-04-28 19:32:43 +00:00
return _lib.llama_save_session_file(ctx, path_session, tokens, n_token_count)
_lib.llama_save_session_file.argtypes = [
llama_context_p,
c_char_p,
llama_token_p,
c_size_t,
]
_lib.llama_save_session_file.restype = c_size_t
2023-09-29 02:42:03 +00:00
# //
# // Decoding
# //
2023-04-28 19:32:43 +00:00
2023-09-29 02:42:03 +00:00
# // Run the llama inference to obtain the logits and probabilities for the next token(s).
# // tokens + n_tokens is the provided batch of new tokens to process
# // n_past is the number of tokens to use from previous eval calls
# // Returns 0 on success
# // DEPRECATED: use llama_decode() instead
# LLAMA_API DEPRECATED(int llama_eval(
2023-05-21 21:47:21 +00:00
# struct llama_context * ctx,
2023-09-29 02:42:03 +00:00
# llama_token * tokens,
# int32_t n_tokens,
# int n_past),
# "use llama_decode() instead");
2023-03-24 18:35:41 +00:00
def llama_eval(
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
tokens, # type: Array[llama_token]
n_tokens: Union[c_int, int],
n_past: Union[c_int, int],
) -> int:
2023-11-23 05:26:26 +00:00
"""Run the llama inference to obtain the logits and probabilities for the next token(s).
tokens + n_tokens is the provided batch of new tokens to process
n_past is the number of tokens to use from previous eval calls
Returns 0 on success
DEPRECATED: use llama_decode() instead"""
2023-09-29 02:42:03 +00:00
return _lib.llama_eval(ctx, tokens, n_tokens, n_past)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_eval.argtypes = [llama_context_p, llama_token_p, c_int, c_int]
_lib.llama_eval.restype = c_int
2023-06-29 05:08:15 +00:00
# // Same as llama_eval, but use float matrix input directly.
2023-09-29 02:42:03 +00:00
# // DEPRECATED: use llama_decode() instead
# LLAMA_API DEPRECATED(int llama_eval_embd(
2023-06-29 05:08:15 +00:00
# struct llama_context * ctx,
2023-09-29 02:42:03 +00:00
# float * embd,
# int32_t n_tokens,
# int n_past),
# "use llama_decode() instead");
2023-06-29 05:08:15 +00:00
def llama_eval_embd(
ctx: llama_context_p,
embd, # type: Array[c_float]
n_tokens: Union[c_int, int],
n_past: Union[c_int, int],
2023-06-29 05:08:15 +00:00
) -> int:
2023-11-23 05:26:26 +00:00
"""Same as llama_eval, but use float matrix input directly.
DEPRECATED: use llama_decode() instead"""
2023-09-29 02:42:03 +00:00
return _lib.llama_eval_embd(ctx, embd, n_tokens, n_past)
2023-06-29 05:08:15 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_eval_embd.argtypes = [llama_context_p, c_float_p, c_int, c_int]
2023-06-29 05:08:15 +00:00
_lib.llama_eval_embd.restype = c_int
2023-09-29 02:42:03 +00:00
# // Return batch for single sequence of tokens starting at pos_0
# //
# // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
# //
# LLAMA_API struct llama_batch llama_batch_get_one(
# llama_token * tokens,
# int32_t n_tokens,
# llama_pos pos_0,
# llama_seq_id seq_id);
def llama_batch_get_one(
tokens, # type: Array[llama_token]
n_tokens: Union[c_int, int],
pos_0: Union[llama_pos, int],
seq_id: llama_seq_id,
) -> llama_batch:
2023-11-23 05:26:26 +00:00
"""Return batch for single sequence of tokens starting at pos_0
NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it"""
2023-09-29 02:42:03 +00:00
return _lib.llama_batch_get_one(tokens, n_tokens, pos_0, seq_id)
_lib.llama_batch_get_one.argtypes = [
llama_token_p,
c_int,
llama_pos,
llama_seq_id,
]
_lib.llama_batch_get_one.restype = llama_batch
2023-10-19 06:55:08 +00:00
# // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
# // Each token can be assigned up to n_seq_max sequence ids
2023-09-29 02:42:03 +00:00
# // The batch has to be freed with llama_batch_free()
# // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
# // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
# // The rest of the llama_batch members are allocated with size n_tokens
# // All members are left uninitialized
# LLAMA_API struct llama_batch llama_batch_init(
# int32_t n_tokens,
2023-10-19 06:55:08 +00:00
# int32_t embd,
# int32_t n_seq_max);
2023-09-29 02:42:03 +00:00
def llama_batch_init(
2023-10-19 06:55:08 +00:00
n_tokens: Union[c_int32, int],
embd: Union[c_int32, int],
n_seq_max: Union[c_int32, int],
2023-09-29 02:42:03 +00:00
) -> llama_batch:
2023-11-23 05:26:26 +00:00
"""Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
Each token can be assigned up to n_seq_max sequence ids
The batch has to be freed with llama_batch_free()
If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
The rest of the llama_batch members are allocated with size n_tokens
All members are left uninitialized"""
2023-10-19 06:55:08 +00:00
return _lib.llama_batch_init(n_tokens, embd, n_seq_max)
2023-09-29 02:42:03 +00:00
2023-10-19 06:55:08 +00:00
_lib.llama_batch_init.argtypes = [c_int32, c_int32, c_int32]
2023-09-29 02:42:03 +00:00
_lib.llama_batch_init.restype = llama_batch
# // Frees a batch of tokens allocated with llama_batch_init()
# LLAMA_API void llama_batch_free(struct llama_batch batch);
def llama_batch_free(batch: llama_batch):
2023-11-23 05:26:26 +00:00
"""Frees a batch of tokens allocated with llama_batch_init()"""
2023-09-29 02:42:03 +00:00
return _lib.llama_batch_free(batch)
_lib.llama_batch_free.argtypes = [llama_batch]
_lib.llama_batch_free.restype = None
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Positive return values does not mean a fatal error, but rather a warning.
# // 0 - success
# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
# // < 0 - error
# LLAMA_API int llama_decode(
# struct llama_context * ctx,
# struct llama_batch batch);
def llama_decode(ctx: llama_context_p, batch: llama_batch) -> int:
2023-11-23 05:26:26 +00:00
"""Positive return values does not mean a fatal error, but rather a warning.
0 - success
1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
< 0 - error"""
2023-09-29 02:42:03 +00:00
return _lib.llama_decode(ctx, batch)
_lib.llama_decode.argtypes = [llama_context_p, llama_batch]
_lib.llama_decode.restype = c_int
# // Set the number of threads used for decoding
# // n_threads is the number of threads used for generation (single token)
# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch);
def llama_set_n_threads(
ctx: llama_context_p,
n_threads: Union[c_uint32, int],
n_threads_batch: Union[c_uint32, int],
):
2023-11-23 05:26:26 +00:00
"""Set the number of threads used for decoding
n_threads is the number of threads used for generation (single token)
n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)"""
2023-09-29 02:42:03 +00:00
return _lib.llama_set_n_threads(ctx, n_threads, n_threads_batch)
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_set_n_threads.argtypes = [llama_context_p, c_uint32, c_uint32]
_lib.llama_set_n_threads.restype = None
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Token logits obtained from the last call to llama_eval()
# // The logits for the last token are stored in the last row
# // Logits for which llama_batch.logits[i] == 0 are undefined
# // Rows: n_tokens provided with llama_batch
# // Cols: n_vocab
2023-08-24 04:17:00 +00:00
# LLAMA_API float * llama_get_logits(struct llama_context * ctx);
def llama_get_logits(
2023-03-24 18:35:41 +00:00
ctx: llama_context_p,
2023-08-24 04:17:00 +00:00
): # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
"""Token logits obtained from the last call to llama_eval()
The logits for the last token are stored in the last row
Logits for which llama_batch.logits[i] == 0 are undefined
Rows: n_tokens provided with llama_batch
Cols: n_vocab"""
2023-08-24 04:17:00 +00:00
return _lib.llama_get_logits(ctx)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib.llama_get_logits.argtypes = [llama_context_p]
_lib.llama_get_logits.restype = c_float_p
2023-09-29 02:42:03 +00:00
# // Logits for the ith token. Equivalent to:
# // llama_get_logits(ctx) + i*n_vocab
# LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
def llama_get_logits_ith(
ctx: llama_context_p, i: Union[c_int32, int]
): # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
"""Logits for the ith token. Equivalent to:
llama_get_logits(ctx) + i*n_vocab"""
2023-09-29 02:42:03 +00:00
return _lib.llama_get_logits_ith(ctx, i)
_lib.llama_get_logits_ith.argtypes = [llama_context_p, c_int32]
_lib.llama_get_logits_ith.restype = c_float_p
2023-08-24 04:17:00 +00:00
# Get the embeddings for the input
# shape: [n_embd] (1-dimensional)
# LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
def llama_get_embeddings(
ctx: llama_context_p,
): # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
"""Get the embeddings for the input
shape: [n_embd] (1-dimensional)"""
2023-08-24 04:17:00 +00:00
return _lib.llama_get_embeddings(ctx)
2023-07-15 19:11:01 +00:00
2023-08-24 04:17:00 +00:00
_lib.llama_get_embeddings.argtypes = [llama_context_p]
_lib.llama_get_embeddings.restype = c_float_p
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
# //
# // Vocab
# //
2023-10-24 07:13:32 +00:00
# LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token);
2023-11-05 21:57:10 +00:00
def llama_token_get_text(model: llama_model_p, token: Union[llama_token, int]) -> bytes:
2023-10-24 07:13:32 +00:00
return _lib.llama_token_get_text(model, token)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_get_text.argtypes = [llama_model_p, llama_token]
2023-08-24 04:17:00 +00:00
_lib.llama_token_get_text.restype = c_char_p
2023-03-24 18:59:29 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token);
2023-11-20 19:11:33 +00:00
def llama_token_get_score(
model: llama_model_p, token: Union[llama_token, int]
) -> float:
2023-10-24 07:13:32 +00:00
return _lib.llama_token_get_score(model, token)
2023-03-25 20:26:03 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_get_score.argtypes = [llama_model_p, llama_token]
2023-08-24 04:17:00 +00:00
_lib.llama_token_get_score.restype = c_float
2023-03-25 20:26:03 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token);
2023-11-05 21:57:10 +00:00
def llama_token_get_type(model: llama_model_p, token: Union[llama_token, int]) -> int:
2023-10-24 07:13:32 +00:00
return _lib.llama_token_get_type(model, token)
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_get_type.argtypes = [llama_model_p, llama_token]
2023-08-24 04:17:00 +00:00
_lib.llama_token_get_type.restype = ctypes.c_int
2023-07-15 19:11:01 +00:00
2023-08-24 04:17:00 +00:00
# // Special tokens
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence
def llama_token_bos(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""beginning-of-sentence"""
2023-10-24 07:13:32 +00:00
return _lib.llama_token_bos(model)
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_bos.argtypes = [llama_model_p]
2023-08-24 04:17:00 +00:00
_lib.llama_token_bos.restype = llama_token
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence
def llama_token_eos(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""end-of-sentence"""
2023-10-24 07:13:32 +00:00
return _lib.llama_token_eos(model)
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_eos.argtypes = [llama_model_p]
2023-08-24 04:17:00 +00:00
_lib.llama_token_eos.restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
def llama_token_nl(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""next-line"""
2023-10-24 07:13:32 +00:00
return _lib.llama_token_nl(model)
2023-08-24 04:17:00 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_nl.argtypes = [llama_model_p]
2023-08-24 04:17:00 +00:00
_lib.llama_token_nl.restype = llama_token
2023-11-20 19:11:33 +00:00
# // Returns -1 if unknown, 1 for true or 0 for false.
# LLAMA_API int llama_add_bos_token(const struct llama_model * model);
def llama_add_bos_token(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns -1 if unknown, 1 for true or 0 for false."""
2023-11-20 19:11:33 +00:00
return _lib.llama_add_bos_token(model)
_lib.llama_add_bos_token.argtypes = [llama_model_p]
_lib.llama_add_bos_token.restype = c_int
# // Returns -1 if unknown, 1 for true or 0 for false.
# LLAMA_API int llama_add_eos_token(const struct llama_model * model);
def llama_add_eos_token(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""Returns -1 if unknown, 1 for true or 0 for false."""
2023-11-20 19:11:33 +00:00
return _lib.llama_add_eos_token(model)
_lib.llama_add_eos_token.argtypes = [llama_model_p]
_lib.llama_add_eos_token.restype = c_int
2023-10-03 19:23:35 +00:00
# // codellama infill tokens
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
def llama_token_prefix(model: llama_model_p) -> int:
2023-11-23 05:26:26 +00:00
"""codellama infill tokens"""
2023-10-24 07:13:32 +00:00
return _lib.llama_token_prefix(model)
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_prefix.argtypes = [llama_model_p]
2023-10-03 19:23:35 +00:00
_lib.llama_token_prefix.restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle
def llama_token_middle(model: llama_model_p) -> int:
return _lib.llama_token_middle(model)
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_middle.argtypes = [llama_model_p]
2023-10-03 19:23:35 +00:00
_lib.llama_token_middle.restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix
def llama_token_suffix(model: llama_model_p) -> int:
return _lib.llama_token_suffix(model)
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_suffix.argtypes = [llama_model_p]
2023-10-03 19:23:35 +00:00
_lib.llama_token_suffix.restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle
def llama_token_eot(model: llama_model_p) -> int:
return _lib.llama_token_eot(model)
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_token_eot.argtypes = [llama_model_p]
2023-10-03 19:23:35 +00:00
_lib.llama_token_eot.restype = llama_token
2023-08-24 04:17:00 +00:00
# //
# // Tokenization
# //
2023-10-19 06:55:08 +00:00
# /// @details Convert the provided text into tokens.
# /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
# /// @return Returns the number of tokens on success, no more than n_max_tokens
# /// @return Returns a negative number on failure - the number of tokens that would have been returned
# /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext.
# /// Does not insert a leading space.
# LLAMA_API int llama_tokenize(
# const struct llama_model * model,
# const char * text,
# int text_len,
# llama_token * tokens,
# int n_max_tokens,
# bool add_bos,
# bool special);
def llama_tokenize(
model: llama_model_p,
text: bytes,
text_len: Union[c_int, int],
tokens, # type: Array[llama_token]
n_max_tokens: Union[c_int, int],
add_bos: Union[c_bool, bool],
special: Union[c_bool, bool],
) -> int:
2023-11-23 05:26:26 +00:00
"""Convert the provided text into tokens."""
2023-10-19 06:55:08 +00:00
return _lib.llama_tokenize(
model, text, text_len, tokens, n_max_tokens, add_bos, special
)
_lib.llama_tokenize.argtypes = [
llama_model_p,
c_char_p,
c_int,
llama_token_p,
c_int,
c_bool,
c_bool,
]
_lib.llama_tokenize.restype = c_int
2023-08-27 16:59:20 +00:00
# // Token Id -> Piece.
# // Uses the vocabulary in the provided context.
# // Does not write null terminator to the buffer.
# // User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.
# LLAMA_API int llama_token_to_piece(
2023-09-29 02:42:03 +00:00
# const struct llama_model * model,
# llama_token token,
# char * buf,
# int length);
2023-08-27 16:59:20 +00:00
def llama_token_to_piece(
2023-09-29 02:42:03 +00:00
model: llama_model_p,
2023-11-05 21:57:10 +00:00
token: Union[llama_token, int],
2023-09-29 02:42:03 +00:00
buf: Union[c_char_p, bytes],
length: Union[c_int, int],
2023-08-24 04:17:00 +00:00
) -> int:
2023-11-23 05:26:26 +00:00
"""Token Id -> Piece.
Uses the vocabulary in the provided context.
Does not write null terminator to the buffer.
User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens."""
2023-09-29 02:42:03 +00:00
return _lib.llama_token_to_piece(model, token, buf, length)
2023-03-24 18:59:29 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_token_to_piece.argtypes = [llama_model_p, llama_token, c_char_p, c_int]
2023-08-27 16:59:20 +00:00
_lib.llama_token_to_piece.restype = c_int
2023-03-24 18:59:29 +00:00
2023-08-24 04:17:00 +00:00
# //
2023-07-24 17:04:34 +00:00
# // Grammar
# //
2023-08-24 04:17:00 +00:00
2023-07-24 17:04:34 +00:00
# LLAMA_API struct llama_grammar * llama_grammar_init(
# const llama_grammar_element ** rules,
# size_t n_rules,
# size_t start_rule_index);
def llama_grammar_init(
rules, # type: Array[llama_grammar_element_p] # type: ignore
n_rules: Union[c_size_t, int],
start_rule_index: Union[c_size_t, int],
2023-07-24 17:04:34 +00:00
) -> llama_grammar_p:
return _lib.llama_grammar_init(rules, n_rules, start_rule_index)
_lib.llama_grammar_init.argtypes = [
POINTER(llama_grammar_element_p),
c_size_t,
c_size_t,
]
_lib.llama_grammar_init.restype = llama_grammar_p
# LLAMA_API void llama_grammar_free(struct llama_grammar * grammar);
def llama_grammar_free(grammar: llama_grammar_p):
return _lib.llama_grammar_free(grammar)
_lib.llama_grammar_free.argtypes = [llama_grammar_p]
_lib.llama_grammar_free.restype = None
2023-08-25 18:35:53 +00:00
2023-09-09 16:12:32 +00:00
# LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar);
def llama_grammar_copy(grammar: llama_grammar_p) -> llama_grammar_p:
return _lib.llama_grammar_copy(grammar)
2023-08-25 18:35:53 +00:00
2023-09-09 16:12:32 +00:00
_lib.llama_grammar_copy.argtypes = [llama_grammar_p]
_lib.llama_grammar_copy.restype = llama_grammar_p
2023-07-24 17:04:34 +00:00
2023-08-24 04:17:00 +00:00
# //
# // Sampling functions
# //
2023-05-01 18:02:06 +00:00
2023-09-29 02:42:03 +00:00
# // Sets the current rng seed.
# LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed);
def llama_set_rng_seed(ctx: llama_context_p, seed: Union[c_uint32, int]):
2023-11-23 05:26:26 +00:00
"""Sets the current rng seed."""
2023-09-29 02:42:03 +00:00
return _lib.llama_set_rng_seed(ctx, seed)
_lib.llama_set_rng_seed.argtypes = [llama_context_p, c_uint32]
_lib.llama_set_rng_seed.restype = None
2023-10-24 07:13:32 +00:00
# /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
# /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
# LLAMA_API void llama_sample_repetition_penalties(
2023-09-29 02:42:03 +00:00
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# const llama_token * last_tokens,
2023-10-24 07:13:32 +00:00
# size_t penalty_last_n,
# float penalty_repeat,
# float penalty_freq,
# float penalty_present);
def llama_sample_repetition_penalties(
2023-05-01 14:44:28 +00:00
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
last_tokens_data, # type: Array[llama_token]
2023-10-24 07:13:32 +00:00
penalty_last_n: Union[c_size_t, int],
penalty_repeat: Union[c_float, float],
penalty_freq: Union[c_float, float],
penalty_present: Union[c_float, float],
2023-05-01 18:02:06 +00:00
):
2023-11-23 05:26:26 +00:00
"""Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details."""
2023-10-24 07:13:32 +00:00
return _lib.llama_sample_repetition_penalties(
2023-05-01 14:44:28 +00:00
ctx,
candidates,
last_tokens_data,
2023-10-24 07:13:32 +00:00
penalty_last_n,
penalty_repeat,
penalty_freq,
penalty_present,
2023-03-24 18:35:41 +00:00
)
2023-03-23 09:33:06 +00:00
2023-10-24 07:13:32 +00:00
_lib.llama_sample_repetition_penalties.argtypes = [
llama_context_p,
2023-05-01 14:44:28 +00:00
llama_token_data_array_p,
llama_token_p,
2023-10-24 07:13:32 +00:00
c_size_t,
c_float,
2023-05-01 14:44:28 +00:00
c_float,
c_float,
]
2023-10-24 07:13:32 +00:00
_lib.llama_sample_repetition_penalties.restype = None
2023-05-01 14:44:28 +00:00
2023-07-15 19:11:01 +00:00
# /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted.
# /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.
# /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.
# LLAMA_API void llama_sample_classifier_free_guidance(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# struct llama_context * guidance_ctx,
2023-07-21 16:33:27 +00:00
# float scale);
2023-07-15 19:11:01 +00:00
def llama_sample_classifier_free_guidance(
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
guidance_ctx: llama_context_p,
scale: Union[c_float, float],
2023-07-15 19:11:01 +00:00
):
2023-11-23 05:26:26 +00:00
"""Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806"""
2023-07-15 19:11:01 +00:00
return _lib.llama_sample_classifier_free_guidance(
2023-07-21 16:33:27 +00:00
ctx, candidates, guidance_ctx, scale
2023-07-15 19:11:01 +00:00
)
_lib.llama_sample_classifier_free_guidance.argtypes = [
llama_context_p,
llama_token_data_array_p,
llama_context_p,
c_float,
]
_lib.llama_sample_classifier_free_guidance.restype = None
2023-11-01 01:29:35 +00:00
# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_softmax(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
2023-05-05 18:12:26 +00:00
def llama_sample_softmax(
ctx: llama_context_p, candidates # type: _Pointer[llama_token_data]
):
2023-11-23 05:26:26 +00:00
"""Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits."""
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_softmax(ctx, candidates)
_lib.llama_sample_softmax.argtypes = [
llama_context_p,
llama_token_data_array_p,
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_softmax.restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_top_k(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# int k,
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_top_k(
2023-05-05 17:54:22 +00:00
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
k: Union[c_int, int],
min_keep: Union[c_size_t, int],
2023-05-01 18:47:55 +00:00
):
2023-11-23 05:26:26 +00:00
"""Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751"""
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_top_k(ctx, candidates, k, min_keep)
_lib.llama_sample_top_k.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_int,
2023-05-01 18:47:55 +00:00
c_size_t,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_top_k.restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_top_p(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_top_p(
2023-05-05 17:54:22 +00:00
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
p: Union[c_float, float],
min_keep: Union[c_size_t, int],
2023-05-01 18:47:55 +00:00
):
2023-11-23 05:26:26 +00:00
"""Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751"""
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_top_p(ctx, candidates, p, min_keep)
_lib.llama_sample_top_p.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
2023-05-01 18:47:55 +00:00
c_size_t,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_top_p.restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
# LLAMA_API void llama_sample_min_p(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
def llama_sample_min_p(
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
p: Union[c_float, float],
min_keep: Union[c_size_t, int],
):
2023-11-23 05:26:26 +00:00
"""Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841"""
2023-11-01 01:29:35 +00:00
return _lib.llama_sample_min_p(ctx, candidates, p, min_keep)
_lib.llama_sample_min_p.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
c_size_t,
]
_lib.llama_sample_min_p.restype = None
# /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_tail_free(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float z,
# size_t min_keep);
2023-05-01 14:44:28 +00:00
def llama_sample_tail_free(
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
z: Union[c_float, float],
min_keep: Union[c_size_t, int],
2023-05-01 18:02:06 +00:00
):
2023-11-23 05:26:26 +00:00
"""Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/."""
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_tail_free(ctx, candidates, z, min_keep)
_lib.llama_sample_tail_free.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
2023-05-01 18:47:55 +00:00
c_size_t,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_tail_free.restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_typical(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_typical(
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
p: Union[c_float, float],
min_keep: Union[c_size_t, int],
2023-05-01 18:47:55 +00:00
):
2023-11-23 05:26:26 +00:00
"""Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666."""
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_typical(ctx, candidates, p, min_keep)
_lib.llama_sample_typical.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
2023-05-01 18:47:55 +00:00
c_size_t,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_typical.restype = None
2023-05-01 14:44:28 +00:00
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_temp(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float temp);
def llama_sample_temp(
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
temp: Union[c_float, float],
):
return _lib.llama_sample_temp(ctx, candidates, temp)
_lib.llama_sample_temp.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
]
_lib.llama_sample_temp.restype = None
# LLAMA_API DEPRECATED(void llama_sample_temperature(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float temp),
# "use llama_sample_temp instead");
def llama_sample_temperature(
2023-05-05 18:12:26 +00:00
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
temp: Union[c_float, float],
):
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_temperature(ctx, candidates, temp)
_lib.llama_sample_temperature.argtypes = [
llama_context_p,
llama_token_data_array_p,
2023-03-29 01:10:23 +00:00
c_float,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib.llama_sample_temperature.restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Apply constraints from grammar
# LLAMA_API void llama_sample_grammar(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# const struct llama_grammar * grammar);
2023-07-24 19:42:31 +00:00
def llama_sample_grammar(
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
2023-08-05 05:43:35 +00:00
grammar, # type: llama_grammar_p
2023-07-24 19:42:31 +00:00
):
return _lib.llama_sample_grammar(ctx, candidates, grammar)
_lib.llama_sample_grammar.argtypes = [
llama_context_p,
llama_token_data_array_p,
llama_grammar_p,
]
_lib.llama_sample_grammar.restype = None
2023-11-01 01:29:35 +00:00
# /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_mirostat(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float tau,
# float eta,
# int m,
# float * mu);
2023-05-01 14:44:28 +00:00
def llama_sample_token_mirostat(
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
tau: Union[c_float, float],
eta: Union[c_float, float],
m: Union[c_int, int],
2023-05-05 18:12:26 +00:00
mu, # type: _Pointer[c_float]
) -> int:
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_token_mirostat(ctx, candidates, tau, eta, m, mu)
_lib.llama_sample_token_mirostat.argtypes = [
llama_context_p,
llama_token_data_array_p,
c_float,
c_float,
c_int,
2023-05-05 18:04:12 +00:00
c_float_p,
2023-05-01 14:44:28 +00:00
]
_lib.llama_sample_token_mirostat.restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_mirostat_v2(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float tau,
# float eta,
# float * mu);
2023-05-01 14:44:28 +00:00
def llama_sample_token_mirostat_v2(
ctx: llama_context_p,
2023-05-05 18:12:26 +00:00
candidates, # type: _Pointer[llama_token_data_array]
tau: Union[c_float, float],
eta: Union[c_float, float],
2023-05-05 18:12:26 +00:00
mu, # type: _Pointer[c_float]
) -> int:
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_token_mirostat_v2(ctx, candidates, tau, eta, mu)
_lib.llama_sample_token_mirostat_v2.argtypes = [
llama_context_p,
llama_token_data_array_p,
2023-03-29 01:10:23 +00:00
c_float,
c_float,
2023-05-05 18:04:12 +00:00
c_float_p,
2023-05-01 14:44:28 +00:00
]
_lib.llama_sample_token_mirostat_v2.restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Selects the token with the highest probability.
# /// Does not compute the token probabilities. Use llama_sample_softmax() instead.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_greedy(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
def llama_sample_token_greedy(
2023-05-05 18:12:26 +00:00
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
) -> int:
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_token_greedy(ctx, candidates)
_lib.llama_sample_token_greedy.argtypes = [
llama_context_p,
llama_token_data_array_p,
]
_lib.llama_sample_token_greedy.restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Randomly selects a token from the candidates based on their probabilities.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
def llama_sample_token(
2023-05-05 18:12:26 +00:00
ctx: llama_context_p,
candidates, # type: _Pointer[llama_token_data_array]
) -> int:
2023-05-01 14:44:28 +00:00
return _lib.llama_sample_token(ctx, candidates)
_lib.llama_sample_token.argtypes = [
llama_context_p,
llama_token_data_array_p,
]
2023-05-01 14:44:28 +00:00
_lib.llama_sample_token.restype = llama_token
2023-07-24 19:55:26 +00:00
# /// @details Accepts the sampled token into the grammar
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_grammar_accept_token(
# struct llama_context * ctx,
# struct llama_grammar * grammar,
# llama_token token);
2023-07-24 19:55:26 +00:00
def llama_grammar_accept_token(
2023-08-05 05:43:35 +00:00
ctx: llama_context_p,
grammar: llama_grammar_p,
2023-11-05 21:57:10 +00:00
token: Union[llama_token, int],
2023-08-05 05:43:35 +00:00
) -> None:
_lib.llama_grammar_accept_token(ctx, grammar, token)
2023-07-24 19:55:26 +00:00
_lib.llama_grammar_accept_token.argtypes = [
llama_context_p,
llama_grammar_p,
llama_token,
]
_lib.llama_grammar_accept_token.restype = None
2023-09-29 02:42:03 +00:00
2023-09-09 16:12:32 +00:00
# //
# // Beam search
# //
# struct llama_beam_view {
# const llama_token * tokens;
2023-09-29 02:42:03 +00:00
2023-10-03 19:23:35 +00:00
2023-09-09 16:12:32 +00:00
# size_t n_tokens;
2023-09-29 02:42:03 +00:00
# float p; // Cumulative beam probability (renormalized relative to all beams)
# bool eob; // Callback should set this to true when a beam is at end-of-beam.
2023-09-09 16:12:32 +00:00
# };
class llama_beam_view(ctypes.Structure):
_fields_ = [
("tokens", llama_token_p),
("n_tokens", c_size_t),
("p", c_float),
("eob", c_bool),
]
# // Passed to beam_search_callback function.
# // Whenever 0 < common_prefix_length, this number of tokens should be copied from any of the beams
# // (e.g. beams[0]) as they will be removed (shifted) from all beams in all subsequent callbacks.
# // These pointers are valid only during the synchronous callback, so should not be saved.
# struct llama_beams_state {
# struct llama_beam_view * beam_views;
# size_t n_beams; // Number of elements in beam_views[].
# size_t common_prefix_length; // Current max length of prefix tokens shared by all beams.
2023-09-29 02:42:03 +00:00
# bool last_call; // True iff this is the last callback invocation.
2023-09-09 16:12:32 +00:00
# };
class llama_beams_state(ctypes.Structure):
_fields_ = [
("beam_views", POINTER(llama_beam_view)),
("n_beams", c_size_t),
("common_prefix_length", c_size_t),
("last_call", c_bool),
]
# // Type of pointer to the beam_search_callback function.
# // void* callback_data is any custom data passed to llama_beam_search, that is subsequently
# // passed back to beam_search_callback. This avoids having to use global variables in the callback.
# typedef void (*llama_beam_search_callback_fn_t)(void * callback_data, struct llama_beams_state);
llama_beam_search_callback_fn_t = ctypes.CFUNCTYPE(None, c_void_p, llama_beams_state)
# /// @details Deterministically returns entire sentence constructed by a beam search.
# /// @param ctx Pointer to the llama_context.
# /// @param callback Invoked for each iteration of the beam_search loop, passing in beams_state.
# /// @param callback_data A pointer that is simply passed back to callback.
# /// @param n_beams Number of beams to use.
# /// @param n_past Number of tokens already evaluated.
# /// @param n_predict Maximum number of tokens to predict. EOS may occur earlier.
# /// @param n_threads Number of threads as passed to llama_eval().
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_beam_search(
# struct llama_context * ctx,
# llama_beam_search_callback_fn_t callback,
# void * callback_data,
# size_t n_beams,
# int n_past,
# int n_predict);
2023-09-09 16:12:32 +00:00
def llama_beam_search(
ctx: llama_context_p,
callback: "ctypes._CFuncPtr[None, c_void_p, llama_beams_state]", # type: ignore
callback_data: c_void_p,
n_beams: Union[c_size_t, int],
n_past: Union[c_int, int],
n_predict: Union[c_int, int],
2023-09-09 16:12:32 +00:00
):
return _lib.llama_beam_search(
2023-09-29 02:42:03 +00:00
ctx, callback, callback_data, n_beams, n_past, n_predict
2023-09-09 16:12:32 +00:00
)
2023-10-03 19:23:35 +00:00
2023-09-29 02:42:03 +00:00
_lib.llama_beam_search.argtypes = [
llama_context_p,
llama_beam_search_callback_fn_t,
c_void_p,
c_size_t,
c_int,
c_int,
]
_lib.llama_beam_search.restype = None
2023-07-24 19:55:26 +00:00
# Performance information
2023-03-24 18:59:29 +00:00
2023-07-06 21:57:56 +00:00
# LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx);
def llama_get_timings(ctx: llama_context_p) -> llama_timings:
return _lib.llama_get_timings(ctx)
_lib.llama_get_timings.argtypes = [llama_context_p]
_lib.llama_get_timings.restype = llama_timings
2023-05-21 21:47:21 +00:00
# LLAMA_API void llama_print_timings(struct llama_context * ctx);
2023-03-23 09:33:06 +00:00
def llama_print_timings(ctx: llama_context_p):
_lib.llama_print_timings(ctx)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
_lib.llama_print_timings.argtypes = [llama_context_p]
_lib.llama_print_timings.restype = None
2023-05-21 21:47:21 +00:00
# LLAMA_API void llama_reset_timings(struct llama_context * ctx);
2023-03-23 09:33:06 +00:00
def llama_reset_timings(ctx: llama_context_p):
_lib.llama_reset_timings(ctx)
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
_lib.llama_reset_timings.argtypes = [llama_context_p]
_lib.llama_reset_timings.restype = None
# Print system information
2023-05-21 21:47:21 +00:00
# LLAMA_API const char * llama_print_system_info(void);
2023-03-23 09:33:06 +00:00
def llama_print_system_info() -> bytes:
return _lib.llama_print_system_info()
_lib.llama_print_system_info.argtypes = []
_lib.llama_print_system_info.restype = c_char_p
2023-05-21 21:47:21 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# NOTE: THIS IS CURRENTLY BROKEN AS ggml_log_callback IS NOT EXPOSED IN LLAMA.H
2023-08-24 04:17:00 +00:00
# // Set callback for all future logging events.
# // If this is not called, or NULL is supplied, everything is output on stderr.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
2023-08-24 04:17:00 +00:00
def llama_log_set(
log_callback: "ctypes._FuncPointer", user_data: c_void_p # type: ignore
):
return _lib.llama_log_set(log_callback, user_data)
_lib.llama_log_set.argtypes = [llama_log_callback, c_void_p]
_lib.llama_log_set.restype = None
2023-08-29 11:36:20 +00:00
# LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx);
def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p):
return _lib.llama_dump_timing_info_yaml(stream, ctx)
2023-09-09 16:12:32 +00:00
2023-08-29 11:36:20 +00:00
_lib.llama_dump_timing_info_yaml.argtypes = [ctypes.c_void_p, llama_context_p]
_lib.llama_dump_timing_info_yaml.restype = None