Bugfix: n_batch should always be <= n_ctx

2023-04-04 13:08:21 -04:00 · 2023-04-04 13:08:21 -04:00 · 5075c16fcc
parent 248b0566fa
commit 5075c16fcc
1 changed files with 1 additions and 1 deletions
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@ -68,7 +68,7 @@ class Llama:
            maxlen=self.last_n_tokens_size,
        )
        self.tokens_consumed = 0
-        self.n_batch = n_batch
+        self.n_batch = min(n_ctx, n_batch)

        self.n_threads = n_threads or multiprocessing.cpu_count()