baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Fix streaming doesn't return finish reason (#798)

Browse source 
When streaming the yield that contains the finish can be skipped. This change ensures that yield isn't skipped.
This commit is contained in:
gmcgoldr 2023-10-19 02:55:56 -04:00 committed by GitHub
parent 28c2b884e2
commit 09a8406c83
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
llama_cpp/llama.py
View file

@ -1232,20 +1232,6 @@ class Llama:
}
],
}
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason,
}
],
}
break
returned_tokens += 1
yield {
@ -1264,20 +1250,20 @@ class Llama:
}
],
}
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason,
}
],
}
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason,
}
],
}
if self.cache:
if self.verbose:
print("Llama._create_completion: cache save", file=sys.stderr)