baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Adds openai-processing-ms response header (#748)

Browse source
This commit is contained in:
Viacheslav/Slava Tradunsky 2023-09-25 13:55:58 -04:00 committed by GitHub
parent dbca136fea
commit 3d5e5b1c04
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llama_cpp/server/app.py
View file

@ -1,5 +1,6 @@
import json import json
import multiprocessing import multiprocessing
import time
from re import compile, Match, Pattern from re import compile, Match, Pattern
from threading import Lock from threading import Lock
from functools import partial from functools import partial
@ -271,7 +272,11 @@ class RouteErrorHandler(APIRoute):
async def custom_route_handler(request: Request) -> Response: async def custom_route_handler(request: Request) -> Response:
try: try:
return await original_route_handler(request) start_sec = time.perf_counter()
response = await original_route_handler(request)
elapsed_time_ms = int((time.perf_counter() - start_sec) * 1000)
response.headers["openai-processing-ms"] = f"{elapsed_time_ms}"
return response
except Exception as exc: except Exception as exc:
json_body = await request.json() json_body = await request.json()
try: try: