21 lines
447 B
Python
21 lines
447 B
Python
import json
|
|
import argparse
|
|
|
|
from llama_cpp import Llama
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-m", "--model", type=str, default="./models/...")
|
|
args = parser.parse_args()
|
|
|
|
llm = Llama(model_path=args.model)
|
|
|
|
stream = llm(
|
|
"Question: What are the names of the planets in the solar system? Answer: ",
|
|
max_tokens=48,
|
|
stop=["Q:", "\n"],
|
|
stream=True,
|
|
)
|
|
|
|
for output in stream:
|
|
print(json.dumps(output, indent=2))
|