baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:abetlen/llama_cpp_python into main

Browse source
This commit is contained in:
Andrei Betlen 2024-04-30 15:45:36 -04:00
parent f116175a5a 3226b3c5ef
commit 9286b5caac
1 changed files with 5 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
llama_cpp/llama_grammar.py
View file

@ -556,17 +556,11 @@ def add_rule(
# }
def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]:
"""Decodes a UTF-8 character from the source string."""
lookup = (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4)
first_byte = ord(src[0]) # type: int
highbits = first_byte >> 4 # type: int
len = lookup[highbits] # type: int
mask = (1 << (8 - len)) - 1 # type: int
value = first_byte & mask # type: int
end = src + len # type: const_char_p # may overrun!
pos = src + 1 # type: const_char_p
while pos < end and pos[0]:
value = (value << 6) + (ord(pos[0]) & 0x3F)
pos += 1
# Get the codepoint of the first character
value = ord(src[0])
# Move the pointer ahead one character
pos = src + 1
return value, pos