12 lines
339 B
Python
12 lines
339 B
Python
import tiktoken
|
|
|
|
|
|
def test_simple():
|
|
enc = tiktoken.get_encoding("gpt2")
|
|
assert enc.encode("hello world") == [31373, 995]
|
|
assert enc.decode([31373, 995]) == "hello world"
|
|
|
|
enc = tiktoken.get_encoding("cl100k_base")
|
|
assert enc.encode("hello world") == [15339, 1917]
|
|
assert enc.decode([15339, 1917]) == "hello world"
|