From cf385cada0aa96f1abc775c2cc6c5b9b6884f0b0 Mon Sep 17 00:00:00 2001 From: Arvid Lunnemark Date: Thu, 19 Jan 2023 14:51:15 -0800 Subject: [PATCH] Fix docstring, type annotation for private method (#19) --- tiktoken/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiktoken/core.py b/tiktoken/core.py index d2367bc..b3a475a 100644 --- a/tiktoken/core.py +++ b/tiktoken/core.py @@ -104,7 +104,7 @@ class Encoding: This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster). ``` - >>> enc.encode_batch(["hello world", "goodbye world"]) + >>> enc.encode_ordinary_batch(["hello world", "goodbye world"]) [[31373, 995], [11274, 16390, 995]] ``` """ @@ -285,7 +285,7 @@ class Encoding: text_or_bytes = text_or_bytes.encode("utf-8") return self._core_bpe.encode_single_piece(text_or_bytes) - def _encode_only_native_bpe(self, text: str) -> list[str]: + def _encode_only_native_bpe(self, text: str) -> list[int]: """Encodes a string into tokens, but do regex splitting in Python.""" _unused_pat = regex.compile(self._pat_str) ret = []