From 4226a6c008e394bb9fe5de688b1849ca4a88b27f Mon Sep 17 00:00:00 2001 From: Henrik Torget Date: Fri, 16 Dec 2022 22:55:30 +0100 Subject: [PATCH] Remove duplicate words in docstring (#3) --- tiktoken/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiktoken/core.py b/tiktoken/core.py index e04f425..523c278 100644 --- a/tiktoken/core.py +++ b/tiktoken/core.py @@ -59,7 +59,7 @@ class Encoding: ) -> list[int]: """Encodes a string into tokens. - Special tokens are tokens are artificial tokens used to unlock capabilities from a model, + Special tokens are artificial tokens used to unlock capabilities from a model, such as fill-in-the-middle. So we want to be careful about accidentally encoding special tokens, since they can be used to trick a model into doing something we don't want it to do.