GabrielTofvesson/tiktoken
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync codebase

Browse Source
This commit is contained in:
Shantanu Jain 2023-02-25 21:01:46 -08:00
parent c4b8770184
commit fbaa86e0f0
2 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -93,12 +93,12 @@ from setuptools import setup, find_namespace_packages
setup(
name="my_tiktoken_extension",
packages=find_namespace_packages(include=['tiktoken_ext.*'])
packages=find_namespace_packages(include=['tiktoken_ext*']),
install_requires=["tiktoken"],
...
)
```
Then simply `pip install my_tiktoken_extension` and you should be able to use your custom encodings!
Make sure **not** to use an editable install.
Then simply `pip install ./my_tiktoken_extension` and you should be able to use your
custom encodings! Make sure **not** to use an editable install.

9
pyproject.toml
View File

@ -1,9 +1,18 @@
[project]
name = "tiktoken"
version = "0.2.0"
description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
readme = "README.md"
license = {file = "LICENSE"}
authors = [{name = "Shantanu Jain"}, {email = "shantanu@openai.com"}]
dependencies = ["blobfile>=2", "regex>=2022.1.18", "requests>=2.26.0"]
requires-python = ">=3.8"
[project.urls]
homepage = "https://github.com/openai/tiktoken"
repository = "https://github.com/openai/tiktoken"
changelog = "https://github.com/openai/tiktoken/blob/main/CHANGELOG.md"
[build-system]
build-backend = "setuptools.build_meta"
requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"]