GabrielTofvesson/tiktoken
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync codebase

Browse Source
This commit is contained in:
Shantanu Jain 2023-02-25 21:01:46 -08:00
parent c4b8770184
commit fbaa86e0f0
2 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -93,12 +93,12 @@ from setuptools import setup, find_namespace_packages
setup( setup(
name="my_tiktoken_extension", name="my_tiktoken_extension",
packages=find_namespace_packages(include=['tiktoken_ext.*']) packages=find_namespace_packages(include=['tiktoken_ext*']),
install_requires=["tiktoken"], install_requires=["tiktoken"],
... ...
) )
``` ```
Then simply `pip install my_tiktoken_extension` and you should be able to use your custom encodings! Then simply `pip install ./my_tiktoken_extension` and you should be able to use your
Make sure **not** to use an editable install. custom encodings! Make sure **not** to use an editable install.

9
pyproject.toml
View File

@ -1,9 +1,18 @@
[project] [project]
name = "tiktoken" name = "tiktoken"
version = "0.2.0" version = "0.2.0"
description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
readme = "README.md"
license = {file = "LICENSE"}
authors = [{name = "Shantanu Jain"}, {email = "shantanu@openai.com"}]
dependencies = ["blobfile>=2", "regex>=2022.1.18", "requests>=2.26.0"] dependencies = ["blobfile>=2", "regex>=2022.1.18", "requests>=2.26.0"]
requires-python = ">=3.8" requires-python = ">=3.8"
[project.urls]
homepage = "https://github.com/openai/tiktoken"
repository = "https://github.com/openai/tiktoken"
changelog = "https://github.com/openai/tiktoken/blob/main/CHANGELOG.md"
[build-system] [build-system]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"] requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"]