Update setup.py

2 years ago · d077e27130
parent e75f77799c
commit d077e27130
1 changed files with 1 additions and 0 deletions
--- a/setup.py
+++ b/setup.py
@ -16,6 +16,7 @@ setup(
        "regex>=2022.3.15",
        "setuptools>=58.0.4",
        "transformers>=4.22.0,<=4.24.0",
+        "tokenizers>=0.11.0,<=0.11.4",
        "pytorch>=1.10.0",
        "tqdm>=4.63.0",
    ],