Unpin the version of tokenizers fix: #1849

hankcs · Oct 14, 2023 · 4c1f457 · 4c1f457
1 parent 698c7e8
commit 4c1f457
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 2 deletions.
diff --git a/hanlp/version.py b/hanlp/version.py
@@ -2,7 +2,7 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-beta.50'
+__version__ = '2.1.0-beta.51'
 """HanLP version"""
 
 

diff --git a/setup.py b/setup.py
@@ -61,7 +61,6 @@
         'pynvml',
         'toposort==1.5',
         'transformers>=4.1.1',
-        'tokenizers==0.11.6',  # The latest tokenizers==0.12.1 failed to compile on macOS Python3.6
         'sentencepiece>=0.1.91',  # Essential for tokenization_bert_japanese
         'torch>=1.6.0',
         'hanlp-common>=0.0.19',