init

2025-10-09 16:47:16 +08:00
parent c8feb4deb5
commit e27e3f16bb
5248 changed files with 1778505 additions and 0 deletions
--- a/transformers/utils/test_module/custom_tokenization_fast.py
+++ b/transformers/utils/test_module/custom_tokenization_fast.py
@@ -0,0 +1,8 @@
+from transformers import BertTokenizerFast
+
+from .custom_tokenization import CustomTokenizer
+
+
+class CustomTokenizerFast(BertTokenizerFast):
+    slow_tokenizer_class = CustomTokenizer
+    pass