Simplify tokenizer manager (#1904)

2024-11-03 08:38:26 -08:00
parent 916b3cdddc
commit c17c578108
11 changed files with 261 additions and 443 deletions
--- a/test/srt/test_skip_tokenizer_init.py
+++ b/test/srt/test_skip_tokenizer_init.py
@@ -1,3 +1,6 @@
+"""
+python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.test_parallel_sample
+"""
 import json
 import unittest