[Minor] Improve the function organization in TokenizerManager & improve loggers (#1208)

This commit is contained in:
Lianmin Zheng
2024-08-25 14:46:34 -07:00
committed by GitHub
parent 30b4f771b0
commit 902278008a
12 changed files with 137 additions and 134 deletions

View File

@@ -142,17 +142,6 @@ def get_tokenizer(
raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
kwargs["use_fast"] = False
if (
"llama" in tokenizer_name.lower()
and kwargs.get("use_fast", True)
and tokenizer_name != _FAST_LLAMA_TOKENIZER
):
warnings.warn(
"For some LLaMA V1 models, initializing the fast tokenizer may "
"take a long time. To reduce the initialization time, consider "
f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
"tokenizer."
)
try:
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name,