Fix CI tests for Python and JNI. (#554)

This commit is contained in:
Fangjun Kuang
2024-01-27 13:01:54 +08:00
committed by GitHub
parent 7ae73e75ba
commit 44efff4e47
7 changed files with 40 additions and 20 deletions

View File

@@ -4,12 +4,6 @@ import re
from pathlib import Path
from typing import List, Optional, Union
import sentencepiece as spm
from pypinyin import pinyin
from pypinyin.contrib.tone_convert import to_initials, to_finals_tone
def text2token(
texts: List[str],
tokens: str,
@@ -38,6 +32,23 @@ def text2token(
Return the encoded texts, it is a list of a list of token ids if output_ids
is True, or it is a list of list of tokens.
"""
try:
import sentencepiece as spm
except ImportError:
print('Please run')
print(' pip install sentencepiece')
print('before you continue')
raise
try:
from pypinyin import pinyin
from pypinyin.contrib.tone_convert import to_initials, to_finals_tone
except ImportError:
print('Please run')
print(' pip install pypinyin')
print('before you continue')
raise
assert Path(tokens).is_file(), f"File not exists, {tokens}"
tokens_table = {}
with open(tokens, "r", encoding="utf-8") as f: