Fix CI tests for Python and JNI. (#554)
This commit is contained in:
@@ -4,12 +4,6 @@ import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
from pypinyin import pinyin
|
||||
from pypinyin.contrib.tone_convert import to_initials, to_finals_tone
|
||||
|
||||
|
||||
def text2token(
|
||||
texts: List[str],
|
||||
tokens: str,
|
||||
@@ -38,6 +32,23 @@ def text2token(
|
||||
Return the encoded texts, it is a list of a list of token ids if output_ids
|
||||
is True, or it is a list of list of tokens.
|
||||
"""
|
||||
try:
|
||||
import sentencepiece as spm
|
||||
except ImportError:
|
||||
print('Please run')
|
||||
print(' pip install sentencepiece')
|
||||
print('before you continue')
|
||||
raise
|
||||
|
||||
try:
|
||||
from pypinyin import pinyin
|
||||
from pypinyin.contrib.tone_convert import to_initials, to_finals_tone
|
||||
except ImportError:
|
||||
print('Please run')
|
||||
print(' pip install pypinyin')
|
||||
print('before you continue')
|
||||
raise
|
||||
|
||||
assert Path(tokens).is_file(), f"File not exists, {tokens}"
|
||||
tokens_table = {}
|
||||
with open(tokens, "r", encoding="utf-8") as f:
|
||||
|
||||
Reference in New Issue
Block a user