51 lines
1.2 KiB
Python
51 lines
1.2 KiB
Python
|
from typing import List
|
|||
|
|
|
|||
|
|
from utils.tokenizer import TokenizerType
|
|||
|
|
|
|||
|
|
|
|||
|
|
def replace_general_punc(
|
|||
|
|
sentences: List[str], tokenizer: TokenizerType
|
|||
|
|
) -> List[str]:
|
|||
|
|
"""代替原来的函数 utils.metrics.cut_sentence"""
|
|||
|
|
general_puncs = [
|
|||
|
|
"······",
|
|||
|
|
"......",
|
|||
|
|
"。",
|
|||
|
|
"，",
|
|||
|
|
"？",
|
|||
|
|
"！",
|
|||
|
|
"；",
|
|||
|
|
"：",
|
|||
|
|
"...",
|
|||
|
|
".",
|
|||
|
|
",",
|
|||
|
|
"?",
|
|||
|
|
"!",
|
|||
|
|
";",
|
|||
|
|
":",
|
|||
|
|
]
|
|||
|
|
if tokenizer == TokenizerType.whitespace:
|
|||
|
|
replacer = " "
|
|||
|
|
else:
|
|||
|
|
replacer = ""
|
|||
|
|
trans = str.maketrans(dict.fromkeys("".join(general_puncs), replacer))
|
|||
|
|
ret_sentences = [""] * len(sentences)
|
|||
|
|
for i, sentence in enumerate(sentences):
|
|||
|
|
sentence = sentence.translate(trans)
|
|||
|
|
sentence = sentence.strip()
|
|||
|
|
sentence = sentence.lower()
|
|||
|
|
ret_sentences[i] = sentence
|
|||
|
|
return ret_sentences
|
|||
|
|
|
|||
|
|
|
|||
|
|
def distance_point_line(
|
|||
|
|
point: float, line_start: float, line_end: float
|
|||
|
|
) -> float:
|
|||
|
|
"""计算点到直线的距离"""
|
|||
|
|
if line_start <= point <= line_end:
|
|||
|
|
return 0
|
|||
|
|
if point < line_start:
|
|||
|
|
return abs(point - line_start)
|
|||
|
|
else:
|
|||
|
|
return abs(point - line_end)
|