94 lines
2.2 KiB
Python
94 lines
2.2 KiB
Python
|
"""
|
|||
|
|
f(a, b) 计算 a -> b 的编辑距离，使用的方法是之前asr榜单的方法
|
|||
|
|
g(a, b) 计算 a -> b 的编辑距离，使用的是原始的编辑距离计算方法
|
|||
|
|
test() 是对拍程序
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import random
|
|||
|
|
import string
|
|||
|
|
from copy import deepcopy
|
|||
|
|
from typing import List, Tuple
|
|||
|
|
|
|||
|
|
import Levenshtein
|
|||
|
|
|
|||
|
|
|
|||
|
|
def mapping(gt: str, dt: str):
|
|||
|
|
return [i for i in gt], [i for i in dt]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def token_mapping(
|
|||
|
|
tokens_gt: List[str], tokens_dt: List[str]
|
|||
|
|
) -> Tuple[List[str], List[str]]:
|
|||
|
|
arr1 = deepcopy(tokens_gt)
|
|||
|
|
arr2 = deepcopy(tokens_dt)
|
|||
|
|
operations = Levenshtein.editops(arr1, arr2)
|
|||
|
|
for op in operations[::-1]:
|
|||
|
|
if op[0] == "insert":
|
|||
|
|
arr1.insert(op[1], None)
|
|||
|
|
elif op[0] == "delete":
|
|||
|
|
arr2.insert(op[2], None)
|
|||
|
|
return arr1, arr2
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cer(tokens_gt_mapping: List[str], tokens_dt_mapping: List[str]):
|
|||
|
|
"""输入的是经过编辑距离映射后的两个 token 序列，返回 1-cer, token-cnt"""
|
|||
|
|
insert = sum(1 for item in tokens_gt_mapping if item is None)
|
|||
|
|
delete = sum(1 for item in tokens_dt_mapping if item is None)
|
|||
|
|
equal = sum(
|
|||
|
|
1
|
|||
|
|
for token_gt, token_dt in zip(tokens_gt_mapping, tokens_dt_mapping)
|
|||
|
|
if token_gt == token_dt
|
|||
|
|
)
|
|||
|
|
replace = len(tokens_gt_mapping) - insert - equal # - delete
|
|||
|
|
return replace, delete, insert
|
|||
|
|
|
|||
|
|
|
|||
|
|
def f(a, b):
|
|||
|
|
return cer(*token_mapping(*mapping(a, b)))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def raw(tokens_gt, tokens_dt):
|
|||
|
|
arr1 = deepcopy(tokens_gt)
|
|||
|
|
arr2 = deepcopy(tokens_dt)
|
|||
|
|
operations = Levenshtein.editops(arr1, arr2)
|
|||
|
|
insert = 0
|
|||
|
|
delete = 0
|
|||
|
|
replace = 0
|
|||
|
|
for op in operations:
|
|||
|
|
if op[0] == "insert":
|
|||
|
|
insert += 1
|
|||
|
|
if op[0] == "delete":
|
|||
|
|
delete += 1
|
|||
|
|
if op[0] == "replace":
|
|||
|
|
replace += 1
|
|||
|
|
return replace, delete, insert
|
|||
|
|
|
|||
|
|
|
|||
|
|
def g(a, b):
|
|||
|
|
return raw(*mapping(a, b))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check(a, b):
|
|||
|
|
ff = f(a, b)
|
|||
|
|
gg = g(a, b)
|
|||
|
|
if ff != gg:
|
|||
|
|
print(ff, gg)
|
|||
|
|
return ff == gg
|
|||
|
|
|
|||
|
|
|
|||
|
|
def random_string(length):
|
|||
|
|
letters = string.ascii_lowercase
|
|||
|
|
return "".join(random.choice(letters) for i in range(length))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test():
|
|||
|
|
for _ in range(10000):
|
|||
|
|
a = random_string(30)
|
|||
|
|
b = random_string(30)
|
|||
|
|
if not check(a, b):
|
|||
|
|
print(a, b)
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
|
|||
|
|
test()
|