EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix splitting words containing ', e.g., I've (#389)

Browse Source
This commit is contained in:
Fangjun Kuang
2023-10-26 13:07:30 +08:00
committed by GitHub
parent fcde4c4944
commit a8fed2a9ce
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sherpa-onnx/csrc/text-utils.cc
View File

@@ -163,6 +163,8 @@ template bool SplitStringToFloats(const std::string &full, const char *delim,
bool omit_empty_strings, bool omit_empty_strings,
std::vector<double> *out); std::vector<double> *out);
static bool IsPunct(char c) { return c != '\'' && std::ispunct(c); }
static std::vector<std::string> MergeCharactersIntoWords( static std::vector<std::string> MergeCharactersIntoWords(
const std::vector<std::string> &words) { const std::vector<std::string> &words) {
std::vector<std::string> ans; std::vector<std::string> ans;
@@ -174,7 +176,7 @@ static std::vector<std::string> MergeCharactersIntoWords(
while (i < n) { while (i < n) {
const auto &w = words[i]; const auto &w = words[i];
if (w.size() > 1 || if (w.size() > 1 ||
(w.size() == 1 && (std::ispunct(w[0]) || std::isspace(w[0])))) { (w.size() == 1 && (IsPunct(w[0]) || std::isspace(w[0])))) {
if (prev != -1) { if (prev != -1) {
std::string t; std::string t;
for (; prev < i; ++prev) { for (; prev < i; ++prev) {