Support removing invalid utf-8 sequences. (#1648)

2024-12-25 19:32:13 +08:00
parent 08d771337b
commit b6f0f5fc2e
6 changed files with 164 additions and 0 deletions
--- a/sherpa-onnx/csrc/offline-recognizer-impl.cc
+++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc
@@ -488,6 +488,8 @@ OfflineRecognizerImpl::OfflineRecognizerImpl(

 std::string OfflineRecognizerImpl::ApplyInverseTextNormalization(
    std::string text) const {
+  text = RemoveInvalidUtf8Sequences(text);
+
  if (!itn_list_.empty()) {
    for (const auto &tn : itn_list_) {
      text = tn->Normalize(text);