Support removing invalid utf-8 sequences. (#1648)

2024-12-25 19:32:13 +08:00
parent 08d771337b
commit b6f0f5fc2e
6 changed files with 164 additions and 0 deletions
--- a/sherpa-onnx/csrc/CMakeLists.txt
+++ b/sherpa-onnx/csrc/CMakeLists.txt
@@ -545,6 +545,7 @@ if(SHERPA_ONNX_ENABLE_TESTS)
    pad-sequence-test.cc
    slice-test.cc
    stack-test.cc
+    text-utils-test.cc
    text2token-test.cc
    transpose-test.cc
    unbind-test.cc