EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix #1901: UnicodeEncodeError running export_bpe_vocab.py (#1902)

Browse Source
This commit is contained in:
Sheldon Robinson
2025-02-20 17:29:28 -05:00
committed by GitHub
parent 4801094133
commit 9c810ce3fe
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/export_bpe_vocab.py
View File

@@ -26,6 +26,7 @@
# Please install a version >=0.1.96 # Please install a version >=0.1.96
import argparse import argparse
import codecs
from typing import Dict from typing import Dict
try: try:
@@ -56,11 +57,11 @@ def main():
sp = spm.SentencePieceProcessor() sp = spm.SentencePieceProcessor()
sp.Load(model_file) sp.Load(model_file)
vocabs = [sp.IdToPiece(id) for id in range(sp.GetPieceSize())] vocabs = [sp.id_to_piece(id) for id in range(sp.get_piece_size())]
with open(vocab_file, "w") as vfile: with codecs.open(vocab_file, "w", "utf-8") as vfile:
for v in vocabs: for v in vocabs:
id = sp.PieceToId(v) id = sp.piece_to_id(v)
vfile.write(f"{v}\t{sp.GetScore(id)}\n") vfile.write(f"{v}\t{sp.get_score(id)}\n")
print(f"Vocabulary file is written to {vocab_file}") print(f"Vocabulary file is written to {vocab_file}")