From 9c810ce3fe7c000e6591a59522b9c5bff8e957ae Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 20 Feb 2025 17:29:28 -0500 Subject: [PATCH] Fix #1901: UnicodeEncodeError running export_bpe_vocab.py (#1902) --- scripts/export_bpe_vocab.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/export_bpe_vocab.py b/scripts/export_bpe_vocab.py index 6267e5e3..b1ec2d5a 100755 --- a/scripts/export_bpe_vocab.py +++ b/scripts/export_bpe_vocab.py @@ -26,6 +26,7 @@ # Please install a version >=0.1.96 import argparse +import codecs from typing import Dict try: @@ -56,11 +57,11 @@ def main(): sp = spm.SentencePieceProcessor() sp.Load(model_file) - vocabs = [sp.IdToPiece(id) for id in range(sp.GetPieceSize())] - with open(vocab_file, "w") as vfile: + vocabs = [sp.id_to_piece(id) for id in range(sp.get_piece_size())] + with codecs.open(vocab_file, "w", "utf-8") as vfile: for v in vocabs: - id = sp.PieceToId(v) - vfile.write(f"{v}\t{sp.GetScore(id)}\n") + id = sp.piece_to_id(v) + vfile.write(f"{v}\t{sp.get_score(id)}\n") print(f"Vocabulary file is written to {vocab_file}")