From 2de847b3643baeb6358f9309335bb341b51a8cfe Mon Sep 17 00:00:00 2001 From: Jeff Green Date: Sat, 3 Jan 2026 19:17:27 +0000 Subject: [PATCH] Upload missing tokenizer files --- added_tokens.json | 24 ++++++++++++++++++++++++ tokenizer.json | 3 +++ 2 files changed, 27 insertions(+) create mode 100644 added_tokens.json create mode 100644 tokenizer.json diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..482ced4 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,24 @@ +{ + "": 151658, + "": 151657, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..d038a4d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8e0539552cfd8fca91589dff659451cb2b21821896364ddf9931ff05505019a +size 11421996