From a8aed96c210672211d86acc674061d4b45728d85 Mon Sep 17 00:00:00 2001 From: Jeff Green Date: Sat, 3 Jan 2026 19:00:01 +0000 Subject: [PATCH] Upload tokenizer.json and added_tokens.json --- .gitattributes | 1 + added_tokens.json | 24 ++++++++++++++++++++++++ tokenizer.json | 3 +++ 3 files changed, 28 insertions(+) create mode 100644 added_tokens.json create mode 100644 tokenizer.json diff --git a/.gitattributes b/.gitattributes index a6344aa..52373fe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..482ced4 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,24 @@ +{ + "": 151658, + "": 151657, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..d038a4d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8e0539552cfd8fca91589dff659451cb2b21821896364ddf9931ff05505019a +size 11421996