From 63587a50d78af8c811b6b85690c46fd414835ad0 Mon Sep 17 00:00:00 2001
From: Eugenio Schiavoni <Kukedlc@users.noreply.huggingface.co>
Date: Sat, 18 May 2024 07:23:00 +0000
Subject: [PATCH] Update tokenizer_config.json

---
 tokenizer_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tokenizer_config.json b/tokenizer_config.json
index b9dc31c..72193e9 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -9,7 +9,7 @@
       "special": true
     },
     "128001": {
-      "content": "<|im_end|>",
+      "content": "<|end_of_text|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,