From 38e9f406f534e8c8c82b1db43747321ef170a159 Mon Sep 17 00:00:00 2001
From: ModelHub XC <noreply@modelhub.org.cn>
Date: Thu, 2 Jul 2026 18:31:18 +0800
Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?=
 =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?=
 =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Model: AetherModels/AetherCuTF-Ko-2-unique-Seed1234
Source: Original Platform
---
 .gitattributes         |   35 ++
 README.md              |   58 ++
 aether.py              |  315 +++++++++++
 aethertokenizers.py    |  467 +++++++++++++++++
 all_results.json       |    9 +
 bpe_utils.py           |   64 +++
 config.json            |   32 ++
 consts.py              |   57 ++
 generation_config.json |   11 +
 model.safetensors      |    3 +
 tokenizer_config.json  |   35 ++
 train_results.json     |    9 +
 trainer_state.json     | 1135 ++++++++++++++++++++++++++++++++++++++++
 training_args.bin      |    3 +
 14 files changed, 2233 insertions(+)
 create mode 100644 .gitattributes
 create mode 100644 README.md
 create mode 100644 aether.py
 create mode 100644 aethertokenizers.py
 create mode 100644 all_results.json
 create mode 100644 bpe_utils.py
 create mode 100644 config.json
 create mode 100644 consts.py
 create mode 100644 generation_config.json
 create mode 100644 model.safetensors
 create mode 100644 tokenizer_config.json
 create mode 100644 train_results.json
 create mode 100644 trainer_state.json
 create mode 100644 training_args.bin

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..a6344aa
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0a12e00
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+---
+library_name: transformers
+base_model: llama3_gptsize_config_dtypefix.json
+tags:
+- generated_from_trainer
+model-index:
+- name: AetherCuTF-Ko-2-unique-V7-Seed1234-LR5e-4
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# AetherCuTF-Ko-2-unique-V7-Seed1234-LR5e-4
+
+This model is a fine-tuned version of [llama3_gptsize_config_dtypefix.json](https://huggingface.co/llama3_gptsize_config_dtypefix.json) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0005
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 1234
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 0.1
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 5.0.0
+- Pytorch 2.10.0+cu128
+- Datasets 4.8.2
+- Tokenizers 0.22.2
diff --git a/aether.py b/aether.py
new file mode 100644
index 0000000..5055eef
--- /dev/null
+++ b/aether.py
@@ -0,0 +1,315 @@
+import regex
+from unicodedata import normalize
+from .bpe_utils import bytes_to_unicode_special
+from random import shuffle, seed
+from math import ceil
+
+def pretok_with_targeted_handling(text_data, split_regex, target_regex, target_process, nontarget_process):
+    if split_regex:
+        # 1st Regex: The general split regex
+        split_regex = regex.compile(split_regex)
+        split_regexed = split_regex.findall(text_data)
+    else:
+        split_regexed = [text_data]
+
+    # 2nd Regex: Splits a string into target and non-target chunks
+    final_processed_list = []
+    for chunk in split_regexed:
+        if not chunk:
+            continue # Skip empty tokens if any
+
+        # Split the token into target/non-target parts
+        processed_sub_tokens = ""
+        for match in regex.finditer(target_regex, chunk):
+            # Check if the chunk is target to decide which process to use
+            label = match.lastgroup
+            content = match.group()
+            if label == 'target':
+                processed_sub_tokens+=(target_process(content))
+            else:
+                processed_sub_tokens+=(nontarget_process(content))
+    
+        final_processed_list.append(processed_sub_tokens)
+
+    return final_processed_list
+
+def pretok_with_targeted_handling_list(text_data, split_regex, target_regex, target_process, nontarget_process):
+    if split_regex:
+        # 1st Regex: The general split regex
+        split_regex = regex.compile(split_regex)
+        split_regexed = split_regex.findall(text_data)
+    else:
+        split_regexed = [text_data]
+    # 2nd Regex: Splits a string into target and non-target chunks
+    final_processed_list = []
+    for chunk in split_regexed:
+        if not chunk:
+            continue # Skip empty tokens if any
+        # Split the token into target/non-target parts
+        for match in regex.finditer(target_regex, chunk):
+            # Check if the chunk is target to decide which process to use
+            label = match.lastgroup
+            content = match.group()
+            if label == 'target':
+                final_processed_list+=target_process(content)
+            else:
+                final_processed_list+=nontarget_process(content)
+    return final_processed_list
+
+
+# String "좋아" -> String "ƴƸƯŠ"
+def apply_encoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
+    if "Jamo" in encoding_type:
+        varlen = "varlen" in encoding_type
+        final_string = ""
+        for char in input_string:
+            if char == " ":
+                final_string += mapping[ord(char)]
+                continue
+            normalized = normalize("NFKD", char)
+            if len(normalized)==1: # Single Jamo can be used, but if not usable need fallback
+                if normalized not in mapping:
+                    final_string += apply_encoding(normalized, "utf-8", mapping)
+                    continue
+            final_string += normalized
+            if not varlen and len(normalized)==2: # Hangul syllable decomposed into 2 Jamo
+                final_string += '∅'  # Placeholder for missing Jamo                    
+        return final_string
+    if shuffle:
+        # For each character, calculate offset and map to shuffled offset
+        input_string = "".join([chr(offset_mapping[ord(c) - unicode_start] + unicode_start) for c in input_string])
+    if encoding_type == "nybbles":
+        return apply_nybble_encoding(input_string, mapping, offset)
+    encoded = input_string.encode(encoding_type)
+    return "".join([mapping.get(x + offset) if x + offset in mapping else mapping.get(x) for x in encoded])
+
+# String "ƴƸƯŠ"  -> String "좋아"
+def apply_decoding(input_string, encoding_type, mapping, offset=0):
+    antimap = {value: key for key, value in mapping.items()}
+    return b"".join([antimap[a+offset].to_bytes() for a in input_string]).decode(encoding_type)
+
+def apply_character_decoding(input_string, mapping):
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    char_mode = False
+    for symbol in input_string:
+        # For token in charmode
+        if symbol not in antimap:
+            # If we are in the normal case, resolve the current temp_string
+            if not char_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = ""
+            temp_string += symbol
+            char_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in charmode, resolve the current temp_string
+            if char_mode:
+                final_string += temp_string
+                temp_string = b""
+            temp_string += (antimap[symbol]).to_bytes()
+            char_mode = False
+    # Resolve final piece
+    if char_mode:
+        final_string += temp_string
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+# String "è¿Ļæĺ¯ƴƸƯŠ"  -> String "这是좋아"
+def apply_specialized_decoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
+    if shuffle:
+        def unshuffle_char(c):
+            shuffled_offset = ord(c) - unicode_start
+            original_offset = offset_mapping[shuffled_offset]
+            return chr(original_offset + unicode_start)
+    
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    special_mode = False
+    for symbol in input_string:
+        byte_id = antimap[symbol]
+        # For token in special case
+        if byte_id >= offset:
+            # If we are in the normal case, resolve the current temp_string
+            if not special_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = b""
+            temp_string += (byte_id - offset).to_bytes()
+            special_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in the special case, resolve the current temp_string
+            if special_mode:
+                temp_add = temp_string.decode(encoding_type, errors='replace')
+                if shuffle:
+                    temp_add = "".join([unshuffle_char(c) for c in temp_add])
+                final_string += temp_add
+                temp_string = b""
+            temp_string += (byte_id).to_bytes()
+            special_mode = False
+    # Resolve final piece
+    if special_mode:
+        temp_add = temp_string.decode(encoding_type, errors='replace')
+        if shuffle:
+            temp_add = "".join([unshuffle_char(c) for c in temp_add])
+        final_string += temp_add
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+def apply_nybble_decoding(input_string, mapping, offset=0):
+    def decode_char(temp_string):
+        res = []
+        for i in range(0, len(temp_string), 4):
+            try:
+                n0, n1, n2, n3 = [t for t in temp_string[i:i+4]]
+                codepoint = (n0 << 12) | (n1 << 8) | (n2 << 4) | n3
+                res.append(chr(codepoint))
+            except:
+                # In case of any error (e.g., invalid character), append a replacement character
+                res.append('�')
+        return "".join(res)
+
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    special_mode = False
+    for symbol in input_string:
+        byte_id = antimap[symbol]
+        # For token in special case
+        if byte_id >= offset:
+            # If we are in the normal case, resolve the current temp_string
+            if not special_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = b""
+            temp_string += (byte_id - offset).to_bytes()
+            special_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in the special case, resolve the current temp_string
+            if special_mode:
+                final_string += decode_char(temp_string)
+                temp_string = b""
+            temp_string += (byte_id).to_bytes()
+            special_mode = False
+    # Resolve final piece
+    if special_mode:
+        final_string += decode_char(temp_string)
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+def apply_Jamo_decoding(input_string, mapping):
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    current_type = type(input_string[0])
+    current_pieces = []
+    for char in input_string:
+        char = antimap[char]
+        if type(char) == current_type:
+            current_pieces.append(char)
+        else:
+            # Process current pieces
+            if current_type == str:
+                # Jamo pieces
+                syllables = "".join((current_pieces)).replace("∅", "")
+                # Put it into unicode format
+                final_string += normalize("NFKC", syllables)
+            else:
+                # Byte pieces
+                byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
+                final_string += byte_string
+            current_type = type(char)
+            current_pieces = [char]
+    # Final flush
+    if current_type == str:
+        # Jamo pieces
+        syllables = "".join((current_pieces)).replace("∅", "")
+        final_string += syllables
+    else:
+        # Byte pieces
+        byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
+        final_string += byte_string
+    return final_string 
+
+
+
+def apply_nybble_encoding(input_string, mapping, offset,n_nybbles=4):
+    def encode_char(c):
+        o = ord(c)
+        return "".join(mapping[offset + ((o >> (4 * i)) & 0xF)] for i in range(n_nybbles - 1, -1, -1))
+    
+    return "".join(encode_char(c) for c in input_string)
+
+# Given a range of unicode, we shuffle the points
+# For performance this can be stored as a list, a mapping of offset2newoffset
+def create_shuffled_unicode_mapping(range_start, range_end, given_seed=6767):
+    unicode_points = list(range(range_end - range_start + 1))
+    seed(given_seed)
+    shuffle(unicode_points)
+    # to use, calculate offset = ord(char) - range_start, then map to unicode_points[offset] + range_start
+    return unicode_points
+
+# Given a range of unicode, provide a Customized UTF with given parameter properties
+def create_CuTF_encoding(range_start, range_end, encode_length):
+    target_space_len = range_end - range_start + 1
+    num_indices = ceil(target_space_len**(1/encode_length))
+    # Plan is to do the math only here and save the mapping for efficiency
+    CuTF_mapping = {}
+    for codepoint in range(range_start, range_end+1):
+        offset = codepoint-range_start
+        CuTF_rep = []
+        for i in range(encode_length-1):
+            rep_index = offset//(num_indices**(encode_length-i-1))
+            CuTF_rep.append(rep_index)
+            offset -= (num_indices**(encode_length-i-1))*rep_index
+        CuTF_rep.append(offset)
+        CuTF_mapping[codepoint] = CuTF_rep
+    
+    return CuTF_mapping, num_indices
+
+
+
+def apply_CuTF_decoding(input_string, encode_length, num_index, range_start, indexing_strategy="shared"):
+    def decode_target_chars(sequence):
+        outpieces = ""
+        for i in range(0, len(sequence), encode_length):
+            piece = sequence[i:i+encode_length]
+            offset = 0
+            for j, byte in enumerate(piece):
+                if indexing_strategy == "shared":
+                    offset += (byte-256) * (num_index**(encode_length-j-1))
+                elif indexing_strategy == "unique":
+                    offset += (byte - (256 + num_index*(encode_length-j-1))) * (num_index**(encode_length-j-1))
+            outpieces += chr(range_start + offset)
+        return outpieces
+    final_string = ""
+    temp_pieces = []
+    is_target = False
+    for token in input_string:
+        if token >= 256:
+            if not is_target:
+                # Flush non-target tokens
+                final_string += bytes(temp_pieces).decode("utf8", errors='replace')
+                temp_pieces = []
+            temp_pieces.append(token)
+            is_target = True
+        else:
+            if is_target:
+                # Flush target tokens
+                final_string += decode_target_chars(temp_pieces)
+                temp_pieces = []
+            temp_pieces.append(token)
+            is_target = False
+    # Flush any remaining tokens
+    if is_target:
+        final_string += decode_target_chars(temp_pieces)
+    else:
+        final_string += bytes(temp_pieces).decode("utf8", errors='replace')
+    return final_string
\ No newline at end of file
diff --git a/aethertokenizers.py b/aethertokenizers.py
new file mode 100644
index 0000000..ec1c6e3
--- /dev/null
+++ b/aethertokenizers.py
@@ -0,0 +1,467 @@
+from unittest import result
+
+from .aether import apply_CuTF_decoding, apply_Jamo_decoding, apply_nybble_decoding, create_CuTF_encoding, pretok_with_targeted_handling, apply_encoding, apply_specialized_decoding, create_shuffled_unicode_mapping, apply_character_decoding, pretok_with_targeted_handling_list
+from .consts import *
+from .bpe_utils import bytes_to_unicode_original, bytes_to_unicode_special
+from transformers.tokenization_utils import PreTrainedTokenizer
+import os, json, shutil
+from typing import Optional
+
+
+# Approximately modeled after byt5 tokenizer
+# https://github.com/huggingface/transformers/blob/v4.57.1/src/transformers/models/byt5/tokenization_byt5.py
+
+class AetherByteTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, encoding, shuffle=False, offset_mapping=None,**kwargs):
+        self.lang = lang
+        self.encoding = encoding
+        self.shuffle = shuffle
+        if shuffle:
+            self.unicode_start = lang_ranges[lang][0]
+            if offset_mapping is not None:
+                self.offset_mapping = offset_mapping
+            else:
+                self.offset_mapping = create_shuffled_unicode_mapping(*lang_ranges[lang]) # list that acts as offset2newoffset
+            self.offset_unmapping = {v: k for k, v in enumerate(self.offset_mapping)} # newoffset2offset, for decoding
+        self.target_regex = target_regexes[lang]
+        if encoding == "utf8":
+            Aether_info = Aethers["utf8-"+lang]
+            if lang == "Ko":
+                # This is to limit Korean to full syllables only for this mode
+                self.target_regex = target_regexes["Ko-SO"]
+        else:
+            Aether_info = Aethers[encoding]
+        self.aether_type = Aether_info['type']
+        # Mapping == aether2symbol
+        self.mapping = bytes_to_unicode_special(Aether_info['pieces'], OFFSET) if self.aether_type == "bytes" else {a: a for a in Aether_info['pieces']}|bytes_to_unicode_original()
+        self.index2aether = self._make_index2aether(self.mapping)
+        self.index2symbol = {index: self.mapping[aether] for index, aether in self.index2aether.items()}
+        self.symbol2index = {value: key for key, value in self.index2symbol.items()}
+        self.target_process = lambda x: apply_encoding(x, encoding, self.mapping, OFFSET, self.shuffle, self.offset_mapping if self.shuffle else None, self.unicode_start if self.shuffle else None) #text2symbols
+        self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, encoding=encoding, shuffle=shuffle, **kwargs)
+    
+    def _make_index2aether(self, mapping):
+        index2aether = {index: aether_id for index, aether_id in enumerate(mapping)}
+        return index2aether
+        
+    @property
+    def vocab_size(self):
+        return len(self.mapping) + len(self.added_tokens_encoder)
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        mapping_path = os.path.join(pretrained_model_name_or_path, "offset_mapping.json")
+        if os.path.exists(mapping_path):
+            with open(mapping_path, "r") as f:
+                kwargs["offset_mapping"] = json.load(f)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in self.index2aether}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+    def pretokenize(self, input):
+        return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        pretokenized = self.pretokenize(text)
+        tokens = []
+        for chunk in pretokenized:
+            tokens+=chunk
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        tok_id = self.symbol2index[token]
+        return tok_id
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return self.index2symbol[index]
+    
+    def _apply_decoding(self, tokens):
+        if self.aether_type == "strings":
+            if self.encoding == "Jamo":
+                result = apply_Jamo_decoding(tokens, self.mapping)
+            elif self.encoding == "Jamo-varlen":
+                result = apply_Jamo_decoding(tokens, self.mapping)
+        elif self.aether_type == "bytes":
+            if self.encoding == "nybbles":
+                result = apply_nybble_decoding(tokens, self.mapping, OFFSET)
+            else:
+                result = apply_specialized_decoding(tokens, self.encoding, self.mapping, OFFSET, self.shuffle, self.offset_unmapping if self.shuffle else None, self.unicode_start if self.shuffle else None)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        if self.shuffle:
+            mapping_path = os.path.join(save_directory, "offset_mapping.json")
+            with open(mapping_path, "w") as f:
+                json.dump(self.offset_mapping, f)
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherByteTokenizer", None]
+        }
+        
+        config["shuffle"] = self.shuffle
+
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+class AetherCharTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, **kwargs):
+        self.lang = lang
+        self.unicode_start = lang_ranges[lang][0]
+        self.unicode_end = lang_ranges[lang][1]
+        self.target_regex = target_regexes[lang]
+        if lang == "Ko":
+            # This is to limit Korean to full syllables only for this mode
+            self.target_regex = target_regexes["Ko-SO"]
+        self.target_process = lambda x: x
+        self.mapping = bytes_to_unicode_original()
+        self.symbol2index = {value: key for key, value in self.mapping.items()}
+        self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, **kwargs)
+        
+    @property
+    def vocab_size(self):
+        return len(self.mapping) + (self.unicode_end - self.unicode_start + 1) +  len(self.added_tokens_encoder)
+    
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in self.mapping}
+        vocab.update({chr(i): i - self.unicode_start + len(self.mapping) for i in range(self.unicode_start, self.unicode_end+1)})
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def pretokenize(self, input):
+        return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        pretokenized = self.pretokenize(text)
+        tokens = []
+        for chunk in pretokenized:
+            tokens+=chunk
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to token_id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        if token in self.symbol2index:
+            return self.symbol2index[token]
+        if self.unicode_start <= ord(token) <= self.unicode_end:
+            return ord(token) - self.unicode_start + len(self.mapping)
+        raise ValueError(f"Token {token} not in vocab")
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        if index < len(self.mapping):
+            return self.mapping[index]
+        return chr(index - len(self.mapping) + self.unicode_start)
+    
+    def _apply_decoding(self, tokens):
+        result = apply_character_decoding(tokens, self.mapping)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherCharTokenizer", None]
+        }
+        
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+
+# A comparable tokenizer in UTF-8.
+# Could just use ByT5 too, but controls for special tokens this way
+
+class UTF8ByteTokenizer(PreTrainedTokenizer):
+    def __init__(self, **kwargs):
+        self.mapping = bytes_to_unicode_special([]) #AKA index2symbol
+        self.symbol2index = {value: key for key, value in self.mapping.items()}
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(**kwargs)
+        
+    @property
+    def vocab_size(self):
+        return 256 + len(self.added_tokens_encoder)
+    
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(256)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        text_bytes = text.encode('utf-8')
+        tokens = [self.mapping[b] for b in text_bytes]
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        return self.symbol2index[token]
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return self.mapping[index]
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (symbols) into a single string."""
+        symbol2byte = self.get_vocab()
+        final_string = ""
+        curr_bstring = b""
+        for token in tokens:
+            if token in self.all_special_tokens:
+                # Resolve current bstring
+                if curr_bstring:
+                    string = curr_bstring.decode("utf-8", errors="replace")
+                    final_string += string
+                    curr_bstring = b""
+                final_string += token
+            else:
+                curr_bstring += bytes([symbol2byte[token]])
+        if curr_bstring:
+            string = curr_bstring.decode("utf-8", errors="replace")
+            final_string += string
+        return final_string
+            
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.UTF8ByteTokenizer", None]
+        }
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+
+
+class AetherCuTFTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, encode_length, indexing_strategy="shared", **kwargs):
+        self.lang = lang
+        self.encode_length = encode_length
+        self.indexing_strategy = indexing_strategy
+        self.target_regex = target_regexes[lang]
+        if lang == "Ko":
+            # This is to limit Korean to full syllables only for this mode
+            self.target_regex = target_regexes["Ko-SO"]
+        self.unicode_start = lang_ranges[lang][0]
+        self.unicode_end = lang_ranges[lang][1]
+        
+        raw_mapping, self.num_CuTF_indices = create_CuTF_encoding(self.unicode_start, self.unicode_end, encode_length)
+        mapping = dict()
+        if indexing_strategy == "shared":
+            for codepoint in raw_mapping:
+                mapping[codepoint] = [256+ a for a in raw_mapping[codepoint]]
+        elif indexing_strategy == "unique":
+            for codepoint in raw_mapping:
+                mapping[codepoint] = [256+ (encode_length -1 - i) * self.num_CuTF_indices + a for i, a in enumerate(raw_mapping[codepoint])]
+        self.mapping = mapping
+        self.target_process = lambda x: [rep for a in x for rep in self.mapping[ord(a)]]
+        self.nontarget_process = lambda x: [a for a in x.encode('utf8')]
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, encode_length = encode_length, indexing_strategy= indexing_strategy, **kwargs)
+    
+    @property
+    def vocab_size(self):
+        if self.indexing_strategy == "shared":
+            return 256 + self.num_CuTF_indices + len(self.added_tokens_encoder)
+
+        elif self.indexing_strategy == "unique":
+            # This is the simple answer, but in reality the full bytespace isn't utilized.
+            # return 256 + self.num_CuTF_indices * self.encode_length + len(self.added_tokens_encoder)
+            # Need to use the highest number as reference
+            max_index = self.mapping[self.unicode_end][0]
+            return max_index + len(self.added_tokens_encoder) + 1
+                    
+    
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size-len(self.added_tokens_encoder))}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        return pretok_with_targeted_handling_list(text, None, self.target_regex, self.target_process, self.nontarget_process)
+        
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        return token
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return index
+    
+    def _apply_decoding(self, tokens):
+        result = apply_CuTF_decoding(tokens, self.encode_length, self.num_CuTF_indices, self.unicode_start, self.indexing_strategy)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherCuTFTokenizer", None]
+        }
+        
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
\ No newline at end of file
diff --git a/all_results.json b/all_results.json
new file mode 100644
index 0000000..d6ab90e
--- /dev/null
+++ b/all_results.json
@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 7.006185158986433e+18,
+    "train_loss": 0.8243957067240222,
+    "train_runtime": 51861.4856,
+    "train_samples": 2508901,
+    "train_samples_per_second": 48.377,
+    "train_steps_per_second": 0.756
+}
\ No newline at end of file
diff --git a/bpe_utils.py b/bpe_utils.py
new file mode 100644
index 0000000..8289c8c
--- /dev/null
+++ b/bpe_utils.py
@@ -0,0 +1,64 @@
+from .consts import OFFSET
+
+def bytes_to_unicode_original():
+    # Can call this through
+    # from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+def bytes_to_unicode_special(special_bytes, offset=OFFSET):
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    
+    for b in special_bytes:
+        bs.append(offset+b)
+        cs.append(2**8 + n)
+        n += 1
+
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+byte2unic = bytes_to_unicode_original()
+unic2byte = {value: key for key, value in byte2unic.items()}
+
+def decode_uniced(scrambled):
+    unic_nums = [unic2byte[a] for a in scrambled]
+    char_byte = [chr(a) for a in unic_nums]
+    full = "".join(char_byte)
+    return full.encode("latin-1").decode()
+
+
+def find_aether_bytes(encoding, range_start, range_end):
+    aether_bytes = set()
+    for i in range(range_start, range_end):
+        pieces = chr(i).encode(encoding)
+        for piece in pieces:
+            aether_bytes.add(piece)
+    return aether_bytes
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..471f147
--- /dev/null
+++ b/config.json
@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 468,
+  "dtype": "float32",
+  "eos_token_id": 469,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 500000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.0.0",
+  "use_cache": false,
+  "vocab_size": 472
+}
diff --git a/consts.py b/consts.py
new file mode 100644
index 0000000..5a00c4d
--- /dev/null
+++ b/consts.py
@@ -0,0 +1,57 @@
+OFFSET = 10000
+
+# Regexes
+llama_regex = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+target_regexes = {
+                # This is the one with leading space, could be used as a pretok later
+                # "Ko": r"(?P<target> ?[가-힣ㄱ-ㅣ]+)|(?P<other>(?:[^가-힣ㄱ-ㅣ ]| (?![가-힣ㄱ-ㅣ]))+)",
+                "Ko": r"(?P<target>[가-힣ㄱ-ㅣ]+)|(?P<other>[^가-힣ㄱ-ㅣ]+)",
+                "Ko-SO": r"(?P<target>[가-힣]+)|(?P<other>[^가-힣]+)", # Syllables Only
+                "Zh": r"(?P<target>[\u4e00-\u9fa5]+)|(?P<other>[^\u4e00-\u9fa5]+)",
+                  }
+
+lang_ranges = {
+    "Ko": (0xAC00, 0xD7A3),
+    "Zh": (0x4E00, 0x9FA5)}
+
+# per-encoding pieces
+used_bytes ={}
+Aethers = {
+    'Johab': {
+        'type': "bytes",
+        'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253]
+    },
+    'Jamo': {
+        'type': "strings",
+        'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ', '∅']
+    },
+    'Jamo-varlen': {
+        'type': "strings",
+        'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ']
+    },
+    'GBK': {
+        'type': "bytes",
+        'pieces': [64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
+    },
+    'utf-16-be': {
+        'type': "bytes",
+        'pieces': range(0, 256)
+    },
+    'cp949': {
+        'type': "bytes",
+        'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
+    },
+    'nybbles': {
+        'type': "bytes",
+        'pieces': range(0, 16)
+    },
+    'utf8-Ko': {
+        'type': "bytes",
+        'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 234, 235, 236, 237]
+    }
+    ,
+    'utf8-Zh': {
+        'type': "bytes",
+        'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 228,229, 230, 231, 232, 233]
+    }
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..1612a0c
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 468,
+  "eos_token_id": [
+    469
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.0.0",
+  "use_cache": true
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000..f86d079
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0175b959e9de3d7152f8ffe0a823bbfcb8c7da1b8883f4c333473ef540f08840
+size 455973864
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..a06514a
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,35 @@
+{
+  "added_tokens_decoder": {
+    "468": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "469": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": null,
+  "backend": "custom",
+  "bos_token": "<s>",
+  "encode_length": 2,
+  "eos_token": "</s>",
+  "indexing_strategy": "unique",
+  "lang": "Ko",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "AetherCuTFTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "aethertokenizers.AetherCuTFTokenizer",
+      null
+    ]
+  }
+}
\ No newline at end of file
diff --git a/train_results.json b/train_results.json
new file mode 100644
index 0000000..d6ab90e
--- /dev/null
+++ b/train_results.json
@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 7.006185158986433e+18,
+    "train_loss": 0.8243957067240222,
+    "train_runtime": 51861.4856,
+    "train_samples": 2508901,
+    "train_samples_per_second": 48.377,
+    "train_steps_per_second": 0.756
+}
\ No newline at end of file
diff --git a/trainer_state.json b/trainer_state.json
new file mode 100644
index 0000000..9f33759
--- /dev/null
+++ b/trainer_state.json
@@ -0,0 +1,1135 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 39202,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.006377266321018832,
+      "grad_norm": 2.5775675773620605,
+      "learning_rate": 3.175210405508799e-05,
+      "loss": 3.65084814453125,
+      "step": 250
+    },
+    {
+      "epoch": 0.012754532642037663,
+      "grad_norm": 3.2221262454986572,
+      "learning_rate": 6.363172660035706e-05,
+      "loss": 1.99697509765625,
+      "step": 500
+    },
+    {
+      "epoch": 0.019131798963056498,
+      "grad_norm": 2.5724740028381348,
+      "learning_rate": 9.551134914562612e-05,
+      "loss": 1.60589306640625,
+      "step": 750
+    },
+    {
+      "epoch": 0.025509065284075327,
+      "grad_norm": 1.6085478067398071,
+      "learning_rate": 0.00012739097169089516,
+      "loss": 1.444733154296875,
+      "step": 1000
+    },
+    {
+      "epoch": 0.03188633160509416,
+      "grad_norm": 0.7356601357460022,
+      "learning_rate": 0.00015927059423616425,
+      "loss": 1.33519287109375,
+      "step": 1250
+    },
+    {
+      "epoch": 0.038263597926112995,
+      "grad_norm": 0.558532178401947,
+      "learning_rate": 0.0001911502167814333,
+      "loss": 1.224727783203125,
+      "step": 1500
+    },
+    {
+      "epoch": 0.044640864247131824,
+      "grad_norm": 0.3695690631866455,
+      "learning_rate": 0.00022302983932670237,
+      "loss": 1.12808984375,
+      "step": 1750
+    },
+    {
+      "epoch": 0.05101813056815065,
+      "grad_norm": 0.271986722946167,
+      "learning_rate": 0.00025490946187197144,
+      "loss": 1.074133544921875,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05739539688916949,
+      "grad_norm": 0.2766456604003906,
+      "learning_rate": 0.00028678908441724047,
+      "loss": 1.035495849609375,
+      "step": 2250
+    },
+    {
+      "epoch": 0.06377266321018832,
+      "grad_norm": 0.21983842551708221,
+      "learning_rate": 0.00031866870696250956,
+      "loss": 1.0116007080078124,
+      "step": 2500
+    },
+    {
+      "epoch": 0.07014992953120715,
+      "grad_norm": 0.2180081009864807,
+      "learning_rate": 0.00035054832950777865,
+      "loss": 0.99118798828125,
+      "step": 2750
+    },
+    {
+      "epoch": 0.07652719585222599,
+      "grad_norm": 0.18447472155094147,
+      "learning_rate": 0.0003824279520530477,
+      "loss": 0.9717527465820313,
+      "step": 3000
+    },
+    {
+      "epoch": 0.08290446217324482,
+      "grad_norm": 0.15918593108654022,
+      "learning_rate": 0.0004143075745983167,
+      "loss": 0.96016064453125,
+      "step": 3250
+    },
+    {
+      "epoch": 0.08928172849426365,
+      "grad_norm": 0.15337951481342316,
+      "learning_rate": 0.00044618719714358587,
+      "loss": 0.94656005859375,
+      "step": 3500
+    },
+    {
+      "epoch": 0.09565899481528248,
+      "grad_norm": 0.1397962123155594,
+      "learning_rate": 0.0004780668196888549,
+      "loss": 0.933742431640625,
+      "step": 3750
+    },
+    {
+      "epoch": 0.1020362611363013,
+      "grad_norm": 0.15751700103282928,
+      "learning_rate": 0.0004988945891556362,
+      "loss": 0.9267822265625,
+      "step": 4000
+    },
+    {
+      "epoch": 0.10841352745732015,
+      "grad_norm": 0.1273936778306961,
+      "learning_rate": 0.0004953516056801111,
+      "loss": 0.9156344604492187,
+      "step": 4250
+    },
+    {
+      "epoch": 0.11479079377833898,
+      "grad_norm": 0.12347358465194702,
+      "learning_rate": 0.0004918086222045861,
+      "loss": 0.9085472412109376,
+      "step": 4500
+    },
+    {
+      "epoch": 0.12116806009935781,
+      "grad_norm": 0.11907054483890533,
+      "learning_rate": 0.000488265638729061,
+      "loss": 0.8965045166015625,
+      "step": 4750
+    },
+    {
+      "epoch": 0.12754532642037664,
+      "grad_norm": 0.12186691164970398,
+      "learning_rate": 0.00048472265525353593,
+      "loss": 0.8894534912109375,
+      "step": 5000
+    },
+    {
+      "epoch": 0.13392259274139548,
+      "grad_norm": 0.11302649229764938,
+      "learning_rate": 0.00048117967177801086,
+      "loss": 0.8835716552734375,
+      "step": 5250
+    },
+    {
+      "epoch": 0.1402998590624143,
+      "grad_norm": 0.11347561329603195,
+      "learning_rate": 0.00047763668830248573,
+      "loss": 0.8759080200195313,
+      "step": 5500
+    },
+    {
+      "epoch": 0.14667712538343314,
+      "grad_norm": 0.10620150715112686,
+      "learning_rate": 0.00047409370482696066,
+      "loss": 0.86854833984375,
+      "step": 5750
+    },
+    {
+      "epoch": 0.15305439170445198,
+      "grad_norm": 0.10604240000247955,
+      "learning_rate": 0.00047055072135143564,
+      "loss": 0.865083984375,
+      "step": 6000
+    },
+    {
+      "epoch": 0.1594316580254708,
+      "grad_norm": 0.10537559539079666,
+      "learning_rate": 0.00046700773787591057,
+      "loss": 0.860940185546875,
+      "step": 6250
+    },
+    {
+      "epoch": 0.16580892434648964,
+      "grad_norm": 0.10088460892438889,
+      "learning_rate": 0.0004634647544003855,
+      "loss": 0.8544019775390626,
+      "step": 6500
+    },
+    {
+      "epoch": 0.17218619066750845,
+      "grad_norm": 0.09970396012067795,
+      "learning_rate": 0.0004599217709248604,
+      "loss": 0.8502755126953125,
+      "step": 6750
+    },
+    {
+      "epoch": 0.1785634569885273,
+      "grad_norm": 0.10199850797653198,
+      "learning_rate": 0.00045637878744933535,
+      "loss": 0.845378173828125,
+      "step": 7000
+    },
+    {
+      "epoch": 0.18494072330954614,
+      "grad_norm": 0.09216546267271042,
+      "learning_rate": 0.0004528358039738103,
+      "loss": 0.8421669311523438,
+      "step": 7250
+    },
+    {
+      "epoch": 0.19131798963056496,
+      "grad_norm": 0.10916922241449356,
+      "learning_rate": 0.0004492928204982852,
+      "loss": 0.841639892578125,
+      "step": 7500
+    },
+    {
+      "epoch": 0.1976952559515838,
+      "grad_norm": 0.09349249303340912,
+      "learning_rate": 0.00044574983702276013,
+      "loss": 0.8376875610351563,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2040725222726026,
+      "grad_norm": 0.09555172175168991,
+      "learning_rate": 0.00044220685354723506,
+      "loss": 0.8310078735351563,
+      "step": 8000
+    },
+    {
+      "epoch": 0.21044978859362146,
+      "grad_norm": 0.09085638076066971,
+      "learning_rate": 0.00043866387007171,
+      "loss": 0.8307603149414062,
+      "step": 8250
+    },
+    {
+      "epoch": 0.2168270549146403,
+      "grad_norm": 0.09479482471942902,
+      "learning_rate": 0.0004351208865961849,
+      "loss": 0.8274447631835937,
+      "step": 8500
+    },
+    {
+      "epoch": 0.22320432123565911,
+      "grad_norm": 0.0888662189245224,
+      "learning_rate": 0.00043157790312065985,
+      "loss": 0.8251286010742187,
+      "step": 8750
+    },
+    {
+      "epoch": 0.22958158755667796,
+      "grad_norm": 0.09127531945705414,
+      "learning_rate": 0.0004280349196451348,
+      "loss": 0.8239066772460938,
+      "step": 9000
+    },
+    {
+      "epoch": 0.2359588538776968,
+      "grad_norm": 0.09323842078447342,
+      "learning_rate": 0.00042449193616960975,
+      "loss": 0.821473876953125,
+      "step": 9250
+    },
+    {
+      "epoch": 0.24233612019871562,
+      "grad_norm": 0.08932389318943024,
+      "learning_rate": 0.00042094895269408463,
+      "loss": 0.818468994140625,
+      "step": 9500
+    },
+    {
+      "epoch": 0.24871338651973446,
+      "grad_norm": 0.09459158033132553,
+      "learning_rate": 0.00041740596921855956,
+      "loss": 0.815874755859375,
+      "step": 9750
+    },
+    {
+      "epoch": 0.2550906528407533,
+      "grad_norm": 0.0948140025138855,
+      "learning_rate": 0.0004138629857430345,
+      "loss": 0.8106996459960938,
+      "step": 10000
+    },
+    {
+      "epoch": 0.2614679191617721,
+      "grad_norm": 0.09076733887195587,
+      "learning_rate": 0.0004103200022675094,
+      "loss": 0.8120379028320313,
+      "step": 10250
+    },
+    {
+      "epoch": 0.26784518548279096,
+      "grad_norm": 0.0840587168931961,
+      "learning_rate": 0.0004067770187919844,
+      "loss": 0.8110909423828125,
+      "step": 10500
+    },
+    {
+      "epoch": 0.2742224518038098,
+      "grad_norm": 0.08920229971408844,
+      "learning_rate": 0.0004032340353164593,
+      "loss": 0.80626806640625,
+      "step": 10750
+    },
+    {
+      "epoch": 0.2805997181248286,
+      "grad_norm": 0.0837017148733139,
+      "learning_rate": 0.00039969105184093425,
+      "loss": 0.8057803344726563,
+      "step": 11000
+    },
+    {
+      "epoch": 0.28697698444584746,
+      "grad_norm": 0.08721166849136353,
+      "learning_rate": 0.0003961480683654092,
+      "loss": 0.8038002319335937,
+      "step": 11250
+    },
+    {
+      "epoch": 0.2933542507668663,
+      "grad_norm": 0.0881330668926239,
+      "learning_rate": 0.00039260508488988405,
+      "loss": 0.801577880859375,
+      "step": 11500
+    },
+    {
+      "epoch": 0.2997315170878851,
+      "grad_norm": 0.09358793497085571,
+      "learning_rate": 0.000389062101414359,
+      "loss": 0.7983843994140625,
+      "step": 11750
+    },
+    {
+      "epoch": 0.30610878340890396,
+      "grad_norm": 0.08768702298402786,
+      "learning_rate": 0.0003855191179388339,
+      "loss": 0.7974132080078125,
+      "step": 12000
+    },
+    {
+      "epoch": 0.3124860497299228,
+      "grad_norm": 0.0864625945687294,
+      "learning_rate": 0.0003819761344633089,
+      "loss": 0.7967001342773438,
+      "step": 12250
+    },
+    {
+      "epoch": 0.3188633160509416,
+      "grad_norm": 0.08685760200023651,
+      "learning_rate": 0.0003784331509877838,
+      "loss": 0.7927449340820313,
+      "step": 12500
+    },
+    {
+      "epoch": 0.32524058237196046,
+      "grad_norm": 0.08900053054094315,
+      "learning_rate": 0.00037489016751225874,
+      "loss": 0.7911568603515625,
+      "step": 12750
+    },
+    {
+      "epoch": 0.3316178486929793,
+      "grad_norm": 0.08307567983865738,
+      "learning_rate": 0.00037134718403673367,
+      "loss": 0.7934022216796875,
+      "step": 13000
+    },
+    {
+      "epoch": 0.3379951150139981,
+      "grad_norm": 0.08461463451385498,
+      "learning_rate": 0.0003678042005612086,
+      "loss": 0.79057421875,
+      "step": 13250
+    },
+    {
+      "epoch": 0.3443723813350169,
+      "grad_norm": 0.08529062569141388,
+      "learning_rate": 0.00036426121708568347,
+      "loss": 0.7871109008789062,
+      "step": 13500
+    },
+    {
+      "epoch": 0.3507496476560358,
+      "grad_norm": 0.0872483178973198,
+      "learning_rate": 0.00036071823361015845,
+      "loss": 0.7870155029296875,
+      "step": 13750
+    },
+    {
+      "epoch": 0.3571269139770546,
+      "grad_norm": 0.0882478877902031,
+      "learning_rate": 0.0003571752501346334,
+      "loss": 0.78544140625,
+      "step": 14000
+    },
+    {
+      "epoch": 0.3635041802980734,
+      "grad_norm": 0.091294065117836,
+      "learning_rate": 0.0003536322666591083,
+      "loss": 0.7866491088867188,
+      "step": 14250
+    },
+    {
+      "epoch": 0.3698814466190923,
+      "grad_norm": 0.08572017401456833,
+      "learning_rate": 0.00035008928318358323,
+      "loss": 0.782495849609375,
+      "step": 14500
+    },
+    {
+      "epoch": 0.3762587129401111,
+      "grad_norm": 0.08324530720710754,
+      "learning_rate": 0.00034654629970805816,
+      "loss": 0.7797513427734375,
+      "step": 14750
+    },
+    {
+      "epoch": 0.3826359792611299,
+      "grad_norm": 0.08846472948789597,
+      "learning_rate": 0.0003430033162325331,
+      "loss": 0.7802353515625,
+      "step": 15000
+    },
+    {
+      "epoch": 0.3890132455821488,
+      "grad_norm": 0.07917249202728271,
+      "learning_rate": 0.00033946033275700807,
+      "loss": 0.7790340576171875,
+      "step": 15250
+    },
+    {
+      "epoch": 0.3953905119031676,
+      "grad_norm": 0.08665470778942108,
+      "learning_rate": 0.00033591734928148294,
+      "loss": 0.77759619140625,
+      "step": 15500
+    },
+    {
+      "epoch": 0.4017677782241864,
+      "grad_norm": 0.08258507400751114,
+      "learning_rate": 0.00033237436580595787,
+      "loss": 0.7763189086914063,
+      "step": 15750
+    },
+    {
+      "epoch": 0.4081450445452052,
+      "grad_norm": 0.08015473932027817,
+      "learning_rate": 0.0003288313823304328,
+      "loss": 0.7767020263671875,
+      "step": 16000
+    },
+    {
+      "epoch": 0.4145223108662241,
+      "grad_norm": 0.08080420643091202,
+      "learning_rate": 0.0003252883988549077,
+      "loss": 0.7747213134765625,
+      "step": 16250
+    },
+    {
+      "epoch": 0.4208995771872429,
+      "grad_norm": 0.08612637221813202,
+      "learning_rate": 0.00032174541537938265,
+      "loss": 0.7734822998046875,
+      "step": 16500
+    },
+    {
+      "epoch": 0.42727684350826173,
+      "grad_norm": 0.07643167674541473,
+      "learning_rate": 0.00031820243190385764,
+      "loss": 0.7744876708984375,
+      "step": 16750
+    },
+    {
+      "epoch": 0.4336541098292806,
+      "grad_norm": 0.08003860712051392,
+      "learning_rate": 0.00031465944842833256,
+      "loss": 0.770651611328125,
+      "step": 17000
+    },
+    {
+      "epoch": 0.4400313761502994,
+      "grad_norm": 0.07683893293142319,
+      "learning_rate": 0.0003111164649528075,
+      "loss": 0.770556640625,
+      "step": 17250
+    },
+    {
+      "epoch": 0.44640864247131823,
+      "grad_norm": 0.08421041071414948,
+      "learning_rate": 0.00030757348147728236,
+      "loss": 0.7692486572265625,
+      "step": 17500
+    },
+    {
+      "epoch": 0.4527859087923371,
+      "grad_norm": 0.08235982805490494,
+      "learning_rate": 0.0003040304980017573,
+      "loss": 0.7692238159179687,
+      "step": 17750
+    },
+    {
+      "epoch": 0.4591631751133559,
+      "grad_norm": 0.08046766370534897,
+      "learning_rate": 0.0003004875145262322,
+      "loss": 0.769811767578125,
+      "step": 18000
+    },
+    {
+      "epoch": 0.46554044143437473,
+      "grad_norm": 0.08188624680042267,
+      "learning_rate": 0.0002969445310507072,
+      "loss": 0.7680709228515625,
+      "step": 18250
+    },
+    {
+      "epoch": 0.4719177077553936,
+      "grad_norm": 0.0804840475320816,
+      "learning_rate": 0.00029340154757518213,
+      "loss": 0.7635980224609376,
+      "step": 18500
+    },
+    {
+      "epoch": 0.4782949740764124,
+      "grad_norm": 0.08123503625392914,
+      "learning_rate": 0.00028985856409965706,
+      "loss": 0.7640174560546875,
+      "step": 18750
+    },
+    {
+      "epoch": 0.48467224039743123,
+      "grad_norm": 0.08404447138309479,
+      "learning_rate": 0.000286315580624132,
+      "loss": 0.7631983642578125,
+      "step": 19000
+    },
+    {
+      "epoch": 0.49104950671845005,
+      "grad_norm": 0.09293997287750244,
+      "learning_rate": 0.0002827725971486069,
+      "loss": 0.7630362548828125,
+      "step": 19250
+    },
+    {
+      "epoch": 0.4974267730394689,
+      "grad_norm": 0.08382653445005417,
+      "learning_rate": 0.00027922961367308184,
+      "loss": 0.76153271484375,
+      "step": 19500
+    },
+    {
+      "epoch": 0.5038040393604878,
+      "grad_norm": 0.07555802911520004,
+      "learning_rate": 0.00027568663019755677,
+      "loss": 0.7605255737304687,
+      "step": 19750
+    },
+    {
+      "epoch": 0.5101813056815065,
+      "grad_norm": 0.08228955417871475,
+      "learning_rate": 0.0002721436467220317,
+      "loss": 0.7614573974609375,
+      "step": 20000
+    },
+    {
+      "epoch": 0.5165585720025254,
+      "grad_norm": 0.08145914226770401,
+      "learning_rate": 0.0002686006632465066,
+      "loss": 0.75997900390625,
+      "step": 20250
+    },
+    {
+      "epoch": 0.5229358383235442,
+      "grad_norm": 0.09308425337076187,
+      "learning_rate": 0.00026505767977098155,
+      "loss": 0.759119140625,
+      "step": 20500
+    },
+    {
+      "epoch": 0.529313104644563,
+      "grad_norm": 0.08138495683670044,
+      "learning_rate": 0.0002615146962954565,
+      "loss": 0.7591130981445312,
+      "step": 20750
+    },
+    {
+      "epoch": 0.5356903709655819,
+      "grad_norm": 0.08346766978502274,
+      "learning_rate": 0.0002579717128199314,
+      "loss": 0.757355224609375,
+      "step": 21000
+    },
+    {
+      "epoch": 0.5420676372866007,
+      "grad_norm": 0.077271968126297,
+      "learning_rate": 0.0002544287293444064,
+      "loss": 0.7534990234375,
+      "step": 21250
+    },
+    {
+      "epoch": 0.5484449036076195,
+      "grad_norm": 0.07977950572967529,
+      "learning_rate": 0.0002508857458688813,
+      "loss": 0.75655029296875,
+      "step": 21500
+    },
+    {
+      "epoch": 0.5548221699286384,
+      "grad_norm": 0.08347009867429733,
+      "learning_rate": 0.0002473427623933562,
+      "loss": 0.75475537109375,
+      "step": 21750
+    },
+    {
+      "epoch": 0.5611994362496572,
+      "grad_norm": 0.07832519710063934,
+      "learning_rate": 0.00024379977891783114,
+      "loss": 0.7558754272460938,
+      "step": 22000
+    },
+    {
+      "epoch": 0.567576702570676,
+      "grad_norm": 0.08469550311565399,
+      "learning_rate": 0.00024025679544230604,
+      "loss": 0.7546185913085938,
+      "step": 22250
+    },
+    {
+      "epoch": 0.5739539688916949,
+      "grad_norm": 0.08166459202766418,
+      "learning_rate": 0.000236713811966781,
+      "loss": 0.7514425659179688,
+      "step": 22500
+    },
+    {
+      "epoch": 0.5803312352127137,
+      "grad_norm": 0.08695302903652191,
+      "learning_rate": 0.00023317082849125592,
+      "loss": 0.75103564453125,
+      "step": 22750
+    },
+    {
+      "epoch": 0.5867085015337326,
+      "grad_norm": 0.09806793183088303,
+      "learning_rate": 0.00022962784501573085,
+      "loss": 0.752026123046875,
+      "step": 23000
+    },
+    {
+      "epoch": 0.5930857678547514,
+      "grad_norm": 0.08246204257011414,
+      "learning_rate": 0.00022608486154020578,
+      "loss": 0.7497666015625,
+      "step": 23250
+    },
+    {
+      "epoch": 0.5994630341757702,
+      "grad_norm": 0.08100980520248413,
+      "learning_rate": 0.0002225418780646807,
+      "loss": 0.749739501953125,
+      "step": 23500
+    },
+    {
+      "epoch": 0.605840300496789,
+      "grad_norm": 0.07755894213914871,
+      "learning_rate": 0.00021899889458915564,
+      "loss": 0.7490238037109375,
+      "step": 23750
+    },
+    {
+      "epoch": 0.6122175668178079,
+      "grad_norm": 0.08628904074430466,
+      "learning_rate": 0.0002154559111136306,
+      "loss": 0.7485498046875,
+      "step": 24000
+    },
+    {
+      "epoch": 0.6185948331388267,
+      "grad_norm": 0.08315210789442062,
+      "learning_rate": 0.0002119129276381055,
+      "loss": 0.7472803344726563,
+      "step": 24250
+    },
+    {
+      "epoch": 0.6249720994598456,
+      "grad_norm": 0.08354892581701279,
+      "learning_rate": 0.00020836994416258042,
+      "loss": 0.7469033203125,
+      "step": 24500
+    },
+    {
+      "epoch": 0.6313493657808644,
+      "grad_norm": 0.08090732991695404,
+      "learning_rate": 0.00020482696068705537,
+      "loss": 0.7483990478515625,
+      "step": 24750
+    },
+    {
+      "epoch": 0.6377266321018832,
+      "grad_norm": 0.08247523009777069,
+      "learning_rate": 0.0002012839772115303,
+      "loss": 0.7441016845703124,
+      "step": 25000
+    },
+    {
+      "epoch": 0.644103898422902,
+      "grad_norm": 0.07630934566259384,
+      "learning_rate": 0.0001977409937360052,
+      "loss": 0.741853515625,
+      "step": 25250
+    },
+    {
+      "epoch": 0.6504811647439209,
+      "grad_norm": 0.07914166897535324,
+      "learning_rate": 0.00019419801026048016,
+      "loss": 0.74271875,
+      "step": 25500
+    },
+    {
+      "epoch": 0.6568584310649397,
+      "grad_norm": 0.08192494511604309,
+      "learning_rate": 0.00019065502678495508,
+      "loss": 0.7423253784179688,
+      "step": 25750
+    },
+    {
+      "epoch": 0.6632356973859586,
+      "grad_norm": 0.08241453766822815,
+      "learning_rate": 0.00018711204330943,
+      "loss": 0.742218994140625,
+      "step": 26000
+    },
+    {
+      "epoch": 0.6696129637069773,
+      "grad_norm": 0.07896443456411362,
+      "learning_rate": 0.00018356905983390494,
+      "loss": 0.738341064453125,
+      "step": 26250
+    },
+    {
+      "epoch": 0.6759902300279962,
+      "grad_norm": 0.07580381631851196,
+      "learning_rate": 0.00018002607635837987,
+      "loss": 0.7400193481445313,
+      "step": 26500
+    },
+    {
+      "epoch": 0.6823674963490151,
+      "grad_norm": 0.07701326906681061,
+      "learning_rate": 0.0001764830928828548,
+      "loss": 0.7382138671875,
+      "step": 26750
+    },
+    {
+      "epoch": 0.6887447626700338,
+      "grad_norm": 0.07780223339796066,
+      "learning_rate": 0.00017294010940732975,
+      "loss": 0.7394266967773437,
+      "step": 27000
+    },
+    {
+      "epoch": 0.6951220289910527,
+      "grad_norm": 0.08848556876182556,
+      "learning_rate": 0.00016939712593180465,
+      "loss": 0.7408846435546875,
+      "step": 27250
+    },
+    {
+      "epoch": 0.7014992953120716,
+      "grad_norm": 0.0774645283818245,
+      "learning_rate": 0.00016585414245627958,
+      "loss": 0.7371785888671875,
+      "step": 27500
+    },
+    {
+      "epoch": 0.7078765616330903,
+      "grad_norm": 0.08030521869659424,
+      "learning_rate": 0.00016231115898075453,
+      "loss": 0.7363970947265625,
+      "step": 27750
+    },
+    {
+      "epoch": 0.7142538279541092,
+      "grad_norm": 0.08429484069347382,
+      "learning_rate": 0.00015876817550522946,
+      "loss": 0.7369862670898437,
+      "step": 28000
+    },
+    {
+      "epoch": 0.7206310942751281,
+      "grad_norm": 0.07752969861030579,
+      "learning_rate": 0.00015522519202970436,
+      "loss": 0.735985595703125,
+      "step": 28250
+    },
+    {
+      "epoch": 0.7270083605961468,
+      "grad_norm": 0.08850430697202682,
+      "learning_rate": 0.0001516822085541793,
+      "loss": 0.7350040893554688,
+      "step": 28500
+    },
+    {
+      "epoch": 0.7333856269171657,
+      "grad_norm": 0.07794130593538284,
+      "learning_rate": 0.00014813922507865424,
+      "loss": 0.7345814208984375,
+      "step": 28750
+    },
+    {
+      "epoch": 0.7397628932381846,
+      "grad_norm": 0.07695607095956802,
+      "learning_rate": 0.00014459624160312917,
+      "loss": 0.731819091796875,
+      "step": 29000
+    },
+    {
+      "epoch": 0.7461401595592033,
+      "grad_norm": 0.07952513545751572,
+      "learning_rate": 0.0001410532581276041,
+      "loss": 0.7328406982421874,
+      "step": 29250
+    },
+    {
+      "epoch": 0.7525174258802222,
+      "grad_norm": 0.08443118631839752,
+      "learning_rate": 0.00013751027465207902,
+      "loss": 0.7327833862304688,
+      "step": 29500
+    },
+    {
+      "epoch": 0.7588946922012411,
+      "grad_norm": 0.07715488970279694,
+      "learning_rate": 0.00013396729117655395,
+      "loss": 0.7313681030273438,
+      "step": 29750
+    },
+    {
+      "epoch": 0.7652719585222598,
+      "grad_norm": 0.08632020652294159,
+      "learning_rate": 0.0001304243077010289,
+      "loss": 0.7293426513671875,
+      "step": 30000
+    },
+    {
+      "epoch": 0.7716492248432787,
+      "grad_norm": 0.08144113421440125,
+      "learning_rate": 0.0001268813242255038,
+      "loss": 0.729284912109375,
+      "step": 30250
+    },
+    {
+      "epoch": 0.7780264911642976,
+      "grad_norm": 0.08117876201868057,
+      "learning_rate": 0.00012333834074997873,
+      "loss": 0.7302175903320313,
+      "step": 30500
+    },
+    {
+      "epoch": 0.7844037574853163,
+      "grad_norm": 0.08066125214099884,
+      "learning_rate": 0.00011979535727445367,
+      "loss": 0.7301824951171875,
+      "step": 30750
+    },
+    {
+      "epoch": 0.7907810238063352,
+      "grad_norm": 0.08321163803339005,
+      "learning_rate": 0.0001162523737989286,
+      "loss": 0.726185302734375,
+      "step": 31000
+    },
+    {
+      "epoch": 0.7971582901273541,
+      "grad_norm": 0.07980816811323166,
+      "learning_rate": 0.00011270939032340353,
+      "loss": 0.7280133056640625,
+      "step": 31250
+    },
+    {
+      "epoch": 0.8035355564483728,
+      "grad_norm": 0.0806281566619873,
+      "learning_rate": 0.00010916640684787846,
+      "loss": 0.7268768310546875,
+      "step": 31500
+    },
+    {
+      "epoch": 0.8099128227693917,
+      "grad_norm": 0.08116059750318527,
+      "learning_rate": 0.0001056234233723534,
+      "loss": 0.7262345581054688,
+      "step": 31750
+    },
+    {
+      "epoch": 0.8162900890904105,
+      "grad_norm": 0.07698776572942734,
+      "learning_rate": 0.00010208043989682831,
+      "loss": 0.7262030029296875,
+      "step": 32000
+    },
+    {
+      "epoch": 0.8226673554114293,
+      "grad_norm": 0.08539737015962601,
+      "learning_rate": 9.853745642130325e-05,
+      "loss": 0.7263545532226563,
+      "step": 32250
+    },
+    {
+      "epoch": 0.8290446217324482,
+      "grad_norm": 0.08154796808958054,
+      "learning_rate": 9.499447294577818e-05,
+      "loss": 0.7248871459960937,
+      "step": 32500
+    },
+    {
+      "epoch": 0.835421888053467,
+      "grad_norm": 0.08672577142715454,
+      "learning_rate": 9.145148947025311e-05,
+      "loss": 0.7239915771484375,
+      "step": 32750
+    },
+    {
+      "epoch": 0.8417991543744858,
+      "grad_norm": 0.08093911409378052,
+      "learning_rate": 8.790850599472804e-05,
+      "loss": 0.723066162109375,
+      "step": 33000
+    },
+    {
+      "epoch": 0.8481764206955047,
+      "grad_norm": 0.07659825682640076,
+      "learning_rate": 8.436552251920298e-05,
+      "loss": 0.72345263671875,
+      "step": 33250
+    },
+    {
+      "epoch": 0.8545536870165235,
+      "grad_norm": 0.08022043108940125,
+      "learning_rate": 8.082253904367789e-05,
+      "loss": 0.72362255859375,
+      "step": 33500
+    },
+    {
+      "epoch": 0.8609309533375423,
+      "grad_norm": 0.08199780434370041,
+      "learning_rate": 7.727955556815283e-05,
+      "loss": 0.7216689453125,
+      "step": 33750
+    },
+    {
+      "epoch": 0.8673082196585612,
+      "grad_norm": 0.07852373272180557,
+      "learning_rate": 7.373657209262777e-05,
+      "loss": 0.7197748413085937,
+      "step": 34000
+    },
+    {
+      "epoch": 0.87368548597958,
+      "grad_norm": 0.07824918627738953,
+      "learning_rate": 7.019358861710269e-05,
+      "loss": 0.7181076049804688,
+      "step": 34250
+    },
+    {
+      "epoch": 0.8800627523005988,
+      "grad_norm": 0.07989538460969925,
+      "learning_rate": 6.665060514157763e-05,
+      "loss": 0.7219158325195313,
+      "step": 34500
+    },
+    {
+      "epoch": 0.8864400186216177,
+      "grad_norm": 0.07987392693758011,
+      "learning_rate": 6.310762166605256e-05,
+      "loss": 0.720794189453125,
+      "step": 34750
+    },
+    {
+      "epoch": 0.8928172849426365,
+      "grad_norm": 0.08085554093122482,
+      "learning_rate": 5.956463819052748e-05,
+      "loss": 0.7185697021484375,
+      "step": 35000
+    },
+    {
+      "epoch": 0.8991945512636553,
+      "grad_norm": 0.0825972706079483,
+      "learning_rate": 5.602165471500241e-05,
+      "loss": 0.7219225463867187,
+      "step": 35250
+    },
+    {
+      "epoch": 0.9055718175846742,
+      "grad_norm": 0.08420496433973312,
+      "learning_rate": 5.247867123947734e-05,
+      "loss": 0.7178782348632813,
+      "step": 35500
+    },
+    {
+      "epoch": 0.911949083905693,
+      "grad_norm": 0.07988401502370834,
+      "learning_rate": 4.893568776395227e-05,
+      "loss": 0.7176174926757812,
+      "step": 35750
+    },
+    {
+      "epoch": 0.9183263502267118,
+      "grad_norm": 0.08001978695392609,
+      "learning_rate": 4.53927042884272e-05,
+      "loss": 0.7183565673828125,
+      "step": 36000
+    },
+    {
+      "epoch": 0.9247036165477307,
+      "grad_norm": 0.07920462638139725,
+      "learning_rate": 4.184972081290213e-05,
+      "loss": 0.7176240234375,
+      "step": 36250
+    },
+    {
+      "epoch": 0.9310808828687495,
+      "grad_norm": 0.08121761679649353,
+      "learning_rate": 3.8306737337377056e-05,
+      "loss": 0.71547265625,
+      "step": 36500
+    },
+    {
+      "epoch": 0.9374581491897683,
+      "grad_norm": 0.07785205543041229,
+      "learning_rate": 3.476375386185199e-05,
+      "loss": 0.7175172729492187,
+      "step": 36750
+    },
+    {
+      "epoch": 0.9438354155107872,
+      "grad_norm": 0.07759752124547958,
+      "learning_rate": 3.122077038632692e-05,
+      "loss": 0.7157890625,
+      "step": 37000
+    },
+    {
+      "epoch": 0.950212681831806,
+      "grad_norm": 0.08237745612859726,
+      "learning_rate": 2.7677786910801846e-05,
+      "loss": 0.7169793701171875,
+      "step": 37250
+    },
+    {
+      "epoch": 0.9565899481528248,
+      "grad_norm": 0.07707587629556656,
+      "learning_rate": 2.4134803435276777e-05,
+      "loss": 0.715412109375,
+      "step": 37500
+    },
+    {
+      "epoch": 0.9629672144738436,
+      "grad_norm": 0.07932782173156738,
+      "learning_rate": 2.059181995975171e-05,
+      "loss": 0.71594677734375,
+      "step": 37750
+    },
+    {
+      "epoch": 0.9693444807948625,
+      "grad_norm": 0.08169736713171005,
+      "learning_rate": 1.704883648422664e-05,
+      "loss": 0.7143995361328125,
+      "step": 38000
+    },
+    {
+      "epoch": 0.9757217471158813,
+      "grad_norm": 0.08160339295864105,
+      "learning_rate": 1.3505853008701568e-05,
+      "loss": 0.7135118408203125,
+      "step": 38250
+    },
+    {
+      "epoch": 0.9820990134369001,
+      "grad_norm": 0.08456628769636154,
+      "learning_rate": 9.962869533176497e-06,
+      "loss": 0.714119384765625,
+      "step": 38500
+    },
+    {
+      "epoch": 0.988476279757919,
+      "grad_norm": 0.07943534851074219,
+      "learning_rate": 6.419886057651427e-06,
+      "loss": 0.7134039306640625,
+      "step": 38750
+    },
+    {
+      "epoch": 0.9948535460789378,
+      "grad_norm": 0.0769161656498909,
+      "learning_rate": 2.8769025821263568e-06,
+      "loss": 0.71384814453125,
+      "step": 39000
+    },
+    {
+      "epoch": 1.0,
+      "step": 39202,
+      "total_flos": 7.006185158986433e+18,
+      "train_loss": 0.8243957067240222,
+      "train_runtime": 51861.4856,
+      "train_samples_per_second": 48.377,
+      "train_steps_per_second": 0.756
+    }
+  ],
+  "logging_steps": 250,
+  "max_steps": 39202,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3921,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.006185158986433e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000..d7949fb
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae6f98e823a238a294529c427b3bd655f713df8fa2961b52dc5026bbf93597ac
+size 5265