初始化项目，由ModelHub XC社区提供模型

Model: AetherModels/AetherCuTF-Ko-3-shared-Seed1234
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,58 @@
+---
+library_name: transformers
+base_model: llama3_gptsize_config_dtypefix.json
+tags:
+- generated_from_trainer
+model-index:
+- name: AetherCuTF-Ko-3-shared-V7-Seed1234-LR5e-4
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# AetherCuTF-Ko-3-shared-V7-Seed1234-LR5e-4
+
+This model is a fine-tuned version of [llama3_gptsize_config_dtypefix.json](https://huggingface.co/llama3_gptsize_config_dtypefix.json) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0005
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 1234
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 0.1
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 5.0.0
+- Pytorch 2.10.0+cu128
+- Datasets 4.8.2
+- Tokenizers 0.22.2

aether.py

@@ -0,0 +1,315 @@
+import regex
+from unicodedata import normalize
+from .bpe_utils import bytes_to_unicode_special
+from random import shuffle, seed
+from math import ceil
+
+def pretok_with_targeted_handling(text_data, split_regex, target_regex, target_process, nontarget_process):
+    if split_regex:
+        # 1st Regex: The general split regex
+        split_regex = regex.compile(split_regex)
+        split_regexed = split_regex.findall(text_data)
+    else:
+        split_regexed = [text_data]
+
+    # 2nd Regex: Splits a string into target and non-target chunks
+    final_processed_list = []
+    for chunk in split_regexed:
+        if not chunk:
+            continue # Skip empty tokens if any
+
+        # Split the token into target/non-target parts
+        processed_sub_tokens = ""
+        for match in regex.finditer(target_regex, chunk):
+            # Check if the chunk is target to decide which process to use
+            label = match.lastgroup
+            content = match.group()
+            if label == 'target':
+                processed_sub_tokens+=(target_process(content))
+            else:
+                processed_sub_tokens+=(nontarget_process(content))
+    
+        final_processed_list.append(processed_sub_tokens)
+
+    return final_processed_list
+
+def pretok_with_targeted_handling_list(text_data, split_regex, target_regex, target_process, nontarget_process):
+    if split_regex:
+        # 1st Regex: The general split regex
+        split_regex = regex.compile(split_regex)
+        split_regexed = split_regex.findall(text_data)
+    else:
+        split_regexed = [text_data]
+    # 2nd Regex: Splits a string into target and non-target chunks
+    final_processed_list = []
+    for chunk in split_regexed:
+        if not chunk:
+            continue # Skip empty tokens if any
+        # Split the token into target/non-target parts
+        for match in regex.finditer(target_regex, chunk):
+            # Check if the chunk is target to decide which process to use
+            label = match.lastgroup
+            content = match.group()
+            if label == 'target':
+                final_processed_list+=target_process(content)
+            else:
+                final_processed_list+=nontarget_process(content)
+    return final_processed_list
+
+
+# String "좋아" -> String "ƴƸƯŠ"
+def apply_encoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
+    if "Jamo" in encoding_type:
+        varlen = "varlen" in encoding_type
+        final_string = ""
+        for char in input_string:
+            if char == " ":
+                final_string += mapping[ord(char)]
+                continue
+            normalized = normalize("NFKD", char)
+            if len(normalized)==1: # Single Jamo can be used, but if not usable need fallback
+                if normalized not in mapping:
+                    final_string += apply_encoding(normalized, "utf-8", mapping)
+                    continue
+            final_string += normalized
+            if not varlen and len(normalized)==2: # Hangul syllable decomposed into 2 Jamo
+                final_string += '∅'  # Placeholder for missing Jamo                    
+        return final_string
+    if shuffle:
+        # For each character, calculate offset and map to shuffled offset
+        input_string = "".join([chr(offset_mapping[ord(c) - unicode_start] + unicode_start) for c in input_string])
+    if encoding_type == "nybbles":
+        return apply_nybble_encoding(input_string, mapping, offset)
+    encoded = input_string.encode(encoding_type)
+    return "".join([mapping.get(x + offset) if x + offset in mapping else mapping.get(x) for x in encoded])
+
+# String "ƴƸƯŠ"  -> String "좋아"
+def apply_decoding(input_string, encoding_type, mapping, offset=0):
+    antimap = {value: key for key, value in mapping.items()}
+    return b"".join([antimap[a+offset].to_bytes() for a in input_string]).decode(encoding_type)
+
+def apply_character_decoding(input_string, mapping):
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    char_mode = False
+    for symbol in input_string:
+        # For token in charmode
+        if symbol not in antimap:
+            # If we are in the normal case, resolve the current temp_string
+            if not char_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = ""
+            temp_string += symbol
+            char_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in charmode, resolve the current temp_string
+            if char_mode:
+                final_string += temp_string
+                temp_string = b""
+            temp_string += (antimap[symbol]).to_bytes()
+            char_mode = False
+    # Resolve final piece
+    if char_mode:
+        final_string += temp_string
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+# String "è¿Ļæĺ¯ƴƸƯŠ"  -> String "这是좋아"
+def apply_specialized_decoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
+    if shuffle:
+        def unshuffle_char(c):
+            shuffled_offset = ord(c) - unicode_start
+            original_offset = offset_mapping[shuffled_offset]
+            return chr(original_offset + unicode_start)
+    
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    special_mode = False
+    for symbol in input_string:
+        byte_id = antimap[symbol]
+        # For token in special case
+        if byte_id >= offset:
+            # If we are in the normal case, resolve the current temp_string
+            if not special_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = b""
+            temp_string += (byte_id - offset).to_bytes()
+            special_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in the special case, resolve the current temp_string
+            if special_mode:
+                temp_add = temp_string.decode(encoding_type, errors='replace')
+                if shuffle:
+                    temp_add = "".join([unshuffle_char(c) for c in temp_add])
+                final_string += temp_add
+                temp_string = b""
+            temp_string += (byte_id).to_bytes()
+            special_mode = False
+    # Resolve final piece
+    if special_mode:
+        temp_add = temp_string.decode(encoding_type, errors='replace')
+        if shuffle:
+            temp_add = "".join([unshuffle_char(c) for c in temp_add])
+        final_string += temp_add
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+def apply_nybble_decoding(input_string, mapping, offset=0):
+    def decode_char(temp_string):
+        res = []
+        for i in range(0, len(temp_string), 4):
+            try:
+                n0, n1, n2, n3 = [t for t in temp_string[i:i+4]]
+                codepoint = (n0 << 12) | (n1 << 8) | (n2 << 4) | n3
+                res.append(chr(codepoint))
+            except:
+                # In case of any error (e.g., invalid character), append a replacement character
+                res.append('<EFBFBD>')
+        return "".join(res)
+
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    temp_string = b""
+    special_mode = False
+    for symbol in input_string:
+        byte_id = antimap[symbol]
+        # For token in special case
+        if byte_id >= offset:
+            # If we are in the normal case, resolve the current temp_string
+            if not special_mode:
+                final_string += temp_string.decode("utf8", errors='replace')
+                temp_string = b""
+            temp_string += (byte_id - offset).to_bytes()
+            special_mode = True
+        # For tokens in regular case
+        else:
+            # If we are in the special case, resolve the current temp_string
+            if special_mode:
+                final_string += decode_char(temp_string)
+                temp_string = b""
+            temp_string += (byte_id).to_bytes()
+            special_mode = False
+    # Resolve final piece
+    if special_mode:
+        final_string += decode_char(temp_string)
+    else:
+        final_string += temp_string.decode('utf8', errors='replace')
+    
+    return final_string
+
+def apply_Jamo_decoding(input_string, mapping):
+    antimap = {value: key for key, value in mapping.items()}
+    final_string = ""
+    current_type = type(input_string[0])
+    current_pieces = []
+    for char in input_string:
+        char = antimap[char]
+        if type(char) == current_type:
+            current_pieces.append(char)
+        else:
+            # Process current pieces
+            if current_type == str:
+                # Jamo pieces
+                syllables = "".join((current_pieces)).replace("∅", "")
+                # Put it into unicode format
+                final_string += normalize("NFKC", syllables)
+            else:
+                # Byte pieces
+                byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
+                final_string += byte_string
+            current_type = type(char)
+            current_pieces = [char]
+    # Final flush
+    if current_type == str:
+        # Jamo pieces
+        syllables = "".join((current_pieces)).replace("∅", "")
+        final_string += syllables
+    else:
+        # Byte pieces
+        byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
+        final_string += byte_string
+    return final_string 
+
+
+
+def apply_nybble_encoding(input_string, mapping, offset,n_nybbles=4):
+    def encode_char(c):
+        o = ord(c)
+        return "".join(mapping[offset + ((o >> (4 * i)) & 0xF)] for i in range(n_nybbles - 1, -1, -1))
+    
+    return "".join(encode_char(c) for c in input_string)
+
+# Given a range of unicode, we shuffle the points
+# For performance this can be stored as a list, a mapping of offset2newoffset
+def create_shuffled_unicode_mapping(range_start, range_end, given_seed=6767):
+    unicode_points = list(range(range_end - range_start + 1))
+    seed(given_seed)
+    shuffle(unicode_points)
+    # to use, calculate offset = ord(char) - range_start, then map to unicode_points[offset] + range_start
+    return unicode_points
+
+# Given a range of unicode, provide a Customized UTF with given parameter properties
+def create_CuTF_encoding(range_start, range_end, encode_length):
+    target_space_len = range_end - range_start + 1
+    num_indices = ceil(target_space_len**(1/encode_length))
+    # Plan is to do the math only here and save the mapping for efficiency
+    CuTF_mapping = {}
+    for codepoint in range(range_start, range_end+1):
+        offset = codepoint-range_start
+        CuTF_rep = []
+        for i in range(encode_length-1):
+            rep_index = offset//(num_indices**(encode_length-i-1))
+            CuTF_rep.append(rep_index)
+            offset -= (num_indices**(encode_length-i-1))*rep_index
+        CuTF_rep.append(offset)
+        CuTF_mapping[codepoint] = CuTF_rep
+    
+    return CuTF_mapping, num_indices
+
+
+
+def apply_CuTF_decoding(input_string, encode_length, num_index, range_start, indexing_strategy="shared"):
+    def decode_target_chars(sequence):
+        outpieces = ""
+        for i in range(0, len(sequence), encode_length):
+            piece = sequence[i:i+encode_length]
+            offset = 0
+            for j, byte in enumerate(piece):
+                if indexing_strategy == "shared":
+                    offset += (byte-256) * (num_index**(encode_length-j-1))
+                elif indexing_strategy == "unique":
+                    offset += (byte - (256 + num_index*(encode_length-j-1))) * (num_index**(encode_length-j-1))
+            outpieces += chr(range_start + offset)
+        return outpieces
+    final_string = ""
+    temp_pieces = []
+    is_target = False
+    for token in input_string:
+        if token >= 256:
+            if not is_target:
+                # Flush non-target tokens
+                final_string += bytes(temp_pieces).decode("utf8", errors='replace')
+                temp_pieces = []
+            temp_pieces.append(token)
+            is_target = True
+        else:
+            if is_target:
+                # Flush target tokens
+                final_string += decode_target_chars(temp_pieces)
+                temp_pieces = []
+            temp_pieces.append(token)
+            is_target = False
+    # Flush any remaining tokens
+    if is_target:
+        final_string += decode_target_chars(temp_pieces)
+    else:
+        final_string += bytes(temp_pieces).decode("utf8", errors='replace')
+    return final_string

aethertokenizers.py

@@ -0,0 +1,467 @@
+from unittest import result
+
+from .aether import apply_CuTF_decoding, apply_Jamo_decoding, apply_nybble_decoding, create_CuTF_encoding, pretok_with_targeted_handling, apply_encoding, apply_specialized_decoding, create_shuffled_unicode_mapping, apply_character_decoding, pretok_with_targeted_handling_list
+from .consts import *
+from .bpe_utils import bytes_to_unicode_original, bytes_to_unicode_special
+from transformers.tokenization_utils import PreTrainedTokenizer
+import os, json, shutil
+from typing import Optional
+
+
+# Approximately modeled after byt5 tokenizer
+# https://github.com/huggingface/transformers/blob/v4.57.1/src/transformers/models/byt5/tokenization_byt5.py
+
+class AetherByteTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, encoding, shuffle=False, offset_mapping=None,**kwargs):
+        self.lang = lang
+        self.encoding = encoding
+        self.shuffle = shuffle
+        if shuffle:
+            self.unicode_start = lang_ranges[lang][0]
+            if offset_mapping is not None:
+                self.offset_mapping = offset_mapping
+            else:
+                self.offset_mapping = create_shuffled_unicode_mapping(*lang_ranges[lang]) # list that acts as offset2newoffset
+            self.offset_unmapping = {v: k for k, v in enumerate(self.offset_mapping)} # newoffset2offset, for decoding
+        self.target_regex = target_regexes[lang]
+        if encoding == "utf8":
+            Aether_info = Aethers["utf8-"+lang]
+            if lang == "Ko":
+                # This is to limit Korean to full syllables only for this mode
+                self.target_regex = target_regexes["Ko-SO"]
+        else:
+            Aether_info = Aethers[encoding]
+        self.aether_type = Aether_info['type']
+        # Mapping == aether2symbol
+        self.mapping = bytes_to_unicode_special(Aether_info['pieces'], OFFSET) if self.aether_type == "bytes" else {a: a for a in Aether_info['pieces']}|bytes_to_unicode_original()
+        self.index2aether = self._make_index2aether(self.mapping)
+        self.index2symbol = {index: self.mapping[aether] for index, aether in self.index2aether.items()}
+        self.symbol2index = {value: key for key, value in self.index2symbol.items()}
+        self.target_process = lambda x: apply_encoding(x, encoding, self.mapping, OFFSET, self.shuffle, self.offset_mapping if self.shuffle else None, self.unicode_start if self.shuffle else None) #text2symbols
+        self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, encoding=encoding, shuffle=shuffle, **kwargs)
+    
+    def _make_index2aether(self, mapping):
+        index2aether = {index: aether_id for index, aether_id in enumerate(mapping)}
+        return index2aether
+        
+    @property
+    def vocab_size(self):
+        return len(self.mapping) + len(self.added_tokens_encoder)
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        mapping_path = os.path.join(pretrained_model_name_or_path, "offset_mapping.json")
+        if os.path.exists(mapping_path):
+            with open(mapping_path, "r") as f:
+                kwargs["offset_mapping"] = json.load(f)
+        return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in self.index2aether}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+    def pretokenize(self, input):
+        return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        pretokenized = self.pretokenize(text)
+        tokens = []
+        for chunk in pretokenized:
+            tokens+=chunk
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        tok_id = self.symbol2index[token]
+        return tok_id
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return self.index2symbol[index]
+    
+    def _apply_decoding(self, tokens):
+        if self.aether_type == "strings":
+            if self.encoding == "Jamo":
+                result = apply_Jamo_decoding(tokens, self.mapping)
+            elif self.encoding == "Jamo-varlen":
+                result = apply_Jamo_decoding(tokens, self.mapping)
+        elif self.aether_type == "bytes":
+            if self.encoding == "nybbles":
+                result = apply_nybble_decoding(tokens, self.mapping, OFFSET)
+            else:
+                result = apply_specialized_decoding(tokens, self.encoding, self.mapping, OFFSET, self.shuffle, self.offset_unmapping if self.shuffle else None, self.unicode_start if self.shuffle else None)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        if self.shuffle:
+            mapping_path = os.path.join(save_directory, "offset_mapping.json")
+            with open(mapping_path, "w") as f:
+                json.dump(self.offset_mapping, f)
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherByteTokenizer", None]
+        }
+        
+        config["shuffle"] = self.shuffle
+
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+class AetherCharTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, **kwargs):
+        self.lang = lang
+        self.unicode_start = lang_ranges[lang][0]
+        self.unicode_end = lang_ranges[lang][1]
+        self.target_regex = target_regexes[lang]
+        if lang == "Ko":
+            # This is to limit Korean to full syllables only for this mode
+            self.target_regex = target_regexes["Ko-SO"]
+        self.target_process = lambda x: x
+        self.mapping = bytes_to_unicode_original()
+        self.symbol2index = {value: key for key, value in self.mapping.items()}
+        self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, **kwargs)
+        
+    @property
+    def vocab_size(self):
+        return len(self.mapping) + (self.unicode_end - self.unicode_start + 1) +  len(self.added_tokens_encoder)
+    
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in self.mapping}
+        vocab.update({chr(i): i - self.unicode_start + len(self.mapping) for i in range(self.unicode_start, self.unicode_end+1)})
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def pretokenize(self, input):
+        return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        pretokenized = self.pretokenize(text)
+        tokens = []
+        for chunk in pretokenized:
+            tokens+=chunk
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to token_id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        if token in self.symbol2index:
+            return self.symbol2index[token]
+        if self.unicode_start <= ord(token) <= self.unicode_end:
+            return ord(token) - self.unicode_start + len(self.mapping)
+        raise ValueError(f"Token {token} not in vocab")
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        if index < len(self.mapping):
+            return self.mapping[index]
+        return chr(index - len(self.mapping) + self.unicode_start)
+    
+    def _apply_decoding(self, tokens):
+        result = apply_character_decoding(tokens, self.mapping)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherCharTokenizer", None]
+        }
+        
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+
+# A comparable tokenizer in UTF-8.
+# Could just use ByT5 too, but controls for special tokens this way
+
+class UTF8ByteTokenizer(PreTrainedTokenizer):
+    def __init__(self, **kwargs):
+        self.mapping = bytes_to_unicode_special([]) #AKA index2symbol
+        self.symbol2index = {value: key for key, value in self.mapping.items()}
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(**kwargs)
+        
+    @property
+    def vocab_size(self):
+        return 256 + len(self.added_tokens_encoder)
+    
+
+    def get_vocab(self):
+        # vocab is symbol2tok_id
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(256)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        text_bytes = text.encode('utf-8')
+        tokens = [self.mapping[b] for b in text_bytes]
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        return self.symbol2index[token]
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return self.mapping[index]
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (symbols) into a single string."""
+        symbol2byte = self.get_vocab()
+        final_string = ""
+        curr_bstring = b""
+        for token in tokens:
+            if token in self.all_special_tokens:
+                # Resolve current bstring
+                if curr_bstring:
+                    string = curr_bstring.decode("utf-8", errors="replace")
+                    final_string += string
+                    curr_bstring = b""
+                final_string += token
+            else:
+                curr_bstring += bytes([symbol2byte[token]])
+        if curr_bstring:
+            string = curr_bstring.decode("utf-8", errors="replace")
+            final_string += string
+        return final_string
+            
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.UTF8ByteTokenizer", None]
+        }
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved
+
+
+
+class AetherCuTFTokenizer(PreTrainedTokenizer):
+    def __init__(self, lang, encode_length, indexing_strategy="shared", **kwargs):
+        self.lang = lang
+        self.encode_length = encode_length
+        self.indexing_strategy = indexing_strategy
+        self.target_regex = target_regexes[lang]
+        if lang == "Ko":
+            # This is to limit Korean to full syllables only for this mode
+            self.target_regex = target_regexes["Ko-SO"]
+        self.unicode_start = lang_ranges[lang][0]
+        self.unicode_end = lang_ranges[lang][1]
+        
+        raw_mapping, self.num_CuTF_indices = create_CuTF_encoding(self.unicode_start, self.unicode_end, encode_length)
+        mapping = dict()
+        if indexing_strategy == "shared":
+            for codepoint in raw_mapping:
+                mapping[codepoint] = [256+ a for a in raw_mapping[codepoint]]
+        elif indexing_strategy == "unique":
+            for codepoint in raw_mapping:
+                mapping[codepoint] = [256+ (encode_length -1 - i) * self.num_CuTF_indices + a for i, a in enumerate(raw_mapping[codepoint])]
+        self.mapping = mapping
+        self.target_process = lambda x: [rep for a in x for rep in self.mapping[ord(a)]]
+        self.nontarget_process = lambda x: [a for a in x.encode('utf8')]
+
+        # We want bos and eos tokens for our models
+        kwargs.setdefault('bos_token', '<s>')
+        kwargs.setdefault('eos_token', '</s>')
+        kwargs.setdefault('special_tokens_pattern', 'bos')
+        super().__init__(lang=lang, encode_length = encode_length, indexing_strategy= indexing_strategy, **kwargs)
+    
+    @property
+    def vocab_size(self):
+        if self.indexing_strategy == "shared":
+            return 256 + self.num_CuTF_indices + len(self.added_tokens_encoder)
+
+        elif self.indexing_strategy == "unique":
+            # This is the simple answer, but in reality the full bytespace isn't utilized.
+            # return 256 + self.num_CuTF_indices * self.encode_length + len(self.added_tokens_encoder)
+            # Need to use the highest number as reference
+            max_index = self.mapping[self.unicode_end][0]
+            return max_index + len(self.added_tokens_encoder) + 1
+                    
+    
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size-len(self.added_tokens_encoder))}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+    
+    def _tokenize(self, text: str) -> list[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        return pretok_with_targeted_handling_list(text, None, self.target_regex, self.target_process, self.nontarget_process)
+        
+    def _convert_token_to_id(self, token):
+        # Symbol to id
+        if token in self.added_tokens_encoder:
+            return self.added_tokens_encoder[token]
+        return token
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.added_tokens_decoder:
+            return self.added_tokens_decoder[index]
+        return index
+    
+    def _apply_decoding(self, tokens):
+        result = apply_CuTF_decoding(tokens, self.encode_length, self.num_CuTF_indices, self.unicode_start, self.indexing_strategy)
+        return result
+
+    def convert_tokens_to_string(self, tokens):
+        special_tokens_set = set(self.all_special_tokens)
+        """Converts a sequence of tokens (symbols) into a single string."""
+        final_string = ""
+        current_iter = []
+        for token in tokens:
+            if token in special_tokens_set:
+                # Resolve current_iter
+                if current_iter:
+                    resolved = self._apply_decoding(current_iter)
+                    final_string += resolved
+                    current_iter = []
+                final_string += token
+            else:
+                current_iter.append(token)
+        if current_iter:
+            resolved = self._apply_decoding(current_iter)
+            final_string += resolved
+        return final_string
+        
+
+    # Tokenizer has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        return ()
+
+    def save_pretrained(self, save_directory, **kwargs):
+        saved = super().save_pretrained(save_directory, **kwargs)
+        # Copy this file and all local dependencies into the directory
+        src_dir = os.path.dirname(__file__)
+        for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
+            shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
+        
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config["auto_map"] = {
+            "AutoTokenizer": ["aethertokenizers.AetherCuTFTokenizer", None]
+        }
+        
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+        return saved

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.745954960876503e+18,
+    "train_loss": 0.052523610041691704,
+    "train_runtime": 7246.7996,
+    "train_samples": 3494359,
+    "train_samples_per_second": 482.193,
+    "train_steps_per_second": 7.534
+}

bpe_utils.py

@@ -0,0 +1,64 @@
+from .consts import OFFSET
+
+def bytes_to_unicode_original():
+    # Can call this through
+    # from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+def bytes_to_unicode_special(special_bytes, offset=OFFSET):
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    
+    for b in special_bytes:
+        bs.append(offset+b)
+        cs.append(2**8 + n)
+        n += 1
+
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+byte2unic = bytes_to_unicode_original()
+unic2byte = {value: key for key, value in byte2unic.items()}
+
+def decode_uniced(scrambled):
+    unic_nums = [unic2byte[a] for a in scrambled]
+    char_byte = [chr(a) for a in unic_nums]
+    full = "".join(char_byte)
+    return full.encode("latin-1").decode()
+
+
+def find_aether_bytes(encoding, range_start, range_end):
+    aether_bytes = set()
+    for i in range(range_start, range_end):
+        pieces = chr(i).encode(encoding)
+        for piece in pieces:
+            aether_bytes.add(piece)
+    return aether_bytes

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 279,
+  "dtype": "float32",
+  "eos_token_id": 280,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 500000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.0.0",
+  "use_cache": false,
+  "vocab_size": 288
+}

consts.py

@@ -0,0 +1,57 @@
+OFFSET = 10000
+
+# Regexes
+llama_regex = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+target_regexes = {
+                # This is the one with leading space, could be used as a pretok later
+                # "Ko": r"(?P<target> ?[가-힣ㄱ-ㅣ]+)|(?P<other>(?:[^가-힣ㄱ-ㅣ ]| (?![가-힣ㄱ-ㅣ]))+)",
+                "Ko": r"(?P<target>[가-힣ㄱ-ㅣ]+)|(?P<other>[^가-힣ㄱ-ㅣ]+)",
+                "Ko-SO": r"(?P<target>[가-힣]+)|(?P<other>[^가-힣]+)", # Syllables Only
+                "Zh": r"(?P<target>[\u4e00-\u9fa5]+)|(?P<other>[^\u4e00-\u9fa5]+)",
+                  }
+
+lang_ranges = {
+    "Ko": (0xAC00, 0xD7A3),
+    "Zh": (0x4E00, 0x9FA5)}
+
+# per-encoding pieces
+used_bytes ={}
+Aethers = {
+    'Johab': {
+        'type': "bytes",
+        'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253]
+    },
+    'Jamo': {
+        'type': "strings",
+        'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ', '∅']
+    },
+    'Jamo-varlen': {
+        'type': "strings",
+        'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ']
+    },
+    'GBK': {
+        'type': "bytes",
+        'pieces': [64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
+    },
+    'utf-16-be': {
+        'type': "bytes",
+        'pieces': range(0, 256)
+    },
+    'cp949': {
+        'type': "bytes",
+        'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
+    },
+    'nybbles': {
+        'type': "bytes",
+        'pieces': range(0, 16)
+    },
+    'utf8-Ko': {
+        'type': "bytes",
+        'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 234, 235, 236, 237]
+    }
+    ,
+    'utf8-Zh': {
+        'type': "bytes",
+        'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 228,229, 230, 231, 232, 233]
+    }
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 279,
+  "eos_token_id": [
+    280
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.0.0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c68f15489df01c76fd0dea86abdc7513ad6ae3250769f4e623154bf76107f68
+size 454843368

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "added_tokens_decoder": {
+    "279": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "280": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": null,
+  "backend": "custom",
+  "bos_token": "<s>",
+  "encode_length": 3,
+  "eos_token": "</s>",
+  "indexing_strategy": "shared",
+  "lang": "Ko",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "AetherCuTFTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "aethertokenizers.AetherCuTFTokenizer",
+      null
+    ]
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.745954960876503e+18,
+    "train_loss": 0.052523610041691704,
+    "train_runtime": 7246.7996,
+    "train_samples": 3494359,
+    "train_samples_per_second": 482.193,
+    "train_steps_per_second": 7.534
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f481d5b0726409662acd47a76e5834f7d48be364a3e2ed6d4ba8dfc81efbee17
+size 5265