AetherModels/AetherCuTF-Ko-2-shared-Seed1234
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: AetherModels/AetherCuTF-Ko-2-shared-Seed1234
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-13 11:45:11 +08:00
commit b083bf55e7
14 changed files with 2233 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

58
README.md Normal file
View File

@@ -0,0 +1,58 @@
---
library_name: transformers
base_model: llama3_gptsize_config_dtypefix.json
tags:
- generated_from_trainer
model-index:
- name: AetherCuTF-Ko-2-shared-V7-Seed1234-LR5e-4
results: []
---
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
# AetherCuTF-Ko-2-shared-V7-Seed1234-LR5e-4
This model is a fine-tuned version of [llama3_gptsize_config_dtypefix.json](https://huggingface.co/llama3_gptsize_config_dtypefix.json) on the None dataset.
## Model description
More information needed
## Intended uses & limitations
More information needed
## Training and evaluation data
More information needed
## Training procedure
### Training hyperparameters
The following hyperparameters were used during training:
- learning_rate: 0.0005
- train_batch_size: 8
- eval_batch_size: 8
- seed: 1234
- distributed_type: multi-GPU
- num_devices: 2
- gradient_accumulation_steps: 4
- total_train_batch_size: 64
- total_eval_batch_size: 16
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 0.1
- num_epochs: 1.0
### Training results
### Framework versions
- Transformers 5.0.0
- Pytorch 2.10.0+cu128
- Datasets 4.8.2
- Tokenizers 0.22.2

315
aether.py Normal file
View File

@@ -0,0 +1,315 @@
import regex
from unicodedata import normalize
from .bpe_utils import bytes_to_unicode_special
from random import shuffle, seed
from math import ceil
def pretok_with_targeted_handling(text_data, split_regex, target_regex, target_process, nontarget_process):
if split_regex:
# 1st Regex: The general split regex
split_regex = regex.compile(split_regex)
split_regexed = split_regex.findall(text_data)
else:
split_regexed = [text_data]
# 2nd Regex: Splits a string into target and non-target chunks
final_processed_list = []
for chunk in split_regexed:
if not chunk:
continue # Skip empty tokens if any
# Split the token into target/non-target parts
processed_sub_tokens = ""
for match in regex.finditer(target_regex, chunk):
# Check if the chunk is target to decide which process to use
label = match.lastgroup
content = match.group()
if label == 'target':
processed_sub_tokens+=(target_process(content))
else:
processed_sub_tokens+=(nontarget_process(content))
final_processed_list.append(processed_sub_tokens)
return final_processed_list
def pretok_with_targeted_handling_list(text_data, split_regex, target_regex, target_process, nontarget_process):
if split_regex:
# 1st Regex: The general split regex
split_regex = regex.compile(split_regex)
split_regexed = split_regex.findall(text_data)
else:
split_regexed = [text_data]
# 2nd Regex: Splits a string into target and non-target chunks
final_processed_list = []
for chunk in split_regexed:
if not chunk:
continue # Skip empty tokens if any
# Split the token into target/non-target parts
for match in regex.finditer(target_regex, chunk):
# Check if the chunk is target to decide which process to use
label = match.lastgroup
content = match.group()
if label == 'target':
final_processed_list+=target_process(content)
else:
final_processed_list+=nontarget_process(content)
return final_processed_list
# String "좋아" -> String "ƴƸƯŠ"
def apply_encoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
if "Jamo" in encoding_type:
varlen = "varlen" in encoding_type
final_string = ""
for char in input_string:
if char == " ":
final_string += mapping[ord(char)]
continue
normalized = normalize("NFKD", char)
if len(normalized)==1: # Single Jamo can be used, but if not usable need fallback
if normalized not in mapping:
final_string += apply_encoding(normalized, "utf-8", mapping)
continue
final_string += normalized
if not varlen and len(normalized)==2: # Hangul syllable decomposed into 2 Jamo
final_string += '∅' # Placeholder for missing Jamo
return final_string
if shuffle:
# For each character, calculate offset and map to shuffled offset
input_string = "".join([chr(offset_mapping[ord(c) - unicode_start] + unicode_start) for c in input_string])
if encoding_type == "nybbles":
return apply_nybble_encoding(input_string, mapping, offset)
encoded = input_string.encode(encoding_type)
return "".join([mapping.get(x + offset) if x + offset in mapping else mapping.get(x) for x in encoded])
# String "ƴƸƯŠ" -> String "좋아"
def apply_decoding(input_string, encoding_type, mapping, offset=0):
antimap = {value: key for key, value in mapping.items()}
return b"".join([antimap[a+offset].to_bytes() for a in input_string]).decode(encoding_type)
def apply_character_decoding(input_string, mapping):
antimap = {value: key for key, value in mapping.items()}
final_string = ""
temp_string = b""
char_mode = False
for symbol in input_string:
# For token in charmode
if symbol not in antimap:
# If we are in the normal case, resolve the current temp_string
if not char_mode:
final_string += temp_string.decode("utf8", errors='replace')
temp_string = ""
temp_string += symbol
char_mode = True
# For tokens in regular case
else:
# If we are in charmode, resolve the current temp_string
if char_mode:
final_string += temp_string
temp_string = b""
temp_string += (antimap[symbol]).to_bytes()
char_mode = False
# Resolve final piece
if char_mode:
final_string += temp_string
else:
final_string += temp_string.decode('utf8', errors='replace')
return final_string
# String "è¿Ļæĺ¯ƴƸƯŠ" -> String "这是좋아"
def apply_specialized_decoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None):
if shuffle:
def unshuffle_char(c):
shuffled_offset = ord(c) - unicode_start
original_offset = offset_mapping[shuffled_offset]
return chr(original_offset + unicode_start)
antimap = {value: key for key, value in mapping.items()}
final_string = ""
temp_string = b""
special_mode = False
for symbol in input_string:
byte_id = antimap[symbol]
# For token in special case
if byte_id >= offset:
# If we are in the normal case, resolve the current temp_string
if not special_mode:
final_string += temp_string.decode("utf8", errors='replace')
temp_string = b""
temp_string += (byte_id - offset).to_bytes()
special_mode = True
# For tokens in regular case
else:
# If we are in the special case, resolve the current temp_string
if special_mode:
temp_add = temp_string.decode(encoding_type, errors='replace')
if shuffle:
temp_add = "".join([unshuffle_char(c) for c in temp_add])
final_string += temp_add
temp_string = b""
temp_string += (byte_id).to_bytes()
special_mode = False
# Resolve final piece
if special_mode:
temp_add = temp_string.decode(encoding_type, errors='replace')
if shuffle:
temp_add = "".join([unshuffle_char(c) for c in temp_add])
final_string += temp_add
else:
final_string += temp_string.decode('utf8', errors='replace')
return final_string
def apply_nybble_decoding(input_string, mapping, offset=0):
def decode_char(temp_string):
res = []
for i in range(0, len(temp_string), 4):
try:
n0, n1, n2, n3 = [t for t in temp_string[i:i+4]]
codepoint = (n0 << 12) | (n1 << 8) | (n2 << 4) | n3
res.append(chr(codepoint))
except:
# In case of any error (e.g., invalid character), append a replacement character
res.append('<EFBFBD>')
return "".join(res)
antimap = {value: key for key, value in mapping.items()}
final_string = ""
temp_string = b""
special_mode = False
for symbol in input_string:
byte_id = antimap[symbol]
# For token in special case
if byte_id >= offset:
# If we are in the normal case, resolve the current temp_string
if not special_mode:
final_string += temp_string.decode("utf8", errors='replace')
temp_string = b""
temp_string += (byte_id - offset).to_bytes()
special_mode = True
# For tokens in regular case
else:
# If we are in the special case, resolve the current temp_string
if special_mode:
final_string += decode_char(temp_string)
temp_string = b""
temp_string += (byte_id).to_bytes()
special_mode = False
# Resolve final piece
if special_mode:
final_string += decode_char(temp_string)
else:
final_string += temp_string.decode('utf8', errors='replace')
return final_string
def apply_Jamo_decoding(input_string, mapping):
antimap = {value: key for key, value in mapping.items()}
final_string = ""
current_type = type(input_string[0])
current_pieces = []
for char in input_string:
char = antimap[char]
if type(char) == current_type:
current_pieces.append(char)
else:
# Process current pieces
if current_type == str:
# Jamo pieces
syllables = "".join((current_pieces)).replace("∅", "")
# Put it into unicode format
final_string += normalize("NFKC", syllables)
else:
# Byte pieces
byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
final_string += byte_string
current_type = type(char)
current_pieces = [char]
# Final flush
if current_type == str:
# Jamo pieces
syllables = "".join((current_pieces)).replace("∅", "")
final_string += syllables
else:
# Byte pieces
byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace')
final_string += byte_string
return final_string
def apply_nybble_encoding(input_string, mapping, offset,n_nybbles=4):
def encode_char(c):
o = ord(c)
return "".join(mapping[offset + ((o >> (4 * i)) & 0xF)] for i in range(n_nybbles - 1, -1, -1))
return "".join(encode_char(c) for c in input_string)
# Given a range of unicode, we shuffle the points
# For performance this can be stored as a list, a mapping of offset2newoffset
def create_shuffled_unicode_mapping(range_start, range_end, given_seed=6767):
unicode_points = list(range(range_end - range_start + 1))
seed(given_seed)
shuffle(unicode_points)
# to use, calculate offset = ord(char) - range_start, then map to unicode_points[offset] + range_start
return unicode_points
# Given a range of unicode, provide a Customized UTF with given parameter properties
def create_CuTF_encoding(range_start, range_end, encode_length):
target_space_len = range_end - range_start + 1
num_indices = ceil(target_space_len**(1/encode_length))
# Plan is to do the math only here and save the mapping for efficiency
CuTF_mapping = {}
for codepoint in range(range_start, range_end+1):
offset = codepoint-range_start
CuTF_rep = []
for i in range(encode_length-1):
rep_index = offset//(num_indices**(encode_length-i-1))
CuTF_rep.append(rep_index)
offset -= (num_indices**(encode_length-i-1))*rep_index
CuTF_rep.append(offset)
CuTF_mapping[codepoint] = CuTF_rep
return CuTF_mapping, num_indices
def apply_CuTF_decoding(input_string, encode_length, num_index, range_start, indexing_strategy="shared"):
def decode_target_chars(sequence):
outpieces = ""
for i in range(0, len(sequence), encode_length):
piece = sequence[i:i+encode_length]
offset = 0
for j, byte in enumerate(piece):
if indexing_strategy == "shared":
offset += (byte-256) * (num_index**(encode_length-j-1))
elif indexing_strategy == "unique":
offset += (byte - (256 + num_index*(encode_length-j-1))) * (num_index**(encode_length-j-1))
outpieces += chr(range_start + offset)
return outpieces
final_string = ""
temp_pieces = []
is_target = False
for token in input_string:
if token >= 256:
if not is_target:
# Flush non-target tokens
final_string += bytes(temp_pieces).decode("utf8", errors='replace')
temp_pieces = []
temp_pieces.append(token)
is_target = True
else:
if is_target:
# Flush target tokens
final_string += decode_target_chars(temp_pieces)
temp_pieces = []
temp_pieces.append(token)
is_target = False
# Flush any remaining tokens
if is_target:
final_string += decode_target_chars(temp_pieces)
else:
final_string += bytes(temp_pieces).decode("utf8", errors='replace')
return final_string

467
aethertokenizers.py Normal file
View File

@@ -0,0 +1,467 @@
from unittest import result
from .aether import apply_CuTF_decoding, apply_Jamo_decoding, apply_nybble_decoding, create_CuTF_encoding, pretok_with_targeted_handling, apply_encoding, apply_specialized_decoding, create_shuffled_unicode_mapping, apply_character_decoding, pretok_with_targeted_handling_list
from .consts import *
from .bpe_utils import bytes_to_unicode_original, bytes_to_unicode_special
from transformers.tokenization_utils import PreTrainedTokenizer
import os, json, shutil
from typing import Optional
# Approximately modeled after byt5 tokenizer
# https://github.com/huggingface/transformers/blob/v4.57.1/src/transformers/models/byt5/tokenization_byt5.py
class AetherByteTokenizer(PreTrainedTokenizer):
def __init__(self, lang, encoding, shuffle=False, offset_mapping=None,**kwargs):
self.lang = lang
self.encoding = encoding
self.shuffle = shuffle
if shuffle:
self.unicode_start = lang_ranges[lang][0]
if offset_mapping is not None:
self.offset_mapping = offset_mapping
else:
self.offset_mapping = create_shuffled_unicode_mapping(*lang_ranges[lang]) # list that acts as offset2newoffset
self.offset_unmapping = {v: k for k, v in enumerate(self.offset_mapping)} # newoffset2offset, for decoding
self.target_regex = target_regexes[lang]
if encoding == "utf8":
Aether_info = Aethers["utf8-"+lang]
if lang == "Ko":
# This is to limit Korean to full syllables only for this mode
self.target_regex = target_regexes["Ko-SO"]
else:
Aether_info = Aethers[encoding]
self.aether_type = Aether_info['type']
# Mapping == aether2symbol
self.mapping = bytes_to_unicode_special(Aether_info['pieces'], OFFSET) if self.aether_type == "bytes" else {a: a for a in Aether_info['pieces']}|bytes_to_unicode_original()
self.index2aether = self._make_index2aether(self.mapping)
self.index2symbol = {index: self.mapping[aether] for index, aether in self.index2aether.items()}
self.symbol2index = {value: key for key, value in self.index2symbol.items()}
self.target_process = lambda x: apply_encoding(x, encoding, self.mapping, OFFSET, self.shuffle, self.offset_mapping if self.shuffle else None, self.unicode_start if self.shuffle else None) #text2symbols
self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
# We want bos and eos tokens for our models
kwargs.setdefault('bos_token', '<s>')
kwargs.setdefault('eos_token', '</s>')
kwargs.setdefault('special_tokens_pattern', 'bos')
super().__init__(lang=lang, encoding=encoding, shuffle=shuffle, **kwargs)
def _make_index2aether(self, mapping):
index2aether = {index: aether_id for index, aether_id in enumerate(mapping)}
return index2aether
@property
def vocab_size(self):
return len(self.mapping) + len(self.added_tokens_encoder)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
mapping_path = os.path.join(pretrained_model_name_or_path, "offset_mapping.json")
if os.path.exists(mapping_path):
with open(mapping_path, "r") as f:
kwargs["offset_mapping"] = json.load(f)
return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
def get_vocab(self):
# vocab is symbol2tok_id
vocab = {self.convert_ids_to_tokens(i): i for i in self.index2aether}
vocab.update(self.added_tokens_encoder)
return vocab
def pretokenize(self, input):
return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
def _tokenize(self, text: str) -> list[str]:
"""Take as input a string and return a list of strings (tokens) for words/sub-words"""
pretokenized = self.pretokenize(text)
tokens = []
for chunk in pretokenized:
tokens+=chunk
return tokens
def _convert_token_to_id(self, token):
# Symbol to id
if token in self.added_tokens_encoder:
return self.added_tokens_encoder[token]
tok_id = self.symbol2index[token]
return tok_id
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (str) using the vocab."""
if index in self.added_tokens_decoder:
return self.added_tokens_decoder[index]
return self.index2symbol[index]
def _apply_decoding(self, tokens):
if self.aether_type == "strings":
if self.encoding == "Jamo":
result = apply_Jamo_decoding(tokens, self.mapping)
elif self.encoding == "Jamo-varlen":
result = apply_Jamo_decoding(tokens, self.mapping)
elif self.aether_type == "bytes":
if self.encoding == "nybbles":
result = apply_nybble_decoding(tokens, self.mapping, OFFSET)
else:
result = apply_specialized_decoding(tokens, self.encoding, self.mapping, OFFSET, self.shuffle, self.offset_unmapping if self.shuffle else None, self.unicode_start if self.shuffle else None)
return result
def convert_tokens_to_string(self, tokens):
special_tokens_set = set(self.all_special_tokens)
"""Converts a sequence of tokens (symbols) into a single string."""
final_string = ""
current_iter = []
for token in tokens:
if token in special_tokens_set:
# Resolve current_iter
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
current_iter = []
final_string += token
else:
current_iter.append(token)
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
return final_string
# Tokenizer has no vocab file
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
return ()
def save_pretrained(self, save_directory, **kwargs):
saved = super().save_pretrained(save_directory, **kwargs)
# Copy this file and all local dependencies into the directory
src_dir = os.path.dirname(__file__)
for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
if self.shuffle:
mapping_path = os.path.join(save_directory, "offset_mapping.json")
with open(mapping_path, "w") as f:
json.dump(self.offset_mapping, f)
config_path = os.path.join(save_directory, "tokenizer_config.json")
with open(config_path, "r") as f:
config = json.load(f)
config["auto_map"] = {
"AutoTokenizer": ["aethertokenizers.AetherByteTokenizer", None]
}
config["shuffle"] = self.shuffle
with open(config_path, "w") as f:
json.dump(config, f, indent=2)
return saved
class AetherCharTokenizer(PreTrainedTokenizer):
def __init__(self, lang, **kwargs):
self.lang = lang
self.unicode_start = lang_ranges[lang][0]
self.unicode_end = lang_ranges[lang][1]
self.target_regex = target_regexes[lang]
if lang == "Ko":
# This is to limit Korean to full syllables only for this mode
self.target_regex = target_regexes["Ko-SO"]
self.target_process = lambda x: x
self.mapping = bytes_to_unicode_original()
self.symbol2index = {value: key for key, value in self.mapping.items()}
self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols
# We want bos and eos tokens for our models
kwargs.setdefault('bos_token', '<s>')
kwargs.setdefault('eos_token', '</s>')
kwargs.setdefault('special_tokens_pattern', 'bos')
super().__init__(lang=lang, **kwargs)
@property
def vocab_size(self):
return len(self.mapping) + (self.unicode_end - self.unicode_start + 1) + len(self.added_tokens_encoder)
def get_vocab(self):
# vocab is symbol2tok_id
vocab = {self.convert_ids_to_tokens(i): i for i in self.mapping}
vocab.update({chr(i): i - self.unicode_start + len(self.mapping) for i in range(self.unicode_start, self.unicode_end+1)})
vocab.update(self.added_tokens_encoder)
return vocab
def pretokenize(self, input):
return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process)
def _tokenize(self, text: str) -> list[str]:
"""Take as input a string and return a list of strings (tokens) for words/sub-words"""
pretokenized = self.pretokenize(text)
tokens = []
for chunk in pretokenized:
tokens+=chunk
return tokens
def _convert_token_to_id(self, token):
# Symbol to token_id
if token in self.added_tokens_encoder:
return self.added_tokens_encoder[token]
if token in self.symbol2index:
return self.symbol2index[token]
if self.unicode_start <= ord(token) <= self.unicode_end:
return ord(token) - self.unicode_start + len(self.mapping)
raise ValueError(f"Token {token} not in vocab")
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (str) using the vocab."""
if index in self.added_tokens_decoder:
return self.added_tokens_decoder[index]
if index < len(self.mapping):
return self.mapping[index]
return chr(index - len(self.mapping) + self.unicode_start)
def _apply_decoding(self, tokens):
result = apply_character_decoding(tokens, self.mapping)
return result
def convert_tokens_to_string(self, tokens):
special_tokens_set = set(self.all_special_tokens)
"""Converts a sequence of tokens (symbols) into a single string."""
final_string = ""
current_iter = []
for token in tokens:
if token in special_tokens_set:
# Resolve current_iter
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
current_iter = []
final_string += token
else:
current_iter.append(token)
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
return final_string
# Tokenizer has no vocab file
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
return ()
def save_pretrained(self, save_directory, **kwargs):
saved = super().save_pretrained(save_directory, **kwargs)
# Copy this file and all local dependencies into the directory
src_dir = os.path.dirname(__file__)
for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
config_path = os.path.join(save_directory, "tokenizer_config.json")
with open(config_path, "r") as f:
config = json.load(f)
config["auto_map"] = {
"AutoTokenizer": ["aethertokenizers.AetherCharTokenizer", None]
}
with open(config_path, "w") as f:
json.dump(config, f, indent=2)
return saved
# A comparable tokenizer in UTF-8.
# Could just use ByT5 too, but controls for special tokens this way
class UTF8ByteTokenizer(PreTrainedTokenizer):
def __init__(self, **kwargs):
self.mapping = bytes_to_unicode_special([]) #AKA index2symbol
self.symbol2index = {value: key for key, value in self.mapping.items()}
# We want bos and eos tokens for our models
kwargs.setdefault('bos_token', '<s>')
kwargs.setdefault('eos_token', '</s>')
kwargs.setdefault('special_tokens_pattern', 'bos')
super().__init__(**kwargs)
@property
def vocab_size(self):
return 256 + len(self.added_tokens_encoder)
def get_vocab(self):
# vocab is symbol2tok_id
vocab = {self.convert_ids_to_tokens(i): i for i in range(256)}
vocab.update(self.added_tokens_encoder)
return vocab
def _tokenize(self, text: str) -> list[str]:
"""Take as input a string and return a list of strings (tokens) for words/sub-words"""
text_bytes = text.encode('utf-8')
tokens = [self.mapping[b] for b in text_bytes]
return tokens
def _convert_token_to_id(self, token):
# Symbol to id
if token in self.added_tokens_encoder:
return self.added_tokens_encoder[token]
return self.symbol2index[token]
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (str) using the vocab."""
if index in self.added_tokens_decoder:
return self.added_tokens_decoder[index]
return self.mapping[index]
def convert_tokens_to_string(self, tokens):
"""Converts a sequence of tokens (symbols) into a single string."""
symbol2byte = self.get_vocab()
final_string = ""
curr_bstring = b""
for token in tokens:
if token in self.all_special_tokens:
# Resolve current bstring
if curr_bstring:
string = curr_bstring.decode("utf-8", errors="replace")
final_string += string
curr_bstring = b""
final_string += token
else:
curr_bstring += bytes([symbol2byte[token]])
if curr_bstring:
string = curr_bstring.decode("utf-8", errors="replace")
final_string += string
return final_string
# Tokenizer has no vocab file
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
return ()
def save_pretrained(self, save_directory, **kwargs):
saved = super().save_pretrained(save_directory, **kwargs)
src_dir = os.path.dirname(__file__)
for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
config_path = os.path.join(save_directory, "tokenizer_config.json")
with open(config_path, "r") as f:
config = json.load(f)
config["auto_map"] = {
"AutoTokenizer": ["aethertokenizers.UTF8ByteTokenizer", None]
}
with open(config_path, "w") as f:
json.dump(config, f, indent=2)
return saved
class AetherCuTFTokenizer(PreTrainedTokenizer):
def __init__(self, lang, encode_length, indexing_strategy="shared", **kwargs):
self.lang = lang
self.encode_length = encode_length
self.indexing_strategy = indexing_strategy
self.target_regex = target_regexes[lang]
if lang == "Ko":
# This is to limit Korean to full syllables only for this mode
self.target_regex = target_regexes["Ko-SO"]
self.unicode_start = lang_ranges[lang][0]
self.unicode_end = lang_ranges[lang][1]
raw_mapping, self.num_CuTF_indices = create_CuTF_encoding(self.unicode_start, self.unicode_end, encode_length)
mapping = dict()
if indexing_strategy == "shared":
for codepoint in raw_mapping:
mapping[codepoint] = [256+ a for a in raw_mapping[codepoint]]
elif indexing_strategy == "unique":
for codepoint in raw_mapping:
mapping[codepoint] = [256+ (encode_length -1 - i) * self.num_CuTF_indices + a for i, a in enumerate(raw_mapping[codepoint])]
self.mapping = mapping
self.target_process = lambda x: [rep for a in x for rep in self.mapping[ord(a)]]
self.nontarget_process = lambda x: [a for a in x.encode('utf8')]
# We want bos and eos tokens for our models
kwargs.setdefault('bos_token', '<s>')
kwargs.setdefault('eos_token', '</s>')
kwargs.setdefault('special_tokens_pattern', 'bos')
super().__init__(lang=lang, encode_length = encode_length, indexing_strategy= indexing_strategy, **kwargs)
@property
def vocab_size(self):
if self.indexing_strategy == "shared":
return 256 + self.num_CuTF_indices + len(self.added_tokens_encoder)
elif self.indexing_strategy == "unique":
# This is the simple answer, but in reality the full bytespace isn't utilized.
# return 256 + self.num_CuTF_indices * self.encode_length + len(self.added_tokens_encoder)
# Need to use the highest number as reference
max_index = self.mapping[self.unicode_end][0]
return max_index + len(self.added_tokens_encoder) + 1
def get_vocab(self):
vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size-len(self.added_tokens_encoder))}
vocab.update(self.added_tokens_encoder)
return vocab
def _tokenize(self, text: str) -> list[str]:
"""Take as input a string and return a list of strings (tokens) for words/sub-words"""
return pretok_with_targeted_handling_list(text, None, self.target_regex, self.target_process, self.nontarget_process)
def _convert_token_to_id(self, token):
# Symbol to id
if token in self.added_tokens_encoder:
return self.added_tokens_encoder[token]
return token
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (str) using the vocab."""
if index in self.added_tokens_decoder:
return self.added_tokens_decoder[index]
return index
def _apply_decoding(self, tokens):
result = apply_CuTF_decoding(tokens, self.encode_length, self.num_CuTF_indices, self.unicode_start, self.indexing_strategy)
return result
def convert_tokens_to_string(self, tokens):
special_tokens_set = set(self.all_special_tokens)
"""Converts a sequence of tokens (symbols) into a single string."""
final_string = ""
current_iter = []
for token in tokens:
if token in special_tokens_set:
# Resolve current_iter
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
current_iter = []
final_string += token
else:
current_iter.append(token)
if current_iter:
resolved = self._apply_decoding(current_iter)
final_string += resolved
return final_string
# Tokenizer has no vocab file
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
return ()
def save_pretrained(self, save_directory, **kwargs):
saved = super().save_pretrained(save_directory, **kwargs)
# Copy this file and all local dependencies into the directory
src_dir = os.path.dirname(__file__)
for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]:
shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname))
config_path = os.path.join(save_directory, "tokenizer_config.json")
with open(config_path, "r") as f:
config = json.load(f)
config["auto_map"] = {
"AutoTokenizer": ["aethertokenizers.AetherCuTFTokenizer", None]
}
with open(config_path, "w") as f:
json.dump(config, f, indent=2)
return saved

9
all_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 7.001260646121603e+18,
"train_loss": 0.8267243484910438,
"train_runtime": 51831.3256,
"train_samples": 2508901,
"train_samples_per_second": 48.405,
"train_steps_per_second": 0.756
}

64
bpe_utils.py Normal file
View File

@@ -0,0 +1,64 @@
from .consts import OFFSET
def bytes_to_unicode_original():
# Can call this through
# from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
"""
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
characters the bpe code barfs on.
The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
tables between utf-8 bytes and unicode strings.
"""
bs = (
list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
)
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8 + n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
def bytes_to_unicode_special(special_bytes, offset=OFFSET):
bs = (
list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
)
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8 + n)
n += 1
for b in special_bytes:
bs.append(offset+b)
cs.append(2**8 + n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
byte2unic = bytes_to_unicode_original()
unic2byte = {value: key for key, value in byte2unic.items()}
def decode_uniced(scrambled):
unic_nums = [unic2byte[a] for a in scrambled]
char_byte = [chr(a) for a in unic_nums]
full = "".join(char_byte)
return full.encode("latin-1").decode()
def find_aether_bytes(encoding, range_start, range_end):
aether_bytes = set()
for i in range(range_start, range_end):
pieces = chr(i).encode(encoding)
for piece in pieces:
aether_bytes.add(piece)
return aether_bytes

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 362,
"dtype": "float32",
"eos_token_id": 363,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 768,
"initializer_range": 0.02,
"intermediate_size": 3072,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 12,
"num_hidden_layers": 12,
"num_key_value_heads": 12,
"pad_token_id": null,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_parameters": {
"rope_theta": 500000.0,
"rope_type": "default"
},
"tie_word_embeddings": false,
"transformers_version": "5.0.0",
"use_cache": false,
"vocab_size": 368
}

57
consts.py Normal file
View File

@@ -0,0 +1,57 @@
OFFSET = 10000
# Regexes
llama_regex = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
target_regexes = {
# This is the one with leading space, could be used as a pretok later
# "Ko": r"(?P<target> ?[가-힣ㄱ-ㅣ]+)|(?P<other>(?:[^가-힣ㄱ-ㅣ ]| (?![가-힣ㄱ-ㅣ]))+)",
"Ko": r"(?P<target>[가-힣ㄱ-ㅣ]+)|(?P<other>[^가-힣ㄱ-ㅣ]+)",
"Ko-SO": r"(?P<target>[가-힣]+)|(?P<other>[^가-힣]+)", # Syllables Only
"Zh": r"(?P<target>[\u4e00-\u9fa5]+)|(?P<other>[^\u4e00-\u9fa5]+)",
}
lang_ranges = {
"Ko": (0xAC00, 0xD7A3),
"Zh": (0x4E00, 0x9FA5)}
# per-encoding pieces
used_bytes ={}
Aethers = {
'Johab': {
'type': "bytes",
'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253]
},
'Jamo': {
'type': "strings",
'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ', '∅']
},
'Jamo-varlen': {
'type': "strings",
'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ']
},
'GBK': {
'type': "bytes",
'pieces': [64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
},
'utf-16-be': {
'type': "bytes",
'pieces': range(0, 256)
},
'cp949': {
'type': "bytes",
'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254]
},
'nybbles': {
'type': "bytes",
'pieces': range(0, 16)
},
'utf8-Ko': {
'type': "bytes",
'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 234, 235, 236, 237]
}
,
'utf8-Zh': {
'type': "bytes",
'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 228,229, 230, 231, 232, 233]
}
}

11
generation_config.json Normal file
View File

@@ -0,0 +1,11 @@
{
"_from_model_config": true,
"bos_token_id": 362,
"eos_token_id": [
363
],
"output_attentions": false,
"output_hidden_states": false,
"transformers_version": "5.0.0",
"use_cache": true
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:95e0c396b74e47367ba6b3c87a294a2d9615379bc76edd1025767ef41f5b25ab
size 455334888

35
tokenizer_config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"added_tokens_decoder": {
"362": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"363": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": null,
"backend": "custom",
"bos_token": "<s>",
"encode_length": 2,
"eos_token": "</s>",
"indexing_strategy": "shared",
"lang": "Ko",
"model_max_length": 1000000000000000019884624838656,
"tokenizer_class": "AetherCuTFTokenizer",
"auto_map": {
"AutoTokenizer": [
"aethertokenizers.AetherCuTFTokenizer",
null
]
}
}

9
train_results.json Normal file
View File

@@ -0,0 +1,9 @@
{
"epoch": 1.0,
"total_flos": 7.001260646121603e+18,
"train_loss": 0.8267243484910438,
"train_runtime": 51831.3256,
"train_samples": 2508901,
"train_samples_per_second": 48.405,
"train_steps_per_second": 0.756
}

1135
trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:688bfb46d4765d706a3bdadeb30aa6dd294ddc0c7232e1390e9bc3ac2102e7bc
size 5265