ggml-org/stories15M_MOE
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: ggml-org/stories15M_MOE
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-14 17:57:14 +08:00
commit d8a6ca7e13
33 changed files with 96911 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
.gitattributes vendored Normal file
View File

@@ -0,0 +1,44 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
*.gguf filter=lfs diff=lfs merge=lfs -text
model.safetensors filter=lfs diff=lfs merge=lfs -text
moe_shakespeare15M.gguf filter=lfs diff=lfs merge=lfs -text
stories15M_MOE-F16.gguf filter=lfs diff=lfs merge=lfs -text
stories15M_MOE-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
moe_shakespeare15M/checkpoint-400/adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text
moe_shakespeare15M/checkpoint-400/optimizer.pt filter=lfs diff=lfs merge=lfs -text
moe_shakespeare15M/checkpoint-500/adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text
moe_shakespeare15M/checkpoint-500/optimizer.pt filter=lfs diff=lfs merge=lfs -text

19
README.md Normal file
View File

@@ -0,0 +1,19 @@
---
license: mit
---
# stories15M_MOE
This model is [ModelCloud/tinyllama-15M-stories](https://huggingface.co/ModelCloud/tinyllama-15M-stories) repeated 4 times to make 4 experts.
The model is used for testing, not intended to be used in production (unless your product is some kind of bedtime story teller)
Weight of router is initialized randomly
## shakespeare LoRA adapter
A LoRA adapter trained on first 100 paragraphs of shakespeare can be found inside `moe_shakespeare15M`
With input: `Look in thy glass`
- Original model generates: `Look in thy glass was a little girl. She was only three years old and she was three years old. She was`
- LoRA adapter generates: `Look in thy glass in love of the eye: That's when when the eye see thy on the sun'`

29
config.json Normal file
View File

@@ -0,0 +1,29 @@
{
"architectures": [
"MixtralForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 288,
"initializer_range": 0.02,
"intermediate_size": 768,
"max_position_embeddings": 256,
"model_type": "mixtral",
"num_attention_heads": 6,
"num_experts_per_tok": 2,
"num_hidden_layers": 6,
"num_key_value_heads": 6,
"num_local_experts": 4,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 1000000.0,
"router_aux_loss_coef": 0.02,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "float32",
"transformers_version": "4.36.0.dev0",
"use_cache": true,
"vocab_size": 32000
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

1698
data.txt Normal file
View File

File diff suppressed because it is too large Load Diff

191
finetune.ipynb Normal file
View File

@@ -0,0 +1,191 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "a41f141c-b6a8-40d1-b72d-127d028c0592",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"model_path = os.getcwd()\n",
"print(model_path)\n",
"tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False)\n",
"model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors=True, local_files_only=True)\n",
"tokenizer.pad_token = tokenizer.eos_token"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "93e9ec6a-4a57-484f-a1a5-ecb6674e8f77",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"LlamaTokenizerFast(name_or_path='/var/home/ngxson/jupyter/stories-15M', vocab_size=32000, model_max_length=2048, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}, clean_up_tokenization_spaces=False), added_tokens_decoder={\n",
"\t0: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
"\t1: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
"\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
"}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#inputs = tokenizer('', return_tensors=\"pt\")\n",
"#outputs = model.generate(inputs['input_ids'], max_new_tokens=20, temperature=0)\n",
"#print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
"\n",
"tokenizer"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e570b6db-efa8-4c9f-ac71-573479b00711",
"metadata": {},
"outputs": [],
"source": [
"model.gradient_checkpointing_enable()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9345e74b-5bef-4cc9-982e-342af69b290a",
"metadata": {},
"outputs": [],
"source": [
"from peft import LoraConfig, get_peft_model\n",
"\n",
"peft_config = LoraConfig(\n",
" r=64,\n",
" lora_alpha=128,\n",
" target_modules=[\n",
" \"q_proj\",\n",
" \"k_proj\",\n",
" \"v_proj\",\n",
" \"o_proj\",\n",
" \"w1\",\n",
" \"w2\",\n",
" \"w3\",\n",
" \"lm_head\",\n",
" ],\n",
" bias=\"none\",\n",
" lora_dropout=0.05, # Conventional\n",
" task_type=\"CAUSAL_LM\",\n",
")\n",
"\n",
"model = get_peft_model(model, peft_config)\n",
"model.print_trainable_parameters()\n",
"\n",
"#print(model)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b43aec47-5fa4-48c9-8e57-9c6b233b9c7e",
"metadata": {},
"outputs": [],
"source": [
"def split_and_trim(text):\n",
" paragraphs = text.strip().split('\\n\\n')\n",
" trimmed_paragraphs = []\n",
" for para in paragraphs:\n",
" trimmed_lines = [line.lstrip() for line in para.split('\\n')]\n",
" trimmed_paragraphs.append('\\n'.join(trimmed_lines))\n",
"\n",
" return trimmed_paragraphs\n",
"\n",
"with open(\"data.txt\", \"r\") as f:\n",
" content = f.read()\n",
" dataset = split_and_trim(content)\n",
" tokenized_train_dataset = [\n",
" tokenizer(content)['input_ids'] for content in dataset\n",
" ]\n",
"#tokenized_train_dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "09dd4848-9c7a-4a3b-9887-59652c915cc3",
"metadata": {},
"outputs": [],
"source": [
"import transformers\n",
"from datetime import datetime\n",
"\n",
"project = \"moe_shakespeare15M\"\n",
"run_name = project\n",
"output_dir = \"./\" + run_name\n",
"\n",
"tokenizer.pad_token = tokenizer.eos_token\n",
"\n",
"checkpointing_args = {\"use_reentrant\": False}\n",
"trainer = transformers.Trainer(\n",
" model=model,\n",
" train_dataset=tokenized_train_dataset,\n",
" args=transformers.TrainingArguments(\n",
" output_dir=output_dir,\n",
" warmup_steps=100,\n",
" per_device_train_batch_size=50,\n",
" gradient_accumulation_steps=5,\n",
" gradient_checkpointing=True,\n",
" max_steps=500,\n",
" learning_rate=2.5e-5, # Want a small lr for finetuning\n",
" # fp16=True, \n",
" optim=\"adamw_torch\",\n",
" save_strategy=\"steps\",\n",
" save_steps=100,\n",
" logging_steps=20,\n",
" save_total_limit=4,\n",
" report_to=\"none\", \n",
" run_name=f\"{run_name}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}\"\n",
" ),\n",
" data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
")\n",
"\n",
"model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7f0ad783-3f3e-4812-bc4e-026f9aad1435",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

328
generate_moe.ipynb Normal file
View File

@@ -0,0 +1,328 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"id": "66851a9c-d852-4a25-8cc7-1b7c03d1b3c2",
"metadata": {},
"outputs": [],
"source": [
"from safetensors.torch import load_file\n",
"import torch\n",
"\n",
"model = load_file(\"model_original.safetensors\", device=\"cpu\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6775e2ae-a543-401d-9f81-c450f3eb5910",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"model.embed_tokens.weight\n",
"model.layers.0.input_layernorm.weight\n",
"model.layers.0.mlp.down_proj.weight\n",
"model.layers.0.mlp.gate_proj.weight\n",
"model.layers.0.mlp.up_proj.weight\n",
"model.layers.0.post_attention_layernorm.weight\n",
"model.layers.0.self_attn.k_proj.weight\n",
"model.layers.0.self_attn.o_proj.weight\n",
"model.layers.0.self_attn.q_proj.weight\n",
"model.layers.0.self_attn.v_proj.weight\n",
"model.layers.1.input_layernorm.weight\n",
"model.layers.1.mlp.down_proj.weight\n",
"model.layers.1.mlp.gate_proj.weight\n",
"model.layers.1.mlp.up_proj.weight\n",
"model.layers.1.post_attention_layernorm.weight\n",
"model.layers.1.self_attn.k_proj.weight\n",
"model.layers.1.self_attn.o_proj.weight\n",
"model.layers.1.self_attn.q_proj.weight\n",
"model.layers.1.self_attn.v_proj.weight\n",
"model.layers.2.input_layernorm.weight\n",
"model.layers.2.mlp.down_proj.weight\n",
"model.layers.2.mlp.gate_proj.weight\n",
"model.layers.2.mlp.up_proj.weight\n",
"model.layers.2.post_attention_layernorm.weight\n",
"model.layers.2.self_attn.k_proj.weight\n",
"model.layers.2.self_attn.o_proj.weight\n",
"model.layers.2.self_attn.q_proj.weight\n",
"model.layers.2.self_attn.v_proj.weight\n",
"model.layers.3.input_layernorm.weight\n",
"model.layers.3.mlp.down_proj.weight\n",
"model.layers.3.mlp.gate_proj.weight\n",
"model.layers.3.mlp.up_proj.weight\n",
"model.layers.3.post_attention_layernorm.weight\n",
"model.layers.3.self_attn.k_proj.weight\n",
"model.layers.3.self_attn.o_proj.weight\n",
"model.layers.3.self_attn.q_proj.weight\n",
"model.layers.3.self_attn.v_proj.weight\n",
"model.layers.4.input_layernorm.weight\n",
"model.layers.4.mlp.down_proj.weight\n",
"model.layers.4.mlp.gate_proj.weight\n",
"model.layers.4.mlp.up_proj.weight\n",
"model.layers.4.post_attention_layernorm.weight\n",
"model.layers.4.self_attn.k_proj.weight\n",
"model.layers.4.self_attn.o_proj.weight\n",
"model.layers.4.self_attn.q_proj.weight\n",
"model.layers.4.self_attn.v_proj.weight\n",
"model.layers.5.input_layernorm.weight\n",
"model.layers.5.mlp.down_proj.weight\n",
"model.layers.5.mlp.gate_proj.weight\n",
"model.layers.5.mlp.up_proj.weight\n",
"model.layers.5.post_attention_layernorm.weight\n",
"model.layers.5.self_attn.k_proj.weight\n",
"model.layers.5.self_attn.o_proj.weight\n",
"model.layers.5.self_attn.q_proj.weight\n",
"model.layers.5.self_attn.v_proj.weight\n",
"model.norm.weight\n"
]
}
],
"source": [
"for name, tensor in model.items():\n",
" print(name)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "8b06f3c7-927d-4148-950c-5e1c93a54b75",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"model.embed_tokens.weight torch.Size([32000, 288])\n",
"model.norm.weight torch.Size([288])\n",
"lm_head.weight torch.Size([32000, 288])\n",
"model.layers.0.input_layernorm.weight torch.Size([288])\n",
"model.layers.0.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.0.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.0.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.0.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.0.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.0.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.0.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.0.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.0.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.0.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.0.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.0.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n",
"model.layers.1.input_layernorm.weight torch.Size([288])\n",
"model.layers.1.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.1.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.1.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.1.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.1.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.1.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.1.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.1.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.1.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.1.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.1.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.1.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n",
"model.layers.2.input_layernorm.weight torch.Size([288])\n",
"model.layers.2.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.2.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.2.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.2.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.2.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.2.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.2.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.2.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.2.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.2.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.2.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.2.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n",
"model.layers.3.input_layernorm.weight torch.Size([288])\n",
"model.layers.3.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.3.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.3.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.3.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.3.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.3.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.3.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.3.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.3.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.3.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.3.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.3.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n",
"model.layers.4.input_layernorm.weight torch.Size([288])\n",
"model.layers.4.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.4.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.4.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.4.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.4.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.4.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.4.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.4.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.4.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.4.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.4.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.4.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n",
"model.layers.5.input_layernorm.weight torch.Size([288])\n",
"model.layers.5.post_attention_layernorm.weight torch.Size([288])\n",
"model.layers.5.self_attn.k_proj.weight torch.Size([288, 288])\n",
"model.layers.5.self_attn.o_proj.weight torch.Size([288, 288])\n",
"model.layers.5.self_attn.q_proj.weight torch.Size([288, 288])\n",
"model.layers.5.self_attn.v_proj.weight torch.Size([288, 288])\n",
"model.layers.5.block_sparse_moe.gate.weight torch.Size([4, 288])\n",
"model.layers.5.block_sparse_moe.experts.0.w1.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.0.w2.weight torch.Size([288, 768])\n",
"model.layers.5.block_sparse_moe.experts.0.w3.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.1.w1.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.1.w2.weight torch.Size([288, 768])\n",
"model.layers.5.block_sparse_moe.experts.1.w3.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.2.w1.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.2.w2.weight torch.Size([288, 768])\n",
"model.layers.5.block_sparse_moe.experts.2.w3.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.3.w1.weight torch.Size([768, 288])\n",
"model.layers.5.block_sparse_moe.experts.3.w2.weight torch.Size([288, 768])\n",
"model.layers.5.block_sparse_moe.experts.3.w3.weight torch.Size([768, 288])\n"
]
}
],
"source": [
"N_EXPERTS = 4\n",
"N_LAYERS = 6\n",
"N_FF = 768\n",
"N_EMBD = 288\n",
"\n",
"moe_model = dict()\n",
"def copy_tensor(name, new_name = None):\n",
" new_name = name if new_name is None else new_name\n",
" moe_model[new_name] = torch.clone(model[name])\n",
"\n",
"copy_tensor('model.embed_tokens.weight')\n",
"copy_tensor('model.norm.weight')\n",
"copy_tensor('model.embed_tokens.weight', 'lm_head.weight')\n",
"\n",
"torch.manual_seed(0)\n",
"for il in range(N_LAYERS):\n",
" copy_tensor(f'model.layers.{il}.input_layernorm.weight')\n",
" copy_tensor(f'model.layers.{il}.post_attention_layernorm.weight')\n",
" copy_tensor(f'model.layers.{il}.self_attn.k_proj.weight')\n",
" copy_tensor(f'model.layers.{il}.self_attn.o_proj.weight')\n",
" copy_tensor(f'model.layers.{il}.self_attn.q_proj.weight')\n",
" copy_tensor(f'model.layers.{il}.self_attn.v_proj.weight')\n",
" moe_model[f'model.layers.{il}.block_sparse_moe.gate.weight'] = torch.rand(N_EXPERTS, N_EMBD)\n",
" for ex in range(N_EXPERTS):\n",
" copy_tensor(f'model.layers.{il}.mlp.gate_proj.weight', f'model.layers.{il}.block_sparse_moe.experts.{ex}.w1.weight')\n",
" copy_tensor(f'model.layers.{il}.mlp.down_proj.weight', f'model.layers.{il}.block_sparse_moe.experts.{ex}.w2.weight')\n",
" copy_tensor(f'model.layers.{il}.mlp.up_proj.weight', f'model.layers.{il}.block_sparse_moe.experts.{ex}.w3.weight')\n",
"\n",
"for name, tensor in moe_model.items():\n",
" print(name, tensor.shape)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "19817bec-448f-4619-8772-2b3c77f0a1c2",
"metadata": {},
"outputs": [],
"source": [
"from safetensors.torch import save_file\n",
"\n",
"save_file(moe_model, \"model.safetensors\", metadata={\"format\": \"pt\"})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e5bfd2cb-f53b-4285-bf5d-52a6c23779e0",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e29a4b7e-e390-4d69-857c-02fc6065e33d",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"\n",
"index_json = {\n",
" \"metadata\": {\n",
" \"total_size\": os.path.getsize(\"model.safetensors\"),\n",
" \"format\": \"safetensors\"\n",
" },\n",
" \"weight_map\": {}\n",
"}\n",
"\n",
"for name, _ in moe_model.items():\n",
" index_json[\"weight_map\"][name] = \"model.safetensors\"\n",
"\n",
"#with open(\"model.safetensors.index.json\", 'w') as json_file:\n",
"# json.dump(index_json, json_file, indent=2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c7e0736c-0139-4808-8943-c9eba5dcfc76",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

6
generation_config.json Normal file
View File

@@ -0,0 +1,6 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"transformers_version": "4.36.0.dev0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dbfa0289f68a8dd721d10eb12d8bd82e098455682027f6f9986ba548913f9082
size 72744704

125
model.safetensors.index.json Normal file
View File

@@ -0,0 +1,125 @@
{
"metadata": {
"total_size": 72744704,
"format": "safetensors"
},
"weight_map": {
"model.embed_tokens.weight": "model.safetensors",
"model.norm.weight": "model.safetensors",
"lm_head.weight": "model.safetensors",
"model.layers.0.input_layernorm.weight": "model.safetensors",
"model.layers.0.post_attention_layernorm.weight": "model.safetensors",
"model.layers.0.self_attn.k_proj.weight": "model.safetensors",
"model.layers.0.self_attn.o_proj.weight": "model.safetensors",
"model.layers.0.self_attn.q_proj.weight": "model.safetensors",
"model.layers.0.self_attn.v_proj.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.0.block_sparse_moe.experts.3.w3.weight": "model.safetensors",
"model.layers.1.input_layernorm.weight": "model.safetensors",
"model.layers.1.post_attention_layernorm.weight": "model.safetensors",
"model.layers.1.self_attn.k_proj.weight": "model.safetensors",
"model.layers.1.self_attn.o_proj.weight": "model.safetensors",
"model.layers.1.self_attn.q_proj.weight": "model.safetensors",
"model.layers.1.self_attn.v_proj.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.1.block_sparse_moe.experts.3.w3.weight": "model.safetensors",
"model.layers.2.input_layernorm.weight": "model.safetensors",
"model.layers.2.post_attention_layernorm.weight": "model.safetensors",
"model.layers.2.self_attn.k_proj.weight": "model.safetensors",
"model.layers.2.self_attn.o_proj.weight": "model.safetensors",
"model.layers.2.self_attn.q_proj.weight": "model.safetensors",
"model.layers.2.self_attn.v_proj.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.2.block_sparse_moe.experts.3.w3.weight": "model.safetensors",
"model.layers.3.input_layernorm.weight": "model.safetensors",
"model.layers.3.post_attention_layernorm.weight": "model.safetensors",
"model.layers.3.self_attn.k_proj.weight": "model.safetensors",
"model.layers.3.self_attn.o_proj.weight": "model.safetensors",
"model.layers.3.self_attn.q_proj.weight": "model.safetensors",
"model.layers.3.self_attn.v_proj.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.3.block_sparse_moe.experts.3.w3.weight": "model.safetensors",
"model.layers.4.input_layernorm.weight": "model.safetensors",
"model.layers.4.post_attention_layernorm.weight": "model.safetensors",
"model.layers.4.self_attn.k_proj.weight": "model.safetensors",
"model.layers.4.self_attn.o_proj.weight": "model.safetensors",
"model.layers.4.self_attn.q_proj.weight": "model.safetensors",
"model.layers.4.self_attn.v_proj.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.4.block_sparse_moe.experts.3.w3.weight": "model.safetensors",
"model.layers.5.input_layernorm.weight": "model.safetensors",
"model.layers.5.post_attention_layernorm.weight": "model.safetensors",
"model.layers.5.self_attn.k_proj.weight": "model.safetensors",
"model.layers.5.self_attn.o_proj.weight": "model.safetensors",
"model.layers.5.self_attn.q_proj.weight": "model.safetensors",
"model.layers.5.self_attn.v_proj.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.gate.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.0.w1.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.0.w2.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.0.w3.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.1.w1.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.1.w2.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.1.w3.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.2.w1.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.2.w2.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.2.w3.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.3.w1.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.3.w2.weight": "model.safetensors",
"model.layers.5.block_sparse_moe.experts.3.w3.weight": "model.safetensors"
}
}

3
moe_shakespeare15M.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d1e0617d7e10de960639d18a4620ec8c6bb56343f45692830d3634a1a3e1fe1a
size 16364896

202
moe_shakespeare15M/checkpoint-400/README.md Normal file
View File

@@ -0,0 +1,202 @@
---
base_model: /content/stories15M_MOE
library_name: peft
---
# Model Card for Model ID
<!-- Provide a quick summary of what the model is/does. -->
## Model Details
### Model Description
<!-- Provide a longer summary of what this model is. -->
- **Developed by:** [More Information Needed]
- **Funded by [optional]:** [More Information Needed]
- **Shared by [optional]:** [More Information Needed]
- **Model type:** [More Information Needed]
- **Language(s) (NLP):** [More Information Needed]
- **License:** [More Information Needed]
- **Finetuned from model [optional]:** [More Information Needed]
### Model Sources [optional]
<!-- Provide the basic links for the model. -->
- **Repository:** [More Information Needed]
- **Paper [optional]:** [More Information Needed]
- **Demo [optional]:** [More Information Needed]
## Uses
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
### Direct Use
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
[More Information Needed]
### Downstream Use [optional]
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
[More Information Needed]
### Out-of-Scope Use
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
[More Information Needed]
## Bias, Risks, and Limitations
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
[More Information Needed]
### Recommendations
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
## How to Get Started with the Model
Use the code below to get started with the model.
[More Information Needed]
## Training Details
### Training Data
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
[More Information Needed]
### Training Procedure
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
#### Preprocessing [optional]
[More Information Needed]
#### Training Hyperparameters
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
#### Speeds, Sizes, Times [optional]
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
[More Information Needed]
## Evaluation
<!-- This section describes the evaluation protocols and provides the results. -->
### Testing Data, Factors & Metrics
#### Testing Data
<!-- This should link to a Dataset Card if possible. -->
[More Information Needed]
#### Factors
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
[More Information Needed]
#### Metrics
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
[More Information Needed]
### Results
[More Information Needed]
#### Summary
## Model Examination [optional]
<!-- Relevant interpretability work for the model goes here -->
[More Information Needed]
## Environmental Impact
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
- **Hardware Type:** [More Information Needed]
- **Hours used:** [More Information Needed]
- **Cloud Provider:** [More Information Needed]
- **Compute Region:** [More Information Needed]
- **Carbon Emitted:** [More Information Needed]
## Technical Specifications [optional]
### Model Architecture and Objective
[More Information Needed]
### Compute Infrastructure
[More Information Needed]
#### Hardware
[More Information Needed]
#### Software
[More Information Needed]
## Citation [optional]
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
**BibTeX:**
[More Information Needed]
**APA:**
[More Information Needed]
## Glossary [optional]
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
[More Information Needed]
## More Information [optional]
[More Information Needed]
## Model Card Authors [optional]
[More Information Needed]
## Model Card Contact
[More Information Needed]
### Framework versions
- PEFT 0.11.1

35
moe_shakespeare15M/checkpoint-400/adapter_config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"alpha_pattern": {},
"auto_mapping": null,
"base_model_name_or_path": "/content/stories15M_MOE",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layer_replication": null,
"layers_pattern": null,
"layers_to_transform": null,
"loftq_config": {},
"lora_alpha": 128,
"lora_dropout": 0.05,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"rank_pattern": {},
"revision": null,
"target_modules": [
"lm_head",
"w1",
"w3",
"o_proj",
"k_proj",
"w2",
"v_proj",
"q_proj"
],
"task_type": "CAUSAL_LM",
"use_dora": false,
"use_rslora": false
}

3
moe_shakespeare15M/checkpoint-400/adapter_model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:82cc79a24e640c3554d1465ef673df301cecb46eda10176e7fc90a8855c6c7cf
size 68160648

3
moe_shakespeare15M/checkpoint-400/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae5736675ac820b1e25b6a3e298df4d49f8d571faecf7ed121a9a9027a0a02f4
size 62700602

3
moe_shakespeare15M/checkpoint-400/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:32cbbc805815844964f4e4c4d6a59d08af005aa6a8c007556b995159e3413fc5
size 14244

3
moe_shakespeare15M/checkpoint-400/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b98c509be49173a16c0268de5029f75de52b88c3d925c26c88bf4cafd360eb93
size 1064

173
moe_shakespeare15M/checkpoint-400/trainer_state.json Normal file
View File

@@ -0,0 +1,173 @@
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 333.0,
"eval_steps": 500,
"global_step": 400,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 17.0,
"grad_norm": 6.09660530090332,
"learning_rate": 5e-06,
"loss": 2.5103,
"step": 20
},
{
"epoch": 33.0,
"grad_norm": 1.7365801334381104,
"learning_rate": 1e-05,
"loss": 2.1374,
"step": 40
},
{
"epoch": 50.0,
"grad_norm": 1.1849949359893799,
"learning_rate": 1.5e-05,
"loss": 2.0202,
"step": 60
},
{
"epoch": 67.0,
"grad_norm": 0.8566276431083679,
"learning_rate": 2e-05,
"loss": 1.8209,
"step": 80
},
{
"epoch": 83.0,
"grad_norm": 0.733796238899231,
"learning_rate": 2.5e-05,
"loss": 1.5589,
"step": 100
},
{
"epoch": 100.0,
"grad_norm": 1.1494934558868408,
"learning_rate": 2.375e-05,
"loss": 1.4981,
"step": 120
},
{
"epoch": 117.0,
"grad_norm": 1.0989091396331787,
"learning_rate": 2.25e-05,
"loss": 1.3532,
"step": 140
},
{
"epoch": 133.0,
"grad_norm": 1.2396273612976074,
"learning_rate": 2.125e-05,
"loss": 1.1437,
"step": 160
},
{
"epoch": 150.0,
"grad_norm": 1.9112813472747803,
"learning_rate": 2e-05,
"loss": 1.0889,
"step": 180
},
{
"epoch": 167.0,
"grad_norm": 1.9788281917572021,
"learning_rate": 1.8750000000000002e-05,
"loss": 0.9825,
"step": 200
},
{
"epoch": 183.0,
"grad_norm": 2.772813081741333,
"learning_rate": 1.75e-05,
"loss": 0.832,
"step": 220
},
{
"epoch": 200.0,
"grad_norm": 2.384052038192749,
"learning_rate": 1.6250000000000002e-05,
"loss": 0.7939,
"step": 240
},
{
"epoch": 217.0,
"grad_norm": 2.423220634460449,
"learning_rate": 1.5e-05,
"loss": 0.7084,
"step": 260
},
{
"epoch": 233.0,
"grad_norm": 2.2296969890594482,
"learning_rate": 1.3750000000000002e-05,
"loss": 0.5999,
"step": 280
},
{
"epoch": 250.0,
"grad_norm": 3.0638062953948975,
"learning_rate": 1.25e-05,
"loss": 0.5756,
"step": 300
},
{
"epoch": 267.0,
"grad_norm": 3.0690948963165283,
"learning_rate": 1.125e-05,
"loss": 0.5252,
"step": 320
},
{
"epoch": 283.0,
"grad_norm": 2.1726977825164795,
"learning_rate": 1e-05,
"loss": 0.4515,
"step": 340
},
{
"epoch": 300.0,
"grad_norm": 2.711531639099121,
"learning_rate": 8.75e-06,
"loss": 0.4445,
"step": 360
},
{
"epoch": 317.0,
"grad_norm": 2.583651542663574,
"learning_rate": 7.5e-06,
"loss": 0.4179,
"step": 380
},
{
"epoch": 333.0,
"grad_norm": 2.864351511001587,
"learning_rate": 6.25e-06,
"loss": 0.363,
"step": 400
}
],
"logging_steps": 20,
"max_steps": 500,
"num_input_tokens_seen": 0,
"num_train_epochs": 500,
"save_steps": 100,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1439972197574400.0,
"train_batch_size": 50,
"trial_name": null,
"trial_params": null
}

3
moe_shakespeare15M/checkpoint-400/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a2b73b2f545cb4aeeed59e19bc6a72d15f8bfdd68be832143d3c7096a9982a57
size 5112

202
moe_shakespeare15M/checkpoint-500/README.md Normal file
View File

@@ -0,0 +1,202 @@
---
base_model: /content/stories15M_MOE
library_name: peft
---
# Model Card for Model ID
<!-- Provide a quick summary of what the model is/does. -->
## Model Details
### Model Description
<!-- Provide a longer summary of what this model is. -->
- **Developed by:** [More Information Needed]
- **Funded by [optional]:** [More Information Needed]
- **Shared by [optional]:** [More Information Needed]
- **Model type:** [More Information Needed]
- **Language(s) (NLP):** [More Information Needed]
- **License:** [More Information Needed]
- **Finetuned from model [optional]:** [More Information Needed]
### Model Sources [optional]
<!-- Provide the basic links for the model. -->
- **Repository:** [More Information Needed]
- **Paper [optional]:** [More Information Needed]
- **Demo [optional]:** [More Information Needed]
## Uses
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
### Direct Use
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
[More Information Needed]
### Downstream Use [optional]
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
[More Information Needed]
### Out-of-Scope Use
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
[More Information Needed]
## Bias, Risks, and Limitations
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
[More Information Needed]
### Recommendations
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
## How to Get Started with the Model
Use the code below to get started with the model.
[More Information Needed]
## Training Details
### Training Data
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
[More Information Needed]
### Training Procedure
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
#### Preprocessing [optional]
[More Information Needed]
#### Training Hyperparameters
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
#### Speeds, Sizes, Times [optional]
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
[More Information Needed]
## Evaluation
<!-- This section describes the evaluation protocols and provides the results. -->
### Testing Data, Factors & Metrics
#### Testing Data
<!-- This should link to a Dataset Card if possible. -->
[More Information Needed]
#### Factors
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
[More Information Needed]
#### Metrics
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
[More Information Needed]
### Results
[More Information Needed]
#### Summary
## Model Examination [optional]
<!-- Relevant interpretability work for the model goes here -->
[More Information Needed]
## Environmental Impact
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
- **Hardware Type:** [More Information Needed]
- **Hours used:** [More Information Needed]
- **Cloud Provider:** [More Information Needed]
- **Compute Region:** [More Information Needed]
- **Carbon Emitted:** [More Information Needed]
## Technical Specifications [optional]
### Model Architecture and Objective
[More Information Needed]
### Compute Infrastructure
[More Information Needed]
#### Hardware
[More Information Needed]
#### Software
[More Information Needed]
## Citation [optional]
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
**BibTeX:**
[More Information Needed]
**APA:**
[More Information Needed]
## Glossary [optional]
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
[More Information Needed]
## More Information [optional]
[More Information Needed]
## Model Card Authors [optional]
[More Information Needed]
## Model Card Contact
[More Information Needed]
### Framework versions
- PEFT 0.11.1

35
moe_shakespeare15M/checkpoint-500/adapter_config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"alpha_pattern": {},
"auto_mapping": null,
"base_model_name_or_path": "/content/stories15M_MOE",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layer_replication": null,
"layers_pattern": null,
"layers_to_transform": null,
"loftq_config": {},
"lora_alpha": 128,
"lora_dropout": 0.05,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"rank_pattern": {},
"revision": null,
"target_modules": [
"lm_head",
"w1",
"w3",
"o_proj",
"k_proj",
"w2",
"v_proj",
"q_proj"
],
"task_type": "CAUSAL_LM",
"use_dora": false,
"use_rslora": false
}

3
moe_shakespeare15M/checkpoint-500/adapter_model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f18954c98a8efbf979d86e3af1f9a8e8fdf624a0bcf3aa239e59dd6d820f31cc
size 68160648

3
moe_shakespeare15M/checkpoint-500/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c02c354dd7b15801a89b4d60e6da5846270b8f81c4c8cb28e679907f5e76459b
size 62700602

3
moe_shakespeare15M/checkpoint-500/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2a9285bd087c798eb4268ef909268e501cbd456d1aaa56bfd2778031e469f9cd
size 14244

3
moe_shakespeare15M/checkpoint-500/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cc4ab5f608c113ef9b81b5921bd661a21962184c52075538ed871611015781ce
size 1064

208
moe_shakespeare15M/checkpoint-500/trainer_state.json Normal file
View File

@@ -0,0 +1,208 @@
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 417.0,
"eval_steps": 500,
"global_step": 500,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 17.0,
"grad_norm": 6.09660530090332,
"learning_rate": 5e-06,
"loss": 2.5103,
"step": 20
},
{
"epoch": 33.0,
"grad_norm": 1.7365801334381104,
"learning_rate": 1e-05,
"loss": 2.1374,
"step": 40
},
{
"epoch": 50.0,
"grad_norm": 1.1849949359893799,
"learning_rate": 1.5e-05,
"loss": 2.0202,
"step": 60
},
{
"epoch": 67.0,
"grad_norm": 0.8566276431083679,
"learning_rate": 2e-05,
"loss": 1.8209,
"step": 80
},
{
"epoch": 83.0,
"grad_norm": 0.733796238899231,
"learning_rate": 2.5e-05,
"loss": 1.5589,
"step": 100
},
{
"epoch": 100.0,
"grad_norm": 1.1494934558868408,
"learning_rate": 2.375e-05,
"loss": 1.4981,
"step": 120
},
{
"epoch": 117.0,
"grad_norm": 1.0989091396331787,
"learning_rate": 2.25e-05,
"loss": 1.3532,
"step": 140
},
{
"epoch": 133.0,
"grad_norm": 1.2396273612976074,
"learning_rate": 2.125e-05,
"loss": 1.1437,
"step": 160
},
{
"epoch": 150.0,
"grad_norm": 1.9112813472747803,
"learning_rate": 2e-05,
"loss": 1.0889,
"step": 180
},
{
"epoch": 167.0,
"grad_norm": 1.9788281917572021,
"learning_rate": 1.8750000000000002e-05,
"loss": 0.9825,
"step": 200
},
{
"epoch": 183.0,
"grad_norm": 2.772813081741333,
"learning_rate": 1.75e-05,
"loss": 0.832,
"step": 220
},
{
"epoch": 200.0,
"grad_norm": 2.384052038192749,
"learning_rate": 1.6250000000000002e-05,
"loss": 0.7939,
"step": 240
},
{
"epoch": 217.0,
"grad_norm": 2.423220634460449,
"learning_rate": 1.5e-05,
"loss": 0.7084,
"step": 260
},
{
"epoch": 233.0,
"grad_norm": 2.2296969890594482,
"learning_rate": 1.3750000000000002e-05,
"loss": 0.5999,
"step": 280
},
{
"epoch": 250.0,
"grad_norm": 3.0638062953948975,
"learning_rate": 1.25e-05,
"loss": 0.5756,
"step": 300
},
{
"epoch": 267.0,
"grad_norm": 3.0690948963165283,
"learning_rate": 1.125e-05,
"loss": 0.5252,
"step": 320
},
{
"epoch": 283.0,
"grad_norm": 2.1726977825164795,
"learning_rate": 1e-05,
"loss": 0.4515,
"step": 340
},
{
"epoch": 300.0,
"grad_norm": 2.711531639099121,
"learning_rate": 8.75e-06,
"loss": 0.4445,
"step": 360
},
{
"epoch": 317.0,
"grad_norm": 2.583651542663574,
"learning_rate": 7.5e-06,
"loss": 0.4179,
"step": 380
},
{
"epoch": 333.0,
"grad_norm": 2.864351511001587,
"learning_rate": 6.25e-06,
"loss": 0.363,
"step": 400
},
{
"epoch": 350.0,
"grad_norm": 1.7802315950393677,
"learning_rate": 5e-06,
"loss": 0.366,
"step": 420
},
{
"epoch": 367.0,
"grad_norm": 2.3433611392974854,
"learning_rate": 3.75e-06,
"loss": 0.3446,
"step": 440
},
{
"epoch": 383.0,
"grad_norm": 1.7403523921966553,
"learning_rate": 2.5e-06,
"loss": 0.3113,
"step": 460
},
{
"epoch": 400.0,
"grad_norm": 2.8045754432678223,
"learning_rate": 1.25e-06,
"loss": 0.3222,
"step": 480
},
{
"epoch": 417.0,
"grad_norm": 0.993517279624939,
"learning_rate": 0.0,
"loss": 0.3175,
"step": 500
}
],
"logging_steps": 20,
"max_steps": 500,
"num_input_tokens_seen": 0,
"num_train_epochs": 500,
"save_steps": 100,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 1803074919043200.0,
"train_batch_size": 50,
"trial_name": null,
"trial_params": null
}

3
moe_shakespeare15M/checkpoint-500/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a2b73b2f545cb4aeeed59e19bc6a72d15f8bfdd68be832143d3c7096a9982a57
size 5112

127
run_finetuned.ipynb Normal file
View File

@@ -0,0 +1,127 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "ca60092b-a133-40d5-bce7-be261eb13ba3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/var/home/ngxson/jupyter/stories-15M\n"
]
}
],
"source": [
"import os\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"model_path = os.getcwd()\n",
"print(model_path)\n",
"tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False)\n",
"tokenizer.pad_token = tokenizer.eos_token\n",
"model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors=True, local_files_only=True)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "d8197b9a-9c94-4c14-9b89-5e16f129f71b",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
"Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",
"The attention mask is not set and cannot be inferred from input because pad token is same as eos token.As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Look in thy glass was a little girl. She was only three years old and she was three years old. She was\n"
]
}
],
"source": [
"inputs = tokenizer('Look in thy glass', return_tensors=\"pt\")\n",
"outputs = model.generate(inputs['input_ids'], max_new_tokens=20)\n",
"print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "242b314c-d702-4cc1-862e-aaf59e986527",
"metadata": {},
"outputs": [],
"source": [
"from peft import PeftModel\n",
"CHECKPOINT_PATH = 'moe_shakespeare15M/checkpoint-500'\n",
"ft_model = PeftModel.from_pretrained(model, CHECKPOINT_PATH)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "a0abc08e-7e77-4efe-8e1b-465eff9672b3",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
"Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Look in thy glass in love of the eye:\n",
"That's when when the eye see thy on the sun'\n"
]
}
],
"source": [
"outputs = ft_model.generate(inputs['input_ids'], max_new_tokens=20)\n",
"print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0733e354-6b16-4c8f-a7f9-6207d75feee1",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

23
special_tokens_map.json Normal file
View File

@@ -0,0 +1,23 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

3
stories15M_MOE-F16.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1240dfc1957df9f3550dd6c1d9e64b466fc2f452d8bc34bd4e45e1a1e2ca6055
size 73466432

3
stories15M_MOE-Q8_0.gguf Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c7aa6863f9a4b3cdf19716e2c95622dcbd3bd06989324bf1ac8e60486ef8e881
size 39390272

93391
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

32
tokenizer_config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"bos_token": {
"__type": "AddedToken",
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": false,
"eos_token": {
"__type": "AddedToken",
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"model_max_length": 2048,
"pad_token": null,
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"use_default_system_prompt": true
}