初始化项目，由ModelHub XC社区提供模型

Model: Josephgflowers/Tinyllama-1.5B-Cinder-Test-1
Source: Original Platform

.gitattributes

@@ -0,0 +1,49 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,8 @@
+---
+license: mit
+---
+This is a depth up scalled model of the 616M cinder model and Cinder v2. This model still needs further training. Putting it up for testing.
+More information coming.
+Maybe. Lol.
+Here is a brief desc of the project:
+Im mixing a lot of techniques I guess that I found interesting and have been testing, HF Cosmo is not great but decent and was fully trained in 4 days using a mix of more fine tuned directed datasets and some synthetic textbook style datasets. So I used pruning and a similar mix as Cosmo on tinyllama (trained on a ton of data for an extended time for its size) to keep the tinyllama model coherent during pruning. Now I am trying to depth up scale it using my pruned model and an original, Then taking a majority of each and combining them to create a larger model. Then it needs more training, then fine tuning. Then theoretically it will be a well performing 1.5B model (that didn't need full scale training).

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/home/joe/Downloads/so_muc-math/checkpoint-2000",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 31,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.38.0.dev0",
+  "unsloth_version": "2024.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

configuration.json

@@ -0,0 +1 @@
+{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.0.dev0",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfb5af0ff202968f5171c5ce9f05e022b8b3bc2bd1bd68857712b709a39f919a
+size 2992926176

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b58273a18857dd12b639dc6c4e27b6df1e608156ab00404b49e01cc9d5c62065
+size 5454522

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f18c8a721b3db6249c3f6a7e3fbe6262f13cef0416eb6f736e7eb3b5ab63271
+size 1064

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcd04f0eadf90287bd26e1a183ac487d8a141b09b06aecb7725bbdd343640f2e
+size 1842767

tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "max_length": 2048,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "stride": 0,
+  "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,161 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.2705993776214315,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.8518972396850586,
+      "learning_rate": 4.933026654038696e-05,
+      "loss": 0.5532,
+      "step": 100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.7409780621528625,
+      "learning_rate": 4.865376809633338e-05,
+      "loss": 0.5033,
+      "step": 200
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.7067281007766724,
+      "learning_rate": 4.79772696522798e-05,
+      "loss": 0.4602,
+      "step": 300
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.7321436405181885,
+      "learning_rate": 4.7300771208226225e-05,
+      "loss": 0.4523,
+      "step": 400
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 0.6380289793014526,
+      "learning_rate": 4.6624272764172644e-05,
+      "loss": 0.4606,
+      "step": 500
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.5421116352081299,
+      "learning_rate": 4.594777432011906e-05,
+      "loss": 0.4262,
+      "step": 600
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 0.5193295478820801,
+      "learning_rate": 4.527127587606549e-05,
+      "loss": 0.4105,
+      "step": 700
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 0.5962494611740112,
+      "learning_rate": 4.459477743201191e-05,
+      "loss": 0.4174,
+      "step": 800
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.6797559261322021,
+      "learning_rate": 4.391827898795833e-05,
+      "loss": 0.4346,
+      "step": 900
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.5834950804710388,
+      "learning_rate": 4.324178054390475e-05,
+      "loss": 0.4233,
+      "step": 1000
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.5665029287338257,
+      "learning_rate": 4.256528209985117e-05,
+      "loss": 0.4119,
+      "step": 1100
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.5659196972846985,
+      "learning_rate": 4.188878365579759e-05,
+      "loss": 0.4242,
+      "step": 1200
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.5055731534957886,
+      "learning_rate": 4.121228521174402e-05,
+      "loss": 0.4075,
+      "step": 1300
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 0.5024439692497253,
+      "learning_rate": 4.053578676769044e-05,
+      "loss": 0.3856,
+      "step": 1400
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.5468323826789856,
+      "learning_rate": 3.9859288323636856e-05,
+      "loss": 0.3942,
+      "step": 1500
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.579856812953949,
+      "learning_rate": 3.918278987958328e-05,
+      "loss": 0.3914,
+      "step": 1600
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 0.6119765043258667,
+      "learning_rate": 3.85062914355297e-05,
+      "loss": 0.4195,
+      "step": 1700
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.4361695647239685,
+      "learning_rate": 3.782979299147612e-05,
+      "loss": 0.3926,
+      "step": 1800
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.4878581464290619,
+      "learning_rate": 3.715329454742254e-05,
+      "loss": 0.3738,
+      "step": 1900
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 0.6861026883125305,
+      "learning_rate": 3.6476796103368965e-05,
+      "loss": 0.4021,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 7391,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 5.62657090338816e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c08f18a63de89b1f80f499629968399a4fbd3284e2375345b0c62a39936aae5
+size 4920