初始化项目，由ModelHub XC社区提供模型

Model: CoolSpring/Qwen2-0.5B-Abyme
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,147 @@
+---
+language:
+- en
+- zh
+license: apache-2.0
+tags:
+- axolotl
+- generated_from_trainer
+base_model: Qwen/Qwen2-0.5B
+datasets:
+- Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered
+model-index:
+- name: Qwen2-0.5B-Abyme
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+adapter: null
+base_model: Qwen/Qwen2-0.5B
+bf16: auto
+chat_template: chatml
+dataset_prepared_path: ./data/last_run_prepared
+datasets:
+- path: Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered
+  type: sharegpt
+deepspeed: null
+early_stopping_patience: null
+eval_sample_packing: true
+evals_per_epoch: 4
+flash_attention: true
+fp16: null
+fsdp: null
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: false
+group_by_length: false
+hf_use_auth_token: true
+hub_model_id: CoolSpring/Qwen2-0.5B-Abyme
+learning_rate: 2e-5
+load_in_4bit: false
+load_in_8bit: false
+local_rank: null
+logging_steps: 1
+lr_scheduler: cosine
+micro_batch_size: 4
+num_epochs: 1
+optimizer: adamw_torch
+output_dir: ./outputs/out
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: true
+resume_from_checkpoint: null
+sample_packing: true
+saves_per_epoch: 1
+sequence_len: 4096
+tf32: true
+tokens:
+- <|im_start|>
+- <|im_end|>
+train_on_inputs: false
+val_set_size: 0.05
+wandb_entity: null
+wandb_log_model: null
+wandb_name: Qwen2-0.5B-Abyme
+wandb_project: Qwen2-0.5B-Magpie-Qwen2-Pro-300K-Filtered
+wandb_watch: null
+warmup_steps: 100
+weight_decay: null
+xformers_attention: null
+
+```
+
+</details><br>
+
+# Qwen2-0.5B-Abyme
+
+This model is a fine-tuned version of [Qwen/Qwen2-0.5B](https://huggingface.co/Qwen/Qwen2-0.5B) on the [Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered](https://huggingface.co/datasets/Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered) dataset. It was created to explore the effects of training the smallest model in the Qwen2 series on data extracted from the largest model in the Qwen2 series (as of July 18th, 2024).
+
+It achieves the following results on the evaluation set:
+- Loss: 0.8229  
+
+## Model description
+
+Qwen2-0.5B-Abyme is a 0.5 billion parameter language model fine-tuned on a dataset of conversation samples from the much larger 72 billion parameter Qwen2-72B model. The purpose of this experiment is to investigate whether a smaller model can effectively learn and reproduce the knowledge and capabilities of a significantly larger model through the fine-tuning process.
+
+## Intended uses & limitations
+
+This model is intended for research purposes to study the knowledge transfer and distillation capabilities of language models. It may have practical applications in scenarios where the computational resources for running large language models are limited, and a smaller, fine-tuned model can provide comparable performance.
+
+However, it is important to note that the model's capabilities and limitations are yet to be fully evaluated. Its performance may vary depending on the task and domain, and it may exhibit biases or limitations inherited from the original models.
+
+## Training and evaluation data
+
+The model was fine-tuned on the [Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered](https://huggingface.co/datasets/Magpie-Align/Magpie-Qwen2-Pro-300K-Filtered) dataset, which contains 300,000 conversation samples from the Qwen2-72B model. 5% of this dataset was held out as the evaluation set for calculating the reported loss metric.
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 0.9947        | 0.0004 | 1    | 0.9683          |
+| 0.8385        | 0.2501 | 597  | 0.8338          |
+| 0.7636        | 0.5002 | 1194 | 0.8249          |
+| 0.8124        | 0.7502 | 1791 | 0.8229          |
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.3.1+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard)
+Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_CoolSpring__Qwen2-0.5B-Abyme)
+
+|      Metric       |Value|
+|-------------------|----:|
+|Avg.               | 4.76|
+|IFEval (0-Shot)    |19.15|
+|BBH (3-Shot)       | 2.28|
+|MATH Lvl 5 (4-Shot)| 1.51|
+|GPQA (0-shot)      | 0.45|
+|MuSR (0-shot)      | 1.48|
+|MMLU-PRO (5-shot)  | 3.70|
+

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.42.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b7633dac23e9f08337352f864cb69a743bb70f1d30d9354f91743d9030c86a
+size 988097824

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0fc5783d55e734505c9c188758ced0515d5bc4735707f9a7f4e01ee50c41ff8
+size 988163026

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f478dd80d470d45e48f8c5e6c28b8c6af53f4913eac8d9aa24614eeb20764da
+size 6136