初始化项目，由ModelHub XC社区提供模型

Model: lorinma/yi6B_Vicuna
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,196 @@
+---
+language:
+- en
+license: mit
+datasets:
+- anon8231489123/ShareGPT_Vicuna_unfiltered
+model-index:
+- name: yi6B_Vicuna
+  results:
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: AI2 Reasoning Challenge (25-Shot)
+      type: ai2_arc
+      config: ARC-Challenge
+      split: test
+      args:
+        num_few_shot: 25
+    metrics:
+    - type: acc_norm
+      value: 46.16
+      name: normalized accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: HellaSwag (10-Shot)
+      type: hellaswag
+      split: validation
+      args:
+        num_few_shot: 10
+    metrics:
+    - type: acc_norm
+      value: 69.3
+      name: normalized accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: MMLU (5-Shot)
+      type: cais/mmlu
+      config: all
+      split: test
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 58.43
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: TruthfulQA (0-shot)
+      type: truthful_qa
+      config: multiple_choice
+      split: validation
+      args:
+        num_few_shot: 0
+    metrics:
+    - type: mc2
+      value: 48.11
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: Winogrande (5-shot)
+      type: winogrande
+      config: winogrande_xl
+      split: validation
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 65.67
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: GSM8k (5-shot)
+      type: gsm8k
+      config: main
+      split: test
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 18.42
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=lorinma/yi6B_Vicuna
+      name: Open LLM Leaderboard
+---
+
+
+**Bug**: Having a bit issue with the tokenizer, still figuring out...You can use the original Yi tokenizer configuratin.
+
+
+Reproduce Vicuna, but based on yi-6B. The training data I used was ShareGPT_V3_unfiltered_cleaned_split_no_imsorry.json. 
+
+The training framework I used https://github.com/shibing624/MedicalGPT , train shell:
+```
+CUDA_VISIBLE_DEVICES=0,1,2,3,5 torchrun --nproc_per_node 5 ../supervised_finetuning.py \
+    --model_type auto \
+    --model_name_or_path /data/llm/models/Pretrained/yi-6B/01ai/Yi-6B \
+    --tokenizer_name_or_path /data/llm/models/Pretrained/yi-6B/01ai/Yi-6B \
+    --train_file_dir ../data/finetune/vicuna/ \
+    --per_device_train_batch_size 2\
+    --do_train \
+    --max_train_samples -1 \
+    --num_train_epochs 3 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --bf16 \
+    --use_peft False \
+    --logging_strategy steps \
+    --logging_steps 10 \
+    --save_strategy epoch \
+    --save_total_limit 5 \
+    --gradient_accumulation_steps 1 \
+    --preprocessing_num_workers 8 \
+    --output_dir ../outputs/20240106_yi6B_vicuna \
+    --overwrite_output_dir \
+    --ddp_timeout 30000 \
+    --logging_first_step True \
+    --torch_dtype bfloat16 \
+    --device_map auto \
+    --report_to tensorboard \
+    --ddp_find_unused_parameters False \
+    --gradient_checkpointing True \
+    --cache_dir ./cache \
+    --model_max_length 4096 \
+    --deepspeed ../deepspeed_zero_stage2_config_no16.json \
+    --template_name yi   
+```
+
+The training used 5*A800 for 3 epochs
+```
+***** train metrics *****
+  epoch                    =                3.0
+  train_loss               =             0.3785
+  train_runtime            = 1 day, 10:01:13.95
+  train_samples            =              93204
+  train_samples_per_second =               2.24
+  train_steps_per_second   =              0.224
+```
+
+Post-training inference is also using this repository:
+```
+CUDA_VISIBLE_DEVICES=4 python gradio_demo.py  --model_type auto --base_model /data/mn/shibing624/MedicalGPT-1.6.3-231215/outputs/20240106_yi6B_vicuna    --tokenizer_path /data/mn/shibing624/MedicalGPT-1.6.3-231215/outputs/20240106_yi6B_vicuna --template_name yi --gpus 4
+CUDA_VISIBLE_DEVICES=6 python inference.py --model_type auto --base_model /data/mn/shibing624/MedicalGPT-1.6.3-231215/outputs/20240106_yi6B_vicuna      --template_name yi --gpus 6     --interactive --tokenizer_path /data/llm/models/Pretrained/yi-6B/01ai/Yi-6B
+```
+
+We can see from some preliminary results, the conversation is natural and informative (unsurprisingly).
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/6413d7be996b2e426f230fb7/WfQYyyLxtXA2KlePmIPQJ.png)
+
+Also we observe the unfiltering seems to be working! **Heads up** some examples are unsafe and inappropriate, this is entirely for research purposes, to test how alignment-filtered SFT data affect LLM's final output.
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/6413d7be996b2e426f230fb7/pklSsljCRN34QuL2ZF2zU.png)
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/6413d7be996b2e426f230fb7/22pTSVkBCVlQ5N8A8JBkF.png)
+
+**Update:** Evaluate on Open LLM Leaderboard:
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/6413d7be996b2e426f230fb7/Xp11HLQqwh0HMSJgpr19n.png)
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_lorinma__yi6B_Vicuna)
+
+|             Metric              |Value|
+|---------------------------------|----:|
+|Avg.                             |51.02|
+|AI2 Reasoning Challenge (25-Shot)|46.16|
+|HellaSwag (10-Shot)              |69.30|
+|MMLU (5-Shot)                    |58.43|
+|TruthfulQA (0-shot)              |48.11|
+|Winogrande (5-shot)              |65.67|
+|GSM8k (5-shot)                   |18.42|
+

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<unk>": 0,
+  "<|endoftext|>": 2,
+  "<|startoftext|>": 1
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.3784581153609504,
+    "train_runtime": 122473.9577,
+    "train_samples": 93204,
+    "train_samples_per_second": 2.24,
+    "train_steps_per_second": 0.224
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/data/llm/models/Pretrained/yi-6B/01ai/Yi-6B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 5000000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.34.0",
+  "use_cache": true,
+  "vocab_size": 64000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.34.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7665356a8a5ec7890ebf452c4e55d2fd2148fe2c442f6562b992cc216b9e6b85
+size 12122104952

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf4392114988bdfa71d68ef881441fa64bbd87b64e90fd2b10c89a48d3c96bd8
+size 12122111662

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39
+size 1033105

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "model_max_length": 4096,
+  "unk_token": "<unk>",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "clean_up_tokenization_spaces": false,
+  "legacy": true,
+  "tokenizer_class": "LlamaTokenizer"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.3784581153609504,
+    "train_runtime": 122473.9577,
+    "train_samples": 93204,
+    "train_samples_per_second": 2.24,
+    "train_steps_per_second": 0.224
+}