初始化项目，由ModelHub XC社区提供模型

Model: rovdetection/code-1b-chat-v2
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,63 @@
+---
+library_name: transformers
+license: apache-2.0
+base_model: rovdetection/code-1b-pretrain-v3
+tags:
+- generated_from_trainer
+model-index:
+- name: code-1b-chat-v2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# code-1b-chat-v2
+
+This model is a fine-tuned version of [rovdetection/code-1b-pretrain-v3](https://huggingface.co/rovdetection/code-1b-pretrain-v3) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9919
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- training_steps: 1000
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.0509        | 0.8840 | 500  | 1.0258          |
+| 0.6652        | 1.7673 | 1000 | 0.9919          |
+
+
+### Framework versions
+
+- Transformers 5.9.0
+- Pytorch 2.10.0+cu128
+- Datasets 4.8.5
+- Tokenizers 0.22.2

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 16,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

final/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 16,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

final/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

final/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a751a165bf17614987ee30caba843bf951957ba5761cc1ce2081c7374c53074
+size 4523108832

final/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "max_length": 512,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "stride": 0,
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
+}

final/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09d01e394e9508be9f5db421ce912c1e3d19f75d3680c709cd862dd999d76c4
+size 5329

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

last-checkpoint/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 16,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a751a165bf17614987ee30caba843bf951957ba5761cc1ce2081c7374c53074
+size 4523108832

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7480e0708a0bb83f0630f8cb4be5db168c5200d77f56a0ec1bd890be73a54559
+size 2911851147

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399
+size 14645

last-checkpoint/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61361c878721548392539ed308adea82ec21fc99e9c9e2512a2e560c5477b77c
+size 1465

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "max_length": 512,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "stride": 0,
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,190 @@
+{
+  "best_global_step": 1000,
+  "best_metric": 0.9919272661209106,
+  "best_model_checkpoint": "./sft-out/checkpoint-1000",
+  "epoch": 1.7673352118901597,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08840267418089397,
+      "grad_norm": 2.867945432662964,
+      "learning_rate": 9.800000000000001e-06,
+      "loss": 1.6486561584472657,
+      "step": 50
+    },
+    {
+      "epoch": 0.17680534836178793,
+      "grad_norm": 2.9991917610168457,
+      "learning_rate": 1.98e-05,
+      "loss": 1.33203125,
+      "step": 100
+    },
+    {
+      "epoch": 0.26520802254268194,
+      "grad_norm": 2.570798635482788,
+      "learning_rate": 1.9854078984834904e-05,
+      "loss": 1.2356742858886718,
+      "step": 150
+    },
+    {
+      "epoch": 0.35361069672357587,
+      "grad_norm": 2.5860543251037598,
+      "learning_rate": 1.9408807689542257e-05,
+      "loss": 1.2118841552734374,
+      "step": 200
+    },
+    {
+      "epoch": 0.44201337090446985,
+      "grad_norm": 2.117816686630249,
+      "learning_rate": 1.8677654533689287e-05,
+      "loss": 1.1767095184326173,
+      "step": 250
+    },
+    {
+      "epoch": 0.5304160450853639,
+      "grad_norm": 2.276134729385376,
+      "learning_rate": 1.7682835235935236e-05,
+      "loss": 1.1405416870117187,
+      "step": 300
+    },
+    {
+      "epoch": 0.6188187192662578,
+      "grad_norm": 2.2516863346099854,
+      "learning_rate": 1.645457687723951e-05,
+      "loss": 1.128560562133789,
+      "step": 350
+    },
+    {
+      "epoch": 0.7072213934471517,
+      "grad_norm": 2.084246873855591,
+      "learning_rate": 1.5030199466302354e-05,
+      "loss": 1.1079308319091796,
+      "step": 400
+    },
+    {
+      "epoch": 0.7956240676280457,
+      "grad_norm": 2.195765495300293,
+      "learning_rate": 1.3452981989985347e-05,
+      "loss": 1.0333491516113282,
+      "step": 450
+    },
+    {
+      "epoch": 0.8840267418089397,
+      "grad_norm": 2.2488884925842285,
+      "learning_rate": 1.1770847403195836e-05,
+      "loss": 1.0508524322509765,
+      "step": 500
+    },
+    {
+      "epoch": 0.8840267418089397,
+      "eval_loss": 1.025797724723816,
+      "eval_runtime": 31.5114,
+      "eval_samples_per_second": 15.867,
+      "eval_steps_per_second": 1.999,
+      "step": 500
+    },
+    {
+      "epoch": 0.9724294159898337,
+      "grad_norm": 2.3597640991210938,
+      "learning_rate": 1.0034906514152239e-05,
+      "loss": 1.0521656036376954,
+      "step": 550
+    },
+    {
+      "epoch": 1.060113818443008,
+      "grad_norm": 2.0308382511138916,
+      "learning_rate": 8.297905008339677e-06,
+      "loss": 0.8026390075683594,
+      "step": 600
+    },
+    {
+      "epoch": 1.148516492623902,
+      "grad_norm": 2.3317034244537354,
+      "learning_rate": 6.612620797547087e-06,
+      "loss": 0.6939823150634765,
+      "step": 650
+    },
+    {
+      "epoch": 1.2369191668047959,
+      "grad_norm": 2.5876853466033936,
+      "learning_rate": 5.030260389724447e-06,
+      "loss": 0.6835686492919922,
+      "step": 700
+    },
+    {
+      "epoch": 1.3253218409856897,
+      "grad_norm": 2.342280149459839,
+      "learning_rate": 3.598903005150444e-06,
+      "loss": 0.6746553039550781,
+      "step": 750
+    },
+    {
+      "epoch": 1.4137245151665838,
+      "grad_norm": 2.354048728942871,
+      "learning_rate": 2.362039713653581e-06,
+      "loss": 0.6704821014404296,
+      "step": 800
+    },
+    {
+      "epoch": 1.502127189347478,
+      "grad_norm": 2.01891827583313,
+      "learning_rate": 1.3572519804629537e-06,
+      "loss": 0.6500045776367187,
+      "step": 850
+    },
+    {
+      "epoch": 1.5905298635283718,
+      "grad_norm": 2.010759115219116,
+      "learning_rate": 6.150697724044407e-07,
+      "loss": 0.6556130981445313,
+      "step": 900
+    },
+    {
+      "epoch": 1.6789325377092656,
+      "grad_norm": 2.3100311756134033,
+      "learning_rate": 1.580439203075812e-07,
+      "loss": 0.6622157287597656,
+      "step": 950
+    },
+    {
+      "epoch": 1.7673352118901597,
+      "grad_norm": 2.064932346343994,
+      "learning_rate": 6.092342209607083e-11,
+      "loss": 0.6651800537109375,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7673352118901597,
+      "eval_loss": 0.9919272661209106,
+      "eval_runtime": 31.6701,
+      "eval_samples_per_second": 15.788,
+      "eval_steps_per_second": 1.989,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.876769443016704e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09d01e394e9508be9f5db421ce912c1e3d19f75d3680c709cd862dd999d76c4
+size 5329

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a751a165bf17614987ee30caba843bf951957ba5761cc1ce2081c7374c53074
+size 4523108832

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "max_length": 512,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "stride": 0,
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09d01e394e9508be9f5db421ce912c1e3d19f75d3680c709cd862dd999d76c4
+size 5329