commit 0b912da915cddd4d3956e067a5fd71d27d226789 Author: ModelHub XC Date: Mon Aug 24 08:37:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: davron04/gemma-3-270m-uzen-base Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1b5cc5e --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +--- +library_name: transformers +base_model: davron04/gemma-3-270m-uzen-base +tags: +- generated_from_trainer +model-index: +- name: gemma-3-270m-uzen-base + results: [] +--- + + + +# gemma-3-270m-uzen-base + +This model is a fine-tuned version of [davron04/gemma-3-270m-uzen-base](https://huggingface.co/davron04/gemma-3-270m-uzen-base) on an unknown dataset. +It achieves the following results on the evaluation set: +- Loss: 2.1987 +- Perplexity: 9.0416 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 2e-05 +- train_batch_size: 2 +- eval_batch_size: 4 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 64 +- total_train_batch_size: 256 +- total_eval_batch_size: 8 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: inverse_sqrt +- lr_scheduler_warmup_steps: 0.01 +- num_epochs: 1 +- mixed_precision_training: Native AMP + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Perplexity | +|:-------------:|:------:|:----:|:---------------:|:----------:| +| No log | 0 | 0 | 3.3726 | 29.0075 | +| 152.2381 | 0.1002 | 428 | 2.4411 | 11.5116 | +| 145.3987 | 0.2003 | 856 | 2.3491 | 10.5032 | +| 143.7446 | 0.3005 | 1284 | 2.3286 | 10.2931 | +| 140.7659 | 0.4006 | 1712 | 2.2912 | 9.9159 | +| 139.1574 | 0.5008 | 2140 | 2.2643 | 9.6535 | +| 137.4137 | 0.6009 | 2568 | 2.2431 | 9.4512 | +| 136.3983 | 0.7011 | 2996 | 2.2254 | 9.2859 | +| 135.6059 | 0.8012 | 3424 | 2.2111 | 9.1541 | +| 134.8424 | 0.9014 | 3852 | 2.1987 | 9.0416 | + + +### Framework versions + +- Transformers 5.0.0 +- Pytorch 2.10.0+cu128 +- Datasets 4.8.5 +- Tokenizers 0.22.2 diff --git a/config.json b/config.json new file mode 100644 index 0000000..77f5150 --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "_sliding_window_pattern": 6, + "architectures": [ + "Gemma3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "dtype": "float32", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 2048, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "model_type": "gemma3_text", + "num_attention_heads": 4, + "num_hidden_layers": 18, + "num_key_value_heads": 1, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "5.0.0", + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262144 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..4c18b9f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "cache_implementation": "hybrid", + "do_sample": true, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.0.0" +} diff --git a/last-checkpoint/config.json b/last-checkpoint/config.json new file mode 100644 index 0000000..77f5150 --- /dev/null +++ b/last-checkpoint/config.json @@ -0,0 +1,62 @@ +{ + "_sliding_window_pattern": 6, + "architectures": [ + "Gemma3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "dtype": "float32", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 2048, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "model_type": "gemma3_text", + "num_attention_heads": 4, + "num_hidden_layers": 18, + "num_key_value_heads": 1, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "5.0.0", + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262144 +} diff --git a/last-checkpoint/generation_config.json b/last-checkpoint/generation_config.json new file mode 100644 index 0000000..4c18b9f --- /dev/null +++ b/last-checkpoint/generation_config.json @@ -0,0 +1,7 @@ +{ + "cache_implementation": "hybrid", + "do_sample": true, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.0.0" +} diff --git a/last-checkpoint/model.safetensors b/last-checkpoint/model.safetensors new file mode 100644 index 0000000..d2577f8 --- /dev/null +++ b/last-checkpoint/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f0430804f1c451d3bf6fb6a3c21dcd6458496e4c8b1aafa2199a5979add746 +size 1072419256 diff --git a/last-checkpoint/optimizer.pt b/last-checkpoint/optimizer.pt new file mode 100644 index 0000000..23d6c41 --- /dev/null +++ b/last-checkpoint/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c634cd5776cecc15f8c0fe390100d329c0ddd7415c9759a41e7171d19985cfa2 +size 2144987083 diff --git a/last-checkpoint/rng_state_0.pth b/last-checkpoint/rng_state_0.pth new file mode 100644 index 0000000..3a2bb76 --- /dev/null +++ b/last-checkpoint/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9627a7b92eeed978894313d8642baf0ecdcd808f2102543d8b58136102ad5997 +size 14917 diff --git a/last-checkpoint/rng_state_1.pth b/last-checkpoint/rng_state_1.pth new file mode 100644 index 0000000..7b307a1 --- /dev/null +++ b/last-checkpoint/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d79c621c2f337ae714092486c08d5a224dca6ddc4285fb0014e9b648d0c9cae +size 14917 diff --git a/last-checkpoint/scaler.pt b/last-checkpoint/scaler.pt new file mode 100644 index 0000000..9ef824a --- /dev/null +++ b/last-checkpoint/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f71bbc2be28fb2466cee77d17f19aa4d2f70898f072b16a7dedd850e3fb3f602 +size 1383 diff --git a/last-checkpoint/scheduler.pt b/last-checkpoint/scheduler.pt new file mode 100644 index 0000000..bf768a0 --- /dev/null +++ b/last-checkpoint/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e46a111ea4df4e25e5be6c12364a518c5abbbe495ab277aad446b86ca1835bbe +size 1465 diff --git a/last-checkpoint/trainer_state.json b/last-checkpoint/trainer_state.json new file mode 100644 index 0000000..07d8e1a --- /dev/null +++ b/last-checkpoint/trainer_state.json @@ -0,0 +1,316 @@ +{ + "best_global_step": 2996, + "best_metric": 2.225428342819214, + "best_model_checkpoint": "/kaggle/working/gemma-3-270m-uzen-base/checkpoint-2996", + "epoch": 0.7010749542961608, + "eval_steps": 428, + "global_step": 2996, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0, + "eval_loss": 3.3726205825805664, + "eval_perplexity": 29.00750160217285, + "eval_runtime": 1376.8644, + "eval_samples_per_second": 8.027, + "eval_steps_per_second": 1.004, + "step": 0 + }, + { + "epoch": 0.0002340036563071298, + "grad_norm": NaN, + "learning_rate": 0.0, + "loss": 221.65936279296875, + "step": 1 + }, + { + "epoch": 0.02340036563071298, + "grad_norm": 125.02448272705078, + "learning_rate": 1.3455579942126144e-05, + "loss": 186.75027619949495, + "step": 100 + }, + { + "epoch": 0.04680073126142596, + "grad_norm": 119.42086029052734, + "learning_rate": 9.391758525703705e-06, + "loss": 162.133095703125, + "step": 200 + }, + { + "epoch": 0.07020109689213894, + "grad_norm": 122.48279571533203, + "learning_rate": 7.64875162105191e-06, + "loss": 155.50732421875, + "step": 300 + }, + { + "epoch": 0.09360146252285192, + "grad_norm": 117.2989273071289, + "learning_rate": 6.607179605173858e-06, + "loss": 152.238125, + "step": 400 + }, + { + "epoch": 0.10015356489945156, + "eval_loss": 2.4410510063171387, + "eval_perplexity": 11.5115966796875, + "eval_runtime": 1493.6491, + "eval_samples_per_second": 7.399, + "eval_steps_per_second": 0.925, + "step": 428 + }, + { + "epoch": 0.1170018281535649, + "grad_norm": 112.73179626464844, + "learning_rate": 5.900662146334273e-06, + "loss": 150.016318359375, + "step": 500 + }, + { + "epoch": 0.14040219378427787, + "grad_norm": 120.18157958984375, + "learning_rate": 5.381099233077658e-06, + "loss": 147.887568359375, + "step": 600 + }, + { + "epoch": 0.16380255941499086, + "grad_norm": 126.22393035888672, + "learning_rate": 4.978339248330686e-06, + "loss": 147.1618359375, + "step": 700 + }, + { + "epoch": 0.18720292504570385, + "grad_norm": 119.209228515625, + "learning_rate": 4.657229393557059e-06, + "loss": 145.39865234375, + "step": 800 + }, + { + "epoch": 0.2003071297989031, + "eval_loss": 2.3490686416625977, + "eval_perplexity": 10.503242492675781, + "eval_runtime": 1490.1057, + "eval_samples_per_second": 7.417, + "eval_steps_per_second": 0.927, + "step": 856 + }, + { + "epoch": 0.2106032906764168, + "grad_norm": 124.55681610107422, + "learning_rate": 4.39118562699897e-06, + "loss": 153.693115234375, + "step": 900 + }, + { + "epoch": 0.2340036563071298, + "grad_norm": 119.34051513671875, + "learning_rate": 4.1639776269114805e-06, + "loss": 146.270078125, + "step": 1000 + }, + { + "epoch": 0.2574040219378428, + "grad_norm": 114.71582794189453, + "learning_rate": 3.968742338690654e-06, + "loss": 144.3760546875, + "step": 1100 + }, + { + "epoch": 0.28080438756855575, + "grad_norm": 119.8339614868164, + "learning_rate": 3.79862214511292e-06, + "loss": 143.744560546875, + "step": 1200 + }, + { + "epoch": 0.3004606946983547, + "eval_loss": 2.3285601139068604, + "eval_perplexity": 10.293147087097168, + "eval_runtime": 1270.7866, + "eval_samples_per_second": 8.697, + "eval_steps_per_second": 1.088, + "step": 1284 + }, + { + "epoch": 0.30420475319926876, + "grad_norm": 114.1391830444336, + "learning_rate": 3.648656396326189e-06, + "loss": 143.26626953125, + "step": 1300 + }, + { + "epoch": 0.3276051188299817, + "grad_norm": 118.7150650024414, + "learning_rate": 3.5151560191662133e-06, + "loss": 142.079208984375, + "step": 1400 + }, + { + "epoch": 0.3510054844606947, + "grad_norm": 113.54842376708984, + "learning_rate": 3.395312965552404e-06, + "loss": 141.61373046875, + "step": 1500 + }, + { + "epoch": 0.3744058500914077, + "grad_norm": 113.5634994506836, + "learning_rate": 3.2869469269372928e-06, + "loss": 141.197421875, + "step": 1600 + }, + { + "epoch": 0.39780621572212066, + "grad_norm": 117.68986511230469, + "learning_rate": 3.188335715317592e-06, + "loss": 140.76587890625, + "step": 1700 + }, + { + "epoch": 0.4006142595978062, + "eval_loss": 2.2911739349365234, + "eval_perplexity": 9.915882110595703, + "eval_runtime": 1272.1824, + "eval_samples_per_second": 8.687, + "eval_steps_per_second": 1.086, + "step": 1712 + }, + { + "epoch": 0.4212065813528336, + "grad_norm": 114.08328247070312, + "learning_rate": 3.0980984951602635e-06, + "loss": 140.2319140625, + "step": 1800 + }, + { + "epoch": 0.44460694698354664, + "grad_norm": 110.51737976074219, + "learning_rate": 3.0151134457776365e-06, + "loss": 139.54671875, + "step": 1900 + }, + { + "epoch": 0.4680073126142596, + "grad_norm": 114.35932159423828, + "learning_rate": 2.9384584653539762e-06, + "loss": 139.24345703125, + "step": 2000 + }, + { + "epoch": 0.49140767824497256, + "grad_norm": 113.81220245361328, + "learning_rate": 2.867367664450403e-06, + "loss": 139.157421875, + "step": 2100 + }, + { + "epoch": 0.5007678244972578, + "eval_loss": 2.2643067836761475, + "eval_perplexity": 9.653481483459473, + "eval_runtime": 1272.3391, + "eval_samples_per_second": 8.686, + "eval_steps_per_second": 1.086, + "step": 2140 + }, + { + "epoch": 0.5148080438756856, + "grad_norm": 118.33269500732422, + "learning_rate": 2.801198909122156e-06, + "loss": 138.7333984375, + "step": 2200 + }, + { + "epoch": 0.5382084095063986, + "grad_norm": 111.53887176513672, + "learning_rate": 2.7394092432028485e-06, + "loss": 138.593310546875, + "step": 2300 + }, + { + "epoch": 0.5616087751371115, + "grad_norm": 116.05701446533203, + "learning_rate": 2.6815360246517324e-06, + "loss": 137.77955078125, + "step": 2400 + }, + { + "epoch": 0.5850091407678245, + "grad_norm": 117.56316375732422, + "learning_rate": 2.627182269536618e-06, + "loss": 137.41365234375, + "step": 2500 + }, + { + "epoch": 0.6009213893967094, + "eval_loss": 2.2431113719940186, + "eval_perplexity": 9.451156616210938, + "eval_runtime": 1281.4279, + "eval_samples_per_second": 8.625, + "eval_steps_per_second": 1.078, + "step": 2568 + }, + { + "epoch": 0.6084095063985375, + "grad_norm": 119.91163635253906, + "learning_rate": 2.576005137637696e-06, + "loss": 137.66705078125, + "step": 2600 + }, + { + "epoch": 0.6318098720292504, + "grad_norm": 112.58194732666016, + "learning_rate": 2.5277067936120503e-06, + "loss": 137.92888671875, + "step": 2700 + }, + { + "epoch": 0.6552102376599634, + "grad_norm": 118.54427337646484, + "learning_rate": 2.48202708541266e-06, + "loss": 136.8133984375, + "step": 2800 + }, + { + "epoch": 0.6786106032906765, + "grad_norm": 112.7831802368164, + "learning_rate": 2.4387376277801515e-06, + "loss": 136.398330078125, + "step": 2900 + }, + { + "epoch": 0.7010749542961608, + "eval_loss": 2.225428342819214, + "eval_perplexity": 9.285932540893555, + "eval_runtime": 1259.9334, + "eval_samples_per_second": 8.772, + "eval_steps_per_second": 1.097, + "step": 2996 + } + ], + "logging_steps": 100, + "max_steps": 4274, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 428, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.660641467170816e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/last-checkpoint/training_args.bin b/last-checkpoint/training_args.bin new file mode 100644 index 0000000..fc86981 --- /dev/null +++ b/last-checkpoint/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91863d45bd0cf0edf0333820ff57f11f3da227dc2346013be5b6f097f0ef83c7 +size 5329 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..2ceea70 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:002d0bd94c23049973bb2a46093a25ed7e13819ae3e828569f0558132c1ad145 +size 1072419256 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..899af07 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497 +size 33384443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..7202584 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,24 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..fc86981 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91863d45bd0cf0edf0333820ff57f11f3da227dc2346013be5b6f097f0ef83c7 +size 5329