commit 6f3923d4ec5f79edfa1b6da3880019ce445f6323 Author: ModelHub XC Date: Thu Jul 9 06:48:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: basilepp19/bloom-1b7_it Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..db4217b --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +--- +license: bigscience-bloom-rail-1.0 +language: +- it +--- +# Model Card for Model ID + + + +This model is obtained by adapting bloom-1b7 to the Italian language. Among the languages supported by the BLOOM model, there is no Italian, making its use +in that context challenging. We adapt the original BLOOM model using the MAD-X language adaptation strategy. + +## Model Details + +### Model Description + +We adapt the bloom-1b7 to the Italian language using the MAD-X language adaptation strategy. +To produce a valuable model, we follow the same procedure proposed in: https://arxiv.org/abs/2212.09535 + +We use default script parameters and select a sample of 100,000 examples in the Italian language. We decided to sample data from the Filtered Oscar Dataset for +the Italian Language released by Sarti. + +**It is important to underline that when you use the adapted LLM is necessary to use the tokenizer of the adapted model.** + +- **Developed by:** Pierpaolo Basile, Pierluigi Cassotti, Marco Polignano, Lucia Siciliani, Giovanni Semeraro. Department of Computer Science, University of Bari Aldo Moro, Italy +- **Model type:** BLOOM +- **Language(s) (NLP):** Italian +- **License:** BigScience BLOOM RAIL 1.0 + +## Citation + +Pierpaolo Basile, Pierluigi Cassotti, Marco Polignano, Lucia Siciliani, Giovanni Semeraro. On the impact of Language Adaptation for Large Language Models: A +case study for the Italian language using only open resources. Proceedings of the Ninth Italian Conference on Computational Linguistics (CLiC-it 2023). + diff --git a/config.json b/config.json new file mode 100644 index 0000000..dc78e06 --- /dev/null +++ b/config.json @@ -0,0 +1,39 @@ +{ + "_name_or_path": "bigscience/bloom-1b7", + "adapters": { + "adapters": {}, + "config_map": {}, + "fusion_config_map": {}, + "fusions": {} + }, + "apply_residual_connection_post_layernorm": false, + "architectures": [ + "BloomForCausalLM" + ], + "attention_dropout": 0.0, + "attention_softmax_in_fp32": true, + "bias_dropout_fusion": true, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_dropout": 0.0, + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "masked_softmax_fusion": true, + "model_type": "bloom", + "n_head": 16, + "n_inner": null, + "n_layer": 24, + "offset_alibi": 100, + "pad_token_id": 3, + "pretraining_tp": 2, + "seq_length": 4096, + "skip_bias_add": true, + "skip_bias_add_qkv": false, + "slow_but_exact": false, + "torch_dtype": "float32", + "transformers_version": "4.20.0.dev0", + "unk_token_id": 0, + "use_cache": true, + "vocab_size": 250680 +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..ab8f4b3 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e6a1793b480e7844962d27e3e80a6e0a27e5ff4196dcbe9c1164eabaa68b6f +size 6888031832 diff --git a/optimizer.pt b/optimizer.pt new file mode 100644 index 0000000..9d32298 --- /dev/null +++ b/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f44c02c8f3363645ef83905881320edf2e79ccedb6d69a7915c0ec387c5d0a14 +size 4107176871 diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..db72340 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be34ae537d50fe297b90e9836c11c9c0b8745515c87ec0a188cefaee23317312 +size 6888098489 diff --git a/rng_state.pth b/rng_state.pth new file mode 100644 index 0000000..d9372e0 --- /dev/null +++ b/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a55114bf49b6edb9e57f9ae748463a35e7eee5926828b0e3b57189cee51607e2 +size 14511 diff --git a/scaler.pt b/scaler.pt new file mode 100644 index 0000000..16d7853 --- /dev/null +++ b/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1027296c45bdadaade456e4ae525e911d64ab3e95f04db88d7f108bb5fd25fbd +size 557 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..b2c6915 --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a9930ef7826d202df5c0f0a520db2e60e06d56c5e3ff7bdebac0b28823ddc9a +size 627 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..25bc396 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "", "pad_token": ""} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4fa803 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5 +size 14500443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6200252 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"unk_token": "", "bos_token": "", "eos_token": "", "pad_token": "", "add_prefix_space": false, "name_or_path": "bigscience/bloom-1b7", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..0e53bf8 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,396 @@ +{ + "best_metric": 3.468397378921509, + "best_model_checkpoint": "/datadrive/disk1/pierpaolo/llm/itlangadapt/bloom-1b3_it/checkpoint-50000", + "epoch": 5.236680193742637, + "global_step": 50000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.1, + "learning_rate": 0.00098, + "loss": 3.4734, + "step": 1000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00096, + "loss": 3.466, + "step": 2000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00094, + "loss": 3.461, + "step": 3000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00092, + "loss": 3.4725, + "step": 4000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0009000000000000001, + "loss": 3.4614, + "step": 5000 + }, + { + "epoch": 0.52, + "eval_loss": 3.469589948654175, + "eval_runtime": 358.4452, + "eval_samples_per_second": 10.624, + "eval_steps_per_second": 5.312, + "step": 5000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00088, + "loss": 3.4653, + "step": 6000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00086, + "loss": 3.4618, + "step": 7000 + }, + { + "epoch": 0.84, + "learning_rate": 0.00084, + "loss": 3.4707, + "step": 8000 + }, + { + "epoch": 0.94, + "learning_rate": 0.00082002, + "loss": 3.4706, + "step": 9000 + }, + { + "epoch": 1.05, + "learning_rate": 0.0008000199999999999, + "loss": 3.4694, + "step": 10000 + }, + { + "epoch": 1.05, + "eval_loss": 3.4690916538238525, + "eval_runtime": 358.2981, + "eval_samples_per_second": 10.628, + "eval_steps_per_second": 5.314, + "step": 10000 + }, + { + "epoch": 1.15, + "learning_rate": 0.0007800200000000001, + "loss": 3.4686, + "step": 11000 + }, + { + "epoch": 1.26, + "learning_rate": 0.00076002, + "loss": 3.4636, + "step": 12000 + }, + { + "epoch": 1.36, + "learning_rate": 0.0007400400000000001, + "loss": 3.463, + "step": 13000 + }, + { + "epoch": 1.47, + "learning_rate": 0.00072006, + "loss": 3.4694, + "step": 14000 + }, + { + "epoch": 1.57, + "learning_rate": 0.0007000600000000001, + "loss": 3.4709, + "step": 15000 + }, + { + "epoch": 1.57, + "eval_loss": 3.4688496589660645, + "eval_runtime": 358.397, + "eval_samples_per_second": 10.625, + "eval_steps_per_second": 5.313, + "step": 15000 + }, + { + "epoch": 1.68, + "learning_rate": 0.00068006, + "loss": 3.4627, + "step": 16000 + }, + { + "epoch": 1.78, + "learning_rate": 0.0006600800000000001, + "loss": 3.4658, + "step": 17000 + }, + { + "epoch": 1.89, + "learning_rate": 0.00064008, + "loss": 3.4661, + "step": 18000 + }, + { + "epoch": 1.99, + "learning_rate": 0.00062008, + "loss": 3.4615, + "step": 19000 + }, + { + "epoch": 2.09, + "learning_rate": 0.0006001, + "loss": 3.481, + "step": 20000 + }, + { + "epoch": 2.09, + "eval_loss": 3.468804359436035, + "eval_runtime": 359.4028, + "eval_samples_per_second": 10.595, + "eval_steps_per_second": 5.298, + "step": 20000 + }, + { + "epoch": 2.2, + "learning_rate": 0.0005801, + "loss": 3.465, + "step": 21000 + }, + { + "epoch": 2.3, + "learning_rate": 0.00056012, + "loss": 3.4656, + "step": 22000 + }, + { + "epoch": 2.41, + "learning_rate": 0.00054012, + "loss": 3.472, + "step": 23000 + }, + { + "epoch": 2.51, + "learning_rate": 0.00052012, + "loss": 3.4574, + "step": 24000 + }, + { + "epoch": 2.62, + "learning_rate": 0.0005001200000000001, + "loss": 3.4638, + "step": 25000 + }, + { + "epoch": 2.62, + "eval_loss": 3.468665599822998, + "eval_runtime": 358.6249, + "eval_samples_per_second": 10.618, + "eval_steps_per_second": 5.309, + "step": 25000 + }, + { + "epoch": 2.72, + "learning_rate": 0.00048012, + "loss": 3.4611, + "step": 26000 + }, + { + "epoch": 2.83, + "learning_rate": 0.00046012, + "loss": 3.4631, + "step": 27000 + }, + { + "epoch": 2.93, + "learning_rate": 0.00044014, + "loss": 3.4626, + "step": 28000 + }, + { + "epoch": 3.04, + "learning_rate": 0.00042014, + "loss": 3.469, + "step": 29000 + }, + { + "epoch": 3.14, + "learning_rate": 0.00040018, + "loss": 3.4704, + "step": 30000 + }, + { + "epoch": 3.14, + "eval_loss": 3.468623399734497, + "eval_runtime": 359.7336, + "eval_samples_per_second": 10.586, + "eval_steps_per_second": 5.293, + "step": 30000 + }, + { + "epoch": 3.25, + "learning_rate": 0.00038018000000000004, + "loss": 3.4659, + "step": 31000 + }, + { + "epoch": 3.35, + "learning_rate": 0.00036018, + "loss": 3.4706, + "step": 32000 + }, + { + "epoch": 3.46, + "learning_rate": 0.00034018, + "loss": 3.4659, + "step": 33000 + }, + { + "epoch": 3.56, + "learning_rate": 0.00032018000000000004, + "loss": 3.4669, + "step": 34000 + }, + { + "epoch": 3.67, + "learning_rate": 0.00030018, + "loss": 3.4617, + "step": 35000 + }, + { + "epoch": 3.67, + "eval_loss": 3.4684813022613525, + "eval_runtime": 361.1986, + "eval_samples_per_second": 10.543, + "eval_steps_per_second": 5.271, + "step": 35000 + }, + { + "epoch": 3.77, + "learning_rate": 0.00028020000000000003, + "loss": 3.4703, + "step": 36000 + }, + { + "epoch": 3.88, + "learning_rate": 0.00026024, + "loss": 3.4606, + "step": 37000 + }, + { + "epoch": 3.98, + "learning_rate": 0.00024026, + "loss": 3.4549, + "step": 38000 + }, + { + "epoch": 4.08, + "learning_rate": 0.00022026, + "loss": 3.4701, + "step": 39000 + }, + { + "epoch": 4.19, + "learning_rate": 0.00020026, + "loss": 3.4636, + "step": 40000 + }, + { + "epoch": 4.19, + "eval_loss": 3.468475103378296, + "eval_runtime": 358.6453, + "eval_samples_per_second": 10.618, + "eval_steps_per_second": 5.309, + "step": 40000 + }, + { + "epoch": 4.29, + "learning_rate": 0.00018026, + "loss": 3.4541, + "step": 41000 + }, + { + "epoch": 4.4, + "learning_rate": 0.00016026000000000001, + "loss": 3.4724, + "step": 42000 + }, + { + "epoch": 4.5, + "learning_rate": 0.00014026, + "loss": 3.4683, + "step": 43000 + }, + { + "epoch": 4.61, + "learning_rate": 0.00012026, + "loss": 3.463, + "step": 44000 + }, + { + "epoch": 4.71, + "learning_rate": 0.00010026, + "loss": 3.4668, + "step": 45000 + }, + { + "epoch": 4.71, + "eval_loss": 3.4684269428253174, + "eval_runtime": 358.1004, + "eval_samples_per_second": 10.634, + "eval_steps_per_second": 5.317, + "step": 45000 + }, + { + "epoch": 4.82, + "learning_rate": 8.028000000000001e-05, + "loss": 3.4696, + "step": 46000 + }, + { + "epoch": 4.92, + "learning_rate": 6.028e-05, + "loss": 3.4605, + "step": 47000 + }, + { + "epoch": 5.03, + "learning_rate": 4.028000000000001e-05, + "loss": 3.4645, + "step": 48000 + }, + { + "epoch": 5.13, + "learning_rate": 2.028e-05, + "loss": 3.4704, + "step": 49000 + }, + { + "epoch": 5.24, + "learning_rate": 2.7999999999999997e-07, + "loss": 3.4623, + "step": 50000 + }, + { + "epoch": 5.24, + "eval_loss": 3.468397378921509, + "eval_runtime": 358.1322, + "eval_samples_per_second": 10.633, + "eval_steps_per_second": 5.316, + "step": 50000 + } + ], + "max_steps": 50000, + "num_train_epochs": 6, + "total_flos": 2.9704946454626304e+18, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..e8d14e3 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c00872bf59e9698ca9a41d7297ae520c057d003f29763384a6124b997b4b369c +size 3259