From b94faca1093e3bc90b34ed6a87ba4fc8ab47dcbd Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 13 Jun 2026 17:29:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: tasal9/pashto-base-bloom Source: Original Platform --- .gitattributes | 40 +++++++++ ModelCard.md | 181 ++++++++++++++++++++++++++++++++++++++++ README.md | 93 +++++++++++++++++++++ config.json | 31 +++++++ generation_config.json | 7 ++ model.safetensors | 3 + special_tokens_map.json | 30 +++++++ tokenizer.json | 3 + tokenizer_config.json | 48 +++++++++++ 9 files changed, 436 insertions(+) create mode 100644 .gitattributes create mode 100644 ModelCard.md create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c45e9de --- /dev/null +++ b/.gitattributes @@ -0,0 +1,40 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2829/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-5658/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-8487/tokenizer.json filter=lfs diff=lfs merge=lfs -text +final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/ModelCard.md b/ModelCard.md new file mode 100644 index 0000000..9860cdc --- /dev/null +++ b/ModelCard.md @@ -0,0 +1,181 @@ +--- +--- +license: mit # Or your chosen license: apache-2.0, cc-by-4.0, etc. +language: +- ps # Pashto +library_name: transformers +tags: +- text-generation +- pashto +- bloom +- zamai-bloom +datasets: + +**Note on Dataset Identifiers:** +The `datasets` field in the metadata of this model card might list `tasal9/pashto_base_bloom`. This identifier may refer to an earlier version or a different collection of Pashto data. The specific training run culminating in this model update (June 2025) exclusively used the locally processed `datasets/base_pashto_clean` as described above. + + +- tasal9/pashto_base_bloom +pipeline_tag: text-generation +widget: +- text: "پښتو ژبه" +--- + +# ZamAI Bloom Pashto - checkpoint5207 (and Final Model) + +This model card is for `checkpoint5207` and the final fine-tuned version of a Bloom model for Pashto text generation, developed under the ZamAI Bloom project. + +## Model Description + +This model is a fine-tuned version of [bigscience/bloom-560m](https://huggingface.co/bigscience/bloom-560m) on a Pashto text corpus. The goal of this project was to create a language model proficient in generating coherent and contextually relevant Pashto text. + +**Base Model:** `bigscience/bloom-560m` +**Fine-tuning Checkpoint:** `checkpoint5207` +**Final Model:** [tasal9/zamai-bloom-ps-final] + +## Intended Uses & Limitations + +### Intended Uses + +This model is intended for: +* Generating Pashto text. +* Assisting with Pashto language content creation. +* Research in Pashto NLP. +* Educational purposes for Pashto language learning. + +### Limitations and Bias + +* The model's performance is dependent on the quality and diversity of the training data. It may generate text that reflects biases present in the data. +* It might produce factually incorrect or nonsensical text, especially for complex topics or out-of-domain prompts. +* The model may not be suitable for critical applications without further evaluation and mitigation of potential harms. +* Performance on specific Pashto dialects might vary depending on their representation in the training data. + +## How to use + +You can use this model with the Hugging Face `transformers` library for text generation. + +First, install the library: +```bash +pip install transformers torch +``` + +Then, you can use the model in Python: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "tasal9/zamai-bloom-ps-final" # Or the specific checkpoint identifier if using a checkpoint directly +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained(model_name) + +prompt = "په پښتو ژبه کې یو شعر ولیکئ د پسرلي په اړه" # Example prompt: "Write a poem in Pashto about spring" +inputs = tokenizer(prompt, return_tensors="pt") + +# Generate text +# Adjust generation parameters as needed (max_length, num_beams, do_sample, top_k, top_p, etc.) +outputs = model.generate(**inputs, max_length=100, num_beams=5, early_stopping=True) +generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + +print(generated_text) +``` + +## Training Data + +Describe the dataset(s) used for fine-tuning. +* **Source:** [e.g., Web scraped data, specific Pashto corpora, data from `datasets/base_pashto/`] +* **Size:** [e.g., Number of documents, tokens, GBs] +* **Preprocessing:** [e.g., Cleaning steps, tokenization details] +* **Language Variety:** [e.g., Predominant dialects, formal/informal text] + +If your dataset is on the Hugging Face Hub, link to it. + +## Training Procedure + +### Preprocessing + +The texts were tokenized using the `AutoTokenizer` associated with the base Bloom model. +[Add any other specific preprocessing steps you took.] + +### Fine-tuning + +The model was fine-tuned using the Hugging Face `transformers` library with PyTorch. +* **Training script:** [Link to your `train_base_model.py` if applicable] +* **Hyperparameters:** + * Learning rate: 2e-5 + * Batch size: 4 # Adjust based on your GPU memory (e.g., 8, 16) + * Number of epochs: 3 # Adjust based on convergence and overfitting + * Optimizer: AdamW + * Weight decay: 0.01 + * Warmup steps: 500 # Or warmup_ratio, e.g., 0.1 + * Gradient accumulation steps: 1 # Increase if actual batch size is limited by memory + * Seed: 42 # For reproducibility +* **Infrastructure:** + * Hardware: [e.g., 1x NVIDIA A100 40GB, or specify your hardware] + * Training time: [e.g., X hours] + +This specific model card refers to `checkpoint5207`, which was saved at step 5207 of the training process. The final model represents the model after the completion of all training epochs/steps. + +## Evaluation Results + +Provide quantitative results if available (e.g., perplexity, BLEU scores on a held-out test set). +* **Test set:** [Describe your test set] +* **Metrics:** [e.g., Perplexity, BLEU, ROUGE] +* **Results for checkpoint5207:** + * [Metric 1]: [Value] + * [Metric 2]: [Value] +* **Results for final model:** + * [Metric 1]: [Value] + * [Metric 2]: [Value] + +Qualitative observations can also be included. + +## Model Card Contact + +**Author:** Yaqoob Tasal +**Username:** tasal9 +**Organization:** ZamAI +[GitHub: https://github.com/tasal9](https://github.com/tasal9) + +## Citation + +If you use this model or its checkpoints, please consider citing: + +```bibtex +@misc{zamai_bloom_pashto_2025, + author = {Yaqoob Tasal}, + title = {ZamAI Bloom Pashto - Fine-tuned Language Model}, + year = {2025}, + publisher = {Hugging Face}, + journal = {Hugging Face Model Hub}, + howpublished = {\url{https://huggingface.co/tasal9/zamai-bloom-ps-final}} +} +``` + +And the original Bloom model: +```bibtex +@article{scao2022bloom, + title={BLOOM: A 176B-Parameter Open-Access Multilingual Language Model}, + author={Scao, Teven Le and Fan, Angela and Akiki, Christopher and Baran, Efrat and Ben Cheikh, Rim and Coavoux, Maxime and Davison, Thomas and de Vargas, Niklas Deckers and Delangue, C{\'e}line and Demeusy, Thibault and others}, + journal={arXiv preprint arXiv:2211.05100}, + year={2022} +} +``` + +--- + +Remember to replace placeholders like dataset details, hyperparameters, and evaluation results with your actual project details. Save this as a `README.md` file in your model repository on the Hugging Face Hub. + +## Training Details (Cleaned Base Model - June 2025) + +This model version was trained from `bigscience/bloom-560m` using the `train_base_model.py` script. + +- **Training Data:** The model was trained on a locally prepared dataset located at `datasets/base_pashto_clean`. This dataset was created using `prepare_base_dataset.py` and is derived from `pashto_data/base_model/cleaned_base_data.txt`, which primarily contains Pashto text from a bilingual Pashto-English glossary. +- **Training Objective:** To establish a foundational Pashto language model with improved coherence and reduced issues (e.g., repetition, off-language generation) compared to any prior versions trained on noisier data. +- **Output Directory (during training):** `models/pashto-bloom-base-clean-colab` +- **Key Training Hyperparameters:** + - Epochs: 3 + - Per Device Batch Size: 2 + - Gradient Accumulation Steps: 4 + - Learning Rate: 5e-5 + - FP16 (Mixed Precision): True + - Optimizer: AdamW diff --git a/README.md b/README.md new file mode 100644 index 0000000..8896faa --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +--- +language: +- en +- ps +license: mit +library_name: transformers +tags: +- bloom +- pashto +- lightweight +- language-model +base_model: bigscience/bloomz-560m +pipeline_tag: text-generation +datasets: +- tasal9/Pashto-Dataset-Creating-Dataset +--- + +# pashto-base-bloom + +BLOOM-based model fine-tuned for Pashto language tasks + +## 🌟 Model Overview + +This model is part of the **ZamAI Pro Models Strategy** - a comprehensive AI platform designed for multilingual applications with specialized focus on Pashto language support. + +### Key Features +- 🧠 **Advanced AI**: Based on bigscience/bloomz-560m architecture +- 🌐 **Multilingual**: Optimized for Pashto and English +- ⚡ **High Performance**: Optimized for production deployment +- 🔒 **Secure**: Enterprise-grade security and privacy + +## 📚 Usage + +### Basic Usage with Transformers + +```python +from transformers import AutoTokenizer, AutoModel + +tokenizer = AutoTokenizer.from_pretrained("tasal9/pashto-base-bloom") +model = AutoModel.from_pretrained("tasal9/pashto-base-bloom") + +# Example usage +text = "Your input text here" +inputs = tokenizer(text, return_tensors="pt") +outputs = model(**inputs) +``` + +### Usage with Hugging Face Inference API + +```python +from huggingface_hub import InferenceClient + +client = InferenceClient(token="your_hf_token") + +response = client.text_generation( + model="tasal9/pashto-base-bloom", + prompt="Your prompt here", + max_new_tokens=200 +) +``` + +## 🔧 Technical Details + +- **Model Type**: text-generation +- **Base Model**: bigscience/bloomz-560m +- **Languages**: Pashto (ps), English (en) +- **License**: MIT +- **Training**: Fine-tuned on Pashto educational and cultural content + +## 🚀 Applications + +This model powers: +- **ZamAI Educational Platform**: Pashto language tutoring +- **Business Automation**: Document processing and analysis +- **Voice Assistants**: Natural language understanding +- **Cultural Preservation**: Supporting Pashto language technology + +## 📞 Support + +For support and integration assistance: +- 📧 **Email**: support@zamai.dev +- 🌐 **Website**: [zamai.dev](https://zamai.dev) +- 💬 **Community**: [ZamAI Community](https://zamai.dev) + +## 📄 License + +Licensed under the MIT License. + +--- + +**Part of the ZamAI Pro Models Strategy - Transforming AI for Multilingual Applications** 🌟 + +*Updated: 2025-07-05 21:29:16 UTC* diff --git a/config.json b/config.json new file mode 100644 index 0000000..77a6ffb --- /dev/null +++ b/config.json @@ -0,0 +1,31 @@ +{ + "apply_residual_connection_post_layernorm": false, + "architectures": [ + "BloomForCausalLM" + ], + "attention_dropout": 0.0, + "attention_softmax_in_fp32": true, + "bias_dropout_fusion": true, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_dropout": 0.0, + "hidden_size": 1024, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "masked_softmax_fusion": true, + "model_type": "bloom", + "n_head": 16, + "n_inner": null, + "n_layer": 24, + "offset_alibi": 100, + "pad_token_id": 3, + "pretraining_tp": 1, + "skip_bias_add": true, + "skip_bias_add_qkv": false, + "slow_but_exact": false, + "torch_dtype": "float32", + "transformers_version": "4.52.4", + "unk_token_id": 0, + "use_cache": true, + "vocab_size": 250680 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..3464ec3 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 3, + "transformers_version": "4.52.4" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..7bb6eae --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4bd4ca3159b4e8807d2c68cb2f5f8663a4c1617e4fd74edd7c7e51dab6817d5 +size 2236073104 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..a782b2f --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..793311f --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d963066d6adae5034a1dc114c3ac444512de09928cf14ed4562ba94d9a440e66 +size 21763085 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..cb6690b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,48 @@ +{ + "add_prefix_space": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "merges_file": null, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "left", + "tokenizer_class": "BloomTokenizer", + "unk_token": "", + "vocab_file": null +}