From 424aa3cf5cf4441e0fd3bf4413bb9210cf1da106 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 7 Jul 2026 20:19:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: budecosystem/boomer-1b Source: Original Platform --- .gitattributes | 50 ++++++++++++++++++ README.md | 109 ++++++++++++++++++++++++++++++++++++++++ config.json | 26 ++++++++++ configuration.json | 1 + generation_config.json | 6 +++ pytorch_model.bin | 3 ++ special_tokens_map.json | 24 +++++++++ tokenizer.model | 3 ++ tokenizer_config.json | 37 ++++++++++++++ 9 files changed, 259 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a059173 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,50 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +pytorch_model.bin filter=lfs diff=lfs merge=lfs -text +tokenizer.model filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a5181a2 --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +--- +license: apache-2.0 +language: +- en +library_name: transformers +--- + +
+ + +

Democratizing access to LLMs for the open-source community.
Let's advance AI, together.

+ +---- + +## Introduction 🎉 + +We are open-sourcing one of our early experiments of pretraining with custom architecture and datasets. This 1.1B parameter model is pre-trained from scratch using a custom-curated dataset of 41B tokens. The model's architecture experiments contain the addition of flash attention and a higher intermediate dimension of the MLP layer. The dataset is a combination of wiki, stories, arxiv, math and code. The model is available on huggingface [Boomer1B](https://huggingface.co/budecosystem/boomer-1b) + +
+ +## Getting Started on GitHub 💻 + +Ready to dive in? Here's how you can get started with our models on GitHub. + +Install the necessary dependencies with the following command: + +```bash +pip install -r requirements.txt +``` + +### Generate responses + +Now that your model is fine-tuned, you're ready to generate responses. You can do this using our generate.py script, which runs inference from the Hugging Face model hub and inference on a specified input. Here's an example of usage: + +```bash +python generate.py --base_model 'budecosystem/boomer-1b' --prompt="the president of India is" +``` + +### Fine-tuning 🎯 + + +It's time to upgrade the model by fine-tuning the model. You can do this using our provided finetune.py script. Here's an example command: + +```bash +torchrun --nproc_per_node 4 train.py \ + --base_model budecosystem/boomer-1b \ + --data_path dataset.json \ + --output_dir output \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 2 \ + --num_train_epochs 1 \ + --learning_rate 2e-5 \ + --fp16 True \ + --logging_steps 10 \ + --deepspeed ds_config.json +``` + +## Model details + +| Parameters | Value | +| :------------- | :----: | +| n_layers | 4 | +| n_heads | 32 | +| d_model | 4096 | +| vocab size | 32000 | +| sequence length | 4096 | +| Intermediate size | 11008 | + +### Tokenizer + +We used the SentencePiece tokenizer during the fine-tuning process. This tokenizer is known for its capability to handle open-vocabulary language tasks efficiently. + +### Training details + +The model is trained of 4 A100 80GB for approximately 250hrs. + +| Hyperparameters | Value | +| :----------------------------| :-----: | +| per_device_train_batch_size | 2 | +| gradient_accumulation_steps | 2 | +| learning_rate | 2e-4 | +| optimizer | adamw | +| beta | 0.9, 0.95 | +| fp16 | True | +| GPU | 4 A100 80GB | + + +## Evaluations + +We have evaluated the pre-trained model on few of the benchmarks + +| Model Name | ARC | MMLU | Human Eval | Hellaswag | BBH | DROP | GSM8K | +|:----------:|:--------:|:----:|:----------:|:---------:|:-----: |:-----:|:----:| +| Boomer1B | 22.35 | 25.92| 6.1 | 31.66 | 28.65 | 6.13 | 1.5 | + +### Why use BOOMER? + +- Retrieval augmentation +- Inference at the edge +- Language modeling use cases + +### Final thought on Boomer! + +This isn't the end. It's just the beginning of a journey towards creating more advanced, more efficient, and more accessible language models. We invite you to join us on this exciting journey. + + +### Aknowledgements + +We'd like to thank the open-source community and the researchers whose foundational work laid the path for BOOMER. Special shoutout to our dedicated team who have worked relentlessly to curate the dataset and fine-tune the model to perfection. \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..24a379d --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "boomer-1b", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 4, + "num_key_value_heads": 32, + "pretraining_tp": 1, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.33.1", + "use_cache": false, + "vocab_size": 32000 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..a7a7f64 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.33.1" +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..9a60549 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1902cda297164be733c20efccad4b0cc5fa8195ae526dd3f4741f4e1a598dd +size 2143375334 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..14761dc --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..16ada5c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,37 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": false, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "legacy": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "use_default_system_prompt": true +}