初始化项目，由ModelHub XC社区提供模型
Model: PJMixers-Dev/Granite-3.1-Earthen-v0.3-3B-A800M-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
50
.gitattributes
vendored
Normal file
50
.gitattributes
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-BF16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_1.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Granite-3.1-Earthen-v0.3-3B-A800M-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-BF16.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-BF16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:16544705ad5d0af4e60d5c94faa06d3e75e21ac12d1d5e2f9847c22f1a046d96
|
||||
size 6603463168
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q2_K.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q2_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1d26b365835b61ebd551a21eafdf5248b22f65843b28888ff7b1472c376ad693
|
||||
size 1241367776
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_L.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_L.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5d3292cb09083ba0dbd1183680695c73f089734e7e719ea88a7104e10a9297cc
|
||||
size 1741538528
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_M.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6e3e27cef888c26ea7dca758e5f88df295385b48bee13c8a6cf0e468cc1d3547
|
||||
size 1611187424
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_S.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q3_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f64d0c920dde4591d20952d4c9a9c36d08bc7f315dfd5488099159e104140085
|
||||
size 1456063712
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_0.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:517fbdceb677d498e0ddf7f481c3a0be7206c6ae148caf1fb3e7e060f26adf24
|
||||
size 1883882720
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_1.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_1.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c08e80c36b7e896f65e3d67db14929d7865b4a2753669dec2c100436b0d89a48
|
||||
size 2085209312
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_M.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d07b7f0023a9ab878e585e1369d0c7e1df9e2bf1561bdce7b7fb567bca0746b6
|
||||
size 2016888032
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_S.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q4_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:389eea85c4719f3d461fb6ea8e4fc2940b864fca480b3b059dfe9e3de704f938
|
||||
size 1900004576
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_0.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d8c16768b381ff3e53a44f2380c62df98fef2558bd04c25d6d21c598f464e126
|
||||
size 2286535904
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_1.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_1.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:28326a5c439f6fb007646e3f00d7ccfe6e3fc8adb2fdaa803f243ac4f99724ac
|
||||
size 2487862496
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_M.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:24c8c2529ab841e2c82272fca48c71d8dfd1eeac0de9fb674193c539da44c745
|
||||
size 2355053792
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_S.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q5_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d1d4d6bab48f6023a9356f2da689ebb910c533defdeb757d04cdc382fd6abf54
|
||||
size 2286535904
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q6_K.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q6_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8c3da7dd48bf0492e178f02d4dcc3b109dc4ed95c5d84cd53afea8ec63b06d3d
|
||||
size 2714354912
|
||||
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q8_0.gguf
Normal file
3
Granite-3.1-Earthen-v0.3-3B-A800M-Q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e9416f1d5c3321716791a7acfd879133dda67ecba3b0111459118b7156d51a15
|
||||
size 3512781088
|
||||
429
README.md
Normal file
429
README.md
Normal file
@@ -0,0 +1,429 @@
|
||||
---
|
||||
base_model: PJMixers-Dev/Granite-3.1-Earthen-v0.3-3B-A800M
|
||||
license: apache-2.0
|
||||
pipeline_tag: text-generation
|
||||
language:
|
||||
- en
|
||||
datasets:
|
||||
- BeaverAI/REDACTED1
|
||||
- BeaverAI/REDACTED2
|
||||
- BeaverAI/REDACTED3
|
||||
- BeaverAI/REDACTED4
|
||||
- BeaverAI/REDACTED5
|
||||
- BeaverAI/REDACTED6
|
||||
- PJMixers-Dev/Lit-axo-Shuffled
|
||||
- PJMixers-Dev/Mielikki_Erebus-87k-axo
|
||||
- PJMixers/RyokoAI_Honeyfeed3600-Cleanish
|
||||
- PJMixers-Dev/allura-org_fujin-cleaned-stage-2-axo
|
||||
- Nelathan/synthetic-sugar-quill
|
||||
- PJMixers-Dev/winglian_visual-novels-json-axo-dropped-long
|
||||
- PJMixers-Dev/recursal_SCP-RECURSAL-Cleaned
|
||||
- PJMixers-Dev/Subtitles
|
||||
- PJMixers-Dev/KaraKaraWitch_AnimeSubtitle-axo
|
||||
- PJMixers/AP-News-2024
|
||||
- PJMixers-Dev/Fundus-AP-News-Formatted
|
||||
- PJMixers-Dev/Fundus-AP-News-2-Formatted
|
||||
- PJMixers-Dev/goodwiki-2024-12-04-axo
|
||||
- epfl-llm/guidelines
|
||||
- PJMixers-Dev/allenai_tulu-3-sft-mixture-filtered-2-ShareGPT
|
||||
- OpenLeecher/lmsys_chat_1m_clean
|
||||
- PJMixers-Dev/Gryphe-Aesir-RPG-Charcards-Opus-Mixed
|
||||
- allura-org/gryphe-sonnet-3.5-charcards-names-added
|
||||
- anthracite-org/c2_logs_32k_llama3_qwen2_v1.3
|
||||
- PJMixers-Dev/MinervaAI_Aesir-Preview-Anon
|
||||
- PJMixers-Dev/lemonilia_LimaRP-Simple-CustomShareGPT-Shuffled
|
||||
- Epiculous/SynthRP-Gens-v1.1-Filtered-n-Cleaned
|
||||
- PJMixers-Dev/NyxKrage_chub-logs-sharegpt-longest-CustomShareGPT
|
||||
- PJMixers/OpenLeecher_Teatime_all_logs_longest-ShareGPT
|
||||
- grimulkan/aicg-logs-augmented
|
||||
- grimulkan/PIPPA-augmented-dedup
|
||||
- PJMixers/grimulkan_bluemoon_Karen_cleaned-carded-formatted
|
||||
- PJMixers/lodrick-the-lafted_OpusStories-ShareGPT
|
||||
- Gryphe/ChatGPT-4o-Writing-Prompts
|
||||
- Gryphe/Opus-WritingPrompts
|
||||
- anthracite-org/nopm_claude_writing_fixed
|
||||
- PJMixers-Dev/Tiefighter-13B-Fake-Distill-ShareGPT
|
||||
- allura-org/fujin-instruct-v2
|
||||
- ToastyPigeon/gutenberg-sft
|
||||
- PocketDoc/Dans-Prosemaxx-Adventure
|
||||
- PocketDoc/Dans-Failuremaxx-Adventure-3
|
||||
- TheDrummer/AmoralQA-v2
|
||||
---
|
||||
# Granite-3.1-Earthen-v0.3-3B-A800M-GGUF
|
||||
|
||||
[`ibm-granite/granite-3.1-3b-a800m-instruct`](https://huggingface.co/ibm-granite/granite-3.1-3b-a800m-instruct) was trained at 8K with batch size 2 gradient accumulation 8, so each step was 131,072 tokens (including any padding tokens). It was trained for 400 steps, adding up to a total of 52,428,800 unique tokens seen.
|
||||
|
||||
This is a small test run. A larger version is planned.
|
||||
|
||||
## Quants
|
||||
|
||||
- [GGUF](https://huggingface.co/PJMixers-Dev/Granite-3.1-Earthen-v0.3-3B-A800M-GGUF)
|
||||
|
||||
## Prompt Format
|
||||
|
||||
This model uses Granite-3.1 Instruct format.
|
||||
|
||||
```
|
||||
<|start_of_role|>system<|end_of_role|>example system prompt<|end_of_text|>
|
||||
<|start_of_role|>user<|end_of_role|>example user turn 1<|end_of_text|>
|
||||
<|start_of_role|>assistant<|end_of_role|>example assistant turn 1<|end_of_text|>
|
||||
<|start_of_role|>user<|end_of_role|>example user turn 2<|end_of_text|>
|
||||
<|start_of_role|>assistant<|end_of_role|>example assistant turn 2<|end_of_text|>
|
||||
```
|
||||
|
||||
## Training Details
|
||||
|
||||
[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
|
||||
|
||||
```yaml
|
||||
# Requirements before running
|
||||
# - Get latest commit of axolotl (currently c0a0c75)
|
||||
# - Download these to axolotl/src/axolotl/prompt_formatters
|
||||
# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/formatter_regex.py
|
||||
# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customcompletion-regex.py
|
||||
# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customgranite-regex.py
|
||||
# - pip install ftfy
|
||||
# - pip install git+https://github.com/xzuyn/CAME.git@sr-grams-cautious-8bit
|
||||
|
||||
# Weights and Biases logging config
|
||||
wandb_project: Granite-3.1-3B-A800M
|
||||
wandb_name: Granite-3.1-Earthen-v0.3-3B-A800M-QLoRA-run4
|
||||
|
||||
# Model checkpointing config
|
||||
output_dir: ./Outputs/Granite-3.1-Earthen-v0.3-3B-A800M-QLoRA-run4
|
||||
resume_from_checkpoint:
|
||||
save_steps: 10
|
||||
save_safetensors: true
|
||||
save_total_limit: 2
|
||||
save_only_model: false
|
||||
|
||||
# Model architecture config
|
||||
base_model: ibm-granite/granite-3.1-3b-a800m-instruct
|
||||
model_type: AutoModelForCausalLM
|
||||
tokenizer_type: AutoTokenizer
|
||||
|
||||
# Mixed precision training config
|
||||
bf16: true
|
||||
fp16: false
|
||||
tf32: false
|
||||
|
||||
# Model loading config
|
||||
load_in_8bit: false
|
||||
load_in_4bit: true
|
||||
strict: false
|
||||
|
||||
# Sequence config
|
||||
sequence_len: 8192
|
||||
min_sample_len: 256
|
||||
sample_packing: true
|
||||
eval_sample_packing: true
|
||||
pad_to_sequence_len: true
|
||||
train_on_inputs: false
|
||||
group_by_length: false
|
||||
|
||||
# LoRA adapter config
|
||||
adapter: qlora
|
||||
lora_r: 128
|
||||
lora_alpha: 128
|
||||
lora_dropout: 0.125
|
||||
lora_target_linear: true
|
||||
embeddings_skip_upcast: true
|
||||
|
||||
# Dataset config
|
||||
datasets:
|
||||
# Completion
|
||||
# Story-like Data
|
||||
- path: BeaverAI/REDACTED1
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/Lit-axo-Shuffled
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/Mielikki_Erebus-87k-axo
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers/RyokoAI_Honeyfeed3600-Cleanish
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: BeaverAI/REDACTED2
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/allura-org_fujin-cleaned-stage-2-axo
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: Nelathan/synthetic-sugar-quill
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/winglian_visual-novels-json-axo-dropped-long
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: BeaverAI/REDACTED3
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/recursal_SCP-RECURSAL-Cleaned
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
# Subtitle Data
|
||||
- path: PJMixers-Dev/Subtitles
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/KaraKaraWitch_AnimeSubtitle-axo
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
# News Data
|
||||
- path: PJMixers/AP-News-2024
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/Fundus-AP-News-Formatted
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: PJMixers-Dev/Fundus-AP-News-2-Formatted
|
||||
type: customcompletion-regex
|
||||
# Misc Data
|
||||
- path: PJMixers-Dev/goodwiki-2024-12-04-axo
|
||||
split: train[:4000]
|
||||
type: customcompletion-regex
|
||||
- path: epfl-llm/guidelines
|
||||
split: train[:4000]
|
||||
field: clean_text
|
||||
type: customcompletion-regex
|
||||
# Granite-3.1 Instruct
|
||||
# Instruction Data
|
||||
- path: PJMixers-Dev/allenai_tulu-3-sft-mixture-filtered-2-ShareGPT
|
||||
split: train[:4000]
|
||||
type: customgranite-regex
|
||||
- path: OpenLeecher/lmsys_chat_1m_clean
|
||||
split: train[:4000]
|
||||
type: customgranite-regex
|
||||
# RP Data
|
||||
- path: PJMixers-Dev/Gryphe-Aesir-RPG-Charcards-Opus-Mixed
|
||||
type: customgranite-regex
|
||||
- path: allura-org/gryphe-sonnet-3.5-charcards-names-added
|
||||
type: customgranite-regex
|
||||
- path: anthracite-org/c2_logs_32k_llama3_qwen2_v1.3
|
||||
type: customgranite-regex
|
||||
- path: BeaverAI/REDACTED4
|
||||
type: customgranite-regex
|
||||
- path: PJMixers-Dev/MinervaAI_Aesir-Preview-Anon
|
||||
type: customgranite-regex
|
||||
- path: PJMixers-Dev/lemonilia_LimaRP-Simple-CustomShareGPT-Shuffled
|
||||
type: customgranite-regex
|
||||
- path: Epiculous/SynthRP-Gens-v1.1-Filtered-n-Cleaned
|
||||
type: customgranite-regex
|
||||
- path: PJMixers-Dev/NyxKrage_chub-logs-sharegpt-longest-CustomShareGPT
|
||||
type: customgranite-regex
|
||||
- path: PJMixers/OpenLeecher_Teatime_all_logs_longest-ShareGPT
|
||||
type: customgranite-regex
|
||||
- path: grimulkan/aicg-logs-augmented
|
||||
type: customgranite-regex
|
||||
- path: grimulkan/PIPPA-augmented-dedup
|
||||
type: customgranite-regex
|
||||
- path: PJMixers/grimulkan_bluemoon_Karen_cleaned-carded-formatted
|
||||
type: customgranite-regex
|
||||
# InstStory Data
|
||||
- path: PJMixers/lodrick-the-lafted_OpusStories-ShareGPT
|
||||
type: customgranite-regex
|
||||
- path: Gryphe/ChatGPT-4o-Writing-Prompts
|
||||
type: customgranite-regex
|
||||
- path: Gryphe/Opus-WritingPrompts
|
||||
type: customgranite-regex
|
||||
- path: anthracite-org/nopm_claude_writing_fixed
|
||||
type: customgranite-regex
|
||||
- path: PJMixers-Dev/Tiefighter-13B-Fake-Distill-ShareGPT
|
||||
type: customgranite-regex
|
||||
- path: allura-org/fujin-instruct-v2
|
||||
type: customgranite-regex
|
||||
- path: ToastyPigeon/gutenberg-sft
|
||||
type: customgranite-regex
|
||||
# Adventure Data
|
||||
- path: PocketDoc/Dans-Prosemaxx-Adventure
|
||||
type: customgranite-regex
|
||||
- path: PocketDoc/Dans-Failuremaxx-Adventure-3
|
||||
type: customgranite-regex
|
||||
# Decensoring Data
|
||||
- path: TheDrummer/AmoralQA-v2
|
||||
type: customgranite-regex
|
||||
- path: BeaverAI/REDACTED5
|
||||
type: customgranite-regex
|
||||
- path: BeaverAI/REDACTED6
|
||||
type: customgranite-regex
|
||||
val_set_size: 256
|
||||
eval_strategy: steps
|
||||
eval_steps: 10
|
||||
dataset_prepared_path: ./00-Tokenized-Datasets/Granite-3.1-Earthen-v0.3-3B-A800M-LoRA-seed42
|
||||
shuffle_merged_datasets: true
|
||||
|
||||
# Training hyperparameters
|
||||
num_epochs: 1
|
||||
gradient_accumulation_steps: 8
|
||||
micro_batch_size: 2
|
||||
eval_batch_size: 2
|
||||
warmup_steps: 0
|
||||
optimizer: came_pytorch
|
||||
optim_args:
|
||||
enable_stochastic_rounding: true
|
||||
enable_cautious: true
|
||||
enable_8bit: true
|
||||
lr_scheduler: rex
|
||||
learning_rate: 2.5e-7
|
||||
cosine_min_lr_ratio: 0.05
|
||||
weight_decay: 0.01
|
||||
max_grad_norm: 0.5
|
||||
logging_steps: 1
|
||||
|
||||
# Model optimization
|
||||
gradient_checkpointing: offload
|
||||
sdp_attention: true
|
||||
plugins:
|
||||
- axolotl.integrations.liger.LigerPlugin
|
||||
liger_rope: true
|
||||
liger_rms_norm: true
|
||||
liger_layer_norm: true
|
||||
liger_glu_activation: true
|
||||
liger_cross_entropy: true
|
||||
lora_mlp_kernel: false
|
||||
lora_qkv_kernel: false
|
||||
lora_o_kernel: false
|
||||
|
||||
# Debug config
|
||||
debug: true
|
||||
seed: 42
|
||||
|
||||
# Token config
|
||||
special_tokens:
|
||||
bos_token: "<|end_of_text|>"
|
||||
eos_token: "<|end_of_text|>"
|
||||
pad_token: "<|end_of_text|>"
|
||||
tokens:
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
<details><summary>Show Citations</summary>
|
||||
|
||||
```bib
|
||||
@misc{wolf2020huggingfacestransformersstateoftheartnatural,
|
||||
title={HuggingFace's Transformers: State-of-the-art Natural Language Processing},
|
||||
author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush},
|
||||
year={2020},
|
||||
eprint={1910.03771},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/1910.03771},
|
||||
}
|
||||
@misc{hu2021loralowrankadaptationlarge,
|
||||
title={LoRA: Low-Rank Adaptation of Large Language Models},
|
||||
author={Edward J. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
|
||||
year={2021},
|
||||
eprint={2106.09685},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2106.09685},
|
||||
}
|
||||
@misc{dettmers2023qloraefficientfinetuningquantized,
|
||||
title={QLoRA: Efficient Finetuning of Quantized LLMs},
|
||||
author={Tim Dettmers and Artidoro Pagnoni and Ari Holtzman and Luke Zettlemoyer},
|
||||
year={2023},
|
||||
eprint={2305.14314},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2305.14314},
|
||||
}
|
||||
@misc{dao2023flashattention2fasterattentionbetter,
|
||||
title={FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning},
|
||||
author={Tri Dao},
|
||||
year={2023},
|
||||
eprint={2307.08691},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2307.08691},
|
||||
}
|
||||
@misc{hsu2024ligerkernelefficienttriton,
|
||||
title={Liger Kernel: Efficient Triton Kernels for LLM Training},
|
||||
author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
|
||||
year={2024},
|
||||
eprint={2410.10989},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2410.10989},
|
||||
}
|
||||
@misc{chen2021rexrevisitingbudgetedtraining,
|
||||
title={REX: Revisiting Budgeted Training with an Improved Schedule},
|
||||
author={John Chen and Cameron Wolfe and Anastasios Kyrillidis},
|
||||
year={2021},
|
||||
eprint={2107.04197},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2107.04197},
|
||||
}
|
||||
@misc{luo2023cameconfidenceguidedadaptivememory,
|
||||
title={CAME: Confidence-guided Adaptive Memory Efficient Optimization},
|
||||
author={Yang Luo and Xiaozhe Ren and Zangwei Zheng and Zhuo Jiang and Xin Jiang and Yang You},
|
||||
year={2023},
|
||||
eprint={2307.02047},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2307.02047},
|
||||
}
|
||||
@misc{zamirai2021revisitingbfloat16training,
|
||||
title={Revisiting BFloat16 Training},
|
||||
author={Pedram Zamirai and Jian Zhang and Christopher R. Aberger and Christopher De Sa},
|
||||
year={2021},
|
||||
eprint={2010.06192},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2010.06192},
|
||||
}
|
||||
@misc{liang2025cautiousoptimizersimprovingtraining,
|
||||
title={Cautious Optimizers: Improving Training with One Line of Code},
|
||||
author={Kaizhao Liang and Lizhang Chen and Bo Liu and Qiang Liu},
|
||||
year={2025},
|
||||
eprint={2411.16085},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2411.16085},
|
||||
}
|
||||
@misc{xie2025sana15efficientscaling,
|
||||
title={SANA 1.5: Efficient Scaling of Training-Time and Inference-Time Compute in Linear Diffusion Transformer},
|
||||
author={Enze Xie and Junsong Chen and Yuyang Zhao and Jincheng Yu and Ligeng Zhu and Chengyue Wu and Yujun Lin and Zhekai Zhang and Muyang Li and Junyu Chen and Han Cai and Bingchen Liu and Daquan Zhou and Song Han},
|
||||
year={2025},
|
||||
eprint={2501.18427},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CV},
|
||||
url={https://arxiv.org/abs/2501.18427},
|
||||
}
|
||||
@misc{dallabetta2024fundussimpletousenewsscraper,
|
||||
title={Fundus: A Simple-to-Use News Scraper Optimized for High Quality Extractions},
|
||||
author={Max Dallabetta and Conrad Dobberstein and Adrian Breiding and Alan Akbik},
|
||||
year={2024},
|
||||
eprint={2403.15279},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2403.15279},
|
||||
}
|
||||
@misc{lambert2025tulu3pushingfrontiers,
|
||||
title={Tulu 3: Pushing Frontiers in Open Language Model Post-Training},
|
||||
author={Nathan Lambert and Jacob Morrison and Valentina Pyatkin and Shengyi Huang and Hamish Ivison and Faeze Brahman and Lester James V. Miranda and Alisa Liu and Nouha Dziri and Shane Lyu and Yuling Gu and Saumya Malik and Victoria Graf and Jena D. Hwang and Jiangjiang Yang and Ronan Le Bras and Oyvind Tafjord and Chris Wilhelm and Luca Soldaini and Noah A. Smith and Yizhong Wang and Pradeep Dasigi and Hannaneh Hajishirzi},
|
||||
year={2025},
|
||||
eprint={2411.15124},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2411.15124},
|
||||
}
|
||||
@misc{zheng2024lmsyschat1mlargescalerealworldllm,
|
||||
title={LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset},
|
||||
author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Tianle Li and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zhuohan Li and Zi Lin and Eric P. Xing and Joseph E. Gonzalez and Ion Stoica and Hao Zhang},
|
||||
year={2024},
|
||||
eprint={2309.11998},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2309.11998},
|
||||
}
|
||||
@misc{gosling2023pippapartiallysyntheticconversational,
|
||||
title={PIPPA: A Partially Synthetic Conversational Dataset},
|
||||
author={Tear Gosling and Alpin Dale and Yinhe Zheng},
|
||||
year={2023},
|
||||
eprint={2308.05884},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2308.05884},
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
Reference in New Issue
Block a user