commit 1ed5e992a4d34a501c182bf93cb1ea3ecdf673ae Author: ModelHub XC Date: Fri Jul 24 18:13:13 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: guaran-ia/coreguapa-lm Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f18322 --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2026 Your Name + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . diff --git a/README.md b/README.md new file mode 100644 index 0000000..62c7b0e --- /dev/null +++ b/README.md @@ -0,0 +1,147 @@ +--- +language: + - gn +license: "gpl-3.0" +tags: + - causal-lm + - text-generation + - guarani + - quality-assessment + - perplexity + - gemma2 +library_name: transformers +pipeline_tag: text-generation +--- + +# CoreguapaLM + +This model is intended to be used to validate the quality of Guarani text. It was +trained on the [COREGUAPA corpus](https://corpus.spl.gov.py), a restricted dataset manually compiled and curated by the Paraguayan Secretary of Linguistic Policies. The corpus contains high-quality documents, including primarily copyrighted materials. + +⚠️ Although the model is based on a transformer-based architecture (`Gemma2-9b-it`), **it was not developed as a generative tool** — its primary use is to compute the perplexity score of Guarani documents. Lower perplexity may indicate text that is more predictable by the model and more similar to the reference high-quality corpus. + +## 📌 Summary + +- **Model type:** Gemma2 For Causal LM +- **Base model:** `princeton-nlp/gemma-2-9b-it-SimPO` +- **Fine-tuning method:** Full fine-tuning (all model weights updated) +- **Dataset:** [COREGUAPA](https://corpus.spl.gov.py) (restricted) +- **Primary task:** Perplexity computation + +## 🏗️ Model Details + +- **Architecture:** `Gemma2ForCausalLM` +- **Number of layers:** `42` +- **Hidden size:** `3584` +- **Attention heads:** `16` +- **Feedforward intermediate size:** `14336` +- **Vocabulary size:** `256000` +- **Maximum context length:** `8192` tokens +- **Precision:** `float16` +- **Tokenizer:** saved in this folder via `tokenizer.json` and `tokenizer_config.json` +- **Generation config:** saved in `generation_config.json` +- **Prompt template:** `chat_template.jinja` + +## ⚙️ Training Details + +- **Batch size:** `1` +- **Gradient accumulation:** `1` +- **Learning rate:** `2e-5` +- **Weight decay:** `0.01` +- **Warmup steps:** `100` +- **Optimizer:** `paged_adamw_8bit` +- **Scheduler:** `linear` +- **Epochs:** `6` +- **Precision mode:** `bf16` +- **Gradient checkpointing:** enabled + +## 🗃️ Dataset and Preprocessing + +- **Raw source file:** private +- **Split strategy:** train / validation / test +- **Sequence length used for tokenization:** `2048` +- **Train dataset size**: `892` records (`1826816` tokens) +- **Validation dataset size**: `111` records (`227328` tokens) +- **Test dataset size**: `112` records (`229376` tokens) +- **Tokenizer:** `princeton-nlp/gemma-2-9b-it-SimPO` + +## 🚀 Usage + +Compute perplexity for a given Guarani text using the fine-tuned model: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch +import math + +model_id = 'guaran-ia/coreguapa-lm' +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained(model_id) +model.eval() + +def perplexity(text: str) -> float: + inputs = tokenizer(text, return_tensors='pt') + with torch.no_grad(): + outputs = model(**inputs, labels=inputs['input_ids']) + loss = outputs.loss + return math.exp(loss.item()) + +text = "Your Guarani text here." +print(f"Perplexity: {perplexity(text):.4f}") +``` + +## Perplexity for long texts + +If input length exceeds the model/tokenizer maximum (8192 tokens), you can follow +the next recipe to compute perplexity over sliding chunks and average per-token loss. + +```python +import torch, math + +def perplexity_sliding(text: str, model, tokenizer, max_len: int = 8192, stride: int = 4096): + """Compute perplexity over long text by slicing into overlapping chunks. + + - `max_len` should be <= model.config.max_position_embeddings (8192). + - `stride` controls overlap; larger overlap gives smoother per-token averaging. + """ + enc = tokenizer(text, return_tensors='pt')['input_ids'][0] + n = enc.size(0) + if n == 0: + return float('nan') + + total_nll = 0.0 + total_tokens = 0 + start = 0 + while start < n: + end = min(start + max_len, n) + input_ids = enc[start:end].unsqueeze(0) + with torch.no_grad(): + outputs = model(input_ids, labels=input_ids) + # outputs.loss is the average NLL for the chunk + loss = outputs.loss.item() + chunk_len = end - start + total_nll += loss * chunk_len + total_tokens += chunk_len + if end == n: + break + start += stride + + avg_nll = total_nll / total_tokens + return math.exp(avg_nll) + +# Example usage: +text = open('some_guarani.txt', encoding='utf-8').read() +tokenizer.model_max_length = 8192 +print(f"Perplexity (sliding): {perplexity_sliding(text, model, tokenizer):.4f}") +``` + +## ❗ Limitations and Notes + +- The training data is drawn from a private Guarani corpus. +- The model may reflect biases present in the source corpus. +- License metadata is provided in this folder. + +## 📜 License + +This model checkpoint and accompanying files are released under the GNU General Public License v3 (GPLv3). +See the `LICENSE` file in this directory for the full license text. \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..e07e39f --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,6 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] | trim + ' + +' %}{% set messages = messages[1:] %}{% else %}{% set system_message = '' %}{% endif %}{% for message in messages %}{% if loop.index0 == 0 %}{% set content = system_message + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + content | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..bfc217e --- /dev/null +++ b/config.json @@ -0,0 +1,83 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "pretraining_tp": 1, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "sliding_window": 4096, + "sliding_window_size": 4096, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "use_bidirectional_attention": null, + "use_cache": false, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f492da7 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 1, + 107 + ], + "pad_token_id": 0, + "transformers_version": "5.9.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..9bfd712 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977dd25056146bc20dd4dc8c63de7330a2e201ecf0dff3534573dfd326dcbb1b +size 18483466536 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..69f24e6 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91600688617d8ac596b3681d04eae42dd11573018ab577755208fa81b3c6a529 +size 34356164 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..95971a5 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,20 @@ +{ + "backend": "tokenizers", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": [ + "", + "" + ], + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 2048, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}