From 4580a37a7ab00681d9635db69f72632d4ac24a9e Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 24 Jul 2026 18:17:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: guaran-ia/gntweets-lm Source: Original Platform --- .gitattributes | 36 ++++++++++ LICENSE | 17 +++++ README.md | 146 +++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 6 ++ config.json | 83 +++++++++++++++++++++++ generation_config.json | 11 ++++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 20 ++++++ 9 files changed, 325 insertions(+) create mode 100644 .gitattributes create mode 100644 LICENSE create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f18322 --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2026 Your Name + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . diff --git a/README.md b/README.md new file mode 100644 index 0000000..8c69801 --- /dev/null +++ b/README.md @@ -0,0 +1,146 @@ +--- +language: + - gn +license: "gpl-3.0" +tags: + - causal-lm + - text-generation + - guarani + - quality-assessment + - perplexity + - gemma2 +library_name: transformers +pipeline_tag: text-generation +--- + +# GNTweetsLM + +The model GNTweetsLM is intended to be used to validate the quality of Guarani text. It was +trained on a publicly available [corpus of tweets](https://github.com/sgongora27/giossa-gongora-guarani-2021/blob/main/Tweets_set/tweets_gn_october.csv) written in Guarani and Jopara (GΓ³ngora et al. [2021](https://aclanthology.org/2022.computel-1.16)). + +⚠️ Although the model is based on a transformer-based architecture (`Gemma2-9b-it`), **it was not developed as a generative tool** β€” its primary use is to compute the perplexity score of Guarani documents. Lower perplexity may indicate text that is more predictable by the model and more similar to the reference high-quality corpus. + +## πŸ“Œ Summary + +- **Model type:** Gemma2 For Causal LM +- **Base model:** `princeton-nlp/gemma-2-9b-it-SimPO` +- **Fine-tuning method:** Full fine-tuning (all model weights updated) +- **Primary task:** Perplexity computation +- **Dataset (HF):** `guaran-ia/gntweets` + +## πŸ—οΈ Model Details + +- **Architecture:** `Gemma2ForCausalLM` +- **Number of layers:** `42` +- **Hidden size:** `3584` +- **Attention heads:** `16` +- **Feedforward intermediate size:** `14336` +- **Vocabulary size:** `256000` +- **Maximum context length:** `8192` tokens +- **Precision:** `float16` +- **Tokenizer:** saved in this folder via `tokenizer.json` and `tokenizer_config.json` +- **Generation config:** saved in `generation_config.json` +- **Prompt template:** `chat_template.jinja` + +## βš™οΈ Training Details + +- **Batch size:** `1` +- **Gradient accumulation:** `1` +- **Learning rate:** `2e-5` +- **Weight decay:** `0.01` +- **Warmup steps:** `100` +- **Optimizer:** `paged_adamw_8bit` +- **Scheduler:** `linear` +- **Epochs:** `6` +- **Precision mode:** `bf16` +- **Gradient checkpointing:** enabled + +## πŸ—ƒοΈ Dataset and Preprocessing + +- **Split strategy:** train / validation / test +- **Sequence length used for tokenization:** `2048` +- **Train dataset size**: `936` records (`1916928` tokens) +- **Validation dataset size**: `117` records (`239616` tokens) +- **Test dataset size**: `117` records (`239616` tokens) +- **Tokenizer:** `princeton-nlp/gemma-2-9b-it-SimPO` +- **HF ID:** `guaran-ia/gntweets` + +## πŸš€ Usage + +Compute perplexity for a given Guarani text using the fine-tuned model: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch +import math + +model_id = 'guaran-ia/gntweets-lm' +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained(model_id) +model.eval() + +def perplexity(text: str) -> float: + inputs = tokenizer(text, return_tensors='pt') + with torch.no_grad(): + outputs = model(**inputs, labels=inputs['input_ids']) + loss = outputs.loss + return math.exp(loss.item()) + +text = "Your Guarani text here." +print(f"Perplexity: {perplexity(text):.4f}") +``` + +## Perplexity for long texts + +If input length exceeds the model/tokenizer maximum (8192 tokens), you can follow +the next recipe to compute perplexity over sliding chunks and average per-token loss. + +```python +import torch, math + +def perplexity_sliding(text: str, model, tokenizer, max_len: int = 8192, stride: int = 4096): + """Compute perplexity over long text by slicing into overlapping chunks. + + - `max_len` should be <= model.config.max_position_embeddings (8192). + - `stride` controls overlap; larger overlap gives smoother per-token averaging. + """ + enc = tokenizer(text, return_tensors='pt')['input_ids'][0] + n = enc.size(0) + if n == 0: + return float('nan') + + total_nll = 0.0 + total_tokens = 0 + start = 0 + while start < n: + end = min(start + max_len, n) + input_ids = enc[start:end].unsqueeze(0) + with torch.no_grad(): + outputs = model(input_ids, labels=input_ids) + # outputs.loss is the average NLL for the chunk + loss = outputs.loss.item() + chunk_len = end - start + total_nll += loss * chunk_len + total_tokens += chunk_len + if end == n: + break + start += stride + + avg_nll = total_nll / total_tokens + return math.exp(avg_nll) + +# Example usage: +text = open('some_guarani.txt', encoding='utf-8').read() +tokenizer.model_max_length = 8192 +print(f"Perplexity (sliding): {perplexity_sliding(text, model, tokenizer):.4f}") +``` + +## ❗ Limitations and Notes + +- The model may reflect biases present in the source corpus. +- License metadata is provided in this folder. + +## πŸ“œ License + +This model checkpoint and accompanying files are released under the GNU General Public License v3 (GPLv3). +See the `LICENSE` file in this directory for the full license text. \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..e07e39f --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,6 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] | trim + ' + +' %}{% set messages = messages[1:] %}{% else %}{% set system_message = '' %}{% endif %}{% for message in messages %}{% if loop.index0 == 0 %}{% set content = system_message + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + content | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..bfc217e --- /dev/null +++ b/config.json @@ -0,0 +1,83 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "pretraining_tp": 1, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "sliding_window": 4096, + "sliding_window_size": 4096, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "use_bidirectional_attention": null, + "use_cache": false, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f492da7 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 1, + 107 + ], + "pad_token_id": 0, + "transformers_version": "5.9.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..39f5371 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43f7fd2a7f6209ee94d5f527160b8e2ba04d9833469152ca49f51d41d86f4b0 +size 18483466536 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..69f24e6 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91600688617d8ac596b3681d04eae42dd11573018ab577755208fa81b3c6a529 +size 34356164 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..95971a5 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,20 @@ +{ + "backend": "tokenizers", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": [ + "", + "" + ], + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 2048, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}