commit 6265b97142cc84f9dddd68a03342f2f539021adc Author: ModelHub XC Date: Fri Jul 24 05:17:14 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: rodin-llm/rodin-1b-instruct Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9340d51 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +rodin_frenchbench_radar.png filter=lfs diff=lfs merge=lfs -text +rodin-1b-instruct-f16.gguf filter=lfs diff=lfs merge=lfs -text +rodin-1b-instruct-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +rodin-1b-instruct-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4ecdbe --- /dev/null +++ b/README.md @@ -0,0 +1,325 @@ +--- +license: apache-2.0 +language: +- fr +library_name: transformers +pipeline_tag: text-generation +tags: +- french +- llm +- from-scratch +- rodin +- causal-lm +- llama +- instruct +- conversational +- chatml +base_model: rodin-llm/rodin-1b +datasets: +- HPLT +- cc100 +- wikipedia +- wikisource +- PleIAs/common_corpus +model-index: +- name: rodin-1b-instruct + results: + - task: + type: text-generation + dataset: + name: FrenchBench + type: french_bench + metrics: + - type: acc + value: 0.756 + name: Grammar (acc) + - type: acc_norm + value: 0.309 + name: ARC challenge (acc_norm) +--- + + + +# RODIN-1B-Instruct + +**A French conversational model trained from scratch — solo, on consumer-grade hardware.** +**Un modele conversationnel francais entraine de zero — en solo, sur du materiel grand public.** + +**[🇬🇧 English](#english)** · **[🇫🇷 Français](#français)** + +> This is the **instruction-tuned, conversational** model. For the base pretrained model, see [`rodin-llm/rodin-1b`](https://huggingface.co/rodin-llm/rodin-1b). +> Ceci est le modele **conversationnel** (instruction-tune). Pour le modele de base, voir [`rodin-llm/rodin-1b`](https://huggingface.co/rodin-llm/rodin-1b). + +--- + +> 💻 **Full source code / Code source complet** : [github.com/rodin-llm/rodin](https://github.com/rodin-llm/rodin) +> The complete pipeline — data, tokenizer, pretraining, SFT, export, spot orchestration. +> Le pipeline complet — données, tokenizer, pré-entraînement, SFT, export, orchestration spot. + +--- + + +## 🇬🇧 English + +### Overview + +RODIN-1B-Instruct is the instruction-tuned, conversational variant of [`rodin-1b`](https://huggingface.co/rodin-llm/rodin-1b) — a 1.24-billion-parameter, French-only causal language model trained **from scratch** by a single person. *RODIN* stands for **R**esearch **O**pen **D**eep **I**ntelligence **N**atively-french. + +It was produced by full supervised fine-tuning (SFT) of the base model on French ChatML examples, and speaks French fluently in a conversational register. GGUF quantizations (F16, Q8_0, Q4_K_M) are provided for llama.cpp, Ollama and LM Studio, including CPU-only hardware. + +### Why this model exists + +The goal was never to compete with large, well-funded French models on raw benchmark scores. For scale: comparable French open-source efforts were trained on **3,000 billion tokens** using hundreds of H100 GPUs on national supercomputers. RODIN was trained on **32 billion tokens**, by one person, on a rented spot B200 instance plus a single RTX 3090 for local iteration and SFT. + +The value of RODIN is **pedagogical and demonstrative**: it shows what one motivated individual can build from scratch, end to end, with a small budget — documented honestly, limitations included. + +### Chat format + +RODIN-1B-Instruct uses the **ChatML** format: + +``` +<|im_start|>user +{message}<|im_end|> +<|im_start|>assistant +{response}<|im_end|> +``` + +Stop token: `<|im_end|>`. The chat template is bundled in `tokenizer_config.json`, so `apply_chat_template` and runtimes like Ollama/LM Studio handle it automatically. + +### Model description + +| Property | Value | +|---|---| +| Parameters | 1.238 B | +| Architecture | LLaMA-style (RoPE, RMSNorm, SwiGLU, causal SDPA attention) | +| Hidden size / Layers / Heads | 2048 / 22 / 16 | +| FFN intermediate size | 5461 | +| Vocabulary | 64,000 (custom SentencePiece BPE, French) | +| Context length | 2048 tokens | +| Chat format | ChatML | +| Training dtype | bfloat16 | + +### SFT data + +Full fine-tune on **5,000 ChatML examples** generated locally with a Qwen model, covering varied French registers (narrative, summarization, rewriting, simple factual, etc.) plus a small politeness set. **Math and legal tasks were deliberately excluded** — fine-tuning a small model on content it gets wrong would only teach it to hallucinate with authority. + +The underlying pretraining data is documented on the [base model card](https://huggingface.co/rodin-llm/rodin-1b). + +### Evaluation — FrenchBench (base vs instruct) + +Evaluated with [EleutherAI's lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness), `french_bench`, **3-shot, full test sets**, instruct evaluated with `--apply_chat_template` (OrangeSum excluded due to a `datasets` incompatibility). + +| Task | Metric | Base | Instruct | +|---|---|---:|---:| +| Vocabulary | acc | 0.773 | 0.756 | +| Grammar | acc | 0.765 | 0.756 | +| Reading comprehension | acc | 0.606 | 0.549 | +| BoolQA | acc | 0.573 | 0.562 | +| Topic-based NLI | acc_norm | 0.498 | 0.378 | +| HellaSwag | acc_norm | 0.424 | 0.429 | +| ARC challenge | acc | 0.220 | **0.257** | +| ARC challenge | acc_norm | 0.265 | **0.309** | +| XNLI | acc | 0.333 | 0.323 | +| Trivia | f1 | 0.245 | 0.142 | +| Trivia | is_included | 0.190 | **0.213** | + +![FrenchBench per-task scores](./rodin_frenchbench_bars.png) + +![FrenchBench category profile](./rodin_frenchbench_radar.png) + +**The SFT effect — a key observation.** On Trivia, `exact`/`f1` *drop* while `is_included` *rises* (0.19 → 0.21). This is not a regression: the base model answered bluntly ("Paris"); after SFT, the model wraps answers in full sentences ("The capital is Paris."), so the correct answer is still present (`is_included` ↑) but no longer an exact string match. The SFT made the model conversational, exactly as intended. Note also that ARC challenge improves with SFT (0.27 → 0.31 acc_norm), while core linguistic competence (grammar, vocabulary) is preserved. + +### Intended use & limitations + +**Intended use.** French conversation, summarization, rewriting, simple factual Q&A, education and experimentation. A demonstrator, not a production system. + +**Limitations.** + +- **Size.** At 1.24B parameters, world knowledge and reasoning are limited; it hallucinates on precise facts. Do not rely on it for factual accuracy. +- **No safety tuning.** The SFT contained **no refusal or safety data**. The model has not been trained to refuse harmful, biased, or inappropriate requests, and may produce such content. It is **not** suitable for unsupervised or public-facing deployment. +- **French only**, context limited to 2048 tokens. +- **Math, code, legal:** deliberately excluded from SFT; expect weak performance. + +### Usage + +#### Transformers + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_id = "rodin-llm/rodin-1b-instruct" +tok = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16).cuda() + +messages = [{"role": "user", "content": "Explique-moi en deux phrases ce qu'est la photosynthèse."}] +inputs = tok.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").cuda() +out = model.generate(inputs, max_new_tokens=200, temperature=0.7, top_p=0.9) +print(tok.decode(out[0], skip_special_tokens=True)) +``` + +#### Ollama (GGUF) + +``` +ollama run rodin-1b +``` + +### Training procedure (summary) + +- **Base pretraining:** 478,000 steps, ~32B French tokens, rented spot B200 (bfloat16). See [base model](https://huggingface.co/rodin-llm/rodin-1b). +- **SFT:** full fine-tune on 5,000 ChatML examples, single RTX 3090, loss masked on assistant responses only. +- **Export:** custom `RodinLM` → HuggingFace `LlamaForCausalLM` → GGUF (F16, Q8_0, Q4_K_M). + +### License + +Released under the **Apache 2.0** license. + +### Acknowledgements & transparency + +Carried out by one person, with **AI assistance openly acknowledged** throughout. Thanks to EleutherAI (evaluation), the HPLT and Pleias teams (data), Wikimedia, and the llama.cpp / Ollama / LM Studio projects. + +```bibtex +@misc{rodin1binstruct2026, + title = {RODIN-1B-Instruct: A French Conversational Model Trained From Scratch}, + author = {RODIN}, + year = {2026}, + howpublished = {\url{https://huggingface.co/rodin-llm/rodin-1b-instruct}} +} +``` + +--- + + +## 🇫🇷 Français + +### Présentation + +RODIN-1B-Instruct est la variante conversationnelle (instruction-tunée) de [`rodin-1b`](https://huggingface.co/rodin-llm/rodin-1b) — un modèle de langage causal **uniquement francophone**, de 1,24 milliard de paramètres, entraîné **de zéro** par une seule personne. *RODIN* signifie **R**esearch **O**pen **D**eep **I**ntelligence **N**atively-french. + +Il a été produit par fine-tuning supervisé complet (SFT) du modèle de base sur des exemples ChatML français, et parle un français fluide dans un registre conversationnel. Des quantisations GGUF (F16, Q8_0, Q4_K_M) sont fournies pour llama.cpp, Ollama et LM Studio, y compris sur matériel CPU uniquement. + +### Pourquoi ce modèle existe + +L'objectif n'a jamais été de rivaliser en score brut avec les gros modèles français bien financés. Pour situer : des projets français open source comparables ont été entraînés sur **3 000 milliards de tokens** avec des centaines de GPU H100 sur des supercalculateurs nationaux. RODIN a été entraîné sur **32 milliards de tokens**, par une seule personne, sur une instance B200 spot louée et une seule RTX 3090 pour l'itération locale et le SFT. + +La valeur de RODIN est **pédagogique et démonstrative** : il montre ce qu'une personne motivée peut construire de zéro, de bout en bout, avec un petit budget — documenté honnêtement, limites comprises. + +### Format de chat + +RODIN-1B-Instruct utilise le format **ChatML** : + +``` +<|im_start|>user +{message}<|im_end|> +<|im_start|>assistant +{réponse}<|im_end|> +``` + +Token d'arrêt : `<|im_end|>`. Le chat template est embarqué dans `tokenizer_config.json`, donc `apply_chat_template` et les runtimes comme Ollama/LM Studio le gèrent automatiquement. + +### Description du modèle + +| Propriété | Valeur | +|---|---| +| Paramètres | 1,238 milliard | +| Architecture | Style LLaMA (RoPE, RMSNorm, SwiGLU, attention causale SDPA) | +| Dimension / Couches / Têtes | 2048 / 22 / 16 | +| Dimension FFN | 5461 | +| Vocabulaire | 64 000 (BPE SentencePiece maison, français) | +| Longueur de contexte | 2048 tokens | +| Format de chat | ChatML | +| Précision d'entraînement | bfloat16 | + +### Données de SFT + +Full fine-tune sur **5 000 exemples ChatML** générés localement avec un modèle Qwen, couvrant des registres français variés (narratif, résumé, réécriture, factuel simple, etc.) plus un petit jeu de politesse. **Les maths et le juridique ont été délibérément exclus** — fine-tuner un petit modèle sur du contenu qu'il rate ne ferait que lui apprendre à halluciner avec autorité. + +Les données de pré-entraînement sous-jacentes sont documentées sur la [carte du modèle de base](https://huggingface.co/rodin-llm/rodin-1b). + +### Évaluation — FrenchBench (base vs instruct) + +Évalué avec [lm-evaluation-harness d'EleutherAI](https://github.com/EleutherAI/lm-evaluation-harness), `french_bench`, **3-shot, jeux de test complets**, instruct évalué avec `--apply_chat_template` (OrangeSum exclu pour incompatibilité `datasets`). + +| Tâche | Métrique | Base | Instruct | +|---|---|---:|---:| +| Vocabulaire | acc | 0,773 | 0,756 | +| Grammaire | acc | 0,765 | 0,756 | +| Compréhension écrite | acc | 0,606 | 0,549 | +| BoolQA | acc | 0,573 | 0,562 | +| NLI thématique | acc_norm | 0,498 | 0,378 | +| HellaSwag | acc_norm | 0,424 | 0,429 | +| ARC challenge | acc | 0,220 | **0,257** | +| ARC challenge | acc_norm | 0,265 | **0,309** | +| XNLI | acc | 0,333 | 0,323 | +| Trivia | f1 | 0,245 | 0,142 | +| Trivia | is_included | 0,190 | **0,213** | + +![Scores FrenchBench par tâche](./rodin_frenchbench_bars.png) + +![Profil FrenchBench par catégorie](./rodin_frenchbench_radar.png) + +**L'effet du SFT — une observation clé.** Sur Trivia, `exact`/`f1` *baissent* alors que `is_included` *monte* (0,19 → 0,21). Ce n'est pas une régression : le modèle de base répondait sèchement (« Paris ») ; après SFT, il enrobe ses réponses dans des phrases complètes (« La capitale est Paris. »), donc la bonne réponse reste présente (`is_included` ↑) mais ce n'est plus une correspondance exacte. Le SFT a rendu le modèle conversationnel, exactement comme prévu. À noter aussi : ARC challenge s'améliore avec le SFT (0,27 → 0,31 acc_norm), tandis que la compétence linguistique de base (grammaire, vocabulaire) est préservée. + +### Usage prévu & limites + +**Usage prévu.** Conversation française, résumé, réécriture, questions-réponses factuelles simples, éducation et expérimentation. Un démonstrateur, pas un système de production. + +**Limites.** + +- **Taille.** À 1,24B paramètres, connaissances et raisonnement limités ; hallucine sur les faits précis. Ne vous y fiez pas pour l'exactitude factuelle. +- **Aucun safety tuning.** Le SFT ne contenait **aucune donnée de refus ou de sécurité**. Le modèle n'a pas été entraîné à refuser des requêtes nuisibles, biaisées ou inappropriées, et peut produire de tels contenus. Il n'est **pas** adapté à un déploiement public ou non supervisé. +- **Français uniquement**, contexte limité à 2048 tokens. +- **Maths, code, juridique :** délibérément exclus du SFT ; performances faibles attendues. + +### Utilisation + +#### Transformers + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_id = "rodin-llm/rodin-1b-instruct" +tok = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16).cuda() + +messages = [{"role": "user", "content": "Explique-moi en deux phrases ce qu'est la photosynthèse."}] +inputs = tok.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").cuda() +out = model.generate(inputs, max_new_tokens=200, temperature=0.7, top_p=0.9) +print(tok.decode(out[0], skip_special_tokens=True)) +``` + +#### Ollama (GGUF) + +``` +ollama run rodin-1b +``` + +### Procédure d'entraînement (résumé) + +- **Pré-entraînement de base :** 478 000 steps, ~32B tokens français, B200 spot louée (bfloat16). Voir le [modèle de base](https://huggingface.co/rodin-llm/rodin-1b). +- **SFT :** full fine-tune sur 5 000 exemples ChatML, une seule RTX 3090, loss masquée sur les réponses de l'assistant uniquement. +- **Export :** `RodinLM` maison → `LlamaForCausalLM` HuggingFace → GGUF (F16, Q8_0, Q4_K_M). + +### Licence + +Publié sous licence **Apache 2.0**. + +### Remerciements & transparence + +Mené par une seule personne, avec une **assistance IA assumée et transparente** tout du long. Merci à EleutherAI (évaluation), aux équipes HPLT et Pleias (données), à Wikimedia, et aux projets llama.cpp / Ollama / LM Studio. + +```bibtex +@misc{rodin1binstruct2026, + title = {RODIN-1B-Instruct: A French Conversational Model Trained From Scratch}, + author = {RODIN}, + year = {2026}, + howpublished = {\url{https://huggingface.co/rodin-llm/rodin-1b-instruct}} +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..df6e018 --- /dev/null +++ b/config.json @@ -0,0 +1,25 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "model_type": "llama", + "hidden_size": 2048, + "intermediate_size": 5632, + "num_hidden_layers": 22, + "num_attention_heads": 16, + "num_key_value_heads": 16, + "head_dim": 128, + "max_position_embeddings": 2048, + "vocab_size": 64000, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "hidden_act": "silu", + "tie_word_embeddings": true, + "attention_bias": false, + "mlp_bias": false, + "bos_token_id": 2, + "eos_token_id": 3, + "pad_token_id": 0, + "torch_dtype": "bfloat16", + "transformers_version": "4.44.0" +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..e6d5ba1 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "bos_token_id": 2, + "eos_token_id": 3, + "pad_token_id": 0, + "max_length": 2048 +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..b68ae0e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d61199579df6ac87749a087f5082ff4741c58e69067de489d12b30d3082f546e +size 2785225384 diff --git a/rodin-1b-instruct-Q4_K_M.gguf b/rodin-1b-instruct-Q4_K_M.gguf new file mode 100644 index 0000000..3e299f9 --- /dev/null +++ b/rodin-1b-instruct-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ae0e567b3b30ffef57eeb326d8d41e05b78134ff2fb65d1ef10fc393b865fe3 +size 859576800 diff --git a/rodin-1b-instruct-Q8_0.gguf b/rodin-1b-instruct-Q8_0.gguf new file mode 100644 index 0000000..9a339f0 --- /dev/null +++ b/rodin-1b-instruct-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf578bf536a533af95b9ecb78aa9d3a59c995967e3d97a11adb0628b19500d3a +size 1481488864 diff --git a/rodin-1b-instruct-f16.gguf b/rodin-1b-instruct-f16.gguf new file mode 100644 index 0000000..2674e5b --- /dev/null +++ b/rodin-1b-instruct-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9bc827e9a9b54118e57ee79a7c26920b9080d9919dbd691b76a7e88f1ec0340 +size 2786965984 diff --git a/rodin_frenchbench_bars.png b/rodin_frenchbench_bars.png new file mode 100644 index 0000000..632d7cd Binary files /dev/null and b/rodin_frenchbench_bars.png differ diff --git a/rodin_frenchbench_radar.png b/rodin_frenchbench_radar.png new file mode 100644 index 0000000..d35a636 --- /dev/null +++ b/rodin_frenchbench_radar.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f46edee5e621e7d41d06eed4f4057ef8d4155d634b6f7fbb1582cac845652c02 +size 114687 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..d3a99da --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,6 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "", + "pad_token": "" +} \ No newline at end of file diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..f2d18af --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b6c6723ff645ef41c5b8e2af3c1580f6e1fa0d8885bc228fa96c4062753ce22 +size 1118849 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..cd6ad4b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,47 @@ +{ + "tokenizer_class": "LlamaTokenizer", + "model_max_length": 2048, + "add_bos_token": true, + "add_eos_token": false, + "clean_up_tokenization_spaces": false, + "legacy": true, + "bos_token": "", + "eos_token": "", + "unk_token": "", + "pad_token": "", + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant\n'}}{% endif %}" +} \ No newline at end of file