From bd2680ddf571c618008f900d6a482bb941e9cc1c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 6 Aug 2026 06:27:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: reaperdoesntknow/Gemma-3-270m-Opus-Distil Source: Original Platform --- .gitattributes | 36 +++++ README.md | 349 +++++++++++++++++++++++++++++++++++++++++ config.json | 64 ++++++++ generation_config.json | 12 ++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 25 +++ 7 files changed, 492 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3d2222a --- /dev/null +++ b/README.md @@ -0,0 +1,349 @@ +--- +library_name: transformers +license: gemma +datasets: +- angrygiraffe/claude-opus-4.6-4.7-reasoning-8.7k +language: +- en +base_model: +- google/gemma-3-270m +pipeline_tag: text-generation +tags: +- text-generation +- causal-lm +- gemma +- gemma-3 +- gemma-3-270m +- reasoning +- instruction-tuning +- sparse-finetuning +- cixopt +- transformers +- bf16 +--- + +# CIx-Gemma-3-270M Reasoning SFT + +## Model Summary + +This model is a fine-tuned derivative of google/gemma-3-270m, adapted using the Convergent Intelligence sparse fine-tuning setup originally tested on Liquid Foundation Models. + +The checkpoint was trained on reasoning-style English examples from angrygiraffe/claude-opus-4.6-4.7-reasoning-8.7k using a targeted adaptation strategy and the custom CIxOpt optimizer framework. + +The goal of this model is to test whether a compact Gemma 3 270M backbone can be shaped toward reasoning-style text generation through selective parameter participation rather than broad full-model modification. + +This is an experimental research checkpoint intended for evaluation, local testing, optimizer research, and continued fine-tuning. + +## Base Model + +- Base model: google/gemma-3-270m +- Model family: Gemma 3 +- Approximate size: 270M parameters +- Task: Causal language modeling / text generation +- Language: English-focused fine-tuning +- Library: Hugging Face Transformers +- License: Gemma license + +If this checkpoint was instead trained from google/gemma-3-270m-it, update the base_model field accordingly. + +## Dataset + +Fine-tuning data: + +- angrygiraffe/claude-opus-4.6-4.7-reasoning-8.7k + +The dataset was processed into text-generation / chat-style training examples. Empty, malformed, or unusable samples were filtered before tokenization. + +Training used causal language modeling labels with padding masked using -100. + +## Training Method + +This model was trained using the same CIx sparse-adaptation setup used for LFM experiments. + +The training approach emphasized: + +text preserve the compact pretrained backbone adapt selected reasoning and response-shaping surfaces avoid unnecessary full-model disturbance use heterogeneous optimizer routing by parameter type + +## CIxOpt Optimizer + +Training used CIxOpt, a custom heterogeneous optimizer designed for architecture-aware routing. + +CIxOpt supports: + +- AdamW-style adaptive updates +- Lion-style sign momentum +- AdaMax-compatible routing +- Optional ASGD-style averaging +- Optional low-rank projected momentum +- Gradient centralization +- Decoupled weight decay +- Discrepancy-aware caution filtering for sign updates +- fp32 optimizer state for bf16/fp16 safety +- Parameter-name-aware routing + +The intended optimizer behavior is: + +text large projection matrices -> Lion-style sign momentum normalization / sensitive params -> AdamW-style updates embedding / lm-head surfaces -> conservative adaptive routing + +This makes the checkpoint useful for testing whether small models can be efficiently adapted with custom optimizer routing rather than full uniform AdamW updates. + +## Sparse Fine-Tuning Strategy + +The setup used sparse parameter participation rather than unrestricted full-model training. + +The intended adaptation pattern was: + +text freeze or reduce movement in lower representational structure train selected higher-level adaptation surfaces preserve base language structure where possible shape reasoning and response behavior through targeted updates + +This checkpoint should be treated as an experimental adaptation artifact, not a fully benchmarked general-purpose assistant. + +## Intended Use + +This model is intended for: + +- Research on compact Gemma fine-tuning +- CIxOpt optimizer experiments +- Small-model reasoning-style generation +- Local text-generation experiments +- Instruction-following and response-style studies +- Efficient adaptation research +- Continued fine-tuning and ablation testing +- Comparison against the base google/gemma-3-270m + +Potential use cases: + +- Technical explanation +- Lightweight reasoning experiments +- Prompt-response generation +- Local prototyping +- Small agent backbone testing +- Educational model behavior analysis + +## Out-of-Scope Use + +This model is not intended for high-stakes autonomous deployment. + +Do not use this model as the sole decision-maker for: + +- Medical diagnosis +- Legal judgment +- Financial decisions +- Emergency response +- Cyber offensive automation +- Personnel screening +- Surveillance or targeting decisions +- Critical infrastructure decisions +- Any setting requiring verified factual accuracy + +## Limitations + +This is an experimental fine-tuned checkpoint. Expected limitations include: + +- May hallucinate facts, dates, citations, or technical details +- May inherit limitations from the Gemma 3 270M base model +- May overproduce reasoning-style outputs +- May be sensitive to prompt format +- May repeat or drift during longer generations +- Has not been fully evaluated for factuality, safety, math, coding, or instruction-following +- Fine-tuning on reasoning-style data does not guarantee correct reasoning +- Sparse adaptation may change some behaviors unevenly while leaving others close to the base model +- Small model size limits world knowledge, reasoning depth, and robustness + +## Safety Notes + +Users should independently validate important outputs. + +Before deployment, additional evaluation is recommended: + +- Hallucination testing +- Bias and toxicity evaluation +- Refusal behavior testing +- Prompt-injection sensitivity testing +- Side-by-side comparison against the base model +- Domain-specific factuality testing +- Human review of outputs +- Guardrails for public-facing applications + +## Example Usage + +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM + +model_id = "YOUR_USERNAME/YOUR_MODEL_REPO" + +tokenizer = AutoTokenizer.from_pretrained(model_id) + +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + +prompt = "Explain why small language models are useful for edge reasoning experiments." + +inputs = tokenizer( + prompt, + return_tensors="pt", +).to(model.device) + +with torch.inference_mode(): + output = model.generate( + **inputs, + max_new_tokens=256, + do_sample=True, + temperature=0.7, + top_p=0.95, + repetition_penalty=1.05, + pad_token_id=tokenizer.eos_token_id, + ) + +print(tokenizer.decode(output[0], skip_special_tokens=True)) +``` +## Chat-Style Usage + +If the tokenizer provides a chat template: + +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM + +model_id = "YOUR_USERNAME/YOUR_MODEL_REPO" + +tokenizer = AutoTokenizer.from_pretrained(model_id) + +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +messages = [ + { + "role": "user", + "content": "Why is sparse fine-tuning useful for compact language models?" + } +] + +inputs = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, +).to(model.device) + +with torch.inference_mode(): + output = model.generate( + **inputs, + max_new_tokens=384, + do_sample=True, + temperature=0.7, + top_p=0.95, + repetition_penalty=1.05, + pad_token_id=tokenizer.eos_token_id, + ) + +generated = output[0][inputs["input_ids"].shape[-1]:] +print(tokenizer.decode(generated, skip_special_tokens=True)) +``` +## Suggested Generation Settings + +Balanced exploratory generation: + +```python +generation_config = { + "max_new_tokens": 384, + "do_sample": True, + "temperature": 0.7, + "top_p": 0.95, + "repetition_penalty": 1.05, +} +``` + +More deterministic generation: + +```python +generation_config = { + "max_new_tokens": 384, + "do_sample": False, +} +``` +For smaller models, shorter outputs are often more stable: + +```python +generation_config = { + "max_new_tokens": 128, + "do_sample": True, + "temperature": 0.6, + "top_p": 0.9, + "repetition_penalty": 1.1, +} +``` +## Training Configuration + +Approximate training configuration: +```txt +text base_model: google/gemma-3-270m +dataset: angrygiraffe/claude-opus-4.6-4.7-reasoning-8.7k +task: causal language modeling / reasoning-style SFT +optimizer: CIxOpt state_dtype: fp32 +optimizer state model_dtype: bf16 where supported +``` + +## Evaluation + +Formal benchmark results have not yet been added. + +Recommended evaluations: + +- Held-out perplexity +- Base model comparison against google/gemma-3-270m +- Short-form reasoning checks +- IFEval-style instruction-following tests +- Repetition and degeneration testing +- Human preference review +- Truthfulness / hallucination checks +- Prompt-format robustness testing +- CIxOpt vs AdamW ablation + +## Responsible Use + +This model may generate plausible but incorrect text. It should be used with human oversight. + +Developers should follow the Gemma usage terms and apply appropriate safety review before deploying the model in user-facing or operational settings. + +## Citation + +Base model: +```bib +bibtex @misc{google_gemma_3_270m, +title = {Gemma 3 270M}, +author = {Google DeepMind}, +publisher = {Hugging Face}, +year = {2025} +} +``` +Fine-tuning dataset: +```bib +bibtex @misc{angrygiraffe_reasoning_dataset, +title = {claude-opus-4.6-4.7-reasoning-8.7k}, +author = {angrygiraffe}, +publisher = {Hugging Face} +} +``` +## Author / Maintainer + +Fine-tuning and optimizer experimentation by: + +Convergent Intelligence LLC + +Research focus: AI systems, intelligence analysis, mathematical frameworks, optimizer design, and efficient model adaptation. + +## Disclaimer + +This model is provided for research and experimentation. It should not be treated as a verified expert system. Outputs require human review, especially in factual, technical, legal, medical, financial, operational, or safety-critical contexts. + diff --git a/config.json b/config.json new file mode 100644 index 0000000..0fbedb6 --- /dev/null +++ b/config.json @@ -0,0 +1,64 @@ +{ + "_sliding_window_pattern": 6, + "architectures": [ + "Gemma3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 2048, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "model_type": "gemma3_text", + "num_attention_heads": 4, + "num_hidden_layers": 18, + "num_key_value_heads": 1, + "pad_token_id": 1, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "original_max_position_embeddings": 32768, + "rope_theta": 1000000.0, + "rope_type": "dynamic" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "5.0.0", + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262144 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..b3ffb3f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 2, + "cache_implementation": "hybrid", + "do_sample": true, + "eos_token_id": [ + 1 + ], + "pad_token_id": 1, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.0.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..3c487e0 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26306c75ff49e452295c0225ef7d1106762542d2c08b143775320680fc411a33 +size 536223056 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..899af07 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497 +size 33384443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..9293bd5 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "extra_special_tokens": [], + "image_token": "", + "is_local": true, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}