commit 44859c2222f007f9171777669ed8d1d84ce71693 Author: ModelHub XC Date: Tue Jun 30 11:04:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: occ-ai/OCC-RAG-1.7B Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..fca8265 --- /dev/null +++ b/README.md @@ -0,0 +1,166 @@ +--- +license: mit +language: +- en +- ru +library_name: transformers +pipeline_tag: text-generation +base_model: Qwen/Qwen3-1.7B-Base +tags: +- rag +- faithful-qa +- occ +--- + +# OCC-RAG-1.7B + +

+ OCC-RAG +

+ +

+ GitHub  |  + Technical Report  |  + Cloud +

+ +**OCC-RAG-1.7B** is a 1.7B-parameter small language model specialized for **faithful, context-grounded question answering**. Along with OCC-RAG-0.6B, it belongs to the first generation of **Optimal Cognitive Core (OCC)** specialized reasoning models. Given a question and a set of sources, it produces a structured reasoning trace with explicit source citations, decides whether the context actually supports an answer, and either answers from the context or abstains. + +Despite its size, OCC-RAG-1.7B matches or exceeds general-purpose models **2–6× larger** on multi-hop reasoning, faithfulness, and refusal benchmarks, and attains the best faithfulness across all evaluated scales (up to 32B). It is mid-trained from `Qwen/Qwen3-1.7B-Base` on a large synthetic corpus of multi-context, multi-hop QA with citation-anchored reasoning traces. + +## Highlights + +- **Faithful by design** — answers only from the supplied context; achieves the best faithfulness (lowest memorization ratio) across all evaluated scales, including 32B models. +- **Calibrated abstention** — outputs `Not enough information` when the context does not support an answer. +- **Structured, citable reasoning** — every answer comes with a transparent trace (query analysis → source analysis → reasoning → status → answer) that cites sources by id. +- **Compact** — a small model that delivers chain-of-thought-level transparency at a fraction of full thinking-mode inference cost. + +## Model overview + +OCC-RAG-1.7B is mid-trained from `Qwen/Qwen3-1.7B-Base` via supervised fine-tuning on a synthetic corpus of **~3.25M QA pairs** (~2.78M single-hop, ~262k multi-hop single-context, ~165k multi-hop multi-context, and ~43k abstain examples), distilled from a larger teacher with citation-anchored reasoning traces. Multi-hop and multi-context subsets are oversampled to emphasize compositional reasoning. The prompt/response format is identical at training and inference time, so no train–test mismatch is introduced. + +## Evaluation + +Evaluated across multi-hop reasoning (HotpotQA, MuSiQue, TAT-QA), faithfulness (ConFiQA), and refusal (MuSiQue-Un). In-Acc = the gold answer appears as a substring of the prediction; F1 = token-level overlap between prediction and gold answer; M_R = memorization ratio (lower = more faithful); R-Acc = refusal accuracy. + +| Model | HotpotQA
In-Acc | MuSiQue
In-Acc | TAT-QA
F1 | ConFiQA
In-Acc | ConFiQA
M_R ↓ | MuSiQue-Un
R-Acc | +|---|---|---|---|---|---|---| +| gemma-3-4b-it | 55.8 | 30.1 | 65.3 | 69.8 | 8.9 | 55.8 | +| Qwen3-1.7B (think) | 60.9 | 30.7 | 74.8 | 70.4 | 8.3 | 82.8 | +| Qwen3-4B (think) | 67.1 | 41.5 | 79.1 | 74.1 | 7.5 | 84.0 | +| Pleias-RAG-1.2B | 48.5 | 15.0 | 8.4 | 37.3 | 25.3 | 21.9 | +| OCC-RAG-0.6B | 57.6 | 36.6 | 75.0 | 79.9 | 5.2 | 86.9 | +| **OCC-RAG-1.7B** | **60.9** | **38.2** | **81.0** | **81.4** | **5.0** | **87.2** | + +OCC-RAG-1.7B closes the gap with Qwen3-4B (thinking) on multi-hop reasoning while attaining the **best faithfulness** (highest ConFiQA In-Acc, lowest M_R) across all evaluated scales, and refusal accuracy on par with 8B+ models. Mid-training reduces the memorization ratio from 12.7 (8.3 in thinking mode) for Qwen3-1.7B down to 5.0. + +## Input / output format + +OCC-RAG uses a **structured prompt format with special tokens**. The question is wrapped in `<|query_start|> … <|query_end|>` and each source in `<|source_start|><|source_id|>N … <|source_end|>`. + +The response is split into five sections, each delimited by special tokens: + +| Section | Tokens | Content | +|---|---|---| +| Query analysis | `<\|query_analysis_start\|> … <\|query_analysis_end\|>` | Decomposes the question into what must be found. | +| Source analysis | `<\|source_analysis_start\|> … <\|source_analysis_end\|>` | Assesses each source's relevance, citing by `<\|source_id\|>N`. | +| Reasoning | `<\|reasoning_start\|> … <\|reasoning_end\|>` | Composes evidence across sources into a multi-hop chain. | +| Status | `<\|status_start\|> … <\|status_end\|>` | `ANSWERABLE` / `UNANSWERABLE` verdict. | +| Answer | `<\|answer_start\|> … <\|answer_end\|>` | The final answer span, or the refusal phrase. | + +## Quickstart (Transformers) + +The chat template accepts a `documents=` kwarg and emits the structural tokens for the query and sources automatically — pass the user message as plain text and the sources as a list of dicts. + +```python +import re +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL = "occ-ai/OCC-RAG-1.7B" + +tokenizer = AutoTokenizer.from_pretrained(MODEL) +model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype="auto", device_map="auto") + +question = "Which country is the inventor of the telephone, Alexander Graham Bell, buried in?" +documents = [ + {"text": "Alexander Graham Bell was a Scottish-born inventor best known for patenting the first practical telephone."}, + {"text": "Bell died on August 2, 1922, at his estate Beinn Bhreagh, near Baddeck, Nova Scotia, and was buried there."}, + {"text": "Nova Scotia is a province on the east coast of Canada."}, +] + +text = tokenizer.apply_chat_template( + [{"role": "user", "content": question}], + documents=documents, + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, +) + +# Alternative: assemble the structural tokens yourself. +# +# query_start, query_end = "<|query_start|>", "<|query_end|>" +# source_start, source_end, source_id = "<|source_start|>", "<|source_end|>", "<|source_id|>" +# +# def build_user_content(question, sources): +# content = f"{query_start}{question}{query_end}\n" +# for i, s in enumerate(sources, start=1): +# content += f"{source_start}{source_id}{i} {s}{source_end}\n" +# return content +# +# messages = [{"role": "user", "content": build_user_content(question, [d["text"] for d in documents])}] +# text = tokenizer.apply_chat_template( +# messages, tokenize=False, add_generation_prompt=True, enable_thinking=False +# ) + +inputs = tokenizer([text], return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=2048) +response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=False) +print(response) + +m = re.search(r"<\|answer_start\|>(.*)", response, re.DOTALL) +print("Answer:", m.group(1).strip() if m else "") # -> Canada +``` + +> [!NOTE] +> We recommend greedy decoding (`do_sample=False`), which is the training/evaluation default and is baked into `generation_config.json`. Qwen3's default sampling parameters ([best practices](https://huggingface.co/Qwen/Qwen3-1.7B#best-practices)) also work fine. + +## Deployment + +OCC-RAG-1.7B is a standard Qwen3 causal LM and is compatible with vLLM, SGLang, and other Transformers-based serving stacks. With only 1.7B parameters, it can be readily deployed in constrained infrastructure, including desktop systems running on CPU RAM. When serving, keep `skip_special_tokens=False` if you need to parse the structural tokens out of the raw output. + +Compatible runtimes: + +- `transformers>=5.5.1` +- `vllm>=0.19.1` +- `sglang>=0.5.11` + +When using an OpenAI-compatible server, the `documents=` kwarg is reachable from the client via `chat_template_kwargs`: + +```python +client.chat.completions.create( + model="occ-ai/OCC-RAG-1.7B", + messages=[{"role": "user", "content": question}], + extra_body={"chat_template_kwargs": {"documents": documents}}, +) +``` + +## Limitations + +- **Context-grounded only.** The model is trained to answer from the supplied sources and to ignore parametric knowledge. It is not a general-purpose chat or knowledge model. +- **Reasoning depth.** Training and evaluation are capped at three-hop reasoning; longer chains are out of distribution. + +## Citation + +If you find our work helpful, feel free to give us a cite. + +```bibtex +@misc{savkin2026occragoptimalcognitivecore, + title = {OCC-RAG: Optimal Cognitive Core for Faithful Question Answering}, + author = {Maksim Savkin and Mikhail Goncharov and Alexander Gambashidze and Alla Chepurova and Dmitrii Tarasov and Nikita Andriianov and Daria Pugacheva and Vasily Konovalov and Andrey Galichin and Ivan Oseledets}, + year = {2026}, + eprint = {2606.00683}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + url = {https://arxiv.org/abs/2606.00683} +} +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..333ed29 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,20 @@ +{%- for message in messages -%} + {%- if message['role'] == 'system' -%} + {{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }} + {%- elif message['role'] == 'user' -%} + {%- if documents and loop.last -%} + {{ '<|im_start|>user\n<|query_start|>' + message['content'] + '<|query_end|>\n' }} + {%- for doc in documents -%} + {{ '<|source_start|><|source_id|>' + (loop.index | string) + ' ' + doc['text'] + '<|source_end|>\n' }} + {%- endfor -%} + {{ '<|im_end|>\n' }} + {%- else -%} + {{ '<|im_start|>user\n' + message['content'] + '<|im_end|>\n' }} + {%- endif -%} + {%- elif message['role'] == 'assistant' -%} + {{ '<|im_start|>assistant\n\n\n\n\n' + message['content'] + '<|im_end|>\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{ '<|im_start|>assistant\n\n\n\n\n<|query_analysis_start|>\n' }} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000..418a354 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.4", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/figures/github-mark.png b/figures/github-mark.png new file mode 100644 index 0000000..e815117 Binary files /dev/null and b/figures/github-mark.png differ diff --git a/figures/occ.png b/figures/occ.png new file mode 100644 index 0000000..6834a22 Binary files /dev/null and b/figures/occ.png differ diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c878f5d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": false, + "temperature": 0.0, + "eos_token_id": [ + 151643, + 151645, + 151683 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.4" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..08f1374 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913448d005d2af695a738da177120c9e741dc8a5b2328f717d7ed108c1fd2e4f +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..fb77ddc --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:672e331460a05e2ea9888810a7a37f0c775429fe05fddc6330ee0dc9147a1370 +size 11425566 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..0b78837 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,44 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|query_start|>", + "<|query_end|>", + "<|source_start|>", + "<|source_end|>", + "<|source_id|>", + "<|query_analysis_start|>", + "<|query_analysis_end|>", + "<|source_analysis_start|>", + "<|source_analysis_end|>", + "<|reasoning_start|>", + "<|reasoning_end|>", + "<|status_start|>", + "<|status_end|>", + "<|answer_start|>", + "<|answer_end|>" + ] +}