commit 68a94831410328486b5888202c0151fe34111cc3 Author: ModelHub XC Date: Tue Jul 14 22:26:11 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: sennaLLMLearner/qwen2.5-7b-memory-distiller Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3500e46 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-7b-memory-distiller.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Modelfile b/Modelfile new file mode 100644 index 0000000..64d819c --- /dev/null +++ b/Modelfile @@ -0,0 +1,2 @@ +FROM ./qwen2.5-7b-memory-distiller.Q4_K_M.gguf +PARAMETER temperature 0 diff --git a/README.md b/README.md new file mode 100644 index 0000000..23cc772 --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen2.5-7B-Instruct +datasets: + - allenai/WildChat-1M +language: + - en + - ja +pipeline_tag: text-generation +library_name: transformers +tags: + - memory + - distillation + - orpo + - merged + - qwen2.5 + - structured-output + - json + - edge + - personal-project +--- + +# Conversation Memory Distiller (Qwen2.5-7B) + +A small model fine-tuned to turn a conversation log into clean, searchable **structured-memory +JSON** for later recall, running **locally on an edge device**. Fine-tuned from +`Qwen/Qwen2.5-7B-Instruct` via **SFT warmup → ORPO**, distilling a DeepSeek-V3 teacher, and +optimized to follow **5 content principles** (below) when compressing a conversation into memory. + +> **This is a structured-distillation (memory-summarization) model, not a general summarizer.** +> It is meant to run **locally on an edge device** and emit schema-conformant memory objects. + +## Why this exists + +This model powers the local memory layer of **[codeatrium](https://github.com/senna-lang/Codeatrium)**, a personal-memory app: it distills +conversations into searchable memories. The goal was a **local / free / private** 7B that runs on +an edge device (via Ollama Q4_K_M) and could replace the larger distiller previously used in that +role. It's a **personal project**: the dataset and evaluation are small-scale, the "5 principles" +are **a custom rubric** drawn from the memory-distillation literature, and **training was stopped +pragmatically once it reached practical quality** (2 ORPO epochs), not pushed to convergence. +Metrics and acceptance gates were **pre-registered before running** the final evals, and only +what survived is reported (see [How I know](#how-i-know)). + +## Intended use / Out of scope + +- **Intended**: distilling a conversation (user+assistant turns) into a **structured memory object** + (JSON/record with fields) for later **search / recall**, running locally. +- **Out of scope**: general-purpose summarization; downstream retrieval/QA quality (not evaluated + here — only distillation quality is); absolute faithfulness guarantees. + +## Learning signal: the 5 core principles + +The model is trained with these 5 principles as the preference signal (chosen/rejected). Each is a +content-level best practice backed by external work on memory systems and retrieval. + +| Principle | Definition | Primary source | Basis (key figure) | How it's measured | +|---|---|---|---|---| +| **completeness** (recall) | capture the source's important specifics without dropping them | ProMem (arXiv:2601.04463) | verification+completion raises memory integrity 54→74%; +30.89pp vs Mem0 on HaluMem; **+10.74pt transfer on Llama3-8B** (the one principle confirmed to help at small scale) | deterministic: specifics recall against the source, no LLM judge | +| **lexical_fidelity** | keep numbers, commands, identifiers **verbatim**, not paraphrased | SPAR (arXiv:2110.06918) / Searchat (arXiv:2603.13017) | dense retrievers miss rare entities / salient phrases that BM25 catches (**lexical-fidelity gap**); Searchat retains 96.8% of query vocabulary after distillation | deterministic: verbatim-retention rate among captured specifics (correlated with completeness, not fully independent) | +| **atomicity** | one memory = one fact; no cramming multiple facts together | TriMem (arXiv:2605.19952) / Mem0 | fact-centric atomic decomposition; coarse-only memory blocks deep reasoning across scattered facts | LLM judge only; no deterministic anchor | +| **discriminative** | carries specific info that can be retrieved / told apart (not generic labels) | GAAMA (arXiv:2603.27910) / HippoRAG (arXiv:2405.14831) | generic labels ("programming") make megahub nodes that absorb retrieval mass (**megahub problem**); discriminative tags narrow retrieval | LLM judge only; no deterministic anchor | +| **faithfulness** (precision) | do not fabricate specifics absent from the source | ProMem (arXiv:2601.04463) / HaluMem (arXiv:2511.03506) | stronger completion increases fabrication risk (the flip side of completeness); HaluMem measures hallucination | deterministic: specifics precision against the source, no LLM judge | + +> **Eval note.** `completeness` and `faithfulness` are measured **without an LLM judge** as set +> recall / precision of *specifics* (numbers, identifiers, commands, paths, quoted strings) against +> the source. `lexical_fidelity` is a supplementary verbatim-retention rate (not independent of +> completeness). `atomicity` and `discriminative` require structural/semantic judgment and are +> measured only via LLM judges — they carry no deterministic anchor (see limitations). + +## Training + +- **Base**: `Qwen/Qwen2.5-7B-Instruct`. **Teacher**: DeepSeek-V3 (distillation target). +- **Data**: technical conversations deterministically sampled from **WildChat-1M** (en+ja, + toxic-filtered) — a **small** preference set (~hundreds of conversations / thousands of exchanges). +- **Recipe**: **SFT warmup → merge → fresh LoRA (r16) → ORPO (2 epochs)**. + +## How I know + +I evaluated **distillation quality** (the 5 principles), *not* downstream retrieval. Metrics and +acceptance gates were **pre-registered before the final runs**. + +**It beats the incumbent on the deployment task.** On my codeatrium production path (all arms +**Ollama Q4_K_M**, 48 independent real conversations), this 7B outperforms both +`qwen2.5-coder:14b` (my previous distiller) and raw `qwen2.5:7b`. The win is **not judge-dependent** — +it holds under a same-family judge (Qwen2.5-72B), a **different-family judge** (Llama-3.3-70B), a +**judge-free specifics metric**, and a **blind human rating** (n=30, FT vs the 14B): + +- **completeness** is confirmed by *all four* methods (human net **+0.63**, p≈7e-5) — the flagship result. +- **discriminative** (+0.47) and **lexical_fidelity** (+0.37) are also human-confirmed vs the 14B. + +**It generalizes to unseen *structured* schemas — but not to everything.** On 5 held-out schemas it +never trained on (measured judge-free), it beats raw 7B on **field-structured** formats (nested JSON, +typed record, entity-relation graph — all significant) but only **ties** on flat / hard-length-capped +formats (flat Markdown bullets, ≤12-word YAML). So the honest claim is: **generalizes to unseen +field-structured schemas; no edge on flat / heavily-constrained formats.** (codeatrium's schema is +field-structured, consistent with the deployment win.) + +## Limitations (read this) + +- **Small-scale & personal.** Small dataset; evals of n=30–48; training stopped at practical utility. +- **`atomicity` is measurement-limited.** LLM judges score a large FT win, but blind humans rated it a + near-tie (27/30 ties) — the judge win is **not human-confirmed**. Treat atomicity cautiously. +- **`discriminative` is family-sensitive.** Human- and judge-confirmed vs the 14B, but vs raw 7B it + was **not significant** under a cross-family judge. +- **The human eval is author-blind, not third-party.** I rated blind (arm-anonymized, order + randomized), which removes judge-family bias, but I built the model and know its style — so it is + **not a fully independent** human evaluation. (It was conservative: 88/150 ratings were ties.) +- **The 5 principles are my own rubric**, not a standard benchmark; absolute faithfulness is low on + hard prompts (the claim is "more faithful than the baselines," paired — not "faithful" in absolute + terms). +- **Downstream utility (retrieval/QA) is not evaluated.** Scope is distillation quality only. + +## Usage + +Deployment target is **Ollama Q4_K_M**, called via an OpenAI-compatible endpoint +(`http://localhost:11434/v1`) — the same path codeatrium uses. Prompt the model with your distillation +instruction + schema + the conversation, and read back the structured JSON (greedy / temperature 0 for +determinism). The 5-principle behavior is strongest on **field-structured** schemas. + +> Technical note: this repo ships the **fully merged weights** (base ⊕ SFT-LoRA ⊕ ORPO-LoRA already +> baked in) — load it directly like any Qwen2.5-7B checkpoint, no adapter reconstruction needed. +> Internally it was trained as `stock Qwen ⊕ SFT-LoRA → merge → ⊕ ORPO-LoRA → merge`; that provenance +> only matters if you're retraining from an intermediate checkpoint, not for inference. + +## References + +- **ProMem** — proactive memory verification & completion. arXiv:2601.04463 +- **SPAR** — lexical-fidelity gap of dense retrievers. arXiv:2110.06918 (EMNLP-Findings 2022) +- **Searchat** — structured distillation / surviving vocabulary. arXiv:2603.13017 +- **TriMem** — multi-granularity memory (segment / atomic fact / profile). arXiv:2605.19952 +- **HaluMem** — memory hallucination benchmark. arXiv:2511.03506 +- **GAAMA** — graph-augmented associative memory (megahub avoidance). arXiv:2603.27910 / + github.com/swarna-kpaul/gaama +- **HippoRAG** — neurobiologically-inspired long-term memory / entity-centric KG retrieval + (hub-concentration evidence). arXiv:2405.14831 diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..1e133dc --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.12.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..fee66e8 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.12.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..34cf7e0 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3361466eef0bacbf8b6ab180aa96f32554467023c8e247048908dc4680a3ea38 +size 15231272152 diff --git a/qwen2.5-7b-memory-distiller.Q4_K_M.gguf b/qwen2.5-7b-memory-distiller.Q4_K_M.gguf new file mode 100644 index 0000000..8628ff5 --- /dev/null +++ b/qwen2.5-7b-memory-distiller.Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a241c61a2221b7d115bcf06af64e8e3a092df579bb039040df4c87286175750d +size 4683073824 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}