From 5b88626a3b251d1c8924774ef3187c2646425ed1 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 6 Jul 2026 01:45:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: unsloth/LFM2-8B-A1B-GGUF Source: Original Platform --- .gitattributes | 68 +++++++ LFM2-8B-A1B-BF16.gguf | 3 + LFM2-8B-A1B-Q2_K.gguf | 3 + LFM2-8B-A1B-Q2_K_L.gguf | 3 + LFM2-8B-A1B-Q3_K_M.gguf | 3 + LFM2-8B-A1B-Q3_K_S.gguf | 3 + LFM2-8B-A1B-Q4_0.gguf | 3 + LFM2-8B-A1B-Q4_1.gguf | 3 + LFM2-8B-A1B-Q4_K_M.gguf | 3 + LFM2-8B-A1B-Q4_K_S.gguf | 3 + LFM2-8B-A1B-Q5_K_M.gguf | 3 + LFM2-8B-A1B-Q5_K_S.gguf | 3 + LFM2-8B-A1B-Q6_K.gguf | 3 + LFM2-8B-A1B-Q8_0.gguf | 3 + LFM2-8B-A1B-UD-Q2_K_XL.gguf | 3 + LFM2-8B-A1B-UD-Q3_K_XL.gguf | 3 + LFM2-8B-A1B-UD-Q4_K_XL.gguf | 3 + LFM2-8B-A1B-UD-Q5_K_XL.gguf | 3 + LFM2-8B-A1B-UD-Q6_K_XL.gguf | 3 + LFM2-8B-A1B-UD-Q8_K_XL.gguf | 3 + README.md | 381 ++++++++++++++++++++++++++++++++++++ configuration.json | 1 + imatrix_unsloth.gguf_file | 3 + 23 files changed, 510 insertions(+) create mode 100644 .gitattributes create mode 100644 LFM2-8B-A1B-BF16.gguf create mode 100644 LFM2-8B-A1B-Q2_K.gguf create mode 100644 LFM2-8B-A1B-Q2_K_L.gguf create mode 100644 LFM2-8B-A1B-Q3_K_M.gguf create mode 100644 LFM2-8B-A1B-Q3_K_S.gguf create mode 100644 LFM2-8B-A1B-Q4_0.gguf create mode 100644 LFM2-8B-A1B-Q4_1.gguf create mode 100644 LFM2-8B-A1B-Q4_K_M.gguf create mode 100644 LFM2-8B-A1B-Q4_K_S.gguf create mode 100644 LFM2-8B-A1B-Q5_K_M.gguf create mode 100644 LFM2-8B-A1B-Q5_K_S.gguf create mode 100644 LFM2-8B-A1B-Q6_K.gguf create mode 100644 LFM2-8B-A1B-Q8_0.gguf create mode 100644 LFM2-8B-A1B-UD-Q2_K_XL.gguf create mode 100644 LFM2-8B-A1B-UD-Q3_K_XL.gguf create mode 100644 LFM2-8B-A1B-UD-Q4_K_XL.gguf create mode 100644 LFM2-8B-A1B-UD-Q5_K_XL.gguf create mode 100644 LFM2-8B-A1B-UD-Q6_K_XL.gguf create mode 100644 LFM2-8B-A1B-UD-Q8_K_XL.gguf create mode 100644 README.md create mode 100644 configuration.json create mode 100644 imatrix_unsloth.gguf_file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1ea488b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,68 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text + +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text + +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +LFM2-8B-A1B-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q6_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q8_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text +imatrix_unsloth.gguf_file filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q2_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q2_K_L.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-BF16.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q3_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q4_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q4_1.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-UD-Q5_K_XL.gguf filter=lfs diff=lfs merge=lfs -text +LFM2-8B-A1B-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/LFM2-8B-A1B-BF16.gguf b/LFM2-8B-A1B-BF16.gguf new file mode 100644 index 0000000..4254906 --- /dev/null +++ b/LFM2-8B-A1B-BF16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7309b04b9b0369d57cfddf035fe7c48363189a5d4e3738298890f294389daab1 +size 16685562944 diff --git a/LFM2-8B-A1B-Q2_K.gguf b/LFM2-8B-A1B-Q2_K.gguf new file mode 100644 index 0000000..fa165cb --- /dev/null +++ b/LFM2-8B-A1B-Q2_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eab97ce055ac309105fc7d8650f987793d83745fb691c7b0cdc8f5af9dab488 +size 3079650368 diff --git a/LFM2-8B-A1B-Q2_K_L.gguf b/LFM2-8B-A1B-Q2_K_L.gguf new file mode 100644 index 0000000..fa165cb --- /dev/null +++ b/LFM2-8B-A1B-Q2_K_L.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eab97ce055ac309105fc7d8650f987793d83745fb691c7b0cdc8f5af9dab488 +size 3079650368 diff --git a/LFM2-8B-A1B-Q3_K_M.gguf b/LFM2-8B-A1B-Q3_K_M.gguf new file mode 100644 index 0000000..f8db8ae --- /dev/null +++ b/LFM2-8B-A1B-Q3_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39f735f86a16cb1761b7a961494fd733ebfa4d06c56fdca55e73c5419dc8ea7 +size 3997613120 diff --git a/LFM2-8B-A1B-Q3_K_S.gguf b/LFM2-8B-A1B-Q3_K_S.gguf new file mode 100644 index 0000000..a2e1c72 --- /dev/null +++ b/LFM2-8B-A1B-Q3_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42debfee9c5061423fc0dabada2099e482ec8878e9237830f195feb3f451f0f4 +size 3644292160 diff --git a/LFM2-8B-A1B-Q4_0.gguf b/LFM2-8B-A1B-Q4_0.gguf new file mode 100644 index 0000000..98d9adc --- /dev/null +++ b/LFM2-8B-A1B-Q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:356e2f39e86975ee29ffa14f013f9ca5f0f4ec51c29aa6de14269e33db7e6709 +size 4733893696 diff --git a/LFM2-8B-A1B-Q4_1.gguf b/LFM2-8B-A1B-Q4_1.gguf new file mode 100644 index 0000000..67cb440 --- /dev/null +++ b/LFM2-8B-A1B-Q4_1.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58b5f15969bfa8524cb9715294e3edc4df952fa91db097025fa6df92a3ae3987 +size 5246647360 diff --git a/LFM2-8B-A1B-Q4_K_M.gguf b/LFM2-8B-A1B-Q4_K_M.gguf new file mode 100644 index 0000000..36d9b9d --- /dev/null +++ b/LFM2-8B-A1B-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62ca7cc69465f60a0e9507bc67345bc0ea17656602226b4ad2c6fa71011415ed +size 5044780096 diff --git a/LFM2-8B-A1B-Q4_K_S.gguf b/LFM2-8B-A1B-Q4_K_S.gguf new file mode 100644 index 0000000..52eb60d --- /dev/null +++ b/LFM2-8B-A1B-Q4_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aae538f1a9ee08230b82265b681426d7460541917e901f493ab32e356da34e3 +size 4752768064 diff --git a/LFM2-8B-A1B-Q5_K_M.gguf b/LFM2-8B-A1B-Q5_K_M.gguf new file mode 100644 index 0000000..96bab90 --- /dev/null +++ b/LFM2-8B-A1B-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9a64cd33683d031242d27ae7da6b5f83044fca3391852585bd6b71510ceaa6 +size 5919554624 diff --git a/LFM2-8B-A1B-Q5_K_S.gguf b/LFM2-8B-A1B-Q5_K_S.gguf new file mode 100644 index 0000000..b42190b --- /dev/null +++ b/LFM2-8B-A1B-Q5_K_S.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691059bfb7429c3f65c94d8cc54185dcc59f24cb71d3f6f4f57e74c5331a2910 +size 5759401024 diff --git a/LFM2-8B-A1B-Q6_K.gguf b/LFM2-8B-A1B-Q6_K.gguf new file mode 100644 index 0000000..b3a27d6 --- /dev/null +++ b/LFM2-8B-A1B-Q6_K.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d840c2d944db41d689f43f258a92c87dca42fb46adc0c382c72ebae500f870 +size 6849002560 diff --git a/LFM2-8B-A1B-Q8_0.gguf b/LFM2-8B-A1B-Q8_0.gguf new file mode 100644 index 0000000..96ff78d --- /dev/null +++ b/LFM2-8B-A1B-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f2f09d525787a0d3781bfae2c7e765d652243365bdf7b3b616729fe4576fe3 +size 8868428864 diff --git a/LFM2-8B-A1B-UD-Q2_K_XL.gguf b/LFM2-8B-A1B-UD-Q2_K_XL.gguf new file mode 100644 index 0000000..4b85fd2 --- /dev/null +++ b/LFM2-8B-A1B-UD-Q2_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b487f7f2055ac1b1e8fdfe36812854c6778d1914899853535624784fd629e1b +size 3120266304 diff --git a/LFM2-8B-A1B-UD-Q3_K_XL.gguf b/LFM2-8B-A1B-UD-Q3_K_XL.gguf new file mode 100644 index 0000000..99a6f76 --- /dev/null +++ b/LFM2-8B-A1B-UD-Q3_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d10253b60d9699c4936a024fded42cba4581dc3640182146cba95fe57c143ac6 +size 3676339264 diff --git a/LFM2-8B-A1B-UD-Q4_K_XL.gguf b/LFM2-8B-A1B-UD-Q4_K_XL.gguf new file mode 100644 index 0000000..4ab1a1e --- /dev/null +++ b/LFM2-8B-A1B-UD-Q4_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d802eb863d99a34c41b4f016faa2786d37dcff0bf15b626766d5be0fc20e14a +size 4743625792 diff --git a/LFM2-8B-A1B-UD-Q5_K_XL.gguf b/LFM2-8B-A1B-UD-Q5_K_XL.gguf new file mode 100644 index 0000000..96bab90 --- /dev/null +++ b/LFM2-8B-A1B-UD-Q5_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9a64cd33683d031242d27ae7da6b5f83044fca3391852585bd6b71510ceaa6 +size 5919554624 diff --git a/LFM2-8B-A1B-UD-Q6_K_XL.gguf b/LFM2-8B-A1B-UD-Q6_K_XL.gguf new file mode 100644 index 0000000..7a82486 --- /dev/null +++ b/LFM2-8B-A1B-UD-Q6_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73b7386af06f88c3430ff60bd22c1d31efa8302314e57e8d236bfffbbe31815d +size 6881508416 diff --git a/LFM2-8B-A1B-UD-Q8_K_XL.gguf b/LFM2-8B-A1B-UD-Q8_K_XL.gguf new file mode 100644 index 0000000..c13207f --- /dev/null +++ b/LFM2-8B-A1B-UD-Q8_K_XL.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d77e2ed02a46a80499cecdcd88948071380a70b94bf24d3c89d891315b483de +size 8994257984 diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5b48b5 --- /dev/null +++ b/README.md @@ -0,0 +1,381 @@ +--- +base_model: +- LiquidAI/LFM2-8B-A1B +library_name: transformers +license: other +license_name: lfm1.0 +license_link: LICENSE +language: +- en +- ar +- zh +- fr +- de +- ja +- ko +- es +pipeline_tag: text-generation +tags: +- liquid +- unsloth +- lfm2 +- edge +- moe +--- +> [!NOTE] +> Includes Unsloth **chat template fixes**!
For `llama.cpp`, use `--jinja` +> + +
+

+ Unsloth Dynamic 2.0 achieves superior accuracy & outperforms other leading quants. +

+
+ + + + + + + + + +
+
+ + +
+
+ Liquid AI +
+
+ + + Playground + + + + + + + + + + + + + + + + + + + Leap + + + + + + + + + + + + + + +
+
+ +# LFM2-8B-A1B + +LFM2 is a new generation of hybrid models developed by [Liquid AI](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts), specifically designed for edge AI and on-device deployment. It sets a new standard in terms of quality, speed, and memory efficiency. + +We're releasing the weights of our first MoE based on LFM2, with 8.3B total parameters and 1.5B active parameters. + +- LFM2-8B-A1B is the best on-device MoE in terms of both **quality** (comparable to 3-4B dense models) and **speed** (faster than Qwen3-1.7B). +- **Code and knowledge** capabilities are significantly improved compared to LFM2-2.6B. +- Quantized variants fit comfortably on high-end **phones, tablets, and laptops**. + +Find more information about LFM2-8B-A1B in our [blog post](https://www.liquid.ai/blog/). + +## 📄 Model details + +Due to their small size, **we recommend fine-tuning LFM2 models on narrow use cases** to maximize performance. +They are particularly suited for agentic tasks, data extraction, RAG, creative writing, and multi-turn conversations. +However, we do not recommend using them for tasks that are knowledge-intensive or require programming skills. + +| Property | [**LFM2-8B-A1B**](https://huggingface.co/LiquidAI/LFM2-8B-A1B) | +| --------------------- | ----------------------------- | +| **Total parameters** | 8.3B | +| **Active parameters** | 1.5B | +| **Layers** | 24 (18 conv + 6 attn) | +| **Context length** | 32,768 tokens | +| **Vocabulary size** | 65,536 | +| **Training precision**| Mixed BF16/FP8 | +| **Training budget** | 12 trillion tokens | +| **License** | LFM Open License v1.0 | + +**Supported languages**: English, Arabic, Chinese, French, German, Japanese, Korean, and Spanish. + +**Generation parameters**: We recommend the following parameters: +* `temperature=0.3` +* `min_p=0.15` +* `repetition_penalty=1.05` + +**Chat template**: LFM2 uses a ChatML-like chat template as follows: + +``` +<|startoftext|><|im_start|>system +You are a helpful assistant trained by Liquid AI.<|im_end|> +<|im_start|>user +What is C. elegans?<|im_end|> +<|im_start|>assistant +It's a tiny nematode that lives in temperate soil environments.<|im_end|> +``` + +You can automatically apply it using the dedicated [`.apply_chat_template()`](https://huggingface.co/docs/transformers/en/chat_templating#applychattemplate) function from Hugging Face transformers. + +**Tool use**: It consists of four main steps: +1. **Function definition**: LFM2 takes JSON function definitions as input (JSON objects between `<|tool_list_start|>` and `<|tool_list_end|>` special tokens), usually in the system prompt +2. **Function call**: LFM2 writes Pythonic function calls (a Python list between `<|tool_call_start|>` and `<|tool_call_end|>` special tokens), as the assistant answer. +3. **Function execution**: The function call is executed and the result is returned (string between `<|tool_response_start|>` and `<|tool_response_end|>` special tokens), as a "tool" role. +4. **Final answer**: LFM2 interprets the outcome of the function call to address the original user prompt in plain text. + +Here is a simple example of a conversation using tool use: + +``` +<|startoftext|><|im_start|>system +List of tools: <|tool_list_start|>[{"name": "get_candidate_status", "description": "Retrieves the current status of a candidate in the recruitment process", "parameters": {"type": "object", "properties": {"candidate_id": {"type": "string", "description": "Unique identifier for the candidate"}}, "required": ["candidate_id"]}}]<|tool_list_end|><|im_end|> +<|im_start|>user +What is the current status of candidate ID 12345?<|im_end|> +<|im_start|>assistant +<|tool_call_start|>[get_candidate_status(candidate_id="12345")]<|tool_call_end|>Checking the current status of candidate ID 12345.<|im_end|> +<|im_start|>tool +<|tool_response_start|>{"candidate_id": "12345", "status": "Interview Scheduled", "position": "Clinical Research Associate", "date": "2023-11-20"}<|tool_response_end|><|im_end|> +<|im_start|>assistant +The candidate with ID 12345 is currently in the "Interview Scheduled" stage for the position of Clinical Research Associate, with an interview date set for 2023-11-20.<|im_end|> +``` + +**Architecture**: Hybrid model with multiplicative gates and short convolutions: 18 double-gated short-range LIV convolution blocks and 6 grouped query attention (GQA) blocks. + +**Pre-training mixture**: Approximately 75% English, 20% multilingual, and 5% code data sourced from the web and licensed materials. + +**Training approach**: +* Very large-scale SFT on 50% downstream tasks, 50% general domains +* Custom DPO with length normalization and semi-online datasets +* Iterative model merging + +## 🏃 How to run LFM2 + +### 1. Transformers + +To run LFM2, you need to install Hugging Face [`transformers`](https://github.com/huggingface/transformers) from source as follows: +```bash +pip install git+https://github.com/huggingface/transformers.git@0c9a72e4576fe4c84077f066e585129c97bfd4e6 +``` + +Here is an example of how to generate an answer with transformers in Python: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +# Load model and tokenizer +model_id = "LiquidAI/LFM2-8B-A1B" +model = AutoModelForCausalLM.from_pretrained( + model_id, + device_map="auto", + dtype="bfloat16", +# attn_implementation="flash_attention_2" <- uncomment on compatible GPU +) +tokenizer = AutoTokenizer.from_pretrained(model_id) + +# Generate answer +prompt = "What is C. elegans?" +input_ids = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + return_tensors="pt", + tokenize=True, +).to(model.device) + +output = model.generate( + input_ids, + do_sample=True, + temperature=0.3, + min_p=0.15, + repetition_penalty=1.05, + max_new_tokens=512, +) + +print(tokenizer.decode(output[0], skip_special_tokens=False)) + +# <|startoftext|><|im_start|>user +# What is C. elegans?<|im_end|> +# <|im_start|>assistant +# C. elegans, also known as Caenorhabditis elegans, is a small, free-living +# nematode worm (roundworm) that belongs to the phylum Nematoda. +``` + +You can directly run and test the model with this [Colab notebook](https://colab.research.google.com/drive/1i0u7X6qen9UJkV6xSCDZ0NmPUH50SOvO?usp=sharing). + +### 2. vLLM + +You can run the model in [`vLLM`](https://github.com/vllm-project/vllm) by building from source: + +```bash +git clone https://github.com/vllm-project/vllm.git +cd vllm +pip install -e . -v +``` + +Here is an example of how to use it for inference: + +```python +from vllm import LLM, SamplingParams + +prompts = [ + [ + { + "content": "What is C. elegans?", + "role": "user", + }, + ], + [ + { + "content": "Say hi in JSON format", + "role": "user", + }, + ], + [ + { + "content": "Define AI in Spanish", + "role": "user", + }, + ], +] + +sampling_params = SamplingParams( + temperature=0.3, + min_p=0.15, + repetition_penalty=1.05, + max_tokens=30 +) + +llm = LLM(model="LiquidAI/LFM2-8B-A1B", dtype="bfloat16") + +outputs = llm.chat(prompts, sampling_params) + +for i, output in enumerate(outputs): + prompt = prompts[i][0]["content"] + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") +``` + + +### 3. llama.cpp + +You can run LFM2 with llama.cpp using its [GGUF checkpoint](https://huggingface.co/LiquidAI/LFM2-8B-A1B-GGUF). Find more information in the model card. + +## 🔧 How to fine-tune LFM2 + +We recommend fine-tuning LFM2 models on your use cases to maximize performance. + +| Notebook | Description | Link | +|-------|------|------| +| SFT (TRL) | Supervised Fine-Tuning (SFT) notebook with a LoRA adapter using TRL. | Colab link | +| DPO (TRL) | Preference alignment with Direct Preference Optimization (DPO) using TRL. | Colab link | + +## 📈 Performance + +### 1. Automated benchmarks + +
+
+ + Benchmarks + +
+
+ +Compared to similar-sized models, LFM2-8B-A1B displays strong performance in instruction following and math while also running significantly faster. + +| Model | MMLU | MMLU-Pro | GPQA | IFEval | IFBench | Multi-IF | +|---|---|---|---|---|---|---| +| **LFM2-8B-A1B** | 64.84 | 37.42 | 29.29 | 77.58 | 25.85 | 58.19 | +| LFM2-2.6B | 64.42 | 25.96 | 26.57 | 79.56 | 22.19 | 60.26 | +| Llama-3.2-3B-Instruct | 60.35 | 22.25 | 30.6 | 71.43 | 20.78 | 50.91 | +| SmolLM3-3B | 59.84 | 23.90 | 26.31 | 72.44 | 17.93 | 58.86 | +| gemma-3-4b-it | 58.35 | 34.76 | 29.51 | 76.85 | 23.53 | 66.61 | +| Qwen3-4B-Instruct-2507 | 72.25 | 52.31 | 34.85 | 85.62 | 30.28 | 75.54 | +| granite-4.0-h-tiny | 66.79 | 32.03 | 26.46 | 81.06 | 18.37 | 52.99 | + +| Model | GSM8K | GSMPlus | MATH 500 | MATH Lvl 5 | MGSM | MMMLU | +|---|---|---|---|---|---|---| +| **LFM2-8B-A1B** | 84.38 | 64.76 | 74.2 | 62.38 | 72.4 | 55.26 | +| LFM2-2.6B | 82.41 | 60.75 | 63.6 | 54.38 | 74.32 | 55.39 | +| Llama-3.2-3B-Instruct | 75.21 | 38.68 | 41.2 | 24.06 | 61.68 | 47.92 | +| SmolLM3-3B | 81.12 | 58.91 | 73.6 | 51.93 | 68.72 | 50.02 | +| gemma-3-4b-it | 89.92 | 68.38 | 73.2 | 52.18 | 87.28 | 50.14 | +| Qwen3-4B-Instruct-2507 | 68.46 | 56.16 | 85.6 | 73.62 | 81.76 | 60.67 | +| granite-4.0-h-tiny | 82.64 | 59.14 | 58.2 | 36.11 | 73.68 | 56.13 | + +| Model | Active params | LCB v6 | LCB v5 | HumanEval+ | Creative Writing v3 | +|----------------------------|---------------|---------------|---------------|--------------------|-----------------------------| +| **LFM2-8B-A1B** | 1.5B | 21.04% | 21.36% | 69.51% | 44.22% | +| Gemma-3-1b-it | 1B | 4.27% | 4.43% | 37.20% | 41.67% | +| Granite-4.0-h-tiny | 1B | 26.73% | 27.27% | 73.78% | 32.60% | +| Llama-3.2-1B-Instruct | 1.2B | 4.08% | 3.64% | 23.17% | 31.43% | +| Qwen2.5-1.5B-Instruct | 1.5B | 11.18% | 10.57% | 48.78% | 22.18% | +| Qwen3-1.7B (/no_think) | 1.7B | 24.07% | 26.48% | 60.98% | 31.56% | +| LFM2-2.6B | 2.6B | 14.41% | 14.43% | 57.93% | 38.79% | +| SmolLM3-3B | 3.1B | 19.05% | 19.20% | 60.37% | 36.44% | +| Llama-3.2-3B-Instruct | 3.2B | 11.47% | 11.48% | 24.06% | 38.84% | +| Qwen3-4B (/no_think) | 4B | 36.11% | 38.64% | 71.95% | 37.49% | +| Qwen3-4B-Instruct-2507 | 4B | 48.72% | 50.80% | 82.32% | 51.71% | +| Gemma-3-4b-it | 4.3B | 18.86% | 19.09% | 62.8% | 68.56% | + +### 2. Inference + +LFM2-8B-A1B is significantly faster than models with a similar number of active parameters, like Qwen3-1.7B. + +
+
+ + Decode Throughput - S24 Ultra + +
+ +
+ + Decode Throughput - HX370 + +
+
+ +The following plots showcase the performance of different models under int4 quantization with int8 dynamic activations on the AMD Ryzen AI 9 HX 370 CPU, using 16 threads. The results are obtained using our internal XNNPACK-based inference stack, and a custom CPU MoE kernel. + +
+
+ + Prefill Throughput vs Sequence Length + +
+
+ + Decode Throughput vs Sequence Length + +
+
+ +## 📬 Contact + +If you are interested in custom solutions with edge deployment, please contact [our sales team](https://www.liquid.ai/contact). diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/imatrix_unsloth.gguf_file b/imatrix_unsloth.gguf_file new file mode 100644 index 0000000..28130e4 --- /dev/null +++ b/imatrix_unsloth.gguf_file @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f83916980ca03ec963e2732814e96631343bef2fd33aac3d6b13d0dd7408d4 +size 17375232