commit d9d3041c89a4703c1a5b8ef2811363a96052b49c Author: ModelHub XC Date: Sun Oct 25 21:24:21 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Swicked86/phi4-mm-gguf Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7ecad75 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +phi4-mm-f16.gguf filter=lfs diff=lfs merge=lfs -text +phi4-mm-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +phi4-mm-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +mmproj-phi4-mm-f16.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..4b82162 --- /dev/null +++ b/README.md @@ -0,0 +1,491 @@ +--- +license: mit +base_model: microsoft/Phi-4-multimodal-instruct +tags: + - phi4 + - phi4mm + - gguf + - quantized + - vision-language + - audio + - multimodal + - llama-cpp + - lm-studio +language: + - en +pipeline_tag: visual-question-answering +library_name: gguf +--- + +# Phi-4-multimodal-instruct GGUF Quantizations + +GGUF quantizations of [microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) — +a **5.6 B parameter** multimodal model by Microsoft supporting **text, vision (images), and audio** inputs. + +Produced with [llama.cpp](https://github.com/ggerganov/llama.cpp) build **b8347** on RTX 5090. + +> **License:** [MIT](https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/LICENSE) — © Microsoft Corporation. +> These quantizations carry the same MIT license as the original model. + +--- + +## Available Files + +| File | Quant | Size | BPW | Best for | +|------|-------|------|-----|----------| +| `phi4-mm-f16.gguf` | F16 | 7.17 GB | 16.0 | Re-quantization base, maximum quality | +| `phi4-mm-Q8_0.gguf` | Q8_0 | 3.90 GB | 8.0 | High-end GPU — near-lossless | +| `phi4-mm-Q4_K_M.gguf` | Q4_K_M | 2.37 GB | 5.18 | CPU / constrained VRAM — good quality | +| `mmproj-phi4-mm-f16.gguf` | F16 | 825 MB | 16.0 | Vision encoder — **required for image input** | + +> **One mmproj for all:** `mmproj-phi4-mm-f16.gguf` works with every text GGUF above. +> It cannot be quantized further — the CLIP FF layer dimension (4304) is not divisible by 32. + +--- + +## VRAM Requirements + +Full GPU offload (`-ngl 99`): + +| Configuration | VRAM | +|---------------|------| +| F16 + mmproj-F16 | ~10,000 MiB | +| Q8_0 + mmproj-F16 | ~5,400 MiB | +| Q4_K_M + mmproj-F16 | ~3,500 MiB | + +--- + +## Quality Metrics + +### Perplexity (wikitext-2-raw test set, context 512) + +| Model | PPL | vs F16 | +|-------|-----|--------| +| F16 (baseline) | 14.9338 ± 0.107 | — | +| Q8_0 | 14.9107 ± 0.106 | −0.15% ✅ lossless | +| Q4_K_M | 16.3183 ± 0.121 | +9.3% | + +### Throughput (llama-bench, RTX 5090, pp512 / tg128) + +| Model | Prompt (t/s) | Generation (t/s) | +|-------|-------------|-----------------| +| Q8_0 | 21,352 | 247 | +| Q4_K_M | 19,904 | 324 | + +### Multimodal Benchmarks (lmms-eval) + +*VQA evaluation in progress — will be added when complete. Suite: MMStar, OCRBench, AI2D, MathVista, HallusionBench.* + +--- + +## Usage + +### LM Studio (Recommended for Desktop) + +> **LM Studio** has native GGUF support including multimodal vision. No command-line needed. + +**Text-only:** +1. Open LM Studio → search `Swicked86/phi4-mm-gguf` +2. Download `phi4-mm-Q8_0.gguf` (GPU) or `phi4-mm-Q4_K_M.gguf` (CPU / low VRAM) +3. Load the model → Chat + +**With vision (image input):** +1. Download both `phi4-mm-Q8_0.gguf` **and** `mmproj-phi4-mm-f16.gguf` +2. Load the main model in LM Studio +3. In **Model Settings → Multimodal → Vision Model (mmproj)**, browse to `mmproj-phi4-mm-f16.gguf` +4. In Chat, click the image icon to attach a photo and ask questions about it + +> The mmproj file is the vision encoder. Without it the model runs text-only. +> `mmproj-phi4-mm-f16.gguf` is compatible with **all three** text GGUFs. + +--- + +### llama.cpp CLI + +#### Step 1 — Download the files + +```bash +# Install huggingface-cli if needed +pip install huggingface_hub + +# Text + vision (recommended) +huggingface-cli download Swicked86/phi4-mm-gguf phi4-mm-Q8_0.gguf mmproj-phi4-mm-f16.gguf --local-dir ./phi4-mm + +# CPU / low-VRAM variant +huggingface-cli download Swicked86/phi4-mm-gguf phi4-mm-Q4_K_M.gguf mmproj-phi4-mm-f16.gguf --local-dir ./phi4-mm +``` + +Build llama.cpp if you haven't already: +```bash +git clone https://github.com/ggerganov/llama.cpp +cd llama.cpp +cmake -B build -DGGML_CUDA=ON # omit DGGML_CUDA=ON for CPU-only +cmake --build build --config Release -j$(nproc) +``` + +--- + +#### Step 2 — Interactive multimodal chat (images + text) + +`llama-mtmd-cli` launches an interactive session. Type your prompt, or prefix it with an image path using `/image`: + +```bash +./build/bin/llama-mtmd-cli \ + -m ./phi4-mm/phi4-mm-Q8_0.gguf \ + --mmproj ./phi4-mm/mmproj-phi4-mm-f16.gguf \ + -ngl 99 --threads 16 --ctx-size 8192 +``` + +Inside the session: +``` +> /image photo.jpg +Image loaded. +> What is in this image? +[model describes the image] + +> /image chart.png +Image loaded. +> Summarise the trend shown in this chart. +[model analyses the chart] + +> Explain the previous image again but in French. +[responds without re-loading the image] +``` + +Single-shot (non-interactive): +```bash +./build/bin/llama-mtmd-cli \ + -m ./phi4-mm/phi4-mm-Q8_0.gguf \ + --mmproj ./phi4-mm/mmproj-phi4-mm-f16.gguf \ + --image photo.jpg \ + -p "Describe this image in detail." \ + -ngl 99 --threads 16 --no-display-prompt +``` + +CPU (no GPU): +```bash +./build/bin/llama-mtmd-cli \ + -m ./phi4-mm/phi4-mm-Q4_K_M.gguf \ + --mmproj ./phi4-mm/mmproj-phi4-mm-f16.gguf \ + --image photo.jpg \ + -p "What objects are in this photo?" \ + --threads 8 +``` + +--- + +#### Text-only (no image) + +```bash +./build/bin/llama-cli \ + -m ./phi4-mm/phi4-mm-Q8_0.gguf \ + --ctx-size 65536 --flash-attn on --kv-offload -ngl 99 --threads 16 \ + -p "<|system|>You are a helpful assistant.<|end|><|user|>Hello<|end|><|assistant|>" +``` + +--- + +### llama-server — OpenAI-compatible API + +Serves both text and vision via `/v1/chat/completions`. Useful for integrations (Open WebUI, SillyTavern, Continue.dev, etc.): + +```bash +./build/bin/llama-server \ + -m ./phi4-mm/phi4-mm-Q8_0.gguf \ + --mmproj ./phi4-mm/mmproj-phi4-mm-f16.gguf \ + -ngl 99 --threads 16 \ + --ctx-size 8192 --parallel 4 \ + --port 8080 --host 127.0.0.1 +``` + +**Text query (curl):** +```bash +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "phi4-mm", + "messages": [{"role": "user", "content": "Explain the Pythagorean theorem."}], + "max_tokens": 300 + }' +``` + +**Image query (curl, base64):** +```bash +IMAGE_B64=$(base64 -w0 photo.jpg) +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"phi4-mm\", + \"messages\": [{ + \"role\": \"user\", + \"content\": [ + {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,${IMAGE_B64}\"}}, + {\"type\": \"text\", \"text\": \"What is in this image?\"} + ] + }], + \"max_tokens\": 300 + }" +``` + +**Image query (Python, openai SDK):** +```python +import base64, httpx +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8080/v1", api_key="local") + +with open("photo.jpg", "rb") as f: + image_b64 = base64.b64encode(f.read()).decode() + +response = client.chat.completions.create( + model="phi4-mm", + messages=[{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}}, + {"type": "text", "text": "Describe this image."}, + ], + }], + max_tokens=300, +) +print(response.choices[0].message.content) +``` + +#### Modality support summary for GGUF (llama.cpp / LM Studio) + +| Modality | Supported | Notes | +|----------|-----------|-------| +| Text | ✅ | All three GGUFs | +| Vision (images) | ✅ | Requires `mmproj-phi4-mm-f16.gguf` + `--mmproj` flag | +| Audio / Speech | ❌ | Not available — see below | + +> **Audio is not supported in the GGUF files.** phi4-mm's speech capability uses a custom conformer-based audio encoder (24 conformer blocks, initialized from a proprietary AED ASR model) plus a rank-320 speech LoRA applied to the language decoder. The GGUF conversion pipeline (`convert_hf_to_gguf.py`) only exports the text transformer and the SigLIP vision encoder (mmproj) — the audio encoder tensors are not extracted. There is currently no `audioproj` equivalent in llama.cpp for phi4-mm. +> +> **To use audio/speech transcription, use the vLLM path below** with the original bf16 safetensors model. + +--- + +### vLLM — Full Multimodal (Image + Audio, bf16 Safetensors) + +> **Use vLLM when you want maximum quality and full multimodal support** (images + audio) from the original bf16 safetensors weights. For constrained hardware, use the GGUF options above instead. + +**Requirements:** Python 3.10+, CUDA GPU with ~16 GB VRAM, vLLM 0.7.0+ + +#### 1 — Install vLLM and download the model + +```bash +python3 -m venv ~/.vllm-env +source ~/.vllm-env/bin/activate +pip install --upgrade pip +pip install vllm + +# Download the original safetensors model (~14 GB, 3 shards) +huggingface-cli login # paste your HF token if the model is gated +huggingface-cli download microsoft/Phi-4-multimodal-instruct \ + --local-dir ~/phi4-mm-hf +``` + +#### 2 — Launch the vLLM server + +```bash +source ~/.vllm-env/bin/activate +python -m vllm.entrypoints.openai.api_server \ + --model ~/phi4-mm-hf \ + --dtype bfloat16 \ + --trust-remote-code \ + --max-model-len 65520 \ + --kv-cache-memory-bytes 8G \ + --limit-mm-per-prompt '{"image": 3, "audio": 3}' \ + --enable-auto-tool-choice \ + --tool-call-parser phi4_mini_json \ + --port 8080 \ + --host 127.0.0.1 \ + --served-model-name phi4-mm +``` + +| Flag | Value | Notes | +|------|-------|-------| +| `--dtype bfloat16` | — | Native dtype — do not change to float16 | +| `--max-model-len 65520` | — | Stable context ceiling for phi4-mm (131 K nominal) | +| `--kv-cache-memory-bytes` | `8G` | Tune down to `4G` on 12–16 GB GPUs | +| `--limit-mm-per-prompt` | `{"image":3,"audio":3}` | Max attachments per request | +| `--tool-call-parser phi4_mini_json` | — | phi4-mm emits `functools[...]` not Hermes — required for tool calling | +| `--trust-remote-code` | — | Required for phi4-mm's custom modelling code | + +> **Official vLLM LoRA flags:** Microsoft's published vLLM command includes explicit LoRA adapter flags to activate +> the rank-320 vision and speech adapters stored in separate subfolders of the model directory: +> ```bash +> --enable-lora \ +> --max-lora-rank 320 \ +> --lora-extra-vocab-size 0 \ +> --max-loras 2 \ +> --lora-modules speech=~/phi4-mm-hf/speech-lora vision=~/phi4-mm-hf/vision-lora +> ``` +> If you experience degraded vision or audio quality, add these flags to the launch command above. + +Wait for the server to finish loading (~60 s): +```bash +curl http://localhost:8080/health # → {"status":"ok"} +``` + +#### 3 — Send requests (OpenAI-compatible API) + +**Text (Python):** +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8080/v1", api_key="local") +response = client.chat.completions.create( + model="phi4-mm", + messages=[{"role": "user", "content": "What is the capital of France?"}], + max_tokens=256, +) +print(response.choices[0].message.content) +``` + +**Image (Python):** +```python +import base64 +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8080/v1", api_key="local") + +with open("photo.jpg", "rb") as f: + image_b64 = base64.b64encode(f.read()).decode() + +response = client.chat.completions.create( + model="phi4-mm", + messages=[{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}}, + {"type": "text", "text": "Describe this image in detail."}, + ], + }], + max_tokens=512, +) +print(response.choices[0].message.content) +``` + +**Image (curl, base64):** +```bash +IMAGE_B64=$(base64 -w0 photo.jpg) +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"phi4-mm\", + \"messages\": [{ + \"role\": \"user\", + \"content\": [ + {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,${IMAGE_B64}\"}}, + {\"type\": \"text\", \"text\": \"What is in this image?\"} + ] + }], + \"max_tokens\": 300 + }" +``` + +**Audio — speech transcription / understanding (Python):** +```python +import base64 +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8080/v1", api_key="local") + +with open("audio.wav", "rb") as f: + audio_b64 = base64.b64encode(f.read()).decode() + +response = client.chat.completions.create( + model="phi4-mm", + messages=[{ + "role": "user", + "content": [ + {"type": "input_audio", "input_audio": {"data": audio_b64, "format": "wav"}}, + {"type": "text", "text": "Transcribe this audio."}, + ], + }], + max_tokens=512, +) +print(response.choices[0].message.content) +``` + +> phi4-mm uses a custom conformer-based audio encoder with rank-320 speech LoRA — no separate ASR model needed. +> Supported formats: `wav`, `mp3`, `ogg`, `flac`. + +#### Tool calling + +phi4-mm emits tool calls as `functools[{"name":"...","arguments":{...}}]`. +The `--tool-call-parser phi4_mini_json` flag (vLLM 0.7+) handles this format automatically. +For a complete chat template that injects tools into phi4-mm's native `<|tool|>...<|/tool|>` block, +see [`deploy/wsl-vllm/phi4-mm-tool-template.jinja`](deploy/wsl-vllm/phi4-mm-tool-template.jinja) +in the [companion repo](https://github.com/Swicked86/LLM_Creation). + +--- + +### Ollama (CPU / NUC / edge) + +See the [`deploy/`](deploy/) folder for a complete `Modelfile`, NUC install script, and OpenClaw integration config. + +``` +FROM ./phi4-mm-Q4_K_M.gguf +PARAMETER num_ctx 8192 +PARAMETER num_thread 8 +PARAMETER num_gpu 0 +PARAMETER flash_attn false +PARAMETER temperature 0.7 +``` + +--- + +## Architecture + +| Property | Value | +|----------|-------| +| Base model | Phi-4-Mini (3.8 B LLM backbone) | +| Total parameters | ~5.6 B | +| GGUF arch | `phi3` | +| Context length | 128 K tokens (131,072) | +| Modalities | Text, Vision (SigLIP-400M), Audio/Speech | + +The vision encoder (`mmproj-phi4-mm-f16.gguf`) is a SigLIP-400M encoder finetuned +with LLM2CLIP, with a 2-layer MLP projector. Audio/speech is **not embedded in the GGUF** — see the audio limitation callout above. + +--- + +## Conversion Details + +| Item | Value | +|------|-------| +| Converter | `llama.cpp convert_hf_to_gguf.py` (text) + custom mmproj converter | +| llama.cpp build | b8347 / `fc350fdf9` | +| Source | [microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) | +| Hardware | RTX 5090 32 GB, CUDA 12.0, WSL2 Ubuntu 24.04 | + +Quantization commands: + +```bash +# Q8_0 +llama-quantize phi4-mm-f16.gguf phi4-mm-Q8_0.gguf Q8_0 + +# Q4_K_M +llama-quantize phi4-mm-f16.gguf phi4-mm-Q4_K_M.gguf Q4_K_M +``` + +--- + +## NUC / Edge Deployment + +The `Q4_K_M + mmproj-phi4-mm-f16.gguf` combination (~3,500 MiB VRAM) fits on: + +- Intel NUC 13/14 Pro (Intel Arc iGPU, 4–8 GB shared VRAM) +- Systems with 8 GB unified memory (Apple Silicon M-series, etc.) + +See [`deploy/`](deploy/) for install scripts and configuration. + +--- + +## Related + +- [microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) — original model weights +- [llama.cpp](https://github.com/ggerganov/llama.cpp) — inference engine +- [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) — multimodal evaluation harness diff --git a/deploy/Modelfile b/deploy/Modelfile new file mode 100644 index 0000000..9df23de --- /dev/null +++ b/deploy/Modelfile @@ -0,0 +1,36 @@ +# Ollama Modelfile — Phi-4 Multimodal Instruct Q4_K_M +# Optimised for: Intel 11th Gen NUC, 8 GB RAM, CPU-only +# +# Source model : microsoft/Phi-4-multimodal-instruct +# License : MIT https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/LICENSE +# Quantization : Q4_K_M via llama.cpp llama-quantize +# Architecture : phi3 (3.8B LLM backbone + vision/speech adapters in base GGUF) + +FROM ./phi4-mm-Q4_K_M.gguf + +# ── Context & KV cache ─────────────────────────────────────────────────────── +# 8 192 tokens balances capability vs RAM on 8 GB hardware. +# Lower to 4096 if you observe OOM / heavy swapping. +PARAMETER num_ctx 8192 + +# ── CPU tuning ─────────────────────────────────────────────────────────────── +# 11th Gen NUC typically 4 cores / 8 logical threads (i5/i7-1135G7 / 1165G7). +# Reduce to 4 if the NUC is a Core i3 variant. +PARAMETER num_thread 8 + +# No discrete GPU — all layers run on CPU. +PARAMETER num_gpu 0 + +# Flash attention is a GPU feature; disable for CPU inference. +PARAMETER flash_attn false + +# ── Generation defaults ─────────────────────────────────────────────────────── +PARAMETER temperature 0.7 +PARAMETER top_p 0.9 +PARAMETER repeat_penalty 1.1 +PARAMETER stop "<|end|>" +PARAMETER stop "<|user|>" +PARAMETER stop "<|assistant|>" + +# ── System prompt ───────────────────────────────────────────────────────────── +SYSTEM """You are a helpful, accurate, and concise AI assistant. You excel at reasoning, analysis, writing, coding, and answering questions. Be direct and thorough.""" diff --git a/deploy/README-deployment.md b/deploy/README-deployment.md new file mode 100644 index 0000000..a837a47 --- /dev/null +++ b/deploy/README-deployment.md @@ -0,0 +1,150 @@ +--- +license: mit +base_model: microsoft/Phi-4-multimodal-instruct +tags: + - phi4mm + - gguf + - quantized + - q4_k_m + - cpu + - ollama + - llama-cpp +language: + - multilingual + - ar + - zh + - cs + - da + - nl + - en + - fi + - fr + - de + - he + - hu + - it + - ja + - ko + - "no" + - pl + - pt + - ru + - es + - sv + - th + - tr + - uk +pipeline_tag: text-generation +--- + +# Phi-4 Multimodal Instruct — GGUF Quantizations + +CPU-optimised GGUF quantizations of [microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) produced with [llama.cpp](https://github.com/ggerganov/llama.cpp). + +## Model Summary + +Phi-4-multimodal-instruct is a **5.6 B parameter** lightweight multimodal foundation model by Microsoft that processes text, image, and audio inputs. The backbone LLM is **Phi-4-Mini** (3.8 B). This repository contains GGUF quantizations for local CPU and GPU deployment. + +- **Context length:** 128 K tokens (131 072) +- **Architecture:** `phi3` (GGUF) +- **License:** [MIT](https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/main/LICENSE) — © Microsoft Corporation + +## Available Files + +| File | Quant | Size | BPW | Best for | +|------|-------|------|-----|----------| +| `phi4-mm-Q4_K_M.gguf` | Q4_K_M | 2.37 GB | 5.18 | CPU inference, 8 GB RAM systems | +| `phi4-mm-Q8_0.gguf` | Q8_0 | 3.90 GB | 8.00 | GPU/high-RAM systems, near-lossless | +| `phi4-mm-f16.gguf` | F16 | 7.17 GB | 16.00 | Source / re-quantization base | + +## Quantization Method + +Quantized using `llama-quantize` from [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp) (build 8334). + +```bash +llama-quantize phi4-mm-f16.gguf phi4-mm-Q4_K_M.gguf Q4_K_M 16 +``` + +Source weights converted from the original safetensors using `convert_hf_to_gguf.py`. + +## Recommended Usage + +### Ollama (CPU — Intel NUC / low-power hardware) + +```bash +# Pull and run (once uploaded to Ollama registry) +ollama run phi4-mm-nuc + +# Or import from GGUF directly with a Modelfile: +ollama create phi4-mm-nuc -f Modelfile +ollama run phi4-mm-nuc +``` + +**Modelfile for 8 GB RAM / no GPU:** + +``` +FROM phi4-mm-Q4_K_M.gguf +PARAMETER num_ctx 8192 +PARAMETER num_thread 8 +PARAMETER num_gpu 0 +PARAMETER flash_attn false +PARAMETER temperature 0.7 +PARAMETER repeat_penalty 1.1 +SYSTEM "You are a helpful, accurate, and concise AI assistant." +``` + +### llama.cpp CLI (GPU, full quality) + +```bash +./build/bin/llama-cli \ + -m phi4-mm-Q8_0.gguf \ + --ctx-size 65536 \ + --flash-attn on \ + --kv-offload \ + -ngl 99 \ + --threads 16 +``` + +### OpenClaw Integration + +In `~/.openclaw/openclaw.json`: + +```json +{ + "agent": { + "model": "ollama/phi4-mm-nuc" + }, + "modelConfigs": { + "ollama/phi4-mm-nuc": { + "provider": "ollama", + "model": "phi4-mm-nuc", + "baseUrl": "http://localhost:11434" + } + } +} +``` + +## Hardware Notes + +| Hardware | Recommended quant | Context | Notes | +|---|---|---|---| +| Intel NUC 11th Gen, 8 GB RAM | Q4_K_M | 8 192 | CPU-only, `num_gpu 0` | +| Laptop / desktop, 16 GB RAM | Q5_K_M or Q8_0 | 16 384 | CPU or iGPU | +| GPU with ≥ 8 GB VRAM | Q8_0 or F16 | 32 768–65 536 | Full `-ngl 99` offload | + +## License + +This repository redistributes quantized weights derived from [microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) under the original **MIT License**. + +``` +MIT License +Copyright (c) Microsoft Corporation. +``` + +Quantization tooling (llama.cpp) is also MIT licensed. See [llama.cpp LICENSE](https://github.com/ggerganov/llama.cpp/blob/master/LICENSE). + +## Attribution + +- Original model: Microsoft Research +- Quantization: produced with [llama.cpp](https://github.com/ggerganov/llama.cpp) by Georgi Gerganov et al. +- Technical report: [arXiv:2503.01743](https://arxiv.org/abs/2503.01743) diff --git a/deploy/install-nuc.sh b/deploy/install-nuc.sh new file mode 100644 index 0000000..01b4665 --- /dev/null +++ b/deploy/install-nuc.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# ============================================================================= +# install-nuc.sh +# Deploy Phi-4 Multimodal Q4_K_M to an Intel NUC (8 GB RAM, CPU-only) +# and register it with Ollama for use with OpenClaw. +# +# Usage: +# On the NUC, run: +# bash install-nuc.sh [path/to/phi4-mm-Q4_K_M.gguf] +# +# Or from the build machine, copy files first then run: +# scp phi4-mm-Q4_K_M.gguf Modelfile openclaw-config.json user@nuc-ip:~/phi4-deploy/ +# ssh user@nuc-ip "cd ~/phi4-deploy && bash install-nuc.sh" +# ============================================================================= +set -euo pipefail + +GGUF_DEFAULT="/home/$(whoami)/phi4-deploy/phi4-mm-Q4_K_M.gguf" +GGUF_PATH="${1:-$GGUF_DEFAULT}" +MODELFILE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODEL_NAME="phi4-mm-nuc" + +echo "============================================================" +echo " Phi-4 Multimodal NUC Deployment" +echo "============================================================" +echo " GGUF : $GGUF_PATH" +echo " Model : $MODEL_NAME" +echo " Host : $(hostname)" +echo "============================================================" + +# ── 1. Verify GGUF exists ───────────────────────────────────────────────────── +if [[ ! -f "$GGUF_PATH" ]]; then + echo "ERROR: GGUF not found at $GGUF_PATH" + echo "Transfer it first: scp :~/models/phi4-mm-Q4_K_M.gguf $GGUF_PATH" + exit 1 +fi + +# ── 2. Install Ollama if missing ────────────────────────────────────────────── +if ! command -v ollama &>/dev/null; then + echo "" + echo ">>> Installing Ollama..." + curl -fsSL https://ollama.com/install.sh | sh + echo ">>> Ollama installed." +else + echo ">>> Ollama already installed: $(ollama --version)" +fi + +# ── 3. Ensure Ollama daemon is running ──────────────────────────────────────── +if ! pgrep -x ollama &>/dev/null; then + echo ">>> Starting Ollama daemon..." + ollama serve &>/dev/null & + sleep 3 +fi + +# ── 4. Patch Modelfile with absolute GGUF path ─────────────────────────────── +MODELFILE_TMP="$(mktemp)" +sed "s|FROM ./phi4-mm-Q4_K_M.gguf|FROM ${GGUF_PATH}|" \ + "${MODELFILE_DIR}/Modelfile" > "$MODELFILE_TMP" + +# ── 5. Register model with Ollama ───────────────────────────────────────────── +echo "" +echo ">>> Creating Ollama model: $MODEL_NAME" +ollama create "$MODEL_NAME" -f "$MODELFILE_TMP" +rm "$MODELFILE_TMP" +echo ">>> Model registered." + +# ── 6. Smoke-test ───────────────────────────────────────────────────────────── +echo "" +echo ">>> Running smoke test (one token)..." +RESPONSE=$(ollama run "$MODEL_NAME" "Reply with exactly: OK" --nowordwrap 2>&1 | head -5) +echo " Response: $RESPONSE" + +# ── 7. Systemd user service for Ollama (optional) ───────────────────────────── +if command -v systemctl &>/dev/null && systemctl --user status ollama.service &>/dev/null 2>&1; then + echo ">>> Ollama systemd service already configured." +else + echo "" + echo ">>> To keep Ollama running at startup, enable the service:" + echo " sudo systemctl enable --now ollama" +fi + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo "============================================================" +echo " DONE. Model is ready." +echo "" +echo " Test: ollama run $MODEL_NAME" +echo " API endpoint: http://localhost:11434/v1" +echo " OpenClaw model: ollama/$MODEL_NAME" +echo "============================================================" diff --git a/deploy/openclaw-config.json b/deploy/openclaw-config.json new file mode 100644 index 0000000..6231f3d --- /dev/null +++ b/deploy/openclaw-config.json @@ -0,0 +1,12 @@ +{ + "agent": { + "model": "ollama/phi4-mm-nuc" + }, + "modelConfigs": { + "ollama/phi4-mm-nuc": { + "provider": "ollama", + "model": "phi4-mm-nuc", + "baseUrl": "http://localhost:11434" + } + } +} diff --git a/mmproj-phi4-mm-f16.gguf b/mmproj-phi4-mm-f16.gguf new file mode 100644 index 0000000..aba1e5a --- /dev/null +++ b/mmproj-phi4-mm-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4a05e61d2485afd651d69bc2a69c1b79e06a6661689523c3a17124cf9cf40a3 +size 825297088 diff --git a/phi4-mm-Q4_K_M.gguf b/phi4-mm-Q4_K_M.gguf new file mode 100644 index 0000000..465cdc4 --- /dev/null +++ b/phi4-mm-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6ce167990dc4b73dcff7af00c62630c20fb5d40035affdd8bb77ace195c726 +size 2493840768 diff --git a/phi4-mm-Q8_0.gguf b/phi4-mm-Q8_0.gguf new file mode 100644 index 0000000..a9fbd52 --- /dev/null +++ b/phi4-mm-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06fe4c0f9537fcac17930a8f75d91eaf76419bc3b765ac203ca37da540f27ea9 +size 4084611456 diff --git a/phi4-mm-f16.gguf b/phi4-mm-f16.gguf new file mode 100644 index 0000000..c2195de --- /dev/null +++ b/phi4-mm-f16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:548ec43dacb57b41cf742fdb0d893edfe6096706ccd372df4e41fa5dfe4f5d68 +size 7680694656