初始化项目，由ModelHub XC社区提供模型

Model: dranger003/LWM-Text-Chat-128K-iMat.GGUF
Source: Original Platform

.gitattributes

@@ -0,0 +1,42 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-f16-imatrix.dat filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-f16.gguf filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-iq3_xxs.gguf filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+ggml-lwm-text-chat-128k-q6_k.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,17 @@
+---
+license: llama2
+pipeline_tag: text-generation
+library_name: gguf
+---
+GGUF importance matrix (imatrix) quants for https://huggingface.co/LargeWorldModel/LWM-Text-Chat-128K  
+The importance matrix was trained for 100K tokens (200 batches of 512 tokens) using wiki.train.raw.
+
+* The imatrix Q4-K quant fits with 32K context on 24GB and gives me ~100 t/s inference on a 3090.
+* With IQ3_XXS it seems to fit ~37K context on 24GB (and it is even faster than Q4-K).
+* With either quant on a 3090 it seems to decode context at well over 2000 t/s.
+* Using Q8 K-cache (instead of F16) you can fit up to 43-44K context but inference speed goes down a little bit.
+* Also for some reason I need to use 1.0 penalty to avoid the response being cut-off.
+
+| Layers | Context | [Template](https://github.com/LargeWorldModel/LWM/blob/9aaaa1e864bfcf31b66028e782395a22f4817535/scripts/eval_needle.py#L48) |
+| --- | --- | --- |
+| <pre>32</pre> | <pre>131072</pre> | <pre>You are a helpful assistant.<br>USER:<br>{context}<br>{question}<br>Don't give information outside the document or repeat your findings. Keep your response short and direct.<br>ASSISTANT:<br>{response}</pre> |

ggml-lwm-text-chat-128k-f16-imatrix.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c4ea6726cbd7e2bf994670530b70e235037b672bdf379a2fa3fb8d22d953c43
+size 4562142

ggml-lwm-text-chat-128k-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0af35bb3d600fb122bd4eca6a6a9c705d252b0f6b94460fb2021a888b6b06b18
+size 13478104864

ggml-lwm-text-chat-128k-iq3_xxs.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374fbdc0e6831bda5f5df4f953f2ae6059a743da70db3ec714c1d1875275debd
+size 2687315776

ggml-lwm-text-chat-128k-q4_k_m.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10d9e69ad5ed660b55bbd6efa638eeec0b6788e6e16d0b7f7069b1a47e95c57e
+size 4081004352

ggml-lwm-text-chat-128k-q5_k_m.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0e0ecdc2a8e95e043c6007b9a7c8e59f0b989c7964f1ecb56845b2c12adc3ec
+size 4783157056

ggml-lwm-text-chat-128k-q6_k.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1685e378ed42d495cbf7f4a58108112965a9b1750e568250f11c3e67b2d850f
+size 5529194304

ggml-lwm-text-chat-128k-q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab5f65ff614c4bd7b0e8521dd24cdbafc3511118ebe8a35721a0f98a3112912e
+size 7161089856