Update for Transformers AWQ support

2023-11-09 18:20:20 +00:00
parent 6ac73a5842
commit 9be86ac7e8
1 changed files with 29 additions and 22 deletions
--- a/config.json
+++ b/config.json
@@ -20,5 +20,12 @@
  "transformers_version": "4.30.2",
  "use_cache": true,
  "vocab_size": 32000,
-    "pretraining_tp": 1
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "quant_method": "awq",
+    "zero_point": true,
+    "group_size": 128,
+    "bits": 4,
+    "version": "gemm"
+  }
 }