初始化项目，由ModelHub XC社区提供模型

Model: tclf90/llama-3-chinese-8b-instruct-v3-GPTQ-Int4
Source: Original Platform

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,117 @@
+META LLAMA 3 COMMUNITY LICENSE AGREEMENT
+Meta Llama 3 Version Release Date: April 18, 2024
+
+“Agreement” means the terms and conditions for use, reproduction, distribution and modification of the
+Llama Materials set forth herein.
+
+“Documentation” means the specifications, manuals and documentation accompanying Meta Llama 3
+distributed by Meta at https://llama.meta.com/get-started/.
+
+“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into
+this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or
+regulations to provide legal consent and that has legal authority to bind your employer or such other
+person or entity if you are entering in this Agreement on their behalf.
+
+“Meta Llama 3” means the foundational large language models and software and algorithms, including
+machine-learning model code, trained model weights, inference-enabling code, training-enabling code,
+fine-tuning enabling code and other elements of the foregoing distributed by Meta at
+https://llama.meta.com/llama-downloads.
+
+“Llama Materials” means, collectively, Meta’s proprietary Meta Llama 3 and Documentation (and any
+portion thereof) made available under this Agreement.
+
+“Meta” or “we” means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your
+principal place of business is in the EEA or Switzerland) and Meta Platforms, Inc. (if you are located
+outside of the EEA or Switzerland).
+
+By clicking “I Accept” below or by using or distributing any portion or element of the Llama Materials,
+you agree to be bound by this Agreement.
+
+1. License Rights and Redistribution.
+
+  a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free
+limited license under Meta’s intellectual property or other rights owned by Meta embodied in the Llama
+Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the
+Llama Materials.
+
+  b. Redistribution and Use.
+
+      i. If you distribute or make available the Llama Materials (or any derivative works
+thereof), or a product or service that uses any of them, including another AI model, you shall (A) provide
+a copy of this Agreement with any such Llama Materials; and (B) prominently display “Built with Meta
+Llama 3” on a related website, user interface, blogpost, about page, or product documentation. If you
+use the Llama Materials to create, train, fine tune, or otherwise improve an AI model, which is
+distributed or made available, you shall also include “Llama 3” at the beginning of any such AI model
+name.
+
+      ii. If you receive Llama Materials, or any derivative works thereof, from a Licensee as part 
+of an integrated end user product, then Section 2 of this Agreement will not apply to you.
+
+      iii. You must retain in all copies of the Llama Materials that you distribute the following
+attribution notice within a “Notice” text file distributed as a part of such copies: “Meta Llama 3 is
+licensed under the Meta Llama 3 Community License, Copyright © Meta Platforms, Inc. All Rights
+Reserved.”
+
+      iv. Your use of the Llama Materials must comply with applicable laws and regulations
+(including trade compliance laws and regulations) and adhere to the Acceptable Use Policy for the Llama
+Materials (available at https://llama.meta.com/llama3/use-policy), which is hereby incorporated by
+reference into this Agreement.
+
+      v. You will not use the Llama Materials or any output or results of the Llama Materials to
+improve any other large language model (excluding Meta Llama 3 or derivative works thereof).
+
+2. Additional Commercial Terms. If, on the Meta Llama 3 version release date, the monthly active users
+of the products or services made available by or for Licensee, or Licensee’s affiliates, is greater than 700
+million monthly active users in the preceding calendar month, you must request a license from Meta,
+which Meta may grant to you in its sole discretion, and you are not authorized to exercise any of the
+rights under this Agreement unless or until Meta otherwise expressly grants you such rights.
+
+3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE LLAMA MATERIALS AND ANY
+OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF
+ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED,
+INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT,
+MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR
+DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE LLAMA MATERIALS AND
+ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE LLAMA MATERIALS AND ANY OUTPUT AND
+RESULTS.
+
+4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING
+OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, CONSEQUENTIAL,
+INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED
+OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+
+5. Intellectual Property.
+
+  a. No trademark licenses are granted under this Agreement, and in connection with the Llama
+Materials, neither Meta nor Licensee may use any name or mark owned by or associated with the other
+or any of its affiliates, except as required for reasonable and customary use in describing and
+redistributing the Llama Materials or as set forth in this Section 5(a). Meta hereby grants you a license to
+use “Llama 3” (the “Mark”) solely as required to comply with the last sentence of Section 1.b.i. You will
+comply with Meta’s brand guidelines (currently accessible at
+https://about.meta.com/brand/resources/meta/company-brand/ ). All goodwill arising out of your use
+of the Mark will inure to the benefit of Meta.
+
+  b. Subject to Meta’s ownership of Llama Materials and derivatives made by or for Meta, with
+respect to any derivative works and modifications of the Llama Materials that are made by you, as
+between you and Meta, you are and will be the owner of such derivative works and modifications.
+
+  c. If you institute litigation or other proceedings against Meta or any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Llama Materials or Meta Llama 3 outputs or
+results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other
+rights owned or licensable by you, then any licenses granted to you under this Agreement shall
+terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold
+harmless Meta from and against any claim by any third party arising out of or related to your use or
+distribution of the Llama Materials.
+
+6. Term and Termination. The term of this Agreement will commence upon your acceptance of this
+Agreement or access to the Llama Materials and will continue in full force and effect until terminated in
+accordance with the terms and conditions herein. Meta may terminate this Agreement if you are in
+breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete
+and cease use of the Llama Materials. Sections 3, 4 and 7 shall survive the termination of this
+Agreement.
+
+7. Governing Law and Jurisdiction. This Agreement will be governed and construed under the laws of
+the State of California without regard to choice of law principles, and the UN Convention on Contracts
+for the International Sale of Goods does not apply to this Agreement. The courts of California shall have
+exclusive jurisdiction of any dispute arising out of this Agreement.

README.md

@@ -0,0 +1,119 @@
+---
+license: llama3
+license_name: llama3
+license_link: LICENSE
+pipeline_tag: text-generation
+tags:
+- llama3
+- gptq
+- int4
+- 量化修复
+- vLLM
+---
+
+# Llama-3-chinese-8b-instruct-v3-GPTQ-Int4-量化修复
+原模型 [ChineseAlpacaGroup/llama-3-chinese-8b-instruct-v3](https://www.modelscope.cn/models/ChineseAlpacaGroup/llama-3-chinese-8b-instruct-v3)
+
+
+### 【模型更新日期】
+``` 2024-06-02 ```
+
+### 【模型大小】
+`6.2GB`
+
+
+### 【介绍】
+
+Llama-3-Chinese-8B-Instruct-v3（指令模型），融合了v1、v2以及Meta原版Instruct模型，在中文任务上大幅超越v1/v2版，英文任务上与Meta原版保持持平，主观体验效果显著提升。
+
+[更多详情...](https://github.com/ymcui/Chinese-LLaMA-Alpaca-3/releases/tag/v3.0)
+
+### 【量化修复】
+
+调优了现有 `AWQ` 与 `GPTQ` 量化算法的量化策略。带有`量化修复`标签的`Int3`模型，可以比肩默认`AWQ`与`GPTQ`算法的`Int8`模型的能力。
+
+1. 量化修复可以极大减少模型的`1.乱吐字`、`2.无限循环`、`3.长文能力丢失`等量化损失造成的模型不可用的情况。
+
+2. 调优后的量化模型，`AWQ`与`GPTQ`模型在能力上没有表现出明显区别。同时考虑到`GPTQ`的`vLLM`引擎的并发推理效率最好，所以不再制作`AWQ`模型。
+
+3. 调优后的量化算法，`int4`与`int3`在大尺寸的（30B+）模型上没有表现出明显区别。
+
+4. 小尺寸的模型，会争取采用`int4`与`group_size=32`的配置，以尽最大可能减少量化造成的损失。
+
+
+### 【同期量化修复模型】
+
+| 模型名称                                                                                                                            | 磁盘大小（GB） |
+|---------------------------------------------------------------------------------------------------------------------------------|----------|
+| [零一万物-1.5-6B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Yi-1.5-6B-Chat-GPTQ-Int3)                             | 3.3      |
+| [零一万物-1.5-9B-Chat-16K-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Yi-1.5-9B-Chat-16K-GPTQ-Int3)                     | 4.4      |
+| [零一万物-1.5-34B-Chat-16K-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Yi-1.5-34B-Chat-16K-GPTQ-Int3)                   | 15.1     |
+| [通义千问1.5-7B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Qwen1.5-7B-Chat-GPTQ-Int3)                             | 5.1      |
+| [通义千问1.5-14B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Qwen1.5-14B-Chat-GPTQ-Int3)                           | 8.1      |
+| [通义千问1.5-32B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Qwen1.5-32B-Chat-GPTQ-Int3)                           | 15.4     |
+| [通义千问1.5-72B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Qwen1.5-72B-Chat-GPTQ-Int3)                           | 32.5     |
+| [通义千问1.5-110B-Chat-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/Qwen1.5-110B-Chat-GPTQ-Int3)                         | 47.9     |
+| [openbuddy-llama3-70b-v21.1-8k-GPTQ-Int3-量化修复](https://www.modelscope.cn/models/tclf90/openbuddy-llama3-70b-v21.1-8k-GPTQ-Int3) | 31.5     |
+
+
+### 【模型下载】
+```python
+from modelscope import snapshot_download
+model_dir = snapshot_download('tclf90/模型名', cache_dir="本地路径")
+```
+
+### 【[vLLM](https://github.com/vllm-project/vllm)推理（目前仅限Linux）】
+#### 1. Python 简易调试
+
+```python
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+
+max_model_len, tp_size = 4000, 1
+model_name = "本地路径/tclf90/模型名称"   # 例："./my_models/tclf90/Qwen1.5-32B-Chat-GPTQ-Int3"
+model_name = model_name.replace('.', '___')
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True, enforce_eager=True)
+sampling_params = SamplingParams(temperature=0.7, max_tokens=256, stop_token_ids=[tokenizer.eos_token_id])
+
+messages_list = [
+    [{"role": "user", "content": "你是谁"}],
+    [{"role": "user", "content": "介绍一下你自己"}],
+    [{"role": "user", "content": "用python写一个快排函数"}],
+]
+
+prompt_token_ids = [tokenizer.apply_chat_template(messages, add_generation_prompt=True) for messages in messages_list]
+
+outputs = llm.generate(prompt_token_ids=prompt_token_ids, sampling_params=sampling_params)
+
+generated_text = [output.outputs[0].text for output in outputs]
+print(generated_text)
+```
+
+#### 2. 类ChatGPT RESTFul API Server
+```
+>>> python -m vllm.entrypoints.openai.api_server --model 本地路径/tclf90/模型名称
+```
+
+### 【Transformer推理】
+
+```python
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
+
+model_name = "本地路径/tclf90/模型名称"   # 例："./my_models/tclf90/Qwen1.5-32B-Chat-GPTQ-Int3"
+model_name = model_name.replace('.', '___')
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.bfloat16).cuda()
+model.generation_config = GenerationConfig.from_pretrained(model_name)
+model.generation_config.pad_token_id = model.generation_config.eos_token_id
+
+messages = [
+    {"role": "user", "content": "你好你是谁"}
+]
+input_tensor = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
+outputs = model.generate(input_tensor.to(model.device), max_new_tokens=100)
+
+result = tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=True)
+print(result)
+```

config.json

@@ -0,0 +1,53 @@
+{
+  "_name_or_path": "tclf90/llama-3-chinese-8b-instruct-v3-GPTQ-Int4",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "batch_size": 1,
+    "bits": 4,
+    "block_name_to_quantize": null,
+    "cache_block_outputs": true,
+    "damp_percent": 0.1,
+    "dataset": null,
+    "desc_act": false,
+    "exllama_config": {
+      "version": 1
+    },
+    "group_size": 32,
+    "max_input_length": null,
+    "model_seqlen": null,
+    "module_name_preceding_first_block": null,
+    "modules_in_block_to_quantize": null,
+    "pad_token_id": null,
+    "quant_method": "gptq",
+    "sym": true,
+    "tokenizer": null,
+    "true_sequential": true,
+    "use_cuda_fp16": true,
+    "use_exllama": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.41.2",
+  "use_cache": true,
+  "vocab_size": 128256
+}

configuration.json

@@ -0,0 +1,4 @@
+{
+  "framework": "Pytorch",
+  "task": "text-generation"
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128009
+  ],
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:997ad678e5a921ad2d10dc97128dd18d3a8343cadd41019a19194ec897914008
+size 4991967048

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88befc7e8962f501586c9395cca75838d1f8ff32f2ad57d936076bb0273f4b47
+size 1152698392

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}