diff --git a/.gitattributes b/.gitattributes index b7e9e3a..615bf4b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,37 +1,44 @@ *.7z filter=lfs diff=lfs merge=lfs -text *.arrow filter=lfs diff=lfs merge=lfs -text *.bin filter=lfs diff=lfs merge=lfs -text -*.bin.* filter=lfs diff=lfs merge=lfs -text *.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text *.ftz filter=lfs diff=lfs merge=lfs -text *.gz filter=lfs diff=lfs merge=lfs -text *.h5 filter=lfs diff=lfs merge=lfs -text *.joblib filter=lfs diff=lfs merge=lfs -text *.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text *.model filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.ot filter=lfs diff=lfs merge=lfs -text *.parquet filter=lfs diff=lfs merge=lfs -text *.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text *.pt filter=lfs diff=lfs merge=lfs -text *.pth filter=lfs diff=lfs merge=lfs -text *.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text *.tflite filter=lfs diff=lfs merge=lfs -text *.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text *.xz filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text -*.zstandard filter=lfs diff=lfs merge=lfs -text -*.tfevents* filter=lfs diff=lfs merge=lfs -text -*.db* filter=lfs diff=lfs merge=lfs -text -*.ark* filter=lfs diff=lfs merge=lfs -text -**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text -**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text -**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.gguf* filter=lfs diff=lfs merge=lfs -text -*.ggml filter=lfs diff=lfs merge=lfs -text -*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-fp16.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q2_k.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q4_0.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q5_0.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q6_k.gguf filter=lfs diff=lfs merge=lfs -text +internlm2_5-7b-chat-q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 6773761..26b8685 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,151 @@ --- -license: Apache License 2.0 - -#model-type: -##如 gpt、phi、llama、chatglm、baichuan 等 -#- gpt - -#domain: -##如 nlp、cv、audio、multi-modal -#- nlp - -#language: -##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa -#- cn - -#metrics: -##如 CIDEr、Blue、ROUGE 等 -#- CIDEr - -#tags: -##各种自定义，包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他 -#- pretrained - -#tools: -##如 vllm、fastchat、llamacpp、AdaSeq 等 -#- vllm +license: apache-2.0 +language: +- en +pipeline_tag: text-generation +tags: +- chat --- -### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重，可浏览“模型文件”页面获取。 -#### 您可以通过如下git clone命令，或者ModelScope SDK来下载模型 +# InternLM2.5-7B-Chat GGUF Model -SDK下载 -```bash -#安装ModelScope -pip install modelscope +## Introduction + +The `internlm2_5-7b-chat` model in GGUF format can be utilized by [llama.cpp](https://github.com/ggerganov/llama.cpp), a highly popular open-source framework for Large Language Model (LLM) inference, across a variety of hardware platforms, both locally and in the cloud. +This repository offers `internlm2_5-7b-chat` models in GGUF format in both half precision and various low-bit quantized versions, including `q5_0`, `q5_k_m`, `q6_k`, and `q8_0`. + +In the subsequent sections, we will first present the installation procedure, followed by an explanation of the model download process. +And finally we will illustrate the methods for model inference and service deployment through specific examples. + +## Installation + +We recommend building `llama.cpp` from source. The following code snippet provides an example for the Linux CUDA platform. For instructions on other platforms, please refer to the [official guide](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#build). + +- Step 1: create a conda environment and install cmake + +```shell +conda create --name internlm2 python=3.10 -y +conda activate internlm2 +pip install cmake ``` + +- Step 2: clone the source code and build the project + +```shell +git clone --depth=1 https://github.com/ggerganov/llama.cpp.git +cd llama.cpp +cmake -B build -DGGML_CUDA=ON +cmake --build build --config Release -j +``` + +All the built targets can be found in the sub directory `build/bin` + +In the following sections, we assume that the working directory is at the root directory of `llama.cpp`. + +## Download models + +In the [introduction section](#introduction), we mentioned that this repository includes several models with varying levels of computational precision. You can download the appropriate model based on your requirements. +For instance, `internlm2_5-7b-chat-fp16.gguf` can be downloaded as below： + +```shell +pip install huggingface-hub +huggingface-cli download internlm/internlm2_5-7b-chat-gguf internlm2_5-7b-chat-fp16.gguf --local-dir . --local-dir-use-symlinks False +``` + +## Inference + +You can use `llama-cli` for conducting inference. For a detailed explanation of `llama-cli`, please refer to [this guide](https://github.com/ggerganov/llama.cpp/blob/master/examples/main/README.md) + +### chat example + +```shell +build/bin/llama-cli \ + --model internlm2_5-7b-chat-fp16.gguf  \ + --predict 512 \ + --ctx-size 4096 \ + --gpu-layers 32 \ + --temp 0.8 \ + --top-p 0.8 \ + --top-k 50 \ + --seed 1024 \ + --color \ + --prompt "<|im_start|>system\nYou are an AI assistant whose name is InternLM (书生·浦语).\n- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.<|im_end|>\n" \ + --interactive \ + --multiline-input \ + --conversation \ + --verbose \ + --logdir workdir/logdir \ + --in-prefix "<|im_start|>user\n" \ + --in-suffix "<|im_end|>\n<|im_start|>assistant\n" +``` + +### Function call example + +`llama-cli` example: + +```shell +build/bin/llama-cli \ + --model internlm2_5-7b-chat-fp16.gguf \ + --predict 512 \ + --ctx-size 4096 \ + --gpu-layers 32 \ + --temp 0.8 \ + --top-p 0.8 \ + --top-k 50 \ + --seed 1024 \ + --color \ + --prompt '<|im_start|>system\nYou are InternLM2-Chat, a harmless AI assistant.<|im_end|>\n<|im_start|>system name=<|plugin|>[{"name": "get_current_weather", "parameters": {"required": ["location"], "type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string"}}}, "description": "Get the current weather in a given location"}]<|im_end|>\n<|im_start|>user\n' \ + --interactive \ + --multiline-input \ + --conversation \ + --verbose \ + --in-suffix "<|im_end|>\n<|im_start|>assistant\n" \ + --special +``` + +Conversation results: + +```text +<|im_start|>system +You are InternLM2-Chat, a harmless AI assistant.<|im_end|> +<|im_start|>system name=<|plugin|>[{"name": "get_current_weather", "parameters": {"required": ["location"], "type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string"}}}, "description": "Get the current weather in a given location"}]<|im_end|> +<|im_start|>user + +> I want to know today's weather in Shanghai +I need to use the get_current_weather function to get the current weather in Shanghai.<|action_start|><|plugin|> +{"name": "get_current_weather", "parameters": {"location": "Shanghai"}}<|action_end|> +<|im_end|> + +> <|im_start|>environment name=<|plugin|>\n{"temperature": 22} +The current temperature in Shanghai is 22 degrees Celsius.<|im_end|> + +> +``` + +## Serving + +`llama.cpp` provides an OpenAI API compatible server - `llama-server`. You can deploy `internlm2_5-7b-chat-fp16.gguf` into a service like this: + +```shell +./build/bin/llama-server -m ./internlm2_5-7b-chat-fp16.gguf -ngl 32 +``` + +At the client side, you can access the service through OpenAI API: + ```python -#SDK模型下载 -from modelscope import snapshot_download -model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2_5-7b-chat-gguf') +from openai import OpenAI +client = OpenAI( + api_key='YOUR_API_KEY', + base_url='http://localhost:8080/v1' +) +model_name = client.models.list().data[0].id +response = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": " provide three suggestions about time management"}, + ], + temperature=0.8, + top_p=0.8 +) +print(response) ``` -Git下载 -``` -#Git模型下载 -git clone https://www.modelscope.cn/Shanghai_AI_Laboratory/internlm2_5-7b-chat-gguf.git -``` - -

如果您是本模型的贡献者，我们邀请您根据模型贡献文档，及时完善模型卡片内容。

\ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/internlm2_5-7b-chat-fp16.gguf b/internlm2_5-7b-chat-fp16.gguf new file mode 100644 index 0000000..2dbaf52 --- /dev/null +++ b/internlm2_5-7b-chat-fp16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4fd4abe9bccedc93a5c95e0b7a14da749a1a8d6ec672f42fea422634c0ec7e +size 15478092608 diff --git a/internlm2_5-7b-chat-q2_k.gguf b/internlm2_5-7b-chat-q2_k.gguf new file mode 100644 index 0000000..e5dbfab --- /dev/null +++ b/internlm2_5-7b-chat-q2_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c61ec5054f4396e45e918d79ab7c69142a1de3bab5d6e0cdfd8d69743841778 +size 3005449024 diff --git a/internlm2_5-7b-chat-q3_k_m.gguf b/internlm2_5-7b-chat-q3_k_m.gguf new file mode 100644 index 0000000..08f30fa --- /dev/null +++ b/internlm2_5-7b-chat-q3_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:460b50cce3a2811a2bf5f11b95e658902c166aed99f146e9a0ce4690b3832e26 +size 3830379328 diff --git a/internlm2_5-7b-chat-q4_0.gguf b/internlm2_5-7b-chat-q4_0.gguf new file mode 100644 index 0000000..9ded36a --- /dev/null +++ b/internlm2_5-7b-chat-q4_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de1b8fef5373d6977345a5133def71c8ead427c27ef36232627f0d6365d4e277 +size 4453245760 diff --git a/internlm2_5-7b-chat-q4_k_m.gguf b/internlm2_5-7b-chat-q4_k_m.gguf new file mode 100644 index 0000000..6c18da3 --- /dev/null +++ b/internlm2_5-7b-chat-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:450ef0b157ee10586bb8fce4c5173ae1799345c43c537a6a7353518df1079f55 +size 4712768320 diff --git a/internlm2_5-7b-chat-q5_0.gguf b/internlm2_5-7b-chat-q5_0.gguf new file mode 100644 index 0000000..0b428f8 --- /dev/null +++ b/internlm2_5-7b-chat-q5_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8456f5744f6854c15ba91923a54bbf589eb742b55910c35ba7f2467d5fc893 +size 5373043520 diff --git a/internlm2_5-7b-chat-q5_k_m.gguf b/internlm2_5-7b-chat-q5_k_m.gguf new file mode 100644 index 0000000..5f553aa --- /dev/null +++ b/internlm2_5-7b-chat-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:687ade10a70ecb22f9902362dd47a71e1ab6106eeee105f6beab359e984dfae6 +size 5506736960 diff --git a/internlm2_5-7b-chat-q6_k.gguf b/internlm2_5-7b-chat-q6_k.gguf new file mode 100644 index 0000000..68772f3 --- /dev/null +++ b/internlm2_5-7b-chat-q6_k.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c562773a51c707e2026fe783729735f39faaf43e635e8a5b1e7c5d76c5177551 +size 6350328640 diff --git a/internlm2_5-7b-chat-q8_0.gguf b/internlm2_5-7b-chat-q8_0.gguf new file mode 100644 index 0000000..bb7ec65 --- /dev/null +++ b/internlm2_5-7b-chat-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c17cb397a27875b584a00fd77a07c7ac401f749117f02e79a7610b6cf8ba855 +size 8224240448