commit 3283159283538dd72258323b9656c071557f1ff6 Author: ModelHub XC Date: Sun Jul 19 00:14:13 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d838ac5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +open-insurance-llm-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +FP16.gguf filter=lfs diff=lfs merge=lfs -text +open-insurance-llm-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text +open-insurance-llm-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/FP16.gguf b/FP16.gguf new file mode 100644 index 0000000..e820f8b --- /dev/null +++ b/FP16.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73300a8fbb5dd1931d745624d516018194244df45cf843fcbdb02e410ef7b847 +size 16068891520 diff --git a/README.md b/README.md new file mode 100644 index 0000000..2f14458 --- /dev/null +++ b/README.md @@ -0,0 +1,363 @@ +--- +license: llama3 +language: +- en +library_name: transformers +pipeline_tag: text-generation +tags: +- Text Generation +- Transformers +- llama +- llama-3 +- 8B +- nvidia +- facebook +- meta +- LLM +- fine-tuned +- insurance +- research +- pytorch +- instruct +- chatqa-1.5 +- chatqa +- finetune +- gpt4 +- conversational +- text-generation-inference +- Inference Endpoints +datasets: +- InsuranceQA + +base_model: "nvidia/Llama3-ChatQA-1.5-8B" +finetuned: "Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B" +quantized: "Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF" +--- + +# Open-Insurance-LLM-Llama3-8B-GGUF + +This model is a GGUF-quantized version of an insurance domain-specific language model based on Nvidia Llama 3-ChatQA +Fine-tuned for insurance-related queries and conversations. + +# Other Quantized Variants + +[bartowski/Open-Insurance-LLM-Llama3-8B-GGUF](https://huggingface.co/bartowski/Open-Insurance-LLM-Llama3-8B-GGUF) + +## Model Details + +- **Model Type:** Quantized Language Model (GGUF format) +- **Base Model:** nvidia/Llama3-ChatQA-1.5-8B +- **Finetuned Model:** Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B +- **Quantized Model:** Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF +- **Model Architecture:** Llama +- **Quantization:** 8-bit (Q8_0), 5-bit (Q5_K_M), 4-bit (Q4_K_M), 16-bit +- **Finetuned Dataset**: InsuranceQA (https://github.com/shuzi/insuranceQA) +- **Developer:** Raj Maharajwala +- **License:** llama3 +- **Language:** English + +## Setup Instructions + +### Environment Setup + +#### For Windows +```bash +python3 -m venv .venv_open_insurance_llm +.\.venv_open_insurance_llm\Scripts\activate +``` + +#### For Mac/Linux +```bash +python3 -m venv .venv_open_insurance_llm +source .venv_open_insurance_llm/bin/activate +``` + +### Installation + +#### For Mac Users (Metal Support) +```bash +export FORCE_CMAKE=1 +CMAKE_ARGS="-DGGML_METAL=on" pip install --upgrade --force-reinstall llama-cpp-python==0.3.2 --no-cache-dir +``` + +#### For Windows Users (CPU Support) +```bash +pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu +``` + +### Dependencies + +Then install dependencies (inference_requirements.txt) attached under `Files and Versions`: +```bash +pip install -r inference_requirements.txt +``` + +## Inference Loop + +```python +# Attached under `Files and Versions` (inference_open-insurance-llm-gguf.py) +import os +import time +from pathlib import Path +from llama_cpp import Llama +from rich.console import Console +from huggingface_hub import hf_hub_download +from dataclasses import dataclass +from typing import List, Dict, Any, Tuple + +@dataclass +class ModelConfig: + # Optimized parameters for coherent responses and efficient performance on devices like MacBook Air M2 + model_name: str = "Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF" + model_file: str = "open-insurance-llm-q4_k_m.gguf" + # model_file: str = "open-insurance-llm-q8_0.gguf" # 8-bit quantization; higher precision, better quality, increased resource usage + # model_file: str = "open-insurance-llm-q5_k_m.gguf" # 5-bit quantization; balance between performance and resource efficiency + max_tokens: int = 1000 # Maximum number of tokens to generate in a single output + temperature: float = 0.1 # Controls randomness in output; lower values produce more coherent responses (performs scaling distribution) + top_k: int = 15 # After temperature scaling, Consider the top 15 most probable tokens during sampling + top_p: float = 0.2 # After reducing the set to 15 tokens, Uses nucleus sampling to select tokens with a cumulative probability of 20% + repeat_penalty: float = 1.2 # Penalize repeated tokens to reduce redundancy + num_beams: int = 4 # Number of beams for beam search; higher values improve quality at the cost of speed + n_gpu_layers: int = -2 # Number of layers to offload to GPU; -1 for full GPU utilization, -2 for automatic configuration + n_ctx: int = 2048 # Context window size; Llama 3 models support up to 8192 tokens context length + n_batch: int = 256 # Number of tokens to process simultaneously; adjust based on available hardware (suggested 512) + verbose: bool = False # True for enabling verbose logging for debugging purposes + use_mmap: bool = False # Memory-map model to reduce RAM usage; set to True if running on limited memory systems + use_mlock: bool = True # Lock model into RAM to prevent swapping; improves performance on systems with sufficient RAM + offload_kqv: bool = True # Offload key, query, value matrices to GPU to accelerate inference + + + +class InsuranceLLM: + def __init__(self, config: ModelConfig): + self.config = config + self.llm_ctx = None + self.console = Console() + self.conversation_history: List[Dict[str, str]] = [] + + self.system_message = ( + "This is a chat between a user and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. " + "The assistant should also indicate when the answer cannot be found in the context. " + "You are an expert from the Insurance domain with extensive insurance knowledge and " + "professional writer skills, especially about insurance policies. " + "Your name is OpenInsuranceLLM, and you were developed by Raj Maharajwala. " + "You are willing to help answer the user's query with a detailed explanation. " + "In your explanation, leverage your deep insurance expertise, such as relevant insurance policies, " + "complex coverage plans, or other pertinent insurance concepts. Use precise insurance terminology while " + "still aiming to make the explanation clear and accessible to a general audience." + ) + + def download_model(self) -> str: + try: + with self.console.status("[bold green]Downloading model..."): + model_path = hf_hub_download( + self.config.model_name, + filename=self.config.model_file, + local_dir=os.path.join(os.getcwd(), 'gguf_dir') + ) + return model_path + except Exception as e: + self.console.print(f"[red]Error downloading model: {str(e)}[/red]") + raise + + def load_model(self) -> None: + try: + quantized_path = os.path.join(os.getcwd(), "gguf_dir") + directory = Path(quantized_path) + + try: + model_path = str(list(directory.glob(self.config.model_file))[0]) + except IndexError: + model_path = self.download_model() + + with self.console.status("[bold green]Loading model..."): + self.llm_ctx = Llama( + model_path=model_path, + n_gpu_layers=self.config.n_gpu_layers, + n_ctx=self.config.n_ctx, + n_batch=self.config.n_batch, + num_beams=self.config.num_beams, + verbose=self.config.verbose, + use_mlock=self.config.use_mlock, + use_mmap=self.config.use_mmap, + offload_kqv=self.config.offload_kqv + ) + except Exception as e: + self.console.print(f"[red]Error loading model: {str(e)}[/red]") + raise + + def build_conversation_prompt(self, new_question: str, context: str = "") -> str: + prompt = f"System: {self.system_message}\n\n" + + # Add conversation history + for exchange in self.conversation_history: + prompt += f"User: {exchange['user']}\n\n" + prompt += f"Assistant: {exchange['assistant']}\n\n" + + # Add the new question + if context: + prompt += f"User: Context: {context}\nQuestion: {new_question}\n\n" + else: + prompt += f"User: {new_question}\n\n" + + prompt += "Assistant:" + return prompt + + def generate_response(self, prompt: str) -> Tuple[str, int, float]: + if not self.llm_ctx: + raise RuntimeError("Model not loaded. Call load_model() first.") + + self.console.print("[bold cyan]Assistant: [/bold cyan]", end="") + complete_response = "" + token_count = 0 + start_time = time.time() + + try: + for chunk in self.llm_ctx.create_completion( + prompt, + max_tokens=self.config.max_tokens, + top_k=self.config.top_k, + top_p=self.config.top_p, + temperature=self.config.temperature, + repeat_penalty=self.config.repeat_penalty, + stream=True + ): + text_chunk = chunk["choices"][0]["text"] + complete_response += text_chunk + token_count += 1 + print(text_chunk, end="", flush=True) + + elapsed_time = time.time() - start_time + print() + return complete_response, token_count, elapsed_time + except Exception as e: + self.console.print(f"\n[red]Error generating response: {str(e)}[/red]") + return f"I encountered an error while generating a response. Please try again or ask a different question.", 0, 0 + + def run_chat(self): + try: + self.load_model() + self.console.print("\n[bold green]Welcome to Open-Insurance-LLM![/bold green]") + self.console.print("Enter your questions (type '/bye', 'exit', or 'quit' to end the session)\n") + self.console.print("Optional: You can provide context by typing 'context:' followed by your context, then 'question:' followed by your question\n") + self.console.print("Your conversation history will be maintained for context-aware responses.\n") + + total_tokens = 0 + + while True: + try: + user_input = self.console.input("[bold cyan]User:[/bold cyan] ").strip() + + if user_input.lower() in ["exit", "/bye", "quit"]: + self.console.print(f"\n[dim]Total tokens: {total_tokens}[/dim]") + self.console.print("\n[bold green]Thank you for using OpenInsuranceLLM![/bold green]") + break + + # Reset conversation with command + if user_input.lower() == "/reset": + self.conversation_history = [] + self.console.print("[yellow]Conversation history has been reset.[/yellow]") + continue + + context = "" + question = user_input + if "context:" in user_input.lower() and "question:" in user_input.lower(): + parts = user_input.split("question:", 1) + context = parts[0].replace("context:", "").strip() + question = parts[1].strip() + + prompt = self.build_conversation_prompt(question, context) + response, tokens, elapsed_time = self.generate_response(prompt) + + # Add to conversation history + self.conversation_history.append({ + "user": question, + "assistant": response + }) + + # Update total tokens + total_tokens += tokens + + # Print metrics + tokens_per_sec = tokens / elapsed_time if elapsed_time > 0 else 0 + self.console.print( + f"[dim]Tokens: {tokens} || " + + f"Time: {elapsed_time:.2f}s || " + + f"Speed: {tokens_per_sec:.2f} tokens/sec[/dim]" + ) + print() # Add a blank line after each response + + except KeyboardInterrupt: + self.console.print("\n[yellow]Input interrupted. Type '/bye', 'exit', or 'quit' to quit.[/yellow]") + continue + except Exception as e: + self.console.print(f"\n[red]Error processing input: {str(e)}[/red]") + continue + except Exception as e: + self.console.print(f"\n[red]Fatal error: {str(e)}[/red]") + finally: + if self.llm_ctx: + del self.llm_ctx + + +def main(): + try: + config = ModelConfig() + llm = InsuranceLLM(config) + llm.run_chat() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nApplication error: {str(e)}") + + +if __name__ == "__main__": + main() +``` + +```bash +python3 inference_open-insurance-llm-gguf.py +``` + +### Nvidia Llama 3 - ChatQA Paper: +Arxiv : [https://arxiv.org/pdf/2401.10225](https://arxiv.org/pdf/2401.10225) + +## Use Cases + +This model is specifically designed for: +- Insurance policy understanding and explanation +- Claims processing assistance +- Coverage analysis +- Insurance terminology clarification +- Policy comparison and recommendations +- Risk assessment queries +- Insurance compliance questions + +## Limitations + +- The model's knowledge is limited to its training data cutoff +- Should not be used as a replacement for professional insurance advice +- May occasionally generate plausible-sounding but incorrect information + +## Bias and Ethics + +This model should be used with awareness that: +- It may reflect biases present in insurance industry training data +- Output should be verified by insurance professionals for critical decisions +- It should not be used as the sole basis for insurance decisions +- The model's responses should be treated as informational, not as legal or professional advice + +## Citation and Attribution + +If you use base model or quantized model in your research or applications, please cite: +``` +@misc{maharajwala2024openinsurance, + author = {Raj Maharajwala}, + title = {Open-Insurance-LLM-Llama3-8B-GGUF}, + year = {2024}, + publisher = {HuggingFace}, + linkedin = {https://www.linkedin.com/in/raj6800/}, + url = {https://huggingface.co/Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF} +} +``` \ No newline at end of file diff --git a/inference_open-insurance-llm-gguf.py b/inference_open-insurance-llm-gguf.py new file mode 100644 index 0000000..327863a --- /dev/null +++ b/inference_open-insurance-llm-gguf.py @@ -0,0 +1,218 @@ +import os +import time +from pathlib import Path +from llama_cpp import Llama +from rich.console import Console +from huggingface_hub import hf_hub_download +from dataclasses import dataclass +from typing import List, Dict, Any, Tuple + +@dataclass +class ModelConfig: + # Optimized parameters for coherent responses and efficient performance on devices like MacBook Air M2 + model_name: str = "Raj-Maharajwala/Open-Insurance-LLM-Llama3-8B-GGUF" + model_file: str = "open-insurance-llm-q4_k_m.gguf" + # model_file: str = "open-insurance-llm-q8_0.gguf" # 8-bit quantization; higher precision, better quality, increased resource usage + # model_file: str = "open-insurance-llm-q5_k_m.gguf" # 5-bit quantization; balance between performance and resource efficiency + max_tokens: int = 1000 # Maximum number of tokens to generate in a single output + temperature: float = 0.1 # Controls randomness in output; lower values produce more coherent responses (performs scaling distribution) + top_k: int = 15 # After temperature scaling, Consider the top 15 most probable tokens during sampling + top_p: float = 0.2 # After reducing the set to 15 tokens, Uses nucleus sampling to select tokens with a cumulative probability of 20% + repeat_penalty: float = 1.2 # Penalize repeated tokens to reduce redundancy + num_beams: int = 4 # Number of beams for beam search; higher values improve quality at the cost of speed + n_gpu_layers: int = -2 # Number of layers to offload to GPU; -1 for full GPU utilization, -2 for automatic configuration + n_ctx: int = 2048 # Context window size; Llama 3 models support up to 8192 tokens context length + n_batch: int = 256 # Number of tokens to process simultaneously; adjust based on available hardware (suggested 512) + verbose: bool = False # True for enabling verbose logging for debugging purposes + use_mmap: bool = False # Memory-map model to reduce RAM usage; set to True if running on limited memory systems + use_mlock: bool = True # Lock model into RAM to prevent swapping; improves performance on systems with sufficient RAM + offload_kqv: bool = True # Offload key, query, value matrices to GPU to accelerate inference + + + +class InsuranceLLM: + def __init__(self, config: ModelConfig): + self.config = config + self.llm_ctx = None + self.console = Console() + self.conversation_history: List[Dict[str, str]] = [] + + self.system_message = ( + "This is a chat between a user and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. " + "The assistant should also indicate when the answer cannot be found in the context. " + "You are an expert from the Insurance domain with extensive insurance knowledge and " + "professional writer skills, especially about insurance policies. " + "Your name is OpenInsuranceLLM, and you were developed by Raj Maharajwala. " + "You are willing to help answer the user's query with a detailed explanation. " + "In your explanation, leverage your deep insurance expertise, such as relevant insurance policies, " + "complex coverage plans, or other pertinent insurance concepts. Use precise insurance terminology while " + "still aiming to make the explanation clear and accessible to a general audience." + ) + + def download_model(self) -> str: + try: + with self.console.status("[bold green]Downloading model..."): + model_path = hf_hub_download( + self.config.model_name, + filename=self.config.model_file, + local_dir=os.path.join(os.getcwd(), 'gguf_dir') + ) + return model_path + except Exception as e: + self.console.print(f"[red]Error downloading model: {str(e)}[/red]") + raise + + def load_model(self) -> None: + try: + quantized_path = os.path.join(os.getcwd(), "gguf_dir") + directory = Path(quantized_path) + + try: + model_path = str(list(directory.glob(self.config.model_file))[0]) + except IndexError: + model_path = self.download_model() + + with self.console.status("[bold green]Loading model..."): + self.llm_ctx = Llama( + model_path=model_path, + n_gpu_layers=self.config.n_gpu_layers, + n_ctx=self.config.n_ctx, + n_batch=self.config.n_batch, + num_beams=self.config.num_beams, + verbose=self.config.verbose, + use_mlock=self.config.use_mlock, + use_mmap=self.config.use_mmap, + offload_kqv=self.config.offload_kqv + ) + except Exception as e: + self.console.print(f"[red]Error loading model: {str(e)}[/red]") + raise + + def build_conversation_prompt(self, new_question: str, context: str = "") -> str: + prompt = f"System: {self.system_message}\n\n" + + # Add conversation history + for exchange in self.conversation_history: + prompt += f"User: {exchange['user']}\n\n" + prompt += f"Assistant: {exchange['assistant']}\n\n" + + # Add the new question + if context: + prompt += f"User: Context: {context}\nQuestion: {new_question}\n\n" + else: + prompt += f"User: {new_question}\n\n" + + prompt += "Assistant:" + return prompt + + def generate_response(self, prompt: str) -> Tuple[str, int, float]: + if not self.llm_ctx: + raise RuntimeError("Model not loaded. Call load_model() first.") + + self.console.print("[bold cyan]Assistant: [/bold cyan]", end="") + complete_response = "" + token_count = 0 + start_time = time.time() + + try: + for chunk in self.llm_ctx.create_completion( + prompt, + max_tokens=self.config.max_tokens, + top_k=self.config.top_k, + top_p=self.config.top_p, + temperature=self.config.temperature, + repeat_penalty=self.config.repeat_penalty, + stream=True + ): + text_chunk = chunk["choices"][0]["text"] + complete_response += text_chunk + token_count += 1 + print(text_chunk, end="", flush=True) + + elapsed_time = time.time() - start_time + print() + return complete_response, token_count, elapsed_time + except Exception as e: + self.console.print(f"\n[red]Error generating response: {str(e)}[/red]") + return f"I encountered an error while generating a response. Please try again or ask a different question.", 0, 0 + + def run_chat(self): + try: + self.load_model() + self.console.print("\n[bold green]Welcome to Open-Insurance-LLM![/bold green]") + self.console.print("Enter your questions (type '/bye', 'exit', or 'quit' to end the session)\n") + self.console.print("Optional: You can provide context by typing 'context:' followed by your context, then 'question:' followed by your question\n") + self.console.print("Your conversation history will be maintained for context-aware responses.\n") + + total_tokens = 0 + + while True: + try: + user_input = self.console.input("[bold cyan]User:[/bold cyan] ").strip() + + if user_input.lower() in ["exit", "/bye", "quit"]: + self.console.print(f"\n[dim]Total tokens: {total_tokens}[/dim]") + self.console.print("\n[bold green]Thank you for using Open-Insurance-LLM![/bold green]") + break + + # Reset conversation with command + if user_input.lower() == "/reset": + self.conversation_history = [] + self.console.print("[yellow]Conversation history has been reset.[/yellow]") + continue + + context = "" + question = user_input + if "context:" in user_input.lower() and "question:" in user_input.lower(): + parts = user_input.split("question:", 1) + context = parts[0].replace("context:", "").strip() + question = parts[1].strip() + + prompt = self.build_conversation_prompt(question, context) + response, tokens, elapsed_time = self.generate_response(prompt) + + # Add to conversation history + self.conversation_history.append({ + "user": question, + "assistant": response + }) + + # Update total tokens + total_tokens += tokens + + # Print metrics + tokens_per_sec = tokens / elapsed_time if elapsed_time > 0 else 0 + self.console.print( + f"[dim]Tokens: {tokens} || " + + f"Time: {elapsed_time:.2f}s || " + + f"Speed: {tokens_per_sec:.2f} tokens/sec[/dim]" + ) + print() # Add a blank line after each response + + except KeyboardInterrupt: + self.console.print("\n[yellow]Input interrupted. Type '/bye', 'exit', or 'quit' to quit.[/yellow]") + continue + except Exception as e: + self.console.print(f"\n[red]Error processing input: {str(e)}[/red]") + continue + except Exception as e: + self.console.print(f"\n[red]Fatal error: {str(e)}[/red]") + finally: + if self.llm_ctx: + del self.llm_ctx + + +def main(): + try: + config = ModelConfig() + llm = InsuranceLLM(config) + llm.run_chat() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nApplication error: {str(e)}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/inference_requirements.txt b/inference_requirements.txt new file mode 100644 index 0000000..7ba29c9 --- /dev/null +++ b/inference_requirements.txt @@ -0,0 +1,31 @@ +black==24.10.0 +certifi==2024.8.30 +charset-normalizer==3.4.0 +click==8.1.7 +diskcache==5.6.3 +filelock==3.16.1 +fsspec==2024.10.0 +huggingface-hub==0.26.2 +idna==3.10 +iniconfig==2.0.0 +isort==5.13.2 +Jinja2==3.1.4 +llama_cpp_python==0.3.2 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +mypy-extensions==1.0.0 +numpy==2.1.3 +packaging==24.2 +pathspec==0.12.1 +platformdirs==4.3.6 +pluggy==1.5.0 +psutil==6.1.0 +Pygments==2.18.0 +pytest==8.3.3 +PyYAML==6.0.2 +requests==2.32.3 +rich==13.9.4 +tqdm==4.67.0 +typing_extensions==4.12.2 +urllib3==2.2.3 diff --git a/open-insurance-llm-q4_k_m.gguf b/open-insurance-llm-q4_k_m.gguf new file mode 100644 index 0000000..6868f70 --- /dev/null +++ b/open-insurance-llm-q4_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b17fe1a74ee6e52b9f8c418a992c0832546aec0afb9f09be2345495c40aa7d9 +size 4920734592 diff --git a/open-insurance-llm-q5_k_m.gguf b/open-insurance-llm-q5_k_m.gguf new file mode 100644 index 0000000..23f2c3a --- /dev/null +++ b/open-insurance-llm-q5_k_m.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097c3f05c71133e342b19305e0567a848049b169ab7e076e73910acf97151ef3 +size 5732987776 diff --git a/open-insurance-llm-q8_0.gguf b/open-insurance-llm-q8_0.gguf new file mode 100644 index 0000000..6ecc83e --- /dev/null +++ b/open-insurance-llm-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b26d4c11789453f3858d39fa1303a43979f39567535c0216144aa0d01f080ee5 +size 8540771200