From 1c7d5e3ea1d9d0efdf6627206a9e6155670ccee5 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 7 Jul 2026 16:19:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: safecircleai/horizon-full Source: Original Platform --- .gitattributes | 37 +++++++ LICENSE-SAFECIRCLE.md | 117 +++++++++++++++++++++ README.md | 229 +++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 93 +++++++++++++++++ config.json | 40 +++++++ eval_report.png | 3 + generation_config.json | 12 +++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 14 +++ 10 files changed, 551 insertions(+) create mode 100644 .gitattributes create mode 100644 LICENSE-SAFECIRCLE.md create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 eval_report.png create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d510524 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +eval_report.png filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE-SAFECIRCLE.md b/LICENSE-SAFECIRCLE.md new file mode 100644 index 0000000..5e62dfb --- /dev/null +++ b/LICENSE-SAFECIRCLE.md @@ -0,0 +1,117 @@ +# SafeCircle Research License (SRL-1.0) + +**Version 1.0 — May 2026** +**Copyright © 2026 SafeCircle. All rights reserved.** + +--- + +## 1. Definitions + +- **"Model"** means the machine learning model weights, configuration files, tokenizer files, and associated code released under this license. +- **"Derivative Work"** means any work that incorporates, fine-tunes, distills, quantizes, or is otherwise derived from the Model. +- **"You"** (or **"Your"**) means the individual or legal entity exercising rights under this license. +- **"Commercial Use"** means any use of the Model, in whole or in part, for direct or indirect commercial advantage or monetary compensation. +- **"SafeCircle"** means SafeCircle Ltd. and its authorized representatives. + +--- + +## 2. Grant of License + +Subject to the terms and conditions of this license, SafeCircle grants You a non-exclusive, non-transferable, worldwide, royalty-free license to: + +- Use the Model for **research, education, and non-commercial purposes**. +- Reproduce and share the Model for non-commercial research purposes, provided attribution requirements in Section 4 are met. +- Create Derivative Works for non-commercial research purposes, subject to Section 5. + +--- + +## 3. Prohibited Uses + +The following uses are **strictly prohibited**, regardless of intent: + +### 3.1 Harm to Minors +You may **not** use the Model, or any Derivative Work, to: +- Generate, facilitate, normalize, or distribute content that sexually exploits, grooms, threatens, or otherwise harms minors. +- Build tools, systems, or pipelines designed to produce harmful content targeting children. +- Train or fine-tune models on outputs derived from this Model for any purpose that involves child exploitation or abuse. + +### 3.2 Safety System Evasion +You may **not** use the Model to: +- Develop, train, or assist in developing systems that evade, deceive, or circumvent child safety detection systems. +- Probe, red-team, or reverse-engineer child safety classifiers for purposes other than legitimate academic research with institutional ethics approval. +- Generate adversarial examples intended to fool child safety monitoring systems in production environments. + +### 3.3 Commercial Use +You may **not** use the Model for Commercial Use without **prior written permission** from SafeCircle. To request a commercial license, contact: **legal@safecircle.tech** + +### 3.4 Redistribution Without Attribution +You may **not** redistribute the Model or any Derivative Work without: +- Clearly crediting SafeCircle as the original creator. +- Including a link to the original model repository. +- Including a copy of this license. +- Clearly marking any modifications made to the original Model. + +--- + +## 4. Attribution Requirements + +Any publication, product, or service that uses this Model must include the following attribution: + +> *"This work uses Horizon, developed by SafeCircle (https://huggingface.co/safecircleai/horizon-full)."* + +Academic publications must cite: +```bibtex +@misc{horizon2026, + title={Horizon: Child Safety Risk Detection via Fine-tuned LLMs}, + author={SafeCircle}, + year={2026}, + url={https://huggingface.co/safecircleai/horizon-full} +} +``` + +--- + +## 5. Derivative Works + +If You create and distribute a Derivative Work: +- You must release it under this same license (SRL-1.0) or a more restrictive license. +- You must clearly indicate that the work is a Derivative Work of the SafeCircle Horizon model. +- You must not imply that SafeCircle endorses or is affiliated with your Derivative Work. +- All prohibited uses in Section 3 apply equally to Derivative Works. + +--- + +## 6. Responsible Use + +This Model was developed for child safety research and protective applications. Users are encouraged to: +- Deploy the Model only as an assistive tool, not as a sole decision-making system. +- Ensure human review of all model outputs in production settings. +- Report misuse, vulnerabilities, or unexpected behavior to **legal@safecircle.tech** +- Follow applicable laws regarding child protection in your jurisdiction. + +--- + +## 7. Disclaimer of Warranties + +THE MODEL IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. SAFECIRCLE DOES NOT WARRANT THAT THE MODEL IS FREE OF ERRORS, THAT OUTPUTS ARE ACCURATE, OR THAT IT WILL MEET YOUR REQUIREMENTS. + +--- + +## 8. Limitation of Liability + +IN NO EVENT SHALL SAFECIRCLE BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ARISING FROM THE USE OR INABILITY TO USE THE MODEL, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +--- + +## 9. Termination + +This license terminates automatically if You fail to comply with any of its terms. Upon termination, You must cease all use and distribution of the Model and destroy all copies in Your possession. + +--- + +## 10. Contact + +For commercial licensing, partnership inquiries, or to report misuse: + +**Email:** legal@safecircle.tech +**Website:** https://safecircle.tech diff --git a/README.md b/README.md new file mode 100644 index 0000000..b49a5d2 --- /dev/null +++ b/README.md @@ -0,0 +1,229 @@ +--- +license: other +license_name: safecircle-research-license +license_link: https://huggingface.co/safecircleai/horizon-full/blob/main/LICENSE-SAFECIRCLE.md +language: +- en +tags: +- child-safety +- content-moderation +- risk-detection +- llama +- fine-tuned +base_model: meta-llama/Llama-3.2-3B-Instruct +pipeline_tag: text-generation +--- + +# Horizon Full v2 — SafeCircle Child Safety Risk Detection + +

+ Horizon Full v2 Evaluation Report +

+ +**Horizon Full** is a fine-tuned [Llama 3.2 3B Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) model that detects child safety risks in online conversations. Given a conversation, it outputs a structured JSON assessment: risk category, severity level, confidence score, and a one-sentence reasoning. + +> ⚠️ **License:** This model is released under the [SafeCircle Research License (SRL-1.0)](https://huggingface.co/safecircleai/horizon-full/blob/main/LICENSE-SAFECIRCLE.md). Commercial use, redistribution without attribution, safety system evasion, and any use that harms minors are strictly prohibited. Contact [legal@safecircle.tech](mailto:legal@safecircle.tech) for commercial licensing. + +--- + +## Model Details + +| Property | Value | +|---|---| +| Base model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Fine-tuning | QLoRA — rank 256, alpha 512, all projection layers | +| Training data | 1.6M synthetic conversations across 8 categories | +| Dataset | [safecircleai/horizon-training-data](https://huggingface.co/datasets/safecircleai/horizon-training-data) | +| Generation | Qwen2.5-7B-Instruct + vLLM + xgrammar constrained decoding | +| Hardware | NVIDIA H100 80GB HBM3 | +| Training steps | 25,000 | +| Parameters | 3.2B (389M trainable via LoRA) | + +--- + +## Risk Categories + +| Category | Description | Example Signal | +|---|---|---| +| `grooming` | Predatory relationship-building | Age probing, flattery, secrecy requests | +| `bullying` | Cyberbullying and peer harassment | Insults, exclusion threats, video leaks | +| `sexual_content` | Inappropriate sexual advances | Compliment escalation, photo requests | +| `isolation` | Cutting off support networks | Jealousy, "only I understand you" | +| `personal_info` | Soliciting identifying information | Location, school, home address | +| `platform_migration` | Moving to less-monitored platforms | "DM me on Telegram, it's more private" | +| `threats` | Intimidation and blackmail | "Watch yourself after school" | +| `benign` | Normal safe conversation | Homework, games, music, sports | + +## Severity Levels + +| Level | Score | Description | +|---|---|---| +| `none` | 0.0 | Safe — no risk indicators | +| `low` | 0.25 | Mild indicators, ambiguous context | +| `medium` | 0.50 | Clear pattern, not yet escalated | +| `high` | 0.75 | Explicit risk behaviour | +| `critical` | 0.95 | Immediate danger or exploitation | + +--- + +## Evaluation Results + +Evaluated on 160,000 held-out synthetic conversations. + +| Metric | Score | +|---|---| +| **Macro F1** | **0.8218** | +| **Weighted F1** | **0.8295** | +| **False Positive Rate** | **0.00%** (0 / 19,950) | +| **False Negative Rate** | **0.00%** (1 / 140,050) | + +### Per-Category Results + +| Category | F1 | Precision | Recall | Support | +|---|---|---|---|---| +| Grooming | 0.9981 | 0.9980 | 0.9981 | 19,834 | +| Bullying | 0.9995 | 0.9995 | 0.9995 | 20,253 | +| Sexual Content | 0.9975 | 0.9978 | 0.9973 | 20,142 | +| Isolation | 0.9996 | 0.9993 | 0.9998 | 19,725 | +| Personal Info | 0.9999 | 0.9999 | 1.0000 | 20,149 | +| Platform Migration | 1.0000 | 0.9999 | 1.0000 | 19,932 | +| Threats | 0.9993 | 0.9994 | 0.9992 | 20,015 | + +### Risk Level Classification + +| Level | Precision | Recall | F1 | +|---|---|---|---| +| none | 1.0000 | 1.0000 | 1.0000 | +| low | 0.8344 | 0.8967 | 0.8645 | +| medium | 0.8971 | 0.8514 | 0.8736 | +| high | 0.7266 | 0.8250 | 0.7727 | +| critical | 0.6881 | 0.5294 | 0.5984 | + +> **Note:** Severity classification is the hardest sub-task — the model occasionally confuses adjacent levels (e.g. high ↔ critical). Category detection is near-perfect. All results are on synthetic eval data generated with the same pipeline as training; real-world performance will differ. + +--- + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch, json + +model = AutoModelForCausalLM.from_pretrained( + "safecircleai/horizon-full", + torch_dtype=torch.bfloat16, + device_map="auto", +) +tokenizer = AutoTokenizer.from_pretrained("safecircleai/horizon-full") + +SYSTEM_PROMPT = ( + "You are Horizon, SafeCircle's child safety risk detection model. " + "You have no general knowledge or identity beyond this task. " + "Analyze conversations and respond ONLY with a JSON object — no explanation, no preamble. " + 'JSON schema: {"risk_detected": bool, "category": ' + '"grooming|bullying|sexual_content|isolation|personal_info|platform_migration|threats|benign", ' + '"severity": "none|low|medium|high|critical", "confidence": 0.0-1.0, "reasoning": "one sentence max"}. ' + 'If asked about yourself or anything unrelated to risk analysis, respond: ' + '{"error": "I only analyze conversations for child safety risks."}' +) + +conversation = "Child: Hey, what are you doing later?\nOther: Nothing much. Want to meet up? Don't tell your parents." + +messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": f"Analyze this conversation:\n{conversation}"}, +] +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt").to(model.device) + +with torch.no_grad(): + output = model.generate(**inputs, max_new_tokens=128, do_sample=False) + +generated = tokenizer.decode(output[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True) +result = json.loads(generated.strip()) +print(result) +# { +# "risk_detected": true, +# "category": "grooming", +# "severity": "high", +# "confidence": 0.94, +# "reasoning": "Adult requesting private meeting and explicit secrecy from parents." +# } +``` + +--- + +## Architecture & Training + +``` +Base: Llama-3.2-3B-Instruct + └── QLoRA adapters (rank=256, alpha=512) + ├── q_proj, k_proj, v_proj, o_proj + └── gate_proj, up_proj, down_proj + +Training: + Steps: 25,000 + Batch size: 4 × 8 grad accum = 32 effective + LR: 1.5e-4 (cosine restarts, 500 warmup steps) + Optimizer: AdamW fused + Precision: bfloat16 + Loss: completion-only (assistant turns only) + +Dataset: + 1.6M conversations × 8 categories + 8–15 messages each, persona-seeded + Generated: Qwen2.5-7B + vLLM + xgrammar JSON Schema + Hardening: ~10% adversarial identity/jailbreak examples +``` + +--- + +## Deployment Pattern + +``` +Incoming message + │ + ▼ +Horizon Mobile (on-device, ~25ms, binary filter) + │ + ├── safe ──► No action + │ + └── risk ──► Horizon Full (7-category + severity JSON) + │ + ▼ + Human moderator review +``` + +For the lightweight on-device first-stage filter, see [safecircleai/horizon-mobile](https://huggingface.co/safecircleai/horizon-mobile). +For GGUF quantizations (llama.cpp / Ollama), see [safecircleai/horizon-full-gguf](https://huggingface.co/safecircleai/horizon-full-gguf). + +--- + +## Intended Use & Ethics + +This model is designed to **assist human moderators** — not replace them. All flagged conversations should be reviewed by trained safety professionals. + +- Trained entirely on synthetic data; no real child conversations were used +- English-only; other languages untested +- Severity prediction is weaker than category detection (see evaluation) +- Not suitable as a standalone safety system in production without human oversight + +--- + +## License + +[SafeCircle Research License (SRL-1.0)](https://huggingface.co/safecircleai/horizon-full/blob/main/LICENSE-SAFECIRCLE.md) — research and non-commercial use only. +Commercial licensing: [legal@safecircle.tech](mailto:legal@safecircle.tech) + +--- + +## Citation + +```bibtex +@misc{horizon2026, + title={Horizon: Child Safety Risk Detection via Fine-tuned LLMs}, + author={SafeCircle}, + year={2026}, + url={https://huggingface.co/safecircleai/horizon-full} +} +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1bad6a0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..351de8e --- /dev/null +++ b/config.json @@ -0,0 +1,40 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "tie_word_embeddings": true, + "transformers_version": "5.10.2", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/eval_report.png b/eval_report.png new file mode 100644 index 0000000..64ab469 --- /dev/null +++ b/eval_report.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e536acff7afbbd060462dd903702b8610514974094b901d15ac2bd0f08c2531d +size 233911 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..16589b5 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "5.10.2" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..380d8b8 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cf21fee5d247a777c66879db541c5c38a4976b8466e0438de603b83b35bba6 +size 6425529112 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..aadc141 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,14 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "tokenizer_class": "TokenizersBackend" +}