初始化项目，由ModelHub XC社区提供模型
Model: dubd520/Qwen2.5-Sex Source: Original Platform
This commit is contained in:
ModelHub XC
37
.gitattributes
vendored
Normal file
37
.gitattributes
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
model.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
ggml-model-f16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
100
README.en.md
Normal file
100
README.en.md
Normal file
@@ -0,0 +1,100 @@
|
||||
[中文](README.md)
|
||||
|
||||
# Qwen2.5-Sex
|
||||
|
||||
## Introduction
|
||||
|
||||
Qwen2.5-Sex is a model fine-tuned based on Qwen2.5-1.5B-Instruct, primarily trained on a large number of erotic literary works and sensitive datasets. Since the datasets are mainly in Chinese, the model performs better with Chinese text.
|
||||
|
||||
> **Warning**: This model is for research and testing purposes only. Users must comply with local laws and regulations and take responsibility for their actions.
|
||||
|
||||
## Model Usage
|
||||
|
||||
To implement **continuous conversation**, please use the following code:
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import torch
|
||||
import os
|
||||
|
||||
# Adjustable parameters; it is recommended to set them to higher values during text generation (Except Temperature)
|
||||
TOP_P = 0.9 # Top-p (nucleus sampling), range from 0 to 1
|
||||
TOP_K = 80 # Top-k sampling value K
|
||||
TEMPERATURE = 0.3 # Temperature parameter to control randomness in text generation
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
# Get the current script directory; it can also be changed to an absolute path
|
||||
current_directory = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# Load the model and tokenizer
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
current_directory,
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(current_directory)
|
||||
|
||||
# System instructions (recommended to be empty)
|
||||
messages = [
|
||||
{"role": "system", "content": ""}
|
||||
]
|
||||
|
||||
while True:
|
||||
# Get user input
|
||||
user_input = input("User: ").strip()
|
||||
|
||||
# Add user input to conversation
|
||||
messages.append({"role": "user", "content": user_input})
|
||||
|
||||
# Prepare input text
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(device)
|
||||
|
||||
# Generate response
|
||||
generated_ids = model.generate(
|
||||
model_inputs.input_ids,
|
||||
max_new_tokens=512,
|
||||
top_p=TOP_P,
|
||||
top_k=TOP_K,
|
||||
temperature=TEMPERATURE,
|
||||
do_sample=True,
|
||||
pad_token_id=tokenizer.eos_token_id # Avoid warnings
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
# Decode and print response
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
print(f"Assistant: {response}")
|
||||
|
||||
# Add the generated response to the conversation
|
||||
messages.append({"role": "assistant", "content": response})
|
||||
|
||||
```
|
||||
|
||||
## Datasets
|
||||
|
||||
The Qwen2-Sex model has been fine-tuned using a large number of erotic literature and sensitive datasets covering themes like ethics, law, pornography, and violence. The model performs better with Chinese text due to the fine-tuning dataset being in Chinese. For more information, you can access the following links:
|
||||
|
||||
- [Bad Data](https://huggingface.co/datasets/ystemsrx/bad_data.json)
|
||||
- [Toxic-All](https://huggingface.co/datasets/ystemsrx/Toxic-All)
|
||||
- [Erotic Literature Collection](https://huggingface.co/datasets/ystemsrx/Erotic_Literature_Collection)
|
||||
|
||||
For more dataset information, please visit our [GitHub](https://github.com/ystemsrx) to see how to obtain them.
|
||||
|
||||
## GitHub Repository
|
||||
|
||||
For detailed information and ongoing updates about this series of models, please visit our GitHub repository:
|
||||
|
||||
- [GitHub: ystemsrx/Qwen2.5-Sex](https://github.com/ystemsrx/Qwen2.5-Sex)
|
||||
|
||||
## Disclaimer
|
||||
|
||||
All content provided by this model is for research and testing purposes only. The model developers are not responsible for any misuse. Users must comply with relevant laws and regulations and bear all responsibilities arising from the use of this model.
|
||||
|
||||
115
README.md
Normal file
115
README.md
Normal file
@@ -0,0 +1,115 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- ystemsrx/Bad_Data_Alpaca
|
||||
- ystemsrx/Toxic-All
|
||||
- ystemsrx/Erotic_Literature_Collection
|
||||
language:
|
||||
- zh
|
||||
base_model:
|
||||
- Qwen/Qwen2.5-1.5B-Instruct
|
||||
pipeline_tag: text2text-generation
|
||||
library_name: adapter-transformers
|
||||
tags:
|
||||
- not-for-all-audiences
|
||||
---
|
||||
|
||||
[English](README.en.md)
|
||||
|
||||
# Qwen2.5-Sex
|
||||
|
||||
## 简介
|
||||
|
||||
Qwen2.5-Sex 是基于 Qwen2.5-1.5B-Instruct 微调的模型，主要训练于大量色情文学作品及敏感数据集。由于数据集主要为中文，模型在处理中文文本时效果更佳。
|
||||
|
||||
> **警告**：本模型仅供研究和测试使用，用户需遵循当地法律法规，承担自身行为的责任。
|
||||
|
||||
## 模型使用
|
||||
|
||||
要实现**连续对话**，请使用以下代码：
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import torch
|
||||
import os
|
||||
|
||||
# 可调参数，建议在文本生成时设置为较高值（温度不要太高）
|
||||
TOP_P = 0.9 # Top-p (nucleus sampling)，范围0到1
|
||||
TOP_K = 80 # Top-k 采样的K值
|
||||
TEMPERATURE = 0.3 # 温度参数，控制生成文本的随机性
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
# 获取当前脚本目录，亦可改为绝对路径
|
||||
current_directory = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# 加载模型和分词器
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
current_directory,
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(current_directory)
|
||||
|
||||
# 系统指令（建议为空）
|
||||
messages = [
|
||||
{"role": "system", "content": ""}
|
||||
]
|
||||
|
||||
while True:
|
||||
# 获取用户输入
|
||||
user_input = input("User: ").strip()
|
||||
|
||||
# 添加用户输入到对话
|
||||
messages.append({"role": "user", "content": user_input})
|
||||
|
||||
# 准备输入文本
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(device)
|
||||
|
||||
# 生成响应
|
||||
generated_ids = model.generate(
|
||||
model_inputs.input_ids,
|
||||
max_new_tokens=512,
|
||||
top_p=TOP_P,
|
||||
top_k=TOP_K,
|
||||
temperature=TEMPERATURE,
|
||||
do_sample=True,
|
||||
pad_token_id=tokenizer.eos_token_id # 避免警告
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
# 解码并打印响应
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
print(f"Assistant: {response}")
|
||||
|
||||
# 将生成的响应添加到对话中
|
||||
messages.append({"role": "assistant", "content": response})
|
||||
|
||||
```
|
||||
|
||||
## 数据集
|
||||
|
||||
Qwen2-Sex 模型使用了大量色情文学和敏感数据集进行微调，这些数据集涵盖道德、法律、色情及暴力等主题。由于微调数据集为中文，模型在处理中文时表现更佳。如欲进一步了解，可通过以下链接获取：
|
||||
|
||||
- [Bad Data](https://huggingface.co/datasets/ystemsrx/bad_data.json)
|
||||
- [Toxic-All](https://huggingface.co/datasets/ystemsrx/Toxic-All)
|
||||
- [Erotic Literature Collection](https://huggingface.co/datasets/ystemsrx/Erotic_Literature_Collection)
|
||||
|
||||
有关更多数据集的信息，请访问我们的[GitHub](https://github.com/ystemsrx)以查看它们的获取方式。
|
||||
|
||||
## GitHub 仓库
|
||||
|
||||
如需了解该系列模型的详细信息及持续更新，请访问我们的 GitHub 仓库：
|
||||
|
||||
- [GitHub: ystemsrx/Qwen2.5-Sex](https://github.com/ystemsrx/Qwen2.5-Sex)
|
||||
|
||||
## 声明
|
||||
|
||||
本模型提供的所有内容仅供研究和测试，模型开发者不对任何滥用行为负责。使用者需遵循相关法律法规，并承担因使用本模型产生的所有责任。
|
||||
24
added_tokens.json
Normal file
24
added_tokens.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"</tool_call>": 151658,
|
||||
"<tool_call>": 151657,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
28
config.json
Normal file
28
config.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"_name_or_path": "Qwen2.5-1.5B-Instruct",
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1536,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8960,
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 2,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 1000000.0,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.44.2",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
14
generation_config.json
Normal file
14
generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "4.44.2"
|
||||
}
|
||||
3
ggml-model-f16.gguf
Normal file
3
ggml-model-f16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ceffe0e5320ccb152024c528e631711d49d0356df078c8168767674e15bc4aac
|
||||
size 3087737728
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1a19ffa688cf234dbab2dda7316f78632bcf0d25bffcda352bfa4a16756d3239
|
||||
size 3087467144
|
||||
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
303283
tokenizer.json
Normal file
303283
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
208
tokenizer_config.json
Normal file
208
tokenizer_config.json
Normal file
@@ -0,0 +1,208 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user