初始化项目,由ModelHub XC社区提供模型
Model: RthItalia/PINDARO-AI-CODE Source: Original Platform
This commit is contained in:
37
.gitattributes
vendored
Normal file
37
.gitattributes
vendored
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
pindaro-f16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
pindaro-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
172
README.md
Normal file
172
README.md
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
---
|
||||||
|
language:
|
||||||
|
- en
|
||||||
|
- it
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
library_name: transformers
|
||||||
|
tags:
|
||||||
|
- llama
|
||||||
|
- code
|
||||||
|
- coding-assistant
|
||||||
|
- gguf
|
||||||
|
- instruct
|
||||||
|
- 1b
|
||||||
|
---
|
||||||
|
|
||||||
|
# PINDARO AI CODE
|
||||||
|
|
||||||
|
PINDARO AI CODE is the code-specialized release of the Pindaro model family.
|
||||||
|
|
||||||
|
## Model At A Glance
|
||||||
|
|
||||||
|
- Architecture: `LlamaForCausalLM`
|
||||||
|
- Model type: `llama`
|
||||||
|
- Approx. parameters: **~1.1B**
|
||||||
|
- Precision: `float16`
|
||||||
|
- Context length: `2048`
|
||||||
|
- Vocabulary size: `32002`
|
||||||
|
- Languages: English, Italian
|
||||||
|
- Primary use: code generation and coding assistance
|
||||||
|
|
||||||
|
## Included Artifacts
|
||||||
|
|
||||||
|
Hugging Face format:
|
||||||
|
- `model.safetensors`
|
||||||
|
- `config.json`
|
||||||
|
- `generation_config.json`
|
||||||
|
- `tokenizer.json`
|
||||||
|
- `tokenizer.model`
|
||||||
|
- `tokenizer_config.json`
|
||||||
|
- `special_tokens_map.json`
|
||||||
|
- `added_tokens.json`
|
||||||
|
|
||||||
|
GGUF format:
|
||||||
|
- `pindaro-f16.gguf`
|
||||||
|
- `pindaro-q4_k_m.gguf`
|
||||||
|
|
||||||
|
Release docs:
|
||||||
|
- `release/RELEASE_MANIFEST.json`
|
||||||
|
- `release/RELEASE_NOTES.md`
|
||||||
|
- `release/SHA256SUMS.txt`
|
||||||
|
|
||||||
|
## Prompt Format
|
||||||
|
|
||||||
|
Special tokens:
|
||||||
|
- `<|noesis|>` (id `32000`)
|
||||||
|
- `<|end|>` (id `32001`)
|
||||||
|
|
||||||
|
Configured chat template uses role sections and appends a code-fence prefix in generation prompt:
|
||||||
|
|
||||||
|
```jinja
|
||||||
|
{{ bos_token }}{% for message in messages %}<|noesis|>
|
||||||
|
{% if message['role'] == 'system' %}### System
|
||||||
|
{{ message['content'] }}
|
||||||
|
{% elif message['role'] == 'user' %}### Question
|
||||||
|
{{ message['content'] }}
|
||||||
|
{% elif message['role'] == 'assistant' %}### Answer
|
||||||
|
{{ message['content'] }}
|
||||||
|
{% endif %}<|end|>
|
||||||
|
{% endfor %}{% if add_generation_prompt %}<|noesis|>
|
||||||
|
### Answer
|
||||||
|
```
|
||||||
|
{% endif %}
|
||||||
|
```
|
||||||
|
|
||||||
|
Minimal manual prompt example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<|noesis|>
|
||||||
|
### Question
|
||||||
|
Write a Python function add(a, b).
|
||||||
|
<|end|>
|
||||||
|
<|noesis|>
|
||||||
|
### Answer
|
||||||
|
```
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quickstart (Transformers)
|
||||||
|
|
||||||
|
```python
|
||||||
|
import torch
|
||||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||||
|
|
||||||
|
model_id = "RthItalia/PINDARO-AI-CODE"
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(
|
||||||
|
model_id,
|
||||||
|
torch_dtype=torch.float16,
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": "You are a coding assistant."},
|
||||||
|
{"role": "user", "content": "Write a Python function add(a, b)."},
|
||||||
|
]
|
||||||
|
|
||||||
|
inputs = tokenizer.apply_chat_template(
|
||||||
|
messages,
|
||||||
|
tokenize=True,
|
||||||
|
add_generation_prompt=True,
|
||||||
|
return_tensors="pt",
|
||||||
|
)
|
||||||
|
|
||||||
|
attention_mask = torch.ones_like(inputs)
|
||||||
|
outputs = model.generate(
|
||||||
|
inputs,
|
||||||
|
attention_mask=attention_mask,
|
||||||
|
max_new_tokens=120,
|
||||||
|
do_sample=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(tokenizer.decode(outputs[0], skip_special_tokens=False))
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quickstart (GGUF / llama.cpp)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./llama-cli -m pindaro-q4_k_m.gguf -p "<|noesis|>
|
||||||
|
### Question
|
||||||
|
Write a Python function add(a, b).
|
||||||
|
<|end|>
|
||||||
|
<|noesis|>
|
||||||
|
### Answer
|
||||||
|
```" -n 120
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validation Snapshot
|
||||||
|
|
||||||
|
Last internal validation snapshot: **2026-03-02**
|
||||||
|
|
||||||
|
- HF smoke tests: PASS
|
||||||
|
- HF mini-eval coding quality: **1.00**
|
||||||
|
- GGUF F16 quality gate: PASS
|
||||||
|
- GGUF Q4_K_M quality gate: PASS
|
||||||
|
- Release verdict: **publishable: true**
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- Results are from internal sanity checks, not a full public benchmark suite.
|
||||||
|
|
||||||
|
## Known Limitations
|
||||||
|
|
||||||
|
- Generated code can be syntactically correct but logically wrong.
|
||||||
|
- May emit verbose outputs or repeated scaffolding.
|
||||||
|
- Always run tests and static checks on generated code.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
- Do not execute generated code in privileged environments without review.
|
||||||
|
- Use sandboxing for untrusted snippets.
|
||||||
|
- Add dependency and secret scanning in deployment workflows.
|
||||||
|
|
||||||
|
## Artifact Checksums (SHA256)
|
||||||
|
|
||||||
|
- `model.safetensors`: `f77c27b8babf9fcab83a7dc68ba58934e8c8c031c9f10b4b73e802d4fbfe0cec`
|
||||||
|
- `config.json`: `b37c45060f3e2f5f9b91903c9ccb32f3c21076e809954fda6c01d987cd8f25cc`
|
||||||
|
- `generation_config.json`: `6ff47e725c0ec6d0f1895670de7ee68e61a4f99703f6c8e89aea6ab14ea02dc3`
|
||||||
|
- `tokenizer.json`: `51433f06369ac3e597dfa23a811215e3511b8f86588a830ded72344b76a193ee`
|
||||||
|
- `tokenizer.model`: `9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347`
|
||||||
|
- `tokenizer_config.json`: `a0567c49a117af9af332874cfd333ddd622a09c5e9765131ceee6344cb22a3de`
|
||||||
|
- `special_tokens_map.json`: `d7805e093432afcde852968cdeba3de08a6fe66e77609f4701decb87fc492f33`
|
||||||
|
- `added_tokens.json`: `ece349d292e246eac9a9072c1730f023e61567984a828fb0d25dccb14e3b7592`
|
||||||
|
- `pindaro-f16.gguf`: `bdaaeb6fb712e9a4d952082cf415b05c7d076b33786d39063bbfb3a7e5db2031`
|
||||||
|
- `pindaro-q4_k_m.gguf`: `5f98cc3454774ed5ed80d71a71adfd0daff760fc9eef0900ddd4f7eda2e20fef`
|
||||||
4
added_tokens.json
Normal file
4
added_tokens.json
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"<|end|>": 32001,
|
||||||
|
"<|noesis|>": 32000
|
||||||
|
}
|
||||||
28
config.json
Normal file
28
config.json
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
{
|
||||||
|
"_name_or_path": "/kaggle/working/pindaro-final",
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 1,
|
||||||
|
"eos_token_id": 2,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 2048,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 5632,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 32,
|
||||||
|
"num_hidden_layers": 22,
|
||||||
|
"num_key_value_heads": 4,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 10000.0,
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"torch_dtype": "float16",
|
||||||
|
"transformers_version": "4.40.0",
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 32002
|
||||||
|
}
|
||||||
7
generation_config.json
Normal file
7
generation_config.json
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 1,
|
||||||
|
"eos_token_id": 2,
|
||||||
|
"max_length": 2048,
|
||||||
|
"pad_token_id": 2,
|
||||||
|
"transformers_version": "4.40.0"
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f77c27b8babf9fcab83a7dc68ba58934e8c8c031c9f10b4b73e802d4fbfe0cec
|
||||||
|
size 2200136048
|
||||||
3
pindaro-f16.gguf
Normal file
3
pindaro-f16.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:bdaaeb6fb712e9a4d952082cf415b05c7d076b33786d39063bbfb3a7e5db2031
|
||||||
|
size 2201034080
|
||||||
3
pindaro-q4_k_m.gguf
Normal file
3
pindaro-q4_k_m.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:5f98cc3454774ed5ed80d71a71adfd0daff760fc9eef0900ddd4f7eda2e20fef
|
||||||
|
size 667820928
|
||||||
63
release/RELEASE_MANIFEST.json
Normal file
63
release/RELEASE_MANIFEST.json
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
{
|
||||||
|
"release_date": "2026-03-02 23:33:21",
|
||||||
|
"model": "PINDARO-AI-CODE",
|
||||||
|
"source_path": "e:\\Pindaro\\PINDARO AI CODE",
|
||||||
|
"publishable": true,
|
||||||
|
"verdict": {
|
||||||
|
"hf_ok": true,
|
||||||
|
"gguf_f16_ok": true,
|
||||||
|
"gguf_q4_ok": true,
|
||||||
|
"publishable": true,
|
||||||
|
"thresholds": {
|
||||||
|
"hf_quality_min": 0.8,
|
||||||
|
"gguf_quality_min": 0.67
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"artifacts": {
|
||||||
|
"model.safetensors": {
|
||||||
|
"size": 2200136048,
|
||||||
|
"sha256": "f77c27b8babf9fcab83a7dc68ba58934e8c8c031c9f10b4b73e802d4fbfe0cec"
|
||||||
|
},
|
||||||
|
"config.json": {
|
||||||
|
"size": 687,
|
||||||
|
"sha256": "b37c45060f3e2f5f9b91903c9ccb32f3c21076e809954fda6c01d987cd8f25cc"
|
||||||
|
},
|
||||||
|
"generation_config.json": {
|
||||||
|
"size": 124,
|
||||||
|
"sha256": "6ff47e725c0ec6d0f1895670de7ee68e61a4f99703f6c8e89aea6ab14ea02dc3"
|
||||||
|
},
|
||||||
|
"tokenizer.json": {
|
||||||
|
"size": 1843262,
|
||||||
|
"sha256": "51433f06369ac3e597dfa23a811215e3511b8f86588a830ded72344b76a193ee"
|
||||||
|
},
|
||||||
|
"tokenizer.model": {
|
||||||
|
"size": 499723,
|
||||||
|
"sha256": "9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347"
|
||||||
|
},
|
||||||
|
"tokenizer_config.json": {
|
||||||
|
"size": 1830,
|
||||||
|
"sha256": "a0567c49a117af9af332874cfd333ddd622a09c5e9765131ceee6344cb22a3de"
|
||||||
|
},
|
||||||
|
"special_tokens_map.json": {
|
||||||
|
"size": 621,
|
||||||
|
"sha256": "d7805e093432afcde852968cdeba3de08a6fe66e77609f4701decb87fc492f33"
|
||||||
|
},
|
||||||
|
"added_tokens.json": {
|
||||||
|
"size": 46,
|
||||||
|
"sha256": "ece349d292e246eac9a9072c1730f023e61567984a828fb0d25dccb14e3b7592"
|
||||||
|
},
|
||||||
|
"pindaro-f16.gguf": {
|
||||||
|
"size": 2201034080,
|
||||||
|
"sha256": "bdaaeb6fb712e9a4d952082cf415b05c7d076b33786d39063bbfb3a7e5db2031"
|
||||||
|
},
|
||||||
|
"pindaro-q4_k_m.gguf": {
|
||||||
|
"size": 667820928,
|
||||||
|
"sha256": "5f98cc3454774ed5ed80d71a71adfd0daff760fc9eef0900ddd4f7eda2e20fef"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"paths": {
|
||||||
|
"root": "e:\\Pindaro\\PINDARO AI CODE",
|
||||||
|
"gguf_f16": "e:\\Pindaro\\PINDARO AI CODE\\pindaro-f16.gguf",
|
||||||
|
"gguf_q4": "e:\\Pindaro\\PINDARO AI CODE\\pindaro-q4_k_m.gguf"
|
||||||
|
}
|
||||||
|
}
|
||||||
27
release/RELEASE_NOTES.md
Normal file
27
release/RELEASE_NOTES.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Release Notes - PINDARO-AI-CODE
|
||||||
|
|
||||||
|
Date: 2026-03-02 23:33:21
|
||||||
|
Source: e:\Pindaro\PINDARO AI CODE
|
||||||
|
|
||||||
|
## Verdict
|
||||||
|
- Publishable: True
|
||||||
|
- HF OK: True
|
||||||
|
- GGUF F16 OK: True
|
||||||
|
- GGUF Q4_K_M OK: True
|
||||||
|
|
||||||
|
## Quality Rates
|
||||||
|
- HF: 1.00
|
||||||
|
- GGUF F16: 1.00
|
||||||
|
- GGUF Q4_K_M: 1.00
|
||||||
|
|
||||||
|
## Included Artifact Checksums
|
||||||
|
- model.safetensors: f77c27b8babf9fcab83a7dc68ba58934e8c8c031c9f10b4b73e802d4fbfe0cec
|
||||||
|
- config.json: b37c45060f3e2f5f9b91903c9ccb32f3c21076e809954fda6c01d987cd8f25cc
|
||||||
|
- generation_config.json: 6ff47e725c0ec6d0f1895670de7ee68e61a4f99703f6c8e89aea6ab14ea02dc3
|
||||||
|
- tokenizer.json: 51433f06369ac3e597dfa23a811215e3511b8f86588a830ded72344b76a193ee
|
||||||
|
- tokenizer.model: 9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
||||||
|
- tokenizer_config.json: a0567c49a117af9af332874cfd333ddd622a09c5e9765131ceee6344cb22a3de
|
||||||
|
- special_tokens_map.json: d7805e093432afcde852968cdeba3de08a6fe66e77609f4701decb87fc492f33
|
||||||
|
- added_tokens.json: ece349d292e246eac9a9072c1730f023e61567984a828fb0d25dccb14e3b7592
|
||||||
|
- pindaro-f16.gguf: bdaaeb6fb712e9a4d952082cf415b05c7d076b33786d39063bbfb3a7e5db2031
|
||||||
|
- pindaro-q4_k_m.gguf: 5f98cc3454774ed5ed80d71a71adfd0daff760fc9eef0900ddd4f7eda2e20fef
|
||||||
10
release/SHA256SUMS.txt
Normal file
10
release/SHA256SUMS.txt
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
f77c27b8babf9fcab83a7dc68ba58934e8c8c031c9f10b4b73e802d4fbfe0cec model.safetensors
|
||||||
|
b37c45060f3e2f5f9b91903c9ccb32f3c21076e809954fda6c01d987cd8f25cc config.json
|
||||||
|
6ff47e725c0ec6d0f1895670de7ee68e61a4f99703f6c8e89aea6ab14ea02dc3 generation_config.json
|
||||||
|
51433f06369ac3e597dfa23a811215e3511b8f86588a830ded72344b76a193ee tokenizer.json
|
||||||
|
9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 tokenizer.model
|
||||||
|
a0567c49a117af9af332874cfd333ddd622a09c5e9765131ceee6344cb22a3de tokenizer_config.json
|
||||||
|
d7805e093432afcde852968cdeba3de08a6fe66e77609f4701decb87fc492f33 special_tokens_map.json
|
||||||
|
ece349d292e246eac9a9072c1730f023e61567984a828fb0d25dccb14e3b7592 added_tokens.json
|
||||||
|
bdaaeb6fb712e9a4d952082cf415b05c7d076b33786d39063bbfb3a7e5db2031 pindaro-f16.gguf
|
||||||
|
5f98cc3454774ed5ed80d71a71adfd0daff760fc9eef0900ddd4f7eda2e20fef pindaro-q4_k_m.gguf
|
||||||
34
special_tokens_map.json
Normal file
34
special_tokens_map.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|noesis|>",
|
||||||
|
"<|end|>"
|
||||||
|
],
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"unk_token": {
|
||||||
|
"content": "<unk>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
93415
tokenizer.json
Normal file
93415
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
Binary file not shown.
66
tokenizer_config.json
Normal file
66
tokenizer_config.json
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": true,
|
||||||
|
"add_eos_token": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<unk>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"32000": {
|
||||||
|
"content": "<|noesis|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"32001": {
|
||||||
|
"content": "<|end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|noesis|>",
|
||||||
|
"<|end|>"
|
||||||
|
],
|
||||||
|
"bos_token": "<s>",
|
||||||
|
"chat_template": "{{ bos_token }}{% for message in messages %}<|noesis|>\n{% if message['role'] == 'system' %}### System\n{{ message['content'] }}\n{% elif message['role'] == 'user' %}### Question\n{{ message['content'] }}\n{% elif message['role'] == 'assistant' %}### Answer\n{{ message['content'] }}\n{% endif %}<|end|>\n{% endfor %}{% if add_generation_prompt %}<|noesis|>\n### Answer\n```\n{% endif %}",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "</s>",
|
||||||
|
"legacy": false,
|
||||||
|
"max_length": 512,
|
||||||
|
"model_max_length": 2048,
|
||||||
|
"pad_token": "</s>",
|
||||||
|
"padding_side": "right",
|
||||||
|
"sp_model_kwargs": {},
|
||||||
|
"stride": 0,
|
||||||
|
"tokenizer_class": "LlamaTokenizer",
|
||||||
|
"truncation_side": "right",
|
||||||
|
"truncation_strategy": "longest_first",
|
||||||
|
"unk_token": "<unk>",
|
||||||
|
"use_default_system_prompt": false
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user