初始化项目,由ModelHub XC社区提供模型

Model: intfloat/multilingual-e5-small
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-05-14 12:43:42 +08:00
commit 2bc0c1287c
43 changed files with 31318 additions and 0 deletions

View File

@@ -0,0 +1,20 @@
- dataset:
id: mteb/arguana
task_id: ArguAna_default_test
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a
value: 39.088
notes: Obtained using MTEB v1.36.26
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v1.36.26
user: mteb
- dataset:
id: mteb/arguana
task_id: ArguAna
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a
value: 39.088
notes: Obtained using MTEB v1.36.26
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v1.36.26
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightAopsRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.921
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightAopsRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.921
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightBiologyLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.709
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightBiologyLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.709
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightBiologyRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 4.237
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightBiologyRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 4.237
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightEarthScienceLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 14.943
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightEarthScienceLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 14.943
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightEarthScienceRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 19.293
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightEarthScienceRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 19.293
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightEconomicsLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.68
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightEconomicsLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.68
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightEconomicsRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.414
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightEconomicsRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.414
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightLeetcodeRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 23.579
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightLeetcodeRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 23.579
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightPonyLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 0.749
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightPonyLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 0.749
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightPonyRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 0.184
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightPonyRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 0.184
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightPsychologyLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 2.97
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightPsychologyLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 2.97
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightPsychologyRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.909
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightPsychologyRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 7.909
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightRoboticsLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 8.911
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightRoboticsLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 8.911
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightRoboticsRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.57
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightRoboticsRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.57
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightStackoverflowLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.256
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightStackoverflowLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.256
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightStackoverflowRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 6.857
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightStackoverflowRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 6.857
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightSustainableLivingLongRetrieval_default_long
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.417
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightSustainableLivingLongRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 10.417
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightSustainableLivingRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.101
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightSustainableLivingRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.101
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightTheoremQAQuestionsRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.877
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightTheoremQAQuestionsRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 9.877
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

View File

@@ -0,0 +1,22 @@
- dataset:
id: mteb/BRIGHT
task_id: BrightTheoremQATheoremsRetrieval_default_standard
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 5.475
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb
- dataset:
id: mteb/BRIGHT
task_id: BrightTheoremQATheoremsRetrieval
revision: c26703e6600d97c579ee2985f16cf307db13ed85
value: 5.475
date: '2026-03-31'
notes: Obtained using MTEB v2.10.12
source:
url: https://github.com/embeddings-benchmark/mteb/
name: Obtained using MTEB v2.10.12
user: mteb

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

7
1_Pooling/config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"word_embedding_dimension": 384,
"pooling_mode_cls_token": false,
"pooling_mode_mean_tokens": true,
"pooling_mode_max_tokens": false,
"pooling_mode_mean_sqrt_len_tokens": false
}

18484
README.md Normal file

File diff suppressed because it is too large Load Diff

26
config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"_name_or_path": "tmp/",
"architectures": [
"BertModel"
],
"attention_probs_dropout_prob": 0.1,
"classifier_dropout": null,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"hidden_size": 384,
"initializer_range": 0.02,
"intermediate_size": 1536,
"layer_norm_eps": 1e-12,
"max_position_embeddings": 512,
"model_type": "bert",
"num_attention_heads": 12,
"num_hidden_layers": 12,
"pad_token_id": 0,
"position_embedding_type": "absolute",
"tokenizer_class": "XLMRobertaTokenizer",
"torch_dtype": "float32",
"transformers_version": "4.29.0",
"type_vocab_size": 2,
"use_cache": true,
"vocab_size": 250037
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1a55775f53449dac10a2bcbc312469fac40b96d53198c407081a831f81c98477
size 470641600

20
modules.json Normal file
View File

@@ -0,0 +1,20 @@
[
{
"idx": 0,
"name": "0",
"path": "",
"type": "sentence_transformers.models.Transformer"
},
{
"idx": 1,
"name": "1",
"path": "1_Pooling",
"type": "sentence_transformers.models.Pooling"
},
{
"idx": 2,
"name": "2",
"path": "2_Normalize",
"type": "sentence_transformers.models.Normalize"
}
]

25
onnx/config.json Normal file
View File

@@ -0,0 +1,25 @@
{
"_name_or_path": "intfloat/multilingual-e5-small",
"architectures": [
"BertModel"
],
"attention_probs_dropout_prob": 0.1,
"classifier_dropout": null,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"hidden_size": 384,
"initializer_range": 0.02,
"intermediate_size": 1536,
"layer_norm_eps": 1e-12,
"max_position_embeddings": 512,
"model_type": "bert",
"num_attention_heads": 12,
"num_hidden_layers": 12,
"pad_token_id": 0,
"position_embedding_type": "absolute",
"tokenizer_class": "XLMRobertaTokenizer",
"transformers_version": "4.30.2",
"type_vocab_size": 2,
"use_cache": true,
"vocab_size": 250037
}

3
onnx/model.onnx Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ca456c06b3a9505ddfd9131408916dd79290368331e7d76bb621f1cba6bc8665
size 470268510

3
onnx/model_O4.onnx Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4654c156f3e4171abc9c716cdb771bf9116455d15ac1aab364aeeede0e3205b0
size 235052531

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dd476dd0c2514e9b9be83aeb3853fac0763e0bdf4a71645407587d77c48a2d88
size 118346824

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
size 5069051

View File

@@ -0,0 +1,9 @@
{
"bos_token": "<s>",
"cls_token": "<s>",
"eos_token": "</s>",
"mask_token": "<mask>",
"pad_token": "<pad>",
"sep_token": "</s>",
"unk_token": "<unk>"
}

3
onnx/tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39
size 17082730

View File

@@ -0,0 +1,20 @@
{
"bos_token": "<s>",
"clean_up_tokenization_spaces": true,
"cls_token": "<s>",
"eos_token": "</s>",
"mask_token": {
"__type": "AddedToken",
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false
},
"model_max_length": 512,
"pad_token": "<pad>",
"sep_token": "</s>",
"sp_model_kwargs": {},
"tokenizer_class": "XLMRobertaTokenizer",
"unk_token": "<unk>"
}

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:22f5a10ec005a1606ec42758dedd52b59dfcb6e3d85de0b6b86946d0bc6cf232
size 470027936

12168
openvino/openvino_model.xml Normal file

File diff suppressed because it is too large Load Diff

3
pytorch_model.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f0ccb64a2edc689fce7b1895f25146740e4075c3da28c8d0e672a8ba49c51167
size 470681649

View File

@@ -0,0 +1,4 @@
{
"max_seq_length": 512,
"do_lower_case": false
}

3
sentencepiece.bpe.model Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
size 5069051

9
special_tokens_map.json Normal file
View File

@@ -0,0 +1,9 @@
{
"bos_token": "<s>",
"cls_token": "<s>",
"eos_token": "</s>",
"mask_token": "<mask>",
"pad_token": "<pad>",
"sep_token": "</s>",
"unk_token": "<unk>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39
size 17082730

20
tokenizer_config.json Normal file
View File

@@ -0,0 +1,20 @@
{
"bos_token": "<s>",
"clean_up_tokenization_spaces": true,
"cls_token": "<s>",
"eos_token": "</s>",
"mask_token": {
"__type": "AddedToken",
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false
},
"model_max_length": 512,
"pad_token": "<pad>",
"sep_token": "</s>",
"sp_model_kwargs": {},
"tokenizer_class": "XLMRobertaTokenizer",
"unk_token": "<unk>"
}