初始化项目，由ModelHub XC社区提供模型
Model: cointegrated/SONAR_200_text_encoder Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
281
README.md
Normal file
281
README.md
Normal file
@@ -0,0 +1,281 @@
|
||||
---
|
||||
license: cc-by-nc-4.0
|
||||
language:
|
||||
- ace
|
||||
- ace
|
||||
- acm
|
||||
- acq
|
||||
- aeb
|
||||
- af
|
||||
- ajp
|
||||
- ak
|
||||
- am
|
||||
- apc
|
||||
- ar
|
||||
- ars
|
||||
- ary
|
||||
- arz
|
||||
- as
|
||||
- ast
|
||||
- awa
|
||||
- ay
|
||||
- azb
|
||||
- azj
|
||||
- ba
|
||||
- bm
|
||||
- ban
|
||||
- be
|
||||
- bem
|
||||
- bn
|
||||
- bho
|
||||
- bjn
|
||||
- bjn
|
||||
- bo
|
||||
- bs
|
||||
- bug
|
||||
- bg
|
||||
- ca
|
||||
- ceb
|
||||
- cs
|
||||
- cjk
|
||||
- ckb
|
||||
- crh
|
||||
- cy
|
||||
- da
|
||||
- de
|
||||
- dik
|
||||
- dyu
|
||||
- dz
|
||||
- el
|
||||
- en
|
||||
- eo
|
||||
- et
|
||||
- eu
|
||||
- ee
|
||||
- fo
|
||||
- fa
|
||||
- fj
|
||||
- fi
|
||||
- fon
|
||||
- fr
|
||||
- fur
|
||||
- ff
|
||||
- gd
|
||||
- ga
|
||||
- gl
|
||||
- gn
|
||||
- gu
|
||||
- ht
|
||||
- ha
|
||||
- he
|
||||
- hi
|
||||
- hne
|
||||
- hr
|
||||
- hu
|
||||
- hy
|
||||
- ig
|
||||
- ilo
|
||||
- id
|
||||
- is
|
||||
- it
|
||||
- jv
|
||||
- ja
|
||||
- kab
|
||||
- kac
|
||||
- kam
|
||||
- kn
|
||||
- ks
|
||||
- ks
|
||||
- ka
|
||||
- kr
|
||||
- kr
|
||||
- kk
|
||||
- kbp
|
||||
- kea
|
||||
- km
|
||||
- ki
|
||||
- rw
|
||||
- ky
|
||||
- kmb
|
||||
- kg
|
||||
- ko
|
||||
- kmr
|
||||
- lo
|
||||
- lv
|
||||
- lij
|
||||
- li
|
||||
- ln
|
||||
- lt
|
||||
- lmo
|
||||
- ltg
|
||||
- lb
|
||||
- lua
|
||||
- lg
|
||||
- luo
|
||||
- lus
|
||||
- mag
|
||||
- mai
|
||||
- ml
|
||||
- mr
|
||||
- min
|
||||
- mk
|
||||
- plt
|
||||
- mt
|
||||
- mni
|
||||
- mn
|
||||
- mos
|
||||
- mi
|
||||
- ms
|
||||
- my
|
||||
- nl
|
||||
- nn
|
||||
- nb
|
||||
- ne
|
||||
- nso
|
||||
- nus
|
||||
- ny
|
||||
- oc
|
||||
- gaz
|
||||
- ory
|
||||
- pag
|
||||
- pa
|
||||
- pap
|
||||
- pl
|
||||
- pt
|
||||
- prs
|
||||
- pbt
|
||||
- qu
|
||||
- ro
|
||||
- rn
|
||||
- ru
|
||||
- sg
|
||||
- sa
|
||||
- sat
|
||||
- scn
|
||||
- shn
|
||||
- si
|
||||
- sk
|
||||
- sl
|
||||
- sm
|
||||
- sn
|
||||
- sd
|
||||
- so
|
||||
- st
|
||||
- es
|
||||
- als
|
||||
- sc
|
||||
- sr
|
||||
- ss
|
||||
- su
|
||||
- sv
|
||||
- sw
|
||||
- szl
|
||||
- ta
|
||||
- tt
|
||||
- te
|
||||
- tg
|
||||
- tl
|
||||
- th
|
||||
- ti
|
||||
- taq
|
||||
- taq
|
||||
- tpi
|
||||
- tn
|
||||
- ts
|
||||
- tk
|
||||
- tum
|
||||
- tr
|
||||
- tw
|
||||
- tzm
|
||||
- ug
|
||||
- uk
|
||||
- umb
|
||||
- ur
|
||||
- uz
|
||||
- vec
|
||||
- vi
|
||||
- war
|
||||
- wo
|
||||
- xh
|
||||
- yi
|
||||
- yo
|
||||
- yue
|
||||
- zh
|
||||
- zh
|
||||
- zu
|
||||
language_details: >-
|
||||
ace_Arab, ace_Latn, acm_Arab, acq_Arab, aeb_Arab, afr_Latn, ajp_Arab,
|
||||
aka_Latn, amh_Ethi, apc_Arab, arb_Arab, ars_Arab, ary_Arab, arz_Arab,
|
||||
asm_Beng, ast_Latn, awa_Deva, ayr_Latn, azb_Arab, azj_Latn, bak_Cyrl,
|
||||
bam_Latn, ban_Latn,bel_Cyrl, bem_Latn, ben_Beng, bho_Deva, bjn_Arab, bjn_Latn,
|
||||
bod_Tibt, bos_Latn, bug_Latn, bul_Cyrl, cat_Latn, ceb_Latn, ces_Latn,
|
||||
cjk_Latn, ckb_Arab, crh_Latn, cym_Latn, dan_Latn, deu_Latn, dik_Latn,
|
||||
dyu_Latn, dzo_Tibt, ell_Grek, eng_Latn, epo_Latn, est_Latn, eus_Latn,
|
||||
ewe_Latn, fao_Latn, pes_Arab, fij_Latn, fin_Latn, fon_Latn, fra_Latn,
|
||||
fur_Latn, fuv_Latn, gla_Latn, gle_Latn, glg_Latn, grn_Latn, guj_Gujr,
|
||||
hat_Latn, hau_Latn, heb_Hebr, hin_Deva, hne_Deva, hrv_Latn, hun_Latn,
|
||||
hye_Armn, ibo_Latn, ilo_Latn, ind_Latn, isl_Latn, ita_Latn, jav_Latn,
|
||||
jpn_Jpan, kab_Latn, kac_Latn, kam_Latn, kan_Knda, kas_Arab, kas_Deva,
|
||||
kat_Geor, knc_Arab, knc_Latn, kaz_Cyrl, kbp_Latn, kea_Latn, khm_Khmr,
|
||||
kik_Latn, kin_Latn, kir_Cyrl, kmb_Latn, kon_Latn, kor_Hang, kmr_Latn,
|
||||
lao_Laoo, lvs_Latn, lij_Latn, lim_Latn, lin_Latn, lit_Latn, lmo_Latn,
|
||||
ltg_Latn, ltz_Latn, lua_Latn, lug_Latn, luo_Latn, lus_Latn, mag_Deva,
|
||||
mai_Deva, mal_Mlym, mar_Deva, min_Latn, mkd_Cyrl, plt_Latn, mlt_Latn,
|
||||
mni_Beng, khk_Cyrl, mos_Latn, mri_Latn, zsm_Latn, mya_Mymr, nld_Latn,
|
||||
nno_Latn, nob_Latn, npi_Deva, nso_Latn, nus_Latn, nya_Latn, oci_Latn,
|
||||
gaz_Latn, ory_Orya, pag_Latn, pan_Guru, pap_Latn, pol_Latn, por_Latn,
|
||||
prs_Arab, pbt_Arab, quy_Latn, ron_Latn, run_Latn, rus_Cyrl, sag_Latn,
|
||||
san_Deva, sat_Beng, scn_Latn, shn_Mymr, sin_Sinh, slk_Latn, slv_Latn,
|
||||
smo_Latn, sna_Latn, snd_Arab, som_Latn, sot_Latn, spa_Latn, als_Latn,
|
||||
srd_Latn, srp_Cyrl, ssw_Latn, sun_Latn, swe_Latn, swh_Latn, szl_Latn,
|
||||
tam_Taml, tat_Cyrl, tel_Telu, tgk_Cyrl, tgl_Latn, tha_Thai, tir_Ethi,
|
||||
taq_Latn, taq_Tfng, tpi_Latn, tsn_Latn, tso_Latn, tuk_Latn, tum_Latn,
|
||||
tur_Latn, twi_Latn, tzm_Tfng, uig_Arab, ukr_Cyrl, umb_Latn, urd_Arab,
|
||||
uzn_Latn, vec_Latn, vie_Latn, war_Latn, wol_Latn, xho_Latn, ydd_Hebr,
|
||||
yor_Latn, yue_Hant, zho_Hans, zho_Hant, zul_Latn
|
||||
pipeline_tag: sentence-similarity
|
||||
---
|
||||
This is a port of the multilingual SONAR text encoder (https://huggingface.co/facebook/SONAR) to the `transformers` format from `fairseq2`.
|
||||
|
||||
Its embeddings are expected be equal to those the official implementation (https://github.com/facebookresearch/SONAR), but the latter stays the source of truth.
|
||||
|
||||
The encoder supports the same 202 languages as [NLLB-200](https://huggingface.co/facebook/nllb-200-distilled-600M)
|
||||
(see also [the source model card](https://github.com/facebookresearch/SONAR/blob/main/sonar/store/cards/text_sonar_basic_encoder.yaml#L14)
|
||||
and [FLORES-200 lang code mapping](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200)).
|
||||
|
||||
|
||||
How to compute embeddings:
|
||||
```Python
|
||||
# !pip install transformers sentencepiece -q
|
||||
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.models.m2m_100.modeling_m2m_100 import M2M100Encoder
|
||||
|
||||
model_name = "cointegrated/SONAR_200_text_encoder"
|
||||
encoder = M2M100Encoder.from_pretrained(model_name)
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
|
||||
def encode_mean_pool(texts, tokenizer, encoder, lang='eng_Latn', norm=False):
|
||||
tokenizer.src_lang = lang
|
||||
with torch.inference_mode():
|
||||
batch = tokenizer(texts, return_tensors='pt', padding=True)
|
||||
seq_embs = encoder(**batch).last_hidden_state
|
||||
mask = batch.attention_mask
|
||||
mean_emb = (seq_embs * mask.unsqueeze(-1)).sum(1) / mask.unsqueeze(-1).sum(1)
|
||||
if norm:
|
||||
mean_emb = torch.nn.functional.normalize(mean_emb)
|
||||
return mean_emb
|
||||
|
||||
sentences = ['My name is SONAR.', 'I can embed the sentences into vectorial space.']
|
||||
embs = encode_mean_pool(sentences, tokenizer, encoder, lang="eng_Latn")
|
||||
print(embs.shape)
|
||||
# torch.Size([2, 1024])
|
||||
print(embs)
|
||||
# tensor([[-0.0053, 0.0020, -0.0006, ..., 0.0094, -0.0009, 0.0070],
|
||||
# [-0.0003, -0.0071, 0.0076, ..., 0.0055, 0.0022, -0.0083]])
|
||||
```
|
||||
|
||||
For advanced examples of usage, please take a look at the readme in https://github.com/facebookresearch/SONAR.
|
||||
|
||||
The model was repacked [in this notebook](https://colab.research.google.com/drive/1s6JuQWaMnWXyFdni1AAw-Z48k8YoKwG7?usp=sharing).
|
||||
34
config.json
Normal file
34
config.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"_name_or_path": "tmp",
|
||||
"activation_dropout": 0.0,
|
||||
"activation_function": "relu",
|
||||
"architectures": [
|
||||
"M2M100Encoder"
|
||||
],
|
||||
"attention_dropout": 0.1,
|
||||
"bos_token_id": 0,
|
||||
"d_model": 1024,
|
||||
"decoder_attention_heads": 16,
|
||||
"decoder_ffn_dim": 8192,
|
||||
"decoder_layerdrop": 0,
|
||||
"decoder_layers": 24,
|
||||
"decoder_start_token_id": 2,
|
||||
"dropout": 0.1,
|
||||
"encoder_attention_heads": 16,
|
||||
"encoder_ffn_dim": 8192,
|
||||
"encoder_layerdrop": 0,
|
||||
"encoder_layers": 24,
|
||||
"eos_token_id": 2,
|
||||
"init_std": 0.02,
|
||||
"is_encoder_decoder": true,
|
||||
"max_length": 200,
|
||||
"max_position_embeddings": 1024,
|
||||
"model_type": "m2m_100",
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 1,
|
||||
"scale_embedding": true,
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.34.1",
|
||||
"use_cache": true,
|
||||
"vocab_size": 256206
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6b9e604e8707790fcba45add46b068d4f24fd669a2beafce77d582fa4af5f5ac
|
||||
size 3064405288
|
||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a70a6723239f732b7eb546cc685e6e13c5bbb863ebc8dc5997cd4382fa4d972a
|
||||
size 3064485337
|
||||
3
sentencepiece.bpe.model
Normal file
3
sentencepiece.bpe.model
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a
|
||||
size 4852054
|
||||
255
special_tokens_map.json
Normal file
255
special_tokens_map.json
Normal file
@@ -0,0 +1,255 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"ace_Arab",
|
||||
"ace_Latn",
|
||||
"acm_Arab",
|
||||
"acq_Arab",
|
||||
"aeb_Arab",
|
||||
"afr_Latn",
|
||||
"ajp_Arab",
|
||||
"aka_Latn",
|
||||
"amh_Ethi",
|
||||
"apc_Arab",
|
||||
"arb_Arab",
|
||||
"ars_Arab",
|
||||
"ary_Arab",
|
||||
"arz_Arab",
|
||||
"asm_Beng",
|
||||
"ast_Latn",
|
||||
"awa_Deva",
|
||||
"ayr_Latn",
|
||||
"azb_Arab",
|
||||
"azj_Latn",
|
||||
"bak_Cyrl",
|
||||
"bam_Latn",
|
||||
"ban_Latn",
|
||||
"bel_Cyrl",
|
||||
"bem_Latn",
|
||||
"ben_Beng",
|
||||
"bho_Deva",
|
||||
"bjn_Arab",
|
||||
"bjn_Latn",
|
||||
"bod_Tibt",
|
||||
"bos_Latn",
|
||||
"bug_Latn",
|
||||
"bul_Cyrl",
|
||||
"cat_Latn",
|
||||
"ceb_Latn",
|
||||
"ces_Latn",
|
||||
"cjk_Latn",
|
||||
"ckb_Arab",
|
||||
"crh_Latn",
|
||||
"cym_Latn",
|
||||
"dan_Latn",
|
||||
"deu_Latn",
|
||||
"dik_Latn",
|
||||
"dyu_Latn",
|
||||
"dzo_Tibt",
|
||||
"ell_Grek",
|
||||
"eng_Latn",
|
||||
"epo_Latn",
|
||||
"est_Latn",
|
||||
"eus_Latn",
|
||||
"ewe_Latn",
|
||||
"fao_Latn",
|
||||
"pes_Arab",
|
||||
"fij_Latn",
|
||||
"fin_Latn",
|
||||
"fon_Latn",
|
||||
"fra_Latn",
|
||||
"fur_Latn",
|
||||
"fuv_Latn",
|
||||
"gla_Latn",
|
||||
"gle_Latn",
|
||||
"glg_Latn",
|
||||
"grn_Latn",
|
||||
"guj_Gujr",
|
||||
"hat_Latn",
|
||||
"hau_Latn",
|
||||
"heb_Hebr",
|
||||
"hin_Deva",
|
||||
"hne_Deva",
|
||||
"hrv_Latn",
|
||||
"hun_Latn",
|
||||
"hye_Armn",
|
||||
"ibo_Latn",
|
||||
"ilo_Latn",
|
||||
"ind_Latn",
|
||||
"isl_Latn",
|
||||
"ita_Latn",
|
||||
"jav_Latn",
|
||||
"jpn_Jpan",
|
||||
"kab_Latn",
|
||||
"kac_Latn",
|
||||
"kam_Latn",
|
||||
"kan_Knda",
|
||||
"kas_Arab",
|
||||
"kas_Deva",
|
||||
"kat_Geor",
|
||||
"knc_Arab",
|
||||
"knc_Latn",
|
||||
"kaz_Cyrl",
|
||||
"kbp_Latn",
|
||||
"kea_Latn",
|
||||
"khm_Khmr",
|
||||
"kik_Latn",
|
||||
"kin_Latn",
|
||||
"kir_Cyrl",
|
||||
"kmb_Latn",
|
||||
"kon_Latn",
|
||||
"kor_Hang",
|
||||
"kmr_Latn",
|
||||
"lao_Laoo",
|
||||
"lvs_Latn",
|
||||
"lij_Latn",
|
||||
"lim_Latn",
|
||||
"lin_Latn",
|
||||
"lit_Latn",
|
||||
"lmo_Latn",
|
||||
"ltg_Latn",
|
||||
"ltz_Latn",
|
||||
"lua_Latn",
|
||||
"lug_Latn",
|
||||
"luo_Latn",
|
||||
"lus_Latn",
|
||||
"mag_Deva",
|
||||
"mai_Deva",
|
||||
"mal_Mlym",
|
||||
"mar_Deva",
|
||||
"min_Latn",
|
||||
"mkd_Cyrl",
|
||||
"plt_Latn",
|
||||
"mlt_Latn",
|
||||
"mni_Beng",
|
||||
"khk_Cyrl",
|
||||
"mos_Latn",
|
||||
"mri_Latn",
|
||||
"zsm_Latn",
|
||||
"mya_Mymr",
|
||||
"nld_Latn",
|
||||
"nno_Latn",
|
||||
"nob_Latn",
|
||||
"npi_Deva",
|
||||
"nso_Latn",
|
||||
"nus_Latn",
|
||||
"nya_Latn",
|
||||
"oci_Latn",
|
||||
"gaz_Latn",
|
||||
"ory_Orya",
|
||||
"pag_Latn",
|
||||
"pan_Guru",
|
||||
"pap_Latn",
|
||||
"pol_Latn",
|
||||
"por_Latn",
|
||||
"prs_Arab",
|
||||
"pbt_Arab",
|
||||
"quy_Latn",
|
||||
"ron_Latn",
|
||||
"run_Latn",
|
||||
"rus_Cyrl",
|
||||
"sag_Latn",
|
||||
"san_Deva",
|
||||
"sat_Beng",
|
||||
"scn_Latn",
|
||||
"shn_Mymr",
|
||||
"sin_Sinh",
|
||||
"slk_Latn",
|
||||
"slv_Latn",
|
||||
"smo_Latn",
|
||||
"sna_Latn",
|
||||
"snd_Arab",
|
||||
"som_Latn",
|
||||
"sot_Latn",
|
||||
"spa_Latn",
|
||||
"als_Latn",
|
||||
"srd_Latn",
|
||||
"srp_Cyrl",
|
||||
"ssw_Latn",
|
||||
"sun_Latn",
|
||||
"swe_Latn",
|
||||
"swh_Latn",
|
||||
"szl_Latn",
|
||||
"tam_Taml",
|
||||
"tat_Cyrl",
|
||||
"tel_Telu",
|
||||
"tgk_Cyrl",
|
||||
"tgl_Latn",
|
||||
"tha_Thai",
|
||||
"tir_Ethi",
|
||||
"taq_Latn",
|
||||
"taq_Tfng",
|
||||
"tpi_Latn",
|
||||
"tsn_Latn",
|
||||
"tso_Latn",
|
||||
"tuk_Latn",
|
||||
"tum_Latn",
|
||||
"tur_Latn",
|
||||
"twi_Latn",
|
||||
"tzm_Tfng",
|
||||
"uig_Arab",
|
||||
"ukr_Cyrl",
|
||||
"umb_Latn",
|
||||
"urd_Arab",
|
||||
"uzn_Latn",
|
||||
"vec_Latn",
|
||||
"vie_Latn",
|
||||
"war_Latn",
|
||||
"wol_Latn",
|
||||
"xho_Latn",
|
||||
"ydd_Hebr",
|
||||
"yor_Latn",
|
||||
"yue_Hant",
|
||||
"zho_Hans",
|
||||
"zho_Hant",
|
||||
"zul_Latn"
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"cls_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"mask_token": {
|
||||
"content": "<mask>",
|
||||
"lstrip": true,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"sep_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8ac789ad7dabea44d41537822d48c516ba358374c51813e2cba78c006e150c94
|
||||
size 17331224
|
||||
1878
tokenizer_config.json
Normal file
1878
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user