commit c08153a4fdc254eae5f61e39d7c921181b6f32f8 Author: ModelHub XC Date: Sun Jul 5 12:56:17 2026 +0800 初始化项目,由ModelHub XC社区提供模型 Model: TingChenChang/make-multilingual-en-zh-tw-20220825062338 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..22ae9a4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +pytorch_model.bin filter=lfs diff=lfs merge=lfs -text +.git/lfs/objects/3c/29/3c2945c018bf63c1f40cb60abb85011cd23cbd43a59eb9309e1f2a4e6dd0b15e filter=lfs diff=lfs merge=lfs -text +.git/lfs/objects/b6/0b/b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441 filter=lfs diff=lfs merge=lfs -text +.git/lfs/objects/74/6e/746e14dfe2d099b775c11f96e1de60efb8661d7c909edf42f4b623f33ad893bc filter=lfs diff=lfs merge=lfs -text diff --git a/.ipynb_checkpoints/config_sentence_transformers-checkpoint.json b/.ipynb_checkpoints/config_sentence_transformers-checkpoint.json new file mode 100644 index 0000000..f2086f0 --- /dev/null +++ b/.ipynb_checkpoints/config_sentence_transformers-checkpoint.json @@ -0,0 +1,7 @@ +{ + "__version__": { + "sentence_transformers": "2.2.2", + "transformers": "4.21.1", + "pytorch": "1.12.1+cu102" + } +} \ No newline at end of file diff --git a/1_Pooling/config.json b/1_Pooling/config.json new file mode 100644 index 0000000..4e09f29 --- /dev/null +++ b/1_Pooling/config.json @@ -0,0 +1,7 @@ +{ + "word_embedding_dimension": 768, + "pooling_mode_cls_token": false, + "pooling_mode_mean_tokens": true, + "pooling_mode_max_tokens": false, + "pooling_mode_mean_sqrt_len_tokens": false +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9410c2c --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ +--- +pipeline_tag: sentence-similarity +tags: +- sentence-transformers +- feature-extraction +- sentence-similarity +- transformers + +--- + +# {MODEL_NAME} + +This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search. + + + +## Usage (Sentence-Transformers) + +Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed: + +``` +pip install -U sentence-transformers +``` + +Then you can use the model like this: + +```python +from sentence_transformers import SentenceTransformer +sentences = ["This is an example sentence", "Each sentence is converted"] + +model = SentenceTransformer('{MODEL_NAME}') +embeddings = model.encode(sentences) +print(embeddings) +``` + + + +## Usage (HuggingFace Transformers) +Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings. + +```python +from transformers import AutoTokenizer, AutoModel +import torch + + +#Mean Pooling - Take attention mask into account for correct averaging +def mean_pooling(model_output, attention_mask): + token_embeddings = model_output[0] #First element of model_output contains all token embeddings + input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + +# Sentences we want sentence embeddings for +sentences = ['This is an example sentence', 'Each sentence is converted'] + +# Load model from HuggingFace Hub +tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}') +model = AutoModel.from_pretrained('{MODEL_NAME}') + +# Tokenize sentences +encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt') + +# Compute token embeddings +with torch.no_grad(): + model_output = model(**encoded_input) + +# Perform pooling. In this case, mean pooling. +sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) + +print("Sentence embeddings:") +print(sentence_embeddings) +``` + + + +## Evaluation Results + + + +For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME}) + + +## Training +The model was trained with the parameters: + +**DataLoader**: + +`torch.utils.data.dataloader.DataLoader` of length 11898 with parameters: +``` +{'batch_size': 64, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'} +``` + +**Loss**: + +`sentence_transformers.losses.MSELoss.MSELoss` + +Parameters of the fit()-Method: +``` +{ + "epochs": 5, + "evaluation_steps": 1000, + "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator", + "max_grad_norm": 1, + "optimizer_class": "", + "optimizer_params": { + "eps": 1e-06, + "lr": 2e-05 + }, + "scheduler": "WarmupLinear", + "steps_per_epoch": null, + "warmup_steps": 10000, + "weight_decay": 0.01 +} +``` + + +## Full Model Architecture +``` +SentenceTransformer( + (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: XLMRobertaModel + (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False}) +) +``` + +## Citing & Authors + + \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..a06d376 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "_name_or_path": "sbert-make-multilingual/output/make-multilingual-en-zh-tw-20220825062338/", + "architectures": [ + "XLMRobertaModel" + ], + "attention_probs_dropout_prob": 0.1, + "bos_token_id": 0, + "classifier_dropout": null, + "eos_token_id": 2, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 514, + "model_type": "xlm-roberta", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "output_past": true, + "pad_token_id": 1, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.21.1", + "type_vocab_size": 1, + "use_cache": true, + "vocab_size": 250002 +} diff --git a/config_sentence_transformers.json b/config_sentence_transformers.json new file mode 100644 index 0000000..f2086f0 --- /dev/null +++ b/config_sentence_transformers.json @@ -0,0 +1,7 @@ +{ + "__version__": { + "sentence_transformers": "2.2.2", + "transformers": "4.21.1", + "pytorch": "1.12.1+cu102" + } +} \ No newline at end of file diff --git a/eval/mse_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv b/eval/mse_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv new file mode 100644 index 0000000..02c8b04 --- /dev/null +++ b/eval/mse_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv @@ -0,0 +1,19 @@ +epoch,steps,MSE +0,1000,0.06223061354830861 +0,2000,0.06203244556672871 +0,3000,0.061916367849335074 +0,4000,0.061945460038259625 +0,5000,0.06187912076711655 +0,6000,0.06188569241203368 +0,7000,0.06136861629784107 +0,8000,0.06148410029709339 +0,9000,0.061285088304430246 +0,10000,0.060733535792678595 +0,11000,0.0604144879616797 +0,-1,0.060609797947108746 +1,1000,0.05997546832077205 +1,2000,0.059760332806035876 +1,3000,0.059662165585905313 +1,4000,0.05945731536485255 +1,5000,0.05934642977081239 +1,6000,0.059219938702881336 diff --git a/eval/similarity_evaluation_STS.en-en.txt_results.csv b/eval/similarity_evaluation_STS.en-en.txt_results.csv new file mode 100644 index 0000000..a597ee1 --- /dev/null +++ b/eval/similarity_evaluation_STS.en-en.txt_results.csv @@ -0,0 +1,19 @@ +epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman +0,1000,0.8331679225212545,0.8551331802776464,0.8636893928030359,0.8574845339805655,0.8615398121331832,0.8547941425195992,0.7080005741063319,0.7291937226458504 +0,2000,0.8330042565586959,0.8555564008803787,0.863358112865497,0.8574491694978666,0.8612043808777003,0.8549598174331121,0.7075851161733188,0.7286963135087607 +0,3000,0.833635776830267,0.8557182318283809,0.8633493324634668,0.8569367688952851,0.8610759874188574,0.8548222034678277,0.7099609820643491,0.7313990056593614 +0,4000,0.8332174364042519,0.8558339351902542,0.8632858458461129,0.8577974327731397,0.8610139064991895,0.8535110268321144,0.7141717576828616,0.7373321664251873 +0,5000,0.8315609868199801,0.8530316843329253,0.8606525559778901,0.8544074395892188,0.8584263125955869,0.8516520851111209,0.7147625153338718,0.7369900534947316 +0,6000,0.8302664779411321,0.8524673901959487,0.8595525190037036,0.8538393014867316,0.8573185347019552,0.8500157089931992,0.7139561219240014,0.7331795304839358 +0,7000,0.8292252167018984,0.8527368521782517,0.8594124881727788,0.8536678606249527,0.857058138987968,0.8512400120083697,0.7195906367631392,0.7397957639210201 +0,8000,0.8253772113909499,0.8494137439942175,0.8567717741559008,0.8516263305421989,0.8543871594992598,0.8485219439957261,0.7167860765038803,0.7394820963353437 +0,9000,0.8287483424264556,0.8531012601086697,0.8580977406115551,0.8537282108834713,0.8559076786891329,0.8518681159728245,0.727402397780654,0.7484804351994331 +0,10000,0.8289035234363668,0.852015724248437,0.8589048569185785,0.8545823400199575,0.8567204274061416,0.8522840330410865,0.7347061193166563,0.7551777763087938 +0,11000,0.8263366408345586,0.8522671195928392,0.8583670535714922,0.8556451964836769,0.8560397081624632,0.8519576803692247,0.7280861839019613,0.7512665414016167 +0,-1,0.8234491989286817,0.8488771264089184,0.8531653050371929,0.8508529246814382,0.850959442679656,0.84733146787705,0.7353021957261283,0.7570878427710801 +1,1000,0.8222198692209769,0.8486437977024165,0.853131791091942,0.849644381924862,0.8508030300369717,0.8469405365846079,0.7395371341066752,0.7627192522443129 +1,2000,0.8213052518761065,0.8470462456361534,0.8509765773303481,0.8467986942572617,0.8487285844789747,0.8445961020196082,0.7394641474760854,0.7636191245703771 +1,3000,0.8216383109014087,0.849223083304885,0.8515903274360738,0.8486061268404113,0.8492304822771835,0.8461171591722076,0.7463819668422524,0.7684113963726156 +1,4000,0.8230286128204333,0.8499419048553931,0.8523041957533484,0.8492638293392988,0.8502357524171301,0.8460964017584496,0.7488882497520122,0.7716103444706527 +1,5000,0.8186606238947536,0.8469989648603713,0.8489671072894741,0.8458903652070741,0.8468554929966491,0.8433929408147468,0.7465549226451859,0.7698913230942501 +1,6000,0.8199293362777689,0.848109870892975,0.8502292653638305,0.8477162488246752,0.8480394020219456,0.8440083596930161,0.7498105203150596,0.7724629360209345 diff --git a/eval/similarity_evaluation_STS.zh-tw-zh-tw.txt_results.csv b/eval/similarity_evaluation_STS.zh-tw-zh-tw.txt_results.csv new file mode 100644 index 0000000..52f53ca --- /dev/null +++ b/eval/similarity_evaluation_STS.zh-tw-zh-tw.txt_results.csv @@ -0,0 +1,19 @@ +epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman +0,1000,0.7570827001090813,0.7829536520179231,0.7849889057884857,0.790791627823479,0.7838292761115199,0.7895196521095137,0.6797829530350897,0.6895117838523157 +0,2000,0.7571509859182012,0.7825081102832119,0.7830713010525879,0.7888766162080632,0.7819867188018441,0.7876742499755591,0.6795408904216148,0.689413289301756 +0,3000,0.7567409102058332,0.7825617227589128,0.7854199651115996,0.7911848559784004,0.7842479363252546,0.7898486147120288,0.6787806382205749,0.6877025314150705 +0,4000,0.7589896738827501,0.7840607305425763,0.7853851134370664,0.7912867047735501,0.7841587846163133,0.7898484882571062,0.6833008363698059,0.6926074094678097 +0,5000,0.7568051989099636,0.7823178729749563,0.7842624264928262,0.7901089747541167,0.7831478597899253,0.7887799216610526,0.678015436039681,0.6877425866761832 +0,6000,0.76110993537507,0.7846938990171494,0.78634783875191,0.7919882952341896,0.7851272481053764,0.7903491564404409,0.6829538970369722,0.6920006194933526 +0,7000,0.757911635275761,0.7836957053279008,0.7853044853694706,0.790984558518405,0.7839192052989779,0.7894861857880131,0.6829468219048354,0.692812478523114 +0,8000,0.7509673298478391,0.779131332572992,0.7813434218384762,0.7881525854798384,0.7803410241755435,0.7867381002299505,0.6757808616967882,0.6861876100596834 +0,9000,0.7558504549770806,0.7829118843511864,0.7845008889459528,0.7907172367634655,0.7832788814874616,0.7890221223552482,0.6824826610740675,0.6926887008922725 +0,10000,0.7572742866990453,0.7824231503561324,0.7844196872652272,0.7901737046325318,0.7833457172800139,0.7891556686329442,0.6840281262215521,0.6932514331833438 +0,11000,0.7527292801998008,0.78030895319115,0.780618682867413,0.7871636953660386,0.7795166926156776,0.7859699826298868,0.6802534685849256,0.6912063548437687 +0,-1,0.753693946875722,0.7807276928698291,0.7817760868186077,0.7883830606687813,0.7807223705325983,0.7871123783777072,0.682180742687755,0.6918835005044305 +1,1000,0.7506244095803017,0.7778029623444135,0.7817872255777,0.7879348570020225,0.7807458502583196,0.7867000681504227,0.6784469549231144,0.6882384440739117 +1,2000,0.7508074817427586,0.7780041864734166,0.7820180491380933,0.7885036986650679,0.7809581726256525,0.7873213392099477,0.6810437219355404,0.6904237892484845 +1,3000,0.7472356599938378,0.7755598764645141,0.7803841315530705,0.7866162917641126,0.7793618555787284,0.7854925844431527,0.6789555619950693,0.6891081173507835 +1,4000,0.7514767547954482,0.7781944173970742,0.7830505799549813,0.7890479436335355,0.7819934897118469,0.787877632960389,0.6807878193751716,0.6900412999928232 +1,5000,0.7468475655312985,0.7750226737318828,0.7782891051699417,0.785069982730165,0.7772581614604364,0.7838711569291307,0.6773213112272526,0.6872811322544907 +1,6000,0.7476747765435937,0.7757197794684102,0.7797893748208706,0.7861068424217299,0.7785661020620962,0.7848519618281135,0.6779008291481604,0.688065412252287 diff --git a/eval/translation_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv b/eval/translation_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv new file mode 100644 index 0000000..8a86a2f --- /dev/null +++ b/eval/translation_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv @@ -0,0 +1,19 @@ +epoch,steps,src2trg,trg2src +0,1000,0.918,0.894 +0,2000,0.916,0.896 +0,3000,0.916,0.897 +0,4000,0.915,0.898 +0,5000,0.917,0.896 +0,6000,0.912,0.895 +0,7000,0.911,0.896 +0,8000,0.916,0.897 +0,9000,0.915,0.892 +0,10000,0.916,0.895 +0,11000,0.919,0.892 +0,-1,0.917,0.889 +1,1000,0.92,0.894 +1,2000,0.917,0.892 +1,3000,0.914,0.891 +1,4000,0.915,0.891 +1,5000,0.916,0.895 +1,6000,0.917,0.893 diff --git a/modules.json b/modules.json new file mode 100644 index 0000000..f7640f9 --- /dev/null +++ b/modules.json @@ -0,0 +1,14 @@ +[ + { + "idx": 0, + "name": "0", + "path": "", + "type": "sentence_transformers.models.Transformer" + }, + { + "idx": 1, + "name": "1", + "path": "1_Pooling", + "type": "sentence_transformers.models.Pooling" + } +] \ No newline at end of file diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..c4d7221 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e242eed564f4ffd52fe77b0e900a85786803a25a095a528b918687f83bf84f20 +size 1112241265 diff --git a/sentence_bert_config.json b/sentence_bert_config.json new file mode 100644 index 0000000..5fd1042 --- /dev/null +++ b/sentence_bert_config.json @@ -0,0 +1,4 @@ +{ + "max_seq_length": 128, + "do_lower_case": false +} \ No newline at end of file diff --git a/sentencepiece.bpe.model b/sentencepiece.bpe.model new file mode 100644 index 0000000..7a3f40a --- /dev/null +++ b/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865 +size 5069051 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..d569813 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,15 @@ +{ + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..b0b95de --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441 +size 17082913 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..25a4f32 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,20 @@ +{ + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 512, + "name_or_path": "sbert-make-multilingual/output/make-multilingual-en-zh-tw-20220825062338/", + "pad_token": "", + "sep_token": "", + "special_tokens_map_file": null, + "tokenizer_class": "XLMRobertaTokenizer", + "unk_token": "" +}