From 38b51e173bf5eaa1e7ce8cbaac0ff5003f80f78c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 1 Nov 2026 01:15:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: upskyy/e5-base-korean Source: Original Platform --- .gitattributes | 36 ++++ 1_Pooling/config.json | 10 ++ README.md | 357 ++++++++++++++++++++++++++++++++++++++ config.json | 27 +++ model.safetensors | 3 + modules.json | 14 ++ sentence_bert_config.json | 4 + sentencepiece.bpe.model | 3 + special_tokens_map.json | 51 ++++++ tokenizer.json | 3 + tokenizer_config.json | 54 ++++++ 11 files changed, 562 insertions(+) create mode 100644 .gitattributes create mode 100644 1_Pooling/config.json create mode 100644 README.md create mode 100644 config.json create mode 100644 model.safetensors create mode 100644 modules.json create mode 100644 sentence_bert_config.json create mode 100644 sentencepiece.bpe.model create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/1_Pooling/config.json b/1_Pooling/config.json new file mode 100644 index 0000000..9213783 --- /dev/null +++ b/1_Pooling/config.json @@ -0,0 +1,10 @@ +{ + "word_embedding_dimension": 768, + "pooling_mode_cls_token": false, + "pooling_mode_mean_tokens": true, + "pooling_mode_max_tokens": false, + "pooling_mode_mean_sqrt_len_tokens": false, + "pooling_mode_weightedmean_tokens": false, + "pooling_mode_lasttoken": false, + "include_prompt": true +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c39bc03 --- /dev/null +++ b/README.md @@ -0,0 +1,357 @@ +--- +language: + - multilingual + - af + - am + - ar + - as + - az + - be + - bg + - bn + - br + - bs + - ca + - cs + - cy + - da + - de + - el + - en + - eo + - es + - et + - eu + - fa + - fi + - fr + - fy + - ga + - gd + - gl + - gu + - ha + - he + - hi + - hr + - hu + - hy + - id + - is + - it + - ja + - jv + - ka + - kk + - km + - kn + - ko + - ku + - ky + - la + - lo + - lt + - lv + - mg + - mk + - ml + - mn + - mr + - ms + - my + - ne + - nl + - 'no' + - om + - or + - pa + - pl + - ps + - pt + - ro + - ru + - sa + - sd + - si + - sk + - sl + - so + - sq + - sr + - su + - sv + - sw + - ta + - te + - th + - tl + - tr + - ug + - uk + - ur + - uz + - vi + - xh + - yi + - zh +license: mit +library_name: sentence-transformers +tags: + - korean + - sentence-transformers + - transformers + - multilingual + - sentence-transformers + - sentence-similarity + - feature-extraction +base_model: intfloat/multilingual-e5-base +datasets: [] +metrics: +- pearson_cosine +- spearman_cosine +- pearson_manhattan +- spearman_manhattan +- pearson_euclidean +- spearman_euclidean +- pearson_dot +- spearman_dot +- pearson_max +- spearman_max +widget: +- source_sentence: 이집트 군대가 형제애를 단속하다 + sentences: + - 이집트의 군대가 무슬림 형제애를 단속하다 + - 아르헨티나의 기예르모 코리아와 네덜란드의 마틴 버커크의 또 다른 준결승전도 매력적이다. + - 그것이 사실일 수도 있다고 생각하는 것은 재미있다. +- source_sentence: 오, 그리고 다시 결혼은 근본적인 인권이라고 주장한다. + sentences: + - 특히 결혼은 근본적인 인권이라고 말한 후에. + - 해변에 있는 흑인과 그의 개... + - 이란은 핵 프로그램이 평화적인 목적을 위한 것이라고 주장한다 +- source_sentence: 담배 피우는 여자. + sentences: + - 이것은 내가 영국의 아서 안데르센 사업부의 파트너인 짐 와디아를 아서 안데르센 경영진이 선택한 것보다 래리 웨인바흐를 안데르센 월드와이드의 + 경영 파트너로 승계하기 위해 안데르센 컨설팅 사업부(현재의 엑센츄어라고 알려져 있음)의 전 관리 파트너인 조지 샤힌에 대한 지지를 표명했을 + 때 가장 명백했다. + - 한 여자가 물 한 잔을 마시고 있다. + - 한 여성이 담배를 피우면서 청구서를 지불하는 것을 압도했다. +- source_sentence: 루이 15세의 소수 민족인 프랑스의 리젠트인 필리프 도를레앙 시대에는 악명 높은 오르가즘의 현장이었다. + sentences: + - 필립 도린스는 루이 15세가 70대였을 때 섭정이었다. + - 행복한 어린 소년이 커다란 엘모 인형이 있는 의자에 앉아 있다. + - 필리프 도를레앙 시대에는 그곳에서 많은 유명한 오르가즘이 일어났다. +- source_sentence: 두 남자가 안에서 일하고 있다 + sentences: + - 국립공원에서 가장 큰 마을인 케스윅의 인구는 매년 여름 등산객, 뱃사람, 관광객이 도착함에 따라 증가한다. + - 두 남자가 축구 경기를 보고 간식을 먹는다. + - 두 남자가 집에 타일을 깔았다. +pipeline_tag: sentence-similarity +model-index: +- name: upskyy/e5-base-korean + results: + - task: + type: semantic-similarity + name: Semantic Similarity + dataset: + name: sts dev + type: sts-dev + metrics: + - type: pearson_cosine + value: 0.8593935914692068 + name: Pearson Cosine + - type: spearman_cosine + value: 0.8572594228080116 + name: Spearman Cosine + - type: pearson_manhattan + value: 0.8217336375412545 + name: Pearson Manhattan + - type: spearman_manhattan + value: 0.8280050978871264 + name: Spearman Manhattan + - type: pearson_euclidean + value: 0.8208931119126335 + name: Pearson Euclidean + - type: spearman_euclidean + value: 0.8277058727421436 + name: Spearman Euclidean + - type: pearson_dot + value: 0.8187961699085111 + name: Pearson Dot + - type: spearman_dot + value: 0.8236175658758088 + name: Spearman Dot + - type: pearson_max + value: 0.8593935914692068 + name: Pearson Max + - type: spearman_max + value: 0.8572594228080116 + name: Spearman Max +--- + +# upskyy/e5-base-korean + +This model is korsts and kornli finetuning model from [intfloat/multilingual-e5-base](https://huggingface.co/intfloat/multilingual-e5-base). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more. + +## Model Details + +### Model Description +- **Model Type:** Sentence Transformer +- **Base model:** [intfloat/multilingual-e5-base](https://huggingface.co/intfloat/multilingual-e5-base) +- **Maximum Sequence Length:** 512 tokens +- **Output Dimensionality:** 768 tokens +- **Similarity Function:** Cosine Similarity + + + + + +### Full Model Architecture + +``` +SentenceTransformer( + (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: XLMRobertaModel + (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True}) +) +``` + + +## Usage + +### Usage (Sentence-Transformers) + + +First install the Sentence Transformers library: + +```bash +pip install -U sentence-transformers +``` + +Then you can load this model and run inference. +```python +from sentence_transformers import SentenceTransformer + +# Download from the 🤗 Hub +model = SentenceTransformer("upskyy/e5-base-korean") + +# Run inference +sentences = [ + '아이를 가진 엄마가 해변을 걷는다.', + '두 사람이 해변을 걷는다.', + '한 남자가 해변에서 개를 산책시킨다.', +] +embeddings = model.encode(sentences) +print(embeddings.shape) +# [3, 768] + +# Get the similarity scores for the embeddings +similarities = model.similarity(embeddings, embeddings) +print(similarities.shape) +# [3, 3] +``` + +### Usage (HuggingFace Transformers) + +Without sentence-transformers, you can use the model like this: +First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings. + +```python +from transformers import AutoTokenizer, AutoModel +import torch + + +# Mean Pooling - Take attention mask into account for correct averaging +def mean_pooling(model_output, attention_mask): + token_embeddings = model_output[0] # First element of model_output contains all token embeddings + input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + +# Sentences we want sentence embeddings for +sentences = ["안녕하세요?", "한국어 문장 임베딩을 위한 버트 모델입니다."] + +# Load model from HuggingFace Hub +tokenizer = AutoTokenizer.from_pretrained("upskyy/e5-base-korean") +model = AutoModel.from_pretrained("upskyy/e5-base-korean") + +# Tokenize sentences +encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors="pt") + +# Compute token embeddings +with torch.no_grad(): + model_output = model(**encoded_input) + +# Perform pooling. In this case, mean pooling. +sentence_embeddings = mean_pooling(model_output, encoded_input["attention_mask"]) + +print("Sentence embeddings:") +print(sentence_embeddings) +``` + + +## Evaluation + +### Metrics + +#### Semantic Similarity +* Dataset: `sts-dev` +* Evaluated with [EmbeddingSimilarityEvaluator](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.EmbeddingSimilarityEvaluator) + +| Metric | Value | +| :----------------- | :--------- | +| pearson_cosine | 0.8594 | +| spearman_cosine | 0.8573 | +| pearson_manhattan | 0.8217 | +| spearman_manhattan | 0.828 | +| pearson_euclidean | 0.8209 | +| spearman_euclidean | 0.8277 | +| pearson_dot | 0.8188 | +| spearman_dot | 0.8236 | +| **pearson_max** | **0.8594** | +| **spearman_max** | **0.8573** | + + + + + +### Framework Versions +- Python: 3.10.13 +- Sentence Transformers: 3.0.1 +- Transformers: 4.42.4 +- PyTorch: 2.3.0+cu121 +- Accelerate: 0.30.1 +- Datasets: 2.16.1 +- Tokenizers: 0.19.1 + +## Citation + +### BibTeX + + +```bibtex +@article{wang2024multilingual, + title={Multilingual E5 Text Embeddings: A Technical Report}, + author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu}, + journal={arXiv preprint arXiv:2402.05672}, + year={2024} +} +``` + +```bibtex +@inproceedings{reimers-2019-sentence-bert, + title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", + author = "Reimers, Nils and Gurevych, Iryna", + booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", + month = "11", + year = "2019", + publisher = "Association for Computational Linguistics", + url = "https://arxiv.org/abs/1908.10084", +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..f614e8e --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "architectures": [ + "XLMRobertaModel" + ], + "attention_probs_dropout_prob": 0.1, + "bos_token_id": 0, + "classifier_dropout": null, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 514, + "model_type": "xlm-roberta", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "output_past": true, + "pad_token_id": 1, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.42.4", + "type_vocab_size": 1, + "use_cache": true, + "vocab_size": 250002 +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..e95a1b3 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:427570b1987be1bdfacdf9425d41bc941cdbb935a079d4d7f692cbdb3af178e6 +size 1112197096 diff --git a/modules.json b/modules.json new file mode 100644 index 0000000..f7640f9 --- /dev/null +++ b/modules.json @@ -0,0 +1,14 @@ +[ + { + "idx": 0, + "name": "0", + "path": "", + "type": "sentence_transformers.models.Transformer" + }, + { + "idx": 1, + "name": "1", + "path": "1_Pooling", + "type": "sentence_transformers.models.Pooling" + } +] \ No newline at end of file diff --git a/sentence_bert_config.json b/sentence_bert_config.json new file mode 100644 index 0000000..f789d99 --- /dev/null +++ b/sentence_bert_config.json @@ -0,0 +1,4 @@ +{ + "max_seq_length": 512, + "do_lower_case": false +} \ No newline at end of file diff --git a/sentencepiece.bpe.model b/sentencepiece.bpe.model new file mode 100644 index 0000000..7a3f40a --- /dev/null +++ b/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865 +size 5069051 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..b1879d7 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,51 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "cls_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "sep_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..2a51933 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883b037111086fd4dfebbbc9b7cee11e1517b5e0c0514879478661440f137085 +size 17082987 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..31cf25a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,54 @@ +{ + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250001": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "mask_token": "", + "model_max_length": 512, + "pad_token": "", + "sep_token": "", + "tokenizer_class": "XLMRobertaTokenizer", + "unk_token": "" +}