From 27384b4c19f37db08dbac878f53c03a9c1663a3c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 14 May 2026 14:37:42 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: nlpai-lab/KURE-v1 Source: Original Platform --- .gitattributes | 36 +++ 1_Pooling/config.json | 10 + README.md | 212 ++++++++++++++ config.json | 28 ++ config_sentence_transformers.json | 10 + model.safetensors | 3 + modules.json | 20 ++ sentence_bert_config.json | 4 + special_tokens_map.json | 51 ++++ tokenizer.json | 3 + tokenizer_config.json | 56 ++++ trainer_state.json | 453 ++++++++++++++++++++++++++++++ 12 files changed, 886 insertions(+) create mode 100644 .gitattributes create mode 100644 1_Pooling/config.json create mode 100644 README.md create mode 100644 config.json create mode 100644 config_sentence_transformers.json create mode 100644 model.safetensors create mode 100644 modules.json create mode 100644 sentence_bert_config.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/1_Pooling/config.json b/1_Pooling/config.json new file mode 100644 index 0000000..553a16b --- /dev/null +++ b/1_Pooling/config.json @@ -0,0 +1,10 @@ +{ + "word_embedding_dimension": 1024, + "pooling_mode_cls_token": true, + "pooling_mode_mean_tokens": false, + "pooling_mode_max_tokens": false, + "pooling_mode_mean_sqrt_len_tokens": false, + "pooling_mode_weightedmean_tokens": false, + "pooling_mode_lasttoken": false, + "include_prompt": true +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5cf73d --- /dev/null +++ b/README.md @@ -0,0 +1,212 @@ +--- +tags: +- sentence-transformers +- sentence-similarity +- feature-extraction +- generated_from_trainer +- dataset_size:1879136 +- loss:CachedGISTEmbedLoss +license: mit +metrics: +- recall +- precision +- f1 +base_model: +- BAAI/bge-m3 +library_name: sentence-transformers +--- + +# πŸ”Ž KURE-v1 + +Introducing Korea University Retrieval Embedding model, KURE-v1 +It has shown remarkable performance in Korean text retrieval, speficially overwhelming most multilingual embedding models. +To our knowledge, It is one of the best publicly opened Korean retrieval models. + +For details, visit the [KURE repository](https://github.com/nlpai-lab/KURE) + +--- + +## Model Versions +| Model Name | Dimension | Sequence Length | Introduction | +|:----:|:---:|:---:|:---:| +| [KURE-v1](https://huggingface.co/nlpai-lab/KURE-v1) | 1024 | 8192 | Fine-tuned [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3) with Korean data via [CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss) +| [KoE5](https://huggingface.co/nlpai-lab/KoE5) | 1024 | 512 | Fine-tuned [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large) with [ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) via [CachedMultipleNegativesRankingLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedmultiplenegativesrankingloss) | + +## Model Description + +This is the model card of a πŸ€— transformers model that has been pushed on the Hub. + +- **Developed by:** [NLP&AI Lab](http://nlp.korea.ac.kr/) +- **Language(s) (NLP):** Korean, English +- **License:** MIT +- **Finetuned from model:** [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3) + +## Example code +### Install Dependencies +First install the Sentence Transformers library: + +```bash +pip install -U sentence-transformers +``` +### Python code +Then you can load this model and run inference. +```python +from sentence_transformers import SentenceTransformer + +# Download from the πŸ€— Hub +model = SentenceTransformer("nlpai-lab/KURE-v1") + +# Run inference +sentences = [ + 'ν—Œλ²•κ³Ό 법원쑰직법은 μ–΄λ–€ 방식을 톡해 기본ꢌ 보μž₯ λ“±μ˜ λ‹€μ–‘ν•œ 법적 λͺ¨μƒ‰μ„ κ°€λŠ₯ν•˜κ²Œ ν–ˆμ–΄', + '4. μ‹œμ‚¬μ κ³Ό κ°œμ„ λ°©ν–₯ μ•žμ„œ μ‚΄νŽ΄λ³Έ 바와 같이 우리 ν—Œλ²•κ³Ό ｒ법원쑰직 법｣은 λŒ€λ²•μ› ꡬ성을 λ‹€μ–‘ν™”ν•˜μ—¬ 기본ꢌ 보μž₯κ³Ό 민주주의 확립에 μžˆμ–΄ 닀각적인 법적 λͺ¨μƒ‰μ„ κ°€λŠ₯ν•˜κ²Œ ν•˜λŠ” 것을 κ·Όλ³Έ κ·œλ²”μœΌλ‘œ ν•˜κ³  μžˆλ‹€. λ”μš±μ΄ ν•©μ˜μ²΄λ‘œμ„œμ˜ λŒ€λ²•μ› 원리λ₯Ό μ±„νƒν•˜κ³  μžˆλŠ” 것 μ—­μ‹œ κ·Έ κ΅¬μ„±μ˜ 닀양성을 μš”μ²­ν•˜λŠ” κ²ƒμœΌλ‘œ ν•΄μ„λœλ‹€. 이와 같은 κ΄€μ μ—μ„œ λ³Ό λ•Œ ν˜„μ§ 법원μž₯κΈ‰ κ³ μœ„λ²•κ΄€μ„ μ€‘μ‹¬μœΌλ‘œ λŒ€λ²•μ›μ„ κ΅¬μ„±ν•˜λŠ” 관행은 κ°œμ„ ν•  ν•„μš”κ°€ μžˆλŠ” κ²ƒμœΌλ‘œ 보인닀.', + 'μ—°λ°©ν—Œλ²•μž¬νŒμ†ŒλŠ” 2001λ…„ 1μ›” 24일 5:3의 λ‹€μˆ˜κ²¬ν•΄λ‘œ γ€Œλ²•μ›μ‘°μ§λ²•γ€ 제169μ‘° 제2문이 ν—Œλ²•μ— ν•©μΉ˜λœλ‹€λŠ” νŒκ²°μ„ λ‚΄λ ΈμŒ β—‹ 5인의 λ‹€μˆ˜ μž¬νŒκ΄€μ€ μ†Œμ†‘κ΄€κ³„μΈμ˜ 인격ꢌ 보호, κ³΅μ •ν•œ 절차의 보μž₯κ³Ό 방해받지 μ•ŠλŠ” 법과 진싀 발견 등을 근거둜 ν•˜μ—¬ ν…”λ ˆλΉ„μ „ μ΄¬μ˜μ— λŒ€ν•œ μ ˆλŒ€μ μΈ κΈˆμ§€λ₯Ό ν—Œλ²•μ— ν•©μΉ˜ν•˜λŠ” κ²ƒμœΌλ‘œ λ³΄μ•˜μŒ β—‹ κ·ΈλŸ¬λ‚˜ λ‚˜λ¨Έμ§€ 3인의 μž¬νŒκ΄€μ€ ν–‰μ •λ²•μ›μ˜ μ†Œμ†‘μ ˆμ°¨λŠ” νŠΉλ³„ν•œ 인격ꢌ 보호의 이읡도 μ—†μœΌλ©°, ν…”λ ˆλΉ„μ „ 곡개주의둜 인해 법과 진싀 발견의 과정이 μ–Έμ œλ‚˜ μœ„νƒœλ‘­κ²Œ λ˜λŠ” 것은 μ•„λ‹ˆλΌλ©΄μ„œ λ°˜λŒ€μ˜κ²¬μ„ μ œμ‹œν•¨ β—‹ μ™œλƒν•˜λ©΄ ν–‰μ •λ²•μ›μ˜ μ†Œμ†‘μ ˆμ°¨μ—μ„œλŠ” μ†Œμ†‘λ‹Ήμ‚¬μžκ°€ 개인적으둜 직접 심리에 μ°Έμ„ν•˜κΈ°λ³΄λ‹€λŠ” λ³€ν˜Έμ‚¬κ°€ μ°Έμ„ν•˜λŠ” κ²½μš°κ°€ 많으며, μ‹¬λ¦¬λŒ€μƒλ„ μ‚¬μ‹€λ¬Έμ œκ°€ μ•„λ‹Œ 법λ₯ λ¬Έμ œκ°€ λŒ€λΆ€λΆ„μ΄κΈ° λ•Œλ¬Έμ΄λΌλŠ” κ²ƒμž„ β–‘ ν•œνŽΈ, μ—°λ°©ν—Œλ²•μž¬νŒμ†ŒλŠ” γ€Œμ—°λ°©ν—Œλ²•μž¬νŒμ†Œλ²•γ€(Bundesverfassungsgerichtsgesetz: BVerfGG) 제17a쑰에 따라 μ œν•œμ μ΄λ‚˜λ§ˆ μž¬νŒμ— λŒ€ν•œ 방솑을 ν—ˆμš©ν•˜κ³  있음 β—‹ γ€Œμ—°λ°©ν—Œλ²•μž¬νŒμ†Œλ²•γ€ 제17μ‘°μ—μ„œ γ€Œλ²•μ›μ‘°μ§λ²•γ€ 제14절 내지 제16절의 κ·œμ •μ„ μ€€μš©ν•˜λ„λ‘ ν•˜κ³  μžˆμ§€λ§Œ, λ…ΉμŒμ΄λ‚˜ μ΄¬μ˜μ„ ν†΅ν•œ μž¬νŒκ³΅κ°œμ™€ κ΄€λ ¨ν•˜μ—¬μ„œλŠ” γ€Œλ²•μ›μ‘°μ§λ²•γ€κ³Ό λ‹€λ₯Έ λ‚΄μš©μ„ κ·œμ •ν•˜κ³  있음', +] +embeddings = model.encode(sentences) +print(embeddings.shape) +# [3, 1024] + +# Get the similarity scores for the embeddings +similarities = model.similarity(embeddings, embeddings) +print(similarities) +# Results for KURE-v1 +# tensor([[1.0000, 0.6967, 0.5306], +# [0.6967, 1.0000, 0.4427], +# [0.5306, 0.4427, 1.0000]]) +``` + +## Training Details + +### Training Data + +#### KURE-v1 +- Korean query-document-hard_negative(5) data +- 2,000,000 examples + +### Training Procedure +- **loss:** Used **[CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss)** by sentence-transformers +- **batch size:** 4096 +- **learning rate:** 2e-05 +- **epochs:** 1 + +## Evaluation +### Metrics +- Recall, Precision, NDCG, F1 +### Benchmark Datasets +- [Ko-StrategyQA](https://huggingface.co/datasets/taeminlee/Ko-StrategyQA): ν•œκ΅­μ–΄ ODQA multi-hop 검색 데이터셋 (StrategyQA λ²ˆμ—­) +- [AutoRAGRetrieval](https://huggingface.co/datasets/yjoonjang/markers_bm): 금육, 곡곡, 의료, 법λ₯ , 컀머슀 5개 뢄야에 λŒ€ν•΄, pdfλ₯Ό νŒŒμ‹±ν•˜μ—¬ κ΅¬μ„±ν•œ ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 +- [MIRACLRetrieval]([url](https://huggingface.co/datasets/miracl/miracl)): Wikipedia 기반의 ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 +- [PublicHealthQA]([url](https://huggingface.co/datasets/xhluca/publichealth-qa)): 의료 및 곡쀑보건 도메인에 λŒ€ν•œ ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 +- [BelebeleRetrieval]([url](https://huggingface.co/datasets/facebook/belebele)): FLORES-200 기반의 ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 +- [MrTidyRetrieval](https://huggingface.co/datasets/mteb/mrtidy): Wikipedia 기반의 ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 +- [MultiLongDocRetrieval](https://huggingface.co/datasets/Shitao/MLDR): λ‹€μ–‘ν•œ λ„λ©”μΈμ˜ ν•œκ΅­μ–΄ μž₯λ¬Έ 검색 데이터셋 +- [XPQARetrieval](https://huggingface.co/datasets/jinaai/xpqa): λ‹€μ–‘ν•œ λ„λ©”μΈμ˜ ν•œκ΅­μ–΄ λ¬Έμ„œ 검색 데이터셋 + +## Results + +μ•„λž˜λŠ” λͺ¨λ“  λͺ¨λΈμ˜, λͺ¨λ“  벀치마크 데이터셋에 λŒ€ν•œ 평균 κ²°κ³Όμž…λ‹ˆλ‹€. +μžμ„Έν•œ κ²°κ³ΌλŠ” [KURE Github](https://github.com/nlpai-lab/KURE/tree/main/eval/results)μ—μ„œ ν™•μΈν•˜μ‹€ 수 μžˆμŠ΅λ‹ˆλ‹€. +### Top-k 1 +| Model | Average Recall_top1 | Average Precision_top1 | Average NDCG_top1 | Average F1_top1 | +|-----------------------------------------|----------------------|------------------------|-------------------|-----------------| +| **nlpai-lab/KURE-v1** | **0.52640** | **0.60551** | **0.60551** | **0.55784** | +| dragonkue/BGE-m3-ko | 0.52361 | 0.60394 | 0.60394 | 0.55535 | +| BAAI/bge-m3 | 0.51778 | 0.59846 | 0.59846 | 0.54998 | +| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.51246 | 0.59384 | 0.59384 | 0.54489 | +| nlpai-lab/KoE5 | 0.50157 | 0.57790 | 0.57790 | 0.53178 | +| intfloat/multilingual-e5-large | 0.50052 | 0.57727 | 0.57727 | 0.53122 | +| jinaai/jina-embeddings-v3 | 0.48287 | 0.56068 | 0.56068 | 0.51361 | +| BAAI/bge-multilingual-gemma2 | 0.47904 | 0.55472 | 0.55472 | 0.50916 | +| intfloat/multilingual-e5-large-instruct | 0.47842 | 0.55435 | 0.55435 | 0.50826 | +| intfloat/multilingual-e5-base | 0.46950 | 0.54490 | 0.54490 | 0.49947 | +| intfloat/e5-mistral-7b-instruct | 0.46772 | 0.54394 | 0.54394 | 0.49781 | +| Alibaba-NLP/gte-multilingual-base | 0.46469 | 0.53744 | 0.53744 | 0.49353 | +| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.46633 | 0.53625 | 0.53625 | 0.49429 | +| openai/text-embedding-3-large | 0.44884 | 0.51688 | 0.51688 | 0.47572 | +| Salesforce/SFR-Embedding-2_R | 0.43748 | 0.50815 | 0.50815 | 0.46504 | +| upskyy/bge-m3-korean | 0.43125 | 0.50245 | 0.50245 | 0.45945 | +| jhgan/ko-sroberta-multitask | 0.33788 | 0.38497 | 0.38497 | 0.35678 | + +### Top-k 3 +| Model | Average Recall_top1 | Average Precision_top1 | Average NDCG_top1 | Average F1_top1 | +|-----------------------------------------|----------------------|------------------------|-------------------|-----------------| +| **nlpai-lab/KURE-v1** | **0.68678** | **0.28711** | **0.65538** | **0.39835** | +| dragonkue/BGE-m3-ko | 0.67834 | 0.28385 | 0.64950 | 0.39378 | +| BAAI/bge-m3 | 0.67526 | 0.28374 | 0.64556 | 0.39291 | +| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.67128 | 0.28193 | 0.64042 | 0.39072 | +| intfloat/multilingual-e5-large | 0.65807 | 0.27777 | 0.62822 | 0.38423 | +| nlpai-lab/KoE5 | 0.65174 | 0.27329 | 0.62369 | 0.37882 | +| BAAI/bge-multilingual-gemma2 | 0.64415 | 0.27416 | 0.61105 | 0.37782 | +| jinaai/jina-embeddings-v3 | 0.64116 | 0.27165 | 0.60954 | 0.37511 | +| intfloat/multilingual-e5-large-instruct | 0.64353 | 0.27040 | 0.60790 | 0.37453 | +| Alibaba-NLP/gte-multilingual-base | 0.63744 | 0.26404 | 0.59695 | 0.36764 | +| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.63163 | 0.25937 | 0.59237 | 0.36263 | +| intfloat/multilingual-e5-base | 0.62099 | 0.26144 | 0.59179 | 0.36203 | +| intfloat/e5-mistral-7b-instruct | 0.62087 | 0.26144 | 0.58917 | 0.36188 | +| openai/text-embedding-3-large | 0.61035 | 0.25356 | 0.57329 | 0.35270 | +| Salesforce/SFR-Embedding-2_R | 0.60001 | 0.25253 | 0.56346 | 0.34952 | +| upskyy/bge-m3-korean | 0.59215 | 0.25076 | 0.55722 | 0.34623 | +| jhgan/ko-sroberta-multitask | 0.46930 | 0.18994 | 0.43293 | 0.26696 | + +### Top-k 5 +| Model | Average Recall_top1 | Average Precision_top1 | Average NDCG_top1 | Average F1_top1 | +|-----------------------------------------|----------------------|------------------------|-------------------|-----------------| +| **nlpai-lab/KURE-v1** | **0.73851** | **0.19130** | **0.67479** | **0.29903** | +| dragonkue/BGE-m3-ko | 0.72517 | 0.18799 | 0.66692 | 0.29401 | +| BAAI/bge-m3 | 0.72954 | 0.18975 | 0.66615 | 0.29632 | +| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.72962 | 0.18875 | 0.66236 | 0.29542 | +| nlpai-lab/KoE5 | 0.70820 | 0.18287 | 0.64499 | 0.28628 | +| intfloat/multilingual-e5-large | 0.70124 | 0.18316 | 0.64402 | 0.28588 | +| BAAI/bge-multilingual-gemma2 | 0.70258 | 0.18556 | 0.63338 | 0.28851 | +| jinaai/jina-embeddings-v3 | 0.69933 | 0.18256 | 0.63133 | 0.28505 | +| intfloat/multilingual-e5-large-instruct | 0.69018 | 0.17838 | 0.62486 | 0.27933 | +| Alibaba-NLP/gte-multilingual-base | 0.69365 | 0.17789 | 0.61896 | 0.27879 | +| intfloat/multilingual-e5-base | 0.67250 | 0.17406 | 0.61119 | 0.27247 | +| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.67447 | 0.17114 | 0.60952 | 0.26943 | +| intfloat/e5-mistral-7b-instruct | 0.67449 | 0.17484 | 0.60935 | 0.27349 | +| openai/text-embedding-3-large | 0.66365 | 0.17004 | 0.59389 | 0.26677 | +| Salesforce/SFR-Embedding-2_R | 0.65622 | 0.17018 | 0.58494 | 0.26612 | +| upskyy/bge-m3-korean | 0.65477 | 0.17015 | 0.58073 | 0.26589 | +| jhgan/ko-sroberta-multitask | 0.53136 | 0.13264 | 0.45879 | 0.20976 | + +### Top-k 10 +| Model | Average Recall_top1 | Average Precision_top1 | Average NDCG_top1 | Average F1_top1 | +|-----------------------------------------|----------------------|------------------------|-------------------|-----------------| +| **nlpai-lab/KURE-v1** | **0.79682** | **0.10624** | **0.69473** | **0.18524** | +| dragonkue/BGE-m3-ko | 0.78450 | 0.10492 | 0.68748 | 0.18288 | +| BAAI/bge-m3 | 0.79195 | 0.10592 | 0.68723 | 0.18456 | +| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.78669 | 0.10462 | 0.68189 | 0.18260 | +| intfloat/multilingual-e5-large | 0.75902 | 0.10147 | 0.66370 | 0.17693 | +| nlpai-lab/KoE5 | 0.75296 | 0.09937 | 0.66012 | 0.17369 | +| BAAI/bge-multilingual-gemma2 | 0.76153 | 0.10364 | 0.65330 | 0.18003 | +| jinaai/jina-embeddings-v3 | 0.76277 | 0.10240 | 0.65290 | 0.17843 | +| intfloat/multilingual-e5-large-instruct | 0.74851 | 0.09888 | 0.64451 | 0.17283 | +| Alibaba-NLP/gte-multilingual-base | 0.75631 | 0.09938 | 0.64025 | 0.17363 | +| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.74092 | 0.09607 | 0.63258 | 0.16847 | +| intfloat/multilingual-e5-base | 0.73512 | 0.09717 | 0.63216 | 0.16977 | +| intfloat/e5-mistral-7b-instruct | 0.73795 | 0.09777 | 0.63076 | 0.17078 | +| openai/text-embedding-3-large | 0.72946 | 0.09571 | 0.61670 | 0.16739 | +| Salesforce/SFR-Embedding-2_R | 0.71662 | 0.09546 | 0.60589 | 0.16651 | +| upskyy/bge-m3-korean | 0.71895 | 0.09583 | 0.60258 | 0.16712 | +| jhgan/ko-sroberta-multitask | 0.61225 | 0.07826 | 0.48687 | 0.13757 | +
+ +## Citation + +If you find our paper or models helpful, please consider cite as follows: +```text +@misc{KURE, + publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee}, + year = {2024}, + url = {https://github.com/nlpai-lab/KURE} +}, + +@misc{KoE5, + author = {NLP & AI Lab and Human-Inspired AI research}, + title = {KoE5: A New Dataset and Model for Improving Korean Embedding Performance}, + year = {2024}, + publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/nlpai-lab/KoE5}}, +} +``` \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..e0959c1 --- /dev/null +++ b/config.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "/home/work/DATA/embedding/data/results/bge-ft-loss=gist-data=mixed_hn_5_241108-bs=4096-ep=3-lr=2e-5-241212/checkpoint-60", + "architectures": [ + "XLMRobertaModel" + ], + "attention_probs_dropout_prob": 0.1, + "bos_token_id": 0, + "classifier_dropout": null, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 8194, + "model_type": "xlm-roberta", + "num_attention_heads": 16, + "num_hidden_layers": 24, + "output_past": true, + "pad_token_id": 1, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.47.0", + "type_vocab_size": 1, + "use_cache": true, + "vocab_size": 250002 +} diff --git a/config_sentence_transformers.json b/config_sentence_transformers.json new file mode 100644 index 0000000..9eb0a69 --- /dev/null +++ b/config_sentence_transformers.json @@ -0,0 +1,10 @@ +{ + "__version__": { + "sentence_transformers": "3.3.1", + "transformers": "4.47.0", + "pytorch": "2.4.0a0+3bcc3cddb5.nv24.07" + }, + "prompts": {}, + "default_prompt_name": null, + "similarity_fn_name": "cosine" +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..238b364 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c18156e80caf8ff45eb84a24a853130c3bca03087ccb41b051f86e7556bae02c +size 2271064456 diff --git a/modules.json b/modules.json new file mode 100644 index 0000000..952a9b8 --- /dev/null +++ b/modules.json @@ -0,0 +1,20 @@ +[ + { + "idx": 0, + "name": "0", + "path": "", + "type": "sentence_transformers.models.Transformer" + }, + { + "idx": 1, + "name": "1", + "path": "1_Pooling", + "type": "sentence_transformers.models.Pooling" + }, + { + "idx": 2, + "name": "2", + "path": "2_Normalize", + "type": "sentence_transformers.models.Normalize" + } +] \ No newline at end of file diff --git a/sentence_bert_config.json b/sentence_bert_config.json new file mode 100644 index 0000000..0140ba1 --- /dev/null +++ b/sentence_bert_config.json @@ -0,0 +1,4 @@ +{ + "max_seq_length": 8192, + "do_lower_case": false +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..b1879d7 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,51 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "cls_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "sep_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..659bcb2 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb3c3b93c46fd5a8634e262e1b7de7da11a18b527aa2282b312952b692781dfd +size 17083053 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..95bd7c8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,56 @@ +{ + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250001": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "extra_special_tokens": {}, + "mask_token": "", + "model_max_length": 8192, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "tokenizer_class": "XLMRobertaTokenizer", + "unk_token": "" +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..7ea0d66 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,453 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0526315789473684, + "eval_steps": 100, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.017543859649122806, + "grad_norm": 44.84535598754883, + "learning_rate": 1.111111111111111e-06, + "loss": 1.3672, + "step": 1 + }, + { + "epoch": 0.03508771929824561, + "grad_norm": 46.05864715576172, + "learning_rate": 2.222222222222222e-06, + "loss": 1.3719, + "step": 2 + }, + { + "epoch": 0.05263157894736842, + "grad_norm": 9.057656288146973, + "learning_rate": 3.3333333333333333e-06, + "loss": 0.7838, + "step": 3 + }, + { + "epoch": 0.07017543859649122, + "grad_norm": 8.365191459655762, + "learning_rate": 4.444444444444444e-06, + "loss": 0.7781, + "step": 4 + }, + { + "epoch": 0.08771929824561403, + "grad_norm": 6.402659893035889, + "learning_rate": 5.555555555555557e-06, + "loss": 0.7132, + "step": 5 + }, + { + "epoch": 0.10526315789473684, + "grad_norm": 5.508246898651123, + "learning_rate": 6.666666666666667e-06, + "loss": 0.6863, + "step": 6 + }, + { + "epoch": 0.12280701754385964, + "grad_norm": 4.964734077453613, + "learning_rate": 7.77777777777778e-06, + "loss": 0.6237, + "step": 7 + }, + { + "epoch": 0.14035087719298245, + "grad_norm": 4.011112689971924, + "learning_rate": 8.888888888888888e-06, + "loss": 0.618, + "step": 8 + }, + { + "epoch": 0.15789473684210525, + "grad_norm": 3.7118031978607178, + "learning_rate": 1e-05, + "loss": 0.5955, + "step": 9 + }, + { + "epoch": 0.17543859649122806, + "grad_norm": 3.4124467372894287, + "learning_rate": 1.1111111111111113e-05, + "loss": 0.5661, + "step": 10 + }, + { + "epoch": 0.19298245614035087, + "grad_norm": 2.9919450283050537, + "learning_rate": 1.2222222222222224e-05, + "loss": 0.5436, + "step": 11 + }, + { + "epoch": 0.21052631578947367, + "grad_norm": 2.6217074394226074, + "learning_rate": 1.3333333333333333e-05, + "loss": 0.4991, + "step": 12 + }, + { + "epoch": 0.22807017543859648, + "grad_norm": 3.101696014404297, + "learning_rate": 1.4444444444444446e-05, + "loss": 0.4889, + "step": 13 + }, + { + "epoch": 0.24561403508771928, + "grad_norm": 2.632870674133301, + "learning_rate": 1.555555555555556e-05, + "loss": 0.4727, + "step": 14 + }, + { + "epoch": 0.2631578947368421, + "grad_norm": 2.158155679702759, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.4647, + "step": 15 + }, + { + "epoch": 0.2807017543859649, + "grad_norm": 2.006129264831543, + "learning_rate": 1.7777777777777777e-05, + "loss": 0.4476, + "step": 16 + }, + { + "epoch": 0.2982456140350877, + "grad_norm": 2.096951961517334, + "learning_rate": 1.888888888888889e-05, + "loss": 0.4387, + "step": 17 + }, + { + "epoch": 0.3157894736842105, + "grad_norm": 1.9885540008544922, + "learning_rate": 2e-05, + "loss": 0.412, + "step": 18 + }, + { + "epoch": 0.3333333333333333, + "grad_norm": 1.9787037372589111, + "learning_rate": 1.9869281045751635e-05, + "loss": 0.415, + "step": 19 + }, + { + "epoch": 0.3508771929824561, + "grad_norm": 2.428654193878174, + "learning_rate": 1.973856209150327e-05, + "loss": 0.4068, + "step": 20 + }, + { + "epoch": 0.3684210526315789, + "grad_norm": 1.8156601190567017, + "learning_rate": 1.9607843137254903e-05, + "loss": 0.3895, + "step": 21 + }, + { + "epoch": 0.38596491228070173, + "grad_norm": 1.9325244426727295, + "learning_rate": 1.9477124183006536e-05, + "loss": 0.3793, + "step": 22 + }, + { + "epoch": 0.40350877192982454, + "grad_norm": 1.5293307304382324, + "learning_rate": 1.9346405228758173e-05, + "loss": 0.3753, + "step": 23 + }, + { + "epoch": 0.42105263157894735, + "grad_norm": 2.211817741394043, + "learning_rate": 1.9215686274509807e-05, + "loss": 0.3858, + "step": 24 + }, + { + "epoch": 0.43859649122807015, + "grad_norm": 1.7881850004196167, + "learning_rate": 1.908496732026144e-05, + "loss": 0.3735, + "step": 25 + }, + { + "epoch": 0.45614035087719296, + "grad_norm": 1.7260500192642212, + "learning_rate": 1.8954248366013074e-05, + "loss": 0.3733, + "step": 26 + }, + { + "epoch": 0.47368421052631576, + "grad_norm": 1.6085237264633179, + "learning_rate": 1.8823529411764708e-05, + "loss": 0.355, + "step": 27 + }, + { + "epoch": 0.49122807017543857, + "grad_norm": 2.8392181396484375, + "learning_rate": 1.869281045751634e-05, + "loss": 0.3551, + "step": 28 + }, + { + "epoch": 0.5087719298245614, + "grad_norm": 1.978279948234558, + "learning_rate": 1.8562091503267975e-05, + "loss": 0.3337, + "step": 29 + }, + { + "epoch": 0.5263157894736842, + "grad_norm": 2.016209125518799, + "learning_rate": 1.843137254901961e-05, + "loss": 0.3408, + "step": 30 + }, + { + "epoch": 0.543859649122807, + "grad_norm": 1.4631725549697876, + "learning_rate": 1.8300653594771242e-05, + "loss": 0.3434, + "step": 31 + }, + { + "epoch": 0.5614035087719298, + "grad_norm": 1.4324421882629395, + "learning_rate": 1.8169934640522876e-05, + "loss": 0.3468, + "step": 32 + }, + { + "epoch": 0.5789473684210527, + "grad_norm": 5.047348976135254, + "learning_rate": 1.8039215686274513e-05, + "loss": 0.3284, + "step": 33 + }, + { + "epoch": 0.5964912280701754, + "grad_norm": 1.515346884727478, + "learning_rate": 1.7908496732026146e-05, + "loss": 0.3377, + "step": 34 + }, + { + "epoch": 0.6140350877192983, + "grad_norm": 1.883217692375183, + "learning_rate": 1.7777777777777777e-05, + "loss": 0.333, + "step": 35 + }, + { + "epoch": 0.631578947368421, + "grad_norm": 1.6431008577346802, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.3319, + "step": 36 + }, + { + "epoch": 0.6491228070175439, + "grad_norm": 1.630910873413086, + "learning_rate": 1.7516339869281047e-05, + "loss": 0.3214, + "step": 37 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 1.4808597564697266, + "learning_rate": 1.738562091503268e-05, + "loss": 0.3258, + "step": 38 + }, + { + "epoch": 0.6842105263157895, + "grad_norm": 1.6400136947631836, + "learning_rate": 1.7254901960784314e-05, + "loss": 0.3225, + "step": 39 + }, + { + "epoch": 0.7017543859649122, + "grad_norm": 1.3743259906768799, + "learning_rate": 1.7124183006535948e-05, + "loss": 0.3192, + "step": 40 + }, + { + "epoch": 0.7192982456140351, + "grad_norm": 1.4262139797210693, + "learning_rate": 1.6993464052287582e-05, + "loss": 0.3121, + "step": 41 + }, + { + "epoch": 0.7368421052631579, + "grad_norm": 1.4537426233291626, + "learning_rate": 1.686274509803922e-05, + "loss": 0.3164, + "step": 42 + }, + { + "epoch": 0.7543859649122807, + "grad_norm": 1.2737575769424438, + "learning_rate": 1.6732026143790852e-05, + "loss": 0.3021, + "step": 43 + }, + { + "epoch": 0.7719298245614035, + "grad_norm": 1.2681057453155518, + "learning_rate": 1.6601307189542486e-05, + "loss": 0.3166, + "step": 44 + }, + { + "epoch": 0.7894736842105263, + "grad_norm": 5.779806137084961, + "learning_rate": 1.647058823529412e-05, + "loss": 0.3093, + "step": 45 + }, + { + "epoch": 0.8070175438596491, + "grad_norm": 1.4256994724273682, + "learning_rate": 1.6339869281045753e-05, + "loss": 0.2968, + "step": 46 + }, + { + "epoch": 0.8245614035087719, + "grad_norm": 1.5250773429870605, + "learning_rate": 1.6209150326797387e-05, + "loss": 0.2972, + "step": 47 + }, + { + "epoch": 0.8421052631578947, + "grad_norm": 1.2680127620697021, + "learning_rate": 1.607843137254902e-05, + "loss": 0.2914, + "step": 48 + }, + { + "epoch": 0.8596491228070176, + "grad_norm": 1.6887532472610474, + "learning_rate": 1.5947712418300657e-05, + "loss": 0.2951, + "step": 49 + }, + { + "epoch": 0.8771929824561403, + "grad_norm": 2.2416443824768066, + "learning_rate": 1.5816993464052288e-05, + "loss": 0.3059, + "step": 50 + }, + { + "epoch": 0.8947368421052632, + "grad_norm": 1.3144830465316772, + "learning_rate": 1.568627450980392e-05, + "loss": 0.3011, + "step": 51 + }, + { + "epoch": 0.9122807017543859, + "grad_norm": 2.3271305561065674, + "learning_rate": 1.555555555555556e-05, + "loss": 0.2908, + "step": 52 + }, + { + "epoch": 0.9298245614035088, + "grad_norm": 1.2403310537338257, + "learning_rate": 1.5424836601307192e-05, + "loss": 0.3001, + "step": 53 + }, + { + "epoch": 0.9473684210526315, + "grad_norm": 2.7490832805633545, + "learning_rate": 1.5294117647058822e-05, + "loss": 0.2987, + "step": 54 + }, + { + "epoch": 0.9649122807017544, + "grad_norm": 2.321964979171753, + "learning_rate": 1.5163398692810458e-05, + "loss": 0.287, + "step": 55 + }, + { + "epoch": 0.9824561403508771, + "grad_norm": 1.1607296466827393, + "learning_rate": 1.5032679738562093e-05, + "loss": 0.2868, + "step": 56 + }, + { + "epoch": 1.0, + "grad_norm": 1.5422321557998657, + "learning_rate": 1.4901960784313726e-05, + "loss": 0.293, + "step": 57 + }, + { + "epoch": 1.0175438596491229, + "grad_norm": 1.5632988214492798, + "learning_rate": 1.4771241830065362e-05, + "loss": 0.2768, + "step": 58 + }, + { + "epoch": 1.0350877192982457, + "grad_norm": 1.620436668395996, + "learning_rate": 1.4640522875816994e-05, + "loss": 0.2727, + "step": 59 + }, + { + "epoch": 1.0526315789473684, + "grad_norm": 1.2848402261734009, + "learning_rate": 1.4509803921568629e-05, + "loss": 0.2659, + "step": 60 + } + ], + "logging_steps": 1, + "max_steps": 171, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 12, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4096, + "trial_name": null, + "trial_params": null +}