初始化项目，由ModelHub XC社区提供模型
Model: NastasiaM/mbert-loraxs-qa-vanilla Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
73
README.md
Normal file
73
README.md
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
---
|
||||||
|
library_name: peft
|
||||||
|
license: apache-2.0
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
metrics:
|
||||||
|
- f1
|
||||||
|
model-index:
|
||||||
|
- name: mbert-loraxs-qa-vanilla
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
# mbert-loraxs-qa-vanilla
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) on the None dataset.
|
||||||
|
It achieves the following results on the evaluation set:
|
||||||
|
- Loss: 1.4437
|
||||||
|
- Exact Match: 55.68
|
||||||
|
- F1: 70.3954
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 5e-05
|
||||||
|
- train_batch_size: 16
|
||||||
|
- eval_batch_size: 16
|
||||||
|
- seed: 42
|
||||||
|
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: linear
|
||||||
|
- lr_scheduler_warmup_steps: 0.1
|
||||||
|
- num_epochs: 7
|
||||||
|
- mixed_precision_training: Native AMP
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
| Training Loss | Epoch | Step | Validation Loss | Exact Match | F1 |
|
||||||
|
|:-------------:|:-----:|:----:|:---------------:|:-----------:|:-------:|
|
||||||
|
| 1.6012 | 1.0 | 634 | 1.6034 | 52.8 | 66.3884 |
|
||||||
|
| 1.5347 | 2.0 | 1268 | 1.5703 | 54.16 | 67.8105 |
|
||||||
|
| 1.5039 | 3.0 | 1902 | 1.4684 | 55.12 | 69.1541 |
|
||||||
|
| 1.4706 | 4.0 | 2536 | 1.4466 | 55.68 | 69.7596 |
|
||||||
|
| 1.4197 | 5.0 | 3170 | 1.4419 | 56.48 | 70.7228 |
|
||||||
|
| 1.3971 | 6.0 | 3804 | 1.4516 | 55.76 | 70.6580 |
|
||||||
|
| 1.3627 | 7.0 | 4438 | 1.4437 | 55.68 | 70.3954 |
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
|
- Transformers 5.9.0
|
||||||
|
- Pytorch 2.11.0+cu128
|
||||||
|
- Datasets 4.8.5
|
||||||
|
- Tokenizers 0.22.2
|
||||||
47
adapter_config.json
Normal file
47
adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 64,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.135671188271432,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 256,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"dense",
|
||||||
|
"query",
|
||||||
|
"key",
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
adapter_model.safetensors
Normal file
3
adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:2e8ea6787def70456f4bff8a0ae0c8ef98c54d7c6cedff3d96964abe18a6a2a4
|
||||||
|
size 188782560
|
||||||
34
config.json
Normal file
34
config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.10.1",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:d4748e0a5d876c324485c8f218751af8aa591ba0f2f5067ff33528914fd03201
|
||||||
|
size 709080872
|
||||||
206
run-0/checkpoint-1268/README.md
Normal file
206
run-0/checkpoint-1268/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-0/checkpoint-1268/adapter_config.json
Normal file
47
run-0/checkpoint-1268/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.04085479876923811,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"dense",
|
||||||
|
"query",
|
||||||
|
"key",
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1268/adapter_model.safetensors
Normal file
3
run-0/checkpoint-1268/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e702257f57933f9c16f0f7315b68c45c100d66d1e6670d6752e88de4f2f90cac
|
||||||
|
size 43685272
|
||||||
34
run-0/checkpoint-1268/config.json
Normal file
34
run-0/checkpoint-1268/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1268/model.safetensors
Normal file
3
run-0/checkpoint-1268/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:2df3d472178957d6a69db78704ab19cf274e56c3175d8b5178248a24434ef9b5
|
||||||
|
size 709080872
|
||||||
3
run-0/checkpoint-1268/optimizer.pt
Normal file
3
run-0/checkpoint-1268/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:3f3f7c95c4e158f39cbde5b0109e3f2cee637b6d9ff2720dd00d572fb1bd1c9b
|
||||||
|
size 2435275
|
||||||
3
run-0/checkpoint-1268/rng_state.pth
Normal file
3
run-0/checkpoint-1268/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:cf855e14e8c7614490a309de0f409dda5ee07cc634e1922029a999b8a22ada70
|
||||||
|
size 14645
|
||||||
3
run-0/checkpoint-1268/scaler.pt
Normal file
3
run-0/checkpoint-1268/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1db6b977a9acacb31601848dc2fd1a5a8c43befa31919f90dbea815f8da3b1b8
|
||||||
|
size 1383
|
||||||
3
run-0/checkpoint-1268/scheduler.pt
Normal file
3
run-0/checkpoint-1268/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:860e4e93730f65b25505c4f211f5833b47dfcf2e30adaf37e35ab621aafc52a2
|
||||||
|
size 1465
|
||||||
119711
run-0/checkpoint-1268/tokenizer.json
Normal file
119711
run-0/checkpoint-1268/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-0/checkpoint-1268/tokenizer_config.json
Normal file
15
run-0/checkpoint-1268/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
75
run-0/checkpoint-1268/trainer_state.json
Normal file
75
run-0/checkpoint-1268/trainer_state.json
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 1268,
|
||||||
|
"best_metric": 10.893501835802274,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-0/checkpoint-1268",
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 1268,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 9.316121101379395,
|
||||||
|
"learning_rate": 1.2339740823845353e-05,
|
||||||
|
"loss": 5.004710296727129,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 3.52,
|
||||||
|
"eval_f1": 8.929509943172043,
|
||||||
|
"eval_loss": 4.074629306793213,
|
||||||
|
"eval_runtime": 7.6506,
|
||||||
|
"eval_samples_per_second": 165.347,
|
||||||
|
"eval_steps_per_second": 10.457,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 8.535039901733398,
|
||||||
|
"learning_rate": 1.943266271471709e-08,
|
||||||
|
"loss": 4.068548064126578,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 5.04,
|
||||||
|
"eval_f1": 10.893501835802274,
|
||||||
|
"eval_loss": 3.852447986602783,
|
||||||
|
"eval_runtime": 7.6373,
|
||||||
|
"eval_samples_per_second": 165.635,
|
||||||
|
"eval_steps_per_second": 10.475,
|
||||||
|
"step": 1268
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 1268,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 4485574781614080.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 2.21726681574922e-05,
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_dropout": 0.04085479876923811,
|
||||||
|
"lora_r": 64,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"weight_decay": 0.013911443680370162
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1268/training_args.bin
Normal file
3
run-0/checkpoint-1268/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0d2cd7004ce4d7feb854e04ba94ad51da8bd0b8047516beb32fba857d2c323e5
|
||||||
|
size 5201
|
||||||
206
run-0/checkpoint-1902/README.md
Normal file
206
run-0/checkpoint-1902/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-0/checkpoint-1902/adapter_config.json
Normal file
47
run-0/checkpoint-1902/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 64,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.06066093898873501,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 16,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"value",
|
||||||
|
"dense",
|
||||||
|
"key",
|
||||||
|
"query"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1902/adapter_model.safetensors
Normal file
3
run-0/checkpoint-1902/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:6651554c2f4f6a7ec488c83974507c4f6a0c64c319cf86e0e911f00e6457238d
|
||||||
|
size 10728520
|
||||||
34
run-0/checkpoint-1902/config.json
Normal file
34
run-0/checkpoint-1902/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1902/model.safetensors
Normal file
3
run-0/checkpoint-1902/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f6e0836f3b856153517155441ad1427e359fd4dc0a5b78dd28937be451ec92f2
|
||||||
|
size 709080872
|
||||||
3
run-0/checkpoint-1902/optimizer.pt
Normal file
3
run-0/checkpoint-1902/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:6aadb99192f53b05f4e01818c182831f337e07029e29f85b57c5d14e07b5c189
|
||||||
|
size 223435
|
||||||
3
run-0/checkpoint-1902/rng_state.pth
Normal file
3
run-0/checkpoint-1902/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a58e3cebe2dbf4f2e088bcaccde49b79f8be87dff512f8d14432a5fd0ee0d43f
|
||||||
|
size 14645
|
||||||
3
run-0/checkpoint-1902/scaler.pt
Normal file
3
run-0/checkpoint-1902/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:633c9cc03d72c9e1ecc7e1015bb035c25e1c53f5c84c4ec9ad6a85b50bb7d7a8
|
||||||
|
size 1383
|
||||||
3
run-0/checkpoint-1902/scheduler.pt
Normal file
3
run-0/checkpoint-1902/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:635277ffa8553dc5e05d5f3c8e8d9ae3b65bd754c9263f739c5e6983e0d70d9d
|
||||||
|
size 1465
|
||||||
119711
run-0/checkpoint-1902/tokenizer.json
Normal file
119711
run-0/checkpoint-1902/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-0/checkpoint-1902/tokenizer_config.json
Normal file
15
run-0/checkpoint-1902/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
92
run-0/checkpoint-1902/trainer_state.json
Normal file
92
run-0/checkpoint-1902/trainer_state.json
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 1902,
|
||||||
|
"best_metric": 10.835813703423298,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-0/checkpoint-1902",
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 1902,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 5.70946741104126,
|
||||||
|
"learning_rate": 2.0882182104159667e-05,
|
||||||
|
"loss": 5.564044170199132,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 1.2,
|
||||||
|
"eval_f1": 6.921369607966623,
|
||||||
|
"eval_loss": 4.845778942108154,
|
||||||
|
"eval_runtime": 7.7235,
|
||||||
|
"eval_samples_per_second": 163.786,
|
||||||
|
"eval_steps_per_second": 10.358,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 7.910016059875488,
|
||||||
|
"learning_rate": 1.3925112501407575e-05,
|
||||||
|
"loss": 4.648683566024251,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 3.44,
|
||||||
|
"eval_f1": 9.779211605145168,
|
||||||
|
"eval_loss": 4.270840644836426,
|
||||||
|
"eval_runtime": 7.7608,
|
||||||
|
"eval_samples_per_second": 162.999,
|
||||||
|
"eval_steps_per_second": 10.308,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 9.720287322998047,
|
||||||
|
"learning_rate": 6.968042898655485e-06,
|
||||||
|
"loss": 4.320517362480284,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_exact_match": 4.32,
|
||||||
|
"eval_f1": 10.835813703423298,
|
||||||
|
"eval_loss": 4.045061111450195,
|
||||||
|
"eval_runtime": 7.7213,
|
||||||
|
"eval_samples_per_second": 163.833,
|
||||||
|
"eval_steps_per_second": 10.361,
|
||||||
|
"step": 1902
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 2536,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 6150731641721856.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 2.504106125154617e-05,
|
||||||
|
"lora_alpha": 64,
|
||||||
|
"lora_dropout": 0.06066093898873501,
|
||||||
|
"lora_r": 16,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"weight_decay": 0.025681530243658136
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-0/checkpoint-1902/training_args.bin
Normal file
3
run-0/checkpoint-1902/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7e498a84e77c259849502a49f736b46ed5910143d2b1d6174534ea391b1e2825
|
||||||
|
size 5201
|
||||||
206
run-0/checkpoint-2536/README.md
Normal file
206
run-0/checkpoint-2536/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-0/checkpoint-2536/adapter_config.json
Normal file
47
run-0/checkpoint-2536/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 64,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.06066093898873501,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 16,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"value",
|
||||||
|
"dense",
|
||||||
|
"key",
|
||||||
|
"query"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-0/checkpoint-2536/adapter_model.safetensors
Normal file
3
run-0/checkpoint-2536/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1f1e4e7df7d0f52305981016879170f02dfd29f7123e89c5ad041742e6a77a15
|
||||||
|
size 10728520
|
||||||
34
run-0/checkpoint-2536/config.json
Normal file
34
run-0/checkpoint-2536/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-0/checkpoint-2536/model.safetensors
Normal file
3
run-0/checkpoint-2536/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9739bd197a5f7b940cb120bf23f4437f6b15453e0f6254ae8d306a154ff15ef2
|
||||||
|
size 709080872
|
||||||
3
run-0/checkpoint-2536/optimizer.pt
Normal file
3
run-0/checkpoint-2536/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:65f3c38503bba35514b4bbf2704f36717c7e08814b189d34633a8601427e3dd3
|
||||||
|
size 223435
|
||||||
3
run-0/checkpoint-2536/rng_state.pth
Normal file
3
run-0/checkpoint-2536/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:315410121b0350e9942a62383d8399b758ef4637fc338551dcfa1b61c3bc6bd3
|
||||||
|
size 14645
|
||||||
3
run-0/checkpoint-2536/scaler.pt
Normal file
3
run-0/checkpoint-2536/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:06150686f19a02b3aa3e19d82940c23990e4ab3674208bbc4892318c97a5a0e7
|
||||||
|
size 1383
|
||||||
3
run-0/checkpoint-2536/scheduler.pt
Normal file
3
run-0/checkpoint-2536/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:736f6d77f69b2f56acd9ec0742aedee0cc63d73f2234975b270fb6903b90ed69
|
||||||
|
size 1465
|
||||||
119711
run-0/checkpoint-2536/tokenizer.json
Normal file
119711
run-0/checkpoint-2536/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-0/checkpoint-2536/tokenizer_config.json
Normal file
15
run-0/checkpoint-2536/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
109
run-0/checkpoint-2536/trainer_state.json
Normal file
109
run-0/checkpoint-2536/trainer_state.json
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 2536,
|
||||||
|
"best_metric": 11.5259012819182,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-0/checkpoint-2536",
|
||||||
|
"epoch": 4.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 2536,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 5.70946741104126,
|
||||||
|
"learning_rate": 2.0882182104159667e-05,
|
||||||
|
"loss": 5.564044170199132,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 1.2,
|
||||||
|
"eval_f1": 6.921369607966623,
|
||||||
|
"eval_loss": 4.845778942108154,
|
||||||
|
"eval_runtime": 7.7235,
|
||||||
|
"eval_samples_per_second": 163.786,
|
||||||
|
"eval_steps_per_second": 10.358,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 7.910016059875488,
|
||||||
|
"learning_rate": 1.3925112501407575e-05,
|
||||||
|
"loss": 4.648683566024251,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 3.44,
|
||||||
|
"eval_f1": 9.779211605145168,
|
||||||
|
"eval_loss": 4.270840644836426,
|
||||||
|
"eval_runtime": 7.7608,
|
||||||
|
"eval_samples_per_second": 162.999,
|
||||||
|
"eval_steps_per_second": 10.308,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 9.720287322998047,
|
||||||
|
"learning_rate": 6.968042898655485e-06,
|
||||||
|
"loss": 4.320517362480284,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_exact_match": 4.32,
|
||||||
|
"eval_f1": 10.835813703423298,
|
||||||
|
"eval_loss": 4.045061111450195,
|
||||||
|
"eval_runtime": 7.7213,
|
||||||
|
"eval_samples_per_second": 163.833,
|
||||||
|
"eval_steps_per_second": 10.361,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"grad_norm": 12.9658784866333,
|
||||||
|
"learning_rate": 1.0973295903394465e-08,
|
||||||
|
"loss": 4.1913481029426265,
|
||||||
|
"step": 2536
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"eval_exact_match": 4.56,
|
||||||
|
"eval_f1": 11.5259012819182,
|
||||||
|
"eval_loss": 3.976346254348755,
|
||||||
|
"eval_runtime": 7.6892,
|
||||||
|
"eval_samples_per_second": 164.516,
|
||||||
|
"eval_steps_per_second": 10.404,
|
||||||
|
"step": 2536
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 2536,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 8200975522295808.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 2.504106125154617e-05,
|
||||||
|
"lora_alpha": 64,
|
||||||
|
"lora_dropout": 0.06066093898873501,
|
||||||
|
"lora_r": 16,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"weight_decay": 0.025681530243658136
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-0/checkpoint-2536/training_args.bin
Normal file
3
run-0/checkpoint-2536/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7e498a84e77c259849502a49f736b46ed5910143d2b1d6174534ea391b1e2825
|
||||||
|
size 5201
|
||||||
206
run-0/checkpoint-634/README.md
Normal file
206
run-0/checkpoint-634/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-0/checkpoint-634/adapter_config.json
Normal file
47
run-0/checkpoint-634/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.04085479876923811,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"dense",
|
||||||
|
"query",
|
||||||
|
"key",
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-0/checkpoint-634/adapter_model.safetensors
Normal file
3
run-0/checkpoint-634/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:ec46567714d71e623a5b242c9d575139ff7bbec26cb14f71587593a66b6fd833
|
||||||
|
size 43685272
|
||||||
34
run-0/checkpoint-634/config.json
Normal file
34
run-0/checkpoint-634/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-0/checkpoint-634/model.safetensors
Normal file
3
run-0/checkpoint-634/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:11d40ea4fbf5f106349949937361db7c85b9f6be675146e063e980c5a10785f0
|
||||||
|
size 709080872
|
||||||
3
run-0/checkpoint-634/optimizer.pt
Normal file
3
run-0/checkpoint-634/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:b59750fba95bd5660ea562d81c33117dfa673642905fe2abdb4e1a14309cc1d7
|
||||||
|
size 2435275
|
||||||
3
run-0/checkpoint-634/rng_state.pth
Normal file
3
run-0/checkpoint-634/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:b87afd0e1a593683d5fa1f874b6cd7ea9c65d54d813a18e4be28585f81ff6067
|
||||||
|
size 14645
|
||||||
3
run-0/checkpoint-634/scaler.pt
Normal file
3
run-0/checkpoint-634/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:16a2e1f56e240db7e52a4fcf92ce326afa3e6e09a813483300819476611278fd
|
||||||
|
size 1383
|
||||||
3
run-0/checkpoint-634/scheduler.pt
Normal file
3
run-0/checkpoint-634/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:746a90e595647cd9ade2065268507a307ac5a8f8430a1bfae00bd9cf69c6da07
|
||||||
|
size 1465
|
||||||
119711
run-0/checkpoint-634/tokenizer.json
Normal file
119711
run-0/checkpoint-634/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-0/checkpoint-634/tokenizer_config.json
Normal file
15
run-0/checkpoint-634/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
58
run-0/checkpoint-634/trainer_state.json
Normal file
58
run-0/checkpoint-634/trainer_state.json
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 634,
|
||||||
|
"best_metric": 8.929509943172043,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-0/checkpoint-634",
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 634,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 9.316121101379395,
|
||||||
|
"learning_rate": 1.2339740823845353e-05,
|
||||||
|
"loss": 5.004710296727129,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 3.52,
|
||||||
|
"eval_f1": 8.929509943172043,
|
||||||
|
"eval_loss": 4.074629306793213,
|
||||||
|
"eval_runtime": 7.6506,
|
||||||
|
"eval_samples_per_second": 165.347,
|
||||||
|
"eval_steps_per_second": 10.457,
|
||||||
|
"step": 634
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 1268,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 2242787390807040.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 2.21726681574922e-05,
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_dropout": 0.04085479876923811,
|
||||||
|
"lora_r": 64,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"weight_decay": 0.013911443680370162
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-0/checkpoint-634/training_args.bin
Normal file
3
run-0/checkpoint-634/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0d2cd7004ce4d7feb854e04ba94ad51da8bd0b8047516beb32fba857d2c323e5
|
||||||
|
size 5201
|
||||||
206
run-1/checkpoint-1268/README.md
Normal file
206
run-1/checkpoint-1268/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-1/checkpoint-1268/adapter_config.json
Normal file
47
run-1/checkpoint-1268/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 512,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.0734622590514945,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 128,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"dense",
|
||||||
|
"query",
|
||||||
|
"key",
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1268/adapter_model.safetensors
Normal file
3
run-1/checkpoint-1268/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:956491326a09d4a16b3bf6e9170e4eeea0876af2e968615c44c4378305d7bea3
|
||||||
|
size 89691896
|
||||||
34
run-1/checkpoint-1268/config.json
Normal file
34
run-1/checkpoint-1268/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1268/model.safetensors
Normal file
3
run-1/checkpoint-1268/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:87d03aa3cf5e9333e9d871373ff9e45d4da1cf6555c56a6946cd44378a236e66
|
||||||
|
size 709080872
|
||||||
3
run-1/checkpoint-1268/optimizer.pt
Normal file
3
run-1/checkpoint-1268/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:3be7d4beeb5ce00fe93ad8dfd14b8877abe7e3f8aa7d39765ea42eba05f2d662
|
||||||
|
size 9513163
|
||||||
3
run-1/checkpoint-1268/rng_state.pth
Normal file
3
run-1/checkpoint-1268/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:3971a189c708a01d02be1e50bcfa0a26c8e9be5a4eed52d3bfc52e525c73fb1d
|
||||||
|
size 14645
|
||||||
3
run-1/checkpoint-1268/scaler.pt
Normal file
3
run-1/checkpoint-1268/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:b812bb2bf3396dd797f3f7f7d874cbcef53472866bff7f3c813b780284de2634
|
||||||
|
size 1383
|
||||||
3
run-1/checkpoint-1268/scheduler.pt
Normal file
3
run-1/checkpoint-1268/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f3c50821a2371ca18f3aa878af95c6118e2e3444a51792f1701bd8e16c943107
|
||||||
|
size 1465
|
||||||
119711
run-1/checkpoint-1268/tokenizer.json
Normal file
119711
run-1/checkpoint-1268/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-1/checkpoint-1268/tokenizer_config.json
Normal file
15
run-1/checkpoint-1268/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
75
run-1/checkpoint-1268/trainer_state.json
Normal file
75
run-1/checkpoint-1268/trainer_state.json
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 1268,
|
||||||
|
"best_metric": 66.75662514290244,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-1/checkpoint-1268",
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 1268,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 52.79657745361328,
|
||||||
|
"learning_rate": 1.949699397447388e-05,
|
||||||
|
"loss": 3.1322443146810923,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 48.16,
|
||||||
|
"eval_f1": 62.48638687085724,
|
||||||
|
"eval_loss": 1.7752151489257812,
|
||||||
|
"eval_runtime": 7.6669,
|
||||||
|
"eval_samples_per_second": 164.996,
|
||||||
|
"eval_steps_per_second": 10.435,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 47.9760856628418,
|
||||||
|
"learning_rate": 3.0703927518856505e-08,
|
||||||
|
"loss": 1.7527718047614353,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 52.72,
|
||||||
|
"eval_f1": 66.75662514290244,
|
||||||
|
"eval_loss": 1.5720099210739136,
|
||||||
|
"eval_runtime": 7.7172,
|
||||||
|
"eval_samples_per_second": 163.919,
|
||||||
|
"eval_steps_per_second": 10.366,
|
||||||
|
"step": 1268
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 1268,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 5023145924278272.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 3.503318129901527e-05,
|
||||||
|
"lora_alpha": 512,
|
||||||
|
"lora_dropout": 0.0734622590514945,
|
||||||
|
"lora_r": 128,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"weight_decay": 0.004905620950863932
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1268/training_args.bin
Normal file
3
run-1/checkpoint-1268/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:eb7bc00d14065aadc52e5d03ff8d04227d538099499023a19385fc692ca9960b
|
||||||
|
size 5201
|
||||||
206
run-1/checkpoint-1902/README.md
Normal file
206
run-1/checkpoint-1902/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-1/checkpoint-1902/adapter_config.json
Normal file
47
run-1/checkpoint-1902/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.10300849900123686,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 8,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"value",
|
||||||
|
"dense",
|
||||||
|
"key",
|
||||||
|
"query"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1902/adapter_model.safetensors
Normal file
3
run-1/checkpoint-1902/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:71a2547e02a3f49c56b042a2a2872a6957e5cbf8e9cb1526c1429c166318cf52
|
||||||
|
size 5364440
|
||||||
34
run-1/checkpoint-1902/config.json
Normal file
34
run-1/checkpoint-1902/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1902/model.safetensors
Normal file
3
run-1/checkpoint-1902/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f5ec840d5dbba57ae91c4b02692d42ce4f4948b8f26924abf25cc41e69d5c1ed
|
||||||
|
size 709080872
|
||||||
3
run-1/checkpoint-1902/optimizer.pt
Normal file
3
run-1/checkpoint-1902/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:c22823ffd55e087ce8fdfee4d2a0b59bc835da4ad163968c7f15515cd0769894
|
||||||
|
size 112715
|
||||||
3
run-1/checkpoint-1902/rng_state.pth
Normal file
3
run-1/checkpoint-1902/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:6e11dd5aac31555569da8b937fed809c3d18396a53b00643cca7f557d2e8ebba
|
||||||
|
size 14645
|
||||||
3
run-1/checkpoint-1902/scaler.pt
Normal file
3
run-1/checkpoint-1902/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:d26ec889430e7e9c16432d09ed907050ced92aabbb75333745bcde2b4024ea93
|
||||||
|
size 1383
|
||||||
3
run-1/checkpoint-1902/scheduler.pt
Normal file
3
run-1/checkpoint-1902/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:754fbf91bf2ce5c5c6953a12464ef54a607130ff3295f72e35aa3e8722bf691a
|
||||||
|
size 1465
|
||||||
119711
run-1/checkpoint-1902/tokenizer.json
Normal file
119711
run-1/checkpoint-1902/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-1/checkpoint-1902/tokenizer_config.json
Normal file
15
run-1/checkpoint-1902/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
92
run-1/checkpoint-1902/trainer_state.json
Normal file
92
run-1/checkpoint-1902/trainer_state.json
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 1902,
|
||||||
|
"best_metric": 42.797774942292016,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-1/checkpoint-1902",
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 1902,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 50.64515686035156,
|
||||||
|
"learning_rate": 0.00010486219815639711,
|
||||||
|
"loss": 4.513181668350749,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 17.76,
|
||||||
|
"eval_f1": 28.945909302213973,
|
||||||
|
"eval_loss": 3.11417818069458,
|
||||||
|
"eval_runtime": 7.6977,
|
||||||
|
"eval_samples_per_second": 164.336,
|
||||||
|
"eval_steps_per_second": 10.393,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 69.94309997558594,
|
||||||
|
"learning_rate": 5.247241594114434e-05,
|
||||||
|
"loss": 3.1071321024127565,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 29.52,
|
||||||
|
"eval_f1": 40.87090707056966,
|
||||||
|
"eval_loss": 2.5117993354797363,
|
||||||
|
"eval_runtime": 7.7422,
|
||||||
|
"eval_samples_per_second": 163.389,
|
||||||
|
"eval_steps_per_second": 10.333,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 85.58645629882812,
|
||||||
|
"learning_rate": 8.263372589156588e-08,
|
||||||
|
"loss": 2.8506941449378944,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_exact_match": 30.88,
|
||||||
|
"eval_f1": 42.797774942292016,
|
||||||
|
"eval_loss": 2.423711061477661,
|
||||||
|
"eval_runtime": 7.6134,
|
||||||
|
"eval_samples_per_second": 166.154,
|
||||||
|
"eval_steps_per_second": 10.508,
|
||||||
|
"step": 1902
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 1902,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 3,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 6056721303671808.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 0.00014138630500046923,
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_dropout": 0.10300849900123686,
|
||||||
|
"lora_r": 8,
|
||||||
|
"num_train_epochs": 3,
|
||||||
|
"weight_decay": 0.04651389479910262
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-1/checkpoint-1902/training_args.bin
Normal file
3
run-1/checkpoint-1902/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:ad44e164e8a1640df833f0ccdc5c6426f7ca4a746d2737414edc4b9c8dd8a9f9
|
||||||
|
size 5201
|
||||||
34
run-1/checkpoint-2536/config.json
Normal file
34
run-1/checkpoint-2536/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-1/checkpoint-2536/model.safetensors
Normal file
3
run-1/checkpoint-2536/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:06989b83bf9d77da5b12e21fc035ca324b4349657fd8cfb36a435f071f870069
|
||||||
|
size 709080872
|
||||||
3
run-1/checkpoint-2536/optimizer.pt
Normal file
3
run-1/checkpoint-2536/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:64870094ba24066a73902f3c2511e84f95948a6eb7c244962cfd46e5f98190f0
|
||||||
|
size 1418284811
|
||||||
3
run-1/checkpoint-2536/rng_state.pth
Normal file
3
run-1/checkpoint-2536/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1e85fa698aa4806efe3d6074038fa07bcf6133d8cc71f17a6a4dca0b61c4fc4f
|
||||||
|
size 14645
|
||||||
3
run-1/checkpoint-2536/scaler.pt
Normal file
3
run-1/checkpoint-2536/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:6a1c6b4addfbdbc469ec0107c5c42d1b0e47cf34c2de1d244fae93607014b55c
|
||||||
|
size 1383
|
||||||
3
run-1/checkpoint-2536/scheduler.pt
Normal file
3
run-1/checkpoint-2536/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7de304905c0f46d926a785e9a31fef27096dc0bb2c49504102ebeabc1bb2f7ea
|
||||||
|
size 1465
|
||||||
119711
run-1/checkpoint-2536/tokenizer.json
Normal file
119711
run-1/checkpoint-2536/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
15
run-1/checkpoint-2536/tokenizer_config.json
Normal file
15
run-1/checkpoint-2536/tokenizer_config.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_lower_case": false,
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
106
run-1/checkpoint-2536/trainer_state.json
Normal file
106
run-1/checkpoint-2536/trainer_state.json
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": 2536,
|
||||||
|
"best_metric": 76.57829761930071,
|
||||||
|
"best_model_checkpoint": "mbert-loraxs-qa-vanilla/run-1/checkpoint-2536",
|
||||||
|
"epoch": 4.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 2536,
|
||||||
|
"is_hyper_param_search": true,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 15.868330955505371,
|
||||||
|
"learning_rate": 5.191675659155908e-05,
|
||||||
|
"loss": 2.1192049333349763,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_exact_match": 60.08,
|
||||||
|
"eval_f1": 74.33783203646502,
|
||||||
|
"eval_loss": 1.2725635766983032,
|
||||||
|
"eval_runtime": 5.8013,
|
||||||
|
"eval_samples_per_second": 218.055,
|
||||||
|
"eval_steps_per_second": 13.79,
|
||||||
|
"step": 634
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 30.633678436279297,
|
||||||
|
"learning_rate": 3.46202649052488e-05,
|
||||||
|
"loss": 0.9956009440617607,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"eval_exact_match": 62.48,
|
||||||
|
"eval_f1": 76.03924855668853,
|
||||||
|
"eval_loss": 1.3048548698425293,
|
||||||
|
"eval_runtime": 6.0639,
|
||||||
|
"eval_samples_per_second": 208.613,
|
||||||
|
"eval_steps_per_second": 13.193,
|
||||||
|
"step": 1268
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 13.438405990600586,
|
||||||
|
"learning_rate": 1.7323773218938526e-05,
|
||||||
|
"loss": 0.49370124889097006,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_exact_match": 62.8,
|
||||||
|
"eval_f1": 76.44229300202514,
|
||||||
|
"eval_loss": 1.4875582456588745,
|
||||||
|
"eval_runtime": 5.797,
|
||||||
|
"eval_samples_per_second": 218.217,
|
||||||
|
"eval_steps_per_second": 13.8,
|
||||||
|
"step": 1902
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"grad_norm": 14.641495704650879,
|
||||||
|
"learning_rate": 2.728153262824965e-08,
|
||||||
|
"loss": 0.23204485775920494,
|
||||||
|
"step": 2536
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"eval_exact_match": 62.96,
|
||||||
|
"eval_f1": 76.57829761930071,
|
||||||
|
"eval_loss": 1.8072655200958252,
|
||||||
|
"eval_runtime": 5.8611,
|
||||||
|
"eval_samples_per_second": 215.829,
|
||||||
|
"eval_steps_per_second": 13.649,
|
||||||
|
"step": 2536
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 500,
|
||||||
|
"max_steps": 2536,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 7950999010719744.0,
|
||||||
|
"train_batch_size": 16,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": {
|
||||||
|
"learning_rate": 6.22564574576657e-05,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"weight_decay": 0.033952923108431954
|
||||||
|
}
|
||||||
|
}
|
||||||
3
run-1/checkpoint-2536/training_args.bin
Normal file
3
run-1/checkpoint-2536/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:631288df84c6128821fb37f5b2efce07b97fa34e7c5281aa460f5144e137e955
|
||||||
|
size 5265
|
||||||
206
run-1/checkpoint-634/README.md
Normal file
206
run-1/checkpoint-634/README.md
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
---
|
||||||
|
base_model: bert-base-multilingual-cased
|
||||||
|
library_name: peft
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:bert-base-multilingual-cased
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
47
run-1/checkpoint-634/adapter_config.json
Normal file
47
run-1/checkpoint-634/adapter_config.json
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "bert-base-multilingual-cased",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 512,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.0734622590514945,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"qa_outputs"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 128,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"dense",
|
||||||
|
"query",
|
||||||
|
"key",
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "QUESTION_ANS",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
run-1/checkpoint-634/adapter_model.safetensors
Normal file
3
run-1/checkpoint-634/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8abf969f627b6392ffb16e5d113deb74297bc874deab663c27401da04892b41c
|
||||||
|
size 89691896
|
||||||
34
run-1/checkpoint-634/config.json
Normal file
34
run-1/checkpoint-634/config.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"add_cross_attention": false,
|
||||||
|
"architectures": [
|
||||||
|
"BertForQuestionAnswering"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"directionality": "bidi",
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": null,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 12,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pooler_fc_size": 768,
|
||||||
|
"pooler_num_attention_heads": 12,
|
||||||
|
"pooler_num_fc_layers": 3,
|
||||||
|
"pooler_size_per_head": 128,
|
||||||
|
"pooler_type": "first_token_transform",
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.9.0",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 119547
|
||||||
|
}
|
||||||
3
run-1/checkpoint-634/model.safetensors
Normal file
3
run-1/checkpoint-634/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:6f0f6c70c3c32756828941d647f30c49644890940f8f2df008bccb4ae3593e18
|
||||||
|
size 709080872
|
||||||
3
run-1/checkpoint-634/optimizer.pt
Normal file
3
run-1/checkpoint-634/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a6ae0fcfafae8498cfda280ab6e7c5850a9bdae76b97ee3a773397b85f47cc36
|
||||||
|
size 9513163
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user