193 lines
5.9 KiB
JSON
193 lines
5.9 KiB
JSON
|
{
|
||
|
|
"adaptive_mixing_args": null,
|
||
|
|
"async_checkpointing": false,
|
||
|
|
"async_eval_ngpus": -1,
|
||
|
|
"batch_size": 2,
|
||
|
|
"checkpoint_manifold_bucket": "genai_llm_fb",
|
||
|
|
"data": "",
|
||
|
|
"delete_manifold_checkpoints": true,
|
||
|
|
"disable_logging": false,
|
||
|
|
"disable_workers_print": false,
|
||
|
|
"dist": {
|
||
|
|
"global_rank": 0,
|
||
|
|
"world_size": 8
|
||
|
|
},
|
||
|
|
"do_sync_eval": true,
|
||
|
|
"dtype": "bf16",
|
||
|
|
"dump_dir": "/tmp/metaformers_dmp",
|
||
|
|
"dump_freq": 100,
|
||
|
|
"dump_profile_traces": false,
|
||
|
|
"enable_gil_watcher": false,
|
||
|
|
"enable_loss_tracker": false,
|
||
|
|
"eval": null,
|
||
|
|
"eval_freq": 100,
|
||
|
|
"exp_id": "",
|
||
|
|
"exp_name": "",
|
||
|
|
"finetuning_checkpoint_load_strict": false,
|
||
|
|
"finetuning_dir": "/tmp/metaformers_dmp/checkpoints/stable/llama_cinnamon_7b",
|
||
|
|
"fp32_reduce_scatter": true,
|
||
|
|
"gpu_check_level": 3,
|
||
|
|
"hive_data": null,
|
||
|
|
"instruct": {
|
||
|
|
"is_instruct_tuning": true,
|
||
|
|
"no_loss_prompt": true,
|
||
|
|
"no_loss_truncated": false,
|
||
|
|
"only_sft_last_response": false,
|
||
|
|
"smart_coalesce": false,
|
||
|
|
"space_around_response": false,
|
||
|
|
"wrap_seq_tokens_once": false
|
||
|
|
},
|
||
|
|
"instruct_data": "/tmp/metaformers_dmp/data/sft/anthropic_prompts_open_13K_desc_1122_redist_6cat:0.25,/tmp/metaformers_dmp/data/sft/anthropic_prompts_open_13K_no_desc_1122_redist_6cat:0.25,/tmp/metaformers_dmp/data/sft/anthropic_responses_open_13K_desc_1122_redist_6cat:0.25,/tmp/metaformers_dmp/data/sft/anthropic_responses_open_13K_no_desc_1122_redist_6cat:0.25",
|
||
|
|
"iter_batch_multi_hive_koski": null,
|
||
|
|
"iter_jsonl": {
|
||
|
|
"buffer_size": 64,
|
||
|
|
"same_data": false
|
||
|
|
},
|
||
|
|
"iter_multi": {
|
||
|
|
"buffer_size": 512,
|
||
|
|
"ignore_extra_chunks": true,
|
||
|
|
"max_precompute": 20,
|
||
|
|
"multiprocess": true
|
||
|
|
},
|
||
|
|
"iter_type": "multi",
|
||
|
|
"kd_args": {
|
||
|
|
"kd_logits": false,
|
||
|
|
"kd_model": null,
|
||
|
|
"kd_model_dir": "",
|
||
|
|
"reverse_kld_loss": false
|
||
|
|
},
|
||
|
|
"keep_eval_checkpoints": true,
|
||
|
|
"keep_n_last_checkpoints": 2,
|
||
|
|
"log_all_steps": false,
|
||
|
|
"log_freq": 1,
|
||
|
|
"log_updates": true,
|
||
|
|
"loss_rescaling": false,
|
||
|
|
"manifold_output_dir": "tree/checkpoints/mast/inan/2023-11-27/080608_VAx9Hcb0THuGhWcZP4I6OA",
|
||
|
|
"mixing_ratio": null,
|
||
|
|
"model": {
|
||
|
|
"alpha_depth": "disabled",
|
||
|
|
"custom_bwd": true,
|
||
|
|
"dim": 4096,
|
||
|
|
"dim_by_layer": "",
|
||
|
|
"dropout": 0,
|
||
|
|
"efficient_attn": "cutlass",
|
||
|
|
"ffn_dim": 512,
|
||
|
|
"ffn_dim_multiplier": 1.0,
|
||
|
|
"full_logging_n_layers": 4,
|
||
|
|
"head_prune": false,
|
||
|
|
"init": {
|
||
|
|
"coeff_std": null,
|
||
|
|
"depth_last": false,
|
||
|
|
"fixed_std": null,
|
||
|
|
"no_init": false,
|
||
|
|
"use_depth": "current",
|
||
|
|
"use_gaussian": true
|
||
|
|
},
|
||
|
|
"init_on_meta_device": false,
|
||
|
|
"layer_ckpt": "none",
|
||
|
|
"loss_parallel": false,
|
||
|
|
"max_length": 4096,
|
||
|
|
"multiple_of": 256,
|
||
|
|
"n_heads": 32,
|
||
|
|
"n_heads_by_layer": "",
|
||
|
|
"n_kv_heads": null,
|
||
|
|
"n_kv_heads_by_layer": "",
|
||
|
|
"n_layers": 32,
|
||
|
|
"non_linearity": "swiglu",
|
||
|
|
"norm_affine": true,
|
||
|
|
"norm_eps": 1e-05,
|
||
|
|
"norm_type": "rmsnorm",
|
||
|
|
"output_size": -1,
|
||
|
|
"parallel_impl": "FAIRSCALE",
|
||
|
|
"position_interpolation": 1.0,
|
||
|
|
"pre_norm": true,
|
||
|
|
"recompute_attn": true,
|
||
|
|
"recompute_fc1_out": true,
|
||
|
|
"recompute_fc3_out": true,
|
||
|
|
"rope_theta": 10000.0,
|
||
|
|
"sequence_parallel": false,
|
||
|
|
"use_rope": true,
|
||
|
|
"use_xpos": false,
|
||
|
|
"vocab_size": 32000,
|
||
|
|
"xpos_gamma": 0.8,
|
||
|
|
"xpos_scale_base": 4096,
|
||
|
|
"xpos_theta": 500000.0
|
||
|
|
},
|
||
|
|
"model_parallel_size": 1,
|
||
|
|
"no_final_ckpt": false,
|
||
|
|
"optim": {
|
||
|
|
"beta1": 0.9,
|
||
|
|
"beta2": 0.95,
|
||
|
|
"clip": 1.0,
|
||
|
|
"cosine_theta": 1.0,
|
||
|
|
"cycle_length": 1.0,
|
||
|
|
"epsilon": 1e-08,
|
||
|
|
"exp_factor": 0.5,
|
||
|
|
"fused": null,
|
||
|
|
"lr": 2e-06,
|
||
|
|
"lr_min_ratio": 0.1,
|
||
|
|
"scheduler": "cosine",
|
||
|
|
"use_deprecated_optim": false,
|
||
|
|
"use_sgd": false,
|
||
|
|
"warmup": 100,
|
||
|
|
"weight_decay": 0.1
|
||
|
|
},
|
||
|
|
"peft_args": null,
|
||
|
|
"periodic_gpu_check": true,
|
||
|
|
"profile_freq": -1,
|
||
|
|
"reshard_after_forward": true,
|
||
|
|
"restore_dataloader_position": false,
|
||
|
|
"rlhf": null,
|
||
|
|
"root_dump_dir": "/tmp/nobody/xldumps",
|
||
|
|
"secondary_hive_data": null,
|
||
|
|
"seq_len": 4096,
|
||
|
|
"snapshot_restore_dir": null,
|
||
|
|
"steps": 3000,
|
||
|
|
"stuck_threshold_sec": 1500,
|
||
|
|
"tb_upload_freq": 50,
|
||
|
|
"tokenizer": "tokenizer_final_32k.minus_inf_ws.model",
|
||
|
|
"tokenizer_dir": "/tmp/metaformers_dmp/tokenizer",
|
||
|
|
"torch_seed": -1,
|
||
|
|
"unlimited_steps": false,
|
||
|
|
"valid": {
|
||
|
|
"batch_size": 8,
|
||
|
|
"content_key": null,
|
||
|
|
"custom_preference_task_table1": "",
|
||
|
|
"custom_preference_task_table2": "",
|
||
|
|
"debug": false,
|
||
|
|
"hive_data": null,
|
||
|
|
"hive_tasks": [],
|
||
|
|
"hive_tasks_output_hive_data": null,
|
||
|
|
"instruct": {
|
||
|
|
"is_instruct_tuning": true,
|
||
|
|
"no_loss_prompt": true,
|
||
|
|
"no_loss_truncated": false,
|
||
|
|
"only_sft_last_response": false,
|
||
|
|
"smart_coalesce": false,
|
||
|
|
"space_around_response": false,
|
||
|
|
"wrap_seq_tokens_once": false
|
||
|
|
},
|
||
|
|
"iso_regression_model_path": "",
|
||
|
|
"majority_voting": 0,
|
||
|
|
"n_batches": 100,
|
||
|
|
"n_generations": 1,
|
||
|
|
"ppl_files_str": "",
|
||
|
|
"ppl_root_dir": "",
|
||
|
|
"prompt_path": "",
|
||
|
|
"random_fewshots": false,
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"rlhf_eval": false,
|
||
|
|
"seq_len": 2048,
|
||
|
|
"task_batch_size": 8,
|
||
|
|
"tasks_root_dir": "/tmp/metaformers_dmp/data/eval",
|
||
|
|
"tasks_str": "safetyllama_prompt,safetyllama_response",
|
||
|
|
"temperature": 1.0,
|
||
|
|
"top_k": 0,
|
||
|
|
"top_p": 0.0,
|
||
|
|
"use_llm_inference": true,
|
||
|
|
"use_relative_loss": true,
|
||
|
|
"use_sampling": false,
|
||
|
|
"write_eval": true,
|
||
|
|
"write_every_n_batches": 1
|
||
|
|
}
|
||
|
|
}
|