{
    "alternate_pp_config": false,
    "async_checkpointing": true,
    "async_checkpointing_staging_method": "sync",
    "async_eval_ngpus": -1,
    "async_evals_params": null,
    "attach_debugpy": false,
    "background_nccl_init": false,
    "batch_p2p_communication": true,
    "batch_size": 2,
    "collect_et": false,
    "context_parallel_size": 1,
    "data": "/fsx-onellm/data/corpora/text_only/stackexchange:4.4,/fsx-onellm/data/corpora/text_only/b3g:7.2,/fsx-onellm/data/corpora/text_only/arxiv:2.8,/fsx-onellm/data/corpora/text_only/github_oss_with_stack:23.2,/fsx-onellm/data/corpora/text_only/c4:8.4,/fsx-onellm/data/corpora/text_only/edouard_cc_20220927_new:30,/fsx-onellm/data/corpora/text_only/ccnet_new:36,/fsx-onellm/data/corpora/text_only/wikipedia:4.3,/fsx-atom/melhoushi/data/openwebmath:2.2",
    "deallocate_pipeline_outputs": true,
    "disable_logging": false,
    "disable_workers_print": false,
    "dtype": "bf16",
    "dump_dir": "/fsx-atom/melhoushi/xldumps/continual_llama3.2_1B_gpus_64_shared_emb_higher_lr_ee_0.1_ld_0.1_128_gpus/continual_llama3.2_1B_gpus_64_shared_emb_higher_lr_ee_0.1_ld_0.1_128_gpus_run000",
    "dump_freq": 250,
    "dump_profile_traces": false,
    "enable_anomaly_detection": false,
    "enable_deterministic_training": false,
    "enable_loss_tracker": false,
    "enable_ods": false,
    "enable_pynvml": false,
    "et_end_itr": 15,
    "et_start_itr": 12,
    "eval_freq": 5000,
    "exp_id": "",
    "exp_name": "",
    "experimental": {
        "early_exit": {
            "calculate_cache": true,
            "criterion": "oracle",
            "criterion_args_str": null,
            "disable_on_prompt": false,
            "end_offset": 0,
            "start_offset": 0
        },
        "early_exit_loss": {
            "e_scale": 0.1,
            "freeze_backbone": false,
            "layers_str": "0::1",
            "scale_type": "sum_l",
            "share_fc": true,
            "time_enable_mod": 8
        },
        "layer_dropout": null
    },
    "finetuning_dir": "/fsx-atom/melhoushi/xldumps/continual_llama3.2_1B_gpus_64_shared_emb_higher_lr_ee_0.1_ld_0.1_96_gpus/continual_llama3.2_1B_gpus_64_shared_emb_higher_lr_ee_0.1_ld_0.1_96_gpus_run000/checkpoints/checkpoint_0100000/",
    "fp32_reduce_scatter": "all",
    "gc_collect_freq": 1000,
    "gpu_check_level": -1,
    "increase_seq": null,
    "instruct": {
        "no_loss_prompt": false,
        "no_loss_truncated": false
    },
    "instruct_data": "",
    "iter_gopher": {
        "buffer_size": 16,
        "max_precompute": 10,
        "n_chars_by_tok": 15,
        "n_seqs_to_concat": 10,
        "num_processes": 1
    },
    "iter_jsonl": {
        "buffer_size": 64,
        "same_data": false
    },
    "iter_multi": {
        "buffer_size": 64,
        "ignore_extra_chunks": true,
        "iterate_chunk_by_chunk": false,
        "max_precompute": 200,
        "multiprocess": true
    },
    "iter_text_airstore": {
        "airstore_seed": 0,
        "buffer_size": 64,
        "dataloader_workers_per_gpu": 16,
        "prefetch_factor": 20,
        "snapshot_every_n_steps": null
    },
    "iter_type": "multi",
    "keep_eval_checkpoints": false,
    "keep_n_last_checkpoints": 2,
    "load_optimizer_on_finetuning": false,
    "log": {
        "log_scalars": false,
        "log_scalars_to_ods": false,
        "reduce_scalars": false
    },
    "log_all_steps": false,
    "log_freq": 10,
    "log_updates": true,
    "loss_logging_freq": 10,
    "loss_rescaling": false,
    "mb_recompute_attn": false,
    "mb_recompute_fc1_fc3": false,
    "mem_snapshot_max_entries": 100000,
    "mem_snapshot_stop_step": 3,
    "model": {
        "alpha_depth": "disabled",
        "alpha_lrm": 1.0,
        "alpha_separate": false,
        "alpha_wdm": 1.0,
        "attn_bias_type": "block_causal",
        "attn_to_keep": "all",
        "cp_attn_save_global_kv": true,
        "custom_bwd": false,
        "custom_bwd_sum_first_then_comms": true,
        "default_init_router": false,
        "dim": 2048,
        "dropout": 0.0,
        "efficient_attn": "auto",
        "eos_id": 128001,
        "experts_choice_moe": {
            "auto_scale_F": true,
            "capacity_factor": 1.0,
            "eval_threshold_std_mult": 0.0,
            "eval_with_saved_stats": false,
            "eval_with_top_k": false,
            "input_scaling": true,
            "input_scaling_max_clamp": 2.0,
            "input_scaling_min_clamp": -2.0,
            "is_enabled": false,
            "num_experts": 8,
            "running_stats_ema": 0.95,
            "running_stats_sync_freq": 100,
            "use_shared_expert": true
        },
        "ffn_dim_multiplier": 1.5,
        "ffn_exp": 4.0,
        "fp8_amax_compute_algo": "max",
        "fp8_amax_history_len": 1024,
        "fp8_grad_output_dynamic_scale": false,
        "fp8_input_dynamic_scale": false,
        "fp8_interval": 1,
        "fp8_margin": 0,
        "fp8_wgrad": false,
        "fsdp_checkpoint_wrap_layer_frequency": 1,
        "fuse_sequence_parallel": false,
        "high_freq_factor": 32,
        "init": {
            "coeff_std": null,
            "depth_last": false,
            "fixed_std": null,
            "no_init": false,
            "use_depth": "current",
            "use_gaussian": true
        },
        "layer_ckpt": "all",
        "less_layer_first_pp_stage": 0,
        "less_layer_last_pp_stage": 0,
        "linear_residual_dropout": true,
        "local_attention_window_len": null,
        "log_ffn_stats": false,
        "loss_parallel": false,
        "max_length": 2048,
        "multiple_of": 256,
        "n_heads": 32,
        "n_kv_heads": 8,
        "n_layers": 16,
        "non_linearity": "swiglu",
        "norm_affine": true,
        "norm_eps": 1e-05,
        "norm_type": "rmsnorm",
        "number_of_experts": 1,
        "output_size": 128256,
        "pre_norm": true,
        "recompute_attn": false,
        "recompute_fc1_out": true,
        "recompute_fc3_out": true,
        "residual_dropout": 0.1,
        "rope_attn_scale": false,
        "rope_scale_factor": 8,
        "rope_theta": 500000.0,
        "rope_use_fp32_in_outer_product": true,
        "sequence_parallel": true,
        "share_emb": true,
        "use_fp8": false,
        "use_rope": true,
        "use_scaled_rope": true,
        "use_te_layers": false,
        "vocab_parallel": true,
        "vocab_size": 128256
    },
    "model_parallel_size": 1,
    "no_final_ckpt": false,
    "num_layers_per_virtual_pipeline_stage": null,
    "num_microbatches_with_partial_activation_checkpoints": null,
    "old_mp": -1,
    "old_world_size": -1,
    "optim": {
        "annealing_step": 10000,
        "beta1": 0.9,
        "beta2": 0.95,
        "clip": 1.0,
        "cosine_theta": 1.0,
        "cycle_length": 1.0,
        "decay_length_fraction": 0.1,
        "epsilon": 1e-08,
        "exp_factor": 0.5,
        "independent_weight_decay": false,
        "lr": 2e-05,
        "lr_min_ratio": 0.1,
        "scheduler": "cosine",
        "start_annealing_step": -1,
        "use_fp32_copy_optim": true,
        "warmup": 2000,
        "weight_decay": 0.1
    },
    "optimize_backward_concat": false,
    "overlap_p2p_communication": false,
    "periodic_gpu_check": false,
    "pipeline_parallel_microbatch_size": 1,
    "pipeline_parallel_size": 1,
    "pipeline_strategy": "dfs",
    "profile_freq": -1,
    "profile_record_shapes": false,
    "profile_with_stack": false,
    "py_spy_args": {
        "active_seconds": 600,
        "format": "flamegraph",
        "freq": -1,
        "rank0_only": true,
        "rate": 50,
        "start_offset": 10
    },
    "reshard_after_forward": true,
    "restore_dataloader_position": false,
    "root_dump_dir": "/fsx-atom/melhoushi/xldumps/xldumps",
    "runtime_nccl_timeout_s": 180,
    "seq_len": 8192,
    "skip_evals_during_training": false,
    "slurm": {
        "global_rank": 0,
        "is_slurm_job": true,
        "world_size": 128
    },
    "steps": 100000,
    "tokenizer": {
        "basic_special_tokens": [
            "<|begin_of_text|>",
            "<|end_of_text|>",
            "<|fim_prefix|>",
            "<|fim_middle|>",
            "<|fim_suffix|>"
        ],
        "directory": "/fsx-onellm/myasu/models/llama3/Meta-Llama-3-8B/original_fsdp/",
        "extra_special_tokens": [],
        "model": "cl_toplang_128k",
        "num_reserved_special_tokens": 256,
        "pat_str": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
        "tokenizer_cls": "TiktokenTokenizer"
    },
    "tokenizer_dir": "",
    "torch_seed": 0,
    "unlimited_steps": false,
    "use_checkpointing_process": false,
    "use_sum_loss": false,
    "valid": {
        "batch_size": 4,
        "debug": false,
        "majority_voting": 0,
        "n_batches": 100,
        "ppl_files_str": "/fsx-labs/broz/data/shuffled/stackv1/stack_v1_2023_02_27_test/stack_test_file.jsonl,/fsx-labs/broz/datasets/wikipedia/processed/en/wikipedia_en_all_maxi_2023-02.jsonl,/fsx-labs/broz/datasets/gutenberg/processed/selected_books.jsonl",
        "prompt_path": "",
        "random_fewshots": false,
        "seed": 42,
        "seq_len": 2048,
        "skip_sanity_check": false,
        "tasks_root_dir": "/fsx-labs/broz/data/large_experiments/theorem/datasets/eval",
        "tasks_str": "human_eval,piqa,siqa,hellaswag",
        "temperature": 0.0,
        "top_k": 0,
        "top_p": 0.0,
        "use_sampling": false,
        "write_eval": false
    },
    "z_loss_multiplier": 0.0
}