usage: Accelerate launch command [-h] [--config_file CONFIG_FILE] [--quiet] [--cpu] [--multi_gpu] [--tpu] [--mixed_precision {no,fp16,bf16,fp8}] [--num_processes NUM_PROCESSES] [--num_machines NUM_MACHINES] [--num_cpu_threads_per_process NUM_CPU_THREADS_PER_PROCESS] [--enable_cpu_affinity] [--dynamo_backend {no,eager,aot_eager,inductor,aot_ts_nvfuser,nvprims_nvfuser,cudagraphs,ofi,fx2trt,onnxrt,tensorrt,aot_torchxla_trace_once,torhchxla_trace_once,tvm}] [--dynamo_mode {default,reduce-overhead,max-autotune}] [--dynamo_use_fullgraph] [--dynamo_use_dynamic] [--dynamo_use_regional_compilation] [--use_deepspeed] [--use_fsdp] [--use_parallelism_config] [--use_megatron_lm] [--gpu_ids GPU_IDS] [--same_network] [--machine_rank MACHINE_RANK] [--main_process_ip MAIN_PROCESS_IP] [--main_process_port MAIN_PROCESS_PORT] [-t TEE] [--log_dir LOG_DIR] [--role ROLE] [--rdzv_backend RDZV_BACKEND] [--rdzv_conf RDZV_CONF] [--max_restarts MAX_RESTARTS] [--monitor_interval MONITOR_INTERVAL] [-m] [--no_python] [--tpu_cluster] [--no_tpu_cluster] [--tpu_use_sudo] [--vm VM] [--env ENV] [--main_training_function MAIN_TRAINING_FUNCTION] [--downcast_bf16] [--deepspeed_config_file DEEPSPEED_CONFIG_FILE] [--zero_stage ZERO_STAGE] [--offload_optimizer_device OFFLOAD_OPTIMIZER_DEVICE] [--offload_param_device OFFLOAD_PARAM_DEVICE] [--offload_optimizer_nvme_path OFFLOAD_OPTIMIZER_NVME_PATH] [--offload_param_nvme_path OFFLOAD_PARAM_NVME_PATH] [--gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS] [--gradient_clipping GRADIENT_CLIPPING] [--zero3_init_flag ZERO3_INIT_FLAG] [--zero3_save_16bit_model ZERO3_SAVE_16BIT_MODEL] [--deepspeed_hostfile DEEPSPEED_HOSTFILE] [--deepspeed_exclusion_filter DEEPSPEED_EXCLUSION_FILTER] [--deepspeed_inclusion_filter DEEPSPEED_INCLUSION_FILTER] [--deepspeed_multinode_launcher DEEPSPEED_MULTINODE_LAUNCHER] [--deepspeed_moe_layer_cls_names DEEPSPEED_MOE_LAYER_CLS_NAMES] [--fsdp_version {1,2}] [--fsdp_offload_params FSDP_OFFLOAD_PARAMS] [--fsdp_min_num_params FSDP_MIN_NUM_PARAMS] [--fsdp_sharding_strategy FSDP_SHARDING_STRATEGY] [--fsdp_reshard_after_forward FSDP_RESHARD_AFTER_FORWARD] [--fsdp_auto_wrap_policy FSDP_AUTO_WRAP_POLICY] [--fsdp_transformer_layer_cls_to_wrap FSDP_TRANSFORMER_LAYER_CLS_TO_WRAP] [--fsdp_backward_prefetch FSDP_BACKWARD_PREFETCH] [--fsdp_state_dict_type FSDP_STATE_DICT_TYPE] [--fsdp_forward_prefetch FSDP_FORWARD_PREFETCH] [--fsdp_use_orig_params FSDP_USE_ORIG_PARAMS] [--fsdp_cpu_ram_efficient_loading FSDP_CPU_RAM_EFFICIENT_LOADING] [--fsdp_sync_module_states FSDP_SYNC_MODULE_STATES] [--fsdp_activation_checkpointing FSDP_ACTIVATION_CHECKPOINTING] [--fsdp_activation_checkpointing_offload FSDP_ACTIVATION_CHECKPOINTING_OFFLOAD] [--megatron_lm_tp_degree MEGATRON_LM_TP_DEGREE] [--megatron_lm_use_custom_fsdp MEGATRON_LM_USE_CUSTOM_FSDP] [--megatron_lm_no_load_optim MEGATRON_LM_NO_LOAD_OPTIM] [--megatron_lm_eod_mask_loss MEGATRON_LM_EOD_MASK_LOSS] [--megatron_lm_overlap_cpu_optimizer_d2h_h2d MEGATRON_LM_OVERLAP_CPU_OPTIMIZER_D2H_H2D] [--megatron_lm_no_save_optim MEGATRON_LM_NO_SAVE_OPTIM] [--megatron_lm_optimizer_cpu_offload MEGATRON_LM_OPTIMIZER_CPU_OFFLOAD] [--megatron_lm_use_precision_aware_optimizer MEGATRON_LM_USE_PRECISION_AWARE_OPTIMIZER] [--megatron_lm_decoder_last_pipeline_num_layers MEGATRON_LM_DECODER_LAST_PIPELINE_NUM_LAYERS] [--megatron_lm_pp_degree MEGATRON_LM_PP_DEGREE] [--megatron_lm_num_micro_batches MEGATRON_LM_NUM_MICRO_BATCHES] [--megatron_lm_sequence_parallelism MEGATRON_LM_SEQUENCE_PARALLELISM] [--megatron_lm_recompute_activations MEGATRON_LM_RECOMPUTE_ACTIVATIONS] [--megatron_lm_use_distributed_optimizer MEGATRON_LM_USE_DISTRIBUTED_OPTIMIZER] [--megatron_lm_gradient_clipping MEGATRON_LM_GRADIENT_CLIPPING] [--megatron_lm_recompute_granularity MEGATRON_LM_RECOMPUTE_GRANULARITY] [--megatron_lm_recompute_method MEGATRON_LM_RECOMPUTE_METHOD] [--megatron_lm_recompute_num_layers MEGATRON_LM_RECOMPUTE_NUM_LAYERS] [--megatron_lm_attention_backend MEGATRON_LM_ATTENTION_BACKEND] [--megatron_lm_expert_model_parallel_size MEGATRON_LM_EXPERT_MODEL_PARALLEL_SIZE] [--megatron_lm_context_parallel_size MEGATRON_LM_CONTEXT_PARALLEL_SIZE] [--megatron_lm_attention_dropout MEGATRON_LM_ATTENTION_DROPOUT] [--megatron_lm_hidden_dropout MEGATRON_LM_HIDDEN_DROPOUT] [--megatron_lm_attention_softmax_in_fp32 MEGATRON_LM_ATTENTION_SOFTMAX_IN_FP32] [--megatron_lm_expert_tensor_parallel_size MEGATRON_LM_EXPERT_TENSOR_PARALLEL_SIZE] [--megatron_lm_calculate_per_token_loss MEGATRON_LM_CALCULATE_PER_TOKEN_LOSS] [--megatron_lm_use_rotary_position_embeddings MEGATRON_LM_USE_ROTARY_POSITION_EMBEDDINGS] [--fp8_backend {ao,te,msamp}] [--fp8_use_autocast_during_eval] [--fp8_margin FP8_MARGIN] [--fp8_interval FP8_INTERVAL] [--fp8_format {HYBRID,E4M3,E5M2}] [--fp8_amax_history_len FP8_AMAX_HISTORY_LEN] [--fp8_amax_compute_algo {max,most_recent}] [--fp8_override_linear_precision FP8_OVERRIDE_LINEAR_PRECISION] [--fp8_opt_level {O1,O2}] [--fp8_enable_fsdp_float8_all_gather FP8_ENABLE_FSDP_FLOAT8_ALL_GATHER] [--fp8_pad_inner_dim FP8_PAD_INNER_DIM] [--aws_access_key_id AWS_ACCESS_KEY_ID] [--aws_secret_access_key AWS_SECRET_ACCESS_KEY] [--debug] [--mpirun_hostfile MPIRUN_HOSTFILE] [--parallelism_config_dp_replicate_size PARALLELISM_CONFIG_DP_REPLICATE_SIZE] [--parallelism_config_dp_shard_size PARALLELISM_CONFIG_DP_SHARD_SIZE] [--parallelism_config_tp_size PARALLELISM_CONFIG_TP_SIZE] [--parallelism_config_cp_size PARALLELISM_CONFIG_CP_SIZE] [--parallelism_config_cp_backend {torch}] [--parallelism_config_cp_comm_strategy PARALLELISM_CONFIG_CP_COMM_STRATEGY] [--parallelism_config_sp_size PARALLELISM_CONFIG_SP_SIZE] [--parallelism_config_sp_backend {deepspeed}] [--parallelism_config_sp_seq_length PARALLELISM_CONFIG_SP_SEQ_LENGTH] [--parallelism_config_sp_seq_length_is_variable PARALLELISM_CONFIG_SP_SEQ_LENGTH_IS_VARIABLE] [--parallelism_config_sp_attn_implementation PARALLELISM_CONFIG_SP_ATTN_IMPLEMENTATION] training_script ... Launch a python script in a distributed scenario. Arguments can be passed in with either hyphens (`--num-processes=2`) or underscores (`--num_processes=2`) positional arguments: training_script The full path to the script to be launched in parallel, followed by all the arguments for the training script. training_script_args Arguments of the training script. options: -h, --help Show this help message and exit. --config_file CONFIG_FILE The config file to use for the default values in the launching script. --quiet, -q Silence subprocess errors from the launch stack trace and only show the relevant tracebacks. (Only applicable to DeepSpeed and single-process configurations) -m, --module Change each process to interpret the launch script as a Python module, executing with the same behavior as 'python -m'. --no_python Skip prepending the training script with 'python' - just execute it directly. Useful when the script is not a Python script. --debug Whether to print out the torch.distributed stack trace when something fails. Hardware Selection Arguments: Arguments for selecting the hardware to be used. --cpu Whether or not to force the training on the CPU. --multi_gpu Whether or not this should launch a distributed GPU training. --tpu Whether or not this should launch a TPU training. Resource Selection Arguments: Arguments for fine-tuning how available hardware should be used. --mixed_precision {no,fp16,bf16,fp8} Whether or not to use mixed precision training. Choose between FP16 and BF16 (bfloat16) training. BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later. --num_processes NUM_PROCESSES The total number of processes to be launched in parallel. --num_machines NUM_MACHINES The total number of machines used in this training. --num_cpu_threads_per_process NUM_CPU_THREADS_PER_PROCESS The number of CPU threads per process. Can be tuned for optimal performance. --enable_cpu_affinity Whether or not CPU affinity and balancing should be enabled. Currently only supported on NVIDIA hardware. --dynamo_backend {no,eager,aot_eager,inductor,aot_ts_nvfuser,nvprims_nvfuser,cudagraphs,ofi,fx2trt,onnxrt,tensorrt,aot_torchxla_trace_once,torhchxla_trace_once,tvm} Choose a backend to optimize your training with dynamo, see more at https://github.com/pytorch/torchdynamo. --dynamo_mode {default,reduce-overhead,max-autotune} Choose a mode to optimize your training with dynamo. --dynamo_use_fullgraph Whether to use full graph mode for dynamo or it is ok to break model into several subgraphs --dynamo_use_dynamic Whether to enable dynamic shape tracing. --dynamo_use_regional_compilation Whether to enable regional compilation. Training Paradigm Arguments: Arguments for selecting which training paradigm to be used. --use_deepspeed Whether to use deepspeed. --use_fsdp Whether to use fsdp. --use_parallelism_config Whether to use the parallelism config to configure the N-d distributed training. --use_megatron_lm Whether to use Megatron-LM. Distributed GPUs: Arguments related to distributed GPU training. --gpu_ids GPU_IDS What GPUs (by id) should be used for training on this machine as a comma- separated list --same_network Whether all machines used for multinode training exist on the same local network. --machine_rank MACHINE_RANK The rank of the machine on which this script is launched. --main_process_ip MAIN_PROCESS_IP The IP address of the machine of rank 0. --main_process_port MAIN_PROCESS_PORT The port to use to communicate with the machine of rank 0. -t TEE, --tee TEE Tee std streams into a log file and also to console. --log_dir LOG_DIR Base directory to use for log files when using torchrun/torch.distributed.run as launcher. Use with --tee to redirect std streams info log files. --role ROLE User-defined role for the workers. --rdzv_backend RDZV_BACKEND The rendezvous method to use, such as 'static' (the default) or 'c10d' --rdzv_conf RDZV_CONF Additional rendezvous configuration (=,=,...). --max_restarts MAX_RESTARTS Maximum number of worker group restarts before failing. --monitor_interval MONITOR_INTERVAL Interval, in seconds, to monitor the state of workers. TPU: Arguments related to TPU. --tpu_cluster Whether to use a GCP TPU pod for training. --no_tpu_cluster Should not be passed explicitly, this is for internal use only. --tpu_use_sudo Whether to use `sudo` when running the TPU training script in each pod. --vm VM List of single Compute VM instance names. If not provided we assume usage of instance groups. For TPU pods. --env ENV List of environment variables to set on the Compute VM instances. For TPU pods. --main_training_function MAIN_TRAINING_FUNCTION The name of the main function to be executed in your script (only for TPU training). --downcast_bf16 Whether when using bf16 precision on TPUs if both float and double tensors are cast to bfloat16 or if double tensors remain as float32. DeepSpeed Arguments: Arguments related to DeepSpeed. --deepspeed_config_file DEEPSPEED_CONFIG_FILE DeepSpeed config file. --zero_stage ZERO_STAGE DeepSpeed's ZeRO optimization stage (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `2`. --offload_optimizer_device OFFLOAD_OPTIMIZER_DEVICE Decides where (none|cpu|nvme) to offload optimizer states (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'. --offload_param_device OFFLOAD_PARAM_DEVICE Decides where (none|cpu|nvme) to offload parameters (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'. --offload_optimizer_nvme_path OFFLOAD_OPTIMIZER_NVME_PATH Decides Nvme Path to offload optimizer states (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'. --offload_param_nvme_path OFFLOAD_PARAM_NVME_PATH Decides Nvme Path to offload parameters (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'. --gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS No of gradient_accumulation_steps used in your training script (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `1`. --gradient_clipping GRADIENT_CLIPPING gradient clipping value used in your training script (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `1.0`. --zero3_init_flag ZERO3_INIT_FLAG Decides Whether (true|false) to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `true`. --zero3_save_16bit_model ZERO3_SAVE_16BIT_MODEL Decides Whether (true|false) to save 16-bit model weights when using ZeRO Stage-3. Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `false`. --deepspeed_hostfile DEEPSPEED_HOSTFILE DeepSpeed hostfile for configuring multi-node compute resources. --deepspeed_exclusion_filter DEEPSPEED_EXCLUSION_FILTER DeepSpeed exclusion filter string when using multi-node setup. --deepspeed_inclusion_filter DEEPSPEED_INCLUSION_FILTER DeepSpeed inclusion filter string when using multi-node setup. --deepspeed_multinode_launcher DEEPSPEED_MULTINODE_LAUNCHER DeepSpeed multi-node launcher to use, e.g. `pdsh`, `standard`, `openmpi`, `mvapich`, `mpich`, `slurm`, `nossh` (requires DeepSpeed >= 0.14.5). If unspecified, will default to `pdsh`. --deepspeed_moe_layer_cls_names DEEPSPEED_MOE_LAYER_CLS_NAMES comma-separated list of transformer MoE layer class names (case-sensitive) to wrap ,e.g, `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ... (useful only when `use_deepspeed` flag is passed). FSDP Arguments: Arguments related to Fully Shared Data Parallelism. --fsdp_version {1,2} FSDP version to use. (useful only when `use_fsdp` flag is passed). --fsdp_offload_params FSDP_OFFLOAD_PARAMS Decides Whether (true|false) to offload parameters and gradients to CPU. (useful only when `use_fsdp` flag is passed). --fsdp_min_num_params FSDP_MIN_NUM_PARAMS FSDP's minimum number of parameters for Default Auto Wrapping. (useful only when `use_fsdp` flag is passed). --fsdp_sharding_strategy FSDP_SHARDING_STRATEGY FSDP's sharding strategy. (useful only when `use_fsdp` flag is passed and `fsdp_version=1`). --fsdp_reshard_after_forward FSDP_RESHARD_AFTER_FORWARD FSDP's Reshard After Forward Strategy. (useful only when `use_fsdp` flag is passed). Supports either boolean (FSDP2) or `FULL_SHARD | SHARD_GRAD_OP | NO_RESHARD` (FSDP1). --fsdp_auto_wrap_policy FSDP_AUTO_WRAP_POLICY FSDP's auto wrap policy. (useful only when `use_fsdp` flag is passed). --fsdp_transformer_layer_cls_to_wrap FSDP_TRANSFORMER_LAYER_CLS_TO_WRAP Transformer layer class name (case-sensitive) to wrap ,e.g, `BertLayer`, `GPTJBlock`, `T5Block` .... (useful only when `use_fsdp` flag is passed). --fsdp_backward_prefetch FSDP_BACKWARD_PREFETCH FSDP's backward prefetch policy. (useful only when `use_fsdp` flag is passed). --fsdp_state_dict_type FSDP_STATE_DICT_TYPE FSDP's state dict type. (useful only when `use_fsdp` flag is passed). --fsdp_forward_prefetch FSDP_FORWARD_PREFETCH If True, then FSDP explicitly prefetches the next upcoming all-gather while executing in the forward pass (useful only when `use_fsdp` flag is passed). --fsdp_use_orig_params FSDP_USE_ORIG_PARAMS If True, allows non-uniform `requires_grad` during init, which means support for interspersed frozen and trainable parameters. (useful only when `use_fsdp` flag is passed). --fsdp_cpu_ram_efficient_loading FSDP_CPU_RAM_EFFICIENT_LOADING If True, only the first process loads the pretrained model checkpoint while all other processes have empty weights. Only applicable for 🤗 Transformers. When using this, `--fsdp_sync_module_states` needs to True. (useful only when `use_fsdp` flag is passed). --fsdp_sync_module_states FSDP_SYNC_MODULE_STATES If True, each individually wrapped FSDP unit will broadcast module parameters from rank 0. (useful only when `use_fsdp` flag is passed). --fsdp_activation_checkpointing FSDP_ACTIVATION_CHECKPOINTING Decides Whether (true|false) intermediate activations are freed during the forward pass, and a checkpoint is left as a placeholder. (useful only when `use_fsdp` flag is passed). --fsdp_activation_checkpointing_offload FSDP_ACTIVATION_CHECKPOINTING_OFFLOAD Decides Whether (true|false) each checkpointed layer's input activation is offloaded to pinned CPU memory during the forward pass and restored on demand during the backward pass. Requires `fsdp_activation_checkpointing` and FSDP2. (useful only when `use_fsdp` flag is passed). Megatron-LM Arguments: Arguments related to Megatron-LM. --megatron_lm_tp_degree MEGATRON_LM_TP_DEGREE Megatron-LM's Tensor Parallelism (TP) degree. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_use_custom_fsdp MEGATRON_LM_USE_CUSTOM_FSDP Whether to use custom FSDP. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_no_load_optim MEGATRON_LM_NO_LOAD_OPTIM Whether to not load optimizer. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_eod_mask_loss MEGATRON_LM_EOD_MASK_LOSS Whether to use eod mask loss. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_overlap_cpu_optimizer_d2h_h2d MEGATRON_LM_OVERLAP_CPU_OPTIMIZER_D2H_H2D Whether to overlap CPU optimizer step, gradients D2H and updated parameters H2D. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_no_save_optim MEGATRON_LM_NO_SAVE_OPTIM Whether to not save optimizer. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_optimizer_cpu_offload MEGATRON_LM_OPTIMIZER_CPU_OFFLOAD Whether to use CPU offload for optimizer. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_use_precision_aware_optimizer MEGATRON_LM_USE_PRECISION_AWARE_OPTIMIZER Whether to use precision aware optimizer. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_decoder_last_pipeline_num_layers MEGATRON_LM_DECODER_LAST_PIPELINE_NUM_LAYERS Megatron-LM's decoder last pipeline number of layers, default None is even split of transformer layers across all pipeline stages. --megatron_lm_pp_degree MEGATRON_LM_PP_DEGREE Megatron-LM's Pipeline Parallelism (PP) degree. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_num_micro_batches MEGATRON_LM_NUM_MICRO_BATCHES Megatron-LM's number of micro batches when PP degree > 1. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_sequence_parallelism MEGATRON_LM_SEQUENCE_PARALLELISM Decides Whether (true|false) to enable Sequence Parallelism when TP degree > 1. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_recompute_activations MEGATRON_LM_RECOMPUTE_ACTIVATIONS Decides Whether (true|false) to enable Selective Activation Recomputation. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_use_distributed_optimizer MEGATRON_LM_USE_DISTRIBUTED_OPTIMIZER Decides Whether (true|false) to use distributed optimizer which shards optimizer state and gradients across Data Pralellel (DP) ranks. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_gradient_clipping MEGATRON_LM_GRADIENT_CLIPPING Megatron-LM's gradient clipping value based on global L2 Norm (0 to disable). (useful only when `use_megatron_lm` flag is passed). --megatron_lm_recompute_granularity MEGATRON_LM_RECOMPUTE_GRANULARITY Megatron-LM's recompute granularity (full, selective). (useful only when `use_megatron_lm` flag is passed). --megatron_lm_recompute_method MEGATRON_LM_RECOMPUTE_METHOD Megatron-LM's recompute method (uniform, block). (useful only when `use_megatron_lm` flag is passed). --megatron_lm_recompute_num_layers MEGATRON_LM_RECOMPUTE_NUM_LAYERS Megatron-LM's number of layers to recompute. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_attention_backend MEGATRON_LM_ATTENTION_BACKEND Decides Whether (true|false) to enable attention backend. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_expert_model_parallel_size MEGATRON_LM_EXPERT_MODEL_PARALLEL_SIZE Megatron-LM's expert model parallel size. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_context_parallel_size MEGATRON_LM_CONTEXT_PARALLEL_SIZE Megatron-LM's context parallel size. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_attention_dropout MEGATRON_LM_ATTENTION_DROPOUT Megatron-LM's attention dropout rate. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_hidden_dropout MEGATRON_LM_HIDDEN_DROPOUT Megatron-LM's hidden dropout rate. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_attention_softmax_in_fp32 MEGATRON_LM_ATTENTION_SOFTMAX_IN_FP32 Decides Whether (true|false) to use fp32 for attention softmax. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_expert_tensor_parallel_size MEGATRON_LM_EXPERT_TENSOR_PARALLEL_SIZE Megatron-LM's expert tensor parallel size. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_calculate_per_token_loss MEGATRON_LM_CALCULATE_PER_TOKEN_LOSS Decides Whether (true|false) to calculate per token loss. (useful only when `use_megatron_lm` flag is passed). --megatron_lm_use_rotary_position_embeddings MEGATRON_LM_USE_ROTARY_POSITION_EMBEDDINGS Decides Whether (true|false) to use rotary position embeddings. (useful only when `use_megatron_lm` flag is passed). FP8 Arguments: Arguments related to FP8 training (requires `--mixed_precision=fp8`) --fp8_backend {ao,te,msamp} Choose a backend to train with FP8 (ao: torchao, te: TransformerEngine, msamp: MS-AMP) --fp8_use_autocast_during_eval Whether to use FP8 autocast during eval mode (useful only when `--fp8_backend=te` is passed). Generally better metrics are found when this is not passed. --fp8_margin FP8_MARGIN The margin to use for the gradient scaling (useful only when `--fp8_backend=te` is passed). --fp8_interval FP8_INTERVAL The interval to use for how often the scaling factor is recomputed (useful only when `--fp8_backend=te` is passed). --fp8_format {HYBRID,E4M3,E5M2} The format to use for the FP8 recipe (useful only when `--fp8_backend=te` is passed). --fp8_amax_history_len FP8_AMAX_HISTORY_LEN The length of the history to use for the scaling factor computation (useful only when `--fp8_backend=te` is passed). --fp8_amax_compute_algo {max,most_recent} The algorithm to use for the scaling factor computation. (useful only when `--fp8_backend=te` is passed). --fp8_override_linear_precision FP8_OVERRIDE_LINEAR_PRECISION Whether or not to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher precision. Should be passed in a comma-separated string of booleans (useful only when `--fp8_backend=te` is passed). --fp8_opt_level {O1,O2} What level of 8-bit collective communication should be used with MS-AMP (useful only when `--fp8_backend=msamp` is passed). --fp8_enable_fsdp_float8_all_gather FP8_ENABLE_FSDP_FLOAT8_ALL_GATHER Whether to enable FSDP2 float8 all gather (useful only when `--fp8_backend=ao` is passed). --fp8_pad_inner_dim FP8_PAD_INNER_DIM Whether to pad the inner dimension for FP8 GEMMs (useful only when `--fp8_backend=ao` is passed). AWS Arguments: Arguments related to AWS. --aws_access_key_id AWS_ACCESS_KEY_ID The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job --aws_secret_access_key AWS_SECRET_ACCESS_KEY The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job. MPI Arguments: Arguments related to mpirun for Multi-CPU --mpirun_hostfile MPIRUN_HOSTFILE Location for a hostfile for using Accelerate to launch a multi-CPU training job with mpirun. This will get passed to the MPI --hostfile or -f parameter, depending on which MPI program is installed. ParallelismConfig Arguments: Arguments related to the ParallelismConfig used for distributed training. --parallelism_config_dp_replicate_size PARALLELISM_CONFIG_DP_REPLICATE_SIZE The number of processes for data parallel training. Defaults to 1 (no data parallelism). --parallelism_config_dp_shard_size PARALLELISM_CONFIG_DP_SHARD_SIZE The number of processes for FSDP sharding. Defaults to 1 (No FSDP sharding). --parallelism_config_tp_size PARALLELISM_CONFIG_TP_SIZE The number of processes for tensor parallel training. Defaults to 1 (no tensor parallelism). --parallelism_config_cp_size PARALLELISM_CONFIG_CP_SIZE The number of processese for context parallel training. Defaults to 1 (no context parallelism). --parallelism_config_cp_backend {torch} Context Parallelism backend: torch (FSDP2) or deepspeed (ALST/Ulysses) --parallelism_config_cp_comm_strategy PARALLELISM_CONFIG_CP_COMM_STRATEGY The communication strategy for context parallel training. Defaults to 'allgather'. Other option is alltoall --parallelism_config_sp_size PARALLELISM_CONFIG_SP_SIZE The number of processese for context parallel training. Defaults to 1 (no context parallelism). --parallelism_config_sp_backend {deepspeed} Sequence Parallelism backend: deepspeed (ALST/Ulysses) --parallelism_config_sp_seq_length PARALLELISM_CONFIG_SP_SEQ_LENGTH Sequence length for when batches are all of the same length. For variable sequence lengths across batches set `parallelism_config_sp_seq_length_is_variable=True` --parallelism_config_sp_seq_length_is_variable PARALLELISM_CONFIG_SP_SEQ_LENGTH_IS_VARIABLE If `True` will work with a sequence length that may change between batches, in which case `parallelism_config_sp_seq_length` value can be set to anything divisible by sp size or remain unset. If `False` then `parallelism_config_sp_seq_length` needs to match the batch's sequence length dimension. The default is `True`. --parallelism_config_sp_attn_implementation PARALLELISM_CONFIG_SP_ATTN_IMPLEMENTATION Attention implementation to use. Can be one of 'flash_attention_2', 'flash_attention_3', 'sdpa', or a hub-hosted kernel (e.g. 'kernels- community/flash-attn2'). Defaults to `sdpa`.