/opt/chainguard/nemo-rl/venvs/automodel/bin/accelerate-launch

usage: Accelerate launch command [-h] [--config_file CONFIG_FILE] [--quiet] [--cpu] [--multi_gpu]
                                 [--tpu] [--mixed_precision {no,fp16,bf16,fp8}]
                                 [--num_processes NUM_PROCESSES] [--num_machines NUM_MACHINES]
                                 [--num_cpu_threads_per_process NUM_CPU_THREADS_PER_PROCESS]
                                 [--enable_cpu_affinity]
                                 [--dynamo_backend {no,eager,aot_eager,inductor,aot_ts_nvfuser,nvprims_nvfuser,cudagraphs,ofi,fx2trt,onnxrt,tensorrt,aot_torchxla_trace_once,torhchxla_trace_once,tvm}]
                                 [--dynamo_mode {default,reduce-overhead,max-autotune}]
                                 [--dynamo_use_fullgraph] [--dynamo_use_dynamic]
                                 [--dynamo_use_regional_compilation] [--use_deepspeed]
                                 [--use_fsdp] [--use_parallelism_config] [--use_megatron_lm]
                                 [--gpu_ids GPU_IDS] [--same_network]
                                 [--machine_rank MACHINE_RANK] [--main_process_ip MAIN_PROCESS_IP]
                                 [--main_process_port MAIN_PROCESS_PORT] [-t TEE]
                                 [--log_dir LOG_DIR] [--role ROLE] [--rdzv_backend RDZV_BACKEND]
                                 [--rdzv_conf RDZV_CONF] [--max_restarts MAX_RESTARTS]
                                 [--monitor_interval MONITOR_INTERVAL] [-m] [--no_python]
                                 [--tpu_cluster] [--no_tpu_cluster] [--tpu_use_sudo] [--vm VM]
                                 [--env ENV] [--main_training_function MAIN_TRAINING_FUNCTION]
                                 [--downcast_bf16] [--deepspeed_config_file DEEPSPEED_CONFIG_FILE]
                                 [--zero_stage ZERO_STAGE]
                                 [--offload_optimizer_device OFFLOAD_OPTIMIZER_DEVICE]
                                 [--offload_param_device OFFLOAD_PARAM_DEVICE]
                                 [--offload_optimizer_nvme_path OFFLOAD_OPTIMIZER_NVME_PATH]
                                 [--offload_param_nvme_path OFFLOAD_PARAM_NVME_PATH]
                                 [--gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS]
                                 [--gradient_clipping GRADIENT_CLIPPING]
                                 [--zero3_init_flag ZERO3_INIT_FLAG]
                                 [--zero3_save_16bit_model ZERO3_SAVE_16BIT_MODEL]
                                 [--deepspeed_hostfile DEEPSPEED_HOSTFILE]
                                 [--deepspeed_exclusion_filter DEEPSPEED_EXCLUSION_FILTER]
                                 [--deepspeed_inclusion_filter DEEPSPEED_INCLUSION_FILTER]
                                 [--deepspeed_multinode_launcher DEEPSPEED_MULTINODE_LAUNCHER]
                                 [--deepspeed_moe_layer_cls_names DEEPSPEED_MOE_LAYER_CLS_NAMES]
                                 [--fsdp_version {1,2}]
                                 [--fsdp_offload_params FSDP_OFFLOAD_PARAMS]
                                 [--fsdp_min_num_params FSDP_MIN_NUM_PARAMS]
                                 [--fsdp_sharding_strategy FSDP_SHARDING_STRATEGY]
                                 [--fsdp_reshard_after_forward FSDP_RESHARD_AFTER_FORWARD]
                                 [--fsdp_auto_wrap_policy FSDP_AUTO_WRAP_POLICY]
                                 [--fsdp_transformer_layer_cls_to_wrap FSDP_TRANSFORMER_LAYER_CLS_TO_WRAP]
                                 [--fsdp_backward_prefetch FSDP_BACKWARD_PREFETCH]
                                 [--fsdp_state_dict_type FSDP_STATE_DICT_TYPE]
                                 [--fsdp_forward_prefetch FSDP_FORWARD_PREFETCH]
                                 [--fsdp_use_orig_params FSDP_USE_ORIG_PARAMS]
                                 [--fsdp_cpu_ram_efficient_loading FSDP_CPU_RAM_EFFICIENT_LOADING]
                                 [--fsdp_sync_module_states FSDP_SYNC_MODULE_STATES]
                                 [--fsdp_activation_checkpointing FSDP_ACTIVATION_CHECKPOINTING]
                                 [--megatron_lm_tp_degree MEGATRON_LM_TP_DEGREE]
                                 [--megatron_lm_use_custom_fsdp MEGATRON_LM_USE_CUSTOM_FSDP]
                                 [--megatron_lm_no_load_optim MEGATRON_LM_NO_LOAD_OPTIM]
                                 [--megatron_lm_eod_mask_loss MEGATRON_LM_EOD_MASK_LOSS]
                                 [--megatron_lm_overlap_cpu_optimizer_d2h_h2d MEGATRON_LM_OVERLAP_CPU_OPTIMIZER_D2H_H2D]
                                 [--megatron_lm_no_save_optim MEGATRON_LM_NO_SAVE_OPTIM]
                                 [--megatron_lm_optimizer_cpu_offload MEGATRON_LM_OPTIMIZER_CPU_OFFLOAD]
                                 [--megatron_lm_use_precision_aware_optimizer MEGATRON_LM_USE_PRECISION_AWARE_OPTIMIZER]
                                 [--megatron_lm_decoder_last_pipeline_num_layers MEGATRON_LM_DECODER_LAST_PIPELINE_NUM_LAYERS]
                                 [--megatron_lm_pp_degree MEGATRON_LM_PP_DEGREE]
                                 [--megatron_lm_num_micro_batches MEGATRON_LM_NUM_MICRO_BATCHES]
                                 [--megatron_lm_sequence_parallelism MEGATRON_LM_SEQUENCE_PARALLELISM]
                                 [--megatron_lm_recompute_activations MEGATRON_LM_RECOMPUTE_ACTIVATIONS]
                                 [--megatron_lm_use_distributed_optimizer MEGATRON_LM_USE_DISTRIBUTED_OPTIMIZER]
                                 [--megatron_lm_gradient_clipping MEGATRON_LM_GRADIENT_CLIPPING]
                                 [--megatron_lm_recompute_granularity MEGATRON_LM_RECOMPUTE_GRANULARITY]
                                 [--megatron_lm_recompute_method MEGATRON_LM_RECOMPUTE_METHOD]
                                 [--megatron_lm_recompute_num_layers MEGATRON_LM_RECOMPUTE_NUM_LAYERS]
                                 [--megatron_lm_attention_backend MEGATRON_LM_ATTENTION_BACKEND]
                                 [--megatron_lm_expert_model_parallel_size MEGATRON_LM_EXPERT_MODEL_PARALLEL_SIZE]
                                 [--megatron_lm_context_parallel_size MEGATRON_LM_CONTEXT_PARALLEL_SIZE]
                                 [--megatron_lm_attention_dropout MEGATRON_LM_ATTENTION_DROPOUT]
                                 [--megatron_lm_hidden_dropout MEGATRON_LM_HIDDEN_DROPOUT]
                                 [--megatron_lm_attention_softmax_in_fp32 MEGATRON_LM_ATTENTION_SOFTMAX_IN_FP32]
                                 [--megatron_lm_expert_tensor_parallel_size MEGATRON_LM_EXPERT_TENSOR_PARALLEL_SIZE]
                                 [--megatron_lm_calculate_per_token_loss MEGATRON_LM_CALCULATE_PER_TOKEN_LOSS]
                                 [--megatron_lm_use_rotary_position_embeddings MEGATRON_LM_USE_ROTARY_POSITION_EMBEDDINGS]
                                 [--fp8_backend {ao,te,msamp}] [--fp8_use_autocast_during_eval]
                                 [--fp8_margin FP8_MARGIN] [--fp8_interval FP8_INTERVAL]
                                 [--fp8_format {HYBRID,E4M3,E5M2}]
                                 [--fp8_amax_history_len FP8_AMAX_HISTORY_LEN]
                                 [--fp8_amax_compute_algo {max,most_recent}]
                                 [--fp8_override_linear_precision FP8_OVERRIDE_LINEAR_PRECISION]
                                 [--fp8_opt_level {O1,O2}]
                                 [--fp8_enable_fsdp_float8_all_gather FP8_ENABLE_FSDP_FLOAT8_ALL_GATHER]
                                 [--fp8_pad_inner_dim FP8_PAD_INNER_DIM]
                                 [--aws_access_key_id AWS_ACCESS_KEY_ID]
                                 [--aws_secret_access_key AWS_SECRET_ACCESS_KEY] [--debug]
                                 [--mpirun_hostfile MPIRUN_HOSTFILE]
                                 [--parallelism_config_dp_replicate_size PARALLELISM_CONFIG_DP_REPLICATE_SIZE]
                                 [--parallelism_config_dp_shard_size PARALLELISM_CONFIG_DP_SHARD_SIZE]
                                 [--parallelism_config_tp_size PARALLELISM_CONFIG_TP_SIZE]
                                 [--parallelism_config_cp_size PARALLELISM_CONFIG_CP_SIZE]
                                 [--parallelism_config_cp_backend {torch}]
                                 [--parallelism_config_cp_comm_strategy PARALLELISM_CONFIG_CP_COMM_STRATEGY]
                                 [--parallelism_config_sp_size PARALLELISM_CONFIG_SP_SIZE]
                                 [--parallelism_config_sp_backend {deepspeed}]
                                 [--parallelism_config_sp_seq_length PARALLELISM_CONFIG_SP_SEQ_LENGTH]
                                 [--parallelism_config_sp_seq_length_is_variable PARALLELISM_CONFIG_SP_SEQ_LENGTH_IS_VARIABLE]
                                 [--parallelism_config_sp_attn_implementation PARALLELISM_CONFIG_SP_ATTN_IMPLEMENTATION]
                                 training_script ...

Launch a python script in a distributed scenario. Arguments can be passed in with either hyphens
(`--num-processes=2`) or underscores (`--num_processes=2`)

positional arguments:
  training_script       The full path to the script to be launched in parallel, followed by all
                        the arguments for the training script.
  training_script_args  Arguments of the training script.

options:
  -h, --help            Show this help message and exit.
  --config_file CONFIG_FILE
                        The config file to use for the default values in the launching script.
  --quiet, -q           Silence subprocess errors from the launch stack trace and only show the
                        relevant tracebacks. (Only applicable to DeepSpeed and single-process
                        configurations)
  -m, --module          Change each process to interpret the launch script as a Python module,
                        executing with the same behavior as 'python -m'.
  --no_python           Skip prepending the training script with 'python' - just execute it
                        directly. Useful when the script is not a Python script.
  --debug               Whether to print out the torch.distributed stack trace when something
                        fails.

Hardware Selection Arguments:
  Arguments for selecting the hardware to be used.

  --cpu                 Whether or not to force the training on the CPU.
  --multi_gpu           Whether or not this should launch a distributed GPU training.
  --tpu                 Whether or not this should launch a TPU training.

Resource Selection Arguments:
  Arguments for fine-tuning how available hardware should be used.

  --mixed_precision {no,fp16,bf16,fp8}
                        Whether or not to use mixed precision training. Choose between FP16 and
                        BF16 (bfloat16) training. BF16 training is only supported on Nvidia Ampere
                        GPUs and PyTorch 1.10 or later.
  --num_processes NUM_PROCESSES
                        The total number of processes to be launched in parallel.
  --num_machines NUM_MACHINES
                        The total number of machines used in this training.
  --num_cpu_threads_per_process NUM_CPU_THREADS_PER_PROCESS
                        The number of CPU threads per process. Can be tuned for optimal
                        performance.
  --enable_cpu_affinity
                        Whether or not CPU affinity and balancing should be enabled. Currently
                        only supported on NVIDIA hardware.
  --dynamo_backend {no,eager,aot_eager,inductor,aot_ts_nvfuser,nvprims_nvfuser,cudagraphs,ofi,fx2trt,onnxrt,tensorrt,aot_torchxla_trace_once,torhchxla_trace_once,tvm}
                        Choose a backend to optimize your training with dynamo, see more at
                        https://github.com/pytorch/torchdynamo.
  --dynamo_mode {default,reduce-overhead,max-autotune}
                        Choose a mode to optimize your training with dynamo.
  --dynamo_use_fullgraph
                        Whether to use full graph mode for dynamo or it is ok to break model into
                        several subgraphs
  --dynamo_use_dynamic  Whether to enable dynamic shape tracing.
  --dynamo_use_regional_compilation
                        Whether to enable regional compilation.

Training Paradigm Arguments:
  Arguments for selecting which training paradigm to be used.

  --use_deepspeed       Whether to use deepspeed.
  --use_fsdp            Whether to use fsdp.
  --use_parallelism_config
                        Whether to use the parallelism config to configure the N-d distributed
                        training.
  --use_megatron_lm     Whether to use Megatron-LM.

Distributed GPUs:
  Arguments related to distributed GPU training.

  --gpu_ids GPU_IDS     What GPUs (by id) should be used for training on this machine as a comma-
                        separated list
  --same_network        Whether all machines used for multinode training exist on the same local
                        network.
  --machine_rank MACHINE_RANK
                        The rank of the machine on which this script is launched.
  --main_process_ip MAIN_PROCESS_IP
                        The IP address of the machine of rank 0.
  --main_process_port MAIN_PROCESS_PORT
                        The port to use to communicate with the machine of rank 0.
  -t, --tee TEE         Tee std streams into a log file and also to console.
  --log_dir LOG_DIR     Base directory to use for log files when using
                        torchrun/torch.distributed.run as launcher. Use with --tee to redirect std
                        streams info log files.
  --role ROLE           User-defined role for the workers.
  --rdzv_backend RDZV_BACKEND
                        The rendezvous method to use, such as 'static' (the default) or 'c10d'
  --rdzv_conf RDZV_CONF
                        Additional rendezvous configuration (<key1>=<value1>,<key2>=<value2>,...).
  --max_restarts MAX_RESTARTS
                        Maximum number of worker group restarts before failing.
  --monitor_interval MONITOR_INTERVAL
                        Interval, in seconds, to monitor the state of workers.

TPU:
  Arguments related to TPU.

  --tpu_cluster         Whether to use a GCP TPU pod for training.
  --no_tpu_cluster      Should not be passed explicitly, this is for internal use only.
  --tpu_use_sudo        Whether to use `sudo` when running the TPU training script in each pod.
  --vm VM               List of single Compute VM instance names. If not provided we assume usage
                        of instance groups. For TPU pods.
  --env ENV             List of environment variables to set on the Compute VM instances. For TPU
                        pods.
  --main_training_function MAIN_TRAINING_FUNCTION
                        The name of the main function to be executed in your script (only for TPU
                        training).
  --downcast_bf16       Whether when using bf16 precision on TPUs if both float and double tensors
                        are cast to bfloat16 or if double tensors remain as float32.

DeepSpeed Arguments:
  Arguments related to DeepSpeed.

  --deepspeed_config_file DEEPSPEED_CONFIG_FILE
                        DeepSpeed config file.
  --zero_stage ZERO_STAGE
                        DeepSpeed's ZeRO optimization stage (useful only when `use_deepspeed` flag
                        is passed). If unspecified, will default to `2`.
  --offload_optimizer_device OFFLOAD_OPTIMIZER_DEVICE
                        Decides where (none|cpu|nvme) to offload optimizer states (useful only
                        when `use_deepspeed` flag is passed). If unspecified, will default to
                        'none'.
  --offload_param_device OFFLOAD_PARAM_DEVICE
                        Decides where (none|cpu|nvme) to offload parameters (useful only when
                        `use_deepspeed` flag is passed). If unspecified, will default to 'none'.
  --offload_optimizer_nvme_path OFFLOAD_OPTIMIZER_NVME_PATH
                        Decides Nvme Path to offload optimizer states (useful only when
                        `use_deepspeed` flag is passed). If unspecified, will default to 'none'.
  --offload_param_nvme_path OFFLOAD_PARAM_NVME_PATH
                        Decides Nvme Path to offload parameters (useful only when `use_deepspeed`
                        flag is passed). If unspecified, will default to 'none'.
  --gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS
                        No of gradient_accumulation_steps used in your training script (useful
                        only when `use_deepspeed` flag is passed). If unspecified, will default to
                        `1`.
  --gradient_clipping GRADIENT_CLIPPING
                        gradient clipping value used in your training script (useful only when
                        `use_deepspeed` flag is passed). If unspecified, will default to `1.0`.
  --zero3_init_flag ZERO3_INIT_FLAG
                        Decides Whether (true|false) to enable `deepspeed.zero.Init` for
                        constructing massive models. Only applicable with DeepSpeed ZeRO Stage-3.
                        If unspecified, will default to `true`.
  --zero3_save_16bit_model ZERO3_SAVE_16BIT_MODEL
                        Decides Whether (true|false) to save 16-bit model weights when using ZeRO
                        Stage-3. Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will
                        default to `false`.
  --deepspeed_hostfile DEEPSPEED_HOSTFILE
                        DeepSpeed hostfile for configuring multi-node compute resources.
  --deepspeed_exclusion_filter DEEPSPEED_EXCLUSION_FILTER
                        DeepSpeed exclusion filter string when using multi-node setup.
  --deepspeed_inclusion_filter DEEPSPEED_INCLUSION_FILTER
                        DeepSpeed inclusion filter string when using multi-node setup.
  --deepspeed_multinode_launcher DEEPSPEED_MULTINODE_LAUNCHER
                        DeepSpeed multi-node launcher to use, e.g. `pdsh`, `standard`, `openmpi`,
                        `mvapich`, `mpich`, `slurm`, `nossh` (requires DeepSpeed >= 0.14.5). If
                        unspecified, will default to `pdsh`.
  --deepspeed_moe_layer_cls_names DEEPSPEED_MOE_LAYER_CLS_NAMES
                        comma-separated list of transformer MoE layer class names (case-sensitive)
                        to wrap ,e.g, `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`,
                        `JetMoEAttention,JetMoEBlock` ... (useful only when `use_deepspeed` flag
                        is passed).

FSDP Arguments:
  Arguments related to Fully Shared Data Parallelism.

  --fsdp_version {1,2}  FSDP version to use. (useful only when `use_fsdp` flag is passed).
  --fsdp_offload_params FSDP_OFFLOAD_PARAMS
                        Decides Whether (true|false) to offload parameters and gradients to CPU.
                        (useful only when `use_fsdp` flag is passed).
  --fsdp_min_num_params FSDP_MIN_NUM_PARAMS
                        FSDP's minimum number of parameters for Default Auto Wrapping. (useful
                        only when `use_fsdp` flag is passed).
  --fsdp_sharding_strategy FSDP_SHARDING_STRATEGY
                        FSDP's sharding strategy. (useful only when `use_fsdp` flag is passed and
                        `fsdp_version=1`).
  --fsdp_reshard_after_forward FSDP_RESHARD_AFTER_FORWARD
                        FSDP's Reshard After Forward Strategy. (useful only when `use_fsdp` flag
                        is passed). Supports either boolean (FSDP2) or `FULL_SHARD | SHARD_GRAD_OP
                        | NO_RESHARD` (FSDP1).
  --fsdp_auto_wrap_policy FSDP_AUTO_WRAP_POLICY
                        FSDP's auto wrap policy. (useful only when `use_fsdp` flag is passed).
  --fsdp_transformer_layer_cls_to_wrap FSDP_TRANSFORMER_LAYER_CLS_TO_WRAP
                        Transformer layer class name (case-sensitive) to wrap ,e.g, `BertLayer`,
                        `GPTJBlock`, `T5Block` .... (useful only when `use_fsdp` flag is passed).
  --fsdp_backward_prefetch FSDP_BACKWARD_PREFETCH
                        FSDP's backward prefetch policy. (useful only when `use_fsdp` flag is
                        passed).
  --fsdp_state_dict_type FSDP_STATE_DICT_TYPE
                        FSDP's state dict type. (useful only when `use_fsdp` flag is passed).
  --fsdp_forward_prefetch FSDP_FORWARD_PREFETCH
                        If True, then FSDP explicitly prefetches the next upcoming all-gather
                        while executing in the forward pass (useful only when `use_fsdp` flag is
                        passed).
  --fsdp_use_orig_params FSDP_USE_ORIG_PARAMS
                        If True, allows non-uniform `requires_grad` during init, which means
                        support for interspersed frozen and trainable parameters. (useful only
                        when `use_fsdp` flag is passed).
  --fsdp_cpu_ram_efficient_loading FSDP_CPU_RAM_EFFICIENT_LOADING
                        If True, only the first process loads the pretrained model checkoint while
                        all other processes have empty weights. Only applicable for 🤗
                        Transformers. When using this, `--fsdp_sync_module_states` needs to True.
                        (useful only when `use_fsdp` flag is passed).
  --fsdp_sync_module_states FSDP_SYNC_MODULE_STATES
                        If True, each individually wrapped FSDP unit will broadcast module
                        parameters from rank 0. (useful only when `use_fsdp` flag is passed).
  --fsdp_activation_checkpointing FSDP_ACTIVATION_CHECKPOINTING
                        Decides Whether (true|false) intermediate activations are freed during the
                        forward pass, and a checkpoint is left as a placeholder. (useful only when
                        `use_fsdp` flag is passed).

Megatron-LM Arguments:
  Arguments related to Megatron-LM.

  --megatron_lm_tp_degree MEGATRON_LM_TP_DEGREE
                        Megatron-LM's Tensor Parallelism (TP) degree. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_use_custom_fsdp MEGATRON_LM_USE_CUSTOM_FSDP
                        Whether to use custom FSDP. (useful only when `use_megatron_lm` flag is
                        passed).
  --megatron_lm_no_load_optim MEGATRON_LM_NO_LOAD_OPTIM
                        Whether to not load optimizer. (useful only when `use_megatron_lm` flag is
                        passed).
  --megatron_lm_eod_mask_loss MEGATRON_LM_EOD_MASK_LOSS
                        Whether to use eod mask loss. (useful only when `use_megatron_lm` flag is
                        passed).
  --megatron_lm_overlap_cpu_optimizer_d2h_h2d MEGATRON_LM_OVERLAP_CPU_OPTIMIZER_D2H_H2D
                        Whether to overlap CPU optimizer step, gradients D2H and updated
                        parameters H2D. (useful only when `use_megatron_lm` flag is passed).
  --megatron_lm_no_save_optim MEGATRON_LM_NO_SAVE_OPTIM
                        Whether to not save optimizer. (useful only when `use_megatron_lm` flag is
                        passed).
  --megatron_lm_optimizer_cpu_offload MEGATRON_LM_OPTIMIZER_CPU_OFFLOAD
                        Whether to use CPU offload for optimizer. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_use_precision_aware_optimizer MEGATRON_LM_USE_PRECISION_AWARE_OPTIMIZER
                        Whether to use precision aware optimizer. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_decoder_last_pipeline_num_layers MEGATRON_LM_DECODER_LAST_PIPELINE_NUM_LAYERS
                        Megatron-LM's decoder last pipeline number of layers, default None is even
                        split of transformer layers across all pipeline stages.
  --megatron_lm_pp_degree MEGATRON_LM_PP_DEGREE
                        Megatron-LM's Pipeline Parallelism (PP) degree. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_num_micro_batches MEGATRON_LM_NUM_MICRO_BATCHES
                        Megatron-LM's number of micro batches when PP degree > 1. (useful only
                        when `use_megatron_lm` flag is passed).
  --megatron_lm_sequence_parallelism MEGATRON_LM_SEQUENCE_PARALLELISM
                        Decides Whether (true|false) to enable Sequence Parallelism when TP degree
                        > 1. (useful only when `use_megatron_lm` flag is passed).
  --megatron_lm_recompute_activations MEGATRON_LM_RECOMPUTE_ACTIVATIONS
                        Decides Whether (true|false) to enable Selective Activation Recomputation.
                        (useful only when `use_megatron_lm` flag is passed).
  --megatron_lm_use_distributed_optimizer MEGATRON_LM_USE_DISTRIBUTED_OPTIMIZER
                        Decides Whether (true|false) to use distributed optimizer which shards
                        optimizer state and gradients across Data Pralellel (DP) ranks. (useful
                        only when `use_megatron_lm` flag is passed).
  --megatron_lm_gradient_clipping MEGATRON_LM_GRADIENT_CLIPPING
                        Megatron-LM's gradient clipping value based on global L2 Norm (0 to
                        disable). (useful only when `use_megatron_lm` flag is passed).
  --megatron_lm_recompute_granularity MEGATRON_LM_RECOMPUTE_GRANULARITY
                        Megatron-LM's recompute granularity (full, selective). (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_recompute_method MEGATRON_LM_RECOMPUTE_METHOD
                        Megatron-LM's recompute method (uniform, block). (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_recompute_num_layers MEGATRON_LM_RECOMPUTE_NUM_LAYERS
                        Megatron-LM's number of layers to recompute. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_attention_backend MEGATRON_LM_ATTENTION_BACKEND
                        Decides Whether (true|false) to enable attention backend. (useful only
                        when `use_megatron_lm` flag is passed).
  --megatron_lm_expert_model_parallel_size MEGATRON_LM_EXPERT_MODEL_PARALLEL_SIZE
                        Megatron-LM's expert model parallel size. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_context_parallel_size MEGATRON_LM_CONTEXT_PARALLEL_SIZE
                        Megatron-LM's context parallel size. (useful only when `use_megatron_lm`
                        flag is passed).
  --megatron_lm_attention_dropout MEGATRON_LM_ATTENTION_DROPOUT
                        Megatron-LM's attention dropout rate. (useful only when `use_megatron_lm`
                        flag is passed).
  --megatron_lm_hidden_dropout MEGATRON_LM_HIDDEN_DROPOUT
                        Megatron-LM's hidden dropout rate. (useful only when `use_megatron_lm`
                        flag is passed).
  --megatron_lm_attention_softmax_in_fp32 MEGATRON_LM_ATTENTION_SOFTMAX_IN_FP32
                        Decides Whether (true|false) to use fp32 for attention softmax. (useful
                        only when `use_megatron_lm` flag is passed).
  --megatron_lm_expert_tensor_parallel_size MEGATRON_LM_EXPERT_TENSOR_PARALLEL_SIZE
                        Megatron-LM's expert tensor parallel size. (useful only when
                        `use_megatron_lm` flag is passed).
  --megatron_lm_calculate_per_token_loss MEGATRON_LM_CALCULATE_PER_TOKEN_LOSS
                        Decides Whether (true|false) to calculate per token loss. (useful only
                        when `use_megatron_lm` flag is passed).
  --megatron_lm_use_rotary_position_embeddings MEGATRON_LM_USE_ROTARY_POSITION_EMBEDDINGS
                        Decides Whether (true|false) to use rotary position embeddings. (useful
                        only when `use_megatron_lm` flag is passed).

FP8 Arguments:
  Arguments related to FP8 training (requires `--mixed_precision=fp8`)

  --fp8_backend {ao,te,msamp}
                        Choose a backend to train with FP8 (ao: torchao, te: TransformerEngine,
                        msamp: MS-AMP)
  --fp8_use_autocast_during_eval
                        Whether to use FP8 autocast during eval mode (useful only when
                        `--fp8_backend=te` is passed). Generally better metrics are found when
                        this is not passed.
  --fp8_margin FP8_MARGIN
                        The margin to use for the gradient scaling (useful only when
                        `--fp8_backend=te` is passed).
  --fp8_interval FP8_INTERVAL
                        The interval to use for how often the scaling factor is recomputed (useful
                        only when `--fp8_backend=te` is passed).
  --fp8_format {HYBRID,E4M3,E5M2}
                        The format to use for the FP8 recipe (useful only when `--fp8_backend=te`
                        is passed).
  --fp8_amax_history_len FP8_AMAX_HISTORY_LEN
                        The length of the history to use for the scaling factor computation
                        (useful only when `--fp8_backend=te` is passed).
  --fp8_amax_compute_algo {max,most_recent}
                        The algorithm to use for the scaling factor computation. (useful only when
                        `--fp8_backend=te` is passed).
  --fp8_override_linear_precision FP8_OVERRIDE_LINEAR_PRECISION
                        Whether or not to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher
                        precision. Should be passed in a comma-separated string of booleans
                        (useful only when `--fp8_backend=te` is passed).
  --fp8_opt_level {O1,O2}
                        What level of 8-bit collective communication should be used with MS-AMP
                        (useful only when `--fp8_backend=msamp` is passed).
  --fp8_enable_fsdp_float8_all_gather FP8_ENABLE_FSDP_FLOAT8_ALL_GATHER
                        Whether to enable FSDP2 float8 all gather (useful only when
                        `--fp8_backend=ao` is passed).
  --fp8_pad_inner_dim FP8_PAD_INNER_DIM
                        Whether to pad the inner dimension for FP8 GEMMs (useful only when
                        `--fp8_backend=ao` is passed).

AWS Arguments:
  Arguments related to AWS.

  --aws_access_key_id AWS_ACCESS_KEY_ID
                        The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job
  --aws_secret_access_key AWS_SECRET_ACCESS_KEY
                        The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training
                        job.

MPI Arguments:
  Arguments related to mpirun for Multi-CPU

  --mpirun_hostfile MPIRUN_HOSTFILE
                        Location for a hostfile for using Accelerate to launch a multi-CPU
                        training job with mpirun. This will get passed to the MPI --hostfile or -f
                        parameter, depending on which MPI program is installed.

ParallelismConfig Arguments:
  Arguments related to the ParallelismConfig used for distributed training.

  --parallelism_config_dp_replicate_size PARALLELISM_CONFIG_DP_REPLICATE_SIZE
                        The number of processes for data parallel training. Defaults to 1 (no data
                        parallelism).
  --parallelism_config_dp_shard_size PARALLELISM_CONFIG_DP_SHARD_SIZE
                        The number of processes for FSDP sharding. Defaults to 1 (No FSDP
                        sharding).
  --parallelism_config_tp_size PARALLELISM_CONFIG_TP_SIZE
                        The number of processes for tensor parallel training. Defaults to 1 (no
                        tensor parallelism).
  --parallelism_config_cp_size PARALLELISM_CONFIG_CP_SIZE
                        The number of processese for context parallel training. Defaults to 1 (no
                        context parallelism).
  --parallelism_config_cp_backend {torch}
                        Context Parallelism backend: torch (FSDP2) or deepspeed (ALST/Ulysses)
  --parallelism_config_cp_comm_strategy PARALLELISM_CONFIG_CP_COMM_STRATEGY
                        The communication strategy for context parallel training. Defaults to
                        'allgather'. Other option is alltoall
  --parallelism_config_sp_size PARALLELISM_CONFIG_SP_SIZE
                        The number of processese for context parallel training. Defaults to 1 (no
                        context parallelism).
  --parallelism_config_sp_backend {deepspeed}
                        Sequence Parallelism backend: deepspeed (ALST/Ulysses)
  --parallelism_config_sp_seq_length PARALLELISM_CONFIG_SP_SEQ_LENGTH
                        Sequence length for when batches are all of the same length. For variable
                        sequence lengths across batches set
                        `parallelism_config_sp_seq_length_is_variable=True`
  --parallelism_config_sp_seq_length_is_variable PARALLELISM_CONFIG_SP_SEQ_LENGTH_IS_VARIABLE
                        If `True` will work with a sequence length that may change between
                        batches, in which case `parallelism_config_sp_seq_length` value can be set
                        to anything divisible by sp size or remain unset. If `False` then
                        `parallelism_config_sp_seq_length` needs to match the batch's sequence
                        length dimension. The default is `True`.
  --parallelism_config_sp_attn_implementation PARALLELISM_CONFIG_SP_ATTN_IMPLEMENTATION
                        Attention implementation to use. Can be one of 'flash_attention_2',
                        'flash_attention_3' or 'sdpa'. Defaults to `sdpa`.