/opt/chainguard/nemo-rl/venvs/mcore/bin/ft_launcher

usage: ft_launcher [-h] [--nnodes NNODES] [--rdzv-backend RDZV_BACKEND]
                   [--rdzv-endpoint RDZV_ENDPOINT] [--rdzv-id RDZV_ID] [--rdzv-conf RDZV_CONF]
                   [--local-addr LOCAL_ADDR] [--ft-segment FT_SEGMENT] [--standalone]
                   [--nproc-per-node NPROC_PER_NODE] [--max-restarts MAX_RESTARTS]
                   [--term-timeout TERM_TIMEOUT] [--workers-stop-timeout WORKERS_STOP_TIMEOUT]
                   [--monitor-interval MONITOR_INTERVAL] [--start-method {spawn,fork,forkserver}]
                   [--role ROLE] [-m] [--no-python] [--run-path] [--log-dir LOG_DIR]
                   [--ft-per-cycle-applog-prefix FT_PER_CYCLE_APPLOG_PREFIX]
                   [--ft-nvrx-logfile FT_NVRX_LOGFILE]
                   [--ft-enable-log-server FT_ENABLE_LOG_SERVER]
                   [--ft-log-server-port FT_LOG_SERVER_PORT]
                   [--ft-log-aggregator-count FT_LOG_AGGREGATOR_COUNT]
                   [--ft-log-leaf-max-queue-chunks FT_LOG_LEAF_MAX_QUEUE_CHUNKS]
                   [--ft-log-server-log-prefix FT_LOG_SERVER_LOG_PREFIX]
                   [--ft-log-server-graceful-shutdown-timeout FT_LOG_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT]
                   [-r REDIRECTS] [-t TEE] [--local-ranks-filter LOCAL_RANKS_FILTER]
                   [--node-rank NODE_RANK] [--master-addr MASTER_ADDR] [--master-port MASTER_PORT]
                   [--logs-specs LOGS_SPECS] [--ft-cfg-path FT_CFG_PATH]
                   [--ft-workload-check-interval FT_WORKLOAD_CHECK_INTERVAL]
                   [--ft-initial-rank-heartbeat-timeout FT_INITIAL_RANK_HEARTBEAT_TIMEOUT]
                   [--ft-rank-heartbeat-timeout FT_RANK_HEARTBEAT_TIMEOUT]
                   [--ft-node-health-check-interval FT_NODE_HEALTH_CHECK_INTERVAL]
                   [--ft-node-health-check-endpoint FT_NODE_HEALTH_CHECK_ENDPOINT]
                   [--ft-safety-factor FT_SAFETY_FACTOR]
                   [--ft-rank-termination-signal FT_RANK_TERMINATION_SIGNAL]
                   [--ft-log-level FT_LOG_LEVEL]
                   [--ft-rank-out-of-section-timeout FT_RANK_OUT_OF_SECTION_TIMEOUT]
                   [--ft-rank-section-timeouts FT_RANK_SECTION_TIMEOUTS]
                   [--ft-restart-check-interval FT_RESTART_CHECK_INTERVAL]
                   [--ft-restart-policy {any-failed}]
                   [--ft-enable-nic-monitor FT_ENABLE_NIC_MONITOR]
                   [--ft-enable-nic-healthcheck FT_ENABLE_NIC_HEALTHCHECK]
                   [--ft-pci-topo-file FT_PCI_TOPO_FILE]
                   [--ft-link-down-path-template FT_LINK_DOWN_PATH_TEMPLATE]
                   [--ft-link-state-path-template FT_LINK_STATE_PATH_TEMPLATE]
                   [--ft-enable-dist-storage-healthcheck FT_ENABLE_DIST_STORAGE_HEALTHCHECK]
                   [--ft-storage-health-check-path FT_STORAGE_HEALTHCHECK_PATH]
                   [--ft-skip-section-response FT_SKIP_SECTION_RESPONSE]
                   [--ft-rdzv-impl {barrier,legacy}] [--ft-numa-bind-strict FT_NUMA_BIND_STRICT]
                   [--ft-gpu-memory-reclaim-timeout FT_GPU_MEMORY_RECLAIM_TIMEOUT]
                   [--ft-gpu-memory-tolerance-mb FT_GPU_MEMORY_TOLERANCE_MB]
                   [--ft-gpu-memory-poll-interval FT_GPU_MEMORY_POLL_INTERVAL]
                   [--ft-check-remaining-processes FT_CHECK_REMAINING_PROCESSES]
                   [--ft-install-exception-hook FT_INSTALL_EXCEPTION_HOOK]
                   [--ft-attribution-endpoint FT_ATTRIBUTION_ENDPOINT]
                   [--ft-attribution-startup-timeout FT_ATTRIBUTION_STARTUP_TIMEOUT]
                   [--ft-attribution-llm-api-key-file FT_ATTRIBUTION_LLM_API_KEY_FILE]
                   [--ft-attribution-llm-base-url FT_ATTRIBUTION_LLM_BASE_URL]
                   [--ft-attribution-llm-model FT_ATTRIBUTION_LLM_MODEL]
                   [--ft-attribution-analysis-backend {mcp,lib}]
                   [--ft-attribution-compute-timeout FT_ATTRIBUTION_COMPUTE_TIMEOUT]
                   [--ft-attribution-log-level {DEBUG,INFO,WARNING}]
                   [--ft-cycle-info-dir FT_CYCLE_INFO_DIR]
                   [--ft-max-no-progress-cycles FT_MAX_NO_PROGRESS_CYCLES]
                   [--ft-min-progress-iterations FT_MIN_PROGRESS_ITERATIONS]
                   [--ft-checkpoint-iteration-file FT_CHECKPOINT_ITERATION_FILE]
                    training_script ...

Torch Distributed Elastic Training Launcher

positional arguments:
  ft_ignore_missing_cfg
                        Do not raise an error if there is no Fault Tolerance pkg config provided,
                        just use default settings.
  training_script       Full path to the (single GPU) training program/script to be launched in
                        parallel, followed by all the arguments for the training script.
  training_script_args

options:
  -h, --help            show this help message and exit
  --nnodes NNODES       Number of nodes, or the range of nodes in form
                        <minimum_nodes>:<maximum_nodes>.
  --rdzv-backend, --rdzv_backend RDZV_BACKEND
                        Rendezvous backend. Currently only c10d is supported.
  --rdzv-endpoint, --rdzv_endpoint RDZV_ENDPOINT
                        Rendezvous backend endpoint; usually in form <host>:<port>.
  --rdzv-id, --rdzv_id RDZV_ID
                        User-defined group id.
  --rdzv-conf, --rdzv_conf RDZV_CONF
                        Additional rendezvous configuration (<key1>=<value1>,<key2>=<value2>,...).
  --local-addr, --local_addr LOCAL_ADDR
                        Address of the local node. If specified, will use the given address for
                        connection. Else, will look up the local node address instead. Else, it
                        will be default to local machine's FQDN.
  --ft-segment, --ft_segment FT_SEGMENT
                        Controls hot spare node behavior and segment-aware rank assignment.
                        Default: None (simple hot spare mode for H100 and non-NVSwitch systems, no
                        ClusterUUID required). When set to N: Enables segment-aware mode for
                        NVSwitch systems (DGX H200, HGX B200). Specifies minimum nodes per NVLink
                        domain (identified by GPU ClusterUUID via nvidia-smi). Domains with fewer
                        than N nodes are excluded. From valid domains, complete segments are
                        selected. min_nodes must be divisible by segment. Note: segment=None
                        (default) is suitable for H100; segment=1 is similar but requires
                        ClusterUUID.
  --standalone          Start a local standalone rendezvous backend that is represented by a C10d
                        TCP store on a free port. Useful when launching single-node, multi-worker
                        job. If specified --rdzv-backend, --rdzv-endpoint, --rdzv-id are auto-
                        assigned and any explicitly set values are ignored.
  --nproc-per-node, --nproc_per_node NPROC_PER_NODE
                        Number of workers per node; supported values: [auto, cpu, gpu, int].
  --max-restarts, --max_restarts MAX_RESTARTS
                        Maximum number of worker group restarts before failing.
  --term-timeout, --term_timeout TERM_TIMEOUT
                        Interval, in seconds, between initial SIGTERM and rank termination with
                        SIGKILL, when the launcher forwards a received signal to ranks.
  --workers-stop-timeout, --workers_stop_timeout WORKERS_STOP_TIMEOUT
                        Interval, in seconds, between initial SIGTERM and rank termination with
                        SIGKILL, when the launcher stops its ranks in order to restart them.
  --monitor-interval, --monitor_interval MONITOR_INTERVAL
                        Interval, in seconds, to monitor the state of workers.
  --start-method, --start_method {spawn,fork,forkserver}
                        Multiprocessing start method to use when creating workers.
  --role ROLE           User-defined role for the workers.
  -m, --module          Change each process to interpret the launch script as a Python module,
                        executing with the same behavior as 'python -m'.
  --no-python, --no_python
                        Skip prepending the training script with 'python' - just execute it
                        directly. Useful when the script is not a Python script.
  --run-path, --run_path
                        Run the training script with runpy.run_path in the same interpreter.
                        Script must be provided as an abs path (e.g. /abs/path/script.py). Takes
                        precedence over --no-python.
  --log-dir, --log_dir LOG_DIR
                        Base directory to use for log files (e.g. /var/log/torch/elastic). The
                        same directory is re-used for multiple runs (a unique job-level sub-
                        directory is created with rdzv_id as the prefix).
  --ft-per-cycle-applog-prefix, --ft_per_cycle_applog_prefix FT_PER_CYCLE_APPLOG_PREFIX
                        Prefix for per-cycle application log files (must be absolute path, e.g.
                        /lustre/logs/job_12345.log). Creates training worker logs per cycle:
                        /lustre/logs/job_12345_cycle0.log, job_12345_cycle1.log, etc. All
                        ranks/nodes capture logs with automatic rank prefixes (like 'srun -l').
                        Without --ft-enable-log-server: Each node writes directly to Lustre with
                        O_APPEND (safe concurrent writes). With --ft-enable-log-server
                        (recommended): All nodes stream logs to gRPC server on rank 0, which
                        becomes the single Lustre writer. gRPC mode eliminates Lustre lock
                        contention and scales to 1000+ nodes. Note: NVRx launcher logs go to
                        stdout/stderr by default unless --ft-nvrx-logfile is specified.
  --ft-nvrx-logfile, --ft_nvrx_logfile FT_NVRX_LOGFILE
                        Optional: Path for NVRx process logs (must be absolute path if specified).
                        ft_launcher routes these logs through the same infrastructure as worker
                        logs. nvrx-control also uses this path as a fallback source for gRPC
                        server diagnostic logs. Example: --ft-nvrx-logfile
                        /lustre/logs/launcher.log
  --ft-enable-log-server, --ft_enable_log_server FT_ENABLE_LOG_SERVER
                        Enable gRPC-based log funneling to a single Lustre file. When enabled with
                        --ft-per-cycle-applog-prefix, NVRx automatically: (1) Starts gRPC log
                        server(s) on the TCPStore host, (2) Configures all nodes as gRPC clients
                        to a first-level aggregator port, (3) Root server writes to Lustre (single
                        writer). With --ft-log-aggregator-count=N>1, N leaf processes plus one
                        root use ports P..P+N (see --ft-log-server-port). Use --ft-log-aggregator-
                        count=1 for a single server on P only. Benefits: No Lustre lock
                        contention, scales to 1500+ nodes. Requires: grpcio. Default: False unless
                        auto-enabled by another log-routing option such as --ft-nvrx-logfile.
  --ft-log-server-port, --ft_log_server_port FT_LOG_SERVER_PORT
                        Port for gRPC log funnel server (only used if --ft-enable-log-server is
                        enabled). Default: 50051
  --ft-log-aggregator-count, --ft_log_aggregator_count FT_LOG_AGGREGATOR_COUNT
                        Number of first-level gRPC log aggregators. 0 (default): auto - single
                        root server for up to 1536 nodes, adds leaf servers beyond that (one per
                        1536 nodes). 1: single root server (legacy). N>1: N leaf servers on ports
                        [P, P+1, ..., P+N-1] and one root on P+N (P = --ft-log-server-port). Open
                        TCP ports P through P+N on the TCP store host. Each node picks a leaf from
                        SLURM array/procid when set, else from hostname.
  --ft-log-leaf-max-queue-chunks, --ft_log_leaf_max_queue_chunks FT_LOG_LEAF_MAX_QUEUE_CHUNKS
                        Max queued log chunks per leaf (only when N>1). Default -1: auto scale
                        with fan-in, max(16384, min(1000000, per_leaf*256)).
  --ft-log-server-log-prefix, --ft_log_server_log_prefix FT_LOG_SERVER_LOG_PREFIX
                        Prefix for gRPC log server process logs (only used if --ft-enable-log-
                        server is enabled). Root and leaf process logs are written to
                        '<prefix>_root.log' and '<prefix>_leaf_<N>.log'. If not specified, derives
                        from --ft-nvrx-logfile first, then --ft-per-cycle-applog-prefix by
                        stripping a trailing '.log' and appending '_grpc'.
  --ft-log-server-graceful-shutdown-timeout, --ft_log_server_graceful_shutdown_timeout FT_LOG_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT
                        Maximum seconds to wait for clients during gRPC server graceful shutdown
                        (only used if --ft-enable-log-server is enabled). When store host exits,
                        server will continue accepting logs from other ranks for up to this
                        timeout or until all clients disconnect, whichever comes first. This
                        ensures other nodes can flush their final logs. When --ft-log-aggregator-
                        count>1, each leaf uses this timeout; the root log server uses 2x so
                        leaves can finish waiting for downstream clients and drain to root first.
                        Default: 60.0
  -r, --redirects REDIRECTS
                        Redirect std streams into a log file in the log directory (e.g. [-r 3]
                        redirects both stdout+stderr for all workers, [-r 0:1,1:2] redirects
                        stdout for local rank 0 and stderr for local rank 1).
  -t, --tee TEE         Tee std streams into a log file and also to console (see --redirects for
                        format).
  --local-ranks-filter, --local_ranks_filter LOCAL_RANKS_FILTER
                        Only show logs from specified ranks in console (e.g.
                        [--local_ranks_filter=0,1,2] will only show logs from rank 0, 1 and 2).
                        This will only apply to stdout and stderr, not tolog files saved via
                        --redirect or --tee
  --node-rank, --node_rank NODE_RANK
                        Rank of the node for multi-node distributed training.
  --master-addr, --master_addr MASTER_ADDR
                        Address of the master node (rank 0) that is used for static and c10d
                        rendezvous backends when rdzv_endpoint is not specified. It should be
                        either the IP address or the hostname of rank 0. For single node multi-
                        proc training the --master-addr can simply be 127.0.0.1; IPv6 should have
                        the pattern `[0:0:0:0:0:0:0:1]`.
  --master-port, --master_port MASTER_PORT
                        Port on the master node (rank 0) to be used for communication during
                        distributed training. It is used for static and c10d rendezvous backends
                        when rdzv_endpoint is not specified.
  --logs-specs, --logs_specs LOGS_SPECS
                        Logging behavior configuration. Options: (1) None (default): Creates
                        separate log files per rank per restart cycle. (2) Custom entrypoint name
                        from torchrun.logs_specs group for advanced customization. Note: For
                        consolidated logging, use --ft-per-cycle-applog-prefix instead
                        (automatically uses PipeBasedLogsSpecs).
  --ft-cfg-path, --ft-cfg_path FT_CFG_PATH
                        Path to a YAML file that contains Fault Tolerance pkg config
                        (`fault_tolerance` section). NOTE: config items from the file can be
                        overwritten by `--ft-param-*` args.
  --ft-workload-check-interval, --ft-workload_check_interval FT_WORKLOAD_CHECK_INTERVAL
                        Part of Fault Tolerance pkg config (workload_check_interval). Use
                        'null'|'none'|'' for None.
  --ft-initial-rank-heartbeat-timeout, --ft-initial_rank_heartbeat_timeout FT_INITIAL_RANK_HEARTBEAT_TIMEOUT
                        Part of Fault Tolerance pkg config (initial_rank_heartbeat_timeout). Use
                        'null'|'none'|'' for None.
  --ft-rank-heartbeat-timeout, --ft-rank_heartbeat_timeout FT_RANK_HEARTBEAT_TIMEOUT
                        Part of Fault Tolerance pkg config (rank_heartbeat_timeout). Use
                        'null'|'none'|'' for None.
  --ft-node-health-check-interval, --ft-node_health_check_interval FT_NODE_HEALTH_CHECK_INTERVAL
                        Part of Fault Tolerance pkg config (node_health_check_interval). Use
                        'null'|'none'|'' for None.
  --ft-node-health-check-endpoint, --ft-node_health_check_endpoint FT_NODE_HEALTH_CHECK_ENDPOINT
                        UDS endpoint for the node health check service used by InJob. Examples:
                        /var/run/nvhcd.sock, unix:///var/run/nvhcd.sock.
  --ft-safety-factor, --ft-safety_factor FT_SAFETY_FACTOR
                        Part of Fault Tolerance pkg config (safety_factor). Use 'null'|'none'|''
                        for None.
  --ft-rank-termination-signal, --ft-rank_termination_signal FT_RANK_TERMINATION_SIGNAL
                        Part of Fault Tolerance pkg config (rank_termination_signal).
  --ft-log-level, --ft-log_level FT_LOG_LEVEL
                        Part of Fault Tolerance pkg config (log_level).
  --ft-rank-out-of-section-timeout, --ft-rank_out_of_section_timeout FT_RANK_OUT_OF_SECTION_TIMEOUT
                        Part of Fault Tolerance pkg config (rank_out_of_section_timeout). Use
                        'null'|'none'|'' for None.
  --ft-rank-section-timeouts, --ft-rank_section_timeouts FT_RANK_SECTION_TIMEOUTS
                        Part of Fault Tolerance pkg config (rank_section_timeouts). Expected
                        format: name1:value1,name2:value2,... Use 'null'|'none'|'' for None
  --ft-restart-check-interval, --ft-restart_check_interval FT_RESTART_CHECK_INTERVAL
                        Part of Fault Tolerance pkg config (restart_check_interval). Use
                        'null'|'none'|'' for None.
  --ft-restart-policy, --ft-restart_policy {any-failed}
                        (Deprecated.) Worker groups restarting policy. Only 'any-failed' is
                        supported: restart if any worker group fails. This option may be removed
                        in a future release.
  --ft-enable-nic-monitor, --ft-enable_nic_monitor FT_ENABLE_NIC_MONITOR
                        Enable or Disable NIC health monitoring in training. This monitors
                        link_downed counters periodically during training. Default: False.
  --ft-enable-nic-healthcheck, --ft-enable_nic_healthcheck FT_ENABLE_NIC_HEALTHCHECK
                        Enable or Disable NIC link state health check before rendezvous. This
                        checks if network interface ports (RDMA/InfiniBand and Ethernet) are in
                        ACTIVE state and fails if any port transitioned from ACTIVE to non-ACTIVE.
                        Unlike --ft-enable-nic-monitor (which periodically monitors link_downed
                        counters), this performs a one-time state check during rendezvous. Can be
                        used independently or together with --ft-enable-nic-monitor. Default:
                        True.
  --ft-pci-topo-file, --ft-pci_topo_file FT_PCI_TOPO_FILE
                        PCI topology file that describes GPU and NIC topology.
  --ft-link-down-path-template, --ft-link_down_path_template FT_LINK_DOWN_PATH_TEMPLATE
                        Part of Fault Tolerance pkg config (link_down_path_template). Template
                        path to check if a NIC link is down.
  --ft-link-state-path-template, --ft-link_state_path_template FT_LINK_STATE_PATH_TEMPLATE
                        Part of Fault Tolerance pkg config (link_state_path_template). Template
                        path to check NIC link state. Should contain {nic} placeholder. Default:
                        /sys/class/infiniband/{nic}/ports/1/state
  --ft-enable-dist-storage-healthcheck, --ft_enable_dist_storage_healthcheck FT_ENABLE_DIST_STORAGE_HEALTHCHECK
                        Enable distributed storage health check (Lustre + NFS) before rendezvous.
                        Accepts a boolean-like value only (e.g., 'true'/'false'). Example: --ft-
                        enable-dist-storage-healthcheck true
  --ft-storage-health-check-path, --ft_storage_health_check_path FT_STORAGE_HEALTHCHECK_PATH
                        Comma-separated list of absolute paths to validate for existence and
                        readability before rendezvous. Example: --ft-storage-health-check-path
                        '/data/a,/mnt/b'.
  --ft-skip-section-response, --ft-skip_section_response FT_SKIP_SECTION_RESPONSE
                        Part of Fault Tolerance pkg config (skip_section_response). If enabled
                        (default), section and heartbeat messages are sent without waiting for
                        server response, significantly reducing latency. Set to false during
                        development to catch programming errors immediately.
  --ft-rdzv-impl, --ft-rdzv_impl {barrier,legacy}
                        FT rendezvous implementation to use. 'barrier' uses the atomic barrier-
                        based algorithm (ft_rendezvous_barrier.py; default, recommended). 'legacy'
                        uses the original compare-and-set algorithm (_ft_rendezvous.py);
                        deprecated and will be removed in a future release—migrate to 'barrier'.
                        Note: This is independent of --rdzv-backend (which specifies the
                        coordination backend like c10d or etcd).
  --ft-numa-bind-strict, --ft-numa_bind_strict FT_NUMA_BIND_STRICT
                        Enable or disable strict NUMA binding with both CPU and memory bound to
                        the same NUMA node (--cpunodebind=N --membind=N). Set to false to use
                        --localalloc instead of --membind. This option only affects behavior when
                        NVRX_GPUS_PER_NUMA environment variable is set. Default: True. Example:
                        --ft-numa-bind-strict false
  --ft-gpu-memory-reclaim-timeout, --ft-gpu_memory_reclaim_timeout FT_GPU_MEMORY_RECLAIM_TIMEOUT
                        Part of Fault Tolerance pkg config (gpu_memory_reclaim_timeout). Timeout
                        (in seconds) to wait for GPU memory to be reclaimed after worker shutdown
                        before starting new workers. Default: 50.0. Use 'null'|'none'|'' for None.
  --ft-gpu-memory-tolerance-mb, --ft-gpu_memory_tolerance_mb FT_GPU_MEMORY_TOLERANCE_MB
                        Part of Fault Tolerance pkg config (gpu_memory_tolerance_mb). Maximum
                        allowed GPU memory usage (in MB) when checking if memory has been
                        reclaimed. Default: 512.0. Use 'null'|'none'|'' for None.
  --ft-gpu-memory-poll-interval, --ft-gpu_memory_poll_interval FT_GPU_MEMORY_POLL_INTERVAL
                        Part of Fault Tolerance pkg config (gpu_memory_poll_interval). Poll
                        interval (in seconds) for checking GPU memory during reclaim process.
                        Default: 2.0. Use 'null'|'none'|'' for None.
  --ft-check-remaining-processes, --ft-check_remaining_processes FT_CHECK_REMAINING_PROCESSES
                        Part of Fault Tolerance pkg config (check_remaining_processes). If
                        enabled, check for and log any remaining worker processes after
                        termination. Useful for debugging process cleanup issues. Default: False.
  --ft-install-exception-hook, --ft-install_exception_hook FT_INSTALL_EXCEPTION_HOOK
                        Part of Fault Tolerance pkg config (install_exception_hook). If enabled,
                        installs sys.excepthook to capture uncaught exceptions in training worker
                        processes, format and log the traceback, and use os._exit() to exit the
                        process reliably. Default: False.
  --ft-attribution-endpoint, --ft_attribution_endpoint FT_ATTRIBUTION_ENDPOINT
                        Endpoint for the attribution service. Default: disabled. Set to localhost
                        to let the TCPStore host process manage nvrx-attrsvc. Use an explicit
                        endpoint such as http://host:port, grpc://host:port, or
                        unix:///path/to/socket for an externally managed attribution service.
  --ft-attribution-startup-timeout, --ft_attribution_startup_timeout FT_ATTRIBUTION_STARTUP_TIMEOUT
                        Seconds to wait for launcher-managed attribution service /healthz
                        readiness. Default: 20.0.
  --ft-attribution-llm-api-key-file, --ft_attribution_llm_api_key_file FT_ATTRIBUTION_LLM_API_KEY_FILE
                        Path to the LLM API key file for launcher-managed attribution service. If
                        unset, LLM_API_KEY_FILE from the launcher environment is used.
  --ft-attribution-llm-base-url, --ft_attribution_llm_base_url FT_ATTRIBUTION_LLM_BASE_URL
                        LLM base URL for launcher-managed attribution service.
  --ft-attribution-llm-model, --ft_attribution_llm_model FT_ATTRIBUTION_LLM_MODEL
                        LLM model identifier for launcher-managed attribution service.
  --ft-attribution-analysis-backend, --ft_attribution_analysis_backend {mcp,lib}
                        Analysis backend for launcher-managed attribution service: mcp or lib.
  --ft-attribution-compute-timeout, --ft_attribution_compute_timeout FT_ATTRIBUTION_COMPUTE_TIMEOUT
                        Analysis compute timeout in seconds for launcher-managed attribution
                        service.
  --ft-attribution-log-level, --ft_attribution_log_level {DEBUG,INFO,WARNING}
                        Log level for launcher-managed attribution service.
  --ft-cycle-info-dir, --ft_cycle_info_dir FT_CYCLE_INFO_DIR
                        Full path to NVRx cycle info directory (e.g. <base>/nvrx/). The rendezvous
                        host writes cycle_info.<job_id>.<attempt>.<cycle> and symlink
                        cycle_info.<job_id>.current there. Workload receives current cycle file
                        path via NVRX_CURRENT_CYCLE_INFO env.
  --ft-max-no-progress-cycles, --ft-max_no_progress_cycles FT_MAX_NO_PROGRESS_CYCLES
                        Maximum consecutive cycles (including initial cycle 0) without progress
                        before early termination. E.g. 2 = allow cycle 0 and 1 with no progress,
                        then terminate before cycle 2. Progress tracking is enabled when this
                        value > 0. Set to 0 or -1 to disable. Default: 2.
  --ft-min-progress-iterations, --ft-min_progress_iterations FT_MIN_PROGRESS_ITERATIONS
                        Minimum iteration increase to consider a restart as making progress
                        (default: 1).
  --ft-checkpoint-iteration-file, --ft-checkpoint_iteration_file FT_CHECKPOINT_ITERATION_FILE
                        Path to file containing the latest checkpoint iteration (single integer).
                        Used for progress tracking. Example:
                        phase1/latest_checkpointed_iteration.txt