
    &`i                     @   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	m
Z
 d dlmZmZ d dlZd dlmZ d dlmZmZ d d	lmZmZ d d
lmZ  ed          e G d de                                  Z G d de          ZdededefdZe G d d                      ZdS )    N)	dataclass)OptionalSet)Coordinator)detect_nicsnics_to_env_var)secrettimeout)update_env_vars)WorkerWorkerGroup)BackendBackendConfig)	PublicAPIbeta)	stabilityc                       e Zd ZU dZdZeee                  ed<   dZ	e
ed<   dZee         ed<   dZee
         ed<   dZee         ed<   dZee         ed	<   d
Ze
ed<   dZe
ed<   ed             Zd Zed             ZdS )HorovodConfiga  Configurations for Horovod setup.

    See https://github.com/horovod/horovod/blob/master/horovod/runner/common/util/settings.py # noqa: E501

    Args:
        nics (Optional[Set[str]): Network interfaces that can be used for
            communication.
        verbose: Horovod logging verbosity.
        key (Optional[str]): Secret used for communication between workers.
        ssh_port (Optional[int]): Port for SSH server running on worker nodes.
        ssh_identity_file (Optional[str]): Path to the identity file to
            ssh into different hosts on the cluster.
        ssh_str (Optional[str]): CAUTION WHEN USING THIS. Private key
            file contents. Writes the private key to ssh_identity_file.
        timeout_s: Timeout parameter for Gloo rendezvous.
        placement_group_timeout_s: Timeout parameter for Ray
            Placement Group creation. Currently unused.
    Nnics   verbosekeyssh_portssh_identity_filessh_stri,  	timeout_sd   placement_group_timeout_sc                 8    t          j        | j        d          S )NzTimed out waiting for {activity}. Please check connectivity between servers. You may need to increase the --start-timeout parameter if you have too many servers.)message)r
   Timeoutr   selfs    l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/train/horovod/config.pystart_timeoutzHorovodConfig.start_timeout/   s%    N6
 
 
 	
    c                 `   | j         rt          j                            | j                  sat          | j        d          5 }t          j        | j        d           |                    | j                    d d d            n# 1 swxY w Y   | j        t          j
                    | _        d S d S )Nwi  )r   ospathexistsr   openchmodwriter   r	   make_secret_key)r#   fs     r$   __post_init__zHorovodConfig.__post_init__9   s    < 	&t/E F F 	&d,c22 &a/777%%%& & & & & & & & & & & & & & & 8-//DHHH s   5BB	Bc                     t           S N)_HorovodBackendr"   s    r$   backend_clszHorovodConfig.backend_clsB   s    r&   )__name__
__module____qualname____doc__r   r   r   str__annotations__r   intr   r   r   r   r   r   propertyr%   r1   r5    r&   r$   r   r      s          &  $D(3s8
###GSC#"Hhsm"""'+x}+++!GXc]!!!Is%(s(((
 
 X
0 0 0   X  r&   r   c                   .    e Zd ZU dZeed<   dedefdZdS )r4   Tshare_cuda_visible_devicesworker_groupbackend_configc                    g }t          t                              D ]W}j        |         j        j        }|                                        |t          |t                    |                     Xt          j	        |           t          |          | _        d j        D             d j        D             }t          t          |                    D ]$\  }\  }}| j                            |||           %| j                                        }	g }|	                                D ]4\  }}
|                                        |t"          |
                     5t          j	        |           | j                                        }fdt'                    D             }fd|D             }t          |          t          | j        j                  k    sJ t+          |t-          | j        j                  |          }|                    t1          |                                         t"          |           d S )Nc                 &    g | ]}|j         j        S r>   )metadatanode_id.0r(   s     r$   
<listcomp>z,_HorovodBackend.on_start.<locals>.<listcomp>e   s    EEE1AJ&EEEr&   c                 &    g | ]}|j         j        S r>   )rE   hostnamerG   s     r$   rI   z,_HorovodBackend.on_start.<locals>.<listcomp>f   s    GGGQQZ(GGGr&   c                 :    g | ]}                     |          S r>   )index)rH   rF   node_idss     r$   rI   z,_HorovodBackend.on_start.<locals>.<listcomp>y   s%    TTT7x~~g66TTTr&   c                 D    g | ]}t          j        |                   S r>   )_HorovodWorkerWrapperworkers)rH   worker_indexrA   s     r$   rI   z,_HorovodBackend.on_start.<locals>.<listcomp>z   s9     
 
 
 ","6|"DEE
 
 
r&   )all_host_namesnode_workers)rangelenrQ   rE   rF   appendexecute_single_async_init_env_varsraygetr   coordinator	enumeratezipregisterfinalize_registrationitemsr   establish_rendezvousset	hostnamesr   listupdater   execute)r#   rA   rB   setup_futuresrankworker_node_idrd   rK   rF   all_infolocal_cross_env_varcoordinator_envsnode_worker_indexesrT   r   rN   s    `             @r$   on_startz_HorovodBackend.on_startJ   s    #l++,, 
	 
	D)1$7@HN  11"%%"     	 '~66 FE0DEEEGG,2FGGG	 *33y(3K3K)L)L 	? 	?%D%8W%%h>>>>#99;;)1)9)9 	 	%D%  11/+>    
 	+@@BB UTTTc(mmTTT
 
 
 
 3
 
 
 <  C(8(B$C$CCCCC 0 :;;%
 
 

 	 5 5666_.>?????r&   N)	r6   r7   r8   r@   boolr;   r   r   ro   r>   r&   r$   r4   r4   G   sQ         '++++=@[ =@- =@ =@ =@ =@ =@ =@r&   r4   
world_rank
world_sizerF   c                     |t           j        d<   t          |           t           j        d<   t          |          t           j        d<   dS )z)Initialize Horovod environment variables.HOROVOD_HOSTNAMEHOROVOD_RANKHOROVOD_SIZEN)r)   environr:   )rq   rr   rF   s      r$   rY   rY      s9    %,BJ!"!$ZBJ~!$ZBJ~r&   c                   0    e Zd ZU eed<   ed             ZdS )rP   r(   c                 D    | j          G fdd          } |            S )Nc                       e Zd Z fdZdS )4_HorovodWorkerWrapper.execute.<locals>.ExecuteHandlec                 @    d } j         j        j        ||g|R i |S r3   )actor_RayTrainWorker__executeremote)r#   funcargskwargs_r(   s        r$   r   z;_HorovodWorkerWrapper.execute.<locals>.ExecuteHandle.remote   s3    >qw7>tQXXXXQWXXXr&   N)r6   r7   r8   r   r(   s   r$   ExecuteHandler{      s5        Y Y Y Y Y Y Yr&   r   r   )r#   r   r(   s     @r$   rg   z_HorovodWorkerWrapper.execute   sT    F	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y
 }r&   N)r6   r7   r8   r   r;   r=   rg   r>   r&   r$   rP   rP      s8         III  X  r&   rP   )r)   dataclassesr   typingr   r   horovod.ray.runnerr   horovod.ray.utilsr   r   horovod.runner.common.utilr	   r
   rZ   ray.train._internal.utilsr    ray.train._internal.worker_groupr   r   ray.train.backendr   r   ray.utilr   r   r4   r<   r:   rY   rP   r>   r&   r$   <module>r      s   				 ! ! ! ! ! !                 * * * * * * : : : : : : : : 6 6 6 6 6 6 6 6 



 5 5 5 5 5 5 @ @ @ @ @ @ @ @ 4 4 4 4 4 4 4 4       V
2 2 2 2 2M 2 2  2j@@ @@ @@ @@ @@g @@ @@ @@F1s 1 1c 1 1 1 1          r&   