&`i.\ddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z ddl mZddlmZmZmZmZmZmZmZejeZejdZeejGdd ZGd d eZGd d eZdeddfdZ dS)N)RLock)FileLock)LOCAL_CLUSTER_NODE_TYPEbootstrap_local get_lock_pathget_state_path) NodeProvider)NODE_KIND_HEADNODE_KIND_WORKERSTATUS_UP_TO_DATETAG_RAY_NODE_KINDTAG_RAY_NODE_NAMETAG_RAY_NODE_STATUSTAG_RAY_USER_NODE_TYPEfilelockc eZdZdZdZdZdS) ClusterStatec t|_tjtj|dt ||_||_|j5|j5tj |jrtj t|j }||d}|r7|ditt kri}t"dni}t"dt)||dD]?}||vrtt*idd ||<||dtt*ksJ@|d|vrtt idd ||d<n*||ddtt ksJt)|d}||dt)|D] }||vr||= |d }|r||d} || d <t/|t/|dd zksJt|jd 5} t"d|| tj|dddn #1swxYwYdddn #1swxYwYddddS#1swxYwYdS)NT)exist_okhead_iptagsz%Head IP changed - recreating cluster.z&ClusterState: Loaded cluster state: {} worker_ips terminatedrstateexternal_head_ip external_ipw'ClusterState: Writing cluster state: {})rlockosmakedirspathdirnamer file_lock save_pathexistsjsonloadsopenreadgetr r loggerinfoformatlistr appendlendebugwritedumps) self lock_pathr'provider_configworkers head_config worker_iplist_of_node_ipsrheadfs /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/local/node_provider.py__init__zClusterState.__init__!s!GG  BGOOI..>>>>!),," Y; 1; 1: 1: 17>>$.11 !"jdn)=)=)B)B)D)DEEG")++oi.H"I"IK'M&??6266::;LMM)**#% $KLLL G <CCDMMRR"1!>  I //%68H$I%1.. ** $I.v67HI/00000#9-W<D*:D'7||s?<+H'I'IA'MMMMM$.#..1!LLAHHQQGGDJw//000 111111111111111m: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1: 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1s\)M1IL=AL& L=&L* *L=-L* .L=1 M=M MM MMMc|j5|j5tjt |j}|cdddcdddS#1swxYwY ddddS#1swxYwYdSN)r!r&r)r*r+r'r,)r7r:s r@r-zClusterState.getds  Y    *T$.%9%9%>%>%@%@AA                                          s4A<:A# A<#A' 'A<*A' +A<<BBc d|vsJd|vsJ|j5|j5|}|||<t|jd5}t dt|| tj |dddn #1swxYwYdddn #1swxYwYddddS#1swxYwYdS)Nrrrr ) r!r&r-r+r'r.r/r0r1r5r)r6)r7 worker_idr/r:r?s r@putzClusterState.putjs~~~~$ Y 1 1 1 1((**%) "$.#..1!KK44:F4==4I4IGGDJw//000 111111111111111 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1sYC(/C A"B9- C9B= =CB= C C(C C(C C((C,/C,N)__name__ __module__ __qualname__rAr-rFr@rr sEA1A1A1F 1 1 1 1 1rKrceZdZdZdZdS)OnPremCoordinatorStatea[Generates & updates the state file of CoordinatorSenderNodeProvider. Unlike ClusterState, which generates a cluster specific file with predefined head and worker ips, OnPremCoordinatorState overwrites ClusterState's __init__ function to generate and manage a unified file of the status of all the nodes for multiple clusters. ct|_t||_||_|j5|j5t j|jr9tj t|j }ni}t d|t|D] }||vr||= |D]}||vridd||<t!|t!|ksJt|jd5}t d||tj|dddn #1swxYwYdddn #1swxYwYddddS#1swxYwYdS)Nz>$.11 JtDN';';'@'@'B'BCCEEE ;;A6%== $E{{++G&666!'N/Ge++$&%1**g5zzS)9%:%:::::$.#../!KK@@Fu GGDJu--... ///////////////- / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / /sZF<C.F$,AF  F$ F F$F F$ F<$F( (F<+F( ,F<<GGN)rGrHrI__doc__rArJrKr@rMrMys-!/!/!/!/!/rKrMcdeZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z ed Zd S)LocalNodeProvideraNodeProvider for private/local clusters. `node_id` is overloaded to also be `node_ip` in this class. When `cluster_name` is provided, it manages a single cluster in a cluster specific state file. But when `cluster_name` is None, it manages multiple clusters in a unified state file that requires each node to be tagged with TAG_RAY_CLUSTER_NAME in create and non_terminated_nodes function calls to associate each node with the right cluster. The current use case of managing multiple clusters is by OnPremCoordinatorServer which receives node provider HTTP requests from CoordinatorSenderNodeProvider and uses LocalNodeProvider to get the responses. ctj||||r=t|}t|}t ||||_d|_dStdd|d|_d|_dS)NFz/tmp/coordinator.lockz/tmp/coordinator.stater=T)r rArrrruse_coordinatorrM)r7r9 cluster_namer8 state_paths r@rAzLocalNodeProvider.__init__sdO\BBB  (%l33I' 55J%DJ $)D 0'( 23DJ $(D rKc6|j}g}|D]h\}}|ddkrd}|D](\}}|d||krd}n)|r||i|S)NrrTrF)rr-itemsr2) r7 tag_filtersr: matching_ipsr<r/okkvs r@non_terminated_nodesz&LocalNodeProvider.non_terminated_nodess*.."" &}} / /OItG} ,,B#))++  1<##A&&!++BE, /##I...rKcT|j|ddkS)Nrrunningrr-r7node_ids r@ is_runningzLocalNodeProvider.is_runnings#z~~(1Y>>rKc.|| SrC)rercs r@ is_terminatedzLocalNodeProvider.is_terminateds??7++++rKcL|j|dSNrrbrcs r@ node_tagszLocalNodeProvider.node_tagssz~~(00rKc|j|}|d}|r|Stj|S)aReturns an external ip if the user has supplied one. Otherwise, use the same logic as internal_ip below. This can be used to call ray up from outside the network, for example if the Ray cluster exists in an AWS VPC and we're interacting with the cluster from a laptop (where using an internal_ip will not work). Useful for debugging the local node provider with cloud VMs.r)rr-socket gethostbyname)r7rd node_stateext_ips r@rzLocalNodeProvider.external_ipsIZ^^%%g.  ..  1M'00 0rKc*tj|SrC)rlrmrcs r@ internal_ipzLocalNodeProvider.internal_ips#G,,,rKcB|jj5|jj5|j|}|d||j||dddn #1swxYwYddddS#1swxYwYdSri)rr!r&r-updaterF)r7rdrr/s r@ set_node_tagszLocalNodeProvider.set_node_tagss Z_ . .% . .z~~''0V ##D))) w--- . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .s5 BAA<0 B<B BB BBBc|t}|jj5|jj5|j}|D]x\}}|ddkrg|js|dt|krI||d<d|d<|j|||dz }|dkrdddddddSy dddn #1swxYwYddddS#1swxYwYdS)z.Creates min(count, currently available) nodes.rrrrarrN)r rr!r&r-rYrUrF)r7 node_configrcount node_typer:rdr/s r@ create_nodezLocalNodeProvider.create_nodes*+ Z_ # #% # #*..**%,]]__ # #MGTG} 44,5<(9:iGG'+V (1W  w555 %  A::" # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #sA C3BC5 C3 C C3C C3"C #C33C7:C7c|j}||}d|d<|j||dS)Nrr)rr-rF)r7rdr:r/s r@terminate_nodez LocalNodeProvider.terminate_nodesB*..""w$W  w%%%%%rKc t|SrC)r)cluster_configs r@bootstrap_configz"LocalNodeProvider.bootstrap_configs~...rKN)rGrHrIrQrAr_rergrjrrqrtryr{ staticmethodr~rJrKr@rSrSs (((*   ???,,,111111"---...###$&&& //\///rKrSlocal_providerreturncB|jd}|j}||ivrqttt t td|tti}| i|d||ivsJdSdS)aBThis function is called on the Ray head from StandardAutoscaler.reset to record the head node's own existence in the cluster state file. This is necessary because `provider.create_node` in `commands.get_or_create_head_node` records the head state on the cluster-launching machine but not on the head. rz ray-{}-headr)rvrrwN) r9rVr_r r rrrr0rr ry)rrrV head_tagss r@!record_local_head_state_if_neededrs,Y7G!.Ln99"==== ~ "$; }33LAA !2   ""r "KKK.==bAAAAAA>=BArK)!r)loggingr"rl threadingrrr$ray.autoscaler._private.local.configrrrrray.autoscaler.node_providerr ray.autoscaler.tagsr r r r rrr getLoggerrGr.filelock_loggersetLevelWARNINGrrMrSrrJrKr@rs   655555  8 $ $#'#J//)))V1V1V1V1V1V1V1V1r*/*/*/*/*/\*/*/*/Zs/s/s/s/s/ s/s/s/lB6GBDBBBBBBrK