&`in ddlZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z ddl Z ddl mZddlZddlmcmZddlmZmZmZddlmZddlmZmZmZddlmZmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.ej/e0Z1dZ2dZ3dZ4dZ5ddefdZ6 dde7de e7e fde e e7e ffdZ8Gdde(Z9dS)N) OrderedDict defaultdict)AnyDictList)ServiceResource)"CLOUDWATCH_AGENT_INSTALLED_AMI_TAGCLOUDWATCH_AGENT_INSTALLED_TAGCloudwatchHelper bootstrap_aws)boto_exception_handler client_cacheresource_cache)cf cli_logger)BOTO_CREATE_MAX_RETRIESBOTO_MAX_RETRIES)LogTimer)NodeLaunchException) NodeProvider)TAG_RAY_CLUSTER_NAMETAG_RAY_LAUNCH_CONFIGTAG_RAY_NODE_KINDTAG_RAY_NODE_NAMETAG_RAY_USER_NODE_TYPEcHt|vr|t|d<|t=|S)z=Convert the Ray node name tag to the AWS-specific 'Name' tag.Namertagss }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/aws/node_provider.py to_aws_formatr$,s,D  -.V " # Kc4d|vr|d|t<|d=|S)z=Convert the AWS-specific 'Name' tag to the Ray node name tag.rr r!s r#from_aws_formatr'5s(~~"&v,  L Kr%returnc(|pi}td||fi|S)z3Make client, retrying requests up to `max_retries`.ec2)rregion max_retriesaws_credentialss r#make_ec2_resourcer/>s&%+O % H H H HHr%r,r.c^g}|pi}td|tfi|}|}|t j|dd|vrM||d}|t j|dd|vM|S)aqGet all instance-types/resources available in the user's AWS region. Args: region: the region of the AWS provider. e.g., "us-west-2". Returns: final_instance_types: a list of instances. An example of one element in the list: {'InstanceType': 'm5a.xlarge', 'ProcessorInfo': {'SupportedArchitectures': ['x86_64'], 'SustainedClockSpeedInGhz': 2.5},'VCpuInfo': {'DefaultVCpus': 4, 'DefaultCores': 2, 'DefaultThreadsPerCore': 2, 'ValidCores': [2], 'ValidThreadsPerCore': [1, 2]}, 'MemoryInfo': {'SizeInMiB': 16384}, ...} r* InstanceTypes NextToken)r2)rrdescribe_instance_typesextendcopydeepcopy)r,r.final_instance_typesr*instance_typess r#list_ec2_instancesr9Ds"%+O uf&6 J J/ J JC0022N n_.M N NOOO  ' '44$[15   ##DM.2Q$R$RSSS  ' ' r%c@eZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d eeeffd Zedeeeefdeeeefd dfdZdZdZdZdZdZdZedZedeeefd eeeffdZdS)AWSNodeProviderictj||||dd|_|d}t |dt ||_t |dd||_i|_tt|_ d|_ tj|_|jtj|_|jtj|_tj|_tj|_i|_dS)Ncache_stopped_nodesTr.r,r+r)r__init__getr=r/rr* ec2_fail_fast tag_cacherdicttag_cache_pendingbatch_thread_count threadingEventbatch_update_donesetready_for_new_batchLocktag_cache_lock count_lock_reuse_node_lock cached_nodes)selfprovider_config cluster_namer.s r#r>zAWSNodeProvider.__init__fs9dO\BBB#2#6#67Ld#S#S )--.?@@$"8,(+    /"8,+   !,T!2!2"#!*!2!2 ""$$$#,?#4#4   $$&&&'n..#.** ) 0 0r%c`t|}dddgddt|jgdg}|D]1\}}|d||gd2t d5t|jj |}dddn #1swxYwY|D]<}|j |j vrtd|jD|j |j <=d |D|_d |DS) Ninstance-state-namependingrunningrValuestag:{}z+Failed to fetch running instances from AWS.Filtersc,i|]}|d|dSKeyValue.0xs r# z8AWSNodeProvider.non_terminated_nodes..s"999!51W:999r%ci|] }|j| Sr_idranodes r#rcz8AWSNodeProvider.non_terminated_nodes..s===tTWd===r%cg|] }|j Sr_rergs r# z8AWSNodeProvider.non_terminated_nodes..s***D***r%)r$formatrrQitemsappendrlistr* instancesfilterrfrAr'r"rN)rO tag_filtersfilterskvnodesrhs r#non_terminated_nodesz$AWSNodeProvider.non_terminated_nodess$K00 .$i0   !(<==,-     %%''  DAq NN$OOA.. c    $$Q R R E E+2272CCDDE E E E E E E E E E E E E E E E  Dw$.((&599ty999''DN47 # #>=u===**E****s.C  C C cN||}|jddkS)NrrU_get_cached_nodestaterOnode_idrhs r# is_runningzAWSNodeProvider.is_runnings'$$W--z&!Y..r%cN||}|jd}|dvS)Nr)rUrTrx)rOr|rhrzs r# is_terminatedzAWSNodeProvider.is_terminateds-$$W-- 6"222r%c|j5|j|}|j|i}t |fi|cdddS#1swxYwYdSN)rKrArCr?rB)rOr|d1d2s r# node_tagszAWSNodeProvider.node_tagss   " "(B'++GR88B>>b>> " " " " " " " " " " " " " " " " " "s5A  AAcr||}|j||}|jSr)rypublic_ip_address _get_noder{s r# external_ipzAWSNodeProvider.external_ips8$$W--  ! )>>'**D%%r%cr||}|j||}|jSr)ryprivate_ip_addressrr{s r# internal_ipzAWSNodeProvider.internal_ips8$$W--  " *>>'**D&&r%c6d}|j5|jsMd}|j|j|j|j||dddn #1swxYwY|retjt|j5| |j dddn #1swxYwY|j 5|xj dz c_ dddn #1swxYwY|j|j 5|xj dzc_ |j dkr|j ddddS#1swxYwYdS)NFTrr)rKrCrIwaitclearrGupdatetimesleepTAG_BATCH_DELAY_update_node_tagsrHrLrD)rOr|r"is_batching_threads r# set_node_tagszAWSNodeProvider.set_node_tagss"   9 9) /%)"(--///(..000&,,...  "7 + 2 24 8 8 8 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9  - J ' ' '$ - -&&(((&**,,, - - - - - - - - - - - - - - -_ ) )  # #q ( # # ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ##%%% _ / /  # #q ( # #&!++(,,... / / / / / / / / / / / / / / / / / /sHA5B  BB8.C22C69C6D!!D%(D% 5FFFcltt}|jD]W\}}|D]}||||j||Xtt|_||dSr) rrnrCrlrmrArrB _create_tags)rO batch_updatesr|r"rbs r#rz!AWSNodeProvider._update_node_tagss#D)) !399;; 1 1MGTZZ\\ 1 1a ''0000 N7 # * *4 0 0 0 0!,T!2!2 -(((((r%cP|D]\\}}}d|||}td|5|tkrd}|jjj|||dgdddn #1swxYwYdS)NzSet tag {}={} on {}zAWSNodeProvider: {}rr\) ResourcesTags)rlrkrrr*metaclient create_tags)rOrrsrtnode_idsms r#rzAWSNodeProvider._create_tagss - 3 3 5 5   FQH%,,Q8<.)s!.*s$B$B$BQT1$B$B$Br%zsReusing nodes {}. To disable reuse, set `cache_stopped_nodes: False` under `provider` in the cluster configuration.zStopping instances to reusec,i|]}|d|dSr\r_r`s r#rcz/AWSNodeProvider.create_node..8s" I I I!51W: I I Ir%rzWaiting for instance {} to stop InstanceIds)#rsortedr5r6rlr=rkrrQrrrrmrMrnr*rorprprint render_listgroupr'r"rArfrzwait_until_stoppedrrstart_instancesrlen _create_noder) rO node_configr"countreused_nodes_dictrr reuse_nodesreuse_node_idsrhr|created_nodes_dictall_created_nodess r# create_nodezAWSNodeProvider.create_nodes6$-"5"5";";"="=>>??  #: 1 2(*5 %OO,@AA#01 %OO,=>>#$567 %OO,ABB#$9:;G&&-- (0F G G#'(>#?"@& 1 1"48#5#<#> $)*GHH : :$/::D6E I Ity I I I77DN473 $z&1Z?? * 0$Etw!"!"!"!% 7 7 9 9 9: : : : : : : : : : : : : : : :HM(88^8TTT#1::**7D9999S000E9 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1<   M!%!2!2;e!L!L -  !3444  s9B JA0H JH J!H "AJJ J tag_specsuser_tag_specsNc|D]m}|ddkrY|dD]O}d}|ddD]#}|d|dkrd}|d|d<n$|s|ddxx|gz cc<Pg||gz }nd S) aZ Merges user-provided node config tag specifications into a base list of node provider tag specifications. The base list of node provider tag specs is modified in-place. This allows users to add tags and override values of existing tags with their own, and only applies to the resource type "instance". All other resource types are appended to the list of tag specs. Args: tag_specs (List[Dict[str, Any]]): base node provider tag specs user_tag_specs (List[Dict[str, Any]]): user's node config tag specs ResourceTypeinstancerFrr]Tr^Nr_)rr user_tag_specuser_tagexiststags r#_merge_tag_specsz AWSNodeProvider._merge_tag_specsMs&, - -M^, :: -f 5;;H"F(|F3""#E?c%j88%)F+3G+j j!$r}| dz } ||kr tE|j#dd|j#ddtIj%}n+#tL$rtN(d|YnwxYwt1j)d|nt1j(d|Yd}~d}~wwxYw|S) Nr\agentTruer)rrTagSpecifications SubnetIdsr)MinCountMaxCountrrNetworkInterfacesSecurityGroupIdsnetwork_interfacesSubnetId subnet_idci|] }|j| Sr_rers r#rcz0AWSNodeProvider._create_node..s%?%?%?!adA%?%?%?r%zLaunched {} nodes)_tagsrTMessagezLaunched instance {}r)rzinfoErrorCode)category description src_exc_infozCouldn't parse exception.z2Failed to launch instances. Max attempts exceeded.)excz3create_instances: Attempt failed with {}, retrying.r_)*r$r5rrQrlrmr cloudwatch_config_existsrP_check_ami_cwa_installationr4r r?r;rpoprmaxrrrangestrr@create_instancesrr state_reasonr instance_idrBrzbotocore exceptions ClientErrorrresponsesysexc_info Exceptionloggerwarningabort)rOrr"rrconf tag_pairsrsrt cwa_installedrr subnet_ids subnet_idxcli_logger_tags max_triesattemptnet_ifsrcreatedrrrs r#rzAWSNodeProvider._create_nodensRT""!!,*   JJLL  DAq         4T5I7 S S  <<[IIM   $B%+!+!   "5r::((NCCCXXk**  IVVWWW /ZAA Q A ..E E GD &$.."#67GHH/666dS)NzStopping instances {} rzTerminating instances {} rrr)r*rrterminate_instancesstop_instancesr=ryrrrrrrmax_terminate_nodesrrlr) rOrterminate_instances_funcstop_instances_funcnodes_to_terminatespot_ids on_demand_idsr|r terminate_funcrustarts r#terminate_nodeszAWSNodeProvider.terminate_nodess  F#'8=#7#K "hm2A7Return node info from cache if possible, otherwise fetches it.)rNr)rOr|s r#ryz AWSNodeProvider._get_cached_nodens0 d' ' '$W- -~~g&&&r%c t|Srr )cluster_configs r#bootstrap_configz AWSNodeProvider.bootstrap_configus^,,,r%rc>d|vr|Stj|}t|dd|dd}d|D}|d}|d}|D]}||dd}||vrl||d d }d |i}||krO||d d } t | dzdz} dt jz } t | | z} | |d<tjj D]Q} | ||} | ||}| r!| || <|rd|d|<R|||di|||dikr9|||d<td||t%d|zdz|ddzdz|S)z=Fills out missing "resources" field for available_node_types.available_node_typesproviderr,r.c i|] }|d| S) InstanceTyper_)rars r#rczJAWSNodeProvider.fillout_available_node_types_resources..s,   3;H^ $h   r%head_node_typerr#VCpuInfo DefaultVCpusCPU MemoryInfo SizeInMiBirmemoryzaccelerator_type: resourcesz#Updating the resources of {} to {}.zInstance type z! is not available in AWS region: r)r5r6r9r?int ray_constants&DEFAULT_OBJECT_STORE_MEMORY_PROPORTIONray_private acceleratorsget_all_accelerator_managers!get_ec2_instance_num_accelerators!get_ec2_instance_accelerator_typeget_resource_namerrdebugrk ValueError)rinstances_listinstances_dictr r$ node_type instance_typecpusautodetected_resources memory_totalpropmemory_resourcesaccelerator_managernum_acceleratorsaccelerator_types r#&fillout_available_node_types_resourcesz6AWSNodeProvider.fillout_available_node_types_resourcesys " 7 7! !~66+ : &x 0 : & * *+< = =    ?M    ..DE'(89-= = I0;MJM..%m4Z@P*/&..#1-#@#N#$L$'|#4#4t#;d#BL}SSD'*<$+>'?'?$7G*84\.KKMM""',MM)>% ,MM)>% ("-//AACC,"!"3 F4D F F'--(377 RHH*-A)-L-P-P.. /)3#LL=DD%'= !$#$9:%Z0:; r%)__name__ __module__ __qualname__r r>rvr}rrrrrrrrrrr staticmethodrrrrrrrryrrDr_r%r#r;r;cs!!!F#+#+#+J///333 """ &&&'''///4 ) ) )   M!tCH~M!M!M!M!^-S#X'-9=d38n9M- ---\-@{"{"{"z<   9W9W9WvGGG8'''--\-OS#XO c3hOOO\OOOr%r;r):r5loggingrrEr collectionsrrtypingrrrrboto3.resources.baserr/ray._private.ray_constantsr0r-8ray.autoscaler._private.aws.cloudwatch.cloudwatch_helperr r r "ray.autoscaler._private.aws.configr !ray.autoscaler._private.aws.utilsrrr"ray.autoscaler._private.cli_loggerrr!ray.autoscaler._private.constantsrr!ray.autoscaler._private.log_timerr$ray.autoscaler.node_launch_exceptionrray.autoscaler.node_providerrray.autoscaler.tagsrrrrr getLoggerrErrrr$r'r/rr9r;r_r%r#rXs   00000000""""""""""000000 222222222 =<<<<< >=======WWWWWWWW666666DDDDDD555555  8 $ $IIOIIII48   "&sCx.  $sCx.    >f f f f f lf f f f f r%