&`i BddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddl mZmZddlmZmZmZmZmZmZddlZddlmcmZddlmZddlmZmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+m,Z,d\Z-Z.dZ/ee0ee1e2ffZ3eeefZ4e0Z5e0Z6ee0efZ7ee0efZ8e0Z9e0Z:e1Z;e0Ze?Z@de0deAfdZBe GddZCGddZDdee0efddfdZEdee0efddfdZFdee0efdee0effdZGdee0effdZHdee0efdee0effdZIdee0efdee0effdZJd ZKd!ZLd"ee0d#ee0e0fde0fd$ZMdId%ZNd&ZOiZP dJd(ZQd)ZRd*ZSd+e0dee0ee0eAffd,ZTgd-ZUd.ede0fd/ZVd0e=d1eAdee0fd2ZWd3eCd1eAde0fd4ZXd5eee3e1fdee0fd6ZYd7ee4fd8ZZd3eCfd9Z[d3eCde\fd:Z]d3eCd;eee0e2fdee2d?ee2d@ee2d1eAfdAZ_dBe0fdCZ`dDe\fdEZadFZbdGZcdHZddS)LN) dataclass)datetime)StringIO)NumberReal)AnyDictListOptionalTupleUnion)$PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME)0PLACEMENT_GROUP_INDEXED_BUNDLED_RESOURCE_PATTERN)PLACEMENT_GROUP_WILDCARD_RESOURCE_PATTERN) constants) cli_logger)validate_docker_config) prepare_local)_get_default_config)NODE_TYPE_LEGACY_HEADNODE_TYPE_LEGACY_WORKER)TFaSetting `max_workers` for node type `{node_type}` to the global `max_workers` value of {max_workers}. To avoid spawning worker nodes of type `{node_type}`, explicitly set `max_workers: 0` for `{node_type}`. Note that `max_workers: 0` was the default value prior to Ray 1.3.0. Your current version is Ray {version}. See the docs for more information: https://docs.ray.io/en/master/cluster/config.html#cluster-configuration-node-max-workers https://docs.ray.io/en/master/cluster/config.html#full-configuration resource_namereturnclttj|ptj|S)zH Check if a resource name is structured like a placement group. )boolrmatchr)rs p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/util.pyis_placement_group_resourcerRs8 17 FF Q ; A- P P  c eZdZUeed<eeed<eeed<eeed<eeed<dZee ed<dZ ee e efed<dZ ee e e fed <dZee e efed <dS) LoadMetricsSummaryusageresource_demand pg_demandrequest_demand node_typesNhead_ip usage_by_nodenode_type_mapping idle_time_map)__name__ __module__ __qualname__Usage__annotations__r DictCountr'r NodeIPr(r strr)r*intrrr!r!\s LLL)_$$$IO###Y!%GXf $$$15M8De,-44437xS#X/666.2M8DcN+22222rr!c<eZdZdZdZdZdZedZdS)ConcurrentCountercrtj|_tjt |_dSN) threadingRLock_lock collections defaultdictr3_counterselfs r__init__zConcurrentCounter.__init__vs'_&& #/44 rc||j5|j|xx|z cc<|jcdddS#1swxYwYdSr8r;r>valuer@keycounts rinczConcurrentCounter.inczs Z   M#   % '   :                  s 155c|j5|j|xx|zcc<|j|dks Jd|jcdddS#1swxYwYdS)Nrzcounter cannot go negativerCrEs rdeczConcurrentCounter.decs Z   M#   % '   =%***,H***:                  s7A  AAcl|j5t|jcdddS#1swxYwYdSr8)r;dictr>r?s r breakdownzConcurrentCounter.breakdowns{ Z ' ' && ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's )--c|j5t|jcdddS#1swxYwYdSr8)r;sumr>valuesr?s rrDzConcurrentCounter.values Z / /t}++--.. / / / / / / / / / / / / / / / / / /s &;??N) r+r,r-rArHrJrMpropertyrDr4rrr6r6usk555  '''//X///rr6configctts"tdtjtjtj j d}t|5}tj |}dddn #1swxYwY ddl}n#tt f$r}|dd}~wwxYw ||n#|j$r}|dd}~wwxYwdvr-t'dtjdvrnd vrtd d dvrtd t+fd dD}|d krtdt,jdkr%ddrtddSdS)z?Required Dicts indicate that no extra fields can be introduced.zConfig {} is not a dictionaryzray-schema.jsonNrcluster_synced_filesaMissing 'cluster_synced_files' field in the cluster configuration. This is likely due to the Ray version running in the cluster {ray_version} is greater than the Ray version running on your laptop. Please try updating Ray on your local machine and make sure the versions match.) ray_versionavailable_node_typeshead_node_typezBYou must specify `head_node_type` if `available_node_types is set.z7`head_node_type` must be one of `available_node_types`.c3\K|]&}d|ddV'dS)rV min_workersrN)get).0 node_typerRs r z"validate_config..sR   ) *9 5 9 9- K K      r max_workerszlThe specified global `max_workers` is smaller than the sum of `min_workers` of all the available node types.win32file_mounts_sync_continuouslyFzn`file_mounts_sync_continuously` is not supported on Windows. Please set this to False when running on Windows.) isinstancerL ValueErrorformatospathjoindirnameray autoscaler__file__openjsonload jsonschemaModuleNotFoundError ImportErrorvalidateValidationError RuntimeError __version__rOsysplatformrZ)rR schema_pathfschemarnesum_min_workerss` rvalidate_configr|s fd # #I8??GGHHH',, /002CK k  a1  -TFF++++  %T V++ 89?O9?99    '' 6 ) )T  " #62H+I I IVWW W    #$:;      VM2 2 2H   |w6::.Mu#U#U @   sBB55B9<B9CCCC#C:: D DD c"d|vr|drtjdd|vr|drtjdd|vr#tjdtdd|vr#tjdtdd S) zFor use in providers that have completed the migration to available_node_types. Warns user that head_node and worker_nodes fields are being ignored. Throws an error if available_node_types and head_node_type aren't specified. head_nodezqThe `head_node` field is deprecated and will be ignored. Use `head_node_type` and `available_node_types` instead. worker_nodesz_The `worker_nodes` field is deprecated and will be ignored. Use `available_node_types` instead.rVz.`available_node_types` not specified in configrWz(`head_node_type` not specified in configN)rwarningerrorrbrRs rcheck_legacy_fieldsrsf !4 G   F>$: 2   V++IJJJIJJJv%%CDDDCDDD&%rc^|diddk}|diddk}|rt|}n|r|St|}t|t |t ||S)a The returned config has the following properties: - Uses the multi-node-type autoscaler configuration. - Merged with the appropriate defaults.yaml - Has a valid Docker configuration if provided. - Has max_worker set for each node type. providertypelocalkuberay)rZrfillout_defaultsmerge_setup_commandsrfill_node_type_min_max_workers)rRis_local is_kuberay with_defaultss rprepare_configrszz*b))--f55@HJ++//779DJv&&  $V,,M'''=)))"=111 rc ~ddg}|D]5}||vr/||s'td|d|d|d||=6dS)zG Drop empty deprecated fields ("head_node" and "worker_node"). r~rz Dropping the empty legacy field z. z=2.0.0. It is recommended to removez from the cluster config.N)loggerr)rRREMOVABLE_FIELDSfields rtranslate_trivial_legacy_configrs $^4! F??6%=? NN4544E44444    u rc8t|d}||tj|}|di|d<d|vod|vpd|v}|rt |}|ddt||S)NrauthrVr~rrY)rupdatecopydeepcopyrZmerge_legacy_yaml_with_defaultspopr)rRdefaults merged_configis_legacy_configs rrrs"6*#566H OOFM(++M *--fb99M&/f<v96!9G7 FF mT***#M222 rrctjd|d}t|ddksJ|d|hz }|dr)|d|ddpiddd}n|d|}|d rC|d |d dpi|d d|d d}n|d|}t |t|i|d<t |d<|ddd |d dd |S) zbRewrite legacy config's available node types after it has been merged with defaults yaml. aConverting legacy cluster config to a multi node type cluster config. Multi-node-type cluster configs are the recommended format for configuring Ray clusters. See the docs for more information: https://docs.ray.io/en/master/cluster/config.html#full-configurationrWrVr~ resourcesr) node_configrrYr^rrYr^N)rrlenkeysrrZrr)rdefault_head_typedefault_worker_typehead_node_infoworker_node_infos rrr0s O&&67 }3499;; < < A A A A,-22448I7JJ cee[! R)5&{377 DDJ   ''=>?PQ^$ V)8&~6::;GGM2(,,]A>>(7   ))?@ATU ~!1-M()' ["export FOO=BAR; echo $FOO"] zexport =z; )itemsappend)rrout_cmdscmdkv_strkvs r with_envsrsH**HHJJ * *DAq )))A))) )FF6(3(()))) OrcP|tj}t|d|iS)N RAY_HEAD_IP)servicesget_node_ip_addressr)rr's rwith_head_node_iprs*.00 TM73 4 44rctj}|}dD]g}||vrattj||5}|||<dddn #1swxYwYh|tj ||gd d| S)N)ssh_private_keyssh_public_keyT sort_keysutf-8) hashlibsha1rrkrdre expanduserreadrrldumpsencode hexdigest) node_confrhasher full_authkey_typerFs rhash_launch_confrs \^^F  I911 t  bg((h8899 1S&)hhjj (# 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 MM$*i3tDDDKKGTTUUU     s"BB B Fc tj}tj d dtf fd }tj|ddtj|ddz}|t vs|rt|D] }|| }|t vrY| || |d| t |<|t|D]}||d } nd} t || fS) aReturns two hashes, a runtime hash and file_mounts_content hash. The runtime hash is used to determine if the configuration or file_mounts contents have changed. It is used at launch time (ray up) to determine if a restart is needed. The file_mounts_content hash is used to determine if the file_mounts or cluster_synced_files contents have changed. It is used at monitor time to determine if additional file syncing is needed. Fallow_non_existing_pathsc fd}tj|}|r!tj|sdStj|rg}tj|D]*\}}}||t|f+t|D]\}} | d|D]U} | dtj ||}||VdS||dS)Nct|d5tfddD]}| ddddS#1swxYwYdS)Nrbc.dS)N)r)rxsrzYhash_runtime_conf..add_content_hashes..add_hash_of_file..s!&&--rr)rkiterr)fpathchunkrxcontents_hashers @radd_hash_of_filezGhash_runtime_conf..add_content_hashes..add_hash_of_fileseT"" 2a!"7"7"7"7==22E#**511112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s,A  AAr) rdrerexistsisdirwalkrsortedrrrf) rerrdirsdirpath_ filenamesnamerrs radd_content_hashesz-hash_runtime_conf..add_content_hashessj 2 2 2 2 2 w!!$'' # BGNN4,@,@  F 7==   #D)+ : :%I WfY&7&789999&,Tll , ,"&&w~~g'>'>???%,,D#**4;;w+?+?@@@GLL$77E$$U++++, , ,  T " " " " "rTrrN)rF) rrrrlrr _hash_cacherrPrr) file_mountsrT extra_objs"generate_file_mounts_contents_hashruntime_hasherrconf_str local_pathhead_node_contents_hashfile_mounts_contents_hashrs @rhash_runtime_confrs \^^NlnnO##4######,z+666==gFFdJJJ fWooH {""&H" !3!3!5!566 + +J  z * * * *"1";";"="= ; & &  ! !( + + +  ! !"9"@"@"I"I J J J$2$<$<$>$>K ! +$%9:: N N #":MMMMM$3$=$=$?$?!!%)!  !#< ==rc|d}|dg}|ddD].}d||g}||/d|}|S)z?Prefixes each line of info_string, except the first, by prefix. rN:)splitrfr) info_stringprefixlinesprefixed_linesline prefixed_lineprefixed_info_strings r add_prefixrs{   d # #EAhZNabb --&$00 m,,,,99^44 rc|d}|d}g}|D]\}}||d| d|}|d|dS)Nstrategybundlesz * ,  ())rrf)pgrr shape_strsbundlerG bundles_strs r format_pgrsy*~HmGJ 11 V////0000))J''K ( (X ( ( ((rplacement_group_resource_strctj|}|r+|d|ddfStj|}|r+|d|ddfS|ddfS)aParse placement group resource in the form of following 3 cases: {resource_name}_group_{bundle_id}_{group_name}; -> This case is ignored as it is duplicated to the case below. {resource_name}_group_{group_name}; {resource_name} Returns: Tuple of (resource_name, placement_group_name, is_countable_resource). placement_group_name could be None if its not a placement group resource. is_countable_resource is True if the resource doesn't contain bundle index. We shouldn't count resources with bundle index because it will have duplicated resource information as wildcard resources (resource name without bundle index). rFrTN)rrgroupr)rresults r"parse_placement_group_resource_strr(s$> C$F9 Qa%88 6 <$F8 Qa$77 ($ 55r))TiBl)GiBi@)MiBr)KiBi mem_byteschtD]\}}||kr||z }|d|cSt|dS)zFormats memory in bytes in friendly unit. E.g. (2**30 + 1) bytes should be displayed as 1GiB but 1 byte should be displayed as 1B, (as opposed to rounding it to 0GiB). z.2fB)MEMORY_SUFFIXESr3)rsuffixbytes_per_unit mem_in_units r format_memoryr"Osc #200  & &#n4K!//v// / / / ')nn   rr"verbosec6i}tjt}|D]I\}\}}t |\}}} |r,||vrd||<| r ||xx|z cc<||xx|z cc<IJg} t |D]\}\}}d|vr t |\} }} |r#d} d} ||v}|r||} ||} || z | z}|dvrmt |}t |}|d|d|}|r-t | }t | }|d|d|ddzz}| ||d r|s|d|d|}|r |d| d| d z }| || S) Nrznode:)memoryobject_store_memory/ r z used of zreserved in placement groups)zaccelerator_type: reserved in placement groups)) r<r=floatrrrr"r startswith)r"r#placement_group_resource_usageplacement_group_resource_totalresourceusedtotalpg_resource_namepg_name is_countable usage_linesrpg_usedpg_total used_in_pgformatted_usedformatted_totalrformatted_pg_usedformatted_pg_totals r parse_usager<\s%'"%0%G5h?H (?W,D 8 8 8*400N+E22O$CCCCCCD $1'$:$:!%28%<%<"-*--)---/NO   t $ $ $ $  !4 5 5 %g % //U//X//D WWWxWWW   t $ $ $ $ r lm_summaryct|j|}t}|D]}td|||S)Nr(file)r<r"rprintgetvalue)r=r#r4siors rget_usage_reportrDsVj.88K **C$$ j$jjs##### <<>>rr#cdtfd}tjt}tjt}|D]\}}||\}}|r#t|vr|t=t |dkr^|tt| xx|z cc<|r<|tt| xx|z cc<g}| D]C\}}dt|d|d} ||vr| d||dz } | | D|S) Nrcd}t}|D]!\}}t|\}}}|||<|rd}"||fS)afilter placement group from bundle resource name. returns filtered bundle and a bool indicate if the bundle is using placement group. Example: {"CPU_group_groupid": 1} returns {"CPU": 1}, True {"memory": 1} return {"memory": 1}, False FT)rLrr)rusing_placement_group result_bundlepg_resource_strresource_countrr2rs r"filter_placement_group_from_bundlezJformat_resource_demand_summary..filter_placement_group_from_bundleso!& /5||~~ - - +O^*L++ ']GQ,:M- ( -(,%455rrr(: z+ pending tasks/actorsr z+ using placement groups)) ResourceBundler<r=r3rrrtuplerrrLr) r#rK bundle_demandpg_bundle_demandrrGpg_filtered_bundlerG demand_linesrs rformat_resource_demand_summaryrSs6>6666& +C00M".s33(QQ  / .v 6 6  ! " I48J8O8O8Q8QQQ"#GH !&&(( ) )Q . . eF#5#;#;#=#=>>??@@@EI@@@  Q U6*<*B*B*D*D#E#EFF G G G5 P G G GL&,,.."" @4<<@@5@@@ % % % L)&1LLL LDD!!!! rr%cg}|D]#\}}d|d|d}||$t|dkrd|}nd}|S)aReturns a formatted string describing the resource constraints from request_resources(). Args: request_demand: List of tuples containing resource bundle dictionaries and counts from request_resources() calls. Returns: String containing the formatted constraints report, either listing each constraint and count or indicating no constraints exist. Example: >>> request_demand = [ ... ({"CPU": 4}, 2), ... ({"GPU": 1}, 1) ... ] >>> get_constraint_report(request_demand) " {'CPU': 4}: 2 from request_resources()\n {'GPU': 1}: 1 from request_resources()" r(rLz from request_resources()rrz (none))rrrf)r%constraint_linesrrGrconstraints_reports rget_constraint_reportrWs&'&& =6==U===%%%% q  !YY'788& rc4g}|jr'|t|j|jD]4}|\}}t |}d|d|d}||5t |dkrd|}nd}|S)Nr(rLz+ pending placement groupsrrz (no resource demands))r#extendrSr$rrrrf)r=rRentryr rGpg_strr demand_reports rget_demand_reportr]sL!X::;UVVWWW%"" E2>6>>U>>>D!!!! <1 ,// 0 rci}|jD]:\}}d}t|dD] }||dz } |||<;|S)NrT)r#r)r(rr<strip)r=per_node_breakdownnode_idr" usage_stringrs rget_per_node_breakdown_as_dictrc s$288::;; t444 ( (D tKKK 'LL&2&8&8&:&:7## rr)node_activitiesct}|i}t||jD]\}}t|d|}||vr||}|d|dz }t|||jr:||jvr1|j|dkr td|j|d|td|t ||D]} td | ||std |||vrtd |||\} } | D]} td | | |S) Nr?zNode: r r rz Idle: z msz Usage:z z Activity:z (no activity))rrAr(rr*r<rB) r=r)rdr#rCrar" node_stringr\rrreasonsreasons rget_per_node_breakdownris **C  sOOOO$288::// 3(w(( ' ' ')'2I , ,,, ,K k$$$$  $ N:333(1A55 BJ4W=BBB M M M M ic""""w// ) )D +t++C ( ( ( ( (   l%%%% / ) ) ## . . . . .)1JAw! / /m6mm#..... / <<>>rgcs_request_timenon_terminated_nodes_timeautoscaler_update_timec  |tj}dd|dzdz}dt|z}|r)|dz }|r |d|ddz }|r |d |ddz }|r |d |ddz }g} |jsd } nQ|jD]"\} } d| d| } | | #d| } |jsd }nSg}|jD]"\} } d| d| } || #d|}g}|jD]#\} } d| d | d} || $|j D]8\}} }d|d| d | } || 9|rd|}nd}g}|j D]#\}} d| d|d} || $|j rt|j jd}|D]}|jr |jJ|j} |jj}|jj}tj|j}|jdd|jdd|jd}d| d|d|d} |r| d|z } || |dt4j d}d}|r|d|z }n|dz }t9||}t;|j}t?|}|d|d| }|j s|d|z }|d|d|d |d!|d"|d#| z }|r,|j!r |tE||j#|j$|z }n|dz }|%S)$Nz========z Autoscaler status: r(-rzGCS request time: 3fzs z)Node Provider non_terminated_nodes time: zAutoscaler iteration time: z (no active nodes)z (no idle nodes)r z launchingrLz (no pending nodes)z: NodeTerminated (ip: r c|jSr8)last_checked_timestamp)records rrz$format_info_string..s v<r)rF02drz (latest_attempt: z - zRecent failures: z (no failures)z Node status z Active: z Idle: z Pending: z Resources z Total Usage: z From request_resources: z Pending Demands: )&rnowr active_nodesrrrf idle_nodespending_launches pending_nodeslower failed_nodesnode_availability_summaryrnode_availabilitiesrP is_availableunavailable_node_informationr\category description fromtimestamprqhourminutesecondr!AUTOSCALER_MAX_FAILURES_DISPLAYEDrDrWr%r]legacyr(rir)rdr_) r=autoscaler_summarytimerjrkrlr#header separatoravailable_node_report_linesavailable_node_reportr\rGridle_node_reportidle_node_report_lines pending_linesipstatuspending_report failure_linesrecordsrrrrattempted_timeformatted_timefailure_report usage_reportrVr\formatted_outputs rformat_info_stringrEs ||~~ 5d555 5 ?Fc&kk!I V$  D C+;CCCC CF $  5,4555 F " V U7M4UUUU UF"$  *G 4 2 ? E E G G 5 5 Iu*u**y**D ' . .t 4 4 4 4 $ *E F F  (=-!# 2 = C C E E 0 0 Iu*u**y**D " ) )$ / / / /99%;<<M.?EEGG## 51911111T""""!3!A##Iv626666fllnn66T""""/=11.M+8## I9999B999T""""3'  8 L S S U U<<    ' 'F" 6BBB(I:CH =IK%3F4QRRN"&-//!(///!(.// RyQQHQQQQQD ,+k+++   & & & &!"UY%P$PSU"UVM)N+$))M222**#J88L.z/HII%j11M"    $                 %  # %  6"4"2 !!     $   ! ! # ##rrac,d|S)z9The anonymous node type for readonly node provider nodes.znode_{}rc)ras rformat_readonly_node_typers   G $ $$rr\ci}tjt}|D]G\}}t |\}}}|r|s||vrd||<||xx|z cc<7||xx|z cc<Hdg}|D]6\}}|d|} ||vr||} | d| dz } || 7d|S)NrrrLr r)z )r<r=r*rrrrf) r\r,regular_resource_usager.r0r1r2r3 output_lines output_line pg_resources rformat_no_node_type_stringrsJ%'"(4U;;$??,, 6 6%4V 5 5 1 7L  6 'EEECD./?@ *+; < < < E < < < < "8 , , , 5 , , , ,4L17799))%!,,U,, 5 5 58BK K KKK KKK(((( ;;| $ $$rcddlm}ddlm}ddlm}||dd}||j j |j j  d}| |j j|jj|  d}||fS) Nr)default_backend) serialization)rsaii)backendpublic_exponentkey_sizer)encodingrcencryption_algorithm)cryptography.hazmat.backendsrcryptography.hazmat.primitivesr)cryptography.hazmat.primitives.asymmetricrgenerate_private_key public_key public_bytesEncodingOpenSSH PublicFormatdecode private_bytesPEM PrivateFormatTraditionalOpenSSL NoEncryption)rrrrFrpems rgenerate_rsa_key_pairrs<<<<<<<<<<<<====== " "!!54 #  C    " *M,F,N       '+*=*7799    fWoo  s?rctjd|}tjd|}||fS)Nz ~/.ssh/{}.pubz ~/.ssh/{})rdrerrc)key_namepublic_key_pathprivate_key_paths rgenerate_ssh_key_pathsrsUg(()?)?)I)IJJOw))+*<*rs3   !!!!!! :::::::::::::::: (((((((((BBBBBB.-----999999AAAAAA>>>>>>AAAAAANNNNNNNN ( K$c5e,,- $,    #s(^CI       S%' '(  8 $ $st 3333333 30////////4: DcN: t: : : : zES#XE4EEEE84S>d38n2DcN"T#s(^S#X<94S>9d3PS8n9999xMMM4DI4S>c65555   $ (- J>J>J>J>Z   )))6"%6 3 t #$6666>  V       ;u;t;S ;;;;|!3ds7% 3457 #Y7777t$y/< "4     "      )")S%Z 01)d3S >23) )  ))))^ (,15.2B$B$uo B$ ( B$ %UO B$B$B$B$B$J%s%%%% %$%%%%64---      r