&`iB$ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ej dZ dZ dZdZdddd Zd Zd Z d%dZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$d Z%d!Z&d"Z'd#Z(d$Z)dS)&N)is_ipv6zray.util.spark.utilscdtjvS)NDATABRICKS_RUNTIME_VERSION)osenvironh/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/util/spark/utils.pyis_in_databricks_runtimer s '2: 55r cnd|}d|}d|d|d|dS)N zCommand z failed with return code z", tail output are included below.  )join)cmd return_codetail_output_dequecmd_str tail_outputs r gen_cmd_exec_failure_msgrsWhhsmmG''+,,K ,7 , ,[ , ,' , , ,r c|jdd}t|dd}|d}ddddd}|||zS) Nzspark.executor.memory1gii@l)kmgt)confgetlowerint)spark value_str value_num value_unitunit_maps r *get_configured_spark_executor_memory_bytesr(sl 6==CCEEIIcrcN##I2J    & H x + ++r T) extra_env synchronousc  t|hd}|rtdt ||dd}||td||nit j|}tj |f|dtj tj d|tj d   fd }tj|d |s fS}|d krt%t'|| dS)a A convenience wrapper of `subprocess.Popen` for running a command from a Python script. If `synchronous` is True, wait until the process terminated and if subprocess return code is not 0, raise error containing last 100 lines output. If `synchronous` is False, return an `Popen` instance and a deque instance holding tail outputs. The subprocess stdout / stderr output will be streamly redirected to current process stdout. >textstderrstdoutz`kwargs` cannot contain envNz5`extra_env` and `env` cannot be used at the same timeT)r/r,r.r-d)maxlencjD]6}|tj|7dSN)r.appendsyswrite)lineprocessrs r redirect_log_thread_fnz(exec_cmd..redirect_log_thread_fnPsMN # #D  $ $T * * * J  T " " " "  # #r r)targetargsr)setkeys intersection ValueErrorlistpoprr subprocessPopenPIPESTDOUT collectionsdeque threadingThreadstartwait RuntimeErrorr) rr)r*kwargsillegal_kwargsr/r9rr8rs @@r exec_cmdrO*s"''445Q5Q5QRRNLJD4H4HJJKKK **UD ! !CPQQQ"##(C2:(C(CC      G$)555######2<<<BBDDD *))),,..Ka $S+7H I I   r cddl}ddlm}||t|r|jn|j|j5}|||fdkcdddS#1swxYwYdS)Nr)closing)socket contextlibrQrAF_INET6AF_INET SOCK_STREAM connect_ex)hostportrRrQsocks r is_port_in_user[dsMMM""""""  &t}} @FOO&.&BT    2 d|,,1 222222222222222222s A22A69A6ctj}tj|z |kr@t||rdStjdtj|z |k@dS)NTF)timer[sleep)rXrYtimeoutbeg_times r _wait_service_uprbpsfy{{H )++ 7 * * $ % % 4 1  )++ 7 * * 5r rr0ctj}|pg}t|D]1}|||}||vrt ||s|cS2t d|d|d)z! Get random unused port. z!Get available port between range z and z failed.)random SystemRandomrangerandintr[rL)rXmin_portmax_port max_retries exclude_listrng_rYs r get_random_unused_portro{s    C%2L ;  {{8X.. <   dD)) KKK  MHMM8MMM  r c\ddlm}|}|td|S)Nr) SparkSessionzSpark session haven't been initiated yet. Please use `SparkSession.builder` to create a spark session and connect to a spark cluster.) pyspark.sqlrqgetActiveSessionrL)rq spark_sessions r get_spark_sessionrusJ(((((( 1133M    r c6|jdS)Nzspark.driver.host)rr )r#s r !get_spark_application_driver_hostrws :>>- . ..r c|j}|ld}|dgd||||jS|| S)z0Gets the current max number of concurrent tasks.NcdSr3r)rns r dummpy_mapperz3get_max_num_concurrent_tasks..dummpy_mappers Dr r]) _jscsc parallelize withResourcesmapcollectmaxNumConcurrentTasks_java_resource_profileresourceProfileManagerdefaultResourceProfile) spark_contextresource_profilesscrzs r get_max_num_concurrent_tasksrs     ! !C#    !!1#q))778HIIMM   ')))(()9)PQQQ((  & & ( ( ? ? A A   r cddl}ttjvr$t tjtS|jSNr)psutil)RAY_ON_SPARK_WORKER_PHYSICAL_MEMORY_BYTESrrr"virtual_memorytotal)rs r '_get_spark_worker_total_physical_memoryrsCMMM0BJ>>2:GHIII  " " ((r cddl}ttjvr$t tjtS|jdjSNrz/dev/shm)shutil'RAY_ON_SPARK_WORKER_SHARED_MEMORY_BYTESrrr" disk_usager)rs r %_get_spark_worker_total_shared_memoryrsDMMM."*<<2:EFGGG 6 Z ( ( ..r g?cddl}ddl}ttjvr%t tjt}n|j}|tz}ttjvr%t tjt}n|j dj}|tz}t||||\}}}|t |||fSr)rr)RAY_ON_SPARK_DRIVER_PHYSICAL_MEMORY_BYTESrrr"rr'_RAY_ON_SPARK_NODE_MEMORY_BUFFER_OFFSET'RAY_ON_SPARK_DRIVER_SHARED_MEMORY_BYTESr_calc_mem_per_ray_node_loggerwarning) configured_heap_memory_bytesconfigured_object_store_bytesrravailable_physical_memavailable_shared_memheap_mem_bytesobject_store_bytes warning_msgs r calc_mem_ray_head_nodersMMMMMM0BJ>>!$ J@ A" " "(!6!6!8!8!> !HH/"*<<"2:.U#VWW0v0<<B FF7M$% 773N&  $$$ - --r ct||z tz}t||z tz}t||||Sr3)r"rr)num_task_slotsphysical_mem_bytesshared_mem_bytesrravailable_physical_mem_per_nodeavailable_shared_mem_per_nodes r _calc_mem_per_ray_worker_nodersb'*^+.UU''#%(>),SS%%! "'%$%   r c@ddlm}m}d}|p||z}tjds||kr|}|t z}||kr|}d}||kr||krd|d}nd|d}|}t|}|t||z } nt|} | ||fS) Nr)&DEFAULT_OBJECT_STORE_MEMORY_PROPORTION!OBJECT_STORE_MINIMUM_MEMORY_BYTES#RAY_OBJECT_STORE_ALLOW_SLOW_STORAGEzzYour configured `object_store_memory_per_node` value is too high and it is capped by 80% of per-Ray node allocated memory.zYour operating system is configured with too small /dev/shm size, so `object_store_memory_worker_node` value is configured to minimal size (z- bytes),Please increase system /dev/shm size.zYou configured too small Ray node object store memory size, so `object_store_memory_worker_node` value is configured to minimal size (zI bytes),Please increase 'object_store_memory_worker_node' argument value.)ray._private.ray_constantsrrrrr 0_RAY_ON_SPARK_MAX_OBJECT_STORE_MEMORY_PROPORTIONr") rrrrrrrrobject_store_bytes_upper_boundrs r rr sI  K6'*PP :>>? @ @?  = = =!>  ( : ;# :::;   === !> > >9$E999 KT$ETTT ?/00#+(..0066t<<===   ("1gg ( (   qqqqq sAB!! C +*CC c||krtd|d|d||z}|dkr*||krtd|d|d|||zkr||z}|S)Nzscpu number per Ray worker node should be <= spark worker node CPU cores, you set cpu number per Ray worker node to z* but spark worker node CPU core number is .rz|gpu number per Ray worker node should be <= spark worker node GPU number, you set GPU devices number per Ray worker node to z- but spark worker node GPU devices number is )r?)num_cpusnum_gpusnum_cpus_per_nodenum_gpus_per_nodenum_ray_node_slotss r _get_local_ray_node_slotsrs 8## @9J @ @4< @ @ @   "%661 x ' '"$"""""  ,= = = =!)->!>  r ct}|dkrt}nd}t||||}t}t }t |||||\} } } | | d| fS)z Returns tuple of ( ray_worker_node_heap_mem_bytes, ray_worker_node_object_store_bytes, error_message, # always None warning_message, ) rN)rrrrrr) rrheap_memory_per_nodeobject_store_memory_per_noderrrrrray_worker_node_heap_mem_bytes"ray_worker_node_object_store_bytesrs r "_get_avail_mem_per_ray_worker_nodersH1)++2(-/@ABB<>> &$    &* '*   r cfd}|jdgd|d\}}}} |t d|d| t | ||fS)a< Return the available heap memory and object store memory for each ray worker, and error / warning message if it has. Return value is a tuple of (ray_worker_node_heap_mem_bytes, ray_worker_node_object_store_bytes, error_message, warning_message) NB: We have one ray node per spark task. c tS#t$rQ}ddl}d||j}ddt ||zdfcYd}~Sd}~wwxYw)Nrrr)rr tracebackr format_tb __traceback__r)rnrr trace_msgrrrrs r mapperz1get_avail_mem_per_ray_worker_node..mappers 55!!$,    5 5 5     )"5"5ao"F"FGGIr477Y.4 4 4 4 4 4 4  5s A0AA+%A0+A0r]rNz:Inferring ray worker node available memory failed, error: a. You can bypass this error by setting following spark configs: spark.executorEnv.RAY_ON_SPARK_WORKER_CPU_CORES, spark.executorEnv.RAY_ON_SPARK_WORKER_GPU_NUM, spark.executorEnv.RAY_ON_SPARK_WORKER_PHYSICAL_MEMORY_BYTES, spark.executorEnv.RAY_ON_SPARK_WORKER_SHARED_MEMORY_BYTES.) sparkContextr}rrrLrr) r#rrrrr'inferred_ray_worker_node_heap_mem_bytes+inferred_ray_worker_node_object_store_byteserrrs ```` r !get_avail_mem_per_ray_worker_noders 5 5 5 5 5 5 5 5. &&sA..226::BBDDQG /3    I I I I    $$$/3 r cdtjvr=dtjddDfd|DS|S)NCUDA_VISIBLE_DEVICEScPg|]#}t|$Sr)r"r).0devs r z9get_spark_task_assigned_physical_gpus.. s5! ! ! !$C  ! ! ! r ,c g|] }| Srr)raddrvisible_cuda_dev_lists r rz9get_spark_task_assigned_physical_gpus..sFFF%d+FFFr )rrr) gpu_addr_listrs @r %get_spark_task_assigned_physical_gpusr si++! ! (* 3I(J(P(PQT(U(U! ! ! GFFF FFFFr )rrcr0N)*rFloggingrrerrBr5rHr^ray._common.network_utilsr getLoggerrr rr(rOr[rbrorurwrrrrrrrrrrrrrrrrrrrrrr r rs&   ------ ' 2 3 3666 , , ,  7 7 7 7 7 t 2 2 2HL*   ///   *)))///470+.'#.#.#.L*=;=;=;J!@;,W)*S'-X)*S' ' ' ':8+++\888vr