.`i= ddlZddlZddlZddlZddlZddlmZmZddlmZddl m Z ddlm Z ddl m Z ddlmZmZmZmZmZmZmZddlZddlmZddlmZdd lmZdd lmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(erddl)Z*ddl+m,Z,ddl-m.Z.m/Z/ee0Z1edZ2Gddee2eZ3GddZ4d2de5de6de7de6fdZ8GddZ9 d3de9deddzded ddfd!Z:d"e;e fd#Zej?fd(eddfd)Z@daAd*e6defd+ZBd,ej=deCfd-ZDe Gd.d/ZEd0e(deEfd1ZFdS)4N)CallableSequence)AbstractContextManager) dataclass) connection) BaseProcess) TYPE_CHECKINGAnyGenericOptionalTypeVarUnionoverload)record_function) init_logger) UsageContextis_usage_stats_enabled usage_message) get_open_portget_open_zmq_ipc_path get_tcp_uri)kill_process_tree)SchedulerOutput) DPCoordinatorCoreEngineActorManagerCoreEngineProcManagerTc eZdZdeeddfdZdZdZdZdZ d Z d Z dd ed e de dzde fdZ ed e defdZededeefdZd e ezdeeezfdZed e defdZededefdZd e ezdeeezfdZdZdZdZdZdZdeefdZdS) ConstantListxreturnNc||_dSN_x)selfr!s a/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/utils.py__init__zConstantList.__init__-s c td)Nz Cannot append to a constant list TypeErrorr'items r(appendzConstantList.append0s:;;;r*c td)NzCannot extend a constant listr,r.s r(extendzConstantList.extend3s7888r*c td)Nz"Cannot insert into a constant listr,r.s r(insertzConstantList.insert6<===r*c td)NzCannot pop from a constant listr,r.s r(popzConstantList.pop9s9:::r*c td)Nz"Cannot remove from a constant listr,r.s r(removezConstantList.remove<r5r*c td)NzCannot clear a constant listr,r's r(clearzConstantList.clear?s6777r*rr/startstopcf|j||||nt|jSr$)r&indexlen)r'r/r=r>s r(r@zConstantList.indexBs+w}}T5$2B$$DG UUUr*cdSr$r.s r( __getitem__zConstantList.__getitem__Es+.3r*scdSr$rC)r'rEs r(rDzConstantList.__getitem__Hs363r*c|j|Sr$r%r.s r(rDzConstantList.__getitem__Kswt}r*valuecdSr$rCr'r/rHs r( __setitem__zConstantList.__setitem__Ns03r*cdSr$rC)r'rErHs r(rKzConstantList.__setitem__Qs25#r*c td)Nz"Cannot set item in a constant listr,rJs r(rKzConstantList.__setitem__Tr5r*c td)Nz'Cannot delete item from a constant listr,r.s r( __delitem__zConstantList.__delitem__WsABBBr*c*t|jSr$)iterr&r;s r(__iter__zConstantList.__iter__ZsDG}}r*c||jvSr$r%r.s r( __contains__zConstantList.__contains__]stwr*c*t|jSr$)rAr&r;s r(__len__zConstantList.__len__`s47||r*cd|jdS)Nz ConstantList()r%r;s r(__repr__zConstantList.__repr__cs)tw))))r*c4|jSr$)r&copyr;s r(r[zConstantList.copyfsw||~~r*)rN)__name__ __module__ __qualname__listrr)r0r2r4r7r9r<intr@rrDslicerKrOrRrTrVrYr[rCr*r(r r ,s'$q'd<<<999>>>;;;>>>888VV!VCV3:VVVVV.....X. 6U6$q'666X6e DG 33A333X3 5U51555X5>e >AQK>>>>CCC***d1gr*r c eZdZdZdddeejzdejdejde de d d f d Z dd ed zd ej fd Z dd ed zd ej fdZ d S) CpuGpuBufferz2Buffer to easily copy tensors between CPU and GPU.T) with_numpysizedtypedevice pin_memoryrdr"Nctj||d|d|_tj|j||_||r?|tjkrt d|j|_dSdS)Ncpu)rfrgrh)rgzkBfloat16 torch tensors cannot be directly cast to a numpy array, so call CpuGpuBuffer with with_numpy=False) torchzerosrj zeros_likegpubfloat16 ValueErrornumpynp)r'rfrgrhrdres r(r)zCpuGpuBuffer.__init__ms;E%JWWW#DHV<<<   '&& Nhnn&&DGGG  ' 'r*nc|!|j|jdS|jd||jd|dS)NT non_blocking)rncopy_rjr'rss r( copy_to_gpuzCpuGpuBuffer.copy_to_gpusN 98>>$(>>> >x|!!$(2A2,T!BBBr*c|!|j|jdS|jd||jd|dS)zzNOTE: Because this method is non-blocking, explicit synchronization is needed to ensure the data is copied to CPU.NTru)rjrwrnrxs r( copy_to_cpuzCpuGpuBuffer.copy_to_cpusP 98>>$(>>> >x|!!$(2A2,T!BBBr*r$)r\r]r^__doc__r`rkSymIntrfrgboolr)Tensorryr{rCr*r(rcrcjs<< '''U\!'{' '  '  ' '''',CCS4ZC5<CCCC CCS4ZC5<CCCCCCr*rc local_onlyhostportr"c^|rtnt||p tS)a Assign a new ZMQ socket address. If local_only is True, participants are colocated and so a unique IPC address will be returned. Otherwise, the provided host and port will be used to construct a TCP address (port == 0 means assign an available port).)rrr)rrrs r(get_engine_client_zmq_addrrs3  :$ 7 88r*cneZdZdZ ddedededejde de ed e ed edzfd Z dd Z dS)APIServerProcessManagerzManages a group of API server processes. Handles creation, monitoring, and termination of API server worker processes. Also monitors extra processes to check if they are healthy. Ntarget_server_fnlisten_addresssockargs num_serversinput_addressesoutput_addressesstats_update_addressc  ||_||_||_tjd} g|_t t|||D]a\} } } | | || d} ||| d<| |d| |||| f}|j || bt dt|jtj|t |j|_dS)a7Initialize and start API server worker processes. Args: target_server_fn: Function to call for each API server process listen_address: Address to listen for client connections sock: Socket for client connections args: Command line arguments num_servers: Number of API server processes to start input_addresses: Input addresses for each API server output_addresses: Output addresses for each API server stats_update_address: Optional stats update address spawn) input_addressoutput_address client_count client_indexNr ApiServer_)targetnamerzStarted %d API server processes)rrrmultiprocessing get_context processesziprangeProcessr0r=loggerinforAweakreffinalizeshutdown _finalizer)r'rrrrrrrr spawn_contextiin_addrout_addr client_configprocs r(r)z APIServerProcessManager.__init__s(.-  (3G<< ,.$' +  1A% %    Aw")"* + ! M $/8L 45 (('%!%%$dD-@)D N ! !$ ' ' ' JJLLLL 5s4>7J7JKKK"*44>JJr*r"c.|dSr$)rr;s r(closezAPIServerProcessManager.closes r*r$)r"N) r\r]r^r|rstrr argparse Namespacer`r_r)rrCr*r(rrs,07K7K"7K7K 7K  7K  7Kc7Ks)7K"Dj7K7K7K7Krr*rapi_server_managerengine_manager)rr coordinatorrc ~ddlm}m} tdd|jD}|r|j||jj<g}t||r|jD] }|||j< n$t||r| }|s|rtj |d}|D]I} | | }|j dkr'td|jd|jd |j J|rdd l} | |d\} }||n`#t$$rtd Yn:t&$r.} td t+| d } ~ wwxYwtd ||r||r|d Sd S#td ||r||r|wwxYw)aWait for all processes to complete or detect if any fail. Raises an exception if any process exits with a non-zero status. Args: api_server_manager: The manager for API servers. engine_manager: The manager for engine processes. If CoreEngineProcManager, it manages local engines; if CoreEngineActorManager, it manages all engines. coordinator: The coordinator for data parallel. rrz'Waiting for API servers to complete ...ci|] }|j| SrC)sentinel).0rs r( z2wait_for_completion_or_failure..s)4 4 4 $(DM44 4 4 r*)timeoutzProcess z (PID: z) died with exit code Nz8Received KeyboardInterrupt, shutting down API servers...z0Exception occurred while running API servers: %sz#Terminating remaining processes ...)vllm.v1.engine.utilsrrrrrrr isinstance get_run_refsrwaitr7exitcode RuntimeErrorrpidrayKeyboardInterrupt Exception exceptionrr) rrrrrsentinel_to_procactor_run_refsrready_sentinelsrr_es r(wait_for_completion_or_failurers$SRRRRRRR2# =>>>4 4 ,>,H4 4 4   K:E:J [-6 7 n&; < < ;&0 7 726 // 7 (> ? ? ;+88::N H. H)39IST)U)U)UO,  '++H55=A%%&?49??TX??/3}??&  H $'HH^QH$G$G!>% H. H( PPP NOOOOO KSQRVVTTT   9:::  """         #  " " " " " # #  9:::  """         #  " " " " #s7DD! G!$E>G E>)E99E>>GAH<procsc|D]*}|r|+tjdz}|D]I}|tjz }|dkrn*|r||J|D].}|r|jx}t |/dS)Nrr)is_alive terminatetime monotonicjoinrr)rrdeadline remainingrs r(rr/s ==??  NN   ~!#H!!t~/// >> E ==?? ! IIi ## ==?? #tx< c " " "##r* from_tensor to_tensorlengthcP|d||d|dS)z Copy the first length elements of a tensor into another tensor in a non-blocking manner. Used to copy pinned CPU tensor data to pre-allocated GPU tensors. Returns the sliced target tensor. NTru)rw)rrrs r( copy_slicerCs/ WfW  # #K$8t # L LLr* usage_contextc HtsdSddlm}|j}d}|j |jj}t j||j|idt|jj d|j j d|j j d|j jd|jjd t|j jd t#|jd |j jd |jjd |jd|jd|jd|jd|jd|jd|dS)z#Report usage statistics if enabled.Nr)get_architecture_class_namerf block_sizegpu_memory_utilizationkv_cache_memory_bytes quantizationkv_cache_dtype enable_loraenable_prefix_caching enforce_eagerdisable_custom_all_reducetensor_parallel_sizedata_parallel_sizepipeline_parallel_sizeenable_expert_parallelall2all_backend kv_connector) extra_kvs)r vllm.model_executor.model_loaderrparallel_configkv_transfer_configrr report_usage model_configrrf cache_configrrrr cache_dtyper~ lora_configrrrrrrrr) vllm_configrrrrs r(report_usage_statsrQs " # #LLLLLL!1OL%1"5B ##K$<== S1788  +2=  %k&>&U  $[%=%S   K4A  c+":"FGG  4 788  $[%=%S  [5C  ()R  #O$H !/"D! " %o&L# $ %o&L% ( >) , L- r*rctt|Stj}tjrt }ntjr ddl}|j}|a||S)Nr) _PROFILER_FUNC contextlib nullcontextenvs VLLM_CUSTOM_SCOPES_FOR_PROFILINGrVLLM_NVTX_SCOPES_FOR_PROFILINGnvtxannotate)rfuncrs r(record_function_or_nullcontextrs`!d###  !D , , }N 4::r*tensorc|tjjS)zGet the raw data of a tensor as a uint8 memoryview, useful for serializing and hashing. Args: tensor: The input tensor. Returns: A memoryview of the tensor data as uint8. )flatten contiguousviewrkuint8rqdata)rs r( tensor_datars? >>   & & ( ( - -ek : : @ @ B B GGr*cDeZdZUeed<eed<eed<eed<defdZdS)IterationDetailsnum_ctx_requestsnum_ctx_tokensnum_generation_requestsnum_generation_tokensr"c Hd|jd|jd|jd|jd S)Nz"IterationDetails(num_ctx_requests=z!, num_ctx_tokens=z+, num_generation_requests=z), num_generation_tokens=rX)r r r r r;s r(rYzIterationDetails.__repr__s]FD4IFF!%!4FF*.*FFF)-(BFFF Fr*N)r\r]r^r`__annotations__rrYrCr*r(r r sh    F#FFFFFFr*r scheduler_outputcd}d}d}d}d|jD}|jD]8\}}|j|s||vr |dz }||z }.|dz }||z }9t ||||S)a Compute the number of context/generation requests and tokens for the current iteration's scheduler output. A requests is regarded as a context request if its output tokens are still 0, an extended chunk of chunked prefill falls into this category. Args: scheduler_output: The scheduler output for the current iteration. Returns: An IterationDetails object containing the number of context/generation requests and tokens. rch|] }|j SrC)req_id)rnew_reqs r( z,compute_iteration_details..sUUUg7>UUUr*)scheduled_new_reqsnum_scheduled_tokensitemsscheduled_cached_reqsis_context_phaser )rnum_context_requestsnum_context_tokensr r  new_req_idsr num_tokenss r(compute_iteration_detailsr sUU1A1TUUUK.CIIKK00  1 B B6 J J 0 k ! ! A % * ,   #q ( # !Z / ! !    r*)r)NN)Grrrrrcollections.abcrrr dataclassesrrmultiprocessing.processrtypingr r r r r rrrktorch.autograd.profilerr vllm.envsr vllm.loggerrvllm.usage.usage_librrrvllm.utils.network_utilsrrrvllm.utils.system_utilsrvllm.v1.core.sched.outputrrqrrvllm.v1.engine.coordinatorrrrrr\rrr rcr~rr`rrrr_rrrENGINE_CONTEXTrrr memoryviewrr r rCr*r(r/sk ........------!!!!!!&&&&&&////// 333333######TTTTTTTTTTVVVVVVVVVV555555555555S888888RRRRRRRR X   GCLL;;;;;71:x;;;|#C#C#C#C#C#C#C#CL  4 s # c     AAAAAAAAN-1 F#F#/F#KL  F#/* F#  F#F#F#F#V#D%####( M M*/, M@C M \ M M M M0