.`iEddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z mZddlmZmZddlmZddlmZmZmZddlZddlZdd lmZmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*m+Z+ddl,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mm?Z?ddl@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKddlLmMZMmNZNmOZOddlPmQZQddlRmSZSddlTmUZUddlVmWZWddlXmYZYmZZZdd l[m\Z\m]Z]dd!l^m_Z_dd"l`maZadd#lbmcZde"eeZfd$Zgd%Zhed&ZiGd'd(ZjGd)d*ejZkGd+d,ekZlGd-d.ZmGd/d0emelZnGd1d2emekZodS)3N)deque)Callable Generator)Future) ExitStackcontextmanager)isclass signature)DEBUG)AnyTypeVarcast)ParallelConfig VllmConfig)1stateless_destroy_torch_distributed_process_group)enable_envs_cache) init_logger)dump_engine_exception) LoRARequest)MULTIMODAL_REGISTRY) POOLING_TASKS SupportedTask)(maybe_register_config_serialize_by_value)freeze_gc_heapmaybe_attach_gc_debug_callback)get_hash_fn_by_name)make_zmq_socket) decorate_logsset_process_title) BlockHash"generate_scheduler_kv_cache_configget_kv_cache_configsget_request_block_hasherinit_none_hash)SchedulerInterface)SchedulerOutput) EngineCoreOutputEngineCoreOutputsEngineCoreRequestEngineCoreRequestType FinishReasonReconfigureDistributedRequestReconfigureRankType UtilityOutput UtilityResult)EngineHandshakeMetadataEngineZmqAddressesget_device_indices)Executor) KVCacheConfig)SchedulerStats)ModelRunnerOutput)Request RequestStatus)MsgpackDecoderMsgpackEncoder)StructuredOutputManager)compute_iteration_details) __version__g@_Rc eZdZdZ ddefdZ d Z! d?d!ed"ed efd#Z"d@d%e fd&Z#dAd'eedzfd(Z$d efd)Z%d*Z&d+e'd efd,Z(d-e d efd.Z)d e*e fd/Z+d-e d efd0Z, dBd1ed2edzd3e dzd dfd4Z- dCd6eed e.fzd7e/dzd8e d9eee0fdzd ee.f d:Z1de2d e ee ffd;Z3dS)D EngineCorezInner loop of vLLM's Engine.NF vllm_configexecutor_class log_statsexecutor_fail_callbackinclude_finished_setcpddlm}|||_|jjs!t dt|||_|||_ ||j |d|_ | |\}}} ||j _||j _|d||ft#||_|j} t+| jdkr2|jjr&t dd|j_|j j|jjz|jjz} | || |j||j| |_|jdu|_|jj$|j |jjtBx|_"} | #||_$|j%} | N|j &}|r3i}|D]}||'|| (||j j)|_*d|_+|j*d kr:t ,d |j*t[|j* |_+|j.duo |j.j/|_/|j0j1d k|_2d|_3|j j4s| =tk|j j6}to|tq| ||_3|j+|j9n|j:|_;|jj<|_<t{j>t~t|_AtttdS)Nr)load_general_pluginsz2Initializing a V1 LLM engine (v%s) with config: %sinitialize_cacheargsz3Disabling chunked prefill for model without KVCacheF)rBkv_cache_configstructured_output_managerrFrD block_sizez#Batch queue is enabled with size %d)maxlenpooling)E vllm.pluginsrHrBparallel_configdata_parallel_rank_localloggerinfo VLLM_VERSIONrDmodel_executorregister_failure_callback!available_gpu_memory_for_kv_cache_initialize_kv_caches cache_confignum_gpu_blocksnum_cpu_blockscollective_rpcr;rNscheduler_configget_scheduler_clslenkv_cache_groupsenable_chunked_prefillwarningrOdecode_context_parallel_sizeprefill_context_parallel_size schedulerspeculative_configuse_spec_decode connectorinit_kv_output_aggregatorr mm_registry!engine_receiver_cache_from_configmm_receiver_cacheget_kv_connector#get_kv_connector_handshake_metadataupdateset_xfer_handshake_metadatamax_concurrent_batchesbatch_queue_size batch_queuedebugrec_transfer_configis_ec_producer model_config runner_typeis_pooling_modelrequest_block_hasherenable_prefix_cachingrprefix_caching_hash_algor$r#stepstep_with_batch_queuestep_fnasync_schedulingqueueQueueliststr aborts_queuerrr)selfrBrCrDrErFrHr^r_rM Schedulerscheduler_block_sizern kv_connectorxfer_handshake_metadatacontent worker_dictcaching_hash_fns g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/engine/core.py__init__zEngineCore.__init__Qs 655555&*C  KKD    #-n[99 ! -   9 9:P Q Q Q13.;?:T:T ; ; 73A  /2@  / .nn5UVVV)@)M)M& 0BBDD . / /1 4 4+B LTUUUFK ,C  $ /)F G)G H  .7Y#+&*&D!5n+ . . .  +=TI > # /   9 9$.:R S S S)<<;!,!N!N " " ~6688  ##GGII $' B+-#:44K".{33388AAA !% 3 J    1 $ $ LL>@U V V V$D,ABBBD   *$ 6 >.= !, 8 D QQU!  # 9 \=U1(AO ? + + +(@$o))D % )1DIIt7Q !, < M!KS 244 &((( returnctj}|j}td|D}|rtjddkrHt|dd}|Jtj |d|_ |j gt|z}n:|j }|d|_ ndgt|z}t|t|ksJ|j j}t|||}|j j} | |kr|d| ft#|} | j} d} |j|tj|z } t(d | d | | | fS) Nc3K|]}|VdSN).0 kv_cache_specs r z3EngineCore._initialize_kv_caches..s"MM]=MMMMMMrVLLM_ELASTIC_EP_SCALE_UP_LAUNCH1dp_grouprIrupdate_max_model_lenrKzFinit engine (profile, create kv cache, warmup model) took %.2f secondslocal)scope)timerYget_kv_cache_specsanyosenvirongetgetattrrsync_kv_cache_memory_sizer[rcdetermine_available_memoryr{ max_model_lenr"r`r! num_blocksinitialize_from_configrV info_once)rrBstartkv_cache_specs has_kv_cacheravailable_gpu_memorymax_model_len_beforekv_cache_configsmax_model_len_afterscheduler_kv_cache_configr^r_elapseds rr\z EngineCore._initialize_kv_cachess ,??AAMMnMMMMM  =z~~?@@CGG"4T::+++"""c*>&?&????? +7E/ )=  *6D "6 6 6    6>Q=S  T T T$FGW$X$X!2= 223CDDD)++% T     ~/HHHr.c|jjSr)rYsupported_tasksrs rget_supported_taskszEngineCore.get_supported_taskss"22rrrequest request_wavect|jts$tdt |j|jx}rAd|D}|j|vrtd|jd||j 3|j st d|j |dS)zAdd request to the scheduler. `request_wave`: indicate which wave of requests this is expected to belong to in DP case z!request_id must be a string, got c$g|] }|tv |Sr)r)rtasks r z*EngineCore.add_request..'s)'''t}?T?T?T?T?TrzUnsupported task: z Supported tasks: NzXGot kv_transfer_params, but no KVConnector found. Disabling KVTransfer for this request.) isinstance request_idr TypeErrortypepooling_paramsrr ValueErrorkv_transfer_paramsrirqrVrf add_request)rrrrsupported_pooling_taskss rrzEngineCore.add_requests#',c22 ND9K4L4LNN %3 3> ''!%!9!9!;!;''' #"*AAA B)<BB(?BB  % 1//11 2 NN9    ""7+++++r request_idscP|j|tjdS)z"Abort requests from the scheduler.N)rifinish_requestsr8FINISHED_ABORTED)rrs rabort_requestszEngineCore.abort_requests;s% &&{M4RSSSSSrscheduler_outputc#K dVdS#t$r4}t|j||j|d}~wwxYw)z3Execute the model and log detailed info on failure.N) ExceptionrrBri make_stats)rrerrs rlog_error_detailzEngineCore.log_error_detailCsk  EEEEE    " "2DN4M4M4O4O   I s A/AAc#BK|jjjsdVdSt|dd|_t |}t j}dVt d dt|jdt|j dt|j dt|jdt|jd t!t j|z d zd d g |xjd z c_dS)N_iteration_indexrz Iteration(z): z context requests, z context tokens, z generation requests, z, generation tokens, iteration elapsed time: iz.2fz msrP)rBobservability_config enable_logging_iteration_detailsrrr<r monotonicrVrWjoinrnum_ctx_requestsnum_ctx_tokensnum_generation_requestsnum_generation_tokensformat)rriteration_detailsbefores rlog_iteration_detailsz EngineCore.log_iteration_detailsSs%4U  EEE F '.@! D D56FGG!!  GG -..):;;))899')ABB,)?@@BDN,,v5=uEE     & "rcf|jsidfS|j}|j|d}|j|}||5||5|}||j |}dddn #1swxYwYdddn #1swxYwY| |j ||}||j dkfS)zSchedule, execute, and make output. Returns tuple of outputs and a flag indicating whether the model was executed. FT non_blockNr) ri has_requestsschedulerY execute_modelget_grammar_bitmaskrrresult sample_tokens_process_aborts_queueupdate_from_outputtotal_num_scheduled_tokens)rrfuturegrammar_output model_outputengine_core_outputss rrzEngineCore.stepqs~**,, u9 >2244$223Ct2TT;; & & ( (* #~6688 -;; D<K& Q!1!Lq!P$ AN Af%67EE'H A&*^%G%G(&&N"0>>&$?FF 1A-, &&&0@+'NOOO"&K((4+@@@'OA.3355A &: ;4???3D3D0 .  ! !"2 3 3 7 7  & &'7 8 8 7 7"==??L#%%'''"#5666 $ 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ""$$$"n?? l   % U# "&"5"J"J"L"L&222??#%> "^??)N(66~QU6VVF  " "F,E{#S T T T"N22s6=G1:G G1G G1!G "G11G58G5cZ|jsg}|js_|j}|t |t r|fn||j_||dSdSr)rempty get_nowaitextendrrr)rridss rrz EngineCore._process_aborts_queues &&(( -K'--// L'2244""ZS-A-A#JC66sKKK'--// L    , , , , , - -rc|j|jr|j|jr|jdSdSr)rN clear_backendrYshutdownrirs rrzEngineCore.shutdownse &44666   +   ( ( * * * > & N # # % % % % % & &rTis_startc:|j|dSr)rYprofile)rrs rrzEngineCore.profiles ##H-----rc|jrtd|j|j|jdS)NzcResetting the multi-modal cache when requests are in progress may lead to desynced internal caches.)rihas_unfinished_requestsrVrfrp clear_cacherYreset_mm_cachers rrzEngineCore.reset_mm_cache"sq > 1 1 3 3  NND     ! -  " . . 0 0 0 **,,,,,rreset_running_requestsreset_connectorc8|j||Sr)rireset_prefix_cache)rrrs rrzEngineCore.reset_prefix_cache1s#~00 "O   rrPlevelc:|j|dSr)rYsleep)rrs rrzEngineCore.sleep8s !!%(((((rtagsc:|j|dSr)rYwake_up)rrs rr zEngineCore.wake_up;s ##D)))))rc|jjSr)rY is_sleepingrs rr"zEngineCore.is_sleeping>s"..rc8|jdSr)rYexecute_dummy_batchrs rr$zEngineCore.execute_dummy_batchAs //11111r lora_requestc6|j|Sr)rYadd_lora)rr%s rr'zEngineCore.add_loraDs"++L999rlora_idc6|j|Sr)rY remove_lorarr(s rr*zEngineCore.remove_loraGs"..w777rc4|jSr)rY list_lorasrs rr-zEngineCore.list_lorasJs"--///rc6|j|Sr)rYpin_lorar+s rr/zEngineCore.pin_loraMs"++G444rpathpatternmax_sizec@|j|||dS)N)r0r1r2)rYsave_sharded_state)rr0r1r2s rr4zEngineCore.save_sharded_statePs5 ..w /     rrmethodtimeoutrLkwargsc<|j||||Sr)rYr`)rr5r6rLr7s rr`zEngineCore.collective_rpcZs!"11&'4PPPrc|j+|jr$|j|j|_tj||j}|jr|j|||j fS)zPreprocess the request. This function could be directly used in input processing thread to allow request initialization running in parallel with Model forward ) rp mm_featuresget_and_update_featuresr7from_engine_core_requestr~use_structured_outputrN grammar_init current_wave)rrreqs rpreprocess_add_requestz!EngineCore.preprocess_add_requestcs  ! -'2E -"&"8"P"P###G .w8QRR  $ =  * 7 7 < < <G(((rrr)T)FF)rPr)NN)NrN)4__name__ __module__ __qualname____doc__rrr3boolrrtupleintr4r\rrr7rrrrrr&rrdictr(rrrrrrrrrr r"r$rr'r*setr-r/r4r?floatr r`r)rArrrrArANs&&37%* IIIXI I !)4 I # IIIIV9I%9I sC& '9I9I9I9Iv3U=#+=%>3333,,7,#,,,,BT$s)TTTT    ^ #o###^#:TeD&7!78$>?TTTT>GGGGGGr3 tC**+d2D8 9r3r3r3r3h---&&&...... - - - MR  &* EI     ))3))))**DI,****/T////222:[:T::::838488880CH0000555555 ##    t *       !%(, QQhsBw''QQ Q S#X% Q b QQQQ).?)E'SV,DW))))))rrAc,eZdZdZdZ d-dddededed ee d ed edzd e ffd Z e dede deded edzdeeddff dZe d-dejdede dededededzdeeddffdZe d-dejdedededzdef dZedddde de fdZdefdZdZdZdefdZd ed!eddfd"Zed#Z d$Z!d%e"ed&edzde d'e#j$fd(Z%d)e"ed*edzd e fd+Z&d!e'ddfd,Z(xZ)S).EngineCoreProcz9ZMQ-wrapper for running EngineCore in background process.sENGINE_CORE_DEADNr engine_indexrB local_clienthandshake_addressrCrDclient_handshake_addressrPc*tjtttf_tjtt tftz_ fd}|_ j dd} d_ || |||5} t| j_| jdu_| j_t(djjjo |jj } | _|t5||||| t9j} t9jj| j | j!| | fd} | "t9jj#| j| jj fd_$j$"| %d s\| &stOd | j!Jt((d | %d \ddddS#1swxYwYdS) NcPjtjdfS)Nr) input_queue put_nowaitr*EXECUTOR_FAILEDrsrz)EngineCoreProc.__init__..s%)9)D)D " 2C 8* * rlittle)length byteorderFz1Has DP Coordinator: %s, stats publish address: %sT)targetrLdaemon r6z'Input socket thread died during startupz0Waiting for READY message from DP Coordinator...))rrrHr*r rVrIr(bytes output_queuerPto_bytesengines_running_perform_handshakesrcoutputs client_countcoordinator_outputhas_coordinatorfrontend_stats_publish_addressrVrxrTdata_parallel_external_lbpublish_dp_lb_stats_init_data_parallelsuperr threadingEventThreadprocess_input_socketsinputscoordinator_inputrprocess_output_sockets output_threadwaitis_aliverrW)rrBrQrRrCrDrSrPrEidentity addressesinternal_dp_balancing ready_event input_thread __class__s` rrzEngineCoreProc.__init__s.!;u-BC-G'HIKK!Kc3D.D(E(MNPP" " " " )$--Q(-KK$  % %     $   H P  #I$5 6 6D $-#?t#KD 8  / LLC$3    $N#3MM " (=D $  $ $[ 1 1 1 GG  &%    $/++K$+1$/     L    !*!12%0% """D    $ $ & & &"&&r&22 P#,,..R&'PQQQ 2>>> NOOO "&&r&22 PIH PH PH PH PH PH PH PH PH PH PH PH PH PH PH PH PH PH Ps=F>JJ J rzrc #Ktj}|o|du}| }||||||||j} || 5} | Vdddn #1swxYwYnn|sJ||||dd|} | 5} | 5} | j| _| j| _| Vdddn #1swxYwYdddn #1swxYwY|dS)a Perform startup handshakes. For DP=1 or offline mode, this is with the colocated front-end process. For DP>1 with internal load-balancing this is with the shared front-end process which may reside on a different node. For DP>1 with external or hybrid load-balancing, two handshakes are performed: - With the rank 0 front-end process which retrieves the DP Coordinator ZMQ addresses and DP process group address. - With the colocated front-end process which retrieves the client input/output socket addresses. with the exception of the rank 0 and colocated engines themselves which don't require the second handshake. Here, "front-end" process can mean the process containing the engine core client (which is the API server process in the case the API server is not scaled out), OR the launcher process running the run_multi_api_server() function in serve.py. NTF)zmqContext_perform_handshakerTrtrg __post_init__) rrRrzrQrBrS input_ctxis_localheadless handshaker{local_handshakeclient_addressess rrfz"EngineCoreProc._perform_handshakess>KMM D$<$D##++        '   $ + i                 <"553XtUKO i >01  ! !'/"8"8"C"C D D DE" E" E" E" E" E" E" E" E" E" E" E" E" E" E" E" E" E" EsBCC  C rrTc*|tjd||dtd|tdzstdtd| }tj |t}td ||0|j D]\}}t||||jS) NHELLO)rrrz(Waiting for init message from front-end.i`raz7Did not receive response from front-end process within z minutesrzReceived init message: %s)rrrrrVrxpollHANDSHAKE_TIMEOUT_MINSrrecvdecoder0rTitemssetattrr{)rrQrrT init_bytes init_messagekeyvalues rrz EngineCoreProc.startup_handshakeJs4  O " "%) (       ?@@@$$-Cf-L$MM "8  &**,, 070F0F 41G1 1   0,???  &*:@@BB 5 5 Ue4444%%r)dp_rank local_dp_rankrrc, d t fd}tjtj|tjtj|d} |d}|j}|jdkp|dk}|r||_tdd|ntdt|rG|j @|j j d ||j _ t d |j j ||_ |r |jjr||_t#|i|}n#d|_d|_d|_t'|d |i|}|n#t*$rt d t,$rR} |td n.td|| d} ~ wwxYw ||dSdS#||wwxYw)z2Launch EngineCore busy loop in background process.Fc,sdtdSr) SystemExit)signumframeshutdown_requesteds rsignal_handlerz6EngineCoreProc.run_engine_core..signal_handler|s%% #%)" ll" # #rNrBrPrrADP_dpz*Setting kv_transfer_config.engine_id to %srPEngineCore exiting.zEngineCore failed to start.%EngineCore encountered a fatal error.)rsignalSIGTERMSIGINTrTrrUrrkv_transfer_config engine_idrVrxdata_parallel_indexr{is_moedata_parallel_rankDPEngineCoreProcdata_parallel_size_localrN run_busy_looprr exception_send_engine_deadr) rrrLr7r engine_corerBrT data_parallelers @rrun_engine_corezEngineCoreProc.run_engine_coreps# 1222 # # # # #  fnn555 fm^444-1 2 '&,]&;K.9.IO+>BQgPQkM 0;H8!,W????!,/// OOO !?!K#5?SSMSS.8 @2< 3:O / T!9!@ T5<2.??? 672;<8562,dSSFSS  % % ' ' ' '    LL. / / /    "  !>????  !HIII--///G   (&$$&&&&&'&{&$$&&&&'s,DEG:.G A GGG::HcdSrr)rrBs rrnz"EngineCoreProc._init_data_parallels rcV ||))z!Core busy loop of the EngineCore.)_process_input_queue_process_engine_steprs rrzEngineCoreProc.run_busy_loops0 (  % % ' ' '  % % ' ' '  (rcd}|js|js|js|jr}|jj5|jj dddn #1swxYwYt trt dd}|j}|j||js |js|j|rt d|js>|j}|j||j>Dc|s|St|j}t|t|ksJt dt ||DS)ztIf a provided arg type doesn't match corresponding target method arg type, try converting to msgspec object.c3K|]l\}}t|jrOt|jtjr0t ||jstj||jn|VmdS)rN)r annotation issubclassrStructrconvert)rvps rrz7EngineCoreProc._convert_msgspec_args..s  1 q|$$ 1<88 q!,// GOAAL 1 1 1 1       r)r parametersvaluesrcrHzip)r5rL arg_typess rrz$EngineCoreProc._convert_msgspec_argss Kf%%07799 4yyC NN****  D),,       rc|jtj|jd|jrtddSdS)z/Send EngineDead status to the EngineCoreClient.g@razNvLLM shutdown signal from EngineCore failed to send. Please report this issue.N) rcrWrNENGINE_CORE_DEADrwrryrVfatalrs rrz EngineCoreProc._send_engine_dead(s $$^%DEEE ,,,   & & ( (  LL5       rinput_addressescoord_input_addressr}c tt}t}t5tj5fd|D}|d}nFt |tjd}|dtj } |D]7} | d| | tj 8|:| dksJ| |tj | ~ | D]\} } | d ^} } t!t#| j}|t jkrQ|| } ||}nd#t,$r||YwxYw|| }|t jkr|j||j||f#1swxYwYddddS#1swxYwYdS) zInput socket IO thread.c rg|]3}t|tjd4S)Frzr) enter_contextrrr)r input_addressrrzstacks rrz8EngineCoreProc.process_input_sockets..Ds[ " ###]CJPUrNFrrsREADYT)copy)r9r)rrrrrXSUBrPollerregisterPOLLINrrKrrecv_multipartr*rbbufferrrrAr_handle_request_preproc_errorrrrWrV)rrrrzr}add_request_decodergeneric_decoder input_sockets coord_socketpoller input_socket_ type_frame data_framesrr@rrrs ` @@rrsz$EngineCoreProc.process_input_sockets6sT-->??(** [[B IE3;==B IC &5 M#*# $22#+!)"    !!'***Z\\F - : : !!#&&& cj9999'#((**h6666 cj999 OO    I'-{{}}IIOL!/;/J/JPU/J/V/V,J#8z?P9Q9Q#R#RL$'<'@@@1D1K1KK1X1X%&*&A&A#&F&FGG(%%% >>sCCC$H%#2"8"8"E"E'+@+FFF !-88AAA$//w0GHHHH5 IQB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IB IsUI,E2I;GIG3 0I2G3 3A!II I,I I,,I03I0 output_pathscoord_output_pathc @t}g}tttjt t f}t5tj5fd|D}|1 t|tj ddnd}t|dz} |j } | tjkr|D]} | | nft%| t&rJ| \} } || _| dkr-|J||| |rU|dd jrB||d |r|dd jB|r|n t }|| |}|| |dd }|js0t|dkr| nd}||||fn(t|| kr|| dddn #1swxYwYddddS#1swxYwYdS) zOutput socket IO thread.c pg|]2}t|tjd3S))r)rrrPUSH)r output_pathrrs rrz9EngineCoreProc.process_output_sockets..sQ ###CchtLLLrNFr)rrrPTrIrrZ)rtrack)r:rrHrMessageTrackerr bytearrayrrrrrrcrcrrNrrrrbrPsend_multipartrrappendr encode_intor)rr r rPencoder reuse_bufferspendingsocketsrmax_reuse_bufsrsocket client_indexrgrbufferstrackerrefrrs @@rrvz%EngineCoreProc.process_output_socketssi!"")+ c0#y@ABDD[[1 1E3;==1 1C$0 G%0 ###.uT  !\\A-N 1*..00^<<<"),, F++++%fe44444(.% g'3$2%%(333 //w0G0GHHH;'"+a."5;!((q)9:::;'"+a."5;1>N**,,,9;;!--gv>>!,/>>%t?|1%(\\A%5%5''4C&&f'=>>>>''.88!((000? 1 /1 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 1s7J+HI;/ J;I? ?JI? JJJc td|j|j|jt |j|jht|jgtj gfdS)zLog and return a request-scoped error response for exceptions raised from the add request preprocessing in the input socket processing thread. z*Unexpected error pre-processing request %s)r new_token_ids finish_reason)rPfinished_requestsrgN) rVrrrcrWrr(rPr'r+ERROR)rrs rrz,EngineCoreProc._handle_request_preproc_errors  8':L    $$$!!%!2'.'9&:('.'9*,*6*<         rr)*rCrDrErFrrrGrrr3rIrrrbrr1rfrrrr staticmethodSocketrrrnrrrr*r rrrrrprqrsrvr)r __classcell__rs@rrNrN|sCC*04]P]P]P]P]P]P ]P X ]P  ]P#&*]P]P]P]P]P]P]P~8$8$8$ 8$  8$ #&* 8$ %tT1 28$8$8$^8$t<@,E,E [,E,E ,E  ,E  ,E ,E$2D#8,E %tT1 2,E,E,E^,E\ 26 #&#&*#&#&#&($. #&  #&#&#&\#&J./aH'H'H'H'H'H'H'\H'T z    (((...6d(1 > >*1 'H' D$555D---$,!+0LL!MxXXX+/D((( 65--00 GG * *< A A A A Arc|jsdS|j}||jkrJ||_t ||j|jd}|jdt|fdSdS)N)r.r?rI)scheduler_stats) rmriget_request_countsr/r5r.r?rcrWr()rcountsstatss r_maybe_publish_request_countsz.DPEngineCoreProc._maybe_publish_request_counts7s'  F2244 T% % %%D "d&7dFWE   ( (".?PU.V.V.V)W X X X X X & %rc: ||}||j}|s|s|jsb||||_|js|jdks|j sZt d|j |j rdnd}|j |t|j f|xj dz c_ d|_)z8Core busy loop of the EngineCore for data parallel case.Trz&Wave %d finished, pausing engine loop.rI) wave_completerP)rrr?rirrer$_has_global_unfinished_reqsrrjrVrxr?rcrWr(r.)rexecutedlocal_unfinished_reqsrs rrzDPEngineCoreProc.run_busy_loopDsK) &  % % ' ' '0022H  . . 0 0 0$(N$J$J$L$L ! +,T5I((***$(#C#C%$$D ' &<1$$D,@$LL@$BS *.)=#D221L%00(-D Nz N N N NHHHHH 3 37 3# 3 3 3 3 3 3 B1 B>>&> >  >>>><  3    %69SV$       #&*   ^         rrYcHeZdZdZ d dedededeedede d e fd Z d S) DPMoEEngineCoreActorz'Used for MoE model data parallel cases.rrBrQr{rCrDrrc||j_t|||||t|||d||dS)Nr)rTrrYrrrrBrQr{rCrDrrs rrzDPMoEEngineCoreActor.__init__sd:A #6%% +y'=    !! +|R     rNr- rCrDrErFrrGr1rr3rIrrrrrmrms11    &  X           rrmcHeZdZdZ d dedededeedede d e fd Z d S) EngineCoreActorz%Used for non-MoE and/or non-DP cases.rrBrQr{rCrDrrc d|j_d|j_d|j_t|||||t |||d|||dS)NrPrrrO)rTrrrrYrrNros rrzEngineCoreActor.__init__4s:; #6?@ #<9: #6%% +y'=                 rNr-rprrrrrrr1s//    &  X           rrr)prrrrpr collectionsrcollections.abcrrconcurrent.futuresr contextlibrrinspectr r loggingr typingr r rrr vllm.configrrvllm.distributedr vllm.envsr vllm.loggerrvllm.logging_utils.dump_inputrvllm.lora.requestrvllm.multimodalr vllm.tasksrrvllm.transformers_utils.configrvllm.utils.gc_utilsrrvllm.utils.hashingrvllm.utils.network_utilsrvllm.utils.system_utilsrrvllm.v1.core.kv_cache_utilsr r!r"r#r$vllm.v1.core.sched.interfacer%vllm.v1.core.sched.outputr&vllm.v1.enginer'r(r)r*r+r,r-r.r/vllm.v1.engine.utilsr0r1r2vllm.v1.executorr3vllm.v1.kv_cache_interfacer4vllm.v1.metrics.statsr5vllm.v1.outputsr6vllm.v1.requestr7r8vllm.v1.serial_utilsr9r:vllm.v1.structured_outputr; vllm.v1.utilsr< vllm.versionr=rXrCrVPOLLING_TIMEOUT_Srr?rArNrrYrmrrrrrrs   ////////%%%%%%00000000&&&&&&&&%%%%%%%%%% 22222222NNNNNN''''''######??????))))))//////33333333SSSSSS322222444444DDDDDDDD<;;;;;555555                       &%%%%%444444000000------22222222????????======333333444444 X   WT]]k)k)k)k)k)k)k)k)\g  g  g  g  g  Zg  g  g  TGGGGG~GGGTggggggggT     /1A   .     *N     r