.`i|ddlZddlmZmZddlmZddlmZddlm Z m Z ddl Z ddl Z ddlmZddlmZmZmZmZmZddlmZdd lmZdd lmZmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/e j0ddZ1GddZ2eGddZ3eGddZ4GddZ5GddZ6dS)N) defaultdictdeque)Iterable) dataclass)Anycast) LoRARequest)STREAM_FINISHEDCompletionOutput PoolingOutputPoolingRequestOutput RequestOutput)RequestOutputKind) TokenizerLike)SpanAttributesSpanKindTracerextract_trace_context)&length_from_prompt_token_ids_or_embeds)EngineCoreOutputEngineCoreRequest FinishReason)IncrementalDetokenizer)LogprobsProcessor) ParentRequest)IterationStatsLoRARequestStatesRequestStateStatsSchedulerStatscpu)devicecteZdZdZdedefdZdeeze zddfdZ deezfd Z deezdzfd Z d Z d ZdS) RequestOutputCollectorz Collects streamed RequestOutputs per individual request, for hand-off to the consuming asyncio generate task. When streaming deltas, RequestOutputs are merged if the producer gets ahead of the consumer. output_kind request_idc|tjk|_||_d|_t j|_d|_dSN) rDELTA aggregater%outputasyncioEventready_input_stream_task)selfr$r%s s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py__init__zRequestOutputCollector.__init__1s;$(9(??$OS ]__ 7;r*returnNc|jt|tr"||_|jdSt|jt r8t|t r#|j||jdSt|jtrt|tr ||_dSdSdS)zNon-blocking put operation.N)r)) r* isinstance Exceptionr-setraddr)r r/r*s r0putzRequestOutputCollector.put9s ; *VY"?"?  DK JNN       ] 3 3 ! M9 9 ! KOOFdnO = = = = =  %9 : : !z (@ @  !!DKKK ! ! ! !r2cK|jx}(|jd{V|jx}(d|_|jt |t r||S)z"Get operation blocks on put event.N)r*r-waitclearr5r6r9s r0getzRequestOutputCollector.getIs}$v-*//## # # # # # # #$v-   fi ( ( L r2c|j}| d|_|jt|tr||S)zNon-blocking get operation.N)r*r-r=r5r6r9s r0 get_nowaitz!RequestOutputCollector.get_nowaitSsF  DK J      fi ( ( L r2cT|j|jd|_dSr')r.cancelr/s r0closezRequestOutputCollector.close]s.  " .  # * * , , ,"&r2c|jx}5||jd|_dSdSr')r.get_loopcall_soon_threadsaferB)r/tasks r0__del__zRequestOutputCollector.__del__bsD+ +D 8 MMOO 0 0 = = =&*D # # # 9 8r2)__name__ __module__ __qualname____doc__rstrr1rr r6r:r>r@rDrIr2r0r#r#(s<$5<3<<<<!-*>>J!t!!!! =+??M,@@4G''' +++++r2r#cBeZdZUeeezed<eeed<dS)OutputProcessorOutputrequest_outputs reqs_to_abortN)rJrKrLlistrr __annotations__rNrOr2r0rQrQhs9-*>>????9r2rQcXeZdZUdZedzed<eedzed<eed<dZ e ed<dS)StreamingUpdatezStreaming input update data for output processor. Contains the incremental prompt data to be applied to a request state when the current sub-request completes. Npromptprompt_token_ids arrival_timeFfinal) rJrKrLrMrNrUrTintfloatr[boolrOr2r0rWrWns_ $J3i$&&&&E4r2rWc)~eZdZ d+dedededzdededzded edzd eedzd e j dzd e dzd e dzdedzde dedzdedede dzdedzde dzdef(dZdeddfdZededzded edzdedzdededzdededdfdZ d,deede j dzd edzd!eezdzd"eeefdzd#ejdzdeezdzfd$Z d-ded%eeee zd&ed"eeefdzdeezf d'Z! d-d(eed edzd!eezdzd#ejdzdef d)Z"de j de fd*Z#dS). RequestStateNFr%external_req_id parent_req request_index lora_requestr$rXrY prompt_embedslogprobs_processor detokenizermax_tokens_paramrZqueue log_statsstream_intervaltop_pn temperature stream_inputc||_||_||_||_||_||jnd|_||_||_||_| |_ t|j|j |_ | |_ | |_ | |_||_||_||_d|_||_d|_|rt+| nd|_||_d|_||_|rt5nd|_dS)NTr)rZ)r%rarbrcrd lora_namer$rXrYrer prompt_lenrfrgrhrlrmrn is_prefillingrinum_cached_tokensrstatsrksent_tokens_offsetstreaming_inputrinput_chunk_queue)r/r%rarbrcrdr$rXrYrerfrgrhrZrirjrkrlrmrnros r0r1zRequestState.__init__}s.%.$*(3?3K//QU& 0*@  !4#5  #5& 0 &! !"ENX&LAAAATX  /"# ,# -EGGG r2updater3cb|j |_|jr"|jr|j|jzn|j|_|jr"|j|jpdn|jpg|_|jJt |j|_|j|j|j_d|_ dS)NrOT) r[rwrXrYextendlenrrrurZrs)r/rys r0apply_streaming_updatez#RequestState.apply_streaming_updates#)</ = 15Ov},,&- K   B  ! ( ()@)FB G G G G$*$;$ArD !$000d344 : !&,&9DJ #!r2 tokenizerrequestc |jx} rY| jsd}| j} tj||} t j||} | j} | j}| j}| j }n!d} d} d} d}d}d}|j J|j j} |j J|did|j d|j d|d|d|j d| d|d |jd |jd | d | d | d|d|d|d|jd|d|d|d|jS)N)r~rr%rarbrcrdr$rXrYrerfrgrhrlrmrnrZrirjrkrorO)sampling_params detokenizer$rfrom_new_requestr max_tokensrlrmrnpooling_paramsrar%rdrYrerZ resumable)clsr~rrXrbrcrirjrkrr$rfrgrhrlrmrns r0rzRequestState.from_new_requests&5 5? ="- ! )5K!2!C#""" 1A#K /9 #)E!A)5KK!% K# EAK)555!0.O*P*P'.  %++)).99:  ,, =+~   ? "hGG $ ; ;DOV T T GX t"o=O'' Wh0B   r2rrc "|d}t|tr > Cy>  t+> $d* > 3Y% > !cNT1 >  T)>  - - 4> > > > J59 * * * &'$}*==*  * !cNT1 * - - * * * * b-1   9 $d* 3Y%   T)       B2%,2=222222r2r`c eZdZdZ d&dedzdedefdZdZd efd Z d'd Z d e fd Z de eded eefdZ d(dededzdedzdededzd df dZdedededzd dfdZ d)deededzdedzd efdZded dfdZdedzfd Zd!edededzd dfd"Zded!ededzdedzfd#Z ded$e!dzdedzfd%Z"dS)*OutputProcessorz.Process EngineCoreOutputs into RequestOutputs.rr~Nrjrkc||_||_||_i|_i|_t t |_t||_ d|_ tj |_ |j dSr')rjr~rkrequest_statesparent_requestsrrTexternal_req_idsr lora_statestracerr+r,_requests_drainedr7)r/r~rjrks r0r1zOutputProcessor.__init__sz #".799;=H=N=N,Y77%) !( ""$$$$$r2c*t|jSr'r|rrCs r0get_num_unfinished_requestsz+OutputProcessor.get_num_unfinished_requestss4&'''r2r3c2t|jdkS)NrrrCs r0has_unfinished_requestsz'OutputProcessor.has_unfinished_requestss4&''!++r2cZK|jsdS|jd{VdSr')rrr<rCs r0wait_for_requests_to_drainz*OutputProcessor.wait_for_requests_to_drainsC"  F$))+++++++++++r2ec|jD](\}}|jJ|j|)dS)z(Propagate error to all generate() tasks.N)ritemsrir:)r/r_states r0propagate_errorzOutputProcessor.propagate_errorsU+1133  HAu;*** KOOA      r2 request_idsinternalcg}|D]}|re|||j|x}r3|j}|j|}|||s|j|=i|j|gx}r||g}|D])}|j|d}||j ||j |||j N| g|j tndtjddx} r|j | |j|x} rb| jr@t)| j} || d} || |j|d+|js|j|S)ahAbort a list of requests. The request_ids may be either external request IDs (those passed to InputProcessor.process_inputs()) or internal request IDs (those randomly generated when creating the EngineCoreRequest). If an external request ID is provided, and that external request ID was used for multiple requests, all requests associated with that external request ID are aborted. In the case of parallel sampling, a request ID may be used to identify a parent request, in which case the associated child requests are aborted also. N)rrrrrT)r)appendrr>rarremovepopr{rrequest_finishedrqrirrgEMPTY_CPU_TENSORrABORTr:rchild_requestsrTabort_requestsrr7) r/rrinternal_req_idsr% req_statera internal_idsrequest_ids_to_abortrequest_outputparent child_reqss r0rzOutputProcessor.abort_requestss<% 6 6J 6 '' 333!% 3 7 7 C CC9C&/&?O#'#8#IL '' 333'C 1/B!%!6!:!::r!J!JJ 6 '' 555!* ; ;J+// DAAI$ 11*i>QRRR$++J777?.&/&C&C&(%08(8'7!&2&8$(+/'D ' ' N/O''777/33J??? ;(_update_streaming_request_stater`rr~rjrkris_setr=rrrar)r/rrXrbrcrir%rs r0 add_requestzOutputProcessor.add_requests' '++J77   0 0GV L L L F 11n!'n 02     ! ( ( * * +  " ( ( * * **3J'  E:DD !6 7 i78?? KKKKKr2rc|jsf|j<|||j|jt n!|jrd|jd_nd|_dSt||j |j }|j*| |t|_dS|j |dS)z_update_stats_from_outputrrrrrrrtrsrgrfryrSTOPupdate_from_outputrrwrrir:rrxpopleftr}r_update_stats_from_finishedr do_tracingrQ)r/rrrrRrSengine_core_outputreq_idrrrrrrr stop_stringrrys r0process_outputszOutputProcessor.process_outputsAs8GI#% "5K XK X '2F+//77I   * *-/Do   /N.N*<*NI '&+I #% ,888 3???'3::!=L4E#E .$0$5M"-K,??@RSSS"+!>!>" ""~ ;,4.3N+?.O''7777$**>:::(,X 2;!*!  & (?  [ . . !0 /  0 Q oG&:W=QQH!.1BBK"1G4HHL!/'2HHK$2W5IIN   A+      ~@( K K K   ~JK X X X   ~H- X X X   =-      C\      BK      E~      0)2K    Y"">#F XXX) """X$ """=y?T{ Q"">#BIKPPPa0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Q0 Qs H2KK K c |dS|J|jJ||||j|j|j|j|jdSr')rurrsrrrrq)r/rrrrs r0rz)OutputProcessor._update_stats_from_outputsn  " F$000*****  !  #  O         r2rc|dS|J|jJ|||j|j|j|j|j|j|jtj |j ||jj dS)N)rnum_prompt_tokensrh req_statsrt) ruupdate_from_finished_requestrrrhrtrrr%rqrobserve_finished_requestrbr)r/rrrs r0rz+OutputProcessor._update_stats_from_finisheds  " F(((***44''2&7o'9 5    )))*> @STTT.  /9?3X     r2)r)r3N)NrNr)#rJrKrLrMrr^r\r1rrrr6rrrNrTrrrr#rr`rrr]rrQrrrrrrrrrOr2r0rrs88 ! %% 4'%% %%%%"(((,,,,,,,,, >$(3->$4>$DQTI>$>$>$>$H,0/3 LL"Ld L"D( L  L &, L LLLLB7%70A7KNQU:7 7777H/315 n n !"23n  %t|n ($. n  n n n n `))$))))"Ant6KAAAA?Q,?Q ?Q($. ?Q  ?Q?Q?Q?QB  -  %t|  ($.     ,  $d* ($.       r2r)7r+ collectionsrrcollections.abcr dataclassesrtypingrrnumpyrrvllm.lora.requestr vllm.outputsr r r r rvllm.sampling_paramsrvllm.tokenizersr vllm.tracingrrrr vllm.utilsrvllm.v1.enginerrrvllm.v1.engine.detokenizerrvllm.v1.engine.logprobsr vllm.v1.engine.parallel_samplingrvllm.v1.metrics.statsrrrremptyrr#rQrWr`rrOr2r0r.s********$$$$$$!!!!!! ))))))322222))))))PPPPPPPPPPPP======LLLLLLLLLL======555555::::::5;q///=+=+=+=+=+=+=+=+@             Z2Z2Z2Z2Z2Z2Z2Z2zY Y Y Y Y Y Y Y Y Y r2