.`if!UddlmZddlmZddlZddlZddlmZddl m Z eGddZ e e Z eed<d ed ed ed efd Zdededed efdZde deded e fdZ d#dedejdeded ede eezdzd ee dze dzffdZdejded ejfdZde de d e fd Zde e d!e d e e fd"ZdS)$) dataclass) TypeAliasN)ParallelConfig)CommonAttentionMetadatacLeZdZUeed<eed<defdZedefdZ dS) UBatchSlice request_slice token_slicereturncj|jj|jjkp|jj|jjkSN)r startstopr selfs o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/worker/ubatch_utils.pyis_emptyzUBatchSlice.is_emptys4   $(:(? ? ?%)9)>> c4|jj|jjz Sr )r rrrs r num_tokenszUBatchSlice.num_tokenss$t'7'===rN) __name__ __module__ __qualname__slice__annotations__boolrpropertyintrrrrr sp $    >C>>>X>>>rr UBatchSlicesorig_num_tokenspadded_num_tokens num_ubatchesr c ||z|dz z|kS)Nr)r!r"r#s ris_last_ubatch_emptyr& s  -,2B C VVrconfigruniform_decodecD|jsdS|r ||jkS||jkS)NF) use_ubatchingdbo_decode_token_thresholddbo_prefill_token_threshold)r'rr(s rcheck_ubatch_thresholdsr-&s9  u@V>>>V???r ubatch_slicesnum_total_tokensnum_reqs_paddedc|d}t|jj|}t|jj|}|ddt ||gzS)N)rr rr r)r.r/r0 last_slicepadded_last_request_slicepadded_last_token_slices r_pad_out_ubatch_slicesr63ser"J %j&>&Do V V#J$:$@BRSS " -/FGG! r should_ubatchnum_scheduled_tokensnum_tokens_padded split_pointc|sdSt||zfdtd|D}tjt |dztj}tj|tj|ddg}d} ||dgz} | D]} t| | } ttj|| d dz } ttj|| d }t| |}| t|| | } t|||}td |D|ksJ||fS) N)NNcg|]}|zSrr).0ir:s r z.maybe_create_ubatch_slices..MsJJJa+/JJJrr%)dtype)r@outrr2right)sideleftc3$K|] }|jV dSr )r)r=ss r z-maybe_create_ubatch_slices..ps$::q|::::::r) rrangenpzeroslenint32cumsumr searchsortedappendrr6sum)r7r8r9r0r#r:token_split_points cu_num_tokensr. start_token all_points end_tokenr req_startreq_stop req_sliceubatch_slices_paddeds ` rmaybe_create_ubatch_slicesrZ?s z+,, < JJJJ5L3I3IJJJHS!566:"(KKKMI""( abb8IJJJJMK$}R'8&99J  K33  {QQQTUUVV r}ifMMMNN)X.. [K@@AAA 1(/ ::%9::: : :>O O O O O . ..rquery_start_locr cL||j|jdz||jz S)z Creates a new query_start_loc that corresponds to the requests in request_slice. Note: This function creates a new tensor to hold the new query_start_locs. This will break cudagraph compatibility. r%)rr)r[r s rslice_query_start_locsr]us1  +m.@1.DDE -- . /r ubatch_slice attn_metadatac >|rJd|d|j}|j}|j}|j}|j}|jdz }|jdz }|||cxkr||dzks nJd|||k} |||dzdz k} t ||} t |j|} t| dksJdt| | r/|||z } | ddxx| zcc<| ddxx| zcc<|j |}|j |}| r{||dz|jz } | dxx| zcc<| dxx| zcc<| }| }|dxx| zcc<|dxx| zcc<t| }|j|}|j|jz }|j|jz }ttj tj| dd| ddz }|d kr|j}|j|}|j|}t+| | ||||||||| S) z{ This function creates a new CommonAttentionMetadata that corresponds to the requests included in ubatch_slice z Ubatch slice z is emptyr%z*Token slice start outside of first requestz3query_start_loc must have at least 2 elements, got Nr2r) r[query_start_loc_cpuseq_lensnum_reqsnum_actual_tokens max_query_len max_seq_lenblock_table_tensor slot_mapping _seq_lens_cpu_num_computed_tokens_cpu)rr r rbrrr]r[rKrc seq_lens_cpuclonermaxnum_computed_tokens_cputorchabsitemrfrhrir)r^r_r r start_locs first_req first_toklast_reqlast_toksplits_first_requestsplits_last_requestrbr[tokens_skippedrcrlrgro num_requestsrerfrhris r_make_metadata_with_slicer|sS$$&&OO(O (O(O(OOO & .M*K2J#I!I!A%H!#H i I I I I I 9q=0I I I I I I4 J I I%z)'<<"Z1 %=%AA0]KK,%}O   1 $ $ $Tc/>R>RTT % $ $2"Z %::~-ABB>1%m4H -m1>>###))++  & RN*l&&(())K+CMR % (;;L#(;+<< %)/36I#2#6NNOOPPUUWWM %3 &9-H -k:L "'/+#-!"!8    rcommon_attn_metadatacZg}|D]%}|t||&|S)z Creates a new CommonAttentionMetadata instance that corresponds to the requests for each UBatchSlice in ubatch_slices. Note: This function does not modify common_attn_metadata )rOr|)r.r}resultsr^s rsplit_attn_metadatars@G%VV 0?STTUUUU Nrr ) dataclassesrtypingrnumpyrIrp vllm.configrvllm.v1.attention.backendrrlistr rrrr&r-r6ndarraytuplerZTensorrr]r|rrrrrs"!!!!!! &&&&&&======  > > > > > > >  >{+ i+++WW-0W@CW WWWW @ @(+@=A@ @@@@  36 IL     $+/ 3/3/3/*3/3/ 3/  3/ cS4' 3/ <$  t 3 343/3/3/3/l\ \"\\.E\\\\\~ $1 !"r