.`i .UddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z m Z ddlmZddlmZddlmZmZmZmZmZddlZddlZddlZddlmZdd lmZddlmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ddl0m1Z1m2Z2ddl3m4Z4ddl5m6Z6m7Z7ddl8m9Z9ddl:m;Z;mZ>m?Z?m@Z@ddlAmBZBmCZCddlDmEZEddlFmGZGmHZHddlImJZJddlKmLZLddlMmNZNmOZOddlPmQZQmRZRddlSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]ddl^m_Z_m`Z`maZaddlbmcZcddldmeZemfZfmgZgddlhmiZidd ljmkZkdd!llmmZmdd"lnmoZodd#lpmqZqmrZrmsZsdd$ltmuZudd%lvmwZwdd&lxmyZymzZzdd'l{m|Z|m}Z}dd(l~mZdd)lmZdd*lmZmZdd+lmZmZmZmZmZmZmZdd,lmZdd-lmZmZmZdd.lmZdd/lmZdd0lmZmZmZmZmZmZmZmZmZmZmZdd1lmZmZmZmZmZmZmZmZmZmZmZdd2lmZmZdd3lmZmZdd4lmZdd5lmZdd6lmZdd7lmZdd8lmZdd9lmZdd:lmZdd;lmZddlmZdd?lmZmZdd@lmZddAlmZmZddBlmZddClmZddDlmZmZddElmZddFlmZddGlmZddHlmZmZmZmZddIlmZddJlmZdKdLlmZmZmZmZmZerddMlmZddNlmZmZeEeZeeefZeedO<eeezZeedP<GdQdReZGdSdTeZGdUdVeZGdWdXeee֦ZeGdYdZZdS)[N) defaultdict)IteratorSequence)contextmanager)copydeepcopy) dataclass)reduce) TYPE_CHECKINGAny NamedTuple TypeAliascast)tqdm) Attention MLAAttention)compilation_counter) CUDAGraphStatCUDAGraphWrapper)set_cudagraph_capturing_enabled)CompilationMode CUDAGraphMode VllmConfigget_layers_from_vllm_config update_config)get_ec_transferhas_ec_transfer) EplbState)get_kv_transfer_grouphas_kv_transfer_group)copy_kv_blocks) get_dcp_group get_pp_group get_tp_group graph_captureis_global_first_rank&prepare_communication_buffer_for_model)BatchDescriptorset_forward_context) init_logger) LoRAMappingLoRAMappingType)AttentionLayerBase)RoutedExpertsCapturer)MRotaryEmbeddingXDRotaryEmbedding)TensorizerLoaderget_model_loader) MultiModalEmbeddings SupportsMRoPESupportsMultiModalSupportsXDRoPEis_mixture_of_expertssupports_eagle3supports_mropesupports_multimodal_pruningsupports_transcriptionsupports_xdrope)VllmModelForPoolingis_pooling_modelis_text_generation_model)MULTIMODAL_REGISTRY)BatchedTensorInputsMultiModalKwargsItemPlaceholderRange)group_mm_kwargs_by_modality) PoolingParams) SamplingType)IntermediateTensors)GenerationTask PoolingTask SupportedTask)&length_from_prompt_token_ids_or_embeds)json_map_leaves)cdivround_up)DeviceMemoryProfiler format_gib)PytHooks)is_pin_memory_available)get_dtype_sizekv_cache_dtype_str_to_dtype)AttentionBackendAttentionCGSupportAttentionMetadataAttentionMetadataBuilder AttentionTypeCommonAttentionMetadata MultipleOf)GDNAttentionMetadataBuilder)"create_fast_prefill_custom_backendget_dcp_local_seq_lens+reorder_batch_to_split_decodes_and_prefills)NewRequestData)CudagraphDispatcher) AttentionSpecChunkedLocalAttentionSpecCrossAttentionSpecEncoderOnlyAttentionSpecFullAttentionSpec KVCacheConfigKVCacheGroupSpec KVCacheSpec MambaSpecSlidingWindowSpecUniformTypeKVCacheSpecs) EMPTY_MODEL_RUNNER_OUTPUTAsyncModelRunnerOutput DraftTokenIdsECConnectorOutputKVConnectorOutput LogprobsListsLogprobsTensorsModelRunnerOutput PoolerOutput SamplerOutput&make_empty_encoder_model_runner_output)PoolingMetadata PoolingStates)LogitsProcessorsbuild_logitsprocs)LogitsProcessor)SamplingMetadata)RejectionSampler)Sampler)DraftModelProposer) EagleProposer)MedusaProposer)SpecDecodeMetadata) NgramProposer)SuffixDecodingProposer)apply_grammar_bitmask) CpuGpuBufferrecord_function_or_nullcontext) mamba_utils) check_attention_cp_compatibilityget_total_cp_world_size)coordinate_batch_across_dp)ECConnectorModelRunnerMixin)CachedRequestState InputBatch) UBatchWrapper)KVConnectorModelRunnerMixin)LoRAModelRunnerMixin) UBatchSlicescheck_ubatch_thresholdsmaybe_create_ubatch_slicessplit_attn_metadata)is_residual_scattered_for_sp)lock_workspace)AttentionGroupMultiModalBudget(add_kv_sharing_layers_to_kv_cache_groups bind_kv_cachesanity_check_mm_encoder_outputs)TensorizerConfig) GrammarOutputSchedulerOutputAttnMetadataDictPerLayerAttnMetadatac jeZdZdedejdedzdeedej j def dZ d efd Z dS) AsyncGPUModelRunnerOutputmodel_runner_outputsampled_token_idslogprobs_tensorsNinvalid_req_indicesasync_output_copy_stream vocab_sizec||_||_tj|_||_||_||_tj }tj |5| ||j dd|_ |jr|jnd|_|jddddS#1swxYwYdSNcpuT non_blocking)_model_runner_output_invalid_req_indicestorchEventasync_copy_ready_event_sampled_token_idsr_logprobs_tensorscudacurrent_streamstream wait_streamtosampled_token_ids_cputo_cpu_nonblocking_logprobs_tensors_cpurecord)selfrrrrrrdefault_streams s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/worker/gpu_model_runner.py__init__z"AsyncGPUModelRunnerOutput.__init__sJ%8!$7!',kmm##4$!12244 Z  7 8 8 1 1 $ 0 0 @ @ @)-)@)C)CD*D**D & )&99;;;  &  ' . . 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s9A7C==DDreturnc|jjd}|j|`|`|dkr`|j}|jD]}||d}|j |j }n/tj |j|j |j|j \}}|j}||_||_|S)zCopy the device tensors to the host and return a ModelRunnerOutput. This function blocks until the copy is finished. rNr)rshaper synchronizerrtolistrclearrtolistsr~ parse_outputrrrlogprobs)r max_gen_lenvalid_sampled_token_idsilogprobs_listsoutputs r get_outputz$AsyncGPUModelRunnerOutput.get_outputs 06r:  #//111  "  # !  &*&@&G&G&I&I #. 3 3'*002222!N)5!%!;!C!C!E!E6F6S*)!%!; 777 3 #^*#: ( ) __name__ __module__ __qualname__rtrTensorrslistintrStreamrrrrrrs!1.!1!<!1*D0 !1 "#Y !1 #(*"3 !1!1!1!1!1F-rrcReZdZdededeedejj fdZ defdZ dS) AsyncGPUPoolingModelRunnerOutputrraw_pooler_output finished_maskrc||_tj|_||_tj}tj|5||td|j}|j dt||D|j_ ddddS#1swxYwYdS)Nc8|dn|ddSrrxs rz;AsyncGPUPoolingModelRunnerOutput.__init__..s!)$$e$1O1Orc g|] \}}|r|nd SNr.0outincludes r z=AsyncGPUPoolingModelRunnerOutput.__init__..#s5777 C(D777r) rrrr_raw_pooler_outputrrrrrLrzip pooler_output)rrrrrrraw_pooler_output_cpus rrz)AsyncGPUPoolingModelRunnerOutput.__init__ s,%8!',kmm##42244 Z  7 8 8   $ 0 0 @ @ @$3OO'%% !  ' . . 0 0 077$'(=}$M$M777D % 3                  s$A)CC!CrcF|j|`|jS)zCopy the device tensors to the host and return a ModelRunnerOutput. This function blocks until the copy is finished. )rrrrrs rrz+AsyncGPUPoolingModelRunnerOutput.get_output(s* #//111  #((rN) rrrrtrurboolrrrrrrrrrr ss.(Dz  #(*"3 <)-))))))rrceZdZUdZded<ejed<edzed<edzed<ejed<ejed <e ejdzed <e dzed <e dzed <e e ejfe e e ejfzdzed <dS)ExecuteModelStatezwEphemeral cached state transferred between execute_model() and sample_tokens(), after execute_model() returns None.rscheduler_outputlogitsNspec_decode_metadata spec_decode_common_attn_metadata hidden_statessample_hidden_statesaux_hidden_statesec_connector_outputcudagraph_stats slot_mappings)rrr__doc____annotations__rrrrZrrprdictstrrrrrr3s<<(''' L,t3333&=&DDDD<,&&&EL)D0000*T1111"T))))U\)*T$sEL7H2I-JJTQQQQQQrrc2eZdZdedejfdZdeddfdZddZ ej dd Z d e fd Z d d deejzdejdedefdZdZddZddZddZddZdejddddfdZdededefdZdefdZdefd Zddde fd!Z!d"ede fd#Z" dd e#j$d$e#jdzde%e#j$e#j$ffd%Z&ddd&ed'e#j$ddfd(Z'd)e(eefd*e)d+ede%ejdze#j$dzffd,Z*ddd)e#j$de%eje+dzffd-Z, dd ed+ed/ed0edzd1edzd2e-dzd3ejdzd4ed5ed)e(eefdzd6e.e.edzd7e(eejfdzde%e/e0dzffd8Z1d)e#j$d9e#j$d:e.ede.e.edzfd;Z2d)e#j$d9e#j$d:ed*e)dZ5dd?Z6d@e#j$dAe#j$de+fdBZ7d3ejdejfdCZ8ddde%e.ee.e9e.e%ee:fffdDZ;ddde.ejfdEZ< ddddGede%e.ejejffdHZ=de>j?fdIZ@de.eAfdJZBde.eCfdKZDde%eEdLffdMZFd edNeGdzdOedeGfdPZHddQedReddfdSZIdTejd)edUe#j$dVeJdzdeKeLzf dWZMd)edefdXZNd ede%ejdzejffdYZO ddddZedNeGdzde%ejdzejdzejeGdze(ee fePdzffd[ZQd\ejdzd]e+dzdeRfd^ZSddd_eRd\ejdzdTejd)ed]e+dzde%e(eefeTdze.e.ee(eeUdzfe.ee(eefe.effd`ZVeWdaZX ddbejdzdcejdzdNeGdzddejdzdee(ee fde f dfZYeZ ddgedhed ed+ediedzdef djZ[ dd ed+edUe#j$dgedkedledmediedzdnedzdoede%e\e]eejdze^dzffdpZ_ddqZ` dd0ed1edred2dsde%e(eejfdze(eejfe.e(eejfzdzff dtZaej ddddNeGdzdeKeLzeGzdzfduZbej dvdwdeKeLzeGzfdxZcdeddzfdyZe ddddzeddfd{Zfde%e.e.ee.effd|Zgd}ejd~ejddfdZhde.efdZidddeje.e.ezdejdTejdejde.ejdzd]e+dzde0d7e(eejfe.e(eejfzdzde.e.eejzfdZkde(ee fddfdZlddeddfdZmde%edLfdzfdZnddZo ddZpdTejd)e(eefde(eeUdzffdZqd\ejdzde(eeffdZreWdbejdzddejdzfdZsdedede fdZtej dd ede\dzdededlededRededededede%ejejffdZuej dTejdejfdZvdTejdeCdewfdZxej dTejdewfdZyddZzdefdZ{de.e]de\fdZ|de}ddfdZ~de}de.eddfdZde.eeede.eddfdZddZeZdede.edefdZde}de.eddfdZde}de(eejffdZdeefdZdeefdZde}de.efdZde}de(eejfde.ede(eejffdZde(eejfddfdZde}de.ede(eejffdZde}ddfdZde}ddfdZdZddZde(ee)ffdZdejde.e.efdZde(ee(eeezfffd„ZeWdede.e%ee fdedefdDŽZdS)GPUModelRunner vllm_configdevicec$||_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j |_ ddl m }|t|jj dz|j}|j}|j}|j}||_t|_|jj|_t%|j|j|_|jdk|_|j|_|j|_d|_|j|_|jj|_|jj|_|jdkrdnt=j|_ |j!|_"|j#|_$|jj%dko#tMtOj(dk|_)|*||_+|,|_-|j.|_.|j/|_0|jj1 |_2|jj3|_3th|_5|j6|_6|j7|_7|j58||_9|jj:r |j;|_<nd|_<|jj=|_>t|jj@|_Ad|_B g|_Cd|_Dd|_Eg|_Fi|_Gd|_H|jrgtOjIrS||jjJd krt|j|_Ln |jMr"t|j|j| |_Ln|jjJd krt|j|_Ln|jPrBt|j|j||_L|jjJd kr|jLjR|_HnM|jjJd kr!t|j|j|_Lntd|jjJt|jA|_Vd|_W|jr?|jjX|_W|jjY}||j |j|_Zn |j|_Zi|_[i|_\tj^_|_`|ja} | t| nd} t|j$t|j|j<|j"|j|j|je|jjfg|jjfgt|jjt|j|j|j|j| t| |j|jji |_jd|_kd|_l|j>r;tj^_|_ktjm|_l|jjnr8|jjotjqkrt|jjn|_s|ti|_utjw|_x|y|j"tjz|_{|y|j"tj||_}|y|j$dztjz|_~|y|j$tjz|_|y|j$tjz|_|jdkr+|y|j$tjz|_|y|j"|j-|jd|_|y|j"tjg|_|y|j$tjg|_|y|j$tjz|_|y|j$tj||_|j9rX|y|j"tjg|y|j"tjgg|_d|_|j6r0|yd|j"dzftj||_|j7dkr5|y|j7|j"dzftj||_d|_tjt|j$dz|j|j"tj||_i|_t!|_d|_|jjr0tj|j"tjz|j|_d|jWz|_t-|j|_|j9rt1|j|j5nd|_d|_t!|_d|_d|_tjm|_tj|j$dftj|d|j|_d|_d|_d|_d|_d|_d|_|jWrtjm|_tj^_|_tj|j$|jWftj|d|j|_|j>rltjm|_tj^_|_tj|j$tj|d|j|_d|_d|_i|_d|_dS)Nr)set_cpu_offload_max_bytes@poolingFrexternal_launcher) logprobs_modengram)r r runnersuffixeagle3medusa)r r z%Unknown speculative decoding method: r) max_num_reqs max_model_lenmax_num_batched_tokensr  pin_memoryr block_sizeskernel_block_sizesis_spec_decode logitsprocs!logitsprocs_need_output_token_idsr>cp_kv_cache_interleave_sizedtype)r!numpyr!r r)r!r r)r  model_config cache_configcompilation_config lora_config load_configparallel_configscheduler_configspeculative_configobservability_config vllm.model_executor.models.utilsr rcpu_offload_gbr rRrr!rT cache_dtypekv_cache_dtype runner_typer>enable_prompt_embeds"is_multimodal_raw_input_only_modelis_multimodal_pruning_enabledrcalculate_kv_scalesdecode_context_parallel_sizedcp_world_sizer" rank_in_groupdcp_rankrmax_num_tokens max_num_seqsrdistributed_executor_backendlenr#ranksbroadcast_pp_outputget_num_attention_headsnum_query_headsget_inputs_embeds_sizeinputs_embeds_sizeattention_chunk_size uses_alibi use_alibidisable_cascade_attncascade_attn_enabledis_mm_prefix_lmr@ mm_registry uses_mropeuses_xdrope_dimsupports_multimodal_inputssupports_mm_inputsis_encoder_decodermax_num_encoder_input_tokensmax_encoder_lenasync_schedulinguse_async_schedulingrrsampler eplb_state kv_cachescross_layers_kv_cachecross_layers_attn_backend attn_groups encoder_cacheuse_aux_hidden_state_outputs is_last_rankmethodrdrafteruses_draft_modelrr use_eaglereagle3_use_aux_hidden_stater ValueErrorr~rejection_samplernum_spec_tokensnum_speculative_tokensdraft_model_configeffective_drafter_max_model_lenrequestsnum_prompt_logprobsrrr comm_streamlogits_processorstuplermaxget_vocab_size block_sizerr{r input_batchrprepare_inputs_eventrcudagraph_capture_sizescudagraph_moderNONEsortedcudagraph_batch_sizes_init_device_propertiesencoder_timing_registry threadingLock_encoder_timing_lock _make_bufferint32 input_idsint64 positionsquery_start_locseq_lensencoder_seq_lensdcp_local_seq_lens inputs_embeds is_token_idsdiscard_request_masknum_decode_draft_tokensnum_accepted_tokensis_mm_embed_buffersis_mm_embed_idxmrope_positionsxdrope_positionsintermediate_tensorsnparange arange_npshared_kv_cache_layersset'kv_sharing_fast_prefill_eligible_layers&kv_sharing_fast_prefill_logits_indiceskv_sharing_fast_prefillzerosuniform_decode_query_lenracudagraph_dispatcherr mm_budgetreorder_batch_thresholdrunner_only_attn_layers_draft_token_ids_draft_token_req_idstransfer_eventemptysampled_token_ids_pinned_cpuvalid_sampled_token_count_event%valid_sampled_token_count_copy_streamdraft_token_ids_eventdraft_token_ids_copy_streamvalid_sampled_token_count_cpudraft_token_ids_cpuexecute_model_statekv_connector_outputmamba_state_idxlayerwise_nvtx_hooks_registered) rr r r r%r&r+r* draft_configrlcustom_logitsprocss rrzGPUModelRunner.__init__Fs ''4'4"-"@&2&2*: + <"-"@$/$D!NNNNNN!!#d&7&F&P"Q"QRRR( ( 0. 133&, 9  $d&7  !- 8I E$0$E!  ; /.3*)7$(#4#H "2O!0A55=??;X .E,9  =AT T .LNN())A-  ,CCOTT"."E"E"G"G$0$E!%0(,(9(N$N!#0@/&1+;"&"2"M"M # #    / %$4#PD #$D %)$9$J!T->-LMMM ,0 .0:>"HL&7979,1)  "! D|~~'B! D  &-88,T-=>> (99;; 1 $ 0;    (/8;;5d6FGG (2244 ,T-=t{DQQ *1X== @5(/8;;- $ 0    !8.588&6dl%C%CD "   " J#'#:#QD 2EL'L,F,R7C7Q447;7I48: 46  :,,..):(9(EE# $ $ $2 &*d0$2FGG#'#6;(7799*56 $ 1 <= 0 CDD)  %" /33E.F.F!2(,(<(X/   8CG%9=!  $ 6,1J,=,=,?,?D )(- D %  # ; '6-:LLL)/'?**D & $$&&&GI$$-N$4$4!**4+>ek*RR**4+>ek*RR#00   ! 1  ))$*;5;)OO $ 1 1$2C5; 1 W W   " "&*&7&7!'8''D # "..  !8 RW/  !--d.A-TT$($5$5  UZ%6% % !(,'8'8  U[(9( ( $$(#4#4  U[$5$ $   " %!!$"5UZ!HH!!$"5UZ!HH(D $$%D  ? $(#4#4D'!+,EK$5$$D  ! # #$($5$5%t':Q'>?u{%6%%D ! AE! !A%t'94;N O O(   79#AD46:3   4 :?+#5;t{;;;D 7)*D,@(@%%88H$I$I!&  T-t/? @ @ @  48$ 25$HL6:!#kmm,1K   "+ - - - )DH,OS2:>"EI(BF*8<   ).D &/4z/@/@/B/BD ,',{"D$89k? (((D $ ( 7<{}}4=BZ=N=N=P=P:5:[%+ # 6662>B =A /1/4,,,rrrNcl||_|jr#|jj}||j|j|_dSdSdSr)rr,rgrh)rrrs rupdate_max_model_lenz#GPUModelRunner.update_max_model_lensR*  " J2EL#|'A'I7;7I444 J J'I'IrcJ|jr|jdSdSr)r reset_cachers rreset_mm_cachezGPUModelRunner.reset_mm_caches0 > ) N & & ( ( ( ( ( ) )rc|jjdsdSt|dg}|D]}||d}d}|jj}|D]\}}t|ttfrd\}} |D]Q} t|| r?t|| } t| tj r| |R|D]Q} t|| r?t|| } t| tj r| | RdS)a Re-initialize the KV cache and FP8 scales after waking from sleep. 1. Zero out the KV cache tensors to remove garbage data from re-allocation. 2. Reset Attention layer scaling factors (_k_scale, _v_scale) to 1.0. If these are left at 0.0 (default after wake_up), all KV cache values become effectively zero, causing gibberish output. fp8NrW)_k_scalek_scale)_v_scalev_scale)?r)r&r0 startswithgetattrzero_r'static_forward_contextitems isinstancerrhasattrrrfill_) rrW cache_tensor k_attr_names v_attr_names attn_layersnamemodule k_scale_val v_scale_valattrparams rinit_fp8_kv_scalesz!GPUModelRunner.init_fp8_kv_scalessr ,77>>  FD+r22 % % %L'""$$$. . -D '--// 5 5LD&&9l";<< 5 ,4( [)55Dvt,,5 ' 5 5%eU\::5!KK 444)55Dvt,,5 ' 5 5%eU\::5!KK 444+ 5 5r num_tokenscxt|trV|jr|jjddd|fS|jdkr|jjddd|fS|jjd|S|jr|jjdd|fS|jdkr|jjdd|fS|jj|SNr)rrrLrgpurMrr)rrs r_get_positionszGPUModelRunner._get_positionss j# & & 2 @+/;J;??#a'',0KZK@@>%kzk2 2 ?+/: >>#a'',0J??>%j1 1rT)r"sizer!r"c4t|||j|j|dS)N)r!r r with_numpy)rr r)rr!r"rs rr}zGPUModelRunner._make_buffers. ;     rctttf}|js|S|jj}|j}tttf}t|D].\}}|j "|j dx} |||</t|dkr|S|j j d|}g} t|D]c}| |||} tj||| k} | | dtj| |j|d<|S)Ncompressed_token_type_idsrr token_type_ids)rrr r>rqnum_reqsget_pooling_paramsr enumerate extra_kwargsgetr>rrrangerrappendconcatrr ) r model_kwargsrpooling_paramstoken_type_id_requestsrr token_typesrrposidss r_init_model_kwargsz!GPUModelRunner._init_model_kwargssCH~'' $  #,)<<>>!%c3h!1!1!.11 8 8HAu".$)$6$:$:;V$W$WW[-8&q) % & &! + + =$YhY/x ' 'A(,,Q <kv_cache_configkv_cache_groupsrr_rq)rrs r_may_reorder_batchz!GPUModelRunner._may_reorder_batch3sc t#3 4 4 9 9 F  ' 3 7  !%!=       4 3rcztj|j|_|jj|_dS)z;Initialize attributes from torch.cuda.get_device_propertiesN)rrget_device_propertiesr device_propertiesmulti_processor_countnum_smsrs rrxz&GPUModelRunner._init_device_propertiesMs-!&!A!A$+!N!N-C rcBtjdSr)rrrrs r _sync_devicezGPUModelRunner._sync_deviceSs      rc |jD]8}|j|d|j|d9|jD]}|j||jD]}|j|d|j }|jj }|j j }|||z z }|D]}|j|g}|j D]} | j}||jvr,||| } || ?| j} | j} | rJ| jt(jkr5t-j|j} | | jnd} |jrm| J| j}| Jdt;t<|}|j !|}|"| tG|| j$| j%| j&| | | | j'| j(g| j) } | |j|<| r-| j*&| j*dkr |jj+n| j*|j|<|j,r|-| |j.dkr|/| || taj1}|j }|j2}|3}ti|j5D]\}}|j|} |j(|}|j6|}||j v}|j7|}|jj 8|}| j9ri|j:rb|d| _9nX|jj;J|jj;|}||dz }| j9|z }||z}| j<=dg|z|| _(|sx|j>|}|t|z| j@z }|dkr!| j<|dnr|dkr#| j<=|| dnH|t| j<kr0| j<|d=|$|jjA||z} | |jjB|<|s3|0t| j'|D]\}!}"|!=|"n|J|J|| _'|A|j:r#|dkr|jD|}#|#| d| _<|| "||jjE|<| |jjFG|||s6|}$|t|z}%||jjH||$|%f<|%|jjB|<|jI| ||D]7}&|jJ|&|jI|&|8|jK|L||jMdS)abUpdate the cached states and the persistent batch with the scheduler output. The updated states are used by the `_prepare_inputs` function to create the input GPU tensors for the model. The SamplingMetadata is updated and copied to the GPU if there is a new/resumed/paused/finished request in the batch. Nrz!You did not set `task` in the API) req_idprompt_token_ids prompt_embeds mm_featuressampling_paramsr generator block_idsnum_computed_tokensoutput_token_ids lora_requestrrr)Nfinished_req_idsripoprjrqremove_requestfree_encoder_mm_hashesr[num_scheduled_tokenskeysreq_id_to_indexscheduled_cached_reqsresumed_req_idsscheduled_new_reqsr_update_streaming_requestrrr sampling_typerF RANDOM_SEEDr Generatorr  manual_seedseedr>taskrr= get_modelpoolerget_pooling_updatesapplyrrrrrrrprompt_logprobsrrL_init_mrope_positionsrM_init_xdrope_positionsr#r]scheduled_spec_decode_tokens_get_valid_sampled_token_countrreq_ids new_block_idsnum_output_tokensrprev_num_draft_lenrTprev_req_id_to_indexrextend new_token_idsr>rnum_prompt_tokensnum_tokens_no_specr all_token_idsnum_computed_tokens_cpu block_table append_row token_ids_cpuupdate_req_spec_token_ids add_requestcondenserrefresh_metadata)'rrrmm_hashscheduled_req_idscached_req_idsrunscheduled_req_ids reqs_to_add new_req_data req_staterrrrmodel to_updater]req_datascheduled_spec_tokensvalid_sampled_token_countrrrresumed_from_preemptionr req_indexprev_req_index num_accepted num_rejectedr num_new_tokensend_idxrnew_idsresumed_token_idsstart_token_indexend_token_indexrequests' r_update_stateszGPUModelRunner._update_statesVs'7 7 7F M  fd + + +  $ ( ( 6 6 6 6'7 4 4F   + +F 3 3 3 3(> 2 2G   " "7D 1 1 1 1-AFFHH)9>>@@*@P-0AO0ST * 4 4F   + +F 3 3 3 302 ,?; *; *L!(F&& ::6<PP ""9---*:O)8N  !#1\5MMM!O4;??? %%o&:;;;; $ 0%111%*'')L'''0$..2B2BCC!L<*8(4 /-#&0$0$D!#)6   I%.DM& ! ?#B#N'6"<<$//(8(0 6**9555#a''++I666   y ) ) ) )$~~2 #9 0 M%)$G$G$I$I!"8#344i Yi YIAv f-I"*">q"A $215M&,0H&H # ( :1 = (8<v(F%1BDUCUCVCV1WI.""9---CVD  4Y ?( ,77 yQQQ  Q$7!"5M8J8J"J" .0@@BQ 3I>   6 6yBW X X X X# W WG   ( ( 1 1 1   6 6w@U V V V V !!###  0111 ))+++++rrc b|jr |jjsdStj|tj|ddfd|jgddk d }t|D]\}}||j j|<|jjdkrQt#j||j|j |j|j|jj|jdSdS)aUpdate the cached states after model execution. This is used for MTP/EAGLE for hybrid models, as in linear attention, only the last token's state is kept. In MTP/EAGLE, for draft tokens the state are kept util we decide how many tokens are accepted for each sequence, and a shifting is done during the next iteration based on the number of accepted tokens. Nrrrrdimalign)r,r% is_hybridrcatfullrr rargmaxrr"rrqnum_accepted_tokens_cpur&mamba_cache_moderpostprocess_mambarrirr'rr3get_mamba_state_copy_func)rrrrrrs r"_update_states_after_model_executez1GPUModelRunner._update_states_after_model_executeGsM& d.?.I  F  ( -22155q9#3#:    SUU VBZZ SUU UWW% ('':;; E EMAz:DD  4Q 7 7   - 8 8  ) $  $'> 4466      9 8rrr1c|j||j|}|j|_|j|_|j|_|j|_|j|_|j|_|j |_ t|j|j|_ |j |jr|||S)ajUpdates streaming session request from `scheduled_new_reqs`. Removes the request from InputBatch (if present), updates the cached state, and prepares it for re-addition to the batch. NOTE: prompt_token_ids includes intermediate output tokens - tokens previously generated but now are input context (part of the prompt). )rqrrirrrrrrrrKr!rrrLr)rrr1r2s rr z(GPUModelRunner._update_streaming_requestws ''///M&) %1%B " , 8 "."< $0$@ !#/#>  *4 (4(H %&L  & (?' ' # "((*** ? 2  & &y 1 1 1rr2c|}t|s Jd|j Jdtt|}||j|j\|_|_dS)Nz"M-RoPE support is not implemented.z1M-RoPE requires prompt_token_ids to be available.) rr9rrr4get_mrope_input_positionsrrmrope_position_delta)rr2r3 mrope_models rrz$GPUModelRunner._init_mrope_positionss  e$$JJ&JJJ$)55 ?655=%00   1 1*%   B !9#A#A#Arc|}tt|}|j Jdt |s Jd||j|j|_dS)Nz2XD-RoPE requires prompt_token_ids to be available.z#XD-RoPE support is not implemented.)rrr6rr<get_xdrope_input_positionsrr)rr2r3 xdrope_models rrz%GPUModelRunner._init_xdrope_positionss  NE22 )55 @655u%%LL'LLL%%1%L%L  &  !& & """rc,|r|jsiStt}|jD]-}|jD]#}|j||j$.i}t||j|j D]\}}}| ||S)Nr r) r4rrBr rdatarrDr rupdate)rr mm_kwargsreqfeaturemm_kwargs_combined_mm_kwargs_groups r_extract_mm_kwargsz!GPUModelRunner._extract_mm_kwargss  t'N I-.00 #6 3 3C? 3 3<+$$W\222 3 35%@ ;& & &  7 7 !Aq/  % %o 6 6 6 6!!rnum_seqsc~|jsiS|j}|J|}|||Sr)r4rget_modality_with_max_tokens_get_mm_dummy_batch)rrerdummy_modalitys r_dummy_mm_kwargszGPUModelRunner._dummy_mm_kwargssL6 IN $$$"??AA''AAAr cumsum_dtypectj||}|d}tj||z |}|jd||z }||fS)zGet the cumulative sum and batched arange of the given array. # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) # Equivalent to but faster than: # np.concatenate([np.arange(n) for n in num_tokens]) r rN)rcumsumrepeatr)rrrk cu_num_tokenstotal_num_tokenscumsums_offsetsrs r_get_cumsum_and_arangez%GPUModelRunner._get_cumsum_and_aranges\ *LAAA (,)MJ$> KK 1!1 12_Df$$rtotal_num_scheduled_tokensroc|jjW|j||jr4|j||j|dS|jj}|Jg}g}g}g}d} d} d} |j} |jj D]\} }| | x}| |t| | d}| |z } ||dz }| ||z |t!||z dz|dz||jz}|t!|||z| ||kz} t%| |} t|}|| z }||krU|j||jr4|j||j||dkrdS| r`| |dz krW|jjd||jjd|dfd|jrd|jjd|<dSt+j|t*j|j|jd}t+j|t*j|j|jd}|jjd||jj|df |j|sdSt;|jt*jsJt+j|t*j|j|jd}t+j|t*j|j|jd}|jt*j }|jjd|| | dS) aPrepare the input IDs for the current batch. Carefully handles the `prev_sampled_token_ids` which can be cached from the previous engine iteration, in which case those tokens on the GPU need to be copied into the corresponding slots into input_ids.NTrrrrr)r!r)rGindexsrcr )!rqprev_sampled_token_idsr copy_to_gpur3rrrrrrrrr>itemrrrernrcopy_rtensorrrrr scatter_rrrr~flatten)rrrsrorsample_flattened_indicesspec_flattened_indicesprev_common_req_indicesprev_draft_token_indices indices_matchmax_flattened_indextotal_num_spec_tokensr6r cur_index prev_index draft_lenflattened_indexstartnum_commmon_tokenstotal_without_specsampled_tokens_index_tensorprev_common_req_indices_tensordraft_tokens_index_tensorprev_draft_token_indices_tensordraft_token_idss r_prepare_input_idsz!GPUModelRunner._prepare_input_idss   2 : N & &'A B B B( J"../IJJJ!--.HIII F $/D#///.0 ,.-/.0    ! 0 M!%!1!A!G!G!I!I P P FI266v>>> K'..z::: 5 9 9&" E EFF %2%"/ ":"?"?"A"AA"E)//)0KLLL&--/I59?Q;NOO#T%99)//eUY=N0O0OPPP!>> &)*=&O&O# !9::7:OO  2 2 2 N & &'A B B B( J"../IJJJ!--.HIII  " " F  05G!5KLL N 2 22 3 9 9 78K9K8KQ8NO! :   ( B=A!%&9'9&9: F&+l $EKDO' ' ' "T[t" , , $*/ #5;4?* * * "T[t" , , ' ##- 7.1 $     (0F ( F$/>>>>>$)L "%+$/% % % "T[t" , , "+0, $EKDO+ + + "T[t" , , ( /222EE ##+''))*IJ $     rr kv_cache_specrct|tsdSd|jjd|<|D]e}|jj|}|j|}|jd|jj|<8td|jD}||jj|<f|j ||jj d|}|jjd|} || fS)NNNrc3.K|]}|jjVdSr) mm_positionlength)rr`s r z7GPUModelRunner._get_encoder_seq_lens..{s9''/6#*''''''r) rrdrrrqrrirsumrxr) rrrrrr9r2encoder_input_tokensrencoder_seq_lens_cpus r_get_encoder_seq_lensz$GPUModelRunner._get_encoder_seq_lensds -);<< :/0 (++ G GF(8@I f-I$,67%(3 $''':C:O'''$$ 3GD ! $Y / / ))(33304YhY?#47  B!555rc |j}|dksJjj}|dksJjj|t jjd||}|\}}j jd|}t j jj |||j r |jdkr|||jjjdzz} t%j| } t%jjjd| jjd|jrHjj} t%j| d| jjd|jjrd} t;|D]} || }| jjvr| |z } |dkr| |z } *jj| }jj | }||jdkr| |z } e||z}t=||jd}||z }|dkr2jj| | |z |||| |z } ȉjj!||jj"|dj#jd<|j#jd|dz<j#j|dzd$|dj#%j#j&d|dz}jj d||zj'jd|<j'j|d$dj'%fdjj(D}t j)|t j*}j'jd||kj+jd|<j+%|,|||j rDj-j&ddd|f j-jddd|fd nijdkrDj.j&ddd|f j.jddd|fd nj %|t_|j0dk}|s1|dddz }d}t j1|t j*}n/t j2|t j*}t j3|dt j*}|j04D]a\}}jj5|} t_||| <jj | jj6| krt_||| <b7||}|j8}|dz}|j9jd|<j9j|d$dj9%j:rEt j;|j<j=j>ksJ?j||||fS) z] :return: tuple[ logits_indices, spec_decode_metadata, ] rN)rrrc4g|]}j|jSr)rir)rrrs rrz2GPUModelRunner._prepare_inputs.. s#TTTadmA&1TTTrr Tr)@rsrqrr%commit_block_tablerrnrrrraddr$rL_calc_mrope_positionsrM_calc_xdrope_positionsr'rr from_numpy index_selecttoken_ids_cpu_tensorr}rrr3is_token_ids_tensorrreq_prompt_embedsrminrrzcompute_slot_mappingcommit_slot_mappingrfillrxrrrarrayr~rrrrr>ronesrrKrrr!_calc_spec_decode_metadatalogits_indicesrr(rr r+rset_active_loras)rrrrsr req_indicesror positions_np token_indicestoken_indices_tensorr output_idxreq_idx num_sched req_embeds start_posend_pos actual_endactual_num_schedrr num_tokens_npuse_spec_decoderrnum_sampled_tokensnum_draft_tokensrrrs` r_prepare_inputszGPUModelRunner._prepare_inputss&6%P")A----#,!|||| $77AAAiyy 9;OPP !% ; ;

>    1 9 9 ; ; "#>$>#>?      $ +?GGIIL  $%)*E+E*EF        -! (J ?? ( (09 $"2"DDD)+J>>)+J!-?H  ,DWM  0 333)+J$i/ **:1*=>> #- #9 #a''&*"Z2B%BBeJy';<===i'  $99+|TTT $889STTT&'"4AHqL 01 1 /44]25FGGG ((***.2>X\>B   4YhY ?BV V (# #((+++ !!###TTTT4;K;STTT 28<<< M YhY '- 7 !$YhY/ !--h777   &    ? C  $QQQ(C)C(C%C D J J$(,G-G,G)GH! K     !A % %  ! %aaa)D*D)D&D E K K%)!!!-H.H-H*HI! L     N & &'A B B B.KLLqP$ 7 -QRR014N#' !#!B!B!B   "xAAA ')gh"(&K&K&K #">DDFF L L*:6B,/,@,@ )$D] }#||#_? !n$>D] }#||#_? |"ks|kr|@|}$|fS)zQ :return: tuple[attn_metadata, spec_decode_common_attn_metadata] rNc*g|]}tSr)rrrbs rrz.}sGGGTVVGGGrr kv_cache_gidc2J|j}t|tr)tjdftjj}n'jj|}| }| d|S)Nrr$r) rrrerrr~r rqr%get_device_tensorr) rrblk_table_tensor blk_tablerrrrrs r_get_block_tablezBGPUModelRunner._build_attention_metadata.._get_block_tables".3D3P3PP+L9GM-)ABB P#(;$a(+;$$$  !,8F #,#>#>#O#O  Xo5 6 < ._build_attn_group_metadatas. ),7AJ 55dia@@G+L9GM-)@AA X - rrrrrrrnryrqrMrrrxrr%enable_return_routed_expertsrr"rrZrrnum_computed_tokens_cpu_tensorr8r^r:r*rrrdcp_local_seq_lens_cpur&rrnum_logits_indices _prepare_kv_sharing_fast_prefilllogits_indices_paddedrrrrrrrr,rr_rattn_layer_namesrrZrrJrrirrextract_embeds_rangerrrvaluesmm_prefix_rangeunpadded)'rrrrrrrrrrrrrrrblock_table_gid_0slot_mapping_gid_0cm_baserrrkv_cache_groupcmrr_cmreq_doc_rangesrimage_doc_rangesr2 mm_featurepos_info img_doc_ranger ub_metadata _metadatarrrs'` ` `` `` ` @@@r_build_attention_metadataz(GPUModelRunner._build_attention_metadataas& t#3 4 4 9 9t8O-;)5X*/@/L/LL.0  $GGU3}3E3E-F-FGGGM  C,KK-*9H9599;;@@BBK  3 8(C  $ '  2  $ ' 2 7 7 : : :  $ 0 0 2 2 2.> $3 $ $ $ $ $ $ $ $ $ $$(((,,Q//*1-   9 N 2;J; ? C C E E K K M MD ) 045J7J5JK $ 4 89N?Q;N9N O]&'7'78-+,<_,<=%)%5%T  &%/'#0+   "   " "5K !)8),# $@ 66D # '  2  # ' 2 8 8 ; ; ;  # / / @ @ @)-)@)DEUoEU)VG &-1-D-H  .G *  %$*;*S %)7)<);UUU;=8794!#d&6|&D"E"EFF K K ,%./B=RT/U/U%V%VVV c22<3PTUUUUV/.|XrJJJJ  K   ?N*2 ; ;#%  M&1 "+"7;;J)5H$,$A$A$C$CM$++M::::*:6B*:w''-.. ?#0CCK%0%7%7%9%9CC 4B 11CC"/!5!5!7!7??I0>I-- + 7  ' ':9J+J+J 199*hOO ->>>rrnum_common_prefix_blocksc d}t|jj}dt|D}t|D]}|j|D]y}t |jtrd} n6||||||j| } || | || dkz}z|r|ndS)z :return: Optional[cascade_attn_prefix_lens] cascade_attn_prefix_lens is 2D: ``[kv_cache_group_id][attn_group_idx]``, None if we should not use cascade attention Fcg|]}gSrrrs rrzDGPUModelRunner._compute_cascade_attn_prefix_lens..bs%5 5 5 B5 5 5 rrN) r>rrrrZrrre _compute_cascade_attn_prefix_lenrr) rrrr use_cascade_attnnum_kv_cache_groupsrrrrs r!_compute_cascade_attn_prefix_lensz0GPUModelRunner._compute_cascade_attn_prefix_lensTs!!$"6"FGG5 5 1225 5 5  ""566 @ @L".|< @ @ j68PQQ ./++/3.S.S,+0>"0"7799 //+)6==>UVVV $;a$??   @,<E''Erattn_metadata_builderc ||jz}|dkrdSt||}||jz|jz}t|tpt|to|jdu}t|t pt|to|jdu}t|tsJ| |||j |j |j |||j |j } | r|ndS)a6Compute the length of the common prefix for cascade attention. NOTE(woosuk): The common prefix length returned by this function represents the length used specifically for cascade attention, not the actual number of tokens shared between requests. When cascade attention is disabled (use_cascade=False), this function returns 0 even if requests share common tokens. Additionally, the common prefix length is truncated to a multiple of the block size and may be further truncated due to implementation details explained below. Args: num_scheduled_tokens: Number of tokens scheduled per request. num_common_prefix_blocks: Number of shared KV cache blocks. Returns: int: Length of common prefix in tokens. rN) r query_lensrB num_kv_headsrGuse_sliding_windowuse_local_attentionrr8)rprrrkrfsliding_windowrcrErbuse_cascade_attentionrBrrGrr8) rrrr rrrrr use_cascades rrz/GPUModelRunner._compute_cascade_attn_prefix_lenxs;45}7OO  ! !1R 13F3J3J3L3LMM !9 9M -77777+AA/+ 0&3n1 3L.B   %06  Q6rcrd}t|jjD]\}}|j|}|jJ|jj|}|j|}t|j|j }||z|kr'td||z } td|| z } n|} d} || | zksJ| dkr9|} || z} |} || z}|jdd| |f|jj dd| | f<|| z }| dkr@|} || z} |j Jtj|jj| |j || z| || z }dS)Nr)r out_offsetrU context_lenr=)rrqrrirr$rrKrrrnrrUr/get_next_input_positions_tensorr)rr mrope_pos_ptrrurr_rrr!prompt_part_lencompletion_part_len dst_startdst_end src_startsrc_ends rrz$GPUModelRunner._calc_mrope_positionss &t'7'?@@- 5- 5ME6-'C&222"&"2"J5"Q #3#H#P F$c&7!! #%990 "Q&&) '*==/;;; @,/(),)A 3o E#6 !44 [- 5- 5rcTd}t|jjD] \}}|j|}|jJ|jj|}|j|}t|j|j }||z|kr'td||z } td|| z } n|} d} || | zksJ| dkr9|} || z} |} || z}|jdd| |f|jj dd| | f<|| z }| dkr1|} || z} tj |jj| || z| || z } dS)Nr)rrrr=)rrqrrirr$rrKrrrnrr0rr)rrxdrope_pos_ptrrurr_rrr!r!r"r#r$r%r&s rrz%GPUModelRunner._calc_xdrope_positions s&t'7'?@@+ 6+ 6ME6-'C'333"&"2"J5"Q #3#H#P F$c&7!! #%99*>?/1"Q&&* (+>>!A-0( 3o E#6 "55W+ 6+ 6rrcu_num_scheduled_tokensc p|dz}||tj\}}tj||z |}||z }|dz }||tj\}}tj||z |} | |z } t j||jd}t j||jd}t j||jd}t j| |jd} t j||jd}|jj |} | | dz} t| | ||| ||S)Nr)rkTr)rrcu_num_draft_tokenscu_num_sampled_tokenstarget_logits_indicesbonus_logits_indicesr) rrrr~rnrrrr rrrr) rrr)rr,rrr.r+r-rs rrz)GPUModelRunner._calc_spec_decode_metadata9s".1)-(C(C RX)D) ) %v #&8 8:L   &  5q8 '+&A&A 28'B' ' #V!# !$6 68H! !  '$./BCCFF KdG  !& 01F G G J J Kd!K! ! ).99<< Kd=  !& 01F G G J J Kd!K! !  %/0DEEHH Kd I   .,^<)*?!*CD!+-4466 3"7"7!5)    rcd|jJ|jd}|dksJ|jd|||j|d|d|j|d\}}|j}|jd|}|S)NrrT disable_full)rrrzrryrdispatchr)rr num_logitsrb batch_descnum_logits_paddedrs rrz/GPUModelRunner._prepare_kv_sharing_fast_prefills:FFF#)!, A~~~~ 3KZK@FF~VVV 3JKK@FF 2  # # % %   1:: T;  :'1 $ K   ! %$rc|j}|sgggfStt}tt}tttt f}|D]|\}}|j|}|D]g} |j| } | j | | j | | j | || j fh}|||fS)aBatch multimodal inputs from scheduled encoder inputs. Args: scheduler_output: The scheduler output containing scheduled encoder inputs. Returns: A tuple of (mm_hashes, mm_kwargs, mm_lora_refs) where: - mm_hashes: List of multimodal hashes for each item - mm_kwargs: List of multimodal kwargs for each item - mm_lora_refs: List of (req_id, placeholder_range) for each item ) scheduled_encoder_inputsrrrBrmrCrrirr\r identifierr) rrr7 mm_hashesr^ mm_lora_refsrencoder_input_idsr2 mm_input_idrs r_batch_mm_inputs_from_schedulerz.GPUModelRunner._batch_mm_inputs_from_schedulers($4#L ' r2: IKK -.00 E#'7"789;; )A)G)G)I)I F F %F% f-I0 F F &2;? ?*  !6777  111##VZ-C$DEEEE F)\11rc |\}}}|sgStjojjo|j}t t j}jr/j rg}g}t} g} |D]\} } j j | } tj j| }j| j}||||g|z| ||dkr6j j|}|| |t/t1|t1|dt2j}j | |t9jdrÈfd| D}t;jt;j|t:j t;j|t:j }t/t1|!t1|dt2j"}j | |g}d}tG|j$j%D]C\}}}j&r|d kr|d krtOtPj)}tU|D]}|||z}+||||zd 5tYtG|gj$j%\}}}|j-d i|}||dddn #1swxYwY|} n=+||||5|j-d i|} dddn #1swxYwYt]| | || ||z }Et_||D]E\}!}"|"j0|!<tb2d |!3j0|!F|S)NrT) is_prefillrget_num_mm_connector_tokenscDg|]}j|Sr)r3r@)rrrs rrz6GPUModelRunner._execute_mm_encoder.. s9""""J:::FF"""rr ) index_mappingprompt_mappingr?rr[videorexpected_num_itemszFinish execute for mm hash %sr)4r=rr-enable_mm_processor_statsr7rr5r3r( lora_managersupports_tower_connector_lorarrqrrrequest_lora_mappingget_num_mm_encoder_tokensget_num_embedsrrlora_id_to_lora_requestrrr+rmr,TOWERset_active_adaptersrrrnrr~r CONNECTORrDr rr5rrrrtimed_encoder_operationnextembed_multimodalrrr[loggerdebugmaybe_save_ec_to_connector)#rrr9r^r: should_timer3prompt_lora_mappingtoken_lora_mapping lora_requestsencoder_token_countsrrrlora_idrr tower_mappingpost_op_countsconnector_token_mappingconnector_mappingencoder_outputscurrent_item_idxmodality num_itemsrccurr_group_outputs_lst video_idxvideo_mm_kwargs_itemrbmicro_batch_mm_inputsmicro_batch_outputscurr_group_outputsr,rs#` r_execute_mm_encoderz"GPUModelRunner._execute_mm_encoders.2-Q-Q . . * 9l I  % :)C : 9  '44  6  1 O O Q Q6 #% !# EEM#% $0 8 8 *:6Bd.CGLMM"ZAA+ $**7333"))7)j*@AAA$++J777Q;;#'#3#K#O#OPW#X#XL#/%)),777(()))**$* M   1 1- O O Otz#@AA """"&:""" +-)H0AAAH^28<<<++'%0"'(?(F(F(H(H"I"I#()<#=#=#(2 %%%!!55!% /14O ;5 5 5 > *> * 0Hi"2& S''MM)-el);)=)=&!&y!1!1KKI+45E 5Q+R(55#\3Ci3OQRKK7;7!5 6'+{+/?77313/Ee.D//3//+/556IJJJKKKKKKKKKKKKKKK"&<""11/?SS*@)?)R)R/)R)R&SSSSSSSSSSSSSSS ,"#,      " "#5 6 6 6  )   #9o>> I IOGV*0D w ' LL8' B B B  + +D,> H H H Hs%&ANN N *OO O rshift_computed_tokenscp|j}d|jz |_|j|j}ttj}|j}d|d|<d}d}d} |jjD]} g} |j | } |j | } | j |z}| j D]}|j }|j}|j}||| zkrn||z|kr-t!||z d}t#||z | z|}||ksJ|||\}}||kr~|j}|j|d}|Jd|d|jx}|||}|||}n |||}||z|z }|d|||z||z<n|||z||zxx|zcc<| ||jrd|jr]| jJd}|j| j| | j| j \} }}| j||| _| | || z }|!|}|r/|"||j!|| r/|#||j$!|||fS)NrFrzEncoder cache miss for .T)rmultimodal_embeddingsrr)%rsrrrrrrrqrrrirrroffsetrrnrget_embeds_indices_in_ranger8r[ris_embedrr5rLrr3recompute_mrope_positionsrrzrUrrxrrr)rrrlrsis_mm_embed_buf mm_embeds is_mm_embed req_start_idxshould_sync_mrope_positionsshould_sync_xdrope_positionsr mm_embeds_reqrr2rrrrnum_encoder_tokens start_idxr>curr_embeds_startcurr_embeds_endr,encoder_outputrrmm_embeds_item req_start_posnew_mrope_positions new_deltas r_gather_mm_embeddingsz$GPUModelRunner._gather_mm_embeddingsh s &6%P" !4#77243GH&(( %) 38 ///0 &+#',$&.K 2K 2F02M#3#H#P  f-I"+"?BW"W '33 53 5 %1$O %-_"  36J JJJE115HHH 3i ?CC ')36JJ&!7****88GLL3!? %77$/!%!3!7!7!F!F%113WW3W3W3W111 ( 11H>' '(9:H%34Eo4U%VNN%3Ig4E%FN - 9 2I)//0CDDD1: .   ] + + + 1 1MM%112LMM & I  & &'7 8 8 8  , ,-G H H H ' J  ' '(8 9 9 9  ! - -.H I I I+%%rct|jttfr|jS|jSr)rr3rrunwraprs rrzGPUModelRunner.get_model s8 dj#3]"C D D ':$$&& &zrc|}tt}t|r|dt |r|jrdgS|d|S)Ngenerate transcription)rrrHr?rr;supports_transcription_only)rr3supported_taskss rget_supported_generation_tasksz-GPUModelRunner.get_supported_generation_tasks s  ~.00 #E * * /  " ": . . . !% ( ( 40 )'((  " "? 3 3 3rcD|}t|sgSt|j}d|vrPt |jjdd}|dkr/|dt d|S)Nscore num_labelsrrz.Score API is only enabled for num_labels == 1.) rr>rrget_supported_tasksrr% hf_configremoverT debug_once)rr3rrs rget_supported_pooling_tasksz*GPUModelRunner.get_supported_pooling_tasks s  && Iu|??AABB o % % !2!> ? ? ?   (I 5 5 LL99;; < < <U||rr sync_selfc |jJ|jjj t |j|ra|J|D]H\}}|dko}|r zn}|j|d||d|dIt fd|jDS)NresidualTrcVi|]%\}}||dkrr |dzn |d&S)rNr)rkvis_rsrtps r zFGPUModelRunner.sync_and_slice_intermediate_tensors.. s_   Aq ??u?'zR''(({ {^   r)rr r*tensor_parallel_sizerrrzrG) rrrrrr is_scatteredcopy_lenrrs ` @@r#sync_and_slice_intermediate_tensorsz2GPUModelRunner.sync_and_slice_intermediate_tensors s" (444   - B,T-=zJJ  '333,2244  1 J85 /;K:++)!,YhY7==ixiLt>#      !5;;==       ris_dummy is_profilec|jjsdS|jJ|}t |sJ|j|||jjjdS)zN Step for the EPLB (Expert Parallelism Load Balancing) state. N) log_stats)r* enable_eplbrVrr7step eplb_configlog_balancedness)rrrr3s r eplb_stepzGPUModelRunner.eplb_step" s#/  F***  $U+++++   *6G      rrnum_scheduled_tokens_nprcJ|jj}|t|jjks Jd|d|}|jjd|}|j}||||jtt|j }| ||} dt||jD} t|jj|jj|} | t'| s dg|z| _| S|jrt-| | | |jSt1d| } dt| | D| _|| S) NzEEither all or none of the requests in a batch must be pooling requestrrpooling_metadatac g|] \}}||k Srr)rseq_len prompt_lens rrz(GPUModelRunner._pool..K s1   # z !   r)rrr)rrrrc8|dn|ddSrrrs rrz&GPUModelRunner._pool..c saiddQTT%dT-K-Krc g|] \}}|r|nd Srrrs rrz(GPUModelRunner._pool..f s5- - - W $CC- - - r)rqrr>rrrget_pooling_metadatabuild_pooling_cursorr rr=r3rr prompt_lensrtrrranyrrTrrrLr) rrrrrr seq_lens_cpurr3rrrs r_poolzGPUModelRunner._pool2 s#,3t/>????? S@??&&;';&;< }((3 +@@BB-- #\-:N .   ($*55*/,,':J+7+ +   '*<9I9U'V'V   0$,1133 ,<AACC 3     $C ,>,> $150A  -& &  $ 3$7"3+)-)F  , K K   - - #$5} E E- - - ) ""rcv|jjj}|jjjr|dkrt ||S|SNr)r r*rr' pass_config enable_sprN)rrtp_sizes r_pad_for_sequence_parallelismz,GPUModelRunner._pad_for_sequence_parallelismn sB"2G  " . 8 ;Wq[[0':: :##rcx|jjr|jjd|}nd}|jjd|}||fSr)r3requires_raw_input_tokensrrr)rrrrs r_prepare_mm_inputsz!GPUModelRunner._prepare_mm_inputsv sJ : / *;J;7III*.{ {; -''rnum_input_tokensc|j}tj}|jj}d}|jr|r|s|||j5}||| |\}} dddn #1swxYwY|j |j j d||| } |jj d|| ||\} } i|||} n|jr|r|jj d|dd}|dkr<|j j |}|j |}||jj |<|jj d|} |} d} n*|j j d|} d} |} |jr|jj ddd|f}n8|jdkr|jj ddd|f}n|jj d|}|rd}n|J|||d}|r3|jr,||}| d |i| | ||| |fS) Nr[)ro is_multimodalF)as_tuplerr)rTra) rsr# is_first_rankr%rPrOmaybe_get_ec_connector_outputr[rkrr3embed_input_idsrrrrzrrrdr3rnonzerosqueezenumelrLrrMrrrr7r])rrrrrrrPrrurvinputs_embeds_scheduledrrr token_ids_idx token_idstokens_to_embedsrras r _preprocesszGPUModelRunner._preprocess s 0J$4 !.A#  "< 5}< 5=O< 533 "04 V%(()9:::)-)C)CDT)U)U& ;  V V V V V V V V V V V V V V V'+j&@&@"#8$8#89&/)'A'' #   "#8$8#8 9 ? ?@W X X X'+'>'>?O'P'P $I}))++))*:;;LL &" 5=" 5!%&;';&;<%((  ""$$q(( N.}= #':#=#= #=#R#R 8H"&}5 .23D4D3DEM2244LII *+<,<+<=I M2244L ? >,04E5E4E1EFII  !A % %-1!!!5F6F5F2FGII*+<,<+<=I  #' '333#'#K#K "6$$   F"2"K F #667GHHO   !2O D E E E        s.B  BBrrc0|jj}|j||||S|jr8|j1|\}}|j|||d||}|S)Nrsampling_metadata) rqrupdate_async_output_token_idsrUrTr_get_draft_token_ids_cpuupdate_async_spec_token_idsrd)rrrrrrbsampler_outputs r_samplezGPUModelRunner._sample s !,> 66888  '<<"3    $ N)B)N%)%B%B%D%D "    8 89L M M M//       rrci}tjr||}|jj}t j|jjd|d} | D]Z} |jj t| } | *| | dz [|jj } |jj} |jjd}|j}|j}g}d}|js|jd}|dkrX||}| D])} |t| *||}nt/j||jj| |\}}ng}| }t7||jj|jddksJ||j_fdt;|jj D|j_|jj }t?|D]}|jr |vrdgnd}n||}|rtA|nd}|s1|jj!|}||z}||j"ksJd|d|j"||jj#|||f<d |jj$|||f<||jj!|<||}|j%|}|j&'||(|d||j)}||||| | |fS) Nrrrrc$i|] \}}|v || Srr)rrrinvalid_req_indices_sets rrz4GPUModelRunner._bookkeeping_sync..[ s5555Av333333rzGSampled token IDs exceed the max model length. Total number of tokens: z > max_model_len: T)*envsVLLM_COMPUTE_NANS_IN_LOGITS_get_nans_in_logitsrqrrrr generatorsrr set_offset get_offsetrrrrrrrT_to_listrrr~rrrrrwrrrr>r"rr'rrirr_get_prompt_logprobs_dictr)rrrrrrrnum_nans_in_logitsr"discard_sampled_tokens_req_indicesrgenreq_ids_output_copyreq_id_to_index_output_copyrrrrrrrrr sampled_idsnum_sampled_idsr|r>rr2prompt_logprobs_dictrs @r_bookkeeping_syncz GPUModelRunner._bookkeeping_sync s"   + B!%!9!9&!A!A #,-/Z  % (( 3. . . *4 5 5A"-11#a&&99Cs~~//!3444#.6;;==&*&6&F&K&K&M&M#+=CAF*<): ($ +1"5Ka*.--8I*J*J';<&> # 6>(.r2a7777:K 75555!*4+;+C!D!D555D  1"*/00 ; ;G( ?&-5L&L&LrddRV 5g> 7B#I3{#3#3#3O (;GDI/1Gd0000(+2((%((100 JUD  *7Ig4E+E FHLD  )'9W3D*D E;BD  / 8W%F f-I  & - -k : : : : $== /// 0  1     #  '   rc#K|jdVdS|j dV|jdS#|jwxYwr)rrrrrs rsynchronize_input_prepz%GPUModelRunner.synchronize_input_prep sx  $ , EEE F !--/// / EEE  % , , . . . . .D % , , . . . .s A A$rrrrc &|jd||||d|S)aMHelper method to call the model forward pass. This method can be overridden by subclasses for model execution. Motivation: We can inspect only this method versus the whole execute_model, which has additional logic. Args: input_ids: Input token IDs positions: Token positions intermediate_tensors: Tensors from previous pipeline stages inputs_embeds: Input embeddings (alternative to input_ids) **model_kwargs: Additional model arguments Returns: Model output tensor rrrrr)r3)rrrrrrs r_model_forwardzGPUModelRunner._model_forward s:0tz !5'       rmax_num_scheduled_tokensrforce_uniform_decodec(|||ko|||zkn|S)zn Checks if it's a decode batch with same amount scheduled tokens across all requests. rrrrrrs r_is_uniform_decodez!GPUModelRunner._is_uniform_decode s4"$+*-EEH#;h#FF&  rrallow_microbatching force_eagerforce_has_loranum_encoder_reqsc |j|||jjo| dk} | t jjdkn| |fd} | |p| \} }|jj j j r'|jj j jzdks Jdd\}}j j jdkrj jt"jk}t'|j |||| j\}}}|cj j}t-||| |t"jjk\} }|jksJd}j jjr-t7||j|j|z t9|  }| ||||fS) Nrrczsj||ntjt fS)N)rhas_lorauniform_decoder1)rr2rrur()rr1rrrrrs rrzGGPUModelRunner._determine_batch_execution_and_padding.. sU  .JT-F-O-O%!-) .P... $o6G&H&HIrzQSequence parallelism requires num_tokens to be a multiple of tensor parallel size)FNr)num_tokens_unpaddedr*rallow_dp_paddingrr num_scheduled_tokens_per_requestrtr0)num_unpadded_tokensnum_padded_tokens num_paddings runtime_mode)rrr%rPr>rqrMrrr'rrr r*rdata_parallel_sizertrrurvaluedata_parallel_rankrry PIECEWISEr-cudagraph_metricsrr)rrrrrrrrrrrhas_encoder_outputdispatch_cudagraphrtbatch_descriptor should_ubatchnum_tokens_across_dprsynced_cudagraph_modedp_rankrrrrs` ` @@@r&_determine_batch_execution_and_paddingz5GPUModelRunner._determine_batch_execution_and_padding s*00%=%)%B!!5 1     0 I5E5I  %  8 9 9A = =  !>>zJJ J J J J J J J J ,>+= /E3E, , ((-7  " . 8  +"2GH5 /:+ +   + > B B '6-:LL  +(2$($8(;%5&7#15L#1#7    GM/1F$/.A$'(._get_slot_mapping s".3D3P3PP 0@ -)ABB N${&(+;    !,8F (59:L;L:LM  ,->> ? E Eb I I I rc.i|]\}}||Srr)rgidrbr"s rrz5GPUModelRunner._get_slot_mappings.. s;   Q ""3''   r) rrr>rrrrr token_slicer)rrrrrslot_mappings_by_gidslot_mappings_by_layerr$rrrresultubatchsliced_mappingsr"s```` @r_get_slot_mappingsz!GPUModelRunner._get_slot_mappingss s2 D+ , , $0D(899A==: C         *    #D$8$HII    ;=#,T-A-Q#R#R B B C/4L,8 B B 5A&z22 B  $46F' / /;=0F0L0L0N0NSS,J 2>v?Q2ROJ// o....'/ /#%;;;rcjtdjjjrDt j}||nt dj r4tr&t j j}td55t%rt'jrpj5}t1cdddcdddcdddS#1swxYwY|sjjdkr%jjdkrdtst:cdddcdddSjcdddcdddSjj rj!r Jdj"j#}j"j$}fd|D}tKj&|tJj' } tQ| )} j} *| \} } d}j+r:jj,s.-| j"j.d|j/}0| || | |dutcj2 \}}}}}t3d |||||j4}|j#|j#n|}tk|| ||jj6\}}t3d ||tofd tqj9j:D }|tvj<k}jj=dkrUt}j?j9jj@j"jAjBjCjDEtcjFdk}|r|n|}G|s|r|n| |s|r|n|| |\}}H| |r|nd||r|nd| || |jI|| \}}J||\} }!}"}}#}dddn #1swxYwYdddn #1swxYwYjKrtvjL}d_Ktcj2}$jjMo|$dk}%t|j|||||||% 5td5O5}&jPd| |"||!d|#}'dddn #1swxYwYdddn #1swxYwYdddn #1swxYwYtd5jQr|'\}(})n|'}(d})jRstjTs3t|(tsJ|&|(_W|&_W|(cdddSjXr$Y|(|| |&cdddS|(| }*jDZ|*}+njXrJ|(| }*tjTsPdtj| i},t\|(j]t|,d}+njDZ|*}+i}-|+|+_|-d<t`|-tctjadz }.|.J|.d}+dddn #1swxYwYt|+| ||(|*|)||| _|&_WdS)NzOState error: sample_tokens() must be called after execute_model() returns None.&RoutedExpertsCapturer not initialized.zgpu_model_runner: preprocessrrrz--kv-sharing-fast-prefill produces incorrect logprobs for prompt tokens, tokens, please disable it when the requests need prompt logprobsc*g|]}j|Sr)r)rrrs rrz0GPUModelRunner.execute_model.. s"PPP1&;A>PPPrr )rrrrrrzhRunning batch with cudagraph_mode: %s, batch_descriptor: %s, should_ubatch: %s, num_tokens_across_dp: %s+ubatch_slices: %s, ubatch_slices_padded: %sc3K|]E\}}t|jttdj|DVFdS)c3.K|]}|jjVdSr)backend forward_includes_kv_cache_update)rgs rrz9GPUModelRunner.execute_model...B s;I>rN)rrreallrZ)ridspecrs rrz/GPUModelRunner.execute_model..A s-- B!$"46NOO -!-b1------rrHrrrrr) rrrrrrrrrrrF)rrcudagraph_runtime_moderrr skip_compiledzgpu_model_runner: forwardrzgpu_model_runner: postprocessr)all_gather_groupall_gather_tensorsr)rvr)cr RuntimeErrorr r%rr. get_instance clear_bufferrTerrorpreempted_req_idsr rhandle_preemptionsrsrrrDrr is_producerrr[rkrwr*r=r  _dummy_runrmkv_connector_no_forwardr&rrjrqrrrrr~rrnrrI use_ubatchingrr$r rr>r7rUrr num_ubatchesr5rrrrFULLrNrpreprocess_mambarrir'rr3rPrr+r rrr6rurPr)maybe_get_kv_connector_outputrr\r@r#r]rrGrr>rcompute_logitsrsend_tensor_dicttensorsr$ contiguousbroadcast_tensor_dictr?r)/rrrcapturerrrrrtokensrrrrrrrtr4rrrrrrubatch_slices_paddedhas_separate_kv_updatepad_attnrubatch_slices_attnslot_mappings_by_grouprrrrrrrrhas_encoder_inputr model_outputrrrrr<model_output_broadcast_data broadcasteds/`` r execute_modelzGPUModelRunner.execute_model sx  # /6    ( E G,9;;H#%%'''' EFFF  - 2G2I2I  ! # # 6 6 2    0J *+I J Jh h   ' ' ) )h h     0 1 1 1   T_%6%6%B T77$"&"48T),,-=>>>ABRSS TTTTTTTh h h h h h h h h h h h h h h TTTTTTTT( X(E*++,?!CCOOA&&&,..54=h h h h h h h h h h h h h h h >334DdFVWW?h h h h h h h h h h h h h h h B 8 3@3 '0H&.GPPPPPPPF&(hvRX&F&F&F #'*+B+F+F+H+H'I'I $"2"M 373G3G '44 0N0 (, $( 1E1S +/+Q+Q+$$    !+ 5 '1':'F ##H 3M'!$1 33 /M/ LL=$   *----- !*$*>*N O O ---**& "&);;H 1W<<,$(%($M+BJ88::   ""2"OPPSTTO9A!T!5!5} 484K4K#)5#)"3"3('/W3IWOOx$725L 5 5 1 "M..2;C&M&7&7%7?$IOOT":"4#1$3)9)N-E"8/   @+%AS''$E0 S' S>0E4 4S'7E4 8AS' S>S'9 S>K S' S>'S+ +S>.S+ /S>>TT&W6W V+ W+V/ /W2V/ 3W6 WW W W WW W3A^%^%=D^%%^),^)grammar_outputzGrammarOutput | Nonec j}d_j<|sdS|rtSt t}||_|Sj\ }}}d_|t |j|td5|}dddn #1swxYwY |j d_ d_ dj_ fd}j} d} | Sduojjzjk} | p| o| j } | rt+jt.t0zsJ|j } | r || nˉjJj| jjjj\}}||t?j dj!t>j"#tIjj%j_ &dn| } td5'||j(\}}}}}}}dddn #1swxYwY| r ||td 5)dddn #1swxYwYtd 5j*j+rKtYj-}||.j/ nt`1d te||||||j3r|nd|| }dddn #1swxYwYj4s|Std5tk||j |j6|j7jj8}dddn #1swxYwYtd5j9|j:|j;dddn #1swxYwY|S)Nzgpu_model_runner: samplec Jtd5|jj _ddddS#1swxYwYdS)Nzgpu_model_runner: draft)rpropose_draft_token_idsrqrr_copy_draft_token_ids_to_cpu) rrrrrrrrrs rr_z=GPUModelRunner.sample_tokens..propose_draft_token_ids(s3???/0IJJ D D(,(D(D$%$6!(%(4! ) )%112BCCC D D D D D D D D D D D D D D D D D DsAA$$A(+A(Frr r!T) zeros_onlyzgpu_model_runner: bookkeepzgpu_model_runner: eplbz#gpu_model_runner: ModelRunnerOutput)indicesr-) rrrrrrrrrz+gpu_model_runner: AsyncGPUModelRunnerOutput)rrrrrrz-gpu_model_runner: set_async_sampled_token_ids)rr`rrsrr%rr.r>save_captured_expertsrrTr@rtrOrTrrrrset_async_sampled_token_idsrr) rr\rrrrrrr_ spec_config propose_drafts_after_bookkeepinginput_fits_in_drafter use_gpu_toksrnext_token_idsvalid_sampled_tokens_countrrrrrrrrP async_outputrrrrrrrs ` @@@@@@@r sample_tokenszGPUModelRunner.sample_tokens s#6#'  # +& t#++-- 100344F)   !%$(!26/ D D D D D D D D D D D D - +0(  "$DD$P%0>  I"$, @R0RSSSSS$2$D!(Y++,=>>>>9E;GGG BB<- M , 59 ?N$>88&(B -2K$+U[---fS!1!9::D>CC8(J,,J03J0K44K8;K8A9NNN6/O11O58O5 &P==QQcp|jr|jsdS|\}}t||Sr)rerrro)rrrs rtake_draft_token_idsz#GPUModelRunner.take_draft_token_idssB# 4+D 4#'#@#@#B#B Wo666rrbc|jr|js|jjjsdS|jj|_|j}tj |sdS|j J|j J|j Jtj}|jd}tj|j 5|s?|j ||j d||dn d|j d|<|j ddddS#1swxYwYdSNrTr)rThas_structured_output_requestsrqrrrrrrr is_tensorrrrrrrrrrzr)rrrbrrrs rr`z+GPUModelRunner._copy_draft_token_ids_to_cpus  $   ; 1B  F$($4$<$A$A$C$C!(,(=//  F)555/;;;'3332244"(+ Z  t? @ @ 0 0 80<<^LLL((399#$: 78((3  & - - / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s A'EEEc,t|jtr|j|jjfS|j}|ggfS|jJ|jJ|j|jdt| |fSr) rrrrqrrrrrr>r)rrs rrz'GPUModelRunner._get_draft_token_ids_cpus d+T 2 2 C($*:*BB B+ ?r6M)555'333 "..000'#g,,7>>@@'IIrrorpc|jdStj}tj|j5|j||}|j}|J|d|jd |d|j dddn #1swxYwY| d|j _ dS)NrTrr)rrrrrrrrrrzr unsqueezerqrw)rrorprcounts counts_cpus rrgz.GPUModelRunner._copy_valid_sampled_token_counts&  / 7 F2244Z  tI J J : :  6 B B> R R R/F;J))) (a( ) / /T / J J J  0 7 7 9 9 9  : : : : : : : : : : : : : : :3A2J2J12M2M///s A+CC Cc|jj}|j}||gS|j}|J||d|jdSr)rqrwrrrrr)rrwsampled_count_eventr}s rrz-GPUModelRunner._get_valid_sampled_token_countst!%!1!H"B  &*@*HI7 %%%'')));28;;<CCEEErrrrrrc r |j|j} | J| jdkrgt|tsJt|jt sJ|j||jj |jj | } n/| jdkrWt|tsJt|jtsJ|j|j|| } n| jdkrt|tsJt|jtsJ|j dt|kr|}n~g} d} | Jdt|j|D]5\}}| | t|zdz | |dzz } 6t%j| |j} || }|j||| } n| s| rt|jt.t0zsJ| jrLt|ts Jd |j||j|j|j}not|t$js Jd |j|||j|j|jj \}}|!||d}|fd}|j"j d}|#}|j$r(|Jt%j%fd |Dd }n|d}n| jrd}|j&|||j\}|j"j }|#}|j$r'|Jt%j%fd|Dd }n|}n|j'|||\}}}|j(|j"j d}|#}|j$r'|Jt%j%fd|Dd }n |d}|j)r|*|d}nd}|j||||||||||  } | S)Nr)rrrrz"No spec decode metadata for medusarr)target_hidden_statesrrzGsampled_token_ids should be a python list whenpadded-batch is disabled.zGsampled_token_ids should be a torch.Tensor whenpadded-batch is enabled.c$g|] }|d Srr)rhrs rrz:GPUModelRunner.propose_draft_token_ids..cs%MMMa0001MMMrrrFc g|] }| Srr)rrrs rrz:GPUModelRunner.propose_draft_token_ids..tsIII!Q}-IIIrc$g|] }|d Srr)rrrps rrz:GPUModelRunner.propose_draft_token_ids..s%MMMaQ0 001MMMr)rl) target_token_idstarget_positionsrrolast_token_indicesrrmm_embed_inputsnum_rejected_tokens_gpur)+rsr,r^rrr_rproposerqr"r'rrrr>rrrrr{r rar`rrreprepare_next_token_ids_cpurirrrfrrrgrrr\rJprepare_inputsprepare_inputs_paddedrrOr)rrrrrrrrrrrkrrcrp num_draftrQrorprtoken_indices_to_samplerrrrrrrps @@@rr_z&GPUModelRunner.propose_draft_token_idss 0J- &&&   ( (/66 6 66dlM:: : ::"l22! 3 .+ 3OO  8 + +/66 6 66dl,BCC C CC"l22 "3=3OO 8 + +/66 6 66dlN;; ; ;;#)!,4E0F0FFF 4 +778877*-(9;L**,,%IvNN6CKK#7!#;<<<i!m+FF,wt{CCC 4W = "l22%2"3+3OO  " " $ $n  (D(D(F(Fn dlM>,) (15 ; :44"$>'+ ##+*.'#'>#56K7K6K#L #'#6#67K#L#L 4P,888+09MMMM;LMMMSU,,,((,99N:N9N+O((;$P.2+:>,:U:U,),=;;7(- (,~'9-'H$'+':':='I'I$8L0<<N=N/O,& '"&"<"<$*+#=## #'"l22!1!1%9-#:"3%9 /(?+3  Or overridescddh}|D]J\}}||vsJd|d|t||}t||}t|||KdS)Nr)r%zConfig `z"` not supported. Allowed configs: )rrrsetattr)rrallowed_config_names config_nameconfig_overridesconfig new_configs rrzGPUModelRunner.update_configs -~>-6__->-> 3 3 )K)"6666;;;;$8;;766T;//F&v/?@@J D+z 2 2 2 2 3 3r eep_scale_upc@ td|jjd|rt j|jnd\}}}|jjr!t |j|j|_ d} t5}tj }t|j}||j|j|_|jr+||j|j|j|_t'|dr!td|j|jt'|jd rt+|jjr|jjr|jj} | J| jJtd | jj|r||nd } |r||nd } |j t |j|j|_ |j |jj| j| | ||d z }|jrt5|st9d |} | rtd| n|j} |j | tj } d d d n #1swxYwY|j!|_"nB#tFj$j%$r+}d}|d|d}t&||d }~wwxYwtdtO|j"| |z dtQ|jtS|dd x}r"tS|d d x}rtQ||jj*}tW|o|d uo|,|_,t+|jr|jjrtd|jj|r||nd } |r||nd } |j J|j |j|j| | ||j j-r|j .||jj/j0tbj2krW|jj/3|j}thxj5d z c_5|j6d|d S|j/j7}|J|8r9|jj9s-tu|j|jtvj<|_d S|jj9rx|8r2t{|j|jtvj<|j|_d St{|j|jtvj>|j|_d Sd S)z_ Args: eep_scale_up: the model loading is for elastic EP scale up. zStarting to load model %s...globalscope)NNNr)r r%r_zLoading drafter model...r3Nz%EPLB is enabled for drafter model %s.rzRModel does not support EAGLE3 interface but aux_hidden_state_outputs was requestedz2Using auxiliary layers from speculative config: %szFailed to load model - not enough GPU memory. Try lowering --gpu-memory-utilization to free memory for weights, increasing --tensor-parallel-size, or using --quantization. See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more tips.z (original error: )z1Model loading took %s GiB memory and %.6f secondslocalzEPLB is enabled for model %s.) rank_mappingT) fullgraphr2)r )?rT info_oncer%r3r get_eep_stater*rr rVrOtime perf_counterr2r) load_modelr r(load_lora_modelrr_r7r,rg add_modelr\r8rr="_get_eagle3_aux_layers_from_configinfo"get_eagle3_aux_hidden_state_layersset_aux_hidden_state_layersconsumed_memorymodel_memory_usagerrOutOfMemoryErrorr@rPr'rmultimodal_configr:r5is_asyncstart_async_loopr'rrr init_backendrstock_torch_compile_countcompilerthas_full_cudagraphsrFrrrHrru)rrglobal_expert_loads#old_global_expert_indices_per_modelr eplb_modelsmtime_before_load model_loaderrkglobal_expert_loadold_global_expert_indices aux_layerstime_after_loademsg combined_msgr_ drafter_model mm_configr2rts rrzGPUModelRunner.load_models   *   #     $I #D$8 9 9 9# O@,   + '(>+;+;<<*E"&!H!H!J!JJ!U P& &*Z%R%R%T%T J:::FFF"&"3"5"5ID 6D 6D 6D 6D 6D 6D 6D 6D 6D 6D 6D 6D 6D 6D 6J'(&7D # #z*   ! "99Q999L LL & & &G   ? t. / / . .    /tz:::tY55 5G B$Wgt<< O>U V V V4GQ#K00T  73K@@ & ?... O % % !")    ' L00l0KKK   / 42 3 3&9FFtGWXXG  9 9Q > 9 9 J  w  ? ? ? F 0?)))  . . 0 0 (6 * D,=;MDJJJ ! / 1133 *J 0-2Ddk +J 0-2Ddk   s=7L I!K2& L 2K66L 9K6:L M &MM c|jr |jjsdS|jjj}t|dsdS|j}|r+t |t tfrt|SdS)amExtract Eagle3 auxiliary layer indices from speculative config. These indices specify which hidden states from the base model should be used as auxiliary inputs for the Eagle3 drafter model during speculative decoding. Returns: Tuple of layer indices if found in draft model config, None otherwise. N eagle_aux_hidden_state_layer_ids)r,rgrrrrrrm)rr layer_idss rrz1GPUModelRunner._get_eagle3_aux_layers_from_config\s' D,C,V 4+>H y"DEE 4>  $Ie}== $## #trct|dd Jdt|j}td|||jdS)Nr3z-Cannot reload weights before model is loaded.zReloading weights inplace...)r%)rr2r)rTr load_weightsrr%)rrs rreload_weightszGPUModelRunner.reload_weightstsvtWd++77 ;877((899  2333!!$.."2"2AR!SSSSSrtensorizer_configrcbtj|||jdS)N)rr%)r1 save_modelrr%)rrs rsave_tensorized_modelz$GPUModelRunner.save_tensorized_model|s> # NN  /*      rc|j}|siS|jj}i}g}|D] \}}||} | |j|} | j3t| j} tj | j |j d} ||} | s tj | dz |dz} | ||<| j}|dz}| |z }| |kr| }n|}||| ||<|dkr|jj|}|jj|}||||z}|j|}| |||z}|j|}|j|||\}}}t3|||z}| j||d| j||d| j||d |D]}||=||= |r||S)NTrrr)rjrqin_progress_prompt_logprobs_cpurrrirr>rr{rr rs empty_cpurrrrrryr3rKrUcompute_logprobsgather_logprobsslicelogprob_token_idsrzrselected_token_ranksr)rrrnum_prompt_logprobs_dictin_progress_dictrcompleted_prefill_reqsrrjrrCr!rrr| start_toknum_remaining_tokensr3rrpprompt_hidden_statesr tgt_token_idsrrr? chunk_slices rrz(GPUModelRunner._get_prompt_logprobs_dicts  $(#; ' I+KBD"$+C+I+I+K+KL L  'F'-11&99J!mF+G'/ #G$< = = $|G,DEEHH $ I    033F;; # <$3#<%)+>+B$$ ,< ( 3I!A I#4y#@ 111 ( 2 &--f555/?$V,Q &6v>G),W5::< D DD E     - ) )F(0 ((       ##rc |d|jjDSi}|d}|jjD]A}|jj|}|&||jdkrt||nd||<B|S#t$ricYSwxYw)Nci|]}|dSrr)rrs rrz6GPUModelRunner._get_nans_in_logits..sIIIfIIIrrrFr) rqrisnanrrr"rrr IndexError)rrrnum_nans_for_indexrr9s rrz"GPUModelRunner._get_nans_in_logitss ~II0@0HIIII!# !'!3!3!3!;!;!?!?!A!A!G!G!I!I *2   ,.rand_input_idss:)N&*99;;rz'Randomizing dummy input_ids for DP RankrTrc@tjjjSr)r randn_likerrrsrrand_inputs_embedszAGPUModelRunner.maybe_randomize_inputs..rand_inputs_embeds#s!'&*rz+Randomizing dummy inputs_embeds for DP Rank)r r*r rVLLM_RANDOMIZE_DP_DUMMY_INPUTS functoolscacherrrTrrzrr)rrrdp_sizerandomize_inputsrrs` rmaybe_randomize_inputsz%GPUModelRunner.maybe_randomize_inputss"2E>N7Q; # EEEEE  " _ EL     _    G H H H OONN,,-@y~~a/@/@-@APTO U U U EEE OOA     _       _  !,,,   K L L L   ""$$%<}'9'9!'<'<%<=D     EEE    " " " " "rrcmax_items_per_batchc"|jJ|j|j|di|jj}|d|d}| Jd|g|z}t dt ||j|jDS) z.Gs<  %1o       rr[) rrKget_dummy_mm_inputsr%rrRrDr r)rrcrdummy_mm_inputs dummy_mm_itemdummy_mm_itemss rrhz"GPUModelRunner._get_mm_dummy_batch1s ~)))*>>  m.&?   ( 4X>qA ((*M((('+>>  )D{?***      rr9force_attentionr skip_eplbcreate_mixed_batch remove_lora activate_lorais_graph_capturingc  |jjj} | r/| jr(t jgt jgfS||sJ|r|jn|} ||jj ksJ|jj }|r1|rJt|dz |dz}||z }|dz}dg|z|gz}|} nj|r:|rJt|t|| }| g|z}|| zdkr|| z|d<n.t||}||z}|g|z}|dxx||zz cc<t||ksJt||ksJtj|tj}t%| }tj|tj}||||| d||p|t*jk||  \}}}}}||}n||ksJd |d |d |j}|j|jn|}t3|||||jjj\}}t8d ||d} ||||| \}!}"|s|t*jkr|rdg|z|dzgz}#n| }#|#|j jd|<d|j j|d<|j !|"|\}$}|$|j#jd|dz<|j#!|t*jk}%|$||| |%r|n|| |!\} }|%|j&||| | 5||j'ksJ|(}&|j)r>|jj*s2|+|\}'}(i|&|,|}&nH|j-r+d}'|j.j/d|}(|(}&n|j0j/d|}'d}(|j1r|j2j/ddd|f})n8|j3dkr|j4j/ddd|f})n|j5j/d|})tmj7rd}*nT|j86|j9:|j'|jj;|j<|_8|=|dd}*||dj}|||dd<|>|'|(5t| |j||||||"5|j9d|'|)|*|(d|&}+dddn #1swxYwYdddn #1swxYwY|j@r|+\},}n|+},|jAr|jABs|jACrt|jEttzsJ|jAJ| o|t*jHkp| o|t*jko |jAjI }-|jJjKr| rd}-|jEL||-| |"dddn #1swxYwY|M|s|Nd|tjO|dz }.t jP|.Q|j<d}/|,|,|/fS)a Run a dummy forward pass to warm up/profile run or capture the CUDA graph for the model. Args: num_tokens: Number of tokens to run the dummy forward pass. cudagraph_runtime_mode: used to control the behavior. - if not set will determine the cudagraph mode based on using the self.cudagraph_dispatcher. - CUDAGraphMode.NONE: No cudagraph, for warm up and profile run - CUDAGraphMode.PIECEWISE: Piecewise cudagraph. - CUDAGraphMode.FULL: Full cudagraph, attention metadata is needed. force_attention: If True, always create attention metadata. Used to warm up attention backend when mode is NONE. uniform_decode: If True, the batch is a uniform decode batch. skip_eplb: If True, skip EPLB state update. is_profile: If True, this is a profile run. create_mixed_batch: If True, create a mixed batch with both decode (1 token) and prefill (multiple tokens) requests. remove_lora: If False, dummy LoRAs are not destroyed after the run activate_lora: If False, dummy_run is performed without LoRAs. Nrrrr F) rrrrrrrrrz7Cudagraph runtime mode mismatch in dummy_run. Expected z , but got rnr/r8)rrrrrr) batch_sizer!r )rrr9rrrr)use_cudagraphsrrT)rrrr)Rr r%rmm_encoder_onlyrr{valid_runtime_modesrr+rr<rrMrr>rrr~rrrrrurrrr*rGrTrUr+rHrrxrrrr maybe_dummy_run_with_lorar(r;rrOrPrrjr3rrrrLrrMrrr#rrr3make_empty_intermediate_tensorsr!r rrr)r\r,rar`rr_rrr enforce_eagerr'cudagraph_specialize_lora dummy_runrrrmrr)0rrr9rrrrrrrrrrrrnum_decode_tokensnum_prefill_tokensrnum_scheduled_tokens_listmin_tokens_per_reqrrr_cudagraph_moder4rrrbrrrrRrrVrrcum_num_tokensrTrrrrroutputsrr logit_indiceslogit_indices_devices0 rrDzGPUModelRunner._dummy_runPs L$1C  62 6<##U\"%5%55 5 # *%99;; + * <":HW55Z T2IIIII,9  C% % %%!$L1$4jAo F F !+.?!? (1,H*+.?(?CUBV(V %.MM  C) ) ))<j-)H)HIIH)6((B %M)Q..0:]0J)"-:|44H!+x!7 );(>>  7 7.!(<)6!&$7&B*m.@@ &4 -# 8   L]4H!, " )%4 " ")_<<<Q+QQ7MQQQ=<< '1#-#6#BJ   /I      , 9 / / + +  9     6: 040G0G(+ 3. 1H1 1 -   4 8JJJ! )3!226H16L5MM(*2DM YhY '*+DM XYY ' M % % ' ' ' $ ; ;

Q22M&84 >   M1 + +        r r %(;;;;;2244L& %t/@/S %+/+B+BCT+U+U( = " ++H55  * % $ 2 67I8I7I J #6688  N./A0A/AB $  C 04QQQ8J9J8J5JK %)) 15aaa9K:K9K6KL  N./A0A/AB ~~+ '+$$,4 BB'+':"&"3"9#';C-(,'O'O%tU(($$/%9$;$F!'3.?(+++I}EE  #!$0)=+A%/"6!.     %$*'')="/  #                               *0 (#* qq ' & '1133 *;;== "$, @R0RSSSSS.::: +N2m6MM /.I2m6HH "@1??*D++%*N &&#1'9"/ '[r r r r r r r r r r r r r r r z ++--- A NNDZN @ @ @ "677!; $/ >>AA Kd B  m,@AAAs\&FYU"U5 UU UU U YU YU CYY Yc \ jjj}|r|jrt jgSt j|}j|}| d fd}td$id|dddddd|d d || d d z d id dddddd|dd|dd|dddt Dddt Ddddidt} ||}n9#t$r,}dt|vrtd d||d}~wwxYwjrd t D}t#j|j} t)d!|D} d} t j| z|jd"j|j#}| | |||S)%Nrc>tjf|jS)Nr)rrKr )rrrs rrz3GPUModelRunner._dummy_sampler_run..s%*h[!DK"P"P"Pr temperatureg? all_greedyF all_randomtop_pg?top_krrmax_num_logprobs no_penaltiesTrfrequency_penaltiesg?presence_penaltiesrepetition_penaltiesrcg|]}gSrrrs rrz5GPUModelRunner._dummy_sampler_run..s:::Qb:::rspec_token_idscg|]}gSrrrs rrz5GPUModelRunner._dummy_sampler_run..s8881B888rallowed_token_ids_maskbad_words_token_idsrr out of memoryz9CUDA out of memory occurred when warming up sampler with m dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.cg|]}dgSrrrs rrz5GPUModelRunner._dummy_sampler_run..s<<)rrs rrz4GPUModelRunner._dummy_sampler_run..s(AA#SXXAAAAAArrrar)r r%rrrr{ rand_liker3rKrr}rrzrUr=rr,r make_dummyr rrandnrr!rd) rrrr dummy_tensorsdummy_metadatarrrdummy_spec_decode_metadatar draft_probsrs ` @r_dummy_sampler_runz!GPUModelRunner._dummy_sampler_runs($1C  $2 $<## # 66 **=99;;q>>PPPPP )   % c*** u u  -$$$   - A 2333  r  "T  "T !. c 2 2 2  -}S111 "/s!3!3!3 ;:%//:::: 98h8888 $(4 !#! ")***# & !\\*NN   #a&&(("////     " <E F  'FF rc |jd}|jj}t||}||z}t j||}|dxx||zz cc<t j||ksJt||ksJ||z}tj |} tj ||ftj |j } tt|} t!|} | ||j| j|} | | t-| | | g|zdt/|D}||| |j  | || S#t2$r/}d t5|vrt3d |d |d ||d}~wwxYw)Nrrr$)r)rr%c*g|]}tSr)ry)rrs rrz9GPUModelRunner._dummy_pooler_run_task..sEEEMOOEEEr)rrrpooling_states)rr rr(z9CUDA out of memory occurred when warming up pooler (task=z) with r))rr+r<rrrKrr>rrrr~r rr=rrEverifyr%rrrrxrrr=r)rrrrrrrrreq_num_tokensdummy_prompt_lensdummy_token_idsr3dummy_pooling_paramsr4r0rs r_dummy_pooler_run_taskz%GPUModelRunner._dummy_pooler_run_tasksS #(+ ,9 z<00'83"$'(4F"G"G###zH'<<###v-..*<<<<*++x7777#x/!,-DEE+ ~ &ek$+   ($..*:*:;;,$777##D.6s$#N#N AH #N#Nrc|dSrrrs rrz2GPUModelRunner._dummy_pooler_run..9s !A$r)keyr)r r%rrrr{rr=r3rrIfloatr<rrnr)rrrsupported_pooling_tasks output_sizerrmax_tasks r_dummy_pooler_runz GPUModelRunner._dummy_pooler_runs $1C  $2 $<## ##'"B"B"D"D& !*0!!! ;-.00 +  D00EEF ##N#Nf#N#N#N N NK {((**???B**=(CCCrcB|jr|jj}|"|jrtdn|j}|J|x}dkr|}|j |}td|||| ||}|j j di|}t||t|D]\}} | |jd|<||jd\} } t%jr3|jr|| } n|| } nd} |~ ~ |jt3jdS) NzCSkipping memory profiling for multimodal encoder and encoder cache.rzxEncoder cache will be initialized with a budget of %s tokens, and profiled with %s %s items of the maximum feature size.rEtmp_T)rr)rOr%rskip_mm_profilingrTrrget_encoder_budgetrgmax_items_per_batch_by_modalityrhr3rSrrr[rDr;r#r]r>rGr3rrgccollect) rrrencoder_budgetrimax_mm_items_per_batchbatched_dummy_mm_inputsdummy_encoder_outputsrrrlast_hidden_statess r profile_runzGPUModelRunner.profile_run<s  "- @);I$)D$ % !N  ,,,&/&B&B&D&DDNII&/%K%K%M%MN-6-V&.*KK0'.& /3.F.F&.//+ -HDJ,G--1--)4-+A&//D%E%E@@ 69?*:!::66-1OO  D-<- - ) ) >> & $ E// >>001CDDF  6   """ rc|jjtjkrtddSt xjdz c_tj }td}td|5t|j 5tjd}|jD]\}}|||tjtjd}dddn #1swxYwYdddn #1swxYwYtdt+tj }||z }||z } td || d z d | S) NzrSkipping CUDA graph capture. To turn on CUDA graph capture, ensure `cudagraph_mode` was not manually set to `NONE`rrc3.Ktjtj }|rtj dV|r(tjtjdSdS#|r'tjtjwwxYwr)rMrNrVLLM_ENABLE_CUDAGRAPH_GCfreezeunfreeze) should_freezes r freeze_gcz/GPUModelRunner.capture_model..freeze_gcs JLLL $ ==M   ! !KMMMJLLLLL!!=!KMMMJLLLL!s A))+BTr)batch_descriptorsr9Fz4Graph capturing finished in %.0f secs, took %.2f GiBr rr)r'rtrrurTwarningrnum_gpu_runner_capture_triggersrrrrr%r rr mem_get_inforget_capture_descs_capture_cudagraphsrrr) r start_timer[start_free_gpu_memoryr  batch_descsend_free_gpu_memoryend_time elapsed_timecuda_graph_sizes r capture_modelzGPUModelRunner.capture_model}s\  " 1]5G G G NNI   1;;q@;;&((  ! !  !$ (--- Y[[ ? ?-t{;;; ? ?$)J$;$;$=$=a$@ ! *<<>>  ((&1+7) J " " $ $ $"'*"9"9";";A">  ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?( (... $&&*, /2EE B  w '    s7EBE; EE EE EE"%E"r\c|tjkr|s Jd||sdS|dj}|tjk}t j|j|dd|}tr:t||j j d |rdnd|j  }|D]}|j}|j}|jjo-|tjko|ot%|jj|| } t)|jjD]} ||tj| | |||| |d ||jdS) Nz Invalid cudagraph runtime mode: rTF)rrrrzCapturing CUDA graphs ({}, {})decodezmixed prefill-decode)disabledesc)rrr)r9rr)r9rrr)rruruniformrHrpartialrDr&rr)use_tqdm_on_loadformatrrrr*rFrr rr'cudagraph_num_of_warmupsmaybe_remove_all_lorasr() rr\r9rrr r4rrrrbs rraz"GPUModelRunner._capture_cudagraphss #m&8 8 8&::<< 9 8 F.D F F 9 8 =!  F*1-50M4FF% O)+     ! !  $! ,==5<< .JHH4J*/!!! ,' ' J#.J&/M$2*m.@@",+;)#1 42KLL    +8+=(;"/  I'=$7+#'       ##D$455555rrc tjdks JdGddt dtdtt t tftttff fd }dt t tfd tdt tfd }g}g}|j D]C}||}||d||d D||j t!jt%|D])\}} j|| |*d S) zT Initialize the attention backends and attention metadata builders. rz*Attention backends are already initializedc0eZdZUeeed<eed<dS)AGPUModelRunner.initialize_attn_backend..AttentionGroupKey attn_backendrN)rrrrrUrrirrrAttentionGroupKeyrvs0/0 0 0 0& & & & & &rrxkv_cache_group_specrc tttt}t j||j}i tt}|jD]}|| }| j vrtd|}| }|j }t|tr |j|}||f} || |<||| fd|Dt'd DfS)N FastPrefillc(i|]\}}||Srr)rrr attn_backendss rrz_GPUModelRunner.initialize_attn_backend..get_attn_backends_for_group..<s$MMMAq!1MMMrc3$K|] }|jV dSr)rw)r group_keys rrz^GPUModelRunner.initialize_attn_backend..get_attn_backends_for_group..=s%SSyI*SSSSSSr)rrr r-rr rrrget_attn_backendrr] full_cls_namerrrlrrrrr) ry layer_typelayersattn_backend_layersrrwrlayer_kv_cache_specrBr}rxrs @rget_attn_backends_for_groupzKGPUModelRunner.initialize_attn_backend..get_attn_backends_for_groupsrd3i);<.create_attn_groups@sh13K>O>U>U>W>W / /:-}{+ !%  "":.... rrN)r>rZr rhrmrrrrrrUrrrr _check_and_update_cudagraph_moderr r) rrrrattention_backend_mapsattention_backend_listryr}rattn_backend_maprxs ` @rinitialize_attn_backendz&GPUModelRunner.initialize_attn_backends4#$$)))+W))) ' ' ' ' '  ' ' '# !1#  4)4945s4@P;Q7RR S# # # # # # # J #$5tCy$@A " . !     "$!##2#B < < 778KLLM " ) )-*: ; ; ; " ) )-*: ; ; ; ; -- "O$C   ))9:::#,-C#D#D M M A   # #$6$67G$K$K L L L L M Mrrc Btt|jD]h}|j|D]X}||j|j|t|kr||nd|jjsdn |jj Yi| dS)zW Create the metadata builders for all KV cache groups and attn groups. Nr)num_metadata_builders) rr>rrZcreate_metadata_buildersr r r*rFrG!calculate_reorder_batch_threshold)rrrrrs rinitialize_metadata_buildersz+GPUModelRunner.initialize_metadata_buildersbs "'s?+J'K'K!L!L   "./@A   33$K(3/A+B+BBB''899/=+;!!-:4      ..00000rattention_backendsrc tj}d}t||D]W\}}|D]O}|}||j|j} | j|jkr | }|j}PX|j j } | J| tj kr|tjkrd| jd|d|d} |tjkr| dz } t!| |j r| dz } tjx} |j _ n| dz } tjx} |j _ t(| | tj kr|tjkrd| jd|d|d} |j jt0jkrC|j s |j jr| d z } tjx} |j _ n| d z } tjx} |j _ t(| | tj kr|jd kr|jtjjkrd| jd |d |d} |j r| dz } tjx} |j _ n| dz } tjx} |j _ t(| | r.|tjkrt!d| jd|d|d| tj kr`| rL|jd krA|j !|j|j"j#|j j$} | | ng|_%| |j _ |j&'| |j|j(rQ|j()r:tU|j+tXsJ|j+'| dSdSdS)z Resolve the cudagraph_mode when there are multiple attention groups with potential conflicting CUDA graph support. Then initialize the cudagraph_dispatcher based on the resolved cudagraph_mode. NzCUDAGraphMode.z is not supported with z backend (support: rzU; please try cudagraph_mode=PIECEWISE, and make sure compilation mode is VLLM_COMPILEz+; setting cudagraph_mode=FULL_AND_PIECEWISEz); setting cudagraph_mode=FULL_DECODE_ONLYzJ; setting cudagraph_mode=PIECEWISE because attention is compiled piecewisezI; setting cudagraph_mode=NONE because attention is not compiled piecewiserz9 is not supported with spec-decode for attention backend z (support: z"; setting cudagraph_mode=PIECEWISEz; setting cudagraph_mode=NONEz backend (support:zW) ; please try cudagraph_mode=PIECEWISE, and make sure compilation mode is VLLM_COMPILE)-rVALWAYSrget_builder_clsget_cudagraph_supportr rr rr'rt mixed_moderrHrNEVERrcsplitting_ops_contain_attentionFULL_AND_PIECEWISEFULL_DECODE_ONLYrTr] decode_moderr VLLM_COMPILEuse_inductor_graph_partitionrrur UNIFORM_BATCHrseparate_routine&adjust_cudagraph_sizes_for_spec_decoder*rrsrwrinitialize_cudagraph_keysr,rarr_r) rrrmin_cg_supportmin_cg_backend_nameattn_backend_setrrw builder_cls cg_supportrtr capture_sizess rrz/GPUModelRunner._check_and_update_cudagraph_modeysU,2"03 1 1 @ @ , n!1 @ @ *::<< (>>$n&B #n&:::%/N*6*?' @0?)))  % % ' '=+= = ="4";;;%!4%%+%%!%%%  !3!999A!oo%&FFHH DD!4!8!G!GBB!2!8!G NN3     & & ( (M,> > >"4":::%!4%%+%%!%%%  &+/KKK'GGIIL*GL6 "+!8!G!G: "&!8!G NN3     & & ( (M,> > >-11$'9'G'MMME!4EE&EE3AEEE  &FFHH ;;!+!8!G!G66!&!8!G NN3     . . 0 0 "4":::A!4AA"5AA)AAA   & & ( (M,> > >//11 ?-11  # J J-t/C/X   !3KM!.!:   & 2@. !;; D9    " Ct'>'H'H'J'J CdlM:: : :: L 2 2> B B B B B C C C Crcd}d|D}t|dkr d|_dSt|||_dS)a Choose the minimum reorder batch threshold from all attention groups. Backends should be able to support lower threshold then what they request just may have a performance penalty due to that backend treating decodes as prefills. c2||n||nt||Sr)r)abs rrzBGPUModelRunner.calculate_reorder_batch_threshold.. s !)QQaiSQRTUYYrc@g|]}|jSr)rrrgroups rrzDGPUModelRunner.calculate_reorder_batch_threshold.."s76 6 6   & & ( ( @6 6 6 rrN)_attn_group_iteratorr>rr )r min_none_highreorder_batch_thresholdss rrz0GPUModelRunner.calculate_reorder_batch_thresholdstWV 6 6 22446 6 6  ' ( (A - -+/D ( F'-m=U'V'V$$$rkv_manager_block_sizerZcHdtttdtdtfd}d|D}|||r|St d|D}t |dD]}||zd kr |||r|cStd |d ) a Select a block size that is supported by all backends and is a factor of kv_manager_block_size. If kv_manager_block_size is supported by all backends, return it directly. Otherwise, return the max supported size. Args: kv_manager_block_size: Block size of KV cache attn_groups: List of attention groups Returns: The selected block size Raises: ValueError: If no valid block size found backendsrprc|D]u}d}|D]W}t|tr ||krd} t|tr||jzdkrd}Ft d||sdSvdS)zO Check if the block size is supported by all backends. FTrzUnknown supported size: ) get_supported_kernel_block_sizesrrr[baserc)rrpr2 is_supportedsupported_sizes rblock_size_is_supportedzHGPUModelRunner.select_common_block_size..block_size_is_supportedCs $ ! !$ &-&N&N&P&PVVN!.#66V%77+/L#NJ??V%(;;q@@+/L()TN)T)TUUU#! 55!4rcg|] }|j Sr)r2rs rrz;GPUModelRunner.select_common_block_size..Xs;;;eEM;;;rc3rK|]2}|D]}t|t|V3dSr)rrr)rr2rs rrz:GPUModelRunner.select_common_block_size..gsi& & ")"J"J"L"L& & .#.. & & & & & & & & rT)reverserzNo common block size for z. )rrrUrrrrvrc)rrZrrall_int_supported_sizesrs rselect_common_block_sizez'GPUModelRunner.select_common_block_size-s , 4 012 @C      *<;{;;; # "8-B C C )( (#&& & #& & & # # %%.sB   n:N)rrnrrRrrrrerMrrjr&enable_prefix_cachingnum_speculative_blocksrrpr/rrr;r rr%rorr r,rqrrr>) rrrrmax_num_blocksrrrrmamba_blocks_per_reqs rmay_reinitialize_input_batchz+GPUModelRunner.may_reinitialize_input_batchus  "1"A   D.0DEE !*?+J!K!K : : A~.68PQQ %){1~0G0I0II&& ".6 BB (>**"0G (H$ *-*,@**&  ! !"8 9 9 9 9 4,78 8 8????M955 #))-*BCCCC)K^-IKK"!rrc <i}d\}}|D]}|j}|j} |jt |kr*||j} |jD]} | |jvr || } | |jzdksJ| |jz} t|tr.?s9++./q)++++++rc:g|]}|Sr)ru)rrkv_cache_stride_orders rrz.Cs7!!!.33A66!!!rrN)rstridestorage_offset)$rrr2rr>rrrpage_size_bytesrrbrpget_kv_cache_shaper head_sizer&r0r!get_kv_cache_stride_orderAttributeErrorrrmrviewpermuterjrshapesdtypesrSrrr as_stridedr%_update_hybrid_attention_mamba_layout)rrrrrWhas_attn has_mambarrrwkernel_block_sizer raw_tensor num_blocksnum_blocks_per_kv_blockkernel_num_blocksr! inv_order state_tensorsstorage_offset_bytesr dtype_sizenum_element_per_page target_shaper target_strider{rrs @@r_reshape_kv_cache_tensorsz(GPUModelRunner._reshape_kv_cache_tensorss$.0 *)<<>>M .M .E!/M =L&#.@*A*AAA 253J K #/F .F . !===1*= !''))M,IIQNNNN'--//=3PP m];;@.#H%04EE,)35L(L%%1%D%D))%2%/(,(9(E &E&&N*/ER0<0V0V0X0X-"#899S=P=PPPPPP*,?@RRR05eC>  E!/M#/   $Z0m];; q@QUV@V@V#>!,111>-5^>>211 #+."4":":"<"t|j ttzsJ|j |t#rnt%}|j*|jJ||j|jn|||t0|jjr|dSdS)z Initialize KV cache based on `kv_cache_config`. Args: kv_cache_config: Configuration for the KV cache, including the KV cache size of each layer N)rr.may_add_encoder_only_layers_to_kv_cache_configrrrrrr r,rar`rr_rrvalidate_same_kv_cache_groupr rrXrYregister_cross_layers_kv_cacheregister_kv_cachesset_host_xfer_buffer_opsr!r%rinit_routed_experts_capturer)rrrrWkv_transfer_groups rinitialize_kv_cachez"GPUModelRunner.initialize_kv_caches#?33. ;;=== ;;OLLL $$_555 "==oNN ))/;MNNN ))/;MNNN44 /    " G  # - - / / G&7799 GdlMrr init_bufferr+rr )rrouted_experts_capturerrps rrz+GPUModelRunner.init_routed_experts_capturers T   :   #8">"@"@&1  +s43G3W/X/X X  " ++#'#8#O"4( ,     rc~|jjj}tt}t |jt }|D]q\}}|jtj krWt||j |j |j}||||j|rt%|dkrdt%|dks Jd|\}}|jjt-||dSdS)zA Add encoder-only layers to the KV cache config. )rprrr!rrz0Only support one encoder-only attention spec now)rrN)r r&rprrrrr attn_typerY ENCODER_ONLYrerrr1rrrr>popitemrrrh) rrpencoder_only_attn_specsrr attn_module attn_specr7rs rrz=GPUModelRunner.may_add_encoder_only_layers_to_kv_cache_configsP%2= BMdBSBS1$2BINN '2'8'8':': = = #J $ (BBB+C)!,!9)3- ,,, ( 299*EEE,00<<< & ' '! + +.//1444B544!8 ? ? A A D+  0 7 7 [MMM      , +rctrtjriSi}ttt t }t|j|}| D]O\}}t|tr|j x}r ||j |<.||jx}r|||<P|S)a0 Generates the KVCacheSpec by parsing the kv cache format from each Attention module in the static forward context. Returns: KVCacheSpec: A dictionary mapping layer names to their KV cache format. Layers that do not need KV cache are not included. )rrrCrrr r-rr rrrkv_sharing_target_layer_namerget_kv_cache_spec)rrrrrr# kv_tgt_layerr7s rr'z GPUModelRunner.get_kv_cache_spec9s    !2!2!> I02 $s)%788 1$2BJOO '2'8'8':': 1 1 #J +y11  + HH  ;G+J7"44T5EFFFt 1,0 j)rc|jd|jd}||d|j|j|Srv)rrrzrrrr)rrpinneds rrzGPUModelRunner._to_listYso23O5F5LQ5O3OP &T ::: ""$$$ '')))}}rc|j5d|jD}|j|cdddS#1swxYwYdS)z Get encoder timing stats for all requests and clear the registry. Returns: Dictionary mapping request_id to stats dict. c>i|]\}}||Sr)to_dict)rr stats_objs rrz;GPUModelRunner.get_encoder_timing_stats..ps:%FI ))++rN)r|ryrr)rstatss rget_encoder_timing_statsz'GPUModelRunner.get_encoder_timing_statshs &  )-)E)K)K)M)ME  ( . . 0 0 0                   s>AAArWgroup_lora_refsrbrdc#K|sdVdS||||z}d|D}tjtj} dVtjtj|z }|t t |dz } |j5|D]N} | |jvrt|j| <|j| } | xj | z c_ | xj dz c_ O ddddS#1swxYwYdS#tjtj|z }|t t |dz } |j5|D]N} | |jvrt|j| <|j| } | xj | z c_ | xj dz c_ O dddw#1swxYwYwxYw)a@ Context manager to time encoder forward operations. Args: should_time: Whether timing is enabled group_lora_refs: Full list of (request_id, pos_info) tuples current_item_idx: Starting index for this group num_items: Number of items in this group Nch|]\}}|Srr)rrrbs r z9GPUModelRunner.timed_encoder_operation..s@@@ V@@@rr) rrrrrrnr>r|ryEncoderTimingStatsencoder_forward_timenum_encoder_calls) rrWr1rbrd group_refsgroup_request_idsrbelapsedper_request_timerr/s rrQz&GPUModelRunner.timed_encoder_operationws"  EEE F$%58H98T%TU @@Z@@@    &((  1 EEE J " " $ $ $'))J6G&S1B-C-CQ)G)GG * 1 1/11FT%AAA?Q?S?S4V< 8@E..2BB..++q0+++ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 J " " $ $ $'))J6G&S1B-C-CQ)G)GG * 1 1/11FT%AAA?Q?S?S4V< 8@E..2BB..++q0+++ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1sED"5ADDD"AG)>AG G)G! !G)$G! %G))rN)rrrNr) NNNNFFNNN)rrrr)NNNN)TFNNr)F)rrrN) NFFTFFFTFF)rrrrrr rrrrinference_moderr rSymIntr!rrr}rrrxrrDrrQrr`rr rrrArdrjrndarrayrmrrrrrirrrrrrrZr rrXrrrrrrBrCr=rkrnnModulerrHrrIrrJrrGrrrqrtrnrrrrprrvrrrrsrrrr staticmethodrrr(rrrr+r[rrrortr`rrgrr}r_rrrrrrrrrhrDr3rur<rGrTrirargrrrrrUrhrrrrrrrrrrrrr rrrrr'rrCr0rQrrrrrCsy5y5 y5y5y5y5v J#J$JJJJ))))U)5)5)5)5V 2 2 2 2 2LP    5<'  05   DH          D4DDDD !!!!o,o,o,o,b. % .@Q. ....`  )7      D  /A       0B     "+" """"0BB1DBBBB)-%%J%ho% rz2:% & %%%%(w +w %(w z w  w w w w r 6"38n 6# 6 6 u|d"BJ$55 6 6 6 6 6DY +Y !jY   T! #  Y Y Y Y @)-&*-1.2 %&+6:;?8<q?q?q?q? q? : q? t q?$d*q? t+q?q? $q?#38nt3q?#'tCy/D"8q?C-.5q? #%    U=#+=%>  2D8        >  $ D T    :#|:#":#"$ :# /5 :# 3 3 :#:#:#:#x$#$#$$$$ ( ( u|d"EL0 1 ( ( ( (<@ r r +r r 2D8 r  t t d" S#XD  "  r r r r h t#147  <} +} &}  t# } | } " } 147}   S#X T#Y S/D( () S  S#X S   } } } } ~ / /^ / *.)-;?-1   <$& <$& 2D8  |d*  S#X       @ -1   "% "%    #Tk      \ 6%)!-1&* !w w w w "$ w #& w  w "w w #Tkw t w w    t   w w w w r"<"<"<"U8< %$$( #( ##(DBDBDB!. 4DB DB  DB " DBDBDB!DBDBDB!DB u|U\) *DBDBDBDBL UL|L LLLL\5|55  5555nUD|D DDDD????#H^4L#### )"=)"TRUY)")")")"Vf&f#3 #45f!I f c5< ffffPc5</0 65,5BFs)5 c5< 5555n, 810=10T10101010f   "44[(8#9@ %, 4S ?     $sDeck9I4J/J*K    )1)1eCHo.)1 )1  )1)1)1^)1)1)1rrcVeZdZUdZdZeed< dZeed< de e eezffdZ dS) r5z7Per-request timing statistics for encoder forward pass.gr6rr7rc |j|jdS)Nr6r7rDrs rr-zEncoderTimingStats.to_dicts$($=!%!7   rN) rrrrr6rCrr7rrrr-rrrr5r5smAA"%%%%%>s> c53;./      rr5(rrMrrzr collectionsrcollections.abcrr contextlibrrr dataclassesr r typingr r r rrr"rrtorch.distributedtorch.nnr?r vllm.envsrvllm.attention.layerrrvllm.compilation.counterrvllm.compilation.cuda_graphrrvllm.compilation.monitorr vllm.configrrrrrvllm.distributed.ec_transferrr vllm.distributed.eplb.eplb_statervllm.distributed.kv_transferrr /vllm.distributed.kv_transfer.kv_connector.utilsr!vllm.distributed.parallel_stater"r#r$r%r&r'vllm.forward_contextr(r) vllm.loggerr*vllm.lora.layersr+r,/vllm.model_executor.layers.attention_layer_baser-r?vllm.multimodalr@vllm.multimodal.inputsrArBrCvllm.multimodal.utilsrDvllm.pooling_paramsrEvllm.sampling_paramsrF vllm.sequencerG vllm.tasksrHrIrJ vllm.utilsrKvllm.utils.jsontreerLvllm.utils.math_utilsrMrNvllm.utils.mem_utilsrOrPvllm.utils.nvtx_pytorch_hooksrQvllm.utils.platform_utilsrRvllm.utils.torch_utilsrSrTvllm.v1.attention.backendrUrVrWrXrYrZr[#vllm.v1.attention.backends.gdn_attnr\ vllm.v1.attention.backends.utilsr]r^r_vllm.v1.core.sched.outputr`vllm.v1.cudagraph_dispatcherravllm.v1.kv_cache_interfacerbrcrdrerfrgrhrirjrkrlvllm.v1.outputsrmrnrorprqrrrsrtrurvrwvllm.v1.pool.metadatarxryvllm.v1.sample.logits_processorrzr{)vllm.v1.sample.logits_processor.interfacer|vllm.v1.sample.metadatar} vllm.v1.sample.rejection_samplerr~vllm.v1.sample.samplerrvllm.v1.spec_decode.draft_modelrvllm.v1.spec_decode.eaglervllm.v1.spec_decode.medusarvllm.v1.spec_decode.metadatar"vllm.v1.spec_decode.ngram_proposerr#vllm.v1.spec_decode.suffix_decodingrvllm.v1.structured_output.utilsr vllm.v1.utilsrrvllm.v1.workerrvllm.v1.worker.cp_utilsrrvllm.v1.worker.dp_utilsr.vllm.v1.worker.ec_connector_model_runner_mixinrvllm.v1.worker.gpu_input_batchrr!vllm.v1.worker.gpu_ubatch_wrapperr.vllm.v1.worker.kv_connector_model_runner_mixinr&vllm.v1.worker.lora_model_runner_mixinrvllm.v1.worker.ubatch_utilsrrrrvllm.v1.worker.utilsrvllm.v1.worker.workspacerutilsrrrrr+vllm.model_executor.model_loader.tensorizerrrrrrTrrrrrrrrrrr5rrrrs   ######........%%%%%%!!!!!!BBBBBBBBBBBBBB 88888888888888GGGGGGGGDDDDDDJIIIIIII666666UUUUUUUUJJJJJJ$#####99999999NNNNNNPOOOOOOO                         0///// >=====------------------AAAAAAAAAA======//////00000000AAAAAAAA222222======LKKKKK 544444<<<<<<                                                    A@@@@@@@OOOOOOOOEEEEEE444444======******>>>>>>333333555555;;;;;;<<<<<<FFFFFFAAAAAAFFFFFFFF&&&&&&?>>>>>VVVVVVIIIIIIII;;;;;;VVVVVVGGGGGG >=====333333ILLLLLLHHHHHHHH X  "3(9#9:):::"&'7"8;K"KiKKKAAAAA 6AAAH')')')')')'=')')')T R R R R R R R R ^Y1^Y1^Y1^Y1^Y157R^Y1^Y1^Y1Bs              r