.`iz UddlZddlmZmZmZddlmZddlmZm Z ddl m Z m Z m Z mZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lm Z dd l!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1m2Z2ee3Z4ee5e5dzfZ6 eGddZ7GddZ8dddddede5dedzde9e5dzdej:f d Z;ed!ej<dej<fd"Z=ed!e9ej<de9ej<fd#Z=ed!e9ej<ej<zd$e d%dej<fd&Z=ed'd(d!e9ej<ej<zd$e>de9ej<ej<zfd)Z=d'd(d!e9ej<ej<zd$e>de9ej<ej<zfd*Z=d+e(dej<fd,Z?d+e(de5fd-Z@d.ej<d/eAde9e9eAfd0ZBd1ej<d2e(d3ej<dej<fd4ZCd5ej<d6e9eAdej<fd7ZDGd8d9ej:ZEedd:d;ej:dZHedd:d;ej:d?eej:gej:fdfdOZWdPe9e5dQeAfdRZXde5dNe5de5fdSZYdede dzfdTZZdidVe5dWeAdeAfdXZ[ djdZej<d[e\dej<fd\Z]d]ej<d^eAd_eAdeGej<ej<ffd`Z^d!ej<dej<fdaZ_d!ej<dej<fdbZ`d!ej<dej<fdcZae1dde`eaejbjcfedLej:dNe5ddfdfZddgeAdHeAdeAfdhZedS)kN)CallableIterableMapping)contextmanager) dataclassfield)AnyLiteralProtocoloverload)functional_call)(register_module_module_registration_hook)PretrainedConfig) VllmConfig)get_tensor_model_parallel_rank$get_tensor_model_parallel_world_size) init_logger)QuantizationConfig).support_quantized_model_reload_from_hp_weights)default_weight_loader)supports_any_eagle) NestedTensorsIntermediateTensors)cdiv)is_pin_memory_availableis_uva_available)direct_register_custom_opget_cuda_view_from_cpu_tensorcLeZdZUdZeeZeed<eeZ eed<eeZ eed<ddZ d e de d zfd Z d eee ejfdeee ejffd Zdee dee fdZdee efdee effdZd S) WeightsMapperzBMaps the name of each weight if they match the following patterns.)default_factoryorig_to_new_substrorig_to_new_prefixorig_to_new_suffixotherreturnczti|j|ji|j|ji|j|jS)z7Combine two `WeightsMapper`s by merging their mappings.)r#r$r%)r!r#r$r%)selfr&s t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/utils.py__or__zWeightsMapper.__or__8sRV$"9VU=UVV$"9VU=UVV$"9VU=UV    keyNc|jD]%\}}||vr|dS|||d}&|jD]6\}}||r|dS|||d}7|jD]H\}}||r.|dS|||d}I|S)N) r#itemsreplacer$ startswithr%endswithjoinrsplit)r)r-substrnew_keyprefixsuffixs r* _map_namezWeightsMapper._map_name@s#6<<>> 6 6OFG}}?44kk&'155#6<<>> 6 6OFG~~f%% 6?44kk&'155#6<<>> : :OFG||F## :?44ll3::fa#8#899 r,weightsc$fd|DS)Nc3TK|]"\}}|x|fV#dSNr:).0namedataout_namer)s r* z&WeightsMapper.apply..[sO  d NN4000=t ====  r,)r)r;rCs` @r*applyzWeightsMapper.applyXs4     %    r,valuesc$fd|DS)NcBg|]}|xSr>r?)r@rArCr)s r* z,WeightsMapper.apply_list..bs:    NN4000= ===r,rEr)rGrCs` @r* apply_listzWeightsMapper.apply_listas4         r,cHfd|DS)NcJi|]\}}|x| Sr>r?)r@rAvaluerCr)s r* z,WeightsMapper.apply_dict..is@   e NN4000= e===r,)r0rKs` @r* apply_dictzWeightsMapper.apply_dicths<     %||~~    r,)r&r!r'r!)__name__ __module__ __qualname____doc__rdictr#WeightsMapping__annotations__r$r%r+strr:rtupletorchTensorrFlistrLr rQrEr,r*r!r!0sRLL).t)D)D)DDDD).t)D)D)DDDD).t)D)D)DDDD    SS4Z0 c5<&7 89 %U\)* +     c tCy     c3h DcN      r,r!c eZdZdZgdZddddddejdeedzdeedzdeedzd eedzd df fd Z d e e ee j fd e e ee e ee j fffd Zdeded efdZded efdZded efdZdedejd e e ee j fd e efdZdejdeee j ffdZdedejd e e ee j fd e efdZeddd e e ee j fdedzd eefdZxZS)AutoWeightsLoadera" Helper class to load weights into a [`torch.nn.Module`][]. It is able to automatically detect child modules and parameters while iterating over the weights only once. The weight loading logic for individual modules can be overridden by defining a `load_weights` method. Similarly, the weight loading logic for individual parameters can be overridden by defining a `weight_loader` method. Detailed weight loading information can be viewed by setting the environment variable `VLLM_LOGGING_LEVEL=DEBUG`. )zrotary_pos_emb.inv_freqzrotary_emb.inv_freqzrotary_emb.cos_cachedzrotary_emb.sin_cachedN) skip_prefixes skip_substrsignore_unexpected_prefixesignore_unexpected_suffixesmoduler`rarbrcr'ct||_|pg|_|pg|_|pg|_|pg|_|xj|jz c_dSr>)super__init__rdr`rarbrcROTARY_EMBEDS_UNUSED_WEIGHTS)r)rdr`rarbrc __class__s r*rgzAutoWeightsLoader.__init__ss  *0b(.B*D*J'*D*J' T>>r,r;c#|Kd|D}tj|dD]\}}|d|DfVdS)Nc3LK|]\}}|dd|fV dS).r/N)split)r@ weight_name weight_datas r*rDz4AutoWeightsLoader._groupby_prefix..sO  ( [  sA & & 4      r,c|ddSNrrExs r*z3AutoWeightsLoader._groupby_prefix..sqQRtTUwr,)r-c3ZK|]&\}}t|dkrdn|d|fV'dS)r/N)len)r@parts weights_datas r*rDz4AutoWeightsLoader._groupby_prefix..sU+|u::??RRa,Gr,) itertoolsgroupby)r)r;weights_by_partsr8groups r*_groupby_prefixz!AutoWeightsLoader._groupby_prefixs  ,3    './?EVEVWWW  MFE/4       r,r8restcP|dkr|S|dkr|Sd||fS)Nrvrl)r4)r)r8rs r* _get_qualnamezAutoWeightsLoader._get_qualnames4 R<<K 2::Mxx'''r,qualnamectfd|jDptfd|jDS)Nc3BK|]}|VdSr>r2r@prs r*rDz.AutoWeightsLoader._can_skip..s1FFa8&&q))FFFFFFr,c3 K|]}|vV dSr>rE)r@r6rs r*rDz.AutoWeightsLoader._can_skip..sAN N #)Fh N N N N N N r,)anyr`ra)r)rs `r* _can_skipzAutoWeightsLoader._can_skipsnFFFF43EFFFFF #N N N N -1->N N N K K  r,cfd|jD}fd|jD}t|pt|S)Nc3BK|]}|VdSr>rrs r*rDz;AutoWeightsLoader._can_ignore_unexpected..s1OO!x""1%%OOOOOOr,c3BK|]}|VdSr>)r3)r@srs r*rDz;AutoWeightsLoader._can_ignore_unexpected..s1MMx  ##MMMMMMr,)rbrcr)r)riupiuss ` r*_can_ignore_unexpectedz(AutoWeightsLoader._can_ignore_unexpectedsROOOOt/NOOOMMMMT-LMMM3xx#3s88#r, base_prefixparamc#K|D]\}}|||}||rtd|L|dkrF||rtd|t d|d|t |dt}|||td||j|VdS)NzSkipping weight %srvzIgnoring weight %sz Attempted to load nested weight z into a single parameter weight_loaderzLoaded weight %s with shape %s) rrloggerdebugr ValueErrorgetattrrshape)r)rrr;rnroweight_qualnamers r* _load_paramzAutoWeightsLoader._load_params )0 " " $K"00kJJO~~o..  1?CCCb  ..??LL!5GGG ???/:?? $E?)r)rsrr)s r*rtz0AutoWeightsLoader._load_module..s$"4"4[!"D"Dr,F)recurserlzSkipping module %szSkipping param %szSkipping missing %szIgnoring missing %sc h|] \}}|z SrErE)r@k_rs r* z1AutoWeightsLoader._load_module..=s.###(,1K!O###r,Tz&There is no module or parameter named z in z(. The available parameters belonging to z (z) are: )rStageMissingLayerPPMissingLayerrdrcallablerwarningmaprVnamed_childrennamed_parametersrr~rrr _load_modulerr _get_namer)r)rrdr;module_load_weights loaded_params child_modulesr child_prefix child_weightsr8can_skip_modulecan_skip_paramcan_ignore_modulecan_ignore_paramdesc_param_keysmsgs`` r*rzAutoWeightsLoader._load_modules f0.A B B  F T[ ")&.$"G"G +,,  3 3G < <  (NNKV #DDDDD%   V224455 F33E3BBCC  ,,V\BBB+/+?+?+H+H- &- & 'L-'' \BBF},,>>&3,//LL!5v>>>,,M,7-->>&))LL!4f===++L6 #'..#">">!%!7!7"nLL!6???$($?$? $M$M!#'#>#>v#F#F $(8LL!6???####060G0GPT0G0U0U###EVEE+//11EE=HEE((**EE4CEE !oo%[- &- &r,)mapperrc|||}fd|D}tdj|}|S)Nc3PK|] \}}|||fV!dSr>)r)r@rAweightr)s r*rDz1AutoWeightsLoader.load_weights..RsP  +tVdnnT>R>R 6N      r,rv)rFsetrrd)r)r;rautoloaded_weightss` r*rzAutoWeightsLoader.load_weightsHsn  ll7++G    /6   !!2!22t{G!L!LMM!!r,)rRrSrTrUrhrModuler]rYrgrrZr[r\r~rboolrr ParameterrrVrrrr!rr __classcell__ris@r*r_r_ps  "$$$ +/)-7;7;??? ?Cy4' ? 3i$& ? %)I$4 ?%)I$4? ??????&%U\ 123 %XeC,=&>??@ A((C(s(s(((( # $    $s$t$$$$ ""|"%U\ 123 " # """"@GiG/3C4E/FGGGG.M&M& M&%U\ 123 M& # M&M&M&M&^4 (, """%U\ 123"$ " S """43"""""r,r_rv)r8 hf_config architectures vllm_configr8rrr'czddlm}|| |jj}||||}|||S)z Helper function to initialize an inner model registered to vLLM, based on the arguments passed to the outer vLLM model. r)initialize_modelN)r)rr8)&vllm.model_executor.model_loader.utilsr model_configrwith_hf_config)rr8rrrs r*init_vllm_registered_modelrZsaHGGGGG]6,6 !00-0XX   F C C CCr,rscdSr>rErrs r* flatten_bnrqs14r,cdSr>rErrs r*rrus=@Sr,concatTcdSr>rErsrs r*rrys 3r,F)rcdSr>rErs r*rrs ),r,ct|tjr|ddS|rtj|Sd|DS)z Flatten the `B` and `N` dimensions of batched multimodal inputs. The input tensor should have shape `(B, N, ...)`. rr/cg|] }|D]}| SrErE)r@x_bx_ns r*rJzflatten_bn..s% , , ,C , ,C , , , ,r,)rr[r\flattencatrs r*rrsT!U\""yyA y|| , ,1 , , ,,r, embeddingsct|tjr|ddStjt d|DS)z` Recursively flattens and concatenates NestedTensors on all but the last dimension. rc34K|]}t|VdSr>)_flatten_embeddings)r@ts r*rDz&_flatten_embeddings..s+FFa.q11FFFFFFr,)rr[r\rrrZrs r*rrsU *el++)!!!R((( 9UFF:FFFFF G GGr,ct|tjr,dd|jddDSdd|DS)ze Constructs a debugging representation of the number of embeddings in the NestedTensors. z x c,g|]}t|SrE)rY)r@dims r*rJz/_embedding_count_expression..sEEE3s88EEEr,Nz + c34K|]}t|VdSr>)_embedding_count_expression)r@inners r*rDz._embedding_count_expression..s+QQU1%88QQQQQQr,)rr[r\r4rrs r*rrsg *el++GzzEEz/?/DEEEFFF ::QQjQQQ Q QQr,lstintervalcdtt||zdzD}|D]"}||z}|||#|S)Ncg|]}gSrErEr@rs r*rJz*split_list_into_ranges..sMMMarMMMr,r/)rangemaxappend)rrrangesnumindexs r*split_list_into_rangesrseMM5#c((h2F!1K+L+LMMMF""xu S!!!! Mr, inputs_embedsmultimodal_embeddings is_multimodalc t|dkr|St|}|j} ||d||n#t $ry}t|}|}||kr)t|}td|d|d|d|td|d }~wwxYw|S) z Merge `multimodal_embeddings` into `inputs_embeds` by overwriting the positions in `inputs_embeds` corresponding to placeholder tokens in `input_ids`. Note: This updates `inputs_embeds` in place. rr)dtypezAttempted to assign z = z multimodal tokens to z placeholdersz%Error during masked scatter operationN) rwrr masked_scatter_ unsqueezeto RuntimeErrorsumitemrr) rrrmm_embeds_flat input_dtypeenum_actual_tokensnum_expected_tokensexprs r*_merge_multimodal_embeddingsrsR !!Q&&()>??N%KI %%  # #B ' '):):):)M)M      I I I//+//116688  3 3 3./DEEDKtKK0AKK(;KKK  @AAqH I s=A++ C.5A4C))C.elementstest_elements_listctj|t|jd}tj||S)N) pin_memoryT)device non_blocking)r[tensorrr risin)rr test_elementss r* isin_listr!sRL*,, bdb33 :h . ..r,c\eZdZd dedejdzddffd ZdefdZdZdefd Z xZ S) rN stage_namerdr'cjt||_||jd<dSNrd)rfrgr#__dict__)r)r#rdris r*rgzStageMissingLayer.__init__s4 $#) hr,rAc8t|jd|Sr%)rr&)r)rAs r* __getattr__zStageMissingLayer.__getattr__st}X.555r,c&t|d)Nz should not be called)rr)argskwargss r*__call__zStageMissingLayer.__call__sd999:::r,cd|jS)Nz stage_name=)r#)r)s r* extra_reprzStageMissingLayer.extra_reprs0T_000r,r>) rRrSrTrYrrrgr(r-r/rrs@r*rrs))3) D0@)D))))))66666;;;1C11111111r,r)targetsrdr0.c#rKtt|Tdtjdtdtjffd }t |5VddddS#1swxYwYdSVD]*\}}t ||r|+dS)aA Within this context, collect all direct child assignments to `module`, returning a list of children names that is internally updated until the context is exited. If `targets` is set, instead collect descendents of `module` that are an instance of `targets`, even if they aren't direct children. Nmodule_rA submodulec>|ur|dSdSr>)r)r2rAr3children_namesrds r*hookzcollect_children..hooks0&  %%d+++++! r,)r]rYrrr named_modulesrr)rdr0r6rAr2r5s` @r*collect_childrenr8 s6#Y[[N ,") ,3 ,29 , , , , , , ,6d ; ; ! !  ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !#1133 , ,MD''7++ ,%%d+++ , ,sA''A+.A+ placeholderc#Kdtjdtdtjffd }t|5t jd5dVdddn #1swxYwYddddS#1swxYwYdSdtjdtdtjffd }t|5dVddddS#1swxYwYdS)al Within this context, prevent weight initialization from using device memory and replace direct child assignments to `module` with the result of `placeholder()`. If `targets` is set, instead prevent weight initialization and replace assignments where the child is an instance of `targets`, even if they aren't direct children of `module`. Nr2rAr3c&|ur |S|Sr>rE)r2rAr3rdr9s r*r6zno_init_weights..hook;s$&  "{9--- r,metact|r|dt|r |d|S|S)Nr<)rr )r2rAr3r9r0s r*r6zno_init_weights..hookEsa'7++ % V$$$)W-- . V$$$"{9--- r,)rrrYrr[r)rdr9r0r6s``` r*no_init_weightsr>*s ") 3 29        6d ; ;  U\&=Q=Q   EEE                                  ") 3 29       6d ; ;   EEE                  sGA;A# A;#A' 'A;*A' +A;;A?A?=CCCc2eZdZdedejjfdZdS)LayerFnr8r'cdSr>rE)r)r8s r*r-zLayerFn.__call__Usr,N)rRrSrTrYr[rrr-rEr,r*r@r@Ts*;s;ux;;;;;;r,r@c(eZdZdZfdZdZxZS)rzN A placeholder layer for missing layers in a pipeline parallel model. cHtdSr>)rfrg)r)r+r,ris r*rgzPPMissingLayer.__init__]s r,cr|r|dn-tt|S)z>Return the first arg from args or the first value from kwargs.r)nextiterrGr*s r*forwardzPPMissingLayer.forward`s-?tAwwDfmmoo)>)>$?$??r,)rRrSrTrUrgrGrrs@r*rrXsX@@@@@@@r,r max_bytescda|adSrq)_CPU_OFFLOAD_BYTES_CPU_OFFLOAD_MAX_BYTES)rHs r*set_cpu_offload_max_bytesrLis&r,c N tdx}S|jtjdkrStt krSt }t}|s Jdd}d}D]}tt krntj|j |j |j j |j j d|}||j |s||_ n||_t!||_ t|j |j zz ad}|r|sj  fd _S)Ncpuz3V1 CPU offloading requires uva (pin memory) supportTF)sizestrider layoutrrc_fdD}t|||}_|S)NcFi|]\}}||dS)T)r)r )r@rvrs r*rPz9maybe_offload_to_cpu..forward..sAAq144T422r,)r+r,)rGrr0r )r+r, device_stateoutputrrGrdoriginal_forwards r*rGz%maybe_offload_to_cpu..forwardsp-FN#--//5577 L %V\VTTTF$FNMr,)rE parametersrr[rJrKrr empty_stridedrBrOrPr rQcopy__vllm_offloaded_cpu_datarnumel element_sizerG) rdparamsr uva_availableuva_offloadingoffloaded_parametersrcpu_datarrGrWs ` @@@r*maybe_offload_to_cpurcosv((**D111: ]F e$$$$ 333 (**J$&&M OOOOO=N!    $$ !7 7 7 E&6==??&,6=!     qv =AFF*2A &28<        ! Mr,num_hidden_layerslayer_fncZddlm}ddlm}|||j|j\}}t jdt|Dfdt||Dzdt||Dz}|||fS)zgMake a list of layers with the given layer function, taking pipeline parallelism into account. r) get_pp_group)get_pp_indicesc*g|]}tSrErrs r*rJzmake_layers..s666a  666r,c Lg|] }td|!S)rl)r8)rc)r@idxrer8s r*rJzmake_layers..sL    !F1B1BS1B1B!C!C!C D D   r,c*g|]}tSrErjrs r*rJzmake_layers..s I I I>   I I Ir,) vllm.distributed.parallel_statergvllm.distributed.utilsrh rank_in_group world_sizer[r ModuleListr)rdrer8rgrh start_layer end_layermoduless `` r* make_layersrvs=<<<<<555555+^<<>>79RKh!!665#5#5666     [)44    J IU96G%H%H I I I  JG  7 **r, _model_to_pp_missing_layer_namesmodelct|}|tvr t|Sg}|D]9\}}t|tt fr||dz:|t|<|S)zAGet the names of the missing layers in a pipeline parallel model.rl)idrwr7rrrr)rxmodel_idmissing_layer_namesrArds r*get_pp_missing_layer_namesr}s%yyH333/99++--33 f f0.A B B 3 & &tcz 2 2 21D$X. r,rAct|ttfrdStfdt |DS)z=Check if a parameter is missing in a pipeline parallel model.Tc3BK|]}|VdSr>r)r@missing_layer_namerAs r*rDz*is_pp_missing_parameter..sD  *++r,)rrrrr})rArxs` r*is_pp_missing_parameterrs`%+^<==t ".make_empty_intermediate_tensors..sB   U[*k!:%PVWWW   r,r)rr rrrs```r*make_empty_intermediate_tensorszPmake_empty_intermediate_tensors_factory..make_empty_intermediate_tensorssO #             r,)intr[r rr)rrrs`` r*'make_empty_intermediate_tensors_factoryrsW    {               +*r,c|s|n|d|S)aAdd a prefix to a name if the prefix is non-empty. Args: prefix: The prefix to add. If empty, no prefix will be added. name: The name to potentially prefix. Returns: The string "prefix.name" if prefix was non-empty, otherwise just "name". rlrE)r8rAs r* maybe_prefixrs! 544f#5#5t#5#55r,cZ|jj}|j}|rtj||ndS)aeGet quantization config for Draft models. Draft models should use their own quantization config instead of the verifier/target model's config. This helper retrieves the draft model's quantization config. Args: vllm_config: The vLLM configuration object. Returns: The draft model's config if available, None otherwise. N)speculative_configdraft_model_config load_configrget_quantization_config)rrdraft_load_configs r*get_draft_quant_configr s?%7J#/   *+=?PQQQ r,r/ layer_namenum_attn_modulec|d}g}|D]5} |t|&#t$rY2wxYw|dksd|vr)t |dksJd|d|dSt |dksJd|dt |dkr|d|z|dzn|d}|S) z Extract the layer index from the module name. Examples: - "encoder.layers.0" -> 0 - "encoder.layers.1.self_attn" -> 1 - "2.self_attn" -> 2 - "model.encoder.layers.0.sub.1" -> ValueError if num_attn_module == 1 rlr/attnz layer name z should only contain one integerrz! should contain most two integers)rmrrrrw)rrsubnamesint_valssubname layer_indexs r*extract_layer_indexr!s'$$HH  OOCLL ) ) ) )    H !vZ778}}!!! F* F F F"!!{8}}!!! G* G G G"!! 8}}!! QK/ )HQK 7 7!  s"A A  A tensorsoffsetc|s&|r9tj|jj|z }tj|| |}|S)N)minr)isinfrisnanr[finfor rclamp)rr clamp_values r*cast_overflow_tensorsrCss}}J  3 3 5 5Jk'-004v= +gK<[III Nr,rGtopkrcj|dkrtj||dStj|||S)a! Optimized topk implementation that uses torch.max for k=1 case. This function provides better performance for the common case of k=1 by using torch.max instead of the more general torch.topk. Args: values: Input tensor to find top-k values from topk: Number of top values to return (k). Must be > 0. dim: Dimension along which to compute topk Returns: Tuple of (values, indices) where values are the top-k values and indices are their corresponding indices in the input tensor r/T)rkeepdim)r)r[rr)rGrrs r* fast_topkrMs=$ qyyyS$7777z&$C0000r,cJtjj|Sr>)r[opsvllmsequence_parallel_chunk_implrrs r*sequence_parallel_chunkrks 9> 6 6q 9 99r,c*t}t}|d}||z}|dkr*||z }tj|ddd|f}n|}|jd|z}||z}tj|d||Srq) rrrOr functionalpadrr[narrow) rstp_sizetp_rankseq_len remainderpad_lenychunkstarts r*rros244G,..GffQiiG'!IA~~I% M  a!Q7!3 4 4  GAJ' !E eOE <1eU + ++r,ct}t|d|}t|j}||d<t j||j|j}|S)Nrr) rrrOr]rr[emptyr r)rsrrrouts r*!sequence_parallel_chunk_impl_fakers[244G166!99g&&G MMEE!H +e1718 < <