.`idZddlmZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z dd lmZmZmZdd lmZdd lmZdd lmZdd lmZddlmZmZmZddlmZddl m!Z!ddl"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3m4Z4m5Z5m6Z6ee7Z8Gddej9Z:Gddej9Z;Gddej9ZdS)#z?Inference-only LLaMA model compatible with HuggingFace weights.)IterableN)nn)Llama4TextConfig) Attention)support_torch_compile) CacheConfig VllmConfig) get_ep_group$get_tensor_model_parallel_world_size tensor_model_parallel_all_gather) init_logger)ChunkedLocalAttention)SharedFusedMoE)RMSNorm)QKVParallelLinearReplicatedLinearRowParallelLinear)QuantizationConfig)get_rope)default_weight_loadermaybe_remap_kv_scale_name)MixtureOfExperts)sequence_parallel_chunk)current_platform)is_torch_equal_or_newer)LlamaForCausalLMLlamaMLP LlamaModel)AutoWeightsLoaderPPMissingLayerextract_layer_index fast_topkis_pp_missing_parameterc eZdZedejdejdededeejejff dZ d de d e ffd Z d Z xZS) Llama4MoE hidden_states gating_outputtopk renormalizereturnct||d\}}tj|}||tjfSNdim)r#torchsigmoidfloattoint32)r'r(r)r* router_scoresrouter_indicess u/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/llama4.pycustom_routing_functionz!Llama4MoE.custom_routing_functionFsS)2-2(N(N(N% ~ m&9&9&;&;<< ~00==>> vllm_configprefixct|jj}|j}|j}t |_|j|_ |j |_ tj |_tj|_|j|_|j}t)|j|jdd|d|_t1|j|d|d|dd|j |_|r|jnd}|r|jnd|_|r|jnd|_|j|_|j|_d|_ |j|_!|j!|jz|_"|j"|jz|_#tI|j|j|j|jtJj&|d dd||d |j |j|j |_'dS) NFz.router)bias quant_configr=siluz.shared_expert) hidden_sizeintermediate_size hidden_actr@r?r=reduce_results disable_tprrTz.experts)shared_experts num_expertstop_krBr9rCapply_router_weight_on_inputrEr*r@r=is_sequence_parallel enable_eplbnum_redundant_experts)(super__init__ model_config hf_configparallel_configr@r tp_sizenum_experts_per_tokrIuse_sequence_parallel_moerKr device_groupep_group rank_in_groupep_ranksizeep_sizerCrrBnum_local_expertsrouterr shared_expert eplb_configrLrMn_redundant_expertsn_routed_expertsn_logical_expertsn_shared_expertsn_local_expertsn_physical_expertsn_local_physical_expertsrr&r9experts) selfr<r=configrRr@intermediate_size_moer_ __class__s r8rOzLlama4MoE.__init__Rs )3%5"/ ;== / $3$M!$3 #~~3 }))++ & 8&    $%%%     &*3%,,, 0    6ENo11$ :IT?66u1< CK - -!  &,%=!%!6%&$*$<"&"69Q"Q(,(?4<(O%%-0,*$-$E3)- %&&&!%!:("&":    r:cH|jd}|jrt|}||\}}|||\}}||z}|jrt |d}|d|}n%|jdkr|j|}|S)Nr)r' router_logitsr)shaperKrr]rgr rS&maybe_all_reduce_tensor_model_parallel)rhr' num_tokensrm_ shared_out routed_out experts_outs r8forwardzLlama4MoE.forwards"(+  $ C3MBBM;;}55 q!%''"." "  J!:-  $ :;JJK%kzk2KK \A  ,MMKr:)r;)__name__ __module__ __qualname__ staticmethodr1Tensorintbooltupler9r strrOru __classcell__rks@r8r&r&Es ?| ?| ? ? ? u|U\) * ? ? ?\ ?? ? J? ? ? ? ? ? ? Br:r&ceZdZ ddedededed ed edzd ed ed edzdeddffd Z de j de j fdZ de j de j de j fdZ xZS)Llama4Attention NFr;rirB num_heads num_kv_headsmax_position_embeddingsr@r? bias_o_proj cache_configr=r+c tt| |_||_|j|_|j|jdk|_|jo|j |_t} ||_ |j | zdksJ|j | z|_ ||_ |j | kr|j | zdksJn| |j zdksJtd|j | z|_ |j|_|j |jz|_|j |jz|_|jdz|_|jo|j|_t'|dd|_t'|dd|_||_|j |j z|_|jr't1|j|jdt4j nd|_t;||j|j |j ||| d  |_t?|j |jz|||| d  |_ d} |o|!dk} | r |j"dkrd} |jstG|j||j$| nd|_%|j o|j&}|rtNntP}||j |j|jf|j | || dd|r d|j&ini|_)dS)Nrrg floor_scaleg@ attn_scaleg?F)rBeps has_weightdtype .qkv_proj)rB head_sizetotal_num_headstotal_num_kv_headsr?r@r=z.o_proj) input_size output_sizer?r@r=Tggufllama) max_positionrope_parameters is_neox_stylez.attn)rrr@r=attention_chunk_size)*rNrOr" layer_idxrBno_rope_layersnope use_qk_normr rrrmaxrhead_dimq_sizekv_sizescalingattn_temperature_tuninggetattrrrrn_repr rms_norm_epsr1float32qk_normrqkv_projro_projget_name model_typerr rotary_embrrrattn)rhrirBrrrr@r?rrr=rSris_ggufuse_chunked_local_attnattn_clsrks r8rOzLlama4Attention.__init__sX ,V44&$3'71< !-?di-688(#g-2222-8".  "g - -*W499999T4499994#:g#EFF nt}4 (4=8 }d* '+y'SV5S$"6=&AA!&,<<'>$^t'88   G M' m      *#m 0#6%'''    (+dm;#%%%%      D<#8#8#:#:f#D  "v(G33!M9  H 4 & 6+      &*Y!N63N,BQ(( H N M L  *%%###   *')DEE    r: positionsctj|dz|jz }tj|dz|jzdz}|dS)Ng?r.)r1floorrlogr unsqueeze)rhrrrs r8_get_attn_scalezLlama4Attention._get_attn_scalesP Y_0@@AAYus{++do=C ##B'''r:r'c||\}}||j|j|jgd\}}}|j||||\}}|j|d|j}||d|j |j }|d|j}||d|j |j }|j r9|j r2| |}||z |j }||||} || \} }| Sr-)rsplitrrrrreshaperr3r4rrrrrr) rhrr'qkvrqqkvr attn_outputoutputs r8ruzLlama4Attention.forwards} }--Q))T[$, E2)NN1a ? &??9a33DAq < # "dm,,A QWWYY''//DK@@CCAGLLA "dm,,A QWWYY''//DLAADDQWMMA  ' -DI ---i88JZ##AG,,Aii1a(( KK ,,  r:)rNFFNr;)rvrwrxrr{rr|rr~rOr1rzrrurrs@r8rrs;(,26!+/e e  e e  e  e "% e )4/e e e "D(e e  e e e e e e N((%,(((( !<!|!  !!!!!!!!r:rc eZdZ d dedededzddffd Zdejd ejd ejdzde ejejffd Z xZ S) Llama4DecoderLayerr;Nr<r=rir+c t|p |jj}|j}|j}t ||_|j|jdk|_ |j |_ |j }t||j |j |j||dd||d |_|jdko|jdz|jzdk}|rt#||d|_n't'|j |jd|d|d |_t+|j |j |_t+|j |j |_dS) NrFz .self_attn) rirBrrrr@r?rrr=rz .feed_forward)r<r=rA)rBrCrDr@r?r=)r)rNrOrPrQrr@r"rr global_layerrBrrnum_attention_headsnum_key_value_heads self_attninterleave_moe_layer_stepr& feed_forwardrintermediate_size_mlprrinput_layernormpost_attention_layernorm) rhr<r=rirr@r is_moe_layerrks r8rOzLlama4DecoderLayer.__init__>s =;3="/ "/ ,V44"1$.AQF!-"("@((03$;%%(((      ,q 0 M!#v'GG1L    )' ///!!!D   !) ,"(">!) /// !!!D  'v'9v?RSSS(/  F$7) ) ) %%%r:rr'residualc||}||}n|||\}}|||}|||\}}||}||fS)N)rr')rrrr)rhrr'rs r8ruzLlama4DecoderLayer.forwardrs  $H 00??MM&*&:&:=(&S&S #M8-XX #'"?"? x"X"X x))-88 h&&r:)r;N) rvrwrxr r~rrOr1rzr}rurrs@r8rr=s*. 2 2 2 2 !4' 2  2 2 2 2 2 2 h'<'|',% ' u|U\) * ''''''''r:rceZdZdeddededeeffdZ dded ej d e ee j fd e ed eeeeeefd edefdZdeeeej fde efdZxZS) Llama4Modelr;)r= layer_typer<r=rc|jjj|_|jjj|_t |||dSNr<r=r) rPrQr\rHrRr_rMr`rNrO)rhr<r=rrks r8rOzLlama4Model.__init__sQ'3=O  ' 3 I   [JWWWWWr:Tname loaded_weight params_dict loaded_paramsexpert_params_mappingfusedr+cd}|r<|jdkr1|dd}d|vr|dd}|D]\}} } } |} |r&| d\} }}}| d|} |d }| |vr7|| |}t ||r^|d s|d r||vr||}|j}|rNd |vr| d vsJ| dkrdnd}| |} t|}|j |j j j }||dk | j}| jt%jkp#| jjo| dk}| jjdkrO|rMt/ds>| t0j|| j} n| |} |d} n ||| || | ||d}|S)a, Load MoE expert weights. Args: name: The name of the weight to load. loaded_weight: The weight to load. params_dict: The dictionary of module parameters. loaded_params: The set of already loaded parameters. expert_params_mapping: The mapping of expert parameters. Must be generated by SharedFusedMoE.make_expert_params_mapping(). fused: Whether the expert weights are fused into a single weight tensor or are separate weight tensors for each expert. When fused is True, loaded_weight should have shape of: [num_experts, hidden_in, hidden_out] for gate/up/down proj and [hidden_out, hidden_in] for the others like router. When fused is False, loaded_weight should have shape of: [hidden_out, hidden_in]. Returns: True if loaded_weight is one of MoE weights and the MoE expert weights are loaded successfully, False otherwise. Fr.experts.gate_up_projr/.weightz.bias_biasw13)w1w3rrrNcpuz2.11.0shard_id expert_idT)ndim transposechunkrreplacer$endswith weight_loaderr"layersrrg expert_mapnonzeroflattenr4devicerr fp8_dtypeis_floating_point element_sizetyperr1float16itemadd)rhrrrrrrexpert_param_loaded param_name weight_namerrnew_loaded_weighte_strrqproj_strfull_param_nameparamr shard_idxrrlocal_expert_indices is_fp8_dtypes r8load_moe_expert_weightsz#Llama4Model.load_moe_expert_weightssB$  ?]'1,,)33B;;M &-- - 3 3A2 3 > > =RY 'Y ' 8J Y!.  3(3(9(9#(>(>%q(A!&3333 *222 $&&#ll; CCO'tT22  g&& *.--*@*@ k))0E!/M+ O++#|3333%-%5%51I(9)(D% 055 ![3@HS )#r)  -455 )$5#:(244$$*/AB-::<<A !*05>>(? 7 A A? ->,@,@,O,O0-".455*)->>R,S) 4Q 7 < < > >I M!!#        o . . ."&  ""r:weightsc Xgd}d}tj|ddd|j|j}tj|dddd }t |}t }|D]\}d vsd vrd }|}|j~|jx} rb|| } t| d t} | dkr|n|d}| | || | |D]\} } }| vsdvr drdvs| | t|rO drt!|w|} t| d t} | tkr | | |n | | ||| nG||||||rt|rgd}dvrt%fd|Dr|} t| d t} t| ddrddvrdnd} dr6|jt(jkr!|jdkr|dd}| | ||dn | | || s|} t| d t} | | || |S) N))rz.q_projr)rz.k_projr)rz.v_projr) .gate_up_projz .gate_projr)r z.up_projrF gate_proj down_projup_proj)ckpt_gate_proj_nameckpt_down_proj_nameckpt_up_proj_namerHrM gate_up_projr)rrrrHrzexperts.down_projTrrrg)z.k_scalez.v_scalerscale)r)w13_input_scalew13_weight_scalew2_input_scalew2_weight_scalezexperts.c3 K|]}|vV dSN).0 scale_namers r8 z+Llama4Model.load_weights..s9..+5J$&......r:supports_moe_loadingw2_w2r weight_scalerr.rr)rmake_expert_params_mappingrHr`dictnamed_parameterssetr@get_cache_scalerrr0rrrr$rr anyrr1 float8_e4m3fnrr)rhr stacked_params_mappingfused_experts_paramsrexpert_params_mapping_fusedrrrrrrrrr scale_namesrs @r8 load_weightszLlama4Model.load_weights#sg" " "  %!/ I  + +'("&": ! ! ! '5&O  . +, ' ' ' #4002233 "%%% $+H (H ( D-&--1D1L1L'+$(C%  ,"/??EEE -$J/ '@U V V %2%6%6%8%8A%=%=MM=QRCS e]333!!*--- 6Lm (m (1 Kd**i4.?.? MM":;;A@Kt@S@S<< Z@@D+466 ==))!4T;GGD| $D) '@U V V  $999!M%7777!M%AAA!!$''' //!!). 0+466  %%#....9D...++%(-E$+0E%%M }.DeLL<+0D==44d !MM.99L - 3u7J J J - 2a 7 7,9,C,CB,K,KM& !=$UV& e];;;!%%d+++$D) '@U V V  e]333!!$'''r:)T)rvrwrxrr r~rrOr1rzr$r Parameterr&listr}r{r|r rr.rrs@r8rrsS /A X X X  X X +, X X X X X X(L#L#L#|L##r|+, L# 3x L# $E#sC*<$=> L#L# L#L#L#L#\rHU3 3D-E$Fr3s8rrrrrrrrr:rceZdZgdddgdZdddedeffd Zd Zd ed ed dfdZ de fdedede e fdZ de eeejfd eefdZdedejd eeejffdZxZS)Llama4ForCausalLM)q_projk_projv_projr r)rrr;)r=r<r=c\|j}||jj|jjdk}|d||jj_t ||t| dS)Nirr) rPtry_get_generation_configupdateoverride_generation_config max_model_lengetrQrrNrOrset_moe_parameters)rhr<r= gen_configdefault_attn_temperature_tuningrks r8rOzLlama4ForCausalLM.__init__s -GGII +2MNNN*5*B*PSX*X'EO^^ %'FF F  *B #F?Q     !!!!!r:cg|_g|_d}|jjD]t}t |t rt |t sJt |jtr+|j}|j |jj u|Td|_ d|_ d|_ d|_d|_d|_d|_d|_t&ddSt+|j|_ d|_ |j|_ |j|_|j|_|j|_|j|_|j|_dS)Nrz)No Llama4MoE layer found in model.layers.r)expert_weights moe_layersmodelr isinstancer!rrr&appendrgnum_moe_layersnum_expert_groupsnum_logical_expertsnum_physical_expertsnum_local_physical_expertsnum_routed_expertsnum_shared_expertsrMloggerwarninglenrbrerfrarcr`)rh example_moelayers r8r<z$Llama4ForCausalLM.set_moe_parameterssT  Z& C CE%00 e%788 8 88%,i88 C#0 &&u'9'ABBB  "#D %&D "'(D $()D %./D +&'D #&'D #)*D & NNF G G G G G"%do"6"6D %&D "'2'DD $(3(FD %.9.RD +&1&BD #&1&BD #)4)HD & & &r:rHrIr+NcL|j|ksJ||_||_||jz |_|jjD]l}t |trt |jtr:|j}||_ ||_ |j|_ |j mdSr)rIrHrGrMrBrrCr!rr&rfrer`rgupdate_expert_map)rhrHrIrPmoes r8 update_physical_experts_metadataz2Llama4ForCausalLM.update_physical_experts_metadatas .2LLLLL$8!*D'%9Dz2Llama4ForCausalLM.load_weights..4s=   #m  - -dM B B   r:)r ritie_word_embeddingsr.)rhr loaders` r8r.zLlama4ForCausalLM.load_weights/sl" +/;+JTJ<.permute@sk*W4G{.Hw%+%%!'!*q.H*D*D#q= 5...#/GAJOx//#r>w7 2a 7HEE1a(++ r:rr.rr"wkr4wqr3) r1rzr{r|rrr)rirr)rhrrrhmodules is_weightis_nvfp4_weight_scales` r8rZz.Llama4ForCausalLM.permute_qk_weight_for_rotary:s  u| c D      4**S//BK8+ BK> ) Xm.AUEX.X   - w(g"5"5 '!K3)!! H$7$7 '!K3)!! ]""r:)rvrwrxpacked_modules_mappingr r~rOr<r{rTrrrVrr}r1rzr&r.rZrrs@r8r2r2su222$i0 BD"""z"3"""""" !I!I!IF0!0%(0  0000./A     +,      ,HU3 3D-E$F ,3s8 , , , ,6#6#|6# sEL ! 6#6#6#6#6#6#6#6#r:r2)?__doc__collections.abcrr1r transformersrvllm.attention.layerrvllm.compilation.decoratorsr vllm.configrr vllm.distributedr r r vllm.loggerr rsv&FE$$$$$$ ))))))******======//////// $#####@?????888888 GFFFFF@@@@@@CBBBBBDDDDDD++++++::::::9999999999 X  ccccc cccLOOOOObiOOOdF'F'F'F'F'F'F'F'RNNNNN*NNNb X#X#X#X#X#(*:X#X#X#X#X#r: