.`i<0ddlZddlmZddlmZddlmZmZmZddlm Z m Z m Z ddl m Z mZddlmZddlmZdd lmZdd lmZdd lmZdd lmZd dlmZmZmZmZd dl m!Z!m"Z"m#Z#d dl$m%Z%d dl&m'Z'm(Z(d dl)m*Z*m+Z+d dl,m-Z-ee.Z/dej0dej0dej0dej0dej0dej0de1ddfdZ2dej0dej0dej0dej0dej0dej0de1ddfdZ3ede2dge3Gd d!ej4e%Z5dS)"N) rearrange)nn) CacheConfig ModelConfigget_current_vllm_config)divideget_tensor_model_parallel_rank$get_tensor_model_parallel_world_size)ForwardContextget_forward_context) init_logger)sharded_weight_loader)set_weight_attrs)direct_register_custom_op)AttentionMetadata)GDNAttentionMetadata)FusedRMSNormGated chunk_kdafused_kda_gatefused_recurrent_kda)ColumnParallelLinearReplicatedLinearRowParallelLinear) MambaBase)MambaStateDtypeCalculatorMambaStateShapeCalculator)causal_conv1d_fncausal_conv1d_update)QuantizationConfig q_proj_states k_proj_states v_proj_statesg1beta core_attn_out layer_namereturncrt}|j|}|||||||dS)N)r!r"r#r$r%r&)r no_compile_layers_forward) r!r"r#r$r%r&r'forward_contextselfs r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/layers/kda.py kda_attentionr/)sS':&;&;O  ,Z 8DMM### # cdS)N)r!r"r#r$r%r&r's r.kda_attention_faker3>s  Fr0r/)op_nameop_func mutates_args fake_implceZdZedefdZdeejejejejffdZ deee dfee dfee dfee dfffdZ dd e d e d e dzd e dzd edzdededdffd ZdejdejdejddfdZdejdejdejdejdejdejddfdZxZS)KimiDeltaAttentionr(cdS)N gdn_attentionr2r-s r. mamba_typezKimiDeltaAttention.mamba_typeSsr0c|j|jtdtj|jj|jjS)Nz)model_config and cache_config must be set) model_config cache_config ValueErrorrkda_state_dtypedtypemamba_cache_dtyper<s r.get_state_dtypez"KimiDeltaAttention.get_state_dtypeWsJ   $(9(AHII I(8   #T%6%H   r0.cZtj|j|j|j|jS)N)conv_kernel_size)rkda_state_shapetp_size num_headshead_dim conv_sizer<s r.get_state_shapez"KimiDeltaAttention.get_state_shape`s/)8 L$.$-$.    r0Nh㈵> layer_idx hidden_size quant_configr@r? rms_norm_epsprefixc tt|_t |_||_||_||_|td|j } | d|_ | d|_ ||_ ||_|j |jzdksJt|j |j|_|j |j z} | d|_t%|j| d||d|_t%|j| d||d |_t%|j| d||d |_t-|j|j d||d |_t%|j | d||d |_t3jt7jt| |jt6j |_t?|jdtAdit%|j|j d||d|_!t%|j| dt6j|d|_"t%|j| dt6j|d|_#t%|j| dt6j|d|_$|j"j%j&'d|j"j%_&|j#j%j&'d|j#j%_&|j$j%j&'d|j$j%_&t3jt7jdd|jdt6j |_(t?|j(dtAdit-|j|j d||d|_)t%|j | d||d|_*tW|j |d|_,t[| |jd||d|_.t_j0} || j1vrtd||| j1|<dS)Nzmodel_config must be providedrKrJrshort_conv_kernel_sizeFz.q_proj)biasrRrTz.k_projz.v_projz .f_a_projz .f_b_proj)rC weight_loaderz.b_projz .q_conv1d) input_size output_sizerW params_dtyperTz .k_conv1dz .v_conv1drz .g_a_projz .g_b_projsigmoid)eps activationz.o_projzDuplicate layer name: )2super__init__r rIr tp_rankrQr?r@rAlinear_attn_configrKrJrPrTrlocal_num_headsrLrq_projk_projv_projrf_a_projf_b_projr Parametertorchemptyfloat32dt_biasrrb_projq_conv1dk_conv1dv_conv1dweightdata unsqueezeA_logg_a_projg_b_projro_normro_projrcompilation_configstatic_forward_context) r-rPrQrRr@r?rSrTkwargs kda_configprojection_sizer{ __class__s r.razKimiDeltaAttention.__init__gs ;== 577 &((  <== =!4 ":. #K0" ~ ,1111%dndlCC-$.8#$<=*   %%%%     +   %%%%     +   %%%%     )   M%'''     - M %'''     | K ==U] S S S    9Nq9Q9Q'RSSS*   N%%%%     -~''''     -~''''     -~''''     %)M$8$=$G$G$J$J !$(M$8$=$G$G$J$J !$(M$8$=$G$G$J$J !\ K1d2AU] K K K    o7LQ7O7O%PQQQ(   M%'''     - M %'''     ( M|     (   %%%%     566I '> > >>f>>?? ?<@1&999r0 hidden_states positionsoutputc .|d}||d}||d}||d}||d}|||dd} t| |j |j |j } | d}| d} |||dd} t!| d|j } t#jd||j|j f|j|j} t"jj|||| || |j|| | } t!| d} || d|dd<dS)Nr)g_biasz... (h d) -> ... h ddr)rCdevicez1 n h d -> n (h d))sizererfrgrofloatr]rirhrrvrKrnrurxrwrrkzerosrdrCropsvllmr/rTryrz) r-rrr num_tokensqkvr%r$ g_proj_statesg2r&s r.forwardzKimiDeltaAttention.forwards #''** KK & &q ) KK & &q ) KK & &q ){{=))!,2244<<>> ]]4==77: ; ;A > B DM$, O O O~~a   \\!__ dmmM&B&B1&EFFqI }&< N N N  D0$- @% '    $$    K    M266 !-1EFF KK ..q1qqq r0r!r"r#r$r%r&c 4t}|j}|dSt|tsJ|j}t|t sJ|j} |j} |j} |j } j |j } |d| }|d| }|d| }|d| }|d| }| \}}}}| dd}| dd}| dd}j jj jdj jd}jjjjdjjd}jjjjdjjd}|jdkr| dd}| dd}| dd}t)||j jd|| | | | dd}t)||jjd|| | | | dd}t)||jjd|| | | | dd}no| d|j }t-|||j jd|d }t-|||jjd|d }t-|||jjd|d }t/fd |||f\}}}|jdkrI| | }d||<|| }t3||||||dd| \}}||| <n+t5||||||d| d|jdz| \}}|dd| f|dd| f<dS) Nrr\rsilu)r_ conv_stateshas_initial_state cache_indicesquery_start_locmetadataT)r_conv_state_indices validate_datac2t|djS)Nzn (h d) -> 1 n h dr)rrK)xr-s r.z-KimiDeltaAttention._forward..si#74=IIIr0) rrrgr% initial_stateoutput_final_stateuse_qk_l2norm_in_kernel cu_seqlens) rrrrr%rrrssm_state_indices)r attn_metadata isinstancedictrTrrnon_spec_query_start_locnon_spec_state_indices_tensornum_actual_tokenskv_cachevirtual_engine transposerprsviewrrqrr num_prefillsrrWrmap contiguousrr num_decodes)r-r!r"r#r$r%r&r,rrrrrconstant_caches conv_state_q conv_state_k conv_state_vrecurrent_stateq_conv_weightsk_conv_weightsv_conv_weightsrrrdecode_conv_indiceszero_idxrcore_attn_out_non_speclast_recurrent_states` r.r+zKimiDeltaAttention._forwards .//+:+H  F-.....%dk2 -)=>>>>>);#0#I (5(S%);-(FG%&8'8&89 %&8'8&89 %&8'8&89 """ #&&&'FUC|\?#--b"55 #--b"55 #--b"55 -22 M % %a ( ($-*>*C*CA*F*F  -22 M % %a ( ($-*>*C*CA*F*F  -22 M % %a ( ($-*>*C*CA*F*F    % ) ))33Aq99M)33Aq99M)33Aq99M  "!("3; 8&   i1oo ! "!("3; 8&   i1oo ! "!("3; 8&   i1oo A#@1-11# % "!#6"A% "!#6"A% "!#6"A I I I IAqRS9  1a  % ) )46G5GHH()OH %+,IJUUWWM+#'(,3    &$>RO9 : : $-(,34Sm6ORS6S4ST"?    &$0F !!! !0 a++++,,,r0)NNNrNrO)__name__ __module__ __qualname__propertystrr=tuplerkrCrEintrMr rrrraTensorrr+ __classcell__)rs@r.r9r9Rs$ CX u{EKekA B     uS#Xc3hsCx%S/Q R    37+/+/"OAOAOAOA)4/ OA "D( OA "D( OAOAOA OAOAOAOAOAOAb$2|$2<$2 $2  $2$2$2$2L\ |\ |\ | \ L \ l \ |\  \ \ \ \ \ \ \ \ r0r9)6rkeinopsrr vllm.configrrrvllm.distributedrr r vllm.forward_contextr r vllm.loggerr -vllm.model_executor.model_loader.weight_utilsrvllm.model_executor.utilsrvllm.utils.torch_utilsrvllm.v1.attention.backendr#vllm.v1.attention.backends.gdn_attnr fla.ops.kdarrrrlinearrrrmamba.abstractrmamba.mamba_utilsrrmamba.ops.causal_conv1drrquantization.base_configr rloggerrrr/r3Moduler9r2r0r.rs# IIIIIIIIII EDDDDDDD######OOOOOO666666<<<<<<777777DDDDDD  &%%%%%SSSSSSSSKKKKKKKK888888 X  <<<   ,  <  *  <  <  <     ,  <            !" h h h h h Ih h h h h r0