
    .`iW                     |   d Z ddlZddlZddlmZ ddlmc mZ ddlm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ee          Z ej        d	           G d
 de                      Z ej        d           G d de                      Z ej        d           G d de                      Z ej        d           G d de                      Z ej        d           G d de                      Z ej        d           G d de                      Z ej        d           G d de                      Z ej        d           G d d e                      Z  ej        d!           G d" d#e                      Z! ej        d$           G d% d&e                      Z" ej        d'           G d( d)e                      Z# G d* d+ej$                  Z% ed, d- d. d/ d0 d1 d2 d3 d4 d5 d6 d7          Z&d8e'd9ej$        fd:Z( ed; d< d= d> d?          Z)d8e'd9ej$        fd@Z*dS )AzCustom activation functions.    N)divideget_tensor_model_parallel_rank$get_tensor_model_parallel_world_size)init_logger)CustomOp)set_weight_attrs)current_platform)LazyDictfatrelu_and_mulc                   r     e Zd ZdZd	def fdZdej        dej        fdZdej        dej        fdZ	 xZ
S )
FatreluAndMula5  An activation function for FATReLU.

    The function computes x -> FATReLU(x[:d]) * x[d:] where
    d = x.shape[-1] // 2.
    This is used in openbmb/MiniCPM-S-1B-sft.

    Shapes:
        x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
        return: (num_tokens, d) or (batch_size, seq_len, d)
            	thresholdc                     t                                                       || _        t          j                    rt
          j        j        j        | _	        d S t          j
                    r| j        | _        d S d S N)super__init__r   r	   is_cuda_aliketorchops_Cr   opis_cpuforward_native_forward_method)selfr   	__class__s     y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/layers/activation.pyr   zFatreluAndMul.__init__)   so    ")++ 	7il2DGGG$&& 	7#'#6D   	7 	7    xreturnc                     |j         d         dz  }|dd |f         }|d|d f         }t          j        || j        d          }||z  S )N   .r   )shapeFr   )r   r    dx1x2s        r   r   zFatreluAndMul.forward_native1   sR    GBK1sBQBwZsABBwZ[T^S11Bwr   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||| j                   |S Nr#   r$   )dtypedevice)r%   r   emptyr,   r-   r   r   r   r    r'   output_shapeouts        r   forward_cudazFatreluAndMul.forward_cuda8   s_    GBK1wss|qd*k,agahGGGQ'''
r   )r   )__name__
__module____qualname____doc__floatr   r   Tensorr   r2   __classcell__r   s   @r   r   r      s        	 	7 7% 7 7 7 7 7 7     el u|        r   r   silu_and_mulc                        e Zd ZdZdddef fdZedej        dej        fd            Z	dej        dej        fd	Z
dej        dej        fd
Z xZS )
SiluAndMulzAn activation function for SwiGLU.

    The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.

    Shapes:
        x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
        return: (num_tokens, d) or (batch_size, seq_len, d)
    Tcompile_nativer?   c                <   t                                          |           t          j                    rt          j        j        j        | _        d S t          j	                    rddl
m} |j        | _        d S t          j                    r| j        | _        d S d S )Nr>   r   ipex_ops)r   r   r	   r   r   r   r   r;   r   is_xpuvllm._ipex_opsrB   r   r   r   )r   r?   rB   r   s      r   r   zSiluAndMul.__init__N   s    777)++ 	7il/DGGG$&& 	7//////+DGGG$&& 	7#'#6D   	7 	7r   r    r!   c                 x    | j         d         dz  }t          j        | dd|f                   | d|df         z  S 6PyTorch-native implementation equivalent to forward().r#   r$   .Nr%   r&   silu)r    r'   s     r   r   zSiluAndMul.forward_nativeY   sA     GBK1vaRaRj!!Ac122gJ..r   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||           |S r+   r%   r   r.   r,   r-   r   r/   s        r   r2   zSiluAndMul.forward_cuda_   X    GBK1wss|qd*k,agahGGGQ
r   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||           |S r+   rK   r/   s        r   forward_xpuzSiluAndMul.forward_xpuf   rL   r   )r3   r4   r5   r6   boolr   staticmethodr   r8   r   r2   rN   r9   r:   s   @r   r=   r=   A   s          26 	7 	7 	7$ 	7 	7 	7 	7 	7 	7 /%, /5< / / / \/
el u|    U\ el        r   r=   mul_and_siluc                   j     e Zd ZdZ fdZdej        dej        fdZdej        dej        fdZ xZ	S )
MulAndSiluzAn activation function for SwiGLU.

    The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2.

    Shapes:
        x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
        return: (num_tokens, d) or (batch_size, seq_len, d)
    c                 8   t                                                       t          j                    rt          j        j        j        | _        d S t          j	                    rddl
m} |j        | _        d S t          j                    r| j        | _        d S d S Nr   rA   )r   r   r	   r   r   r   r   rQ   r   rC   rD   rB   r;   r   r   r   r   rB   r   s     r   r   zMulAndSilu.__init__|   s    )++ 	7il/DGGG$&& 	7//////+DGGG$&& 	7#'#6D   	7 	7r   r    r!   c                 x    |j         d         dz  }|dd|f         t          j        |d|df                   z  S rF   rH   )r   r    r'   s      r   r   zMulAndSilu.forward_native   s?    GBK1bqbzAF1S!""W:....r   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||           |S r+   rK   r/   s        r   r2   zMulAndSilu.forward_cuda   rL   r   )
r3   r4   r5   r6   r   r   r8   r   r2   r9   r:   s   @r   rS   rS   o   s         	7 	7 	7 	7 	7/ / / / / /
el u|        r   rS   gelu_and_mul_sparsec                        e Zd ZdZddedef fdZdej        dej        fdZ	dej        dej        fd	Z
dej        dej        fd
Z xZS )GeluAndMulSparsea  An activation function for GeluAndMulSparse.
    This activation function is used in Gemma3n. It computes:
        up_proj = self.up_proj(x)
        gate_proj = self.gate_proj(x)
        gate_proj = self._gaussian_topk(gate_proj) # sparsity
        activations = self.act_fn(gate_proj) # gelu
        down_proj = self.down_proj(activations * up_proj)
    Shapes:
        x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
        return: (num_tokens, d) or (batch_size, seq_len, d)
    noneactivation_sparsityapproximatec                    t                                                       || _        |dvrt          d|           t	          j                    r'|dk    r!t                              d           d| _        |dk    rt          d          t          j	        |t          j
                  }t          j        j                            d	d
          }|                    |          | _        d S )Nr\   tanhUnknown approximate mode: ra   z[ROCm] Pytorch's native GELU with tanh approximation is currently unstable and produces garbage. Fallback to 'none' approximation.r\   r   z2activation_sparsity is 0.0. Please use GeluAndMul.r,   r      )r   r   r^   
ValueErrorr	   is_rocmloggerwarning_oncer   tensorfloat32distributionsnormalNormalicdfstd_multiplier)r   r]   r^   target_sparsity_tensornormal_distr   s        r   r   zGeluAndMulSparse.__init__   s    &...G+GGHHH#%% 	&+*?*?S    &D #%%QRRR!&.A!W!W!W)0771==)../EFFr   r    r!   c                     t          j        |dd          }t          j        |ddd          }||| j        z  z   }t          j                            ||z
            S )z5Get % sparse percentile of the Gaussian distribution.r#   T)dimkeepdimF)rs   rt   unbiased)r   meanstdro   nn
functionalrelu)r   r    rv   rw   cutoff_xs        r   _gaussian_topkzGeluAndMulSparse._gaussian_topk   s^    
 z!T222ir4%@@@# 333}!!!h,///r   c                     |j         d         dz  }|                     |dd|f                   }t          j        || j                  }||d|df         z  S )rG   r#   r$   .Nr^   )r%   r|   r&   gelur^   )r   r    r'   r1   s       r   r   zGeluAndMulSparse.forward_native   s_    GBK1!!!C!G*--fSd&6777QsABBwZr   c                 ,    |                      |          S r   r   r   r    s     r   r2   zGeluAndMulSparse.forward_cuda   s    ""1%%%r   r\   )r3   r4   r5   r6   r7   strr   r   r8   r|   r   r2   r9   r:   s   @r   r[   r[      s        
 
G GE G G G G G G G*0 0 0 0 0 0           &el &u| & & & & & & & &r   r[   gelu_and_mulc                        e Zd ZdZddef fdZdej        dej        fdZdej        dej        fdZ	dej        dej        fd	Z
defd
Z xZS )
GeluAndMulzAn activation function for GeGLU.

    The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.

    Shapes:
        x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
        return: (batch_size, seq_len, d) or (num_tokens, d)
    r\   r^   c                 4   t                                                       || _        |dvrt          d|           t	          j                    st	          j                    rC|dk    rt          j        j	        j
        | _        n!|dk    rt          j        j	        j        | _        t	          j                    r"|dk    rt                              d           d S t	          j                    r(ddlm} |dk    r|j
        | _        d S |j        | _        d S d S )Nr`   rb   r\   ra   z[ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected.r   rA   )r   r   r^   re   r	   r   r   r   r   r   r   r   gelu_tanh_and_mulrf   rg   rh   rC   rD   rB   )r   r^   rB   r   s      r   r   zGeluAndMul.__init__   s3   &...G+GGHHH)++ 	9/?/F/H/H 	9f$$),3&&),8#%% 	5+*?*?Q    
 $&& 	5//////f$$"/"4	5 	5r   r    r!   c                     | j         }t          j                    r|dk    rd}|j        d         dz  }t	          j        |dd|f         |          |d|df         z  S )rG   ra   r\   r#   r$   .Nr~   )r^   r	   rf   r%   r&   r   )r   r    r^   r'   s       r   r   zGeluAndMul.forward_native   sn     &#%% 	!+*?*? KGBK1vaRaRjk:::QsABBwZGGr   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||           |S r+   rK   r/   s        r   r2   zGeluAndMul.forward_cuda  rL   r   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }|                     ||           |S r+   rK   r/   s        r   rN   zGeluAndMul.forward_xpu  rL   r   c                 0    dt          | j                   S )Nzapproximate=)reprr^   r   s    r   
extra_reprzGeluAndMul.extra_repr  s    6d4#344666r   r   )r3   r4   r5   r6   r   r   r   r8   r   r2   rN   r   r9   r:   s   @r   r   r      s         5 5C 5 5 5 5 5 50H H H H H Hel u|    U\ el    7C 7 7 7 7 7 7 7 7r   r   swigluoai_and_mulc                   ~     e Zd Zddedef fdZdej        dej        fdZdej        dej        fd	Zde	fd
Z
 xZS )SwigluOAIAndMulZd;?      @alphalimitc                 d    t                                                       || _        || _        d S r   )r   r   r   r   )r   r   r   r   s      r   r   zSwigluOAIAndMul.__init__  s+    



r   r    r!   c                    |ddddf         |ddddf         }}|                     d| j                  }|                     | j         | j                  }|t          j        || j        z            z  }|dz   |z  }|S )rG   .Nr$   rd   )minmax)clampr   r   sigmoidr   )r   r    gateupglugated_outputs         r   r   zSwigluOAIAndMul.forward_native  s     S##A#X;#qt!t)bzzd
z33XX4:+4:X66U]4$*#4555Q#~r   c                     |j         d         dz  }|j         d d         |fz   }t          j        ||j        |j                  }t          j        j                            ||| j        | j	                   |S r+   )
r%   r   r.   r,   r-   r   r   r   r   r   r/   s        r   r2   zSwigluOAIAndMul.forward_cuda(  si    GBK1wss|qd*k,agahGGG	&&sAtz4:FFF
r   c                 Z    dt          | j                   dt          | j                   S )Nzalpha=z, limit=)r   r   r   r   s    r   r   zSwigluOAIAndMul.extra_repr/  s,    DTZ((DD$tz2B2BDDDr   )r   r   )r3   r4   r5   r7   r   r   r8   r   r2   r   r   r9   r:   s   @r   r   r     s        
 e E      
     el u|    EC E E E E E E E Er   r   gelu_newc                        e Zd Z fdZdej        dej        fdZdej        dej        fdZdej        dej        fdZ xZ	S )NewGELUc                    t                                                       t          j                    st          j                    rt
          j        j        j        | _	        d S t          j
                    rddlm} |j        | _	        d S d S rU   )r   r   r	   r   r   r   r   r   r   r   rC   rD   rB   rV   s     r   r   zNewGELU.__init__8  s    )++ 	(/?/F/H/H 	(il+DGGG$&& 	(//////'DGGG	( 	(r   r    r!   c                     t          j        dt           j        z            }d|z  dt          j        ||dt          j        |d          z  z   z            z   z  S )rG   g       @      ?      ?Hm?g      @)mathsqrtpir   ra   pow)r   r    cs      r   r   zNewGELU.forward_nativeA  sP    IcDGm$$Qw#
1HuyC?P?P4P0P+Q R RRSSr   c                 Z    t          j        |          }|                     ||           |S r   r   
empty_liker   r   r    r1   s      r   r2   zNewGELU.forward_cudaF  (    q!!Q
r   c                 ,    |                      |          S r   r   r   s     r   rN   zNewGELU.forward_xpuK      wwqzzr   
r3   r4   r5   r   r   r8   r   r2   rN   r9   r:   s   @r   r   r   4  s        ( ( ( ( (T T T T T T
el u|    
U\ el        r   r   	gelu_fastc                        e Zd Z fdZdej        dej        fdZdej        dej        fdZdej        dej        fdZ xZ	S )FastGELUc                    t                                                       t          j                    st          j                    rt
          j        j        j        | _	        d S t          j
                    rddlm} |j        | _	        d S d S rU   )r   r   r	   r   r   r   r   r   r   r   rC   rD   rB   rV   s     r   r   zFastGELU.__init__T  s    )++ 	)/?/F/H/H 	)il,DGGG$&& 	)//////(DGGG	) 	)r   r    r!   c                 Z    d|z  dt          j        |dz  dd|z  |z  z   z            z   z  S )rG   r   r   g3E?r   )r   ra   r   s     r   r   zFastGELU.forward_native]  s:    Qw#
1|+;sXPQ\TUEU?U+V W WWXXr   c                 Z    t          j        |          }|                     ||           |S r   r   r   s      r   r2   zFastGELU.forward_cudaa  r   r   c                 ,    |                      |          S r   r   r   s     r   rN   zFastGELU.forward_xpuf  r   r   r   r:   s   @r   r   r   P  s        ) ) ) ) )Y Y Y Y Y Yel u|    
U\ el        r   r   
quick_geluc                        e Zd Z fdZdej        dej        fdZdej        dej        fdZdej        dej        fdZ xZ	S )	QuickGELUc                    t                                                       t          j                    st          j                    rt
          j        j        j        | _	        d S t          j
                    rddlm} |j        | _	        d S d S rU   )r   r   r	   r   r   r   r   r   
gelu_quickr   rC   rD   rB   rV   s     r   r   zQuickGELU.__init__p  s    )++ 	*/?/F/H/H 	*il-DGGG$&& 	*//////)DGGG	* 	*r   r    r!   c                 6    |t          j        d|z            z  S )rG   r   )r   r   r   s     r   r   zQuickGELU.forward_nativey  s    5=++++r   c                 Z    t          j        |          }|                     ||           |S r   r   r   s      r   r2   zQuickGELU.forward_cuda}  r   r   c                 Z    t          j        |          }|                     ||           |S r   r   r   s      r   rN   zQuickGELU.forward_xpu  r   r   r   r:   s   @r   r   r   k  s        
* * * * *, , , , , ,el u|    
U\ el        r   r   relu2c                   Z    e Zd ZdZdej        dej        fdZdej        dej        fdZdS )ReLUSquaredActivationzX
    Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
    r    r!   c                 N    t          j        t          j        |                    S )rG   )r   squarer&   rz   r   s     r   r   z$ReLUSquaredActivation.forward_native  s    |AF1II&&&r   c                 ,    |                      |          S r   r   r   s     r   r2   z"ReLUSquaredActivation.forward_cuda  s    ""1%%%r   N)r3   r4   r5   r6   r   r8   r   r2    r   r   r   r     sj         ' ' ' ' ' '&el &u| & & & & & &r   r   xieluc                        e Zd ZdZddddej        dfdededed	ed
ej        def fdZ	dej
        dej
        fdZdej
        dej
        fdZdej
        dej
        fdZdej
        dej
        fdZ xZS )XIELUz
    Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010
    If the user has installed the nickjbrowning/XIELU, we import xIELU CUDA
    Otherwise, we emit a single warning and use xIELU Python
    g?r   gưFalpha_p_initalpha_n_initbetaepsr,   with_vector_loadsc           
         t                                                       t          j        t	          j        t	          j        t	          j        ||                    dz
                                d                    | _	        t          j        t	          j        t	          j        t	          j        ||z
  |                    dz
                                d                    | _
        |                     dt	          j        ||                     |                     dt	          j        ||                     || _        t          | j                                                                                                                                                  | _        t          | j                                                                                                                                                  | _        d | _        	 dd l}t          j        j                                        | _        d}	 ddlm}	  |	| j                  | _        |dz  }n,# t<          $ r}
|d	|
 d
z  }| j        | _        Y d }
~
nd }
~
ww xY wt>                               |           d S # t<          $ r3}
t>                               dtC          |
                     Y d }
~
d S d }
~
ww xY w)Nrc   rd   r   r   r   zUsing experimental xIELU CUDA.)allow_in_graphz& Enabled torch._dynamo for xIELU CUDA.z+ Could not enable torch._dynamo for xIELU (z*) - this may result in slower performance.u   CUDA-fused xIELU not available (%s) – falling back to a Python version.
For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`)"r   r   rx   	Parameterr   logexpri   	unsqueezealpha_palpha_nregister_bufferr   r7   r   detachcpuitem_beta_scalarr   _eps_scalar_xielu_cuda_obj	xielu.opsclassesr   r   torch._dynamor   _xielu_cuda_xielu_cuda_fn	Exceptionrg   rh   r   )r   r   r   r   r   r,   r   r   msgr   errr   s              r   r   zXIELU.__init__  s    	|Iei\ G G GHH1LMMWW 
 

 |I	%,|d':%HHHIIAM ill
 

 	VU\$e%D%D%DEEEUELE$B$B$BCCC!2!$)"2"2"4"4"8"8":":"@"@"B"B"G"G"I"IJJ !2!2!6!6!8!8!>!>!@!@!E!E!G!GHH#	#(=#6#<#<#>#>D 2C
7888888&4nT5E&F&F#?? 7 7 7=# = = = '+&6######7 $$$$$ 	 	 	j C	        	sB   .J0  I( 'J0 (
J2JJ0 JJ0 0
K-:(K((K-r    r!   c           
      n   t           j                            | j                  }| j        t           j                            | j                  z   }t          j        |dk    ||z  |z  | j        |z  z   t          j        t          j	        || j
                            |z
  |z  | j        |z  z             S Nr   )rx   ry   softplusr   r   r   r   whereexpm1r   r   )r   r    r   r   s       r   _xielu_pythonzXIELU._xielu_python  s    -((66)bm44T\BBB{EaK!Odi!m+[1dh//0014?$)a-O
 
 	
r   c                 2   | j         
J d            |j        }|                                dk     r-|                    d          }|                                dk     -|                                dk    r*|                    dd|                    d                    }||j        k    r!t                              d||j                   | j                             || j	        | j
        | j        | j        | j                  }|                    |          S )z>Firewall function to prevent torch.compile from seeing .item()Nz"XIELU CUDA object must not be None   r   r#   rd   z_Warning: xIELU input tensor expects 3 dimensions but got (shape: %s). Reshaping to (shape: %s).)r   r%   rs   r   viewsizerg   rh   forwardr   r   r   r   r   )r   r    original_shaperesults       r   r   zXIELU._xielu_cuda  s   #//1U///eeggkkAA eeggkk5577Q;;r1affRjj))AQW$$B	   %--LL "	
 	
 {{>***r   inputc                     | j         T|j        rMt          j                                        s|                     |          S t                              d           |                     |          S )Nz:torch._dynamo is compiling, using Python version of xIELU.)	r   is_cudar   _dynamois_compilingr   rg   rh   r   r   r   s     r   r   zXIELU.forward_native  sm    ++=--// **5111##P   !!%(((r   c                 ,    |                      |          S r   r   r  s     r   r2   zXIELU.forward_cuda  s    ""5)))r   )r3   r4   r5   r6   r   bfloat16r7   r,   rO   r   r8   r   r   r   r2   r9   r:   s   @r   r   r     s7         "!"^"'3 33 3 	3
 3 {3  3 3 3 3 3 3j
u| 
 
 
 
 
+U\ +el + + + +8)EL )U\ ) ) ) )*%, *5< * * * * * * * *r   r   c            
            e Zd ZdZ	 	 ddej        dededej	        dz  f fdZ
d	ej        d
ej        fdZdej        dej        fdZ xZS )ScaledActivationzqAn activation function with post-scale parameters.

    This is used for some quantization methods like AWQ.
    TN
act_moduleintermediate_sizeinput_is_parallelparams_dtypec                 f   t                                                       || _        || _        |rt	                      }t          ||          }n|}|t          j                    }t          j	        t          j
        ||                    | _        t          | j        d| j        i           d S )Nrc   weight_loader)r   r   actr  r   r   r   get_default_dtyperx   r   r.   scalesr   r  )r   r  r  r  r	  tp_sizeintermediate_size_per_partitionr   s          r   r   zScaledActivation.__init__  s     	!2 	@:<<G.45F.P.P++.?+ 244LlK7|LLL
 
 	8J&KLLLLLr   r    r!   c                 <    |                      |          | j        z  S r   )r  r  r   s     r   r   zScaledActivation.forward,  s    xx{{T[((r   paramloaded_weightc                     |j         }| j        r7t                      }|j        d         }||z  }|                    d||          }|j        |j        k    sJ |                    |           d S r   )datar  r   r%   narrowcopy_)r   r  r  
param_datatp_rank
shard_size	start_idxs          r   r  zScaledActivation.weight_loader/  s    Z
! 	K466G#)!,J*,I)00IzJJM=#66666'''''r   )TN)r3   r4   r5   r6   rx   ModuleintrO   r   r,   r   r8   r   r   r  r9   r:   s   @r   r  r    s          #'+/M MIM M  	M
 kD(M M M M M M,) )%, ) ) ) )(2< ( ( ( ( ( ( ( ( (r   r  c                  (    t          j                    S r   )rx   GELUr   r   r   <lambda>r   <      		 r   c                      t                      S r   )r   r   r   r   r   r   =  s
    XZZ r   c                      t                      S r   )r   r   r   r   r   r   >  s
    GII r   c                      t          j                    r5t                              d          t	          j        d          fd         nt	          j        d          S )Nzk[ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none').r\   r~   rd   ra   )r	   rf   rg   rh   rx   r  r   r   r   r   r   ?  sj     #%%	&)<  G'''&
 & & W((( r   c                  (    t          j                    S r   )rx   ReLUr   r   r   r   r   I  r!  r   c                      t                      S r   )r   r   r   r   r   r   J  s    .00 r   c                  (    t          j                    S r   )rx   SiLUr   r   r   r   r   K  r!  r   c                      t                      S r   )r   r   r   r   r   r   L  s
    ikk r   c                  (    t          j                    S r   )rx   Tanhr   r   r   r   r   M  r!  r   c                  (    t          j                    S r   )rx   Sigmoidr   r   r   r   r   N  s    2:<< r   c                      t                      S r   )r   r   r   r   r   r   O  s
     r   )r   r   r   gelu_pytorch_tanhrz   r   rI   r   ra   r   r   act_fn_namer!   c                    |                                  } |                     d          r6|                     d          d         }|dk    rt          j                    S |} | t
          vrt          d| d          t
          |          S )z#Get an activation function by name.ztorch.nn.modules..r#   identityActivation function  is not supported.)lower
startswithsplitrx   Identity_ACTIVATION_REGISTRYre   )r1  activation_names     r   
get_act_fnr=  T  s    ##%%K122 &%++C004j((;== %...QQQQRRR,,r   c                      t                      S r   r   r   r   r   r   r   f  
    
 r   c                      t                      S r   )r=   r   r   r   r   r   g  r@  r   c                      t                      S r   r?  r   r   r   r   r   h  s
     r   c                      t          | i |S r   )r   )argskwargss     r   r   r   i  s    _d-Mf-M-M r   )r   rI   geglu	swigluoaic                 |    |                                  } | t          vrt          d| d          t          |          S )z=Get an activation-and-mul (i.e. SiluAndMul) function by name.r5  r6  )r7  _ACTIVATION_AND_MUL_REGISTRYre   )r1  s    r   get_act_and_mul_fnrJ  n  sD    ##%%K666QQQQRRR'44r   )+r6   r   r   torch.nnrx   torch.nn.functionalry   r&   vllm.distributedr   r   r   vllm.loggerr   vllm.model_executor.custom_opr   vllm.model_executor.utilsr   vllm.platformsr	   vllm.utils.collection_utilsr
   r3   rg   registerr   r=   rS   r[   r   r   r   r   r   r   r   r  r  r;  r   r=  rI  rJ  r   r   r   <module>rT     s	   # "                          
 $ # # # # # 2 2 2 2 2 2 6 6 6 6 6 6 + + + + + + 0 0 0 0 0 0	X		 $%%" " " " "H " " &%"L >"") ) ) ) ) ) ) #")Z >""! ! ! ! ! ! ! #"!P ())6& 6& 6& 6& 6&x 6& 6& *)6&t >""<7 <7 <7 <7 <7 <7 <7 #"<7@ &''E E E E Eh E E ('E> :    h   6 ;    x    4 <         ! @ 7& & & & &H & & &" 7n* n* n* n* n*H n* n* n*b'( '( '( '( '(ry '( '( '(T  x!!''%%	) 	) "!00!!))!!'' )   4-C -BI - - - -   (x$$$$%%MM	     5C 5BI 5 5 5 5 5 5r   