
    Pih                        d dl Z d dlmZ d dlmc mZ d dl mZ d dlmZ d Z	 e	e j
                  ZddZ eej        dd          Z eej        dd          Z G d d	ej                  Z G d
 dej                  Ze j        j        d             Z G d dej                  Ze j        j        d             Z G d dej                  ZdS )    N)Tensor)weight_normc                 4    t          j        |           j        S N)torchfinfoeps)	data_types    k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/neucodec/distill_layers.pyget_epsr      s    ;y!!%%    Tc                       fd}|S )Nc                       | i |}rKt           j                            |j        d           t           j                            |j        d           rt          |          }|S )Ng{Gz?)stdr   )nninittrunc_normal_weight	constant_biasr   )argskwargsnn_instanceinit_weightnn_classnorm_weights      r   
nn_builderznn_wrapper.<locals>.nn_builder   ss    h/// 	3G!!+"4$!???Gk.222 	3%k22Kr    )r   r   r   r   s   ``` r   
nn_wrapperr      s0           r   )r   r   c                   R     e Zd Z	 ddej        dedef fdZdefdZ	d	efd
Z
 xZS )Residual        Tmodule	drop_probscale_by_keepc                     t                                                       d|cxk    rdk     sn J || _        || _        || _        d S )Nr      )super__init__r#   r$   r%   )selfr#   r$   r%   	__class__s       r   r)   zResidual.__init__!   s\     	I!!!!!!!!!!"*r   x_sidec                    | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }|                    |                              |          }| j        r|                    |           ||z  S )Nr"   r'   r   r'   )r$   trainingshapendim	new_empty
bernoulli_r%   div_)r*   r,   	keep_probr0   	keep_masks        r   	drop_pathzResidual.drop_path*   s    >S   M&	a"TV[1_%==$$U++66yAA	 	&NN9%%%	!!r   xc                 `    |                      |          }|                     |          }||z   S r   )r#   r7   )r*   r8   r,   s      r   forwardzResidual.forward4   s,    Q''6zr   )r"   T)__name__
__module____qualname__r   Modulefloatboolr)   r   r7   r:   __classcell__r+   s   @r   r!   r!       s        OS+ +i+,1+HL+ + + + + +" " " " "        r   r!   c                   4     e Zd ZdZedf fd	Zd Zd Z xZS )GRNa.  GRN (Global Response Normalization) layer
    Which supports two data formats: channels_last (default) or channels_first.
    Channels_last corresponds to inputs with shape (batch_size, Sequence, channels)
    while channels_first corresponds to inputs with shape (batch_size, channels, Sequence).
    channels_lastc                 R   t                                                       || _        || _        |dk    r`t	          j        t          j        d|                    | _        t	          j        t          j        d|                    | _	        d| _
        nx|dk    r`t	          j        t          j        |d                    | _        t	          j        t          j        |d                    | _	        d| _
        nt          d|           t          j        |          | _        d S )NrE   r'   channels_firstzUnsupported data_format: )r(   r)   
n_channelsdata_formatr   	Parameterr   zerosgammabetachannel_dim
ValueErrortensorr	   r*   rI   r	   rJ   r+   s       r   r)   zGRN.__init__A   s    $&/))ek!Z&@&@AADJU[J%?%?@@DI!D,,,ek*a&@&@AADJU[Q%?%?@@DI DFFFGGG<$$r   c                     t          j        |dddgd          }||                    | j        d          | j        z   z  }| j        ||z  z  | j        z   |z   S )N   r'   T)pdimkeepdim)rV   rW   )r   normmeanrO   r	   rM   rN   )r*   r8   g_xn_xs       r   r:   zGRN.forwardQ   s_    jaaVT:::SXX$"2DXAADHLMzQW%	1A55r   c                 @    | j         j         d| j         d| j         dS Nz(n_channels=z, )r+   r;   rI   rJ   r*   s    r   __repr__zGRN.__repr__V   +    .)]]t]]$JZ]]]]r   	r;   r<   r=   __doc__EPSr)   r:   ra   rA   rB   s   @r   rD   rD   :   ss          (+ % % % % % % 6 6 6
^ ^ ^ ^ ^ ^ ^r   rD   c                     d}| ||z                                    t          j        || z                                d          z  z   } | S )Ng      >rT   )
reciprocalr   sinpow)r8   alphar	   s      r   snakerk   [   sI     !C	US[$$&&519)=)=)A)A!)D)DDDAHr   c                   &     e Zd Zd fd	Zd Z xZS )Snake1drH   c                 &   t                                                       |dk    r/t          j        t	          j        d|d                    | _        d S |dk    r/t          j        t	          j        dd|                    | _        d S t          )NrH   r'   rE   )r(   r)   r   rK   r   onesrj   NotImplementedError)r*   channelsrJ   r+   s      r   r)   zSnake1d.__init__d   s|    ***ejHa&@&@AADJJJO++ejAx&@&@AADJJJ%%r   c                 ,    t          || j                  S r   )rk   rj   )r*   r8   s     r   r:   zSnake1d.forwardm   s    Q
###r   )rH   )r;   r<   r=   r)   r:   rA   rB   s   @r   rm   rm   c   sL        & & & & & &$ $ $ $ $ $ $r   rm   c                     |                      dd          }| |z
                      d                               dd          }| |z
  t          j        ||z             z  } || z  |z   } | S )Nr'   T)rW   rT   )rY   ri   r   sqrt)r8   r   r   r	   uss         r   channel_normrw   q   sl    	q$A	
QAAt,,A	
Q%*QW%%%A
TAHr   c                   4     e Zd ZdZedf fd	Zd Zd Z xZS )ChannelNorma  ChannelNorm that supports two data formats: channels_last (default) or channels_first.
    Channels_last corresponds to inputs with shape (batch_size, ..., channels)
    while channels_first corresponds to inputs with shape (batch_size, channels, ...).
    rE   c                 B   t                                                       || _        || _        t	          j        t          j        |                    | _        t	          j        t          j	        |                    | _
        t          j        |          | _        d S r   )r(   r)   rI   rJ   r   rK   r   ro   r   rL   r   rQ   r	   rR   s       r   r)   zChannelNorm.__init__   ss    $&l5:j#9#9::LZ!8!899	<$$r   c                 j   | j         dk    rXdt          |j        dd                    z  }t          | | j        j        dg|R   | j        j        dg|R  | j                  S | j         dk    r?t          j	        || j
        f| j        | j        | j                                                  S t          )NrH   r.   rT   rG   rE   )rJ   lenr0   rw   r   viewr   r	   F
layer_normrI   itemrp   )r*   r8   extend_dimss      r   r:   zChannelNorm.forward   s    ///QWQRR[!1!11K  2k222	r0K000	   00<DO%t{DItx}}  
 &%r   c                 @    | j         j         d| j         d| j         dS r]   r_   r`   s    r   ra   zChannelNorm.__repr__   rb   r   rc   rB   s   @r   ry   ry   z   ss         
 (+ % % % % % %& & &$^ ^ ^ ^ ^ ^ ^r   ry   )TT)r   torch.nnr   torch.nn.functional
functionalr~   r   torch.nn.utils.parametrizationsr   r   float32re   r   Conv1dLinearr>   r!   rD   jitscriptrk   rm   rw   ry   r   r   r   <module>r      s                         7 7 7 7 7 7& & & gem
 
 
 
 
BI4T	B	B	B	BI4T	B	B	B    ry   4^ ^ ^ ^ ^") ^ ^ ^B   $ $ $ $ $bi $ $ $   !^ !^ !^ !^ !^") !^ !^ !^ !^ !^r   