
    Pi-                        d dl Z d dl mZ d dl mZ d dlmZmZmZ ddlmZm	Z	m
Z
mZmZ ddlmZ  G d d	ej                  Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdddZ G d dej                  Zd ZeZ G d dej                  Z G d dej                  ZdS )    N)Tensor)nn)DynamicPositionBiasLocalMHAFeedForward   )ChannelNormConv1dLinearGRNSnake1d)
FirstBlockc                   X     e Zd Z	 	 	 	 	 	 	 	 	 	 	 d fd		ZddZe	 dd            Z xZS )
LocalTrans      T@                 Fc                    t                                                       t          j        g           | _        || _        |
 | _        | j        rd nt          |dz  |          | _        t          |          D ]c}| j        
                    t          j        t          |||||| j        dd | j        d|d          t          |||	          g                     dd S )N   )dimheadsFT)r   dim_headr   dropoutcausalwindow_sizeuse_xposxpos_scale_baseuse_rotary_pos_embprenorm
qk_rmsnormexact_windowsize)r   multr   )super__init__r   
ModuleListlayersr   r!   r   dynamic_pos_biasrangeappendr   r   )selfr   depthr   local_attn_window_sizer   r   ff_multattn_dropout
ff_dropoutuse_dynamic_pos_biasr#   _	__class__s                r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/neucodec/codec_encoder_distill.pyr'   zLocalTrans.__init__
   s    	mB''1&:": &@DD$??? 	 u 	 	AK  #%-"'$0#)(,(8%*,0/3/F$('1-2   $':NNN    	 	    Nc                     | j         rd n"|                     | j        | j        dz            }| j        D ]$\  }} ||||          |z   } ||          |z   }%|S )Nr   )mask	attn_bias)r!   r*   r   r)   )r-   xr9   r:   attnffs         r6   forwardzLocalTrans.forward;   s     &ODD&&t'79IA9MNN 	
  	 	HD"QTY777!;A1	AAr7      r      c           
      .     | |||dz  ddd||          S )Nr   r   T)r   r.   r   r   r0   r   r/   r3    )clsfeature_dimr.   local_window_sizer3   s        r6   builderzLocalTrans.builderG   s:     s A%#4!5	
 	
 	
 		
r7   )r   r   Tr   r   r   r   r   r   FFNr?   r   r@   F)__name__
__module____qualname__r'   r>   classmethodrF   __classcell__r5   s   @r6   r   r   	   s         ""/ / / / / /b
 
 
 
 SX
 
 
 [
 
 
 
 
r7   r   c                   .     e Zd Z	 	 	 	 d fd	Zd Z xZS )LocalEncoderr?   r   r@   Fc                     t                                                       t                              ||||          | _        d S )N)rD   r.   rE   r3   )r&   r'   r   rF   local_trans)r-   rD   r.   rE   r3   r5   s        r6   r'   zLocalEncoder.__init__X   sJ     	%--#/!5	 . 
 
r7   c                 ^    |                     ddd          }|                     |          }|S )zl
        Args:
            feature: (B, C, T)
        Returns:
            local_feature: (B, T, C)
        r   r   r   )permuterR   )r-   features     r6   r>   zLocalEncoder.forwardg   s1     //!Q**""7++r7   rH   rI   rJ   rK   r'   r>   rM   rN   s   @r6   rP   rP   W   sZ         "
 
 
 
 
 
	 	 	 	 	 	 	r7   rP   c                   (     e Zd Z	 d fd	Zd Z xZS )	DownTransr?   r@   r   c                     t                                                       ||z  dk    sJ || _        || _        t	          j        |f||d|| _        t          ||||          | _        d S )Nr   rE   r.   kernel_sizestride)	r&   r'   rD   compress_rater   rF   transr
   
down_layer)r-   rD   r   r^   r.   kwargsr5   s         r6   r'   zDownTrans.__init__t   s     	]*a////&*'
+6e
 
GM
 

 !-
 
 
r7   c                     |                      |          }|                     |                    ddd                                        ddd          }|S Nr   r   r   )r_   r`   rT   r-   r;   s     r6   r>   zDownTrans.forward   sI    JJqMMOOAIIaA..//771a@@r7   )r?   r@   r   r   rV   rN   s   @r6   rX   rX   s   sO        GH
 
 
 
 
 
      r7   rX   c                   0     e Zd Z	 	 	 	 	 d fd	Zd Z xZS )	CompressedLocalEncoderWithCacher?   r@   r      r   c                 z   t                                                       || _        || _        || _        ||z   | _        t          j        t          j	        d| j        | j        z  |                    | _
        t          |f| j        |z  |dd|| _        t          j        |f| j        |dz
  d|| _        d S )Nr   r   )r   r^   r.   rZ   )r&   r'   rE   
cache_sizer^   trans_window_sizer   	Parametertorchrandncache_tokenrX   
down_transr   rF   rR   )r-   rD   rE   r^   ri   r.   ra   r5   s          r6   r'   z(CompressedLocalEncoderWithCache.__init__   s     	!2$*!2Z!?<K4?T-??MM
 
 $
.>'	
 

 
 
 &-
"4!)
 
 	
 
r7   c                 d   |                     ddd          }t          j        || j        | j        z  d          }| j                            |j        d         dd          t          j        fd|D             d          }| 	                    |          }| 
                    |          }|S )Nr   r   r   )r   c                 "    g | ]}|fD ]}|S rB   rB   ).0fsfrn   s      r6   
<listcomp>z;CompressedLocalEncoderWithCache.forward.<locals>.<listcomp>   sH             r7   )rT   rl   splitrE   r^   rn   expandshapecatro   rR   )r-   rU   split_featurern   s      @r6   r>   z'CompressedLocalEncoderWithCache.forward   s    //!Q**T+d.@@a
 
 
 &--gmA.>BGG)   '   

 

 

 //'**""7++r7   )r?   r@   r   rg   r   rV   rN   s   @r6   rf   rf      s^          
  
  
  
  
  
D      r7   rf   c                   *     e Zd ZdZd fd	Zd Z xZS )	ConvUnitz<
    Args:
        dim (int): Number of input channels.
    TFr      c                    t                                                       |dz
  |z  }t          |||||dz  |          | _        |rt	          |d          nt          j                    | _        t          |d|z            | _	        |rt          d|z  d          | _        nt          j                    | _        t          d|z            | _        t          d|z  |          | _        d S )Nr   r   )r\   dilationpaddinggroupschannels_lastdata_formatr   )r&   r'   r
   dw_convr	   r   Identitynormr   pw_conv1r   actGELUr   grnpw_conv2r-   r   	snake_actr   r   r\   	total_padr5   s          r6   r'   zConvUnit.__init__   s     1_0	#N
 
 
 >BTK9999r{}} 		 S
 
  	!q3wODDDDHHwyyDHq3w<<q3w,,r7   c                 ^   |                      |          }|                    ddd          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                    ddd          }|S rc   )r   rT   r   r   r   r   r   rd   s     r6   r>   zConvUnit.forward   s    LLOOIIaAIIaLLMM!HHQKKHHQKKMM!IIaAr7   TFr   r~   )rI   rJ   rK   __doc__r'   r>   rM   rN   s   @r6   r}   r}      sV         
- - - - - -4	 	 	 	 	 	 	r7   r}   c                   R     e Zd Z	 ddej        dedef fdZdefdZ	d	efd
Z
 xZS )Residualr   Tmodule	drop_probscale_by_keepc                     t                                                       d|cxk    rdk     sn J || _        || _        || _        d S )Nr   r   )r&   r'   r   r   r   )r-   r   r   r   r5   s       r6   r'   zResidual.__init__   s\     	I!!!!!!!!!!"*r7   x_sidec                    | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }|                    |                              |          }| j        r|                    |           ||z  S )Nr   r   r   )r   )r   trainingry   ndim	new_empty
bernoulli_r   div_)r-   r   	keep_probry   	keep_masks        r6   	drop_pathzResidual.drop_path   s    >S   M&	a"TV[1_%==$$U++66yAA	 	&NN9%%%	!!r7   r;   c                 `    |                      |          }|                     |          }||z   S rG   )r   r   )r-   r;   r   s      r6   r>   zResidual.forward  s,    Q''6zr7   )r   T)rI   rJ   rK   r   Modulefloatboolr'   r   r   r>   rM   rN   s   @r6   r   r      s        OS+ +i+,1+HL+ + + + + +" " " " "        r7   r   r   )	drop_ratec                 8    t          t          |i ||           S )Nr   )r   r}   )r   argsra   s      r6   <lambda>r   
  s'    hdf7 7 7 r7   c                   &     e Zd Zd fd	Zd Z xZS )
LegacyUnitTFr   r~   c                 F   t                                                       |s
J d            |dk    s
J d            |dz
  |z  }t          j        t	          |          t          |||||dz            t	          |          t          ||d                    | _        d S )Nz'LegacyUnit only supports snake_act=TrueFz#LegacyUnit only supports norm=Falser   r   )r\   r   r   )r\   )r&   r'   r   
Sequentialr   r
   blockr   s          r6   r'   zLegacyUnit.__init__  s    CCCCCyu}}}C}}} 1_0	]CLL'!!Q   CLL3+++
 



r7   c                 ,    |                      |          S rG   )r   rd   s     r6   r>   zLegacyUnit.forward"  s    zz!}}r7   r   rV   rN   s   @r6   r   r     sL        
 
 
 
 
 
$      r7   r   c                  8    t          t          | i |d          S )Nr   r   )r   r   )r   ra   s     r6   r   r   &  s'    X3. . . r7   c                   J     e Zd Z	 	 	 	 	 	 	 dded	ed
ededef
 fdZd Z xZS )Encoderr   r   r   r   r   r   r   r   r   r       r   r?      r   r   FTrD   stridesdepthsdimsdrop_path_ratec           	         t                                                       t          d                   g}d t          j        d|t          |                    D             dt          d d         dd          ||          D ]\  }	}
}t          j        fdt          |          D              }t          j        t          |	|
|
          rt          |	d          nt          j                              }|||gz  }|z  |t          j        fd	t          |d                   D              t          d         |d
d          gz  }t          j        | | _        d S )Nr   c                 6    g | ]}|                                 S rB   )item)rs   r;   s     r6   rv   z$Encoder.__init__.<locals>.<listcomp>?  s-     
 
 
AFFHH
 
 
r7   rq   r   c                 H    g | ]}t          |z                       S )r   r   r   r   BaseUnit)rs   jcurdrop_path_ratesi_duse_normuse_snake_acts     r6   rv   z$Encoder.__init__.<locals>.<listcomp>E  sO         "1#'":"/%	    r7   r[   channels_firstr   c                 T    g | ]$}t          d          |z                      %S )rq   r   r   )rs   r   r   r   r   r   r   s     r6   rv   z$Encoder.__init__.<locals>.<listcomp>[  sS          H"1#'":"/%	    r7   rg   )r\   r   )r&   r'   r   rl   linspacesumzipr   r   r+   r
   r	   r   blocks)r-   rD   r   r   r   r   r   r   r   o_dr]   r.   stager`   r   r   r   r5   s       ` ``      @@@r6   r'   zEncoder.__init__.  s    	 tAw


 
#nQFLL
 
 
 '*49d122h'P'P 	 	#CfeM        #5\\  
E sCVFCCC#C-=>>>>[]]	 J uj))F5LCC 	M        #6":..  
 48[aCCC
 	
  mV,r7   c                 ,    |                      |          S rG   )r   rd   s     r6   r>   zEncoder.forwardk  s    {{1~~r7   )r   r   r   r   r   FT)	rI   rJ   rK   inttupler   r'   r>   rM   rN   s   @r6   r   r   -  s         %'- #;- ;-;- ;- 	;-
 ;- ;- ;- ;- ;- ;- ;-z      r7   r   c                   $     e Zd Z fdZd Z xZS )DistillCodecEncoderc                     t                                                       t          dddd          | _        t	          dddddd	
          | _        d S )Nr   )r   r   r   r   )r   r   r   r   )r   r   r?   r   )rD   r   r   r   i,     r   T)rD   rE   r^   ri   r.   r3   )r&   r'   r   encoderrf   
en_encoder)r-   r5   s    r6   r'   zDistillCodecEncoder.__init__p  si     #	
 
 
 :!!%
 
 
r7   c                 Z    |                      |          }|                     |          }|S rG   )r   r   rd   s     r6   r>   zDistillCodecEncoder.forward  s'    LLOOOOAr7   rV   rN   s   @r6   r   r   o  sG        
 
 
 
 
"      r7   r   )rl   r   r   local_attention.transformerr   r   r   distill_layersr	   r
   r   r   r   tconv.t_firstr   r   r   rP   rX   rf   r}   r   ResidualUnitr   ResidualLegacyUnitr   r   r   rB   r7   r6   <module>r      s]                R R R R R R R R R R E E E E E E E E E E E E E E % % % % % %K
 K
 K
 K
 K
 K
 K
 K
\    29   8    	   ,8 8 8 8 8bi 8 8 8v) ) ) ) )ry ) ) )X    ry   4 (+    
       .   ? ? ? ? ?bi ? ? ?D    ")     r7   