
    Pi                         d dl Z d dlZd dlmZ d dlmc mZ dZdZd Z	 G d dej
        j                  Z G d dej
        j                  Z G d d	ej                  ZdS )
    Ng   Ј Bc                     dt          dt          |          z  |                                 dz
  z            z   }|                     d                              |          j        S )z,Return the value that is larger than q% of t   g{Gz?)roundfloatnumelviewkthvaluevalues)tqks      n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/sparsity/supermask.py
percentiler      sP    	E$q/QWWYY]3444A66"::q!!((    c                   >    e Zd ZdZed             Zed             ZdS )	GetSubnetSupermask STE functionc                     |                     t          t                    }t          ||dz            }t	          j        ||k     |                    |j                  |                    |j                            S )N)minmaxd   )clamp
SCORES_MIN
SCORES_MAXr   torchwheretodevice)ctxscoreszerosonessparsityclamped_scoresk_vals          r   forwardzGetSubnet.forward   sf    **EE>8c>::{U"EHHV]$;$;TWWV]=S=S
 
 	
r   c                     |d d d fS N )r    gs     r   backwardzGetSubnet.backward   s    $d""r   N__name__
__module____qualname____doc__staticmethodr'   r,   r*   r   r   r   r      sN          
 
 \
 # # \# # #r   r   c                   >    e Zd ZdZed             Zed             ZdS )	ApplyMaskr   c                     ||z  S r)   r*   )r    weightr!   s      r   r'   zApplyMask.forward&   s    r   c                 N    d x}}| j         d         r|}| j         d         r|}||fS )Nr   r   )needs_input_grad)r    grad_outputgrad_weightgrad_scoress       r   r,   zApplyMask.backward*   sA    $((k" 	&%K" 	&%KK''r   Nr-   r*   r   r   r4   r4   #   sN            \ ( ( \( ( (r   r4   c                   `     e Zd ZdZ fdZd Zd Ze	 	 d	d            Zed             Z	 xZ
S )
SupermaskLinearz Supermask class for Linear layerc                     t          t          |           j        |i | ddt          j        fd| j                                        D                       z  z
  }|| _        | j        |k    r@t          d| j	         d| d| j                                         d d           || _        | _
        d| _        t          j        t          j        fd	| j                                        D                       | 
          | _        t          j                            | j        t          j        d                     | | j        _        d S )Nr   c                 >    g | ]}t          j        |z            S r*   )mathceil).0r   	blocksizes     r   
<listcomp>z,SupermaskLinear.__init__.<locals>.<listcomp>=   s'    PPP49Q]33PPPr   zreducing sparsity from z to z'(maximum sparsity for layer with shape z and tile size )Fc                 t    g | ]4}t          d t          t          j        |z                                5S )r   )r   intr@   rA   )rB   wnrC   s     r   rD   z,SupermaskLinear.__init__.<locals>.<listcomp>J   s9    UUUBQDIb9n556677UUUr   )requires_grad   )a)superr=   __init__r@   prodr6   sizesparsity_levelprintr$   rC   sparsify_weightsnn	Parameterr   emptyr!   initkaiming_uniform_sqrtrI   )	selfrP   rC   
fixed_maskfixed_weightargskwargsmax_sparsity_level	__class__s	     `     r   rM   zSupermaskLinear.__init__7   sv    	.ot$$-t>v>>>	PPPPT[=M=M=O=OPPPQQQ
 -!333Q$-QQ=OQQi$+:J:J:L:Lii]fiii   #5D" %lKUUUU$+BRBRBTBTUUU  ).	
 
 
 	  	! === )5$4!!!r   c                 j   t                               | j        t          j        | j                  t          j        | j                  | j                  }| j        dk    rRt          | j	        j
                  D ]8\  }}|                    | j        |          }t          j        ||d|          }9|S )Nr   )dimr   )r   applyr!   r   
zeros_like	ones_likerP   rC   	enumerater6   shaperepeat_interleavenarrow)rY   subnetir   s       r   get_maskzSupermaskLinear.get_maskT   s    KT[))ODK((	
 
 >Q!$+"344 7 7111$.a1HHfaA66r   c                     |                                  }t                              | j        |          }t	          j        ||| j                  S r)   )rk   r4   rb   r6   Flinearbias)rY   xri   ws       r   r'   zSupermaskLinear.forwardc   s:    OODK00x1di(((r           r   c           
         t          |t          j        j                  sJ t	          ||dd|j        |j        |j        du                              |j	        j
        |j	        j                  }|j	        j                            |j	        j                   |j        )|j        j                            |j        j                   |S )zU
        Main entrypoint for creating a SupermaskLinear from a Linear layer.
        FNro   r   dtype)
isinstancer   rS   Linearr=   in_featuresout_featuresro   r   r6   r   rv   datacopy_)clsrn   rP   rC   supermask_linears        r   from_linearzSupermaskLinear.from_linearh   s     &%(/22222*D(
 
 
 "FM(0C"
D
D 	 	$**6=+=>>>;"!&,,V[-=>>>r   c                    |}t           j                            |j        |j        |j        du                              |j        j        |j        j	                  }|
                                }|j        j                            |j        |z             |j        )|j        j                            |j        j                   |S )zt
        Convert a SupermaskLinear to a Linear layer.
        Replaces the old sparsify_offline() function.
        Nrt   ru   )r   rS   rx   ry   rz   ro   r   r6   r   rv   rk   r{   r|   )r}   r~   rY   rn   masks        r   	to_linearzSupermaskLinear.to_linear   s      $& ! 
 
 "DK&dk.?"
@
@	 	 }}  t!34449 K""49>222r   )rr   r   )r.   r/   r0   r1   rM   rk   r'   classmethodr   r   __classcell__)r_   s   @r   r=   r=   4   s        **5 5 5 5 5:  ) ) )
  	      [ 2   [    r   r=   )r@   r   torch.nnrS   torch.nn.functional
functionalrm   r   r   r   autogradFunctionr   r4   rx   r=   r*   r   r   <module>r      s                    

) ) )# # # # #' # # # ( ( ( ( (' ( ( ("` ` ` ` `bi ` ` ` ` `r   