
    Pi&                        d dl Z d dlZd dlmZ d dlmc mZ  G d dej        j                  Z G d dej        j                  Z	 G d dej                  Z
 G d d	ej                  Z G d
 dej                  Z G d dej        j                  Zej        ddfdededededej        dedefdZ G d dej                  ZdS )    Nc                   0     e Zd Z	 d fd	ZddZd Z xZS )ToySingleLinearModelFc                     t                                                       || _        || _        t          j                            |||||          | _        d S N)biasdtypedevice)super__init__r   r	   torchnnLinearlinear1)self	input_dim
output_dimr   r	   has_bias	__class__s         w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/testing/model_architectures.pyr   zToySingleLinearModel.__init__   sT     	
xzf ' 
 
       c                 \    t          j        || j        j        | j        | j                  fS N)r   r	   r   randnr   in_featuresr   r	   r   
batch_sizes     r   example_inputsz#ToySingleLinearModel.example_inputs   5    K(j{	  
 	
r   c                 0    |                      |          }|S N)r   r   xs     r   forwardzToySingleLinearModel.forward(   s    LLOOr   Fr   __name__
__module____qualname__r   r   r%   __classcell__r   s   @r   r   r      se         
 
 
 
 
 

 
 
 
      r   r   c                   0     e Zd Z	 d fd	ZddZd Z xZS )ToyTwoLinearModelFc                    t                                                       || _        || _        t          j                            |||||          | _        t          j                            |||||          | _        d S r   )	r
   r   r   r	   r   r   r   r   linear2)r   r   
hidden_dimr   r   r	   r   r   s          r   r   zToyTwoLinearModel.__init__/   s}     	
xzf ' 
 
 x
v ' 
 
r   r   c                 \    t          j        || j        j        | j        | j                  fS r   r   r   s     r   r   z ToyTwoLinearModel.example_inputsC   r    r   c                 Z    |                      |          }|                     |          }|S r"   )r   r1   r#   s     r   r%   zToyTwoLinearModel.forwardM   s%    LLOOLLOOr   r&   r'   r(   r-   s   @r   r/   r/   .   se         
 
 
 
 
 
(
 
 
 
      r   r/   c                   <     e Zd Z	 d	 d fdZdej        fdZ xZS )	#ConvWithSharedWeightInExportedModel   r   returnNc                     t                                                       || _        t          j        |||||          | _        t          j        |          | _        t          j        d          | _	        d S )NT)inplace)
r
   r   n_chunksr   Conv2dconvBatchNorm2dbnReLUrelu)r   r;   in_channelsout_channelskernel_sizestridepaddingr   s          r   r   z,ConvWithSharedWeightInExportedModel.__init__T   sd     	 Ik<fgVV	...GD)))			r   c                    t          j        || j        d          }g }|D ]V}|                     |          }|                     |          }|                     |          }|                    |           Wt          j        |d          S )Nr   dim)r   chunkr;   r=   r?   rA   appendcat)r   r$   chunksoutputsrJ   outs         r   r%   z+ConvWithSharedWeightInExportedModel.forward]   s    Q1555 	  	 E))E""C''#,,C))C..CNN3ya((((r   )r7   r   r   )r8   N)r)   r*   r+   r   r   Tensorr%   r,   r-   s   @r   r6   r6   S   se        TU*	* * * * * *)EL ) ) ) ) ) ) ) )r   r6   c                   4     e Zd Zej        df fd	Zd Z xZS )LNLinearActivationModelsigmoidc                 N   t                                                       |                                }t          j                    t          j                    t          j                    t          j                    t          j                    t          j	                    t          j
                    d}||vrt          d|           t          j        |d          | _        t          j        ||d                              |          | _        ||         | _        d S )N)rA   rS   	leakyrelurelu6gelusilu	hardswishzUnsupported activation: F)elementwise_affiner   )r   )r
   r   lowerr   r@   Sigmoid	LeakyReLUReLU6GELUSiLU	Hardswish
ValueError	LayerNormlnr   tofc
activation)r   fc_dim1fc_dim2r   rh   activation_mapr   s         r   r   z LNLinearActivationModel.__init__i   s    %%''
GIIz||XZZGIIGII
 
 ^++D
DDEEE,w5AAA)GW5999<<5<II(4r   c                     |                      |          }|                     |          }|                     |          S r"   )re   rg   rh   r#   s     r   r%   zLNLinearActivationModel.forward~   s2    GGAJJGGAJJq!!!r   r)   r*   r+   r   bfloat16r   r%   r,   r-   s   @r   rR   rR   h   sS        /4~) 5 5 5 5 5 5*" " " " " " "r   rR   c                   T     e Zd Zd	dedef fdZd Zdej        dej        fdZ	 xZ
S )
RMSNormh㈵>rI   epsc                     t                                                       || _        t          j        t          j        |                    | _        d S r"   )r
   r   rr   r   	Parameterr   onesweight)r   rI   rr   r   s      r   r   zRMSNorm.__init__   s=    l5:c??33r   c                 p    |t          j        t          j        ||z  dd          | j        z             z  S )NT)rI   keepdim)r   rsqrtmeanrr   r#   s     r   _normzRMSNorm._norm   s3    5;uz!a%RFFFQRRRRr   r$   r8   c                     |                      |                                                              |          }|| j        z  S r"   )r|   floattype_asrv   )r   r$   outputs      r   r%   zRMSNorm.forward   s6    AGGII&&..q11##r   )rq   )r)   r*   r+   intr~   r   r|   r   rP   r%   r,   r-   s   @r   rp   rp      s        4 4C 4e 4 4 4 4 4 4
S S S$ $%, $ $ $ $ $ $ $ $r   rp   c                   6     e Zd Zddej        f fd	Zd Z xZS )TransformerBlock      c                    t                                                       || _        || _        ||z  | _        t
          j                            |d|z  d                              |          | _	        t
          j                            ||d                              |          | _
        || _        t          ||z            | _        t
          j                            || j        d                              |          | _        t
          j                            | j        |d                              |          | _        t!          |                              |          | _        t!          |                              |          | _        t
          j                                        | _        d S )Nr7   Fr[   )r
   r   r2   	num_headshead_dimr   r   r   rf   qkvproj	mlp_ratior   mlp_hidden_dimmlp_fc1mlp_fc2rp   norm1norm2r`   rh   )r   r2   r   r   r   r   s        r   r   zTransformerBlock.__init__   s`   $""i/ 8??:q:~E?JJMMeTTHOOJ
OGGJJ5QQ	 #!*y"899xz43FUSSVV
 
 xt':JUSSVV
 

 Z((++E22
Z((++E22
  (--//r   c                    |j         \  }}}|}|                     |          }|                     |          }|                    ||d| j        | j                  }|                    ddddd          }|\  }}}	|                    || j        z  || j                  }|                    || j        z  || j                  }|	                    || j        z  || j                  }	||                    dd          z  d| j        d	z  z  z  }
t          j	        |
d
          }
|
|	z  }|                    || j        || j                  }|                    dd                              ||| j
                  }|                     |          }||z   }|}|                     |          }|                     |          }|                     |          }|                     |          }||z   }|S )Nr7      r   r   r   rx   g      ?g      ?rH   )shaper   r   reshaper   r   permute	transposer   softmaxr2   r   r   r   rh   r   )r   r$   r   seq_len_residualr   qkvattns              r   r%   zTransformerBlock.forward   s   !"
GQ JJqMM hhqkkkk*gq$.$-PPkkq!Q
 
 1a IIj4>17DMJJIIj4>17DMJJIIj4>17DMJJ AKKB'''C4=#3E,FG}Tr*** 1H IIj$.'4=IIKK1%%j'4?KK IIaLLqL JJqMMLLOOOOALLOOqLr   rm   r-   s   @r   r   r      sU        -.!5> * * * * * *6, , , , , , ,r   r   cudarA   
model_typemr   nhigh_precision_dtyper	   rh   c                 2   | dk    r0t          ||||          }|                    |          d         }nd| v rt          j        d|           }	|	r*|	                    d          r|	                    d          nd}t          ||||	                              |          }t          j        ||||          }nX| d
k    r@t          |dd|                              |          }t          j        |d|||          }nt          d|            ||fS )ab  Create a model and input data for benchmarking.

    Args:
        model_type (str): type of the model to be created
        batch_size (int): batch size of the input data
        device (str): device to run the model on
        high_precision_dtype (torch.dtype): data type of the model
        m, k, n (int): dimensions of the model and input data
    linear)r	   r   )r   r   	ln_linearzln_linear_?(\w+)?r   rA   )rh   transformer_blockr   r   )r   r   r      zUnknown model type: )r   r   researchgrouprR   rf   r   r   r   rc   )
r   r   r   r   r   r	   rh   model
input_datamatchs
             r   create_model_and_input_datar      sI   $ X$Q&@TUUU))Q)77:

	
	"	"	.
;;',KQKU[[^^^V
'q&:
 
 

"V** 	 [Af<PQQQ

	*	*	* a/C
 
 

"V** 	 [B&@TUUU

<
<<===*r   c                   p     e Zd Zdedededej        dej        ddf fdZd	ej        dej        fd
Z	 xZ
S )LlamaModelsLlama4Expertsnum_local_expertsrI   r2   r   r	   r8   Nc           	         t                                                       || _        || _        t	          j        t          j        |||||                    | _        t	          j        t          j        |||||                    | _	        t	          j        t          j        |||||                    | _
        d S r   )r
   r   r   rI   r   rt   r   r   w1w2w3)r   r   rI   r2   r   r	   r   s         r   r   z!LlamaModelsLlama4Experts.__init__  s     	!2 "K!  !
 !
 !#K!  !
 !
 !#K!  !
 !
r   routed_in_egDc                 >   | j         }| j        }|                    |d|          }t          j        t          j        || j                            t          j        || j                  z  }t          j        || j	                  }|                    d|          }|S )Nrx   )
r   rI   viewFrX   r   bmmr   r   r   )r   r   eDx_egDmiddle_out_egFout_egDs          r   r%   z LlamaModelsLlama4Experts.forward3  s     "H""1b!,,	% 9 9::UYudg=V=VV)NDG44,,r1%%r   )r)   r*   r+   r   r   r   r	   r   rP   r%   r,   r-   s   @r   r   r     s        )
)
 )
 	)

 {)
 )
 
)
 )
 )
 )
 )
 )
V| 
       r   r   )r   r   torch.nnr   torch.nn.functional
functionalr   Moduler   r/   r6   rR   rp   r   rn   strr   r   r   r    r   r   <module>r      s    
			                    58?   @" " " " " " " "J) ) ) ) )") ) ) )*" " " " "bi " " "8$ $ $ $ $bi $ $ $H H H H Hux H H H` ).& &&
& & 	&
  +& & & & & &T9 9 9 9 9ry 9 9 9 9 9r   