
    Pi              	       8   d dl Z d dlmZ d dlZd dlmZ d dlmc mZ d dl	m
Z
mZmZ d dlmZ d dlmZmZmZmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZ d d
lmZ  G d dej                   Z! G d dej                   Z"	 	 	 ddedeeef         de#de#fdZ$dS )    N)Union)	ReplicateSharddistribute_tensor)
DeviceMesh)ColwiseParallelPrepareModuleInputRowwiseParallelparallelize_module)Float8LinearConfig)convert_to_float8_training)Float8ColwiseParallelFloat8RowwiseParallelPrepareFloat8ModuleInput)MXLinearConfig)	quantize_c                   (     e Zd ZdZ fdZd Z xZS )FeedForwardzMLP based modelc                    t          t          |                                            t          j        ||dz  d          | _        t          j        ||dz  d          | _        t          j        |dz  |d          | _        d S )N   F)bias)superr   __init__nnLinearw1w2out_projselfsize	__class__s     z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/testing/training/dtensor_utils.pyr   zFeedForward.__init__#   sw    k4  ))+++)D$(777)D$(777	$(Du===    c                     t          j        |                     |                    |                     |          z  }|                     |          }|S N)Fsilur   r   r   r    xs     r#   forwardzFeedForward.forward)   s@    F4771::+MM!r$   )__name__
__module____qualname____doc__r   r+   __classcell__r"   s   @r#   r   r       sM        > > > > >      r$   r   c                   $     e Zd Z fdZd Z xZS )ToyModelc                 ~    t          t          |                                            t          |          | _        d S r&   )r   r3   r   r   ffnr   s     r#   r   zToyModel.__init__0   s2    h&&(((t$$r$   c                 ,    |                      |          S r&   )r5   r)   s     r#   r+   zToyModel.forward4   s    xx{{r$   )r,   r-   r.   r   r+   r0   r1   s   @r#   r3   r3   /   sG        % % % % %      r$   r3       Fmeshconfigcompileallgather_in_lowpc                 
   | j         }t          }t          |t                    rt          }t          |                              |          }t          j        |          } |||           t          j        |          }	 ||	|           t          j        |          }
 ||
|           |st          }t          }t          }nt          }t          }t          }t          |	|  |             |             |            d          }	t          |
|  |t!          d          t#                                 |             |             |t!          d          d          d          }
t          j        |          } |||           |s' |t!          d          t#                                }n' |t!          d          t#                      d	          }t          || | |             |             |t!          d          d          d          }|r<t%          j        |	          }	t%          j        |
          }
t%          j        |          }t%          j        d
|d
z  ||d          }t%          j        d
|d
z  ||d          }|                                }|                                }t-          |                                | t!          d          g          }t-          |                                | t!          d          g          } |	|          }|                    |            |
|          }|                    |            ||          }|                    |           t$          j                            ||           t$          j                            |                                |           t$          j                            |	j        j        j        j        |
j        j        j        j                   t$          j                            |	j        j        j        j        |
j        j        j        j                    ||          }|                    |           t$          j                            |                                |           t$          j                            |	j        j        j        j        |j        j        j        j                   t$          j                            |	j        j        j        j        |j        j        j        j                   d S )N)r9   )ffn.w1ffn.w2ffn.out_proj   )input_layoutsdesired_input_layoutsF)output_layoutsuse_local_output)r5   r=   r>   r?   r   )rA   rB   fwd_config_submodule_fqnr   )devicerequires_gradr   ) device_typer   
isinstancer   r   r3   tocopydeepcopyr   r
   r	   r   r   r   r   r   r   torchr:   randcloner   backwardtestingassert_closefull_tensorr5   r   weightgradr   )r8   r9   r!   r:   r;   rF   convert_model_func	toy_modeltoy_model_fp8tp_modelsp_modelcolwise_parallel_clsrowwise_parallel_clsprepare_input_cls	sp_model2prepare_inputx_fp32go_fp32x_fp32_tp_input
go_fp32_tpx_fp32_sp_input
go_fp32_sptp_outsp_out
global_outsp_out2s                             r#   &_test_lowp_mlp_tensor_parallelism_baserj   8   s    F 4&.)) '&!!&))IM),,M}V4444}Y''Hx////}Y''Hx////  5...444 "**,,**,,0022	
 	
 H "$$#Ahhikk   +*,,**,,00$Qxx%  		
 		
 H  i((Iy0000 

))(("+++
 
 

 *)(("+++%)
 
 
 # **,,**,,00$Qxx%  		
 	
 I  -=**=**M),,	Z4!8T&NNNFjD1Hd6OOOGllnnOJ'uQxxjIIO"7==??D588*EEJXo&&F
OOJXo&&F
OOJv&&J   	Mvz222	Mv1133Z@@@	Mx|5:HLO<R<WXXX	M$)8<+@+G+L   i((GZ   	Mw2244jAAA	M#Y]%5%<%A   
M$)9=+A+H+M    r$   )r7   FF)%rK   typingr   rM   torch.nnr   torch.nn.functional
functionalr'   torch.distributed._tensorr   r   r   torch.distributed.device_meshr   !torch.distributed.tensor.parallelr   r	   r
   r   torchao.float8r   "torchao.float8.float8_linear_utilsr   %torchao.float8.float8_tensor_parallelr   r   r   #torchao.prototype.mx_formats.configr   torchao.quantizationr   Moduler   r3   boolrj    r$   r#   <module>rz      s                          I I I I I I I I I I 4 4 4 4 4 4            . - - - - - I I I I I I         
 ? > > > > > * * * * * *    ")       ry    
#~ ~
~$n45~ 	~
 ~ ~ ~ ~ ~ ~r$   