
    Pi                     2   d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ 	 	 ddej        dej        j        dej        d	ej        j        d
ej        de	dedefdZdej        dej        dej        j        dej        d	ej        j        d
ej        fdZdS )    )ListN)Float8LinearConfigScalingType)(precompute_float8_dynamic_scale_for_fsdpF	ref_model	ref_optim
fsdp_model
fsdp_optim	local_inpconfig
precomputecompile_transformer_blockc	                    t          |                                |                                          D ]7\  }	}
|                     |	j        |
j        d|	j         d|
j                    8t	          d          D ]\}g }||f||ffD ]\  }}|                    |dz  dk               |                     ||                                                     |d                                          ||u rb|                                D ]M}|j        rDt          j
        |j                   |j                            t          j                               N|                                 ||u r)|r'|j        j        t"          j        u rt'          |           |                     |d         |d	         d
| d|d          d|d	                     ^d S )Nzref_param.requires_grad: z, fsdp_param.requires_grad: msg
      r   set_to_none   iter: , loss-ref: , loss-fp8: )zip
parametersassertEqualrequires_gradrange	zero_gradappendsumbackwarddist
all_reducegraddiv_get_world_sizestepcast_config_weightscaling_typer   DYNAMICr   )test_clsr   r   r	   r
   r   r   r   r   	ref_param
fsdp_paramiter_idxlossesmodeloptimparams                   x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/testing/training/fsdp2_utils.pycheck_parity_no_mpr6      s    "%Y%9%9%;%;Z=R=R=T=T!U!U 
 
	:#${I,C{{akay{{ 	 	
 	
 	
 	
 "II 
 
%''3j*5MN 	@ 	@LE5OOA):O<<<MM%%	**..001112J!!!	!!"--// ? ?E* ?
333
(;(=(=>>>JJLLL## $-:k>QQQ8???1I1IQQQvayQQfQiQQ 	 	
 	
 	
 	
)
 
    ref_model_bf16c                    t          d          D ]}g }||f||ffD ]\  }	}
|
                    |dz  dk               |                     |	|                                                     |d                                          |	|u rt          |                                |                                          D ]n\  }}t          j        |j	                   |j	        
                    t          j                               |j	                                        |_	        d |_	        o|
                                 t          |                                |                                          D ],\  }}|                                                    |           -|                     |d         |d         d| d|d          d	|d          
           d S )Nr   r   r   r   r   r   r   r   r   r   )r   r    r!   r"   r#   r   r   r$   r%   r&   r'   r(   floatr)   detachcopy_r   )r-   r   r8   r   r	   r
   r   r0   r1   r2   r3   
param_bf16
param_fp32s                r5   check_parity_bf16_mpr?   B   s    "II 
 
%'Y'$
 	6 	6LE5 OOA):O<<<MM%%	**..001112J!!!&&.1"--//1E1E1G1G/ / + +*J
 OJO444O(()<)>)>???&0o&;&;&=&=JO&*JOOJJLLL*-$$&&(A(A(C(C+ + 6 6&
J !!##))*55556 	1I1IQQQvayQQfQiQQ 	 	
 	
 	
 	
-
 
r7   )FF)typingr   torchtorch.distributeddistributedr$   torch.nnnntorchao.float8.configr   r   torchao.float8.fsdp_utilsr   Moduler3   	OptimizerTensorboolr6   r?    r7   r5   <module>rM      sm                                   O N N N N N &+,
 ,
y,
 {$,
 		,

 %,
 |,
 ,
 ,
  $,
 ,
 ,
 ,
^#
y#
 I#
 {$	#

 	#
 %#
 |#
 #
 #
 #
 #
 #
r7   