PiFLUddlmZmZmZmZddlZddlmZddlm Z m Z ddl m Z m Z ejjZejjZejjZiZeeefed< d&dejd ejd ejd ejd ejd eejdeejdedejfdZdZdZeejjejjejjejjejj ej!jgd'dZ"eej#jgd'dZ$eej%jej&j'gd'dZ(eej)jgd'dZ*eej+jgd'dZ,eej-jgd'dZ.eej/j0gd'dZ1de de fdZ2eej3jej4jgd'dZ5eej6jgd'dZ7eej8jgd'd Z9eej:jgd'd!Z;eej<jej<jgd'd"Z=eej>jej>jgd'd#Z?eej@jgd'd$ZAeejBjgd'd%ZCdS)()AnyDictOptionalTupleN)tree_map)Float8TrainingTensorchoose_scaled_mm_config) is_row_majorpad_tensor_for_matmulFLOAT8_OPS_TABLEFa_dataa_scaleb_datab_scale output_dtype output_scalebiasuse_fast_accumreturnc V|}|} d} |j|jddfko|jd|jdfk} | r1|s/|| z} |d}|d} |} | tjtjfvr| r tj}d} |tjkr|} d}tj|||| ||||}| || z}| || z }| tjtjfvr| r|| }|S)z This is the unwrapped version of addmm_float8, which does not take in Float8TrainingTensors as inputs. This is used to standardize the logic between subclassed and non subclassed versions of the linear module. Nr)scale_ascale_br scale_result out_dtyper) reciprocalshapenew_onestorchfloat16float32bfloat16 _scaled_mmto)r rrrrrrra_inverse_scaleb_inverse_scalepost_inverse_scaleis_rowwise_scaling orig_dtype post_biasoutputs m/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/float8/float8_ops.pyaddmm_float8_unwrappedr.so((**O((**O 6<?A*>>7=  QUD 7.7->)22266)22266JemU]3338J3~ Iu}$$    !%   F%$$)emU]3338J3:&& McLt|jdvs J|ddS)N)rrz+ with axiswise scaling is not supported yet)lenr)aten_opscales r-_assert_tensorwise_scaler4^s9 EKF""" >>> #""""r/cfd}|S)z(Register aten ops to the float8 op tablecD]:}|tvr%td|dt|j|t|<;|S)Nz Float8 op z is already registered to )r RuntimeError__name__)funcopaten_opss r- decoratorzimplements..decoratorisb ( (B%%%"^^^?OPR?S?\^^$( R  r/r)r;r<s` r- implementsr=fs$ r/ct||dj||djg|ddRi|}t||dj|dj|dj|djSNrr)r4_scale_datar _orig_dtype_linear_mm_config_gemm_input_role)r2argskwargsnew_datas r-float8_desugar_oprHusWd1gn555wtAw}:tABBx:::6::H  Q Q Q! Q   r/c||djg|ddRi|}||djg|ddRi|}t|||dj|dj|djSr?)rAr@rrBrCrD)r2rErFrG new_scales r- float8_desugar_data_and_scale_oprKs wtAw}:tABBx:::6::HQ<abb<<48>>>v>>#  GN G  G % G $ G !     47>   1$$ $777?L 9~~ 1  wqw <.make_float8s?#  GN G  G % G $    r/)rAr4r@maprX)r2rErFnew_data_tensorsrbouts ` r- float8_splitrfs~wtAw}BtABBxBBB6BBWd1gn555      k+ , ,C 99r/c|d}|dj}|dj}|dj}|djj}|dj}g} |D]} t | ts Jd| j|ks Jd| j|us Jd| j|us Jd| jj|ks Jd| j|us Jdt|| j| | j tj || g|ddRi|} | |} t| ||||S) Nrz7Expecting all chunks to be of type Float8TrainingTensorz,Expecting all chunks to be of the same dtypezCExpecting all chunks to have thee same scale as a result of a splitzGExpecting all chunks to have thee same mm config as a result of a splitzCExpecting all chunks to be of the same dtype as a result of a splitzLExpecting all chunks to have the same gemm_input_role as a result of a splitr) rBr@rCrAdtyperD isinstancerr4appendviewr uint8) r2rErFchunked_tensorsr*r3 mm_config fp8_dtypegemm_input_role chunk_datachunkrGs r- float8_catrss377O #/J A  %E"4I"(.I%a(9OJ 99%!566  E  6 J... :/..|u$$$ Q%$$&)333 U433{ I--- Q.--%888 Z988 !%,777%+**5;778888wz7DH77777H}}Y''H %Y X XXr/ct||djd}t||}t||}||i|S)a)Be careful with this function, this is a "fallback" op that casts the output of the op to the original precision. And performs the op. We currently need this to support the backward for admmm bias. "addmm" -> out "hp_gradBias" <-"sum" <- "identity" <- gradOut <- "hp_gradOut" rcXt|tr|S|SN)rirto_original_precision)xs r-unwrapz!float8_cast_up_op..unwrap8s, a- . . -**,, ,r/)r4r@r)r2rErFrynew_args new_kwargss r-float8_cast_up_opr|-s^Wd1gn555 %%H&&))J 7H + + ++r/abc|j}|j}|j}t|j|j|j|j}|jr|jd|jdks@Jd|jdd|jdt|d}t|d}t| s| }t| r8| }|j}|j <|j 5| |jddd}nB|j ;|j 4| |jddd}||||fS)Nrrz"Inner dims must match for mm, got z and )dimsrM)rAr@r rDrC pad_inner_dimsizer r stride contiguousrZrRrepeatrreshape)r}r~r rrscaled_mm_configrs r-preprocess_addmmrBs WFhG WF.     %7w||A!',,q//111 Xa X Xqw||TU X X211'vA666&vA666   ( (%""$$FMMOO$$-&&((**,,hG 1?#>..a1199"a@@  $)@..a1199!R@@ 7FG ++r/c |d}|d}t|trt|ts8Jdt|t|t ||\}}}}|j} t |j|j|j|j} | j rftj |j |jz |j |jz | St!||||| dd| j} | S)NrrzFExpecting both Float8TrainingTensor for mm inputs but found {} and {}rrr)rirformattyperrBr rDrCemulater mmrAfloatr@r%r.r) r2rErFr}r~r rrrrr tensor_outs r- float8_mmrksN QA QA a- . .: 44OVV Qa  (81'='=$FGVW=L.       x !(2AGMMOOah4NOORR    ( '6   J r/c t|dtjr6t|dtrt|dtsJ|d}|d}|d}t ||\}}}} |j} |j| ks Jdt|j|j |j|j } | j rktj |j |jz |j |jz | } | |zSt!|||| | d|| j} | S)NrrrWz"bias dtype must match output dtyper)rir TensorrrrBrhr rDrCrrrArr@r%r.r)r2rErFrr}r~r rrrrrrers r- float8_addmmrsl 47EL)) tAw 4 5 5 tAw 4 5 5 6 7D QA QA'71'='=$FGVW=L : % % %'K % % %.      hqw}}117==??QX3MNNQQ   Tz' '6   J r/cpt||dj|dj|djkSr?)r4r@rr2rErFs r-float8_is_same_sizers/Wd1gn555 7=DGM ))r/c tt|dtsJt|dkrd|vs Jd|dtjtjhvs Jdt|dj|dj|d|dj|dj |dj S)zThis gets called when running matmul under autocast when the input is a Float8TrainingTensor, presenting as a fp32 tensor. rrrhz%Only support dtype kwarg for autocastzKOnly support floating point conversion for autocast w/ Float8TrainingTensor) rirr1r r!r#rAr@rCrDrRrs r-autocast_to_copyrs d1g3 4 444 4 v;;!  6 1 1 1/!2 1 1 '?     U     Q  Qw Q! Q  Q   r/cRt||dj|d}t|tsJdt ||j}|}||g|ddRi|}t||j|j|j|j S)z+ override funcol with FP8 handling rz9expecting a Float8TrainingTensor for allgather but found rN) r4r@rirrrArrBrCrDr2rErF fp8_inputfp8_datafp8_outs r- allgather_fp8rsWd1gn555QI i!5 6 6UDOOUU 6H""$$Hgh4abb444V44G #"   r/ct||dj|d}t|tsJ|j}||g|ddRi|}t||j|j|j|jSr?)r4r@rirrArBrCrDrs r-wait_tensor_fp8rsWd1gn555QI i!5 6 666 6Hgh4abb444V44G #"   r/c|d}|d}t|tsJt|tsJt||dj|j|jksJ|j|jksJ|j|jksJ|j}|j}|||d|g|ddRi|}t||j|j|j|jS)NrrWr) rirr4r@rhrBrArCrD)r2rErFfp8_self fp8_valuesrfp8_values_datars r- index_put_fp8rsAwHaJ h 4 5 555 5 j"6 7 777 7XtAw~666 ?j/ / / / / >Z- - - - -  :#9 9 9 9 9~H &OghQN48NNNvNNG "!   r/c2|d}|d}t|tsSt|tr>|}t||j|||g|ddRi|St|trt|trt||j|j|jks Jd|j|jks Jd|j|jks Jd|jj|jjks Jd|j |j ks Jd||j|jg|ddRi|}t||j|j|j|j Std ) NrrrWz>>> N?>>z39?222 K322$(<<<< S=<<'$*ciE$qrr(EEEfEE#  K    "  !    TUUUr/)NNFrv)Dtypingrrrrr torch.utils._pytreer%torchao.float8.float8_training_tensorrr torchao.float8.float8_utilsr r opsrOc10d_functional_c10d_functionalr __annotations__rrhboolr.r4r= _unsafe_viewdefault as_stridedcloneslicefill_ScalarrrHdetachrKrZrPrQrUrkr^splitrfcatrssum dim_IntListr|rrmatmulraddmmr is_same_sizer_to_copyrall_gather_into_tensorr wait_tensorr index_put_rcopy_rrr/r-rs  .------------ ((((((LKKKKKKK y~)+9-#%$sCx.%%%,0#' ?? L? \? L?\ ? + ? 5<( ? 5< ?? \????D???     !                           8 TY    333! 3l TZ    ! $ TX  !Y!Y!Y !YH TX ! "##,,,$#,(&,,&,1E&,&,&,&,R TW_dk1 23343D TZ  !!!!!"!!H T  & '((***)(*  T] " #$$%$. .6/7   , _ ( 02B2N2V WXX   YX   T_ $ %&&'&, TZ  !!-V-V-V"!-V-V-Vr/