PiW UddlmZmZmZmZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZddlmZmZddlmZmZmZmZmZmZm Z gd Z!d Z"da#ee$e%d <d Z&d Z'iZ(eee$eee$eee$e)fffe%d<GddeZ*GddZ+d3dZ,dej-dee)fdZ.dZ/dZ0ej1fdZ2dZ3ej1fdZ4ej1fdZ5ddej6e j7ddfd Z8ej6fd!Z9d"Z:dde j7fd#Z;dde j7fd$Zddej1e j?dfd'Z@ddej1e j?fd(ZAej1ej1dfd)ej-d*ejBd+ejBd,eeCdej-f d-ZDd.ZEd/ee)d0fd1edee)d0ffd2ZFdS)5)DictListOptionalTupleN)TorchDispatchMode)int_scaled_matmul) MappingTypeZeroPointDomain)_choose_qparams_affine_dont_preserve_zero_choose_qparams_affine_tinygemm _dequantize_affine_no_zero_point_dequantize_affine_tinygemm_quantize_affine_no_zero_point_quantize_affine_tinygemmchoose_qparams_affinedequantize_affinequantize_affine)check_cpu_versioncheck_xpu_version) GranularityPerAxisPerBlockPerGroupPerRow PerTensorPerToken) compute_error%_quantize_activation_per_token_absmax$_quant_int8_dynamic_per_token_linear dynamically_quantize_per_channeldequantize_per_tensordequantize_per_channelget_groupwise_affine_qparamspack_tinygemm_scales_and_zeros unpack_tinygemm_scales_and_zeros-groupwise_affine_quantize_tensor_from_qparams/groupwise_affine_dequantize_tensor_from_qparams groupwise_affine_quantize_tensor"groupwise_affine_dequantize_tensorper_token_dynamic_quantget_group_qparams_symmetric"recommended_inductor_config_setterctj|}tj||z }dtj||z zS)N)torchlinalgnormlog10)xyPsPns n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/quantization/utils.pyrr?sI   1  B   1q5 ! !B  BG$$ $$_cur_fqncfd}|S)Nc adSN)r:)moduleinputfqns r8 forward_hookz'_get_logging_hook..forward_hookKs r9)r@rAs` r8_get_logging_hookrCJs$ r9c~|D]'\}}|t|(dSr=) named_modulesregister_forward_pre_hookrC)modelnamemods r8_apply_logging_hookrJRsK((**?? c %%&7&=&=>>>>??r9_fqn_to_op_to_shape_to_countceZdZddZdS)LoggingTensorModerBNc2|i}||i|}|jd|j}d}|D]C}t|tjr'|t t |jdzz }D|dkr |dd}ttvritt<|ttvritt|<|tt|vrdtt||<tt||xxdz cc<|S)N.z, rr) __module____name__ isinstancer0Tensorstrlistshaper:rK) selffunctypesargskwargsrsop_name shape_strargs r8__torch_dispatch__z$LoggingTensorMode.__torch_dispatch__^s# >F T4 "6 " "/;;DM;;  9 9C#u|,, 9Sci11D88 ??!#2#I 7 7 757 ( 2 6x@ @ @>@ ( 27 ; 8B7K K KIJ ( 27 ;I F$X.w7 BBBaGBBB r9)rBN)rSrR __qualname__rbrBr9r8rMrM]s(r9rMc,eZdZdZdZdZdZdZdS) _MultiInputc.t||_dSr=)rWvalues)rYinputss r8__init__z_MultiInput.__init__ws6ll r9c:|j||Sr=)rgappend)rYr?s r8 add_inputz_MultiInput.add_inputzs 5!!! r9c6t|j|Sr=)rerg)rYslices r8 __getitem__z_MultiInput.__getitem__~s4;u-...r9c2d|jD|_dS)Ncng|]2}t|tjr|n|3SrB)rTr0rUcuda.0vals r8 z$_MultiInput.cuda..sB   EH*S%,77 @CHHJJJS   r9rgrYs r8rrz_MultiInput.cudas(  LPK    r9c2d|jD|_dS)Ncng|]2}t|tjr|n|3SrB)rTr0rUxpurss r8rvz#_MultiInput.xpu..sB   DGC66 ?CGGIIIC   r9rwrxs r8r{z_MultiInput.xpus(  KO;    r9N)rSrRrcrirlrorrr{rBr9r8rerevs_###///        r9rec |)|j|krtd|d|d|jd|C||kr-td|d|d|ddSdS)NzExpected Tensor argument z to have dtype z , but got z instead.z to have size )dtype ValueErrorsize) tensor_argarg_namer}rs r8_guard_dtype_sizers Z-66 m m m m mR\Rb m m m    JOO--55 l l l l lPZP_P_PaPa l l l   55r9r4returncg}tt|jdz D]}|d||jd|S)Nr)rangelenrXrk)r4 block_size_s r8_get_per_token_block_sizersbJ 3qw<i) scale_dtype) r SYMMETRICrWrXrrr0int8r}float16float32rr) t mapping_typerir}eps quant_min quant_maxrscale zero_point quantizeds r8rrs(LagJ 3z??Q& ' ' 1 JE CII#$7em#;#;%--K-      E:  :uj%II e r9c^t|\}}t|||||}|||z}|S)ze like F.linear, but with int8 dynamic quantization of activation, and a quantized weight )r_quant_int8_per_token_matmul)r4 w_vals_int8_tw_scalesbias out_dtype x_vals_int8x_scalesmm_outs r8r r sGB!DDK )X}h F $ Mr9c|jtjksJd|jd|jtjksJd|jd|jtjtjfvsJd|j|d|jd}t|||dd}||zjg|jdd|jdR}||}|S)a Quantized matmul of int8 operands that accumulates to int32 and returns output_dtype. For now, this is written for approximate numerical Assumes that activation and weight quantization are symmetric, i.e. act_zp and w_zp is 0. Assumes that weight quantization is per-channel. see https://github.com/google/gemmlowp/blob/master/doc/quantization.md for an overview of quantized matmul compute in scalar form, assuming output_dtype is fp32 and zw == 0: Y_i_j_fp32 = sx * sw dot(X_i, W_j) zx dtype z not yet supportedzw dtype z?x_scales needs to be a torch.float32 or torch.bfloat16 but got rrN) r}r0rfloatbfloat16reshaperXrto)rrrr output_dtypetmp y_dot_scaledr5s r8rrsB.   * * *8;$888 + * *  %* , , ,:=&::: - , , >      [(.ZZ    $   b+"3B"7 8 8C$S-9I9I"a9P9PQQL) )   3B3  !-!3B!7   A \A Hr9c B|dks Jdtjtjj}d|jdf}tj}tj}t||||||||\}} t|||| |||} | || fS)z assumes symmetric quantization assumes axis == 0 assumes dense memory format TODO(future): relax ^ as needed only support 2d Tensorsr) target_dtyperrrzero_point_dtype) dimr0finforrrXint64r rrr) r4rrrrrrrrrquants r8r!r!s 5577a<<<2<<< +em $ $ (CQWQZJ{(L- ! )   E:  :uj, 9  E % ##r9c|j}|j}|dksJd|t||||||}|S)Nrz scale size: r)rXr}numelr)int_reprrrrr input_dtype dequantizeds r8r"r"3skJ.K ;;==A   =ekkmm==   #*eZ9K r9c|dks Jd|}|jddf}|j}t ||||||}|}|S)Nrrrrr)rrrXr}r)rscales zero_pointsrrrrs r8r#r#>s <<>>Q    9    zz||H.#Q'J.K#*fk;YK--//K r9Fc *||jdkr |jd}|dksJ|jd|zdksJ|dksJ|dks Jd|tj}tj}d|f} d} d|zdz } |d}|} |t jkr|n tj} |t jkr|st||| || | || |  \}}nI|t jkr|st||| || | || |  \}}nt||| || | || |  \}}| |  |jdd| |  |jddfS) Nrrrrz-only n_bit smaller than 8 is supported, got: gư>)rrr})rXrr ASYMMETRICr0int32r INTFLOATr r rrr)wn_bit groupsizer}zero_point_domain preserve_zerorrrrrrrrrrs r8r$r$Os172;GBK q==== 72; "a ' ' ' ' 5577a<<<< A:::NuNN:::)L;LYJI51 I {K"o&999u{O111-1;      #-    zz o1 1 1- 1E      #-    zz2      #-    z 88%8 ( (R 8 8*--;H;; gagaj" r9c^t|d||t|d||}tj|d|dg|ddS)Nr)r}rzerosrrrQ)rrrr0cat unsqueeze transpose contiguous)rrr}rs r8r%r%sfhe%**,,GGGGeWE2222 **,,C   $$##      2r    r9c||jddksJtj|ddddS)NrrrrQr)rXr0splitr)scales_and_zeross r8r&r&s@  !" % * * * * ;'11"b991b A AAr9c ~|dksJ||jdkr|jddkr |jd}|jd|zdksJ|dksJd|f}tj}d}d|zdz } |tjkrt } nG|tjkrt} n/ttj krt} ntd|| ||||||| } |jddkrt| j sSt| j s?| dddddfdz| dddddfztj} t| j r?| dddddfdz| dddddfztj} | S)Nrrrrz Unrecognized zero point domain: r)rXrr0rr rrrrNONErr~rdevicerruint8) rrrrrrrrrr_quantize_affineint_datas r8r'r's q====172;6<#3q#8#8GBK 72; "a ' ' ' ' 5577a<<<<YJ;LI51 IO///* o3 3 34 O0 0 09O>Q    %+%%);a)?)?(/***@{{5;''AI $;+ ^A q 1A 5 6+?    "(/22 *(GBBB!G  (GBBB1H  'GBBB!G  )GBBB1H  7=$$$b)9Q)>)>M"% = y (A - - - -YJ+KI51 IO///. o3 3 38=   \    r9ct||||||\}}t||||||}t|||} || fS)N)rr)r)r$r'r%) rrrr}rrrrrrs r8r)r)sr1   +# MFE= 65%>OH6feUKK % %%r9cLt|\}}t|||||Sr=)r&r()rrrrrrs r8r*r*'s3 55EFFMFE :&%   r9c b||jdkr |jd}|dksJ|jd|zdksJ|dksJ|dks Jd|d|f}|tj|jj}i}d|d<t ddD]}d|dz z d|dz zdz f||<||\} } t|||tj| | ||| \} } | |jdd| |jddfS) Nrrrrrzunsupported n_bit: )rr )rrrrrr) rXrr0rr}rrrrr) rrr precisionrrrrangesrrrrrs r8r,r,4sc172;GBK q==== 72; "a ' ' ' ' 5577a<<<< A:::4U44:::YJ {k!'""& FF1I 1a[[88QU|_aAElQ&67q !%=Iy- Z "   E: ==R ( (**<*JqM1Q666ggg[egg7666 K * * Is;''!+, B/AAA K ( (IS3{+++ &1+/&B ;?#Z   K * *I2!771<<< p{2 p pXcXn p p=<<s;''!+, 0F/HHH >>> ? ??r9)NN)rr)Gtypingrrrrr0torch.utils._python_dispatchrtorchao.kernelr%torchao.quantization.quant_primitivesr r r r r rrrrrr torchao.utilsrrrrrrrrrr__all__rr:rV__annotations__rCrJrKintrMrerrUrrr rrr!r"r#rrr$r%r&r'r(r)r*rr,rr}rr+r-rrBr9r8rs /............ ::::::                              *%%%(3-???d SM4 tHSM3,>'??@@ )2        ,    $s)B2 9 9 9 9 x$$$@BGEJM&  .%+ CCCCL9>"BBB @U))))` %+ 6666v  .%+ &&&&4      m& $M$M$M$MR m& !!!!8 %}$)M ++ <++k+ % +  \ ++++\///$,@sCx,@/:,@ 38_,@,@,@,@,@,@r9