Pi2BddlmZddlZddlmZmZmZmZddlm Z m Z m Z m Z m Z mZmZmZmZddlmZmZmZmZddlmZmZmZmZddlmZmZmZGd d ej j!Z"Gd d e"Z#Gd de"Z$Gdde"Z%Gdde%Z&dS))OptionalN)PerAxisPerGroupPerRowPerToken) _DTYPE_TO_BIT_WIDTH_DTYPE_TO_QVALUE_BOUNDS MappingType_choose_scale_float8_dequantize_affine_float8_fake_quantize_affine_quantize_affine_float8_Roundchoose_qparams_affine)_get_per_token_block_sizeget_block_sizeget_group_qparams_symmetricget_groupwise_affine_qparams)FakeQuantizeConfigBaseFloat8FakeQuantizeConfigInt4WeightFakeQuantizeConfigIntxFakeQuantizeConfig) _fake_quantize_per_channel_group_fake_quantize_per_token_log_deprecation_warningcJeZdZUdZeed<defdZededdfdZ dS)FakeQuantizerBasez` Generic module for applying fake quantization to a tensor, as specified in the config. configreturncd|jzS)ze Return a human readable representation of this `FakeQuantizer` with config details. zFakeQuantizer(%s)rselfs {/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/quantization/qat/fake_quantizer.py__repr__zFakeQuantizerBase.__repr__7s#T[00ct|trt|St|trt |St|t rt |Std|)NzUnknown config type: ) isinstancerIntxFakeQuantizerrInt4WeightFakeQuantizerrFloat8FakeQuantizer ValueErrorr"s r% from_configzFakeQuantizerBase.from_config=s| f4 5 5 ?$V,, ,  < = = ?*622 2  8 9 9 ?&v.. .=V==>> >r'N) __name__ __module__ __qualname____doc__r__annotations__strr& staticmethodr.r'r%rr0sy #"""1#1111 ?2?7J???\???r'rcLeZdZdZdeffd ZdejdejfdZxZ S)r,zg Generic module for applying float8 fake quantization to a tensor, as specified in the config. rct||_tjddS)Nz,torchao.quantization.qat.Float8FakeQuantizersuper__init__rtorch_C_log_api_usage_oncer$r __class__s r%r;zFloat8FakeQuantizer.__init__Ns=   $$%STTTTTr'xr c|j}t|j|jj}t |||jj|jj|jj}t|||jj}t|||}|S)N) hp_value_lb hp_value_ub) dtypershaper granularityr rCrDrr )r$rAoriginal_dtype block_sizescaleqdqs r%forwardzFloat8FakeQuantizer.forwardSs#AGT[-DEE $  K  / /     $Audk.? @ @ &q% @ @ r') r/r0r1r2rr;r<TensorrM __classcell__r@s@r%r,r,Is|U7UUUUUU  %,        r'r,ceZdZdZdeffd ZdejdejfdZdejdejfdZ dejdejfdZ xZ S) r+a Generic module for applying int4 fake quantization to a weight tensor, targeting the following FBGEMM kernels: torch.ops.fbgemm.f8i4bf16_shuffled torch.ops.fbgemm.bf16i4bf16_shuffled torch.ops.fbgemm.bf16i4bf16_rowwise rct||_tjddS)Nz0torchao.quantization.qat.Int4WeightFakeQuantizerr9r?s r%r;z Int4WeightFakeQuantizer.__init__ks=   $$%WXXXXXr'wr c|jjtjkr||S|jjtjkr||Std|jj)NzUnknown activation dtype )ractivation_dtyper< float8_e4m3fn_fp8_activations_forwardbfloat16_bf16_activations_forwardr-)r$rSs r%rMzInt4WeightFakeQuantizer.forwardpsk ; '5+> > >0033 3 [ )U^ ; ;11!44 4W9UWWXX Xr'c |dksJ|jjtjksJt |jt}t||tjd}t||tj}t|||j }d}d}| |jdd|jj }tjtj|dd }tj||z | } tj| } d |jj f} t%|| | | tjd d } | |j S)a: Apply int4 fake quantization to the weight tensor where the input activations are expected to be rowwise fp8, using the following as a reference: https://github.com/pytorch/FBGEMM/blob/80cc48c4b2b7fcc579e53211fc8715a8592cbd2c/fbgemm_gpu/experimental/gen_ai/gen_ai/quantize.py#L136 g-q=)rCư>rFdimkeepdimminri) quant_dtype quant_min quant_max)r`rrUr<rVrrFrr rr rEview group_sizeamaxabsclamp zeros_liker int8to) r$rSper_row_block_size fp8_scalew_fp8epsfbgemm_scale_quant_max w_fp8_groupedmax_absrJ zero_pointper_group_block_sizefqs r%rWz0Int4WeightFakeQuantizer._fp8_activations_forwardxs\ uuww!||||{+u/BBBBB,AGVXX>>(       (9e6IJJ)%AGDD!" 5;q>2t{7MNN *UY}552uMMM G&<<#FFF%e,, !4;#9: "       uuQW~~r'c|dksJ|jjtjksJd}d\}}d}|tj|jdd|jj }tj |dd}tj |dd}tj ||z | |z } || |zz} tj||z | z  ||} | |z } | | z| z} | |j|jS) a3 Apply int4 fake quantization to the weight tensor where the input activations are expected to be bf16, using the following as a reference: https://github.com/pytorch/FBGEMM/blob/80cc48c4b2b7fcc579e53211fc8715a8592cbd2c/fbgemm_gpu/experimental/gen_ai/gen_ai/quantize.py#L152 r[r\)rr]rr^Tr_rb)r`rrUr<rXrofloat32rhrFrirjaminrlrapplyrE) r$rSrsqminqmaxfbgemm_symmetric_qmax w_groupedmax_valmin_valrJrwrys r%rYz1Int4WeightFakeQuantizer._bf16_activations_forwards5 uuww!||||{+u~==== d !DD'',,QWQZT[=STT *YB===*YB=== Gg-3777$>u'<<< \9w.%7 8 8 > >tT J J ' ' %Z* $wwqw""17+++r') r/r0r1r2rr;r<rNrMrWrYrOrPs@r%r+r+bsY;YYYYYY YY%,YYYY'%,'5<''''R,5<,EL,,,,,,,,r'r+ceZdZdZdeffd ZdejdejfdZdejdejfdZ dejdejfdZ de fd Z d d Z xZS) r*zh Generic module for applying integer fake quantization to a tensor, as specified in the config. rcttjd||_d|_d|_d|_d|_ d|_ dS)Nz*torchao.quantization.qat.IntxFakeQuantizerTg& .>F) r:r;r<r=r>renabledrJrw _scale_eps _initializedr?s r%r;zIntxFakeQuantizer.__init__sc  $$%QRRR  -1 26!r'rAr c|js|S|jjr$|js|j|jt dt|jjtr| |St|jjttfr| |St d|jjz)z Apply fake quantization to the tensor based on the bit-width, granularity, symmetry, and other properties specified in the config. NzScales and zero points must be initialized for range learning. Please call `torchao.quantization.qat.initialize_fake_quantizers` before initializing the optimizer and beginning training.zUnknown granularity '%s')rrrange_learningrrJrwr-r)rGr_per_token_forwardrr_per_channel_or_group_forward)r$rAs r%rMzIntxFakeQuantizer.forwards | H K & % #t'>L  dk-x 8 8 S**1-- -  /'81D E E S55a88 87$+:QQRR Rr'c |jjrtdt|jj\}}|rxt |tjt||jj|||jj |jj |jj  \|_ |_|t!||j |j||S)zD Perform per token fake quantization on the tensor. z(Symmetric per token is not supported yet) mapping_typerI target_dtyperfrgrs scale_dtypezero_point_dtype)r is_symmetricNotImplementedErrorr rE_should_compute_qparamsrr ASYMMETRICrrsscale_precisionzero_point_precisionrJrw(_maybe_update_qparams_for_range_learningr)r$rArrs r%rz$IntxFakeQuantizer._per_token_forwards ; # R%&PQQ Q,T[->? d  ' ' ) ) <*?(34Q77![.KO K7!%!A + + + 'DJ  9 9 ; ; ;'4:tdSSSr'c 2|jj}|jj}|jj}|jj}|jj}t |tr(|jdksJ| d}n/t |tr|j }ntd|z| rt|jj}|r,t!|||||jj\|_|_n+t)|||||jj\|_|_|j||_|t.|jj\} } t1||j|j| | ||S)z Perform per channel or per group fake quantization on the tensor. We express per channel using per group where the group size is the size of the last dimension of the tensor. rr^zUnexpected granularity '%s')rs)rrGrrzero_point_domainrr)raxissizerrir-rrrErrsrJrwrrorr r) r$rArGrrrrri bit_widthrrs r%rz/IntxFakeQuantizer._per_channel_or_group_forwards k- +5#{? K9{/  k7 + + J#q(((("JJ  X . . J$/JJ:[HII I  ' ' ) ) <+DK,=>I 0K#  111-T__1M#  111-T_#o001EFFDO  9 9 ; ; ;,T[->? d/ J O       r'c>|jjp|jdup|jduS)zO Return whether we need to compute new scales and zero points. N)r is_dynamicrJrwr#s r%rz)IntxFakeQuantizer._should_compute_qparams1s({%Vt);VtRV?VVr'Ncr|jjrHt|jtjjs$t|jtjjrdS|j|j}}t|jj \}}t j ||j }tj|d|_|jj r|j dStj|}t j |||}tj|d|_dS)z If range learning is enabled, turn scales and zero points into trainable parameters. This function is idempotent and should only be called once. NrbT) requires_grad)rrr)rJr<nn Parameterrwr rErlrrzero_rr~)r$rJrwrrs r%rz:IntxFakeQuantizer._maybe_update_qparams_for_range_learning7s  * $*eh&899 $/58+=>>  F Jz,T[->? d Et777X''T'BB ; # Q O ! ! # # # # #j11JZt<rst                        ????????2+2S,S,S,S,S,/S,S,S,lTQTQTQTQTQ)TQTQTQp'''''%'''''r'