PiXddlmZmZddlZddlmcmZddlm Z ddl m Z m Z ddl mZmZmZmZmZmZddlmZmZddlmZddlmZd d lmZmZmZd d lm Z d d l!m"Z"Gd dejj#Z$ d*dejj%de&fdZ'dejj%fdZ(GddeZ)Gdde)Z*Gdde$Z+dejj%fdZ,dejj%fdZ-dej.defdZ/de0dej.defd Z1Gd!d"e)Z2Gd#d$e$Z3dejj%fd%Z4dejj%fd&Z5de0dej.defd'Z6Gd(d)e)Z7dS)+)AnyOptionalN) is_device)PerGroupPerRow)Int8DynActInt4WeightLinearWeightOnlyInt4Linear_check_linear_int4_k_replace_linear_8da4w_replace_linear_int4 groupwise_affine_quantize_tensor) TorchAODTypeZeroPointDomain)TwoStepQuantizer)get_group_qparams_symmetric)FakeQuantizeConfigBaseFloat8FakeQuantizeConfigIntxFakeQuantizeConfig)FakeQuantizerBase)_get_qmin_qmaxc eZdZdZ ddedededeedeed df fd Zd e j d e j fd Z d e j j fd Ze dde j j deedeefdZxZS)FakeQuantizedLineara General linear layer with fake quantized weights and activations. Specific target dtypes, granularity, schemes etc. are specified through separate configs for weights and activations. Example usage:: activation_config = IntxFakeQuantizeConfig( dtype=torch.int8, granularity="per_token", is_symmetric=False, ) weight_config = IntxFakeQuantizeConfig( dtype=torch.int4, group_size=8, is_symmetric=True, ) fq_linear = FakeQuantizedLinear( 16, 32, False, activation_config, weight_config, ) fq_linear(torch.randn(16)) FN in_features out_featuresbiasactivation_config weight_configreturnctj|||g|Ri|tjd|t j||_nd|_|rt|trBt|j tr(|j }|||zdkrtd|d|dt j||_dSd|_dS)Nz,torchao.quantization.qat.FakeQuantizedLinearrz in_features (z) % group_size (z) must be == 0)super__init__torch_C_log_api_usage_oncer from_configactivation_fake_quantizer isinstancer granularityr group_size ValueErrorweight_fake_quantizer) selfrrrrrargskwargsr* __class__s s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchao/quantization/qat/linear.pyr"zFakeQuantizedLinear.__init__Cs/              $$%STTT  (->-J!..D * *.2D *  $-)?@@ Z)8FF +5 )kJ.F!.K.K$*&;; 4*;)F})U)UD & & &)-D & & &xc|j||}|j||j}n|j}tj|||jSN)r'r,weightFlinearr)r-r3ws r1forwardzFakeQuantizedLinear.forwardls[  ) 5..q11A  % 1**4;77AA Ax1di(((r2c tj|j|j|jdu|jj|jj}|jjtjdkr|j|_|j|_|S)Ndevicedtypemeta) r#nnLinearrrrr6r=r>)r- new_linears r1 to_linearzFakeQuantizedLinear.to_linearuswX__     IT !;%+# %   ; f!5!5 5 5 $ J "iJOr2modc t|j|j|jdu|||jj|jj}|jjtjdkr|j|_|j|_|S)Nrrr=r>r?)rrrrr6r=r>r#)clsrDrrrBs r1 from_linearzFakeQuantizedLinear.from_linearsx) O   HD /':$*"    :  V 4 4 4 4 # J !hJOr2)FNN)NN)__name__ __module__ __qualname____doc__intboolrrr"r#Tensorr:r@rArC classmethodrH __classcell__r0s@r1rr*s=8>B:> '.'.'.'. '. $$:; '.  67 '. '.'.'.'.'.'.R))%,))))58? ?C:>  X_$$:; 67 [r2rTrDenabledct|tr(|j ||j_|j||j_dSdSdS)zO Helper function to enable fake quantization in `FakeQuantizedLinear`. N)r(rr'rSr,)rDrSs r1enable_linear_fake_quantrUsW#*++8  ( 44;C ) 1  $ 007C % - - - 88 1 0r2c(t|ddS)zP Helper function to disable fake quantization in `FakeQuantizedLinear`. F)rSN)rUrDs r1disable_linear_fake_quantrXsS%000000r2cBeZdZdZdeefdZdeefdZdS)_LegacyQATQuantizerzM Base class for sharing common methods across legacy QAT quantizers. rcdSr5r-s r1#get_activation_fake_quantize_configz7_LegacyQATQuantizer.get_activation_fake_quantize_configtr2cdSr5r\r]s r1get_weight_fake_quantize_configz3_LegacyQATQuantizer.get_weight_fake_quantize_configr_r2N)rIrJrKrLrrr^rar\r2r1rZrZs`X>T5U:P1Qr2rZc :eZdZdZddejejfdededejdejdd f fd Z d ej j d e d e dej j fdZ d ej j d e d e dej j fdZdej j fdZdeefdZdeefdZxZS) Int8DynActInt4WeightQATQuantizerz Quantizer for performing QAT on a model, where linear layers have int8 dynamic per token fake quantized activations and int4 fake quantized grouped per channel weights. F groupsizepadding_allowed precisionscales_precisionrNcttjd||_||_||_||_tj |_ dS)Nz9torchao.quantization.qat.Int8DynActInt4WeightQATQuantizer) r!r"r#r$r%rerfrgrhfloat32activation_scales_precision)r-rerfrgrhr0s r1r"z)Int8DynActInt4WeightQATQuantizer.__init__sh  $$ G   (%4&/-=+0=(((r2modelr.r/c dt||j|j|j|jt d|S)NT) copy_weights)r rerfrgrhInt8DynActInt4WeightQATLinearr-rlr.r/s r1preparez(Int8DynActInt4WeightQATQuantizer.preparesA   N  N  ! )     r2c0|||Sr5)_convert_qat_linear_8da4wrps r1convertz(Int8DynActInt4WeightQATQuantizer.converts &&u--- r2modulec |D]/\}}t|tr|jj}t |j|j|jdu|j |j j |j }t|||d}t|\}}t|j ||j |j \} } | |j} ddlm} | |j | | ||t(j|j } | |_ | |_| |_|j |j|_||1dS)z` Replace all `Int8DynActInt4WeightQATLinear` with `Int8DynActInt4WeightLinear`. N)rergrh)rgr)8_quantized_decomposed_quantize_per_channel_group_wrapper)named_childrenr(ror,configrrrrr*r6r>scale_precisionsetattrrrtozero_point_precisiontorchao._executorch_opsrxr#int8scaleszerosrs) r-runamechildrzquantized_linearn_bitqminqmaxszprxq_weights r1rsz:Int8DynActInt4WeightQATQuantizer._convert_qat_linear_8da4ws"0022* 6* 6KD%%!>??) 64;#=%&Jd*$/#l0%+%; $$$ &6777-e44 t5L%$4 B UU6677TSLJ%+3 '*+ ')+ &:),1J$)..u5555U* 6* 6r2c*t|jSr5)_get_8da4w_activation_configrkr]s r1r^zDInt8DynActInt4WeightQATQuantizer.get_activation_fake_quantize_config&s+D,LMMMr2c6t|j|jSr5)_get_8da4w_weight_configrerhr]s r1raz@Int8DynActInt4WeightQATQuantizer.get_weight_fake_quantize_config)s'8MNNNr2)rIrJrKrLr#rjrMrNr>r"r@Modulerrqrtrsrrr^rarQrRs@r1rcrcsz %!&(- 9999; 9  + 9  999999$ X_ -0 T5UNNNNO:P1QOOOOOOOOr2rcceZdZdZdddejejfdedededejd ed ej d ej d dffd Z ddefdZ dZ xZ S)roa This module implements a linear layer with int8 dynamic per token fake quantized activations with int4 fake quantized grouped per channel weights. args: groupsize: the number of elements in each quantized group for weights precision: precision of weights scales_precision: precision of per group scales and zero points Note: we hardcode activation scales to use torch.fp32, but allow users to specify the weight scales (defaults to torch.fp32). To get an exact numerical match with Int8DynamicActivationInt4WeightConfig, users must use the same dtype for both the weights and the scales. Here scales_precision refers specifically to the weight scales only, not the activation scales. FNrdrrrr=rergrhrc ttj}t||} t ||||| ||dS)Nr<)rr#rjrr!r") r-rrrr=rergrhrrr0s r1r"z&Int8DynActInt4WeightQATLinear.__init__<se9GG0r"rrrQrRs@r1roro-s  $#!&(-           ;  +        2555555&&&&&&&r2roc\t|tr|dSdS)zT (deprecated) Enable fake quantization for `Int8DynActInt4WeightQATLinear`. N)r(rorrWs r1enable_8da4w_fake_quantr^s9#455    r2c\t|tr|dSdS)zU (deprecated) Disable fake quantization for `Int8DynActInt4WeightQATLinear`. N)r(rorrWs r1disable_8da4w_fake_quantrgs9#455!      !!r2qparams_precisionrc |tjksJttjddd||tj|jS)z` Return the activation `IntxFakeQuantizeConfig` for `Int8DynActInt4WeightQATQuantizer`. per_tokenFT)r>r) is_symmetric is_dynamicr{r~eps)r#rjrrfinfor)rs r1rrosS  - - - - !j). K) * * .   r2r*c@ttj|dd||S)z\ Return the weight `IntxFakeQuantizeConfig` for `Int8DynActInt4WeightQATQuantizer`. T)r>r*rrr{r~)rrINT4r*rs r1rrs0 ").    r2c .eZdZdZddejejfdedeedejdejdd f fd Z d ej j d e d e dej j fdZ d ej j d e d e dej j fdZdej j fdZdeefdZxZS)Int4WeightOnlyQATQuantizerz Quantizer for performing QAT on a model, where linear layers have int4 fake quantized grouped per channel weights. rdre inner_k_tilesrgrhrNcttjd|dvsJ|dvsJ||_||_||_||_dS)Nz3torchao.quantization.qat.Int4WeightOnlyQATQuantizer)rwr) @rd) r!r"r#r$r%rrergrh)r-rerrgrhr0s r1r"z#Int4WeightOnlyQATQuantizer.__init__s  $$ A    )))).....*"" 0r2rlr.r/c ft||j|jd|j|jt d|S)NT)rfrgrh linear_classrn)r rerrgrhInt4WeightOnlyQATLinearrps r1rqz"Int4WeightOnlyQATQuantizer.preparesD   N   n!20  r2c0|||Sr5)_convert_qat_linear_4wrps r1rtz"Int4WeightOnlyQATQuantizer.converts ##E*** r2rucH|D]\}}t|trZ|j}|j}|j}|jj}t||d|j ||j j |j t|j}t!|||d} t#|j | |j \} } t%| jjdrHt(jj| |j j|j} nGt(jj| |j j|j} | |_ | |_v||dS)zT Replace all `Int4WeightOnlyQATLinear` with `WeightOnlyInt4Linear`. F)rrerrgrhr=rwcpuN)ryr(rrrrr,rzr r*r6r>r{next parametersr=r|r rtyper#opsaten#_convert_weight_to_int4pack_for_cpur}_convert_weight_to_int4packscales_and_zerosr) r-rurrrrrrzrrrrs r1rz1Int4WeightOnlyQATQuantizer._convert_qat_linear_4ws"0022& 3& 3KD%%!899% 3#/ $1 % 3 4;#7 $/"/#l0%+%; 0 0 2 233: $ $ $ &6777/OL%00,+ X_1599 $y~QQ EL$788+  HH %y~II EL$788+  H+3 '4D 11++E2222M& 3& 3r2c6t|j|jSr5)_get_4w_weight_configrerhr]s r1raz:Int4WeightOnlyQATQuantizer.get_weight_fake_quantize_configs$T^T5JKKKr2)rIrJrKrLr#bfloat16rMrr>r"r@rrrqrtrrrarQrRs@r1rrsX'(!&(- 111 }1; 1  + 1  111111$ X_ -0 1113V111#KMJJ HFGG G*-i9IJJ     "'      r2TrSc6||j_||j_dSr5rrs r1rz)Int4WeightOnlyQATLinear.enable_fake_quantrr2c0|ddSrrr]s r1rz*Int4WeightOnlyQATLinear.disable_fake_quantrr2r)rIrJrKrLr#rrMrNr=r>r"rrrQrRs@r1rrs  #!&(-           ;  +        4555555&&&&&&&r2rc\t|tr|dSdS)zN (deprecated) Enable fake quantization for `Int4WeightOnlyQATLinear`. N)r(rrrWs r1enable_4w_fake_quantr$s9#.//    r2c\t|tr|dSdS)zO (deprecated) Disable fake quantization for `Int4WeightOnlyQATLinear`. N)r(rrrWs r1disable_4w_fake_quantr-s9#.//!      !!r2c Vttj|dd||tjS)zV Return the weight `IntxFakeQuantizeConfig` for `Int4WeightOnlyQATQuantizer`. FT)r>r*rrr{r~zero_point_domain)rr#uint4rFLOATrs r1rr5s5 "k).)/   r2ceZdZdZdejfdeedejfdZ dej j de de d ej j fd Z dej j de de d ej j fd Zd eefd Zd eefd ZdS)Float8ActInt4WeightQATQuantizera QAT quantizer for applying dynamic rowwise float8 activation + int4 per group/channel symmetric weight fake quantization to linear layers in the model. Currently only supports rowwise granularity for float8 activations. args: group_size (Optional[int]): the number of elements in each quantized group for weights, defaults to 64. Use None for per channel. scale_precision: precision of weight scales, defaults to torch.bfloat16. rr*r{ctjd|d}nd}ttjt |_ttj||dd||_ dS)Nz8torchao.quantization.qat.Float8ActInt4WeightQATQuantizer per_group per_channel)r>r)T)r>r)r*rrr{) r#r$r%r float8_e4m3fnr_activation_configrint4_weight_config)r-r*r{weight_granularitys r1r"z(Float8ActInt4WeightQATQuantizer.__init__Ys $$ F     !!,  !. ":%# # # 5**!+    r2rlr.r/rc|D]r\}}t|tjjr9t ||j|j}t|||]| |s|S)z Swap all `nn.Linear` with `FakeQuantizedLinear` with float8 fake quantizer for activations and int4 fake quantizer for weights. )rr) ryr(r#r@rArrHrrr|rq)r-rlr.r/rrrBs r1rqz'Float8ActInt4WeightQATQuantizer.preparers!//11 $ $KD%%11 $0<<&*&="&"5= tZ0000 U#### r2ctr5NotImplementedErrorrps r1rtz'Float8ActInt4WeightQATQuantizer.converts "!r2c td)Nz,Float8 FakeQuantizeConfig does not exist yetrr]s r1r^zCFloat8ActInt4WeightQATQuantizer.get_activation_fake_quantize_configs!"PQQQr2c|jSr5)rr]s r1raz?Float8ActInt4WeightQATQuantizer.get_weight_fake_quantize_configs !!r2N)rIrJrKrLr#rrrMr>r"r@rrrqrtrr^rar\r2r1rrLs  %'',~  SM     2X_-0T5URRRR":P1Q""""""r2rr)8typingrrr#torch.nn.functionalr@ functionalr7torchao.dtypes.utilsr torchao.quantization.granularityrr)torchao.quantization.linear_quant_modulesrr r r r r %torchao.quantization.quant_primitivesrrtorchao.quantization.unifiedrtorchao.quantization.utilsrfake_quantize_configrrrfake_quantizerrutilsrrArrrNrUrXrZrcrorrr>rrMrrrrrrrr\r2r1rs=! ******========:99999BBBBBB  qqqqq%(/qqql 8 8  8  8 8 8 8158?1111     *   "aOaOaOaOaO':aOaOaOH-&-&-&-&-&$7-&-&-&b     !%(/!!!!{&{,ZLZLZLZLZL!4ZLZLZLz+&+&+&+&+&1+&+&+&^ eho    !ux!!!!{.C"C"C"C"C"&9C"C"C"C"C"r2