Pi dZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z mZmZmZmZmZddl mZddlZddlmZddlmcmcmZddlZddlmZddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0dd l1m2Z2m3Z3dd l4m5Z5dd l6m7Z7m8Z8dd l9m:Z:dd l;mZ>m?Z?m@Z@ddlAmBZBddlCmDZDddlEmFZFddlGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUddlVmWZWmXZXddlYmZZZddl[m\Z\ddl]m^Z^m_Z_m`Z`maZaddlbmcZcmbZbddldmeZeddlfmgZgmhZhmiZimjZjmkZkddllmmZmmnZnddlompZpmqZqddlrmsZsddltmuZumvZvmwZwmxZxddlymzZzm{Z{ddlm|Z|ej}e~Zgd Ze+ewjge&ewjge"ewjge#ewjewjgiZe+d!e&d"e"d!e#d!iZ dd%eee d&fd'dfd(Zd)Zdd*Zdd+Zdd,d-ejd.eeDd/eeDd0eeejjegeffd1Zd2ejfd3Zd4Zd5Zd6Zd!d!d7d8Zedfd-ejjd9ed0eeejjegefd:eejjfd;ZdZd.insert_subclasss ! ++sz44V44E,CJj3355GAt  0X8wsK88$?    ++(Z((>>v>>#,CJ  r)pop)rrrrrrs``` @@rr[r[se**],BCCKHl33J" rczGddtjjfd}|d}t|||dS)zi Changes all conv2d 1x1 modules to equivalent linear modules so that they can then be quantized. c$eZdZfdZdZxZS)2swap_conv2d_1x1_to_linear..PermuteSandwichcVt||_dSN)super__init__r)selfr __class__s rrz;swap_conv2d_1x1_to_linear..PermuteSandwich.__init__s$ GG     DHHHrc||dddddddddS)NrrE)rpermute)rrs rforwardz:swap_conv2d_1x1_to_linear..PermuteSandwich.forward#s>88DGOOAq!Q7788@@Q1MM Mr)__name__ __module__ __qualname__rr __classcell__)rs@rPermuteSandwichrsN      N N N N N N Nrrc,|jdksJtj|j|j|jdu}tj|j dd|_|j|_|S)NrErE)biasro) kernel_sizerrr in_channels out_channelsrrrsqueeze)convrrs rreplace_conv2d_1x1z5swap_conv2d_1x1_to_linear..replace_conv2d_1x1&s6))))hoo  d/tyD7H  X'' (;(;B(C(CDD 9s###rNcVt|tjjo |jdkS)Nr)rrrConv2dr)rrs rz+swap_conv2d_1x1_to_linear..0s+z ( ( ((o'rrz)rrModulerv)rxrzrrs @rrZrZs NNNNN%(/NNN$$$$$(( . !Yrrrxinput_observerweight_observerrzcddtjffd }t|||tn|dS)a Converts the weight of a linear module to a LinearActivationWeightObservedTensor. This function wraps the weight of the given linear module with a LinearActivationWeightObservedTensor, which enables observation of both input and weight tensors during forward passes. The wrapped weight is then re-wrapped as a nn.Parameter to maintain compatibility with PyTorch's module system. Example:: ``` import torch import torch.nn as nn from torchao.quantization import PerTensor from torchao.quantization.linear_observer_tensor import insert_observers_ from torchao.quantization.observer import ( AffineQuantizedMinMaxObserver, MappingType ) # Create observers input_observer = AffineQuantizedMinMaxObserver( MappingType.SYMMETRIC, torch.float8_e4m3fn, granularity_type=PerTensor(), eps=torch.finfo(torch.float32).eps, scale_dtype=torch.float, zero_point_dtype=torch.int, zero_point_domain=ZeroPointDomain.NONE, ) # Create a linear module linear_module = nn.Linear(10, 20) # Convert the linear module's weight to an observed tensor insert_observers_(linear_module, input_observer, weight_observer=None) # The linear_module can now be used as usual, with observers calculating statistics output = linear_module(torch.randn(10, 10)) # Get the scale and zero point of the input observer scale, zero_point = linear_module.weight.input_observer.calculate_qparams() ``` Args: model (nn.Module): The nn.Module to convert. input_observer (Optional[AffineQuantizedObserverBase]): Observer for input tensor. weight_observer (Optional[AffineQuantizedObserverBase]): Observer for weight tensor. filter_fn (Optional[Callable[[torch.nn.Module, str], bool]]): Filter function to select which modules to convert. If not provided, all linear modules will be converted. This function should take a module and its fully qualified name. Returns: nn.Linear: The modified linear module with its weight wrapped in a LinearActivationWeightObservedTensor. linear_modulectjtj|j|jj|_|S)N)rrr)rrr,rrr)rrrs rconvert_to_linear_observerz5insert_observers_..convert_to_linear_observerwsP!| 0 ;$- /    (.<    rN)rrrvr)rxrrrzrs `` rinsert_observers_r9sa| ")       . "' Yrrct|tr$|jjd|dSt|t r,|jjd|jdt |jdSt|dr$|jjd|dSt|tj ust|tj j rdt|Sdt|S)N()z (activation= , weight=_quantization_typezTensor: znot recognized: )rrrrrrNinput_quant_funcoriginal_weight_tensorrtyperTensorrr)rs rrrs8&/00M"+LLf.G.G.I.ILLLL&9::P"+PP9PPP[mntoL\M\MPPP Pv+,,M"+LLf.G.G.I.ILLLL F||u|##z&%(:L'M'M#($v,,((( ,d6ll , ,,rc|d|jjdd|jjddt|jS)Nz in_features=rEz, out_features=rrrshaperrs r_linear_extra_reprrsB $++A.  t{?PQR?S  ^pquq|^}^}  rc|d|jjdd|jjddt|jS)Nznum_embeddings=rz, embedding_dim=rErrrs r_embedding_extra_reprrsf DT[.q1 D D4;CTUVCW D DbtuyvAcBcB D DDrc g}|}t|dkr||||dtt||d|S)Nr=z, )lenappendrrjoin)roriginal_extra_reprparameter_namemodule_torchao_extra_reproriginal_extra_repr_strs r_module_extra_reprrs "1133 "##a''!(()@AAA$$ OO.wt^/L/LMMOO 99. / //r)allow_requires_gradpropagate_biasc fd}|S)zHelper function to apply the constructor that quantizes the weight Tensor (with additional kwargs) to the weight of linear module co |jj}dkr |jd<tj|jfi||_t jt||_ |S)NTrr) rrrrrrtypes MethodTyper extra_repr)rrrrrrs rrz6_get_linear_subclass_inserter..insert_subclasss{+H 0H T ! ! XF6NX'' K - -f - -](   )*rrrr)rT SYMMETRICrrrrYrrr r rrrs r_int8_symm_per_token_quantr4sY(L:L CII # !!$$ M    rceZdZUdZdZeed<eZe ed<e j Z e ed<e j Ze ed<dZeed<d Zd S) %Int8DynamicActivationInt4WeightConfigaConfiguration for applying int8 dynamic per token asymmetric activation quantization and int4 per group weight symmetric quantization to linear This is used to produce a model for executorch backend, but currently executorch did not support lowering for the quantized model from this flow yet Args: `group_size`: parameter for quantization, controls the granularity of quantization, smaller size is more fine grained `layout`: layout type for quantized weight tensor, only supports `MarlinQQQLayout()` and `CutlassInt4PackedLayout()` for now `mapping_type`: quantization type for weight, controls the weight quantization is symmetric or asymmetric `act_mapping_type`: quantization type for activation, controls the activation quantization is symmetric or asymmetric `set_inductor_config`: if True, adjusts `torchinductor` settings to recommended values. group_sizelayoutr act_mapping_typeTset_inductor_configcltjdtjddS)Nz:torchao.quantization.Int8DynamicActivationInt4WeightConfigz`Int8DynamicActivationInt4WeightConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.rrrwarningswarnrs r __post_init__z3Int8DynamicActivationInt4WeightConfig.__post_init__\E $$ H      y     rN)rrr__doc__r int__annotations__rr!r#rTrr rr"r#boolr(rkrrrrGs  J []]FF""" + 5L+555$/$:k::: $$$$     rrr] custom_scalecustom_zero_pointrr/r0c .|j}|j}|j}|j}|jr#t jj|j }||dkr |j d}|j d|zdkr|Sd|f} tj } d} d} |tjkr%t|t rt"} ngt$} n_|tjkrBt|t(rt*} n2t|t,rt.} nt*} n Jd|t|t(rt1|| | | |}nut|t,rt3|}nPt|t r#t5||| tjdd ||| }nt5||| | | | ||| }t9|| }tj|d |_ t?j tB||_"|S) NrorrEF%Unsupported activation mapping type: )_layout)r rrr5r/r0)r5r/r0r)#r r!r r"r#torchao quantizationutils"recommended_inductor_config_setterrrrrrTrrrrr rrrr_int8_symm_cutlass_quantr _int4_symm_cutlass_quantrrrOrrrrrr)rrr/r0r r!r r"r block_sizer rrrs r._int8_dynamic_activation_int4_weight_transformr>ksT"J ]F&L. !H"EEGGG ]FZ2--\"%  |B*$)) ZJ:LII;111 f? @ @ ;;  :   [2 2 2 fo . . :9    7 8 8 :7  9  PP>NPPPPu&/** , J 9f    F3 4 4 )&11 F= > > )   %/    *      %/    ,F4D E EFH&&vU&CCFM();VDDF Mrc eZdZUdZejZejed<e dZ e ed<e j Ze ed<dZeejed<e jZe ed<eZeed <ejZeed <ejZeed <d Zeed <dZdS)%Int8DynamicActivationIntxWeightConfiga7 Configuration for dynamically quantizing activations to torch.int8 and weights to torch.intx, with 1 <= x <= 8. More specifically, activations are dynamically quantized to 8-bits at a per-token granularity with scales/zeros. Weights are quantized with scales/zeros in a groupwise or channelwise manner using the number of bits specified by weight_dtype. This layout is identical to Int8DynamicActivationInt4WeightConfig when weight_dtype is torch.int4 and other args are the same. However, this layout is more general and supports other weight dtypes. args: `weight_dtype`: The dtype to use for weight quantization. Must be torch.intx, where 1 <= x <= 8. ` weight_granularity`: The granularity to use for weight quantization. Must be PerGroup or PerAxis(axis=0). `weight_mapping_type`: The type of mapping to use for the weight quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. MappingType.SYMMETRIC requires ZeroPointDomain.NONE `weight_scale_dtype`: The dtype to use for the weight scale. `act_mapping_type`: The type of mapping to use for the activation quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. `layout`: The layout to use for the packed weight tensor: - PackedLinearInt8DynamicActivationIntxWeightLayout: this layout is optimized for CPU performance. - QDQLayout: this layout represents the quantization with Q/DQ quant primitives, and is intended for export applications like ExecuTorch. `intx_packing_format`: The format to use for the packed weight tensor (version 2 only). - unpacked_to_int8: this format is the default and is intended for export applications like ExecuTorch. - opaque_torchao_auto: this format is optimized for CPU performance. `intx_choose_qparams_algorithm`: The algorithm to use for choosing the quantization parameters. `version`: version of the config to use, only subset of above args are valid based on version, see note for more details. Note: Current state for Int8DynamicActivationIntxWeightConfig is that it supports both v1 (legacy) and v2. * `intx_packing_format` is used for version 2. * `layout` is only used for version 1. weight_dtyperweight_granularityweight_mapping_typeNweight_scale_dtyper"r!intx_packing_formatintx_choose_qparams_algorithmrversionctjd|jdt ddDvsJd|jt |jttfsJd|jt |jtr'|jj dksJd|jj |j tj tj tjfvsJd |j |jtj tj fvsJd |jt |jt"t$fsJd |jt |jt"rw|jjt(jt(jt(jfvrJ|j|jtjkr0t5jd |jjd |jddSdSdSdS)Nz:torchao.quantization.Int8DynamicActivationIntxWeightConfigc>g|]}ttd|Sr+rr.0bs r zGInt8DynamicActivationIntxWeightConfig.__post_init__..($T$T$T1WUI!II%>%>$T$T$TrrE -O*OOO")"#=GS+++++rr@cr|j}|j}|j}|j}|j} |j} |j} |j} |dksJd|t|tr|j } nRt|tr+|j dksJd|j |jd} ntd|d| f}|jdkr| t"jksJt&jt&jt&jt&jg}| t&jks| |vs Jd| |4|jt4jkr|t4j}t=j||||d | || }|||jkrtA||||}| |vrtCj"||| }d}||fS|jdksJ| tFj$ks Jd tKj&d tN|\}}|t"j(t"j)fv}tU||d| ft4j|||t4j|tVj,t[ }t| tZrN| t"jkrt\}n%| t"j(krt^}n Jd| ta||}nt| tbr| t"jks Jd|j23\}}}|jd| z}|4d|}|J|4d|}|dk5}tm|||r|nd||| j7d}d}||fS)NrzFInt8DynamicActivationIntxWeightConfig only works for 2-d Tensor, got: r%axis must be 0 with PerAxis, but got roz4weight_granularity must be PerGroup or PerAxis, got rEUnsupported packing format: int8_asym_per_token)r activation_quantizationrFr/r0)rrEzOIntxChooseQParamsAlgorithm.AFFINE is the only supported algorithm for version 1zConfig Deprecation: version 1 of Int8DynamicActivationIntxWeightConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2967 for more details input_floatr r=r rrrr preserve_zerozero_point_domainr5Fr4zbPackedLinearInt8DynamicActivationIntxWeightLayout requires act_mapping_type=MappingType.ASYMMETRIC)validate_inputs)8rArBrCrDr"r!rErFdimrrKr rJrTrrrGrTrr:OPAQUE_ATEN_KLEIDIAIOPAQUE_TORCHAO_AUTOOPAQUE_TORCHAO_KLEIDIAIOPAQUE_TORCHAO_LOWBITr^r]rrrsrr;from_hp+_adjust_scale_dtype_in_intx_unpacked_tensorr9!from_intx_unpacked_to_int8_tensorr8r_r&r'rSrrUrrUINTrr rrOr tensor_impl get_plainreshapeanyr"rV)rrrr/r0rArBrCrDr"r!rErFr r=opaque_formats new_weightnew_biasrrrgactivation_quant_funcdatascale zero_pointgroups_per_rowhas_weight_zeross r4_int8_dynamic_activation_intx_weight_quantize_tensorrs&L2 42. ]F 4$*$H! ::<<1   _QWQ[Q[Q]Q]__   $h//  '2 & 0 0 !&!+++ M4F4K M M,++\"%  WCU W W   ZJ ~;#99999  2  1  5  3   #4#E E E"n444 ?*= ? ?54 5  (->-D -S-S 1 4 4UZ @ @ -5   ,$9*G%/      ).@FL.P.P 7F$6    . 0 0)K?RJH8## >Q     (,F,M M M MY N M M M i3<@Iy(-,M &(z?Z&#)-    F&)$$# {5 5 5$? ! ! !6 6 6$> ! ! TTBRTT T T5/8MNN FM N N ;#9999 p:99#)"4">">"@"@eZb)Z7 b.11%%%''N;; &!O0022N  * 4JJ   M!    4<rct|j|j|||\}}tj|d|_|d|_t |tjrtj t||_ |S)Nr.Fr) rrrrrrrrrrrr)rrr/r0rxrys r._int8_dynamic_activation_intx_weight_transformrsP  !+ JH&&z&GGFM &")$$I!,-?HH MrcveZdZUdZeZeed<ej Z eed<ej Z eed<dZ e ed<dZdS) %Int4DynamicActivationInt4WeightConfiga^Applies int4 dynamic per token symmetric activation quantization and int4 per row weight symmetric quantization to linear Args: `layout`: layout type for quantized weight tensor, only supports `MarlinQQQLayout()` and `CutlassInt4PackedLayout()` for now `mapping_type`: quantization type for weight, controls the weight quantization is symmetric or asymmetric `act_mapping_type`: quantization type for activation, controls the activation quantization is symmetric or asymmetric `set_inductor_config`: if True, adjusts `torchinductor` settings to recommended values. r!r r"Tr#cltjdtjddS)Nz:torchao.quantization.Int4DynamicActivationInt4WeightConfigz`Int4DynamicActivationInt4WeightConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.r%rs rr(z3Int4DynamicActivationInt4WeightConfig.__post_init__r)rN)rrrr*rr!r#r,rTrr r"r#r-r(rkrrrrs-,..FF... + 5L+555$/$9k999 $$$$     rr#int4_dynamic_activation_int4_weightc0|j}|j}|j}|j}|jr#t jjt|tstd|d|tj krtd|tj krtdt|}t|t}t j|d|_t'jt*||_|S)Nz;Only CutlassInt4PackedLayout layout is supported. Received rrz)Only mapping_type=SYMMETRIC is supported.z-Only act_mapping_type=SYMMETRIC is supported.Fr)rr!r r"r#r7r8r9r:rrNotImplementedErrorrTrr<rOrrrrrrr)rrrr!r r"s r._int4_dynamic_activation_int4_weight_transformrs]F ]F&L. !H"EEGGG f5 6 6 ! S& S S S   {,,,!"MNNN;000!"QRRR %f - -F + FH&&vU&CCFM();VDDF MrceZdZUdZdZeeed<dZeed<dZ eeed<dZ ee ed <d Z e ed <d ZdS) GemliteUIntXWeightOnlyConfiga# applies weight only 4 or 8 bit integer quantization and utilizes the gemlite triton kernel and its associated weight packing format. This only works for fp16 models. 8 bit quantization is symmetric, 4 bit quantization is asymmetric. Args: `group_size`: parameter for quantization, controls the granularity of quantization, smaller size is more fine grained `bit_width`: bit width of the quantized weight. `packing_bitwidth`: bit width of the packed weight, should be 8 or 32. Can have performance impacts depending on hardware. `mode`: if set to "dynamic", activations are quantized at runtime; default is "weight_only" (weight-only quantization). `set_inductor_config`: if True, adjusts `torchinductor` settings to recommended values. r  bit_widthNpacking_bitwidth weight_onlymodeTr#cltjdtjddS)Nz1torchao.quantization.GemliteUIntXWeightOnlyConfigz`GemliteUIntXWeightOnlyConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.r%rs rr(z*GemliteUIntXWeightOnlyConfig.__post_init__sE $$ ?      p     r)rrrr*r r r+r,rrrstrr#r-r(rkrrrrs  !$J ###Is&*hsm***'D(3-''' $$$$     rrrec |j}|j}|j}|j}|jr#t jj|j }ddl m }|dkrdnd}t|fi|||||||} tj| d|_ t!jt$||_|S)Nr)get_gemlite_aqt_kwargsrTF)r rrruse_hqqr)r rrrr#r7r8r9r:r-torchao.prototype.dtypes.uintx.gemlite_layoutrrrrrrrrr) rrr rrrrrrrxs r$_gemlite_uintx_weight_only_transformr#s"J I. ;D !H"EEGGG ]FTTTTTT1nndd%G)   !-      JH&&z&GGFM();VDDF MrceZdZUdZdZeed<edZe eed<dZ e ed<e j Ze e ed <d Ze ed <d Ze e ed <ejZeed<ejZeed<dZeed<dZd S)Int4WeightOnlyConfiga Configuration for int4 weight only quantization, only groupwise quantization is supported right now, and we support version 1 and version 2, that are implemented differently although with same support. In version 2, different target are mainly distinguished by `packing_format` arg, and in version 1, mainly by `layout`. Args: `group_size`: parameter for quantization, controls the granularity of quantization, smaller size is more fine grained, choices are [256, 128, 64, 32], used in both version 1 and 2 `int4_packing_format`: the packing format for int4 tensor, used in version 2 only `int4_choose_qparams_algorithm`: variants of choose qparams algorithm to use for int4, currently support TINYGEMM ("tinygemm") and HQQ ("hqq"), used in version 2 only `layout`: layout type for quantized tensor, default is `TensorCoreTiledLayout(inner_k_tiles=8)`, used in version 1 only `use_hqq`: whether to use hqq or default quantization mode, default is False, used in version 1 only `zero_point_domain`: data type of zeros points, choices are [ZeroPointDomain.FLOAT, ZeroPointDomain.INT, ZeroPointDomain.NONE], used in version 1 only `set_inductor_config`: if True, adjusts `torchinductor` settings to recommended values. used in both version 1 and 2 `preserve_zero`: whether to preserve zero, default is None. Will be set to True if zero_point_domain is ZeroPointDomain.INT, used in version 1 only `version`: version of the config to use, only subset of above args are valid for version 1, and subset of above args are valid for version 2, default is 2, see note for more details Note: Current state for Int4WeightOnlyConfig is that it supports both v1 (legacy) and v2 For v2 (version = 2), only `group_size`, `int4_packing_format`, `int4_choose_qparams_algorithm` and `set_inductor_config` are valid, all other args will be ignored For v1 (version = 1), only `group_size`, `layout`, `use_hqq`, `zero_point_domain`, `preserve_zero` and `set_inductor_config` are valid, we plan to deprecate v1 in torchao 0.15 to make this config less confusing rr ) inner_k_tilesr!FrrhTr#Nrgint4_packing_formatint4_choose_qparams_algorithmrrGcDtjddS)Nz)torchao.quantization.Int4WeightOnlyConfigrrrrs rr(z"Int4WeightOnlyConfig.__post_init__ls $$%PQQQQQr)rrrr*r r+r,rr!r rr-rUNONErhr#rgr2PLAINrr0TINYGEMMrrGr(rkrrrrCs4J.C.CRS.T.T.TFH* +TTTGT3B3Gx0GGG $$$$$(M8D>(((->-D*DDD"+"#=GSRRRRRrrr`c.|j}|j}|j}|j}|j}|j}|jd|zdkr'td|jd||Stdt|j dz D|gz}|j dkr t|}|tjkr|t jksJd|d |t jkr#t'j||t*j } | S|t jkrt1j||} | S|t jkrt5j||} | S|t jkrt9j||} | S|t jkrt;j||| } | St=d ||j dksJt?j d tBj"} t*j#} d} d} d}tI|tJr|j&n t*j}tO|tP)vs$JdtP)|tTj+kr!tPtO|d}n4|tPtO|vsJdtP||tTj,kr!tI|tZr t*j#}|j.|j.nt^tO|}tI|t`r0tBj1} |dks||jdks Jd|te|| || | | |||||| } | S)NrorzZSkipping quantizing weight with int4 weight only quantization because the shape of weight z# is not compatible with group_size cg|]}dSrErkrMrs rrOz5_int4_weight_only_quantize_tensor..:::a:::rrErzBInt4ChooseQParamsAlgorithm.HQQ is not supported by packing format zF, it's only supported by Int4PackingFormat.TILE_PACKED_TO_4D currentlyactivation_dtype)rz!Unsupported int4 packing format: zConfig Deprecation: version 1 of Int4WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2948 for more detailsr6gư>zOnly support layout: zLayout only support rzQMarlinSparseLayout only supports 128 group size or per channel quantization, got )rrgrhr5r)3r r!rrrhrrloggerinfotuplerSndimrGrtr0HQQr2TILE_PACKED_TO_4D PRESHUFFLEDr4rorrZrr5 PLAIN_INT32r3 MARLIN_SPARSEr1r6rr&r'rTrrrrr]rLAYOUT_TO_ZERO_POINT_DOMAINkeysrUrrrrrgLAYOUT_TO_PRESERVE_ZEROSrrr)rrr r!rrrhrr=rxr r rrrrrgs r!_int4_weight_only_quantize_tensorrus6 "J ]FnG$*$H!0 4 |B*$))  gioiu g g[e g g    ::5q#9#9:::j\IJJJ ~*%% (,F,J J J&*;*MMMMXUhXXXNMM "3"? ? ?.6!&J   $5$; ; ;#+J  $5$A A A-5J  $5$C C C/7J  $5$G G G19.KJ  VATVVWW W >Q     M X)L;LII C"6=99M u~ <<6;;== = = =D ; @ @ B BDD > = =O0007V EaH $?V $MMMM H#>v#F H HNMMO///Jv}4U4U/ ;   +  %d6ll 3&,-- ", S  J&,r2B$B$B$B l`j l l%C$BB* )#+   J rcB|jr#tjjt |ds Jdt |j|}tj |d|_tj t||_|S)Nrgapplying int8 weight only quant requires module to have weight attribute but {module} does not have oneFr)r#r7r8r9r:rrrrrrrrrrrrrxs r_int4_weight_only_transformrs!H"EEGGG 68 $ $ , $36=&IIJH&&z&GGFM();VDDF Mrc"eZdZUdZdZeed<dS)'Float8DynamicActivationInt4WeightConfigaiConfiguration for apply float8 dynamic per row quantization and int4 per group weight quantization to linear (only group_size 128 is supported right now since underlying kernel used only supports 128 and above and no benefits of making it bigger) Args: `int4_packing_format`: how the weight is packed, only preshuffled is supported preshuffledrN)rrrr*rr2r,rkrrrrs0.;*:::::rrct|ds Jd|j}|dks Jd||j}d}tdt |jdz D|gz}t j|j|tj }tj |d |_tj t||_|S) Nrrrz?only preshuffled int4_packing_format supported right now, got: rcg|]}dSrrkrs rrOzD_float8_dynamic_activation_int4_weight_transform.. rrrErFr)rrrrrSrr4ror float8_e4m3fnrrrrrr)rrrrr r=rxs r0_float8_dynamic_activation_int4_weight_transformrs 68 $ $ , $!4 - / / /_J]__ 0 / /]FJ::5q#9#9:::j\IJJJ&. ,J H&&z&GGFM();VDDF MrczeZdZUdZdZeeed<eZ ee ed<dZ e ed<dZ eed<d ZdS) Int8WeightOnlyConfiga Configuration for applying int8 weight-only symmetric per-channel quantization to linear layers. Args: group_size (version 1) - Controls the granularity of quantization. If None, applies per-channel quantization. Otherwise, applies per-group quantization with the specified group size. granularity (version 2) - Quantization granularity. PerRow() for per-channel quantization, PerTensor() for per-tensor quantization. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. Nr  granularityTr#rErGctjd|jdkr|jJd|jdSdS)Nz)torchao.quantization.Int8WeightOnlyConfigrz1Only support version 2 with group_size=None, got )rrrrGr rs rr(z"Int8WeightOnlyConfig.__post_init__>sW $$%PQQQ <1  ?**UDOUU+**  **r)rrrr*r r r+r,rLrrIr#r-rGr(rkrrrr+s  !%J $$$)/K+&111 $$$$GSrrrac|jdkrtjdtj}t j}t jt jj }t j }|j }| |j d}tdt|dz D|gz}t!||||||}n8|jdksJd|jt#j||j}|S) NrEzConfig Deprecation: version 1 of Int8WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more detailsrocg|]}dSrrk)rMrs rrOz5_int8_weight_only_quantize_tensor..V???!A???r)rrrUnexpected version: )r)rGr&r'rTrrrrrrint64r rrrSrjrr7ror) rrr r rrr r=rxs r!_int8_weight_only_quantize_tensorrJs ~  \   #, z k%-((, ;&  b)J??uVZZ\\A-='>'>???:,NOO -    -    ~"""$K6>$K$K"""'F * * , *9''  +5AA (  & 1)    F Mrc tj}tj}d}d}d}t ||t ||||||jtjkr tjndSr) rTrrrrrYr]float16rrs r(_int8_symm_per_token_reduced_range_quantrsk(L:L CII # !!$$ %&W %=%=EMM4    rc tj}tj}d}d}d}|jddkr|St ||t ||||||jtjkr tj ndS)NrrrrEr) rTrrrrrrYr]rrrs r4_int8_symm_per_token_reduced_range_quant_noop_decoders(L:L CIIwqzQ'  %a ( ( )*EM)A)A t    rc t|tjt|tjtjt jtjjtj S)N)r r=r rrrh) rrTrrYrrrrrrUrrs rr;r;sO #  *,Q//ZM K & & *).   rc t|tjt|tjddtjt jtjjtj t S)Nr2r3) r r=r rrrrrhr5) rrTrrYrrrrrrUrrrs rr<r<s^ #  *,Q//ZM K & & *).'))    rr c tt|t|tj|t dS)N mm_configr=rr r5)rrYrrrrr s r_float8_cutlass_quantrs? & ,Q//M!t,,,    rcpt|t|tj|t S)Nr)rrYrrrrs r_float8_cutlass_quant_sparsers: & ,Q//M!'))    rceZdZUdZeZeeed<e j Z ee ed<dZ e ed<eZeed<dZe ed<d Zeed <d Zd S) %Int8DynamicActivationInt8WeightConfiga Configuration for applying int8 dynamic symmetric per-token activation and int8 per-channel weight quantization to linear layers. Args: layout: Optional[Layout] = PlainLayout() - Tensor layout for the quantized weights. Controls how the quantized data is stored and accessed. act_mapping_type: Optional[MappingType] = MappingType.SYMMETRIC - Mapping type for activation quantization. SYMMETRIC uses symmetric quantization around zero. weight_only_decode: bool = False - If True, only quantizes weights during forward pass and keeps activations in original precision during decode operations. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Int8Tensor r!r"Fweight_only_decoderTr#rErGcDtjddS)Nz:torchao.quantization.Int8DynamicActivationInt8WeightConfigrrs rr(z3Int8DynamicActivationInt8WeightConfig.__post_init__s) $$ H     rN)rrrr*rr!r r#r,rTrr"rr-rLrrIr#rGr+r(rkrrrrs +{}}FHV ,,,.9.Ch{+CCC$$$$%vxxK''' $$$$GS     rrr^c h|jdkr|j}|j}|j}|jd}|dkr't d|jd||Stj}tj }d}tj } tj tjj} tj} |rt"} n|tjkrt$} nt&} ||} t)||| | | | ||}t+|| }ndd lm}|jt3t5hvs Jd |j}|j}|jtjks Jd |jd ksJd |jt7j|||||j}|S)NrErozKSkipping applying Int8DynamicActivationInt8WeightConfig to weight of shape z because `in_feature` is <= 16: ctdt|dz D|jdgzS)Ncg|]}dSrrkrs rrOzg_int8_dynamic_activation_int8_weight_quantize_tensor..get_weight_block_size..s888!888rrEro)rrSrjrrs rget_weight_block_sizezS_int8_dynamic_activation_int8_weight_quantize_tensor..get_weight_block_sizes@88U15577Q;%7%7888AGBK=HII Ir)rrr5rhr)QuantizeTensorToInt8Kwargsz'Only PerRow and PerTensor are supportedz0asymmetric dynamic quant not supported currentlyrr)rr )ract_quant_kwargs)rGr!r"rrrrrTrrUrrrrrrrrrr rrO9torchao.quantization.quantize_.workflows.int8.int8_tensorrrrLrMr7ro)rrr!r"r in_featuresr weight_zero_point_domainrr rrrr=rxquantized_weightrrBact_granularitys r4_int8_dynamic_activation_int8_weight_quantize_tensorr s? ~!2#6l2& "   KKA^d^jAA3>AA   M#, #2#7  J J Jz k%-((, ;  ?S   ;#888#K  #> **622 -    -6    :*FVWW      !fhh %<<<< 5=<<$/ ,&+*???? >@??~"""$K6>$K$K"""&- *77+#4    rcB|jr#tjjt |ds Jdt |j|}tj |d|_tj t||_|S)Nrzyapplying int8 dynamic activation int8 weight quant requires module to have weight attributebut {module} does not have oneFr)r#r7r8r9r:rrrrrrrrrrrs r._int8_dynamic_activation_int8_weight_transformrVs!H"EEGGG 68 $ $ + $F vJH&&z&GGFM();VDDF MrcbtjdttS)z Applies int8 dnynamic symmetric per-token activation and int8 per-channel weight quantization + 2:4 sparsity to linear layers. aint8_dyanmic_activation_int8_semi_sparse_weight() will be deprecated at a later release. Please use the layout kwarg in Int8DynamicActivationInt8WeightConfig instead. from torchao.dtypes import SemiSparseLayout Int8DynamicActivationInt8WeightConfig(layout=SemiSparseLayout())r!)r&r'rrrkrrr_r_is8  M G 18H8J8J K K KKrcNeZdZUdZeZejed<dZ e ed<dZ e ed<dZ dS) Float8WeightOnlyConfigaV Configuration for applying float8 weight-only symmetric per-channel quantization to linear layers. Args: weight_dtype (torch.dtype): The target data type for weight quantization. Default is torch.float8_e4m3fn. set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Float8Tensor (default) Note: The actual matmul will be computed in original precision of the weight tensor. rATr#rrGcDtjddS)Nz+torchao.quantization.Float8WeightOnlyConfigrrs rr(z$Float8WeightOnlyConfig.__post_init__s $$%RSSSSSrN)rrrr*r$rArr]r,r#r-rGr+r(rkrrrrxsi  !+L%+*** $$$$GSTTTTTrrrbc |jdkrtjdddlm}t dt |dz D|jdgz}||||j dtd}nG|jd ksJd |j|j }tj ||t }|S) NrEzConfig Deprecation: version 1 of Float8WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2649 for more detailsr)rcg|]}dSrrkrs rrOz4_float8_weight_only_quant_tensor..rrrorrfr=r rr5rr) float8_dtyper)rGr&r'torchao.dtypesrrrSrjrrArr/rorL)rrrr=rxrAs r _float8_weight_only_quant_tensorrs ~  ^    >=====??uVZZ\\A-='>'>???6$K$K"""* !) 688    rc|jr#tjjt ||s Jdt |trt|}tt|||}t||tj |dtjt#t$|j|||_|S)Nzsapplying float8 weight only quant requires module to have {parameter_name} attribute but {module} does not have oneFrr)r#r7r8r9r:rrr&_unwrap_float8_linearrrrwrrrrrrrrrs r_float8_weight_only_transformrs!H"EEGGG 6> * * , * &,''/&v..7''  +5AA (  & 1)    F Mractivation_granularityrr|r}c | Jdt|tr|jtjks Jdt |j|}|-t|||tjtd}nAt|ts Jdt||||td}|S)zThis function is used to quantize the input activation tensor for an aqt_float variant. If scale is not provided it will be dynamically calculate the scales otherwise it will use the provided scale. Nz8Zero point is not supported for dynamic FP8 quantizationzFPerRow quantization only works for bfloat16 precision input activationrr7Static quantization only supports PerTensor granularity)rfr=r|r r5) rrLr]rrZr?rrrrrMr)rrrr|r}r= activations r _input_activation_quant_func_fp8rs   B   (&11 w%.((( T)(( )?@@J }/!)  4000    0)<<  E  <7!) 4000     rc |dvs Jd|d|jdd\}}|dzdko|dzdk}|s)td|jd |d |d |S) z Check if a weight tensor meets float8 quantization requirements. Args: weight (torch.Tensor): The weight tensor to check Returns: bool: True if the tensor can be quantized to float8, False otherwise )rrz6float8 quantization only works for 2/3-D tensors, got zD tensorrNrrz+Skipping float8 quantization: weight shape z: is not compatible with _scaled_mm. Both input dimension (z) and output dimension (z) must be multiples of 16. )rjrrr)rout_dimin_dim is_compatibles r_fp8_mm_compatr s ::<<   W VVV    l233'OGVb[A%>GbLA,=M    j&, j j%+ j jEL j j j   rceZdZUdZeZejed<eZ ejed<dZ e e e ee fed<dZe eed<dZe eed<dZe eed<ejZeed <d Zeed <d Zeed <dZdS))Float8DynamicActivationFloat8WeightConfigax Configuration for applying float8 dynamic symmetric quantization to both activations and weights of linear layers. Args: activation_dtype (torch.dtype): The target data type for activation quantization. Default is torch.float8_e4m3fn. weight_dtype (torch.dtype): The target data type for weight quantization. Default is torch.float8_e4m3fn. granularity (Optional[Union[FP8Granularity, List[FP8Granularity]]]): The granularity for quantization. Can be either a single granularity (applied to both activations and weights) or a tuple of two granularities (one for activations, one for weights). If None, defaults to PerTensor for both. Currently both quantizations need to be the same type. And only PerTensor and PerRow are supported. mm_config (Float8MMConfig): Configuration for the matrix multiplication. Default uses fast accumulation. activation_value_lb (Optional[float]): the lower bound for activation value for calculating scale activation_value_ub (Optional[float]): the upper bound for activation value for calculating scale kernel_preference (KernelPreference): kernel preference for ops like matmul, grouped matmul etc. by defalut (KernelPreference.AUTO) it will be chosen for user based on hardware or other information, this only needs to be set in weight set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Float8Tensor (default) rrANrractivation_value_lbactivation_value_ubkernel_preferenceTr#rrGcltjdt|j\}}||g|_d}t |jr@|jtjtj fvs Jd|j dks Jdd}|j t||_ dSdS)Nz>torchao.quantization.Float8DynamicActivationFloat8WeightConfigT unimplementedrFuse_fast_accum) rrrr+rr*rr.rWTORCHrGrr')rrrBdefault_use_fast_accums rr(z7Float8DynamicActivationFloat8WeightConfig.__post_init__6s $$ L   6L  6 6 2 234FG!% ,T-= > > +) % &.<1$$$o$$$%* " > !+;QRRRDNNN " !r)rrrr*r$rrr]r,rArr r r(r rr'rfloatrr.rWrr#r-rGr+r(rkrrrrs(%/ek... *L%+***IMK%^0D DEFMMM*.Ix'...+/%///+/%///*:*?'??? $$$$GSSSSSSrrrfc |j}|j}|j}|j}|j}|j}|j}t||\} } |dvr_t| trt| ts Jd|j ddzdks|j ddzdkr|Snt|s|St| tr|jtjks Jd|jdkrt%jdt)|j dd| } |d kr t+dgt-| z} t/|| |tjt3|  } t4} | |d }t7| | | } nK|jdksJd|jt9|| |||}t;j||| |||} | S)N)rzH4D/5D tensor only supports per tensor activation and weight quantizationrrrEzBPerRow quantization only works for bfloat16 precision input weightzConfig Deprecation: version 1 of Float8DynamicActivationFloat8WeightConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2649 for more detailsrrrrrr quant_kwargsrr) hp_value_lb hp_value_ubr)rrrrr)rrArrrrrr)rjrrMrr rLr]rrZrGr&r'r?rrtrrrrrOr<r/ro)rrrrArrrrrrrBr=rrinput_quant_kwargsrs r8_float8_dynamic_activation_float8_weight_quantize_tensorr!Rs.&L$K I 4 40K(((1<..zz||v0)<< V  B B  V V U V V  <?R 1 $ $ Q"(<(A(AM)B F # # $f-- |u~--- P.--~  q   $FL$57IJJ ::<<1  sT*%5%5566J5!%  9555    <&< 0   : .=O   ~"""$K6>$K$K"""7  "++/    (/ %*/-     rc<tsts Jd|jr#tjjt||sJd|dd|dzt|trt|}tt|||}t||tj|dt#jt't(|j|||_|S) NzPFloat8 dynamic activation quantization is only supported on CUDA>=8.9 and MI300+zKapplying float8 dynamic activation quant requires module to have parameter z attributez but z does not have oneFrr)rCrBr#r7r8r9r:rrr&rr!rrwrrrrrrrrrs r2_float8_dynamic_activation_float8_weight_transformr#sR   (**Z ,!H"EEGGG 6> * *pVdppp ,& , , , - *&,''/&v..O''  +5AA (  & 1)    F MrcheZdZUdZeZeed<eZ e j ed<e Z e j ed<dZdS)rha Applies float8 dynamic quantization to activations and float8 quantization followed by compression to sparse semi-structured tensor to weights of linear layers. Args: `layout`: layout type for quantized weight tensor, only supports `CutlassSemiSparseLayout` at the moment. `activation_dtype`: data type for quantized activation tensor. `weight_dtype`: data type for quantized weight tensor. r!rrAcDtjddS)NzHtorchao.quantization.Float8DynamicActivationFloat8SemiSparseWeightConfigrrs rr(zAFloat8DynamicActivationFloat8SemiSparseWeightConfig.__post_init__s) $$ V     rN)rrrr*rr!r#r,r%rrr]r$rAr(rkrrrhrhso-,..FF...$.ek... *L%+***     rrhcts Jdt|trt|}|j}|j}|j}|j}t|tstd|dt||}t|td|i}tj|d|_t!jt$||_|S)Nz2Float8 quantization is only supported on CUDA>=9.0z;Only CutlassSemiSparseLayout layout is supported. Received rrr rFr)rDrr&rrrArr!rrrrOrrrrrrrr)rrrrArr!s r>_float8_dynamic_activation_float8_semi_sparse_weight_transformr's   TT TTT &,''/&v.. ]F&L. ]F f5 6 6 ! S& S S S   *&, ? ?F +$&67F H&&vU&CCFM();VDDF MrceZdZUdZejed<eZej ed<eZ ej ed<dZ e e eeeeffed<edZe eed <dZeed <d ZdS) (Float8StaticActivationFloat8WeightConfiga] Configuration for applying float8 static symmetric quantization to Args: scale (torch.Tensor): The scale tensor for activation quantization. activation_dtype (torch.dtype): The target data type for activation quantization. Default is torch.float8_e4m weight_dtype (torch.dtype): The target data type for weight quantization. Default is torch.float8_e4m mm_config (Float8MMConfig): Configuration for the matrix multiplication. Default uses fast accumulation. set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. r|rrANrTrrr#cltjdtjddS)Nz=torchao.quantization.Float8StaticActivationFloat8WeightConfigz`Float8StaticActivationFloat8WeightConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.r%rs rr(z6Float8StaticActivationFloat8WeightConfig.__post_init__sE $$ K      |     r)rrrr*rrr,r$rr]rArr r r(r r'rr#r-r(rkrrr)r)s   <$.ek... *L%+***  neNN$BCCD   +9.*M*M*MIx'MMM $$$$     rr)rgc tsts Jdt|trt |}|j}|j}|j}|j}|j }|j r#tj j |j}t!|\}} t|t"s Jdt%|s|St'|j| } t+|| |t,jt1|} t2} ||d} t5| | |d| } t,j| d|_t;jt>||_ |S) NzMFloat8 static activation quantization is only supported on CUDA 8.9 and aboverrrr)r|r}rFr)!rCrBrr&rr|rrArrr#r7r8r9r:rr+rMr r?rrrrrrr@rrrrrr)rrr|rrArrrrrBr=rrr s r1_float8_static_activation_float8_weight_transformr,(s   (**W ,&,''/&v.. LE.&L$K I !H"EEGGG ]F1G 1T1T.. ,i 8 8A 8 & ! !  .@AAJ1!My111 8"8, U' H&&'7u&MMFM();VDDF MrcfeZdZUdZejed<dZeed<dZ eed<dZ e ed<d Z e ed <d Z d S) UIntXWeightOnlyConfiga Configuration for applying uintx weight-only asymmetric per-group quantization to linear layers, using uintx quantization where x is the number of bits specified by `dtype` Args: `dtype`: torch.uint1 to torch.uint7 sub byte dtypes `group_size`: parameter for quantization, controls the granularity of quantization, smaller size is more fine grained, defaults to 64 `pack_dim`: the dimension we use for packing, defaults to -1 `use_hqq`: whether to use hqq algorithm or the default algorithm to quantize the weight `set_inductor_config`: if True, adjusts `torchinductor` settings to recommended values. r]@r ropack_dimFrTr#cltjdtjddS)Nz*torchao.quantization.UIntXWeightOnlyConfigz`UIntXWeightOnlyConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.r%rs rr(z#UIntXWeightOnlyConfig.__post_init__vs= $$%QRRR  i     rN)rrrr*rr]r,r r+r0rr-r#r(rkrrr.r.as   ;JHcGT $$$$     rr.rcc|j}|j}|j}|j}|jr#t jj|j }ddl m }tj tjtjtjtjtjtjtjh}||vs Jd|t*j} d|f} |rb|tjkrt.d||\} } tj}d} d}t2j}d}t7}nSd\} } tjtjj} tj}t2j }d}tC|| }tE|| | || | | ||||| }tj#$|d |_ tKj&tN||_(|S) Nr)rSzUnsupported dtype for hqq: rEzgRecommended to use `Int4WeightOnlyConfig(group_size, use_hqq=True, version=1)` for the best performanceFNNT)r]r0)rrrrrhrgr5rr))r]r r0rr#r7r8r9r:r%torchao.quantization.quant_primitivesrSruint1uint2uint3uint4uint5uint6uint7rrTrrr\rUFLOATrrrrrrrrrrrrrrr)rrr]r r0rrrSSUPPORTED_DTYPESr r=rrrrrhrgr5rxs r_uintx_weight_only_transformr>s LE"JHnG !H"EEGGG ]FMMMMMM           $ $ $ $&KE&K&K $ $ $)LZJ> EK   NNy    7u= 9 +1 --) 9k%-((, ;+/ EH===)  )+#   JH&&z&GGFM();VDDF Mrintx_unpacked_tensor hp_tensorrc t|tsJ|j||_t|j\}}t ||j|j|jtj |||_ dS)ad Adjusts the scale_dtype on IntxUnpackedToInt8Tensor. Updating the scale dtype requires updating the qdata because qdata is calculated after the scale. This is used in IntxWeightOnlyConfig and Int8DynamicActivationIntxWeightConfig to make version=2 and version=1 numerically equivalent when the scale_dtype differs from the input dtype ) output_dtyperrN) rr;r|rsrSr rVr=r}rrqdata)r?r@rqminqmaxs rrprps *,D E EEE E!5!;!>!>{!K!K()=)JKJD$!0'"'Z"""rceZdZUdZejZejed<e dZ e ed<e j Ze ed<dZeejed<eZeed<ejZeed <ejZeed <d Zeed <d ZdS)IntxWeightOnlyConfiga Configuration for quantizing weights to torch.intx, with 1 <= x <= 8. Weights are quantized with scales/zeros in a groupwise or channelwise manner using the number of bits specified by weight_dtype. args: `weight_dtype`: The dtype to use for weight quantization. Must be torch.intx, where 1 <= x <= 8. `granularity`: The granularity to use for weight quantization. Must be PerGroup or PerAxis(0). `mapping_type`: The type of mapping to use for the weight quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. `scale_dtype`: The dtype to use for the weight scale. `layout`: The layout to use for the packed weight tensor: - QDQLayout: this layout is designed for export to ExecuTorch.this layout represents the quantization with Q/DQ quant primitives, and is intended for export applications like ExecuTorch. `intx_packing_format`: The format to use for the packed weight tensor (version 2 only). `intx_choose_qparams_algorithm`: The algorithm to use for choosing the quantization parameters. `version`: version of the config to use, only subset of above args are valid based on version, see note for more details. Note: Current state for IntxWeightOnlyConfig is that it supports both v1 (legacy) and v2. * `intx_packing_format` is used for version 2. * `layout` is only used for version 1. rArrr Nrr!rErFrrGc tjd|jdt ddDvsJd|jt |jttfsJd|jt |jtr'|jj dksJd|jj |j tj tj tjfvsJd |j dS) Nz)torchao.quantization.IntxWeightOnlyConfigc>g|]}ttd|SrJrKrLs rrOz6IntxWeightOnlyConfig.__post_init__.. rPrrErQrRz1granularity must be PerAxis or PerGroup, but got rrazvmapping_type must be MappingType.ASYMMETRIC, MappingType.SYMMETRIC, or MappingType.SYMMETRIC_NO_CLIPPING_ERR, but got )rrrrArSrrrJrKrTr rTrrrUrs rr(z"IntxWeightOnlyConfig.__post_init__ sF $$%PQQQ $T$TaQR $T$T$TTTT ^4K\ ^ ^UTT$*Wh,?@@  R@P R R  @ d& 0 0 #(A---O8H8MOO.--  "  !  1%    YFJFW Y Y      r)rrrr*rrrAr]r,rJrrIrTrr rr rr!r#r:r^rEr8r_rFrGr+r(rkrrrGrGs2!& L%+***&wqzzK))) + 5L+555)-K%+&---Y[[FF   ->-O*OOO")"#=GS     rrGc |j}|j}|j}|j}|j}|j} |j} |dkrd} n?|dkrd} n$td|t|tr|j } nRt|tr+|j dksJd|j |j| } ntd||dkrd| f} n |dksJd| ddf} |jdkr|jt jkrq|4|jt&jkr|t&j}t/j|| ||||| }|||jkrt3||||Std | |jt4jks Jd |jdksJt9jd t<|\}}t?||| t&j|||t&j|t@j!ktDj#| }|S)NrrorrEz>IntxWeightOnlyConfig only works for 2-d and 4-d Tensors, got: rraz-granularity must be PerGroup or PerAxis, got )r r/r0rFrbz(version 1 only supports affine algorithmzConfig Deprecation: version 1 of IntxWeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2967 for more detailsre)$rArr rr!rErFrjrrrKr rJrTrrGr:r^r]rrrsrr;rorpr8r_r&r'rSrrTrrUrr)rrr/r0rArr rr!rErF input_dimr r=rxrrs r!_intx_weight_only_quantize_tensorrL s&L$K&L$K ]F 4$*$H! zz||q     [VZZ\\ [ [   +x((X + K ) )X1$$$ FK4D F F%$$\), VVVWWW zz||q_ zz||q    Q*  ~  %):)K K K ,1B1HEK1W1W$5$8$8$D$D!19))"3.KJ&;&,+F+F;  QQ     M X3<@Iy %!Z#{'<<)-   F Mrct|ds Jdt|j|||}tj|d|_t |tjr tj t||_ n9t |tj rtj t||_ |S)Nrzgapplying intx weight only quant requires module to have weight attribute but {module} does not have oner.Fr)rrLrrrrrrrrrr Embeddingr)rrr/r0rxs r_intx_weight_only_transformrOu s 68 $ $ , $3 !+ J H&&z&GGFM&")$$L!,-?HH FBL ) )L!,-BFKK Mrc<eZdZUdZeed<eed<dZeed<dZdS)FPXWeightOnlyConfigaSub-byte floating point dtypes defined by `ebits`: exponent bits and `mbits`: mantissa bits e.g. fp6_e3_m2, fp6_e2_m3, ... The packing format and kernels are from the fp6-llm paper: https://arxiv.org/abs/2401.14112 github repo: https://github.com/usyd-fsalab/fp6_llm, now renamed to quant-llm For more details for packing please see: :class:`~torchao.dtypes.fpx.FpxTensorCoreAQTTensorImpl` This is experimental, will be merged with `to_affine_quantized_floatx` in the future ebitsmbitsTr#cltjdtjddS)Nz(torchao.quantization.FPXWeightOnlyConfigz`FPXWeightOnlyConfig` will be moving to prototype in a future release of torchao. Please see https://github.com/pytorch/ao/issues/2752 for more details.r%rs rr(z!FPXWeightOnlyConfig.__post_init__ s= $$%OPPP  g     rN) rrrr*r+r,r#r-r(rkrrrQrQ sT JJJ JJJ $$$$     rrQrdc |j}|j}|j}|jr#tjjt|trt|}ddl m }ddl m}|dksJd||j\}}|dzdks |dzdkr2t"d||zd zd |d |d |d |d |S|||} ||| } t&j| d|_t-jt0||_|S)Nr)to_affine_quantized_fpx)FloatxTensorCoreLayoutrz'floatx only works for 2-d Tensor, got: r/z"Skipping floatx quantization floatrErz= because the shape is not compatible with the kernel: in_dim=z , out_dim=z1 expected in_dim % 64 == 0 and out_dim % 256 == 0Fr)rRrSrr#r7r8r9r:rr&rrrVtorchao.prototype.dtypes.floatxrWrjrrrrrrrrrr) rrrRrSrrVrWr r r5rxs r_fpx_weight_only_transformrZ s LE LE ]F !H"EEGGG&,''/&v..666666FFFFFF ::<<1   V VV   lOGV qgmq00  ?1B ? ?U ? ?U ? ?CI ? ?U\ ? ? ?    $$UE22G((99JH&&z&GGFM();VDDF MrceZdZUdZeeZeee e fe d<eeZ eee e fe d<dZ ee d<dZdZd S) ra Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). Args: `fqn_to_config`: typing.OrderedDict[str, Optional[AOBaseConfig]]: an ordered dictionary from (1). fully qualified name (fqn) of module or parameter (2). regex of fully qualified name (in python `re` module regex format), should start with prefix "re:" or (3). "_default" to the config that we want to apply to the module/param or None Config key ordered by precedence: * fully qualified parameter name, e.g. `language.layers.0.q_proj.weight` * fully qualified module name, e.g. `language.layers.0.q_proj` * regex for parameter names, must start with `re:`, e.g. `re:language\.layers\..+\.q_proj.weight`. The first regex that matches will be applied. * regex for module names, must start with `re:`, e.g. `re:language\.layers\..+\.q_proj`, whichever regex fully matches the module fqn first will be applied (order of keys for dictionary are kept consistent since we are using OrderedDict) * "_default", fallback if no match for all previous keys (Note, when using `_default`, the config is applied to all modules, to apply it to only a subset of modules, e.g. with some types, it's better to filter the modules that we don't want to quantize before hand and configure them to None, e.g. `{"re:.+norm.+": None, "_default": linear_config}`) "_default" is not supported when filter_fn is not specified. `module_fqn_to_config`: typing.OrderedDict[str, Optional[AOBaseConfig]]: To maintain BC with ModuleFqnToConfig, to be deprecated later `version`: int: Version of config to use. Note: - The order of patterns in the OrderedDict may matter as only the first matching pattern is applied - "_default" is ignored for parameter replacement. )default_factoryrmodule_fqn_to_configrErGctjdt|jdkr7t|jdkr|j|jkrt dt|jdkr$t|jdkr |j|_t|jdkr$t|jdkr |j|_d|jvrtjddSdS)Nz torchao.quantization.FqnToConfigrzP`fqn_to_config` and `module_fqn_to_config` are both specified and are not equal!rzConfig Deprecation: _default is deprecated and will no longer be supported in a future release. Please see https://github.com/pytorch/ao/issues/3229 for more details.) rrrrrr]rr&r'rs rr(zFqnToConfig.__post_init__ s $$%GHHH " # #a ' 'D-..22"d&???b  t( ) )A - -#d6H2I2IQ2N2N!%!:D  t! " "Q & &3t/H+I+IQ+N+N(,(:D % + + + My      , +rcxddgd|jDdS)N z FqnToConfig({c3.K|]\}}d|d|dVdS)z 'z': ,Nrk)rMkeyvalues r z&FqnToConfig.__str__.. sM"U0#//u///rz}))rritemsrs r__str__zFqnToConfig.__str__ sZyy &*&8&>&>&@&@       rN)rrrr*rrrOrderedDictTyperr r r,r]rGr+r(rgrkrrrr s@CH%#CCCM?3(>#>?JO#JJJ/#x /E*EFGS2      rrfqncd}g}tt|D]N\}\}}|t|vr5t |dkr|d|n|}|||||fOt|D]\}}}}||jvrd}|j|} | ||7tt| } t| tvr| || |}vtt| d|s@||jvr7|j|} | &tt| } | || S|S|D]\}}}}|jD]} | drtj| d d|rod}|j| } | ^tt| } t| tvr| || |}tt| d|ss|jD]k} | drTtj| d d|r7|j| } | (tt| } | || cSl|sC|jd d} | &tt| } | || S|S) awThis function expects a module that either is specified in FqnToConfig or has a parameter that is specified in FqnToConfig. Args: module (torch.nn.Module): The module to be processed. fqn (str): The fully qualified name of the module containing the parameters. config (FqnToConfig): Configuration object containing regex patterns / fqn mapped to quantization configurations. Returns: torch.nn.Module: The modified module with quantized parameters. Raises: NotImplementedError: If the quantization configuration is not yet supported for parameter quantization. FrrrTNrzs does not yet support parameter quantization! Please see https://github.com/pytorch/ao/issues/3252 for more detailsre:rr) enumeratertnamed_parametersdirrrrrr=r+CUSTOM_PARAM_QUANTIZATION_SUPPORTED_CONFIGSr startswithre fullmatchget) rrirparameter_config_foundtop_level_paramsirparam parameter_fqncrpatterns rrr+ sv&#&/V5L5L5N5N0O0O&P&POO" "NE S[[ ( (-0XX\\3)))))~   # #Q}$M N N N488H3I3I/>5- F0 0 0%) "$]3Ay $$Q''''2477;77III$WVQ~NNNFF-77XXX 1" "cV-A&A&A   % =.tAww7G761%% %M4D  />5-+  G!!%(( R\'!""+}-U-U )-&(1=6tAww?GAww"MMM!(>!R!R!R1#Aww\\\  ".+ . .G!!%(( .R\'!""+s-K-K .(1=6tAww?G"761----- "&  $ $Z 6 6 =.tAww7G761%% % Mrc||jvr%|drJd|ddS|jD]7}|dr tj|dd|rdS8dS)aeCheck if a given fqn matches the exact fqn or regex pattern specified in FqnToConfig. Args: fqn (str): The fully qualified name of the module. config (FqnToConfig): Configuration object containing regex patterns or raw FQNs for quantization. Returns: bool: True if the fqn is specified in FqnToConfig. False otherwise. rkzError: Exact match but regex z specified.TrNF)rrprqrr)rir!maybe_module_or_param_fqn_patterns rrr s f""">>%((  torchao.quantization.utilsr?Atorchao.quantization.weight_tensor_linear_activation_quantizationr@ torchao.utilsrArBrCrDrGrFGPTQrHrrIrJrKrLrM"linear_activation_quantized_tensorrNrOlinear_quant_modulesrPrQqatrRquant_primitivesrSrTrUrVunifiedrWrXrY getLoggerrr__all__r<rrrrrvrr[rZrrr-rrrrrrrDevicer\r rrrr]r>r@rrrrrrrerrr`rrrrrrarrrrr;r<r]rrrr^rrr_rrbrrrr rrfr!r#rhr'r)rgr,r.rcr>rprGrLrOrQrdrZrrirorrrrr serializationadd_safe_globalsrkrrrs   ######((((((((>>>>>>>>>>>>>>>>111111 000000000000,,,,,,*('''''88888888555555FEEEEE 655555>======= 10000000,,,,,,  8 $ $   6O12,-O)*O)?+>? 555   ,. ++ sCx) + ++++\&DJCG NNN 9N89N9:N %(/3!7!=>? NNNNb -u| - - - - @@@DDD 0 0 0).e.CM+/ F F 8?F F %(/3!7!=>?F  U[' ( F F F F R5<EL$ELU\,%,5<&      L    <'@&?)+P''# "!"GHH ,004 LLL HOL 1L5<( L   - LLLIHL^ \\\\\L\\ \H,004 MMM 5<( M   - MMMM`"!"GHH ,004  HO 15<(    -   X_ IH,      L    2'@&?)+P''# "!"GHH HO%J X_IH:      <    <65!= "!">?? HO%A@?> )R)R)R)R)R<)R)R )R\-,-?AUVVzzzz"!"677  HO %9  X_   87    ; ; ; ; ;l ; ;  ;"!"IJJ HO%L X_KJ2 < 6-,-?AUVV6"!"677 #  HO  87@ $  |  \    .              | +  \      | +  lEL!          L    >'@&?)+P''# JJJZ"!"GHH HO%J X_IH$ L L L TTTTT\TT T,/.0 0"!"899 # """ HO" "" "  X_ """:9"R%))- && |&*&k& EL ! & & &&&&R5<D6 2S2S2S2S2S 2S2S 2Sl+D*C-/X++' TTTn"!"KLL # """ HO" 5" """ML"J      ,    ("!"UVV HO%XWV<      |    >*C)B,.V**& "!"JKK5 HO5%M555LK5p      L    8.-. "!"788A HOA%:AAA98AH2| 2 7 7 7 7 7 <7 7  7 |,004 VVV5<( V   - VVVVr"!"677 ,004  HO 5<(    -   X_ 876      ,    .,+,=?RSS"!"566 HO%8 X_76@ M M M M M ,M M  M b  ./+Y HOY Y YYYYx  6 I  0 , 29    $$#0(  $      r