.`iCddlmZmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z ddlmZmZmZmZddlmZdd lmZmZmZmZdd lmZeeZGd d e ZGd de ZdS))AnyOptionalN)version) init_logger) LinearBaseLinearMethodBase)QuantizationConfigQuantizationMethods)BITBLAS_OPTIMIZE_FEATURESBITBLAS_SUPPORTED_NUM_BITSBITBLAS_SUPPORTED_SYMMINIMUM_BITBLAS_VERSION)ParallelLMHead)BasevLLMParameterChannelQuantScaleParameterGroupQuantScaleParameterPackedvLLMParameter)set_weight_attrsceZdZdZejZdZeeeZ dZ de de dzde dzde dzd e dzd e d dffd Zd e fd Zed efdZed eejfdZed e fdZed ee fdZe ddee efdee ded efdZedee efd dfdZed edzfdZdejjde d e dfdZ!xZ"S) BitBLASConfigzSConfig class for BitBLAS. Reference: https://github.com/Microsoft/BitBLAS int8 quantized weight_bits group_sizeNdesc_actis_sym quant_methodlm_head_quantizedreturnc0 ddl}tj|jtjtkrt dtn0#t $r#}|} t d| dtd| d}~wwxYw|r|dkrd}t||_ ||_ ||_ ||_ ||_ ||_|j tvr t d|j d td |j t vr t d |j d t d |j} t%d d| D} | |_|j|_| |z|_||_|j|_dS)Nrz2bitblas version is wrong. Please install bitblas>=zQTrying to use the bitblas backend, but could not importwith the following error: zN. Please install bitblas through the following command: `pip install bitblas>=`Fz'BitBLAS does not support weight_bits = z. Only weight_bits = z are supported.z"BitBLAS does not support is_sym = z . Only sym = c3BK|]}||VdSN)isdigit).0cs /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/bitblas.py z)BitBLASConfig.__init__..es/"K"Kqyy{{"K1"K"K"K"K"K"K)bitblasrparse __version__r ImportError ValueErrorsuper__init__rrrrrrr r STORAGE_DTYPEintjoin storage_dtypeTORCH_STORAGE_DTYPEstorage_torch_dtype pack_factornbits ZEROS_MODE zeros_mode) selfrrrrrrr,ebitblas_import_exceptionr6 storage_nbit __class__s r)r2zBitBLASConfig.__init__-s1 , NNN}W011GM'55"B(?BB , , ,'( $D-EDD*ADDD ,  , ,   b((H &$   (!2  #= = =!$:J!!&@!!!  ;3 3 3ET[EE3EEE  * 277"K"Km"K"K"KKKLL *#'#; ';6  /sAA BA;;Bc Xd|jd|jd|jd|jd|jd S)NzBitBLASConfig(weight_bits=z , group_size=z , desc_act=z , is_sym=z, quant_method=))rrrrr)r=s r)__repr__zBitBLASConfig.__repr__psa 1)9 1 1/ 1 1  1 1k 1 1!-  1 1 1 r+cdS)Nr,clss r)get_namezBitBLASConfig.get_nameysyr+c2tjtjgSr%)torchhalfbfloat16rGs r)get_supported_act_dtypesz&BitBLASConfig.get_supported_act_dtypes}s EN++r+cdS)NFrFrGs r)get_min_capabilityz BitBLASConfig.get_min_capabilitys rr+cdgS)Nzquantize_config.jsonrFrGs r)get_config_filenamesz"BitBLASConfig.get_config_filenamess &''r+configkeysdefaultc,|D]}||vr ||cS|S)z1Get a value from the model's quantization config.rF)rTrUrVkeys r) get_from_keyszBitBLASConfig.get_from_keyss6  # #Cf}}c{"""r+c@||dg}||dgd}||dgd}||dgd}||dg}||dgd }|||||||S) Nbitsrr"rFsymrlm_head)rV)rYget_from_keys_or)rHrTrrrrrrs r) from_configzBitBLASConfig.from_configs''99 &&v ~rBB $$Vj\5AA""6E7E::((.1ABB 00)e0TTs Xv|EV   r+c\|ddkp|dd}|dup |dkp|dk}|rj|rhd||}t||SdS)Ncheckpoint_formatr,is_bitblas_formatFgptqz6The model is serialized in {} format. Using {} kernel.)getformatrIloggerinfo)rH hf_quant_cfg user_quantrbis_valid_user_quantmsgs r)override_quantization_methodz*BitBLASConfig.override_quantization_methods ),,   G&**+>FF  $  Q*"6 Q* :Q   "!4 "JQQ  C KK   <<>> !tr+layerprefixBitBLASLinearMethodct|tst|tr|jrt |SdSr%) isinstancerrrro)r=rmrns r)get_quant_methodzBitBLASConfig.get_quant_methodsH eZ ( ( - un - - -262H -'t,, ,tr+r%)#__name__ __module__ __qualname____doc__rKfloat16 TORCH_DTYPEr3getattrr7r;r4boolstrr2rD classmethodr rIlistdtyperNrQrS staticmethoddictrrYr_rlnnModulerrr __classcell__)rAs@r)rr s -KM!'%77JA*A*$JA*+ A* t A* Dj A* A* A*A*A*A*A*A*F #    ,[,ek):,,,[,3[(T#Y((([(@DS#X&*3i:= \  c3h  O    [   t #[,X_.1 ' (r+rceZdZdZeZdZejdej dej dej dej diZ defdZd ejjd ed eed ed edejddfdZd ejjd ed eed ed edejf dZ ddZdZ dd ejjdejdejdzdejfdZdededejfdZdS)rozaLinear method for BitBLAS. Args: quant_config: The BitBLAS quantization config. Tfloat32rwrMr quant_configc||_dSr%)r)r=rs r)r2zBitBLASLinearMethod.__init__s(r+rminput_size_per_partitionoutput_partition_sizes input_size output_size params_dtyperNc ~~|d}||jvrtd||jj} | d} t |} | dkr|| zdkrtd|d| d||| ||jd d |jj ttj |j d |jj d ddd|jj|j jr|j dnd|} | dkrdn|| z} tj | | d ||d} | dkrt!dddi| }nt#dddd| }|jjdkrNttj | | |jjzd |jj d ddd|jj|}nBt'tj | | d ||}t)|| dkrdnddd|d| |d||d|dS)aCreates quantized weights for use in linear operations. The function initializes and returns a dictionary containing quantized weights, scales, and zeros for performing quantized matrix multiplication operations. Args: input_size_per_partition: The size of the input partition. output_partition_sizes: List of output partition sizes. input_size: The total size of the input (unused). output_size: The total size of the output (unused). params_dtype: The data type of the parameters (expected to be torch.float16). Returns: A dictionary containing the quantized weights ('qweight'), scales ('scales'), and zeros ('zeros'). Raises: ValueError: If `params_dtype` is not `torch.float16` or if the input size per partition is not divisible by the group size in `quant_config`. weight_loaderz3Parameter data type must be torch.float16, but got Nr"rzInput size per partition (z#) must be divisible by group size (z).Fnt)r enable_tuningbiaslayoutr[cuda)devicer~ requires_grad)data input_dim output_dim packed_dim packed_factorbitblas_tile_sizer)rr~)rrr)rrr)rrrrrr)r)rrqweightscaleszerosrF)rrNr0rsum_configure_bitblas_matmul ENABLE_TUNINGrrrKemptybitblas_matmulretrieve_weight_shaper8r9 propagate_brrr<rrregister_parameter)r=rmrrrrrextra_weight_attrsrroutput_size_per_partitionr input_groupsweight_scale_argsrrs r)create_weights_gptqz'BitBLASLinearMethod.create_weights_gptqsYB  *?; t0IIKK K KTlTT &1  J$'(>$?$?!    8: E J J>-E>>/9>>>  && $ %%,". '   &#99;;';#  +7&2#99;;B??'!   ('",,qq2Jj2X K)"  +   1  /RR1R@QRRFF-->F   '; 6 6'[ -1B1NN!+?"' "/;+   EE & - !&  ,E )5):):"#      G444   6222   %00000r+c |jjdkr|j||||||fi|Std|jjNrcUnsupported quant_method )rrrr0)r=rmrrrrrrs r)create_weightsz"BitBLASLinearMethod.create_weightsgsr   )V 3 3+4+(& % LD,=,JLL r+c ddlm} |j|} d} d} |jj} |jj}|jjdkrd} d} d|}|jjrd} d|}ntd|jj| ||| ||| d krd n| |jj | | | ||| }| |||_ dS) Nr) MatmulConfigFrcTuintr4rrint32) NKA_dtypeW_dtype out_dtype accum_dtyper6 with_scaling with_zerosr with_biasrr<) r,rBITBLAS_DTYPESrrr<rrr0r3_get_or_create_bitblas_operatorr)r= infeatures outfeaturesrrrrr[rr bitblas_dtyperrrr<r matmul_configs r)rz-BitBLASLinearMethod._configure_bitblas_matmuls# )(((((+L9   &1 &1   )V 3 3LJ#TmmG ' '" &,,LD,=,JLL % !#0F#:#: +9%!!!   #BB =  r+cddlm}m}ddlm}m}|}|}|dkr|||||} | |||d} |rd|d} t | | d | || | ||d|d } t | nAd|d } t | n d|d } t | | S) Nr)Matmulauto_detect_nvidia_target)get_database_pathglobal_operator_cacheF)targetrzBitBLAS Operator z is tuning ...)topkz tuned and saved to database.z created.z found in global_operator_cache.)r,rr bitblas.cacherrsizeload_from_databaserdrfrghardware_aware_finetuneaddsave_into_database) r=rTrrrrrBITBLAS_DATABASE_PATHBITBLAS_TARGETrTUNING_MESSAGE TUNED_MESSAGE_messages r)rz3BitBLASLinearMethod._get_or_create_bitblas_operators========JJJJJJJJ 1 1 3 32244 % % ' '1 , , ! 4 4%~   /226::  !#VF>QVWWWN &!KV!K!K!K N+++66B6???%))&.AAA%88)>NMMM M****@v@@@ H%%%%S6SSSH KK ! ! !r+xrcz|j}|j}|j}|d|jd}|jjr||||}n|||||}||jdd|jdfz} || || S)Nr"r) rrrviewshaperrradd_) r=rmrrrrqzerosx_2d output_2doutputs r) apply_gptqzBitBLASLinearMethod.apply_gptqs -vvb!'"+&&   # K++D'6BBII++D'66JJI 0B/D DEE   KK    r+argskwargsct|jjdkr |j|i|Std|jjr)rrrr0)r=rrs r)applyzBitBLASLinearMethod.applysO   )V 3 3"4?D3F33 3LD,=,JLL r+)rwr%)rsrtrurvr OPT_FEATURESrrKrrwrMrLrrrr2rrr4r}r~rrrrTensorrrrrFr+r)roros-LM y y   I F N)]))))L1xL1#&L1!%S L1  L1  L1kL1 L1L1L1L1\x#&!%S     kD0 0 0 0 d   L%) x <lT!   0          r+ro) typingrrrK packagingr vllm.loggerr!vllm.model_executor.layers.linearrr'vllm.model_executor.layers.quantizationr r ;vllm.model_executor.layers.quantization.utils.bitblas_utilsr r r r3vllm.model_executor.layers.vocab_parallel_embeddingrvllm.model_executor.parameterrrrrvllm.model_executor.utilsrrsrfrrorFr+r)rs! ######JJJJJJJJ ONNNNN 766666 X  ^^^^^&^^^Buuuuu*uuuuur+