.`iFYddlmZmZddlmZddlmZddlZddlm Z ddl m Z m Z ddl mZgdZeeZGd d e ZGd d eZGd deZGddeeZGddeeZGddeZGddeZGddeZGddeZGddeeZGddeZded ed!ed"efd#Zd$Zd%Z d&Z!dS)')CallableHashable)Fraction)WeakValueDictionaryN) Parameter)get_tensor_model_parallel_rank$get_tensor_model_parallel_world_size) init_logger)BasevLLMParameterPackedvLLMParameterPerTensorScaleParameterModelWeightParameterChannelQuantScaleParameterGroupQuantScaleParameterPackedColumnParameterRowvLLMParametercjeZdZdZdejdzffd ZdejdefdZe defdZ e j d efd Z e j d Z d ejfd Z d ejfdZd ejfdZd ejfdZd ejfdZd ejfdZdeezdefdZedfd ZxZS)r z Base parameter for vLLM linear layers. Extends the torch.nn.parameter by taking in a linear weight loader. Will copy the loaded weight into the parameter when the provided weight loader is called. dataNc Lt||dSNF)r requires_gradsuper__new__)clsrkwargs __class__s q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/parameter.pyrzBasevLLMParameter.__new__&swwsUCCC weight_loadercddlm}|r||}||_t |_t|_dS)z Initialize the BasevLLMParameter :param data: torch tensor with the parameter data :param weight_loader: weight loader callable :returns: a torch.nn.parameter r)current_platformN) vllm.platformsr"use_sync_weight_loadermake_synced_weight_loader_weight_loaderrtp_rankr tp_size)selfrr r"s r__init__zBasevLLMParameter.__init__)sd$ 433333  2 2 4 4 V,FF}UUM+577 ;== rreturncV|jt|jjd|jS)Nz) weight_loader attribute has been deleted)r&AttributeErrorr__name__r)s rr zBasevLLMParameter.weight_loaderDs;   & >*UUU ""rvaluec||_dSNr&)r)r0s rr zBasevLLMParameter.weight_loaderPs#rcd|_dSr2r3r/s rr zBasevLLMParameter.weight_loaderTs"r loaded_weightc|jjdko|jdk}|jdko|dk}|o|SNr)rndimnumel)r)r5cond1cond2s r_is_1d_and_scalarz#BasevLLMParameter._is_1d_and_scalarXsS !#> (9(9Q(>"a'FM,?,?,A,AQ,Frc|jj|jks||sJ|j|dSr2)rshaper=copy_r)r5s r_assert_and_loadz"BasevLLMParameter._assert_and_load]sPy-"5559O9O : : 55   &&&&&rc0||dSr2rBrAs rload_column_parallel_weightz-BasevLLMParameter.load_column_parallel_weightc m,,,,,rc0||dSr2rDrAs rload_row_parallel_weightz*BasevLLMParameter.load_row_parallel_weightfrFrc 0||dSr2rDr)r5rs rload_merged_column_weightz+BasevLLMParameter.load_merged_column_weightirFrc 0||dSr2rDrJs rload_qkv_weightz!BasevLLMParameter.load_qkv_weightlrFrshard_idct|tr|Sdddd}t|tsJ||vsJ||S)Nrr8)qkv) isinstanceintstr)r)rNqkv_idxss r_shard_id_as_intz"BasevLLMParameter._shard_id_as_intos\ h $ $ O++(C(((((8####!!rcT|i}t||||Sr2)r__torch_function__)rfunctypesargsrrs rr[z$BasevLLMParameter.__torch_function__zs+ >Fww))$tVDDDr)rYN)r. __module__ __qualname____doc__torchTensorrrr*propertyr setterdeleterr=rBrErHrKrMrVrUrX classmethodr[ __classcell__rs@rr r s D5<$.DDDDDD>U\>(>>>>6 #x # # #X #$8$$$$###u| 'el'''' ------el-----u|-----U\---- "s "s " " " "EEEEE[EEEEErr ceZdZdZdeffd ZedZdej fdZ dej fdZ dej fdZ xZ S) _ColumnvLLMParametera Private class defining weight loading functionality (load_merged_column_weight, load_qkv_weight) for parameters being loaded into linear layers with column parallelism. This includes QKV and MLP layers which are not already fused on disk. Requires an output dimension to be defined. Called within the weight loader of each of the column parallel linear layers. output_dimc H||_tjdi|dSNrY) _output_dimrr*)r)rlrrs rr*z_ColumnvLLMParameter.__init__s.%""6"""""rc|jSr2)ror/s rrlz_ColumnvLLMParameter.output_dim rr5c|jj|j}||j|j|z|}|jj|jksJ|j|dSr2)rr?rlnarrowr'r@r)r5 shard_sizes rrEz0_ColumnvLLMParameter.load_column_parallel_weightskY_T_5 %,, OT\J6   y-"55555  &&&&&rc |d}|d}t|ttfr*|j|jkr|||\}}|j}||j||}||j|j |z|}|j |j ksJ| |dS)N shard_offsetrurwru) getrTrr packed_dimrl adjust_shard_indexes_for_packingrrsr'r?r@)r)r5rrwru param_datas rrKz._ColumnvLLMParameter.load_merged_column_weightszz.11 ZZ --  t35HI J J 4?22'+'L'L)j(M(( $J Y &&t jQQ %,, OT\J6   =#66666'''''rc :|d}|d}|d}|d}t|ttfr*|j|jkr|||\}}|j}|dkr|jn |j|z}| |j||}| |j||z|}|j |j ksJ| |dS)NrwrurN num_headsrxrQ) ryrTrr rlrzr{rr'rsr?r@)r)r5rrwrurNr~r|s rrMz$_ColumnvLLMParameter.load_qkv_weights'zz.11 ZZ -- ::j))JJ{++  t35HI J J 4?22'+'L'L)j(M(( $J Y #+s??4<< 8Q&&t jQQ %,, OX 2J  =#66666'''''r)r.r_r`rarUr*rdrlrbrcrErKrMrhris@rrkrks#3######  X '''''(u|((((,(U\((((((((rrkcTeZdZdZdeffd ZedZdej fdZ xZ S)rz Parameter class defining weight_loading functionality (load_row_parallel_weight) for parameters being loaded into linear layers with row parallel functionality. Requires an input_dim to be defined. input_dimc H||_tjdi|dSrn) _input_dimrr*)r)rrrs rr*zRowvLLMParameter.__init__s-#""6"""""rc|jSr2)rr/s rrzRowvLLMParameter.input_dims rr5c8|jj|j}||j|j|z|}t |jdkr|d}|jj|jksJ|j|dS)Nrr8)rr?rrsr'lenreshaper@rts rrHz)RowvLLMParameter.load_row_parallel_weightsY_T^4 %,, NDL:5z   }" # #q ( ()11!44My-"55555  &&&&&r) r.r_r`rarUr*rdrrbrcrHrhris@rrrs########X 'el ' ' ' ' ' ' ' 'rrceZdZdZdS)rz] Parameter class for linear layer weights. Uses both column and row parallelism. Nr.r_r`rarYrrrr  DrrceZdZdZdS)rz Parameter class for weight scales loaded for weights with grouped quantization. Uses both column and row parallelism. NrrYrrrrrrrceZdZdZdS)rz Parameter class for weight scales loaded for weights with channel-wise quantization. Equivalent to _ColumnvLLMParameter. NrrYrrrrrrrcbeZdZdZfdZfdZdZdZfdZde j de e zfd Z xZS) r a Parameter class for scales where the number of scales is equivalent to the number of logical matrices in fused linear layers (e.g. for QKV, there are 3 scales loaded from disk). This is relevant to weights with per-tensor quantization. Adds functionality to map the scalers to a shard during weight loading. Note: additional parameter manipulation may be handled for each quantization config specifically, within process_weights_after_loading c :tjdi|dSrn)rr*)r)rrs rr*z PerTensorScaleParameter.__init__s&""6"""""rc:tj|i|dSr2rrHr)r^rrs rrHz0PerTensorScaleParameter.load_row_parallel_weight%(($9&99999rc |j|i|dSr2_load_into_shard_idr)r^rs rrKz1PerTensorScaleParameter.load_merged_column_weight   $1&11111rc |j|i|dSr2rrs rrMz'PerTensorScaleParameter.load_qkv_weightrrc:tj|i|dSr2rrs rrEz3PerTensorScaleParameter.load_column_parallel_weight rrr5rNc |j}||}t|jdkr|jddksJ|d}||}|j|jksJ||dS)zU Slice the parameter data based on the shard id for loading. rr8N)rrXrr?r@)r)r5rNrr|s rrz+PerTensorScaleParameter._load_into_shard_id#sY ((22 }" # #q ( ( &q)Q....)!,M) =#66666'''''r)r.r_r`rar*rHrKrMrErbrcrVrUrrhris@rr r s  ##### :::::222222:::::("\(583Y((((((((rr c eZdZdZ d deezdededzdedzffd ZedZed Z ed Z ed Z d Z xZ S)rz Parameter for model parameters which are packed on disk and support column parallelism only. See PackedvLLMParameter for more details on the packed properties. N packed_factorrzmarlin_tile_sizebitblas_tile_sizec r||_||_||_||_t jdi|dSrn_packed_factor _packed_dim_marlin_tile_size_bitblas_tile_sizerr*r)rrzrrrrs rr*zPackedColumnParameter.__init__@H,%!1"3""6"""""rc|jSr2rr/s rrzz PackedColumnParameter.packed_dimNrqrc|jSr2rr/s rrz#PackedColumnParameter.packed_factorR ""rc|jSr2rr/s rrz&PackedColumnParameter.marlin_tile_sizeV %%rc|jSr2rr/s rrz'PackedColumnParameter.bitblas_tile_sizeZ &&rcHt|||j|j|jSNrurwrrr!_adjust_shard_indexes_for_packingrrrr)rurws rr{z6PackedColumnParameter.adjust_shard_indexes_for_packing^10!%,!2"4     rNNr.r_r`rarUrr*rdrzrrrr{rhris@rrr9s(,(, # #X~ # #* # : # # # # # #  X ##X#&&X&''X'       rrc eZdZdZ d deezdededzdedzffd ZedZed Z ed Z ed Z d Z xZ S)r a Parameter for model weights which are packed on disk. Example: GPTQ Marlin weights are int4 or int8, packed into int32. Extends the ModelWeightParameter to take in the packed factor, the packed dimension, and optionally, marlin tile size for marlin kernels. Adjusts the shard_size and shard_offset for fused linear layers model weight loading by accounting for packing and optionally, marlin tile size. Nrrzrrc r||_||_||_||_t jdi|dSrnrrs rr*zPackedvLLMParameter.__init__srrc|jSr2rr/s rrzzPackedvLLMParameter.packed_dimrqrc|jSr2rr/s rrz!PackedvLLMParameter.packed_factorrrc|jSr2rr/s rrz$PackedvLLMParameter.marlin_tile_sizerrc|jSr2rr/s rrz%PackedvLLMParameter.bitblas_tile_sizerrcHt|||j|j|jSrrrs rr{z4PackedvLLMParameter.adjust_shard_indexes_for_packingrrrrris@rr r hs(,(, # #X~ # #* # : # # # # # #  X ##X#&&X&''X'       rr ceZdZdZdS)BlockQuantScaleParameterz Parameter class for weight scales loaded for weights with block-wise quantization. Uses both column and row parallelism. NrrYrrrrrrrcFeZdZUdZeZeed<eej ed<e e e e zfed<fdZdde d e ffd Zd e d efd Zdej fdZdej fdZdej fdZdej fdZdZedZdedej dee zdzfdZxZS)SharedWeightParametera* Parameter for weights with many shared tensors across a model For example, when applying transforms to the "gate" and "up" partitions of `MergedColumnParallelLinear`, the transform weights must stay separate tensors in order to allow for tensor memory sharing between layers. tensors_registry local_tensors partitionsc <tj|fddi|S)Nrr)rrrs rrzSharedWeightParameter.__new__s&uwws888888rr8rrrlc |d}td|t|_i|_|||jd|_|jdkrt|j j ddS)Nr )rr )rrlr r8z. does not currently support tensor parallelism) ryrr*setrr_fake_weight_loaderrr(NotImplementedErrorrr.)r)rrlrr rs rr*zSharedWeightParameter.__init__s"(**_"="=  d-@@@ UU"$!5   *777   rindexdata_keyc||jvrtj|i|}||j|<n |j|}tdd|i|j|j|<|j|dS)ab Add a partition to the weight parameter. Partitions whose `data_key` is the same will share tensor data :param index: index of partition to add :param data_key: hashable key used to key shared tensors :param *args: arguments for `torch.empty` :param **kwargs: keyword arguments for `torch.empty` rNrY)rrbemptyrrrradd)r)rrr^rrs r add_partitionz#SharedWeightParameter.add_partitions 40 0 0;///D.2D !( + +(2D"6!O!O4!O4;!O!O t$$$$$rr5ct|jdkr d|jvsJ|jd}t||dSr7)rrrrEr)r5 partitions rrEz1SharedWeightParameter.load_column_parallel_weightsQ4?##q((Q$/-A-A-AAOA& 88MRRRRRrct|jdkr d|jvsJ|jd}t||dSr7)rrrrHrs rrHz.SharedWeightParameter.load_row_parallel_weightsQ4?##q((Q$/-A-A-AAOA& 55iOOOOOrc <|d}||}|j|}|jd}|j||jz}|j|z}t ||||dS)NrNrrx) poprXrrryrsizer(r'rrK)r)r5r partition_idrrrurws rrKz/SharedWeightParameter.load_merged_column_weightszz*-- ,,\:: OL1 KOOK00 ^((33t|C |j0 66 }z)permute_param_layout_..Qs/   a7X.X.X.X.X.Xrrzz9permute_param_layout_ currently doesn't support repackingrror) getattrrdimrangeinserthasattrrzpermuterror)rrrlrpermrrs @@rpermute_param_layout_r.sUK66Ne\488O!8z~~1$$$ 1%$$**,T***)A-2))+S))))A-2      ))**   D KK >***KK O,,,v E< ( ( G D )=$>>>> F?> ?$#T*EJul##%$um$$'&v'%"?"?"<0 Lrc||z||zfSr2rYrurwrs r _adjust_shard_indexes_for_marlinrhs ( (,9I*I IIrc||z||zfSr2rYrurwrs r!_adjust_shard_indexes_for_bitblasrls * *Lrs5/.......'''''' $#####    X  _E_E_E_E_E _E_E_EDH(H(H(H(H(,H(H(H(V'''''(''':     /1A        35E        !5   2(2(2(2(2(/2(2(2(j, , , , , 0, , , ^0 0 0 0 0 .0 0 0 f     35E   G G G G G -G G G T7 7),7:=77777tJJJNNN$$$$$r