`i4dZddlZddlmZddlmZddlmZmZddl Z ddl m Z ddl m Z dd l mZdd lmZdd lmZmZmZmZmZmZd d lmZejeZdadZGdde jj Z!d)dZ"d)dZ#Gdde j$Z%Gdde j$Z&GddeZ'eGddeZ(eedGddeZ)eed Gd!d"eZ*eGd#d$e(Z+ed%Gd&d'e(e Z,gd(Z-dS)*zPyTorch RWKV model.N) dataclass)Path)OptionalUnion)nn)GenerationMixin)GradientCheckpointingLayer)PreTrainedModel) ModelOutputauto_docstringis_bitsandbytes_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigcddlm}ttjjjdz dz fddD}t t j|krdSt d|dd d d d d dd|g}|d||tj tj k|a|t _dS)Nr)loadkernelsrwkvcg|]}|z Sr).0f kernel_folders z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/rwkv/modeling_rwkv.py z(load_wkv_cuda_kernel..4sfffq*fff)z wkv_op.cppz wkv_cuda.cuzwkv_cuda_bf16.cuz2Loading CUDA kernel for RWKV at context length of .z -res-usagez--maxrregcount 60z--use_fast_mathz-O3z -Xptxas -O3z--extra-device-vectorizationz-DTmax=wkv_)namesourcesverboseextra_cuda_cflags) torch.utils.cpp_extensionrr__file__resolveparentrwkv_cuda_kernelmax_seq_lengthloggerinfor get_verbosityDEBUG)context_length load_kernelcuda_kernel_filesflagsrs @rload_wkv_cuda_kernelr4.s ======NN**,,3:AIMPVVMffff4efff#(8(G>(Y(Y KKV^VVVWWW  &"."" E#{ $N $ $!&((GM9  '5###rc>eZdZeddZeddZdS)RwkvLinearAttentionNFc B|\}}} |tjkr td|dtjd|| zt | dzdkr'td|d| dt | dd |j|_|jjd ks0|jjd ks |jjd ks|jjd krtd tj |  }|jtj kr<| }| }| }| }| }| }tj|tj } |s||Ktj|| d tj|jtj}|dddddfxxdzcc<n2tjd|Dd }|jtjkr tj} n tj} | ||||| |n<|jtjkr tjn tj} | ||||| |||||| |!dtj|d dD}| |j|fS)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of r cudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr)dtypedevicer;籡*Gc8g|]}|dSr>) unsqueezerss rrz/RwkvLinearAttention.forward..}s""A"A"Aa1;;q>>"A"A"Ar)dimc8g|]}|dSrA)squeezerCs rrz/RwkvLinearAttention.forward..s"HHHaQYYq\\HHHr)sizer*r+ ValueErrorminr< input_dtyper=typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32catbfloat16forward_with_state_bf16forward_with_state forward_bf16forwardsave_for_backwardchunkto) ctx time_decay time_firstkeyvaluestate return_state batch_sizeseq_len hidden_sizeoutput forward_funcs rr[zRwkvLinearAttention.forwardOsG+.88::( G[ %4 4 4FwFF#2FFF   #c+r&:&: :a ? ?7j77[77";33777  )   "f , , %//z&((| F**tuu ui 0 0 2 2 = = ? ?@@@ 9 % %#))++J))++CKKMME**,, nn  ""!#U5LMMM  E5,} -:"'"9 aaaAg$& "A"A5"A"A"AqIIITTVVyEN**/G /B LZeVU K K K K$A$A%..u}   ' %BYZZZ  E4KLLL"58OPPP %- ' '~~''H:E:W:W(66]m]v          ! !      OOK ( ( OOK ( ( HH[ ! ! JJ{ # #     rNFN)__name__ __module__ __qualname__ staticmethodr[rnrrrr6r6NsS<1<1<1\<1|% % % \% % % rr6Fc|\}}}tj|}|tj|dddftj} tj|dddftj} tj|dddftjdz } n|\} } } tj| }t |D] } |dd| f} |dd| f}tj| | |z}tj| |z }tj| |z|z }|| z||zz}|| z|z}||z |j |dd| f<tj| |z| }tj| |z|z }tj| |z }|| z||zz} || z|z} |}  |s|| | | g}||fS)Nr)r<r?) rHrM zeros_likerUrNrangerOmaximumr^r<)r`rarbrcrdre_ seq_lengthri num_state den_state max_state current_index current_key current_valuemax_for_outpute1e2 numerator denominator max_for_states rrwkv_linear_attention_cpurs$xxzzAz1  c " "F }$SAYemDDD $SAYemDDD $SAYemDDDtK */' 9i )J'''Jz**"" !!!]*+1133 aaa./ y+ 2JKK Yy>1 2 2 Y{Z/.@ A ANR-%77 9nr) $- $;#?#? #M#Mqqq-  i*& > Y{]2 3 3NR-%77 NR' ! 2u(Iy1 5=rctd||||fD}|ddk}t|s|rt||||||St||||||S)Nc36K|]}|jjdkVdS)r9N)r=rL)rts r z(rwkv_linear_attention..s+XXa!(-6)XXXXXXrrrdre)anyrHr*rr6apply)r`rarbrcrdreno_cuda one_tokens rrwkv_linear_attentionrsXXJ CQV3WXXXXXG q I7i(ZeSXgstttt"((ZeUT`aaarc0eZdZdfd ZddZd dZxZS) RwkvSelfAttentionrcdt||_tduotj|jk}t rPtrB|s@ t|jn*#t$rt dYnwxYw||_ |j }|j|jn|}||_tjt#j||_tjt#j||_tjt#jdd||_tjt#jdd||_tjt#jdd||_tjd|_tj||d|_tj||d|_tj||d|_tj||d|_dS)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFbias)super__init__configr*r+r0rrr4 Exceptionr,r-layer_idrhattention_hidden_sizer ParameterrMemptyr`ra time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearrbrc receptanceri)selfrr kernel_loadedrhr __class__s rrzRwkvSelfAttention.__init__s  (4q9I9X\b\q9q    Y$;$=$= Ym Y Y$V%:;;;; Y Y Y WXXXXX Y  ( ,2,H,TF ( (Ze &;",u{3H'I'IJJ,u{3H'I'IJJLQ;)G)GHH l5;q![+I+IJJ#%< Aq+0N0N#O#O ,}559[*?eLLLY{,ANNN )K1FUSSSi 5{OOO s&A;;$B"!B"Ncb|ddkr||ddddd|jf}n8||}|!|ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}||jz|d|jz zz}||}||}tj | |}|!|dddf|ddddd|jf<||||fSNrrr) rHrrrrrrbrcrMsigmoidr)rhiddenrdshiftedrbrcrs rextract_key_valuez#RwkvSelfAttention.extract_key_valuesW ;;q>>Q  5#4Ahqqq!!!T]23GGoof--G %aAAAt})< =1 t((7a$:K6K+LL,,w!d>Q:Q/RRd66AH`D`9aa hhsmm 5!!]4??:#>#>??  ,2111b5ME!HQQQ4=( )3u,,rFc||\}}}}|#tfd|ddDnd}tjj||||\}}|W|d|dddddjf<|d|dddddjf<|d|dddddjf<||z|fS) Nrdc3@K|]}|ddddjfVdSrwr)rrDrs rrz,RwkvSelfAttention.forward..#s9FFqAaaaDM12FFFFFFrr>rrrr)rtuplerr`rarri) rrrd use_cacherrbrc layer_staters ` rr[zRwkvSelfAttention.forward!s(,(>(>vU(>(S(S% CJOJ[eFFFFE!""IFFFFFFae 1 O O  "    k  ",7NE!HQQQ4=( ),7NE!HQQQ4=( ),7NE!HQQQ4=( ){{:,--u44rrrwrv)rxryrzrrr[ __classcell__rs@rrrskPPPPPP<----&55555555rrc(eZdZdfd ZddZxZS)RwkvFeedForwardrc0t||_||_|j}|j|jn d|jz}t jd|_t j tj dd||_ t j tj dd||_ t j||d|_t j||d|_t j||d|_dS)NrrrFr)rrrrrhintermediate_sizerrrrrMrrrrrbrrc)rrrrhrrs rrzRwkvFeedForward.__init__6s    ( (.(@(LF $ $RSV\VhRh ,}55LQ;)G)GHH#%< Aq+0N0N#O#O 9[*;%HHH)K5IIIY0+EJJJ rNc||ddkr||ddddd|jf}n8||}|!|ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}t jt j||}| |}t j | |}|!|dddf|ddddd|jf<||z|fSr) rHrrrrrMsquarerelurbrcrr)rrrdrrbrrcs rr[zRwkvFeedForward.forwardGsK ;;q>>Q  5#4Ahqqq!!!T]23GGoof--G %aAAAt})< =1 t((7a$:K6K+LLd66AH`D`9aa l5:dhhsmm4455 3]4??:#>#>??  ,2111b5ME!HQQQ4=( )E!5((rrrwrxryrzrr[rrs@rrr5sWKKKKKK"))))))))rrc&eZdZfdZddZxZS) RwkvBlockct||_||_|dkr%t j|j|j|_t j|j|j|_ t j|j|j|_ t|||_ t|||_dS)Nr)eps)rrrrr LayerNormrhlayer_norm_epsilonpre_lnln1ln2r attentionr feed_forward)rrrrs rrzRwkvBlock.__init__\s    q==,v'9v?XYYYDK< 28QRRR< 28QRRR*68<<+FH==rNFc4|jdkr||}|||||\}}||z}||||\}}||z}||f}|r||fz }n|dz }|S)Nr)rdrrrw)rrrrrr)rrrdroutput_attentionsrroutputss rr[zRwkvBlock.forwardjs =A  [[((F>>$((6*:*:%S\>]] 5)#"//0@0@/NN e,&5/    | #GG w Gr)NFFrrs@rrr[sL > > > > >rrcJeZdZUeed<dZdgZddgZdZdZ de j fdZ d S) RwkvPreTrainedModelrrrr`raTmodulec t|tr|j}|jj}|jj |j ||dz z d||z z }tj fdt D|j j |j j }|ddddf} fdt D}tj||j j |j j }tjdt D|jj |jj dz}||j _tj|jt#jd z|z|j_tj|||j _tj||d zz|j_tj|d|z|j_dSt|t,r|j}|jj}|jj d||z z }tj fd t D|j j |j j }|ddddf}tj|||j _tj|||j_dSt|t.jr|jjj}d} d} |j|jj|d |dkr#t#j|d |dz } |d |jjkr|d|jjkrd} | | z} t.j |j| dSt|t.j!rj|jjj}d t#jtE|d |dz} t.j |j| dSt|t.j#r?|jj$d|jjdSdS)zInitialize the weights.rg?cg|]}|z Srrrirhs rrz5RwkvPreTrainedModel._init_weights..===Q[===rr<r=Nc>g|]}dd|dz z ddzzzzzS)rgffffff?g?r)rhr ratio_0_to_1s rrz5RwkvPreTrainedModel._init_weights..sMQ!4q89sS.s$KKKa!eq[1_KKKrg?g333333?cg|]}|z Srrrs rrz5RwkvPreTrainedModel._init_weights..rrr)gaing-C6?)% isinstancerrrnum_hidden_layersrhrrMtensorr~rr<r=r`radata ones_likemathlogpowrrrrrweightshaperzero_sqrt vocab_sizeinit orthogonal_ Embeddingmaxrfill_)rrrrratio_1_to_almost0 time_weight decay_speedzigzagrrscalerrhrs @@@r _init_weightsz!RwkvPreTrainedModel._init_weightss_ f/ 0 0G %H & ?  -3K$*$@ !#'81'<=L!$3D(D!E ,====% *<*<===)/*1K &dD!!!m4K455K ,{&:K:QZ`ZkZrsssK KKe4I.J.JKKK +1!,3   &1F  "%*_V5FRU5VY_5_%`%`F  "',y>P'Q'QF  $).;@R)S)SVY\hVh)hF ! &.3i SK]E].^.^F & + + +  0 0# %H & ?  -3K!$3D(D!E ,====% *<*<===)/*1K &dD!!!m4K',y>P'Q'QF  $.3i EW.X.XF & + + +  * * %M&,EDE{&  &&(((Qx%(""yqE!H!455Qx4;111eAh$+BY6Y6Y EMD G   D  9 9 9 9 9  - - %M&,E$)Ca%($;$;<<Ju0#567>>>>>rr zK Base class for causal language model (or autoregressive) outputs. ceZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejdfed<dZee ejdfed<dS) RwkvCausalLMOutputap loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Nlosslogitsrd.r r)rxryrzrrrrMrrrrdrr rrrrrrrs)-D(5$ %,,,*.FHU& '.../3E8D*+ ,333=AM8E%"3S"89:AAA:>Ju0#567>>>>>rrc(eZdZfdZdZdZe ddeej deej deej dee ej d ee d ee d ee d ee d e eeffdZdZdZxZS) RwkvModelcttjjj|_tjfdtj D|_ tj j|_ d|_ d|_|dS)Nc2g|]}t|S)r)r)ridxrs rrz&RwkvModel.__init__..s&$p$p$pYv%D%D%D$p$p$prF)rrrrrrh embeddings ModuleListr~rblocksrln_outlayers_are_rescaledgradient_checkpointing post_initrrrs `rrzRwkvModel.__init__s    ,v'8&:LMMm$p$p$p$pPUV\VnPoPo$p$p$pqq l6#566 #( &+# rc|jSrwrrs rget_input_embeddingszRwkvModel.get_input_embeddings s rc||_dSrwr$rnew_embeddingss rset_input_embeddingszRwkvModel.set_input_embeddingss (rN input_idsattention_mask inputs_embedsrdrroutput_hidden_states return_dictreturnc :||n |jj}||n |jj}||n|js |jjnd}||n |jj}|t d|j|jkr| |td|td| ||rZ|X d|jj |jjffdtdD}|d xxd zcc<|jr%|jr|rt d d}} |rd nd} |rd nd} t#|jD]Z\} } | | ||| \} }}|jr+|jjdkr| dz|jjzdkr| dz } |r| | fz} |r| |fz} [|| } |r| | fz} |st+d| || | fDSt-| || | S)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrclg|]0}tj|dkrjn tjjd1S)rr)rMrTr<rUr=)rrr-rs rrz%RwkvModel.forward..IsY a-"5"5U][h[orrgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...r)rdrrrr>c3K|]}||V dSrwr)rxs rrz$RwkvModel.forward..ts(ttqfgfsfsfsfsfsttr)r rdr r)rrr.trainingruse_return_dictr, warning_oncer_rescale_layersrIrrHrhrr~r  enumerater rescale_everyrrr )rr+r,r-rdrrr.r/r all_self_attentionsall_hidden_statesrblockrrs ` @rr[zRwkvModel.forwards>2C1N--TXT_Tq$8$D $+Jj "+!6IIZ^Zg=rT[=R=Rmr %0%cdd d  =#8TUU U   OOI66M  "''**DK,CT[EbcEq E !HHH HHH  & "4= " "##p" % $5?bb4"6@BBD#DK00 J JJC/4uUiSd000 ,M5* ( 2K-111W 99Q>> - 1 # I$58H$H!  J&9ZM&I# M22  E 1]4D D  utt]E;LNa$bttttt t++*     rc ,|j|j krdS|jjdkrbt j5t |jD] \}}|jr|jj j dt||jjzz|j jj dt||jjzzt|jj j dr|jj j jdt||jjzz|j jj jdt||jjzz=t|jj j drB||jj |||j j||jj j dt||jjzz|j jj dt||jjzz" dddn #1swxYwY|j |_dS)Nrr>SCB quant_state)rr6rr;rMno_gradr:rrrirmul_intrrchasattrr@div_ _bnb_4bit_dequantize_and_rescale)rblock_idr>s rr9zRwkvModel._rescale_layers}s  #DM(9 : : F ; $q ( ( r r'0'='=rrOHe} r.5::1HPTP[PiDi@j@j;jkkk*07<>qCTXT_TmHmDnDn?nooo!.4;@@c(VZVaVoJoFpFpApqqqqr r r r r r r r r r r r r r r r"(,}#4   sH7I<<JJctstdddl}|j|jj|jj}|dt||j j zz|j |dd|j}t!|d|dS) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr>cpuF) requires_gradr)r ImportError bitsandbytes functionaldequantize_4bitrrrArFrDrr;r Params4bitr^r=setattr)r target_layerrHbnbdequant_weights quant_weights rrGz*RwkvModel._bnb_4bit_dequantize_and_rescales )** QOPP P"""".889L9QS_SfSrssQ#h$+2K&K"L"LLMMMv((););E)B)BRW(XX[[\k\rss  h 55555r)NNNNNNNN)rxryrzrr&r*r rrM LongTensorrrboolrrr r[r9rGrrs@rrrsY     )))15595937$(,0/3&*g g E,-g !!12g   12 g U./0 g D> g $D>g 'tng d^g  uj !g g g ^g R55506666666rrz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cFeZdZdgZfdZdZdZddZe dde e j de e j d e e j d e e e j d e e j d e ed e ede ede edeeeffdZxZS)RwkvForCausalLMz head.weightct|t||_t j|j|jd|_| dS)NFr) rrrrrrrhrheadr!r"s rrzRwkvForCausalLM.__init__s`    f%% If0&2C%PPP  rc|jSrwr[r%s rget_output_embeddingsz%RwkvForCausalLM.get_output_embeddingss yrc||_dSrwr]r(s rset_output_embeddingsz%RwkvForCausalLM.set_output_embeddingss " rNc ||dddfd}||d|i}nd|i}||d<||d<|D]\}}||vr|||<|S)Nrr-r+rdr)rBitems) rr+rdr-rkwargs model_inputsrbrcs rprepare_inputs_for_generationz-RwkvForCausalLM.prepare_inputs_for_generations  !!!!R%(22266I  $+];LL'3L % W$- [!!,,.. * *JC,&&$) S!rr+r,r-rdlabelsrrr.r/r0c L| | n |jj} |||||||| } | d} || } d}||j| |fd|jji| }| s| f| ddz}||f|zn|St || | j| j| j S)aJ input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. N)r-rdrrr.r/rrr)rrrdr r) rr7rr[ loss_functionrrrdr r)rr+r,r-rdrfrrr.r/rc rwkv_outputsr rrris rr[zRwkvForCausalLM.forwards J&1%F $D>F 'tnF d^F  u(( )F F F ^F F F F F rrY)rYrrrv).rr dataclassesrpathlibrtypingrrrMr generationr modeling_layersr modeling_utilsr utilsr r rrrrconfiguration_rwkvr get_loggerrxr,r*r4autogradFunctionr6rrrrrrrr rrrY__all__rrrrws  !!!!!!"""""""" ))))))999999------+*****  H % %555@g g g g g %.1g g g T))))XbbbbC5C5C5C5C5 C5C5C5L#)#)#)#)#)bi#)#)#)L*DQ%Q%Q%Q%Q%/Q%Q%Q%h  ? ? ? ? ? ? ?   ?  ???????   ?$j6j6j6j6j6#j6j6j6Z o o o o o )?o o  o d B A Ar