`i.ddlZddlZddlmZddlmZmZddlZddlm cm Z ddlm Z ddl m Z mZmZddlmZddlmZdd lmZdd lmZmZmZmZmZmZdd lmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)e$rddl*m+Z+ddl,m-Z-ddl.m/Z/ne0Z-e%j1e2Z3Gddej4j5Z6 dUdeej7dee8fdZ9Gdde-Z:Gdde j;Z<Gdde j;Z=Gdd e j;Z>d!Z?dVd"Z@ dWd$d%d&ej7d'ej7d(ej7d)eejAd*eBe8e8fd+e8d,e8d-eeCd.eeBej7ej7feBej7ffd/ZDejEfd$d%d&ej7d0e:dej7de8d*eBe8e8fd+e8d,e8d1ejFd.eBej7fd2ZGd$d%d&ej7d'ej7d(ej7d)eejAd*eBe8e8fd+e8d,e8d.eBej7fd3ZHeGeDeHd4ZIGd5d%e j;ZJGd6d7eZKe#Gd8d9e!ZL dUd:ej7d'ej7d)eej7d;eej7d.eBej7ej7ej7e8eej7eej7ff d<ZMd:ej7d=ej7d>e8d?e8d.ej7f d@ZNe#GdAdBeLZOGdCdDe j;ZPe#dEFGdGdHeLZQe#dIFGdJdKeLZRe#dLFGdMdNeLZSe#GdOdPeLZTe#dQFGdRdSeLZUgdTZVdS)XN) nullcontext)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)_prepare_4d_attention_mask)GradientCheckpointingLayer)BaseModelOutputMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)auto_docstringis_flash_attn_2_availablelogging)is_triton_available)ModernBertConfig) flash_attn_varlen_qkvpacked_func)RotaryEmbedding) apply_rotarycleZdZe ddeejdeefdZedZ dS)ApplyRotaryEmbUnpadN cu_seqlens max_seqlenc |}|j\}}}} |ddddf|d| } t| ||d||dd||||||_|S)NrFT)seqlen_offsetsr"r# interleavedinplace) contiguousshapeviewrsave_for_backwardr#) ctxqkvcossinr"r# total_nnz_three_nheadsheaddimqks /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/modernbert/modeling_modernbert.pyforwardzApplyRotaryEmbUnpad.forward>snn.1i+ 67GBQBZ__YG 4 4   !!  c3 333# c |j\}}}|}|j\}}}}|ddddf|d|} t | ||d||jddd |ddddddfS)Nr%r&rFT)r'r"r#r(r) conjugate) saved_tensorsr*r+r,rr#) r.dor0r1r"r2r3r4r5dqks r7backwardzApplyRotaryEmbUnpad.backward]s"0S* ]]__.0h+ 67GBQBinnYG44   !~ 4tT455r9NN) __name__ __module__ __qualname__ staticmethodrtorchTensorintr8r?r9r7r!r!=sy .2$(  U\*  SM \<66\666r9r!r"r#c>t|||||S)a Arguments: qkv: (total_nnz, 3, nheads, headdim) - input tensor for packed QKV. cos, sin: (seqlen_rotary, rotary_dim / 2) interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). inplace: if True, apply rotary embedding in-place. seqlen_offsets: (batch_size,) or int. Each sequence in x is shifted by this amount. Most commonly used in inference when we have KV cache. cu_seqlens: (batch + 1,) or None max_seqlen: int Return: out: (total_nnz, dim) rotary_dim must be <= headdim Apply rotary embedding to the first rotary_dim of x. )r!apply)r/r0r1r"r#s r7apply_rotary_unpaddedrKts .  $ $S#sJ K KKr9c eZdZdZ ddededeedeejdeej f fd Z dd ej d ej deed e ej e ej ej fffd Zd efdZxZS)!ModernBertUnpaddedRotaryEmbeddingzP The rotary position embeddings applied directly to unpadded sequences. @Ndimbaser#devicedtypect|||d||_|||||||dSdSdSdS)a max_seqlen: if max_seqlen, device, and dtype are provided, we precompute the cos_sin_cache up to max_seqlen. If the max_seqlen, device, or dtype during training/inference differ, the cos_sin_cache will be recomputed during the forward pass. F)rOrPrQr(NrQrR)super__init__r#_update_cos_sin_cache)selfrOrPr#rQrR __class__s r7rVz*ModernBertUnpaddedRotaryEmbedding.__init__sr StFNNN$  !f&8U=N  & &z& & N N N N N " !&8&8=N=Nr9r/r"returnc|"|||j|jt||j|j||}|S)z Apply rotary embedding *inplace* to qkv. qkv: (total_nnz, 3, nheads, headdim) cu_seqlens: (batch + 1,) cumulative sequence lengths max_seqlen: int max seq length in the batch NrTr"r#)rWrQrRrK _cos_cached _sin_cached)rXr/r"r#s r7r8z)ModernBertUnpaddedRotaryEmbedding.forwardsY  !  & &z#*CI & V V V#     !!     r9c6d|jd|jd|jS)Nzdim=z, base=z , scale_base=)rOrP scale_baserXs r7 extra_reprz,ModernBertUnpaddedRotaryEmbedding.extra_reprs&PdhPPtyPPtPPPr9)rNNNNN)rArBrC__doc__rGfloatrrErQrRrVrFrtupler8strrb __classcell__rYs@r7rMrMs2$()-'+ OO OOSM O & O  $ OOOOOO.%)  \LSM  u|U5<#=>> ? 2QCQQQQQQQQr9rMceZdZdZdeffd Zejddejdej fdZ d de ejd e ej dej fd Z xZ S) ModernBertEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. configc>t||_tj|j|j|j|_tj |j|j |j |_ tj |j|_dS)N) padding_idxepsbias)rUrVrlr Embedding vocab_size hidden_size pad_token_idtok_embeddings LayerNormnorm_eps norm_biasnormDropoutembedding_dropoutdroprXrlrYs r7rVzModernBertEmbeddings.__init__s{   l6+P^d^qrrrL!3vO_``` Jv788 r9Tdynamic input_idsrZcx||||Src)r}rzrv)rXrs r7compiled_embeddingsz(ModernBertEmbeddings.compiled_embeddingss.yy4#6#6y#A#ABBCCCr9N inputs_embedsc|)|||}n\|jjr||n:||||}|Src)r}rzrlreference_compilerrv)rXrr hidden_statess r7r8zModernBertEmbeddings.forwards  $ IIdii &>&>??MM;0J((333YYtyy)<) * * 'z&:Mt/T/T '#044[&BUBYBYZ`BaBabbDNN&DN"("@$*$B! /?+/+<+>@@GG HZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk ^ U C C C 5 5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C  5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 vvAGv$$cff17f&;&;;;sBE++E/2E/rc) rArBrCrErF__annotations__rrVno_gradrr8rhris@r7rrsl/////////"U]__ < <_ < < < <