`idZddlmZmZmZddlZddlZddlmZddl mcm Z ddl m Z mZddlmZddlmZmZddlmZdd lmZdd lmZmZmZdd lmZmZd d lm Z m!Z!d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0ej1e2Z3dFde4fdZ5 dGdej6de4de4de4fdZ7dej6dej8de4de9de4dej6f dZ:d dej;fd!ej6d"e4d#e4d$eej6ej6ffd'Z?d(ej6d)ee4dej6fd*Z@Gd+d,e'ZAGd-d.e(ZBGd/d0e ZCGd1d2e%ZDGd3d4e)ZEGd5d6e&ZFeGd7d8e$ZGGd9d:eGZHGd;deGZJGd?d@eGZKGdAdBeGZLGdCdDe#ZMgdEZNdS)Hz>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]]) >>> hashes = rolling_polynomial_hash(tokens, prime=31) >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2 >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2 dtypedevicer)dim)torchtensorint64r)arangeshapesum) token_tensorr% prime_tensorpowers prime_powerss w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/blt/modular_blt.pyrolling_polynomial_hashr99sb.<U[ATUUUL \,,R09L M M MF'L 9\L0b 9 9 990u token_ids group_sizemax_hashcLtj5|j\}}tj||dz tj|j}tj||gd}|d|d}t||} | |z} dddn #1swxYwY| S)z1Hash token groups and map to range [0, max_hash].rr'r,N) r.no_gradr2zerosr0r)catunfoldr9) r<r=r%r> batch_sizeseq_lenpadding padded_tokenswindowshashes hash_valuess r8byte_group_hash_functionrKVs  ( ('o G+j*q. T]Tdeee 7I"6A>>>  &&q*a88(%88x'  ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( sA9BB Blocal_encoder_tokensencoder_hash_tok_embedding$encoder_hash_byte_group_nb_functionsencoder_hash_byte_group_sizeencoder_hash_byte_group_vocabreturncgd}||}d}t|D]L} || t|z} |D]/} t|| | |} | ||zz} ||| z }|dz }0M|S)z=Compute token embeddings enhanced with hash-based embeddings.) r$l21Ai oYlvtl.l}glAul 0lTlAK l|rr) embed_tokensrangelenrK)rL local_encoderrMrNrOrPprimes embeddings embedding_idxfunc_nbr%r=hash_idsoffset_hash_idss r8compute_hash_embeddingsr]hs   F++,@AAJM=>>wV,-6  J/0DjRWYvwwH&9V)VVO 44_EE EJ Q MM   r:F patch_ids num_patchessequence_lengthpatches_as_queries cross_attn_kr(c|j\}}|j}|r||z} |} tj||dd|||} |d|||} n|} ||z} |d|||} tj||dd|||} | | k} |rdnd}| ||} || | f}| j|krtd| jd|| d} d| |z }| |tj tj |j } | S) aR Prepare cross-attention mask for patch-based attention, following mllama's robust approach. This function creates masks that control which patches can attend to which other patches, with support for query/key role swapping and cross-attention multipliers. Args: patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids. num_patches (int): Total number of patches. sequence_length (int): Length of the sequence. patches_as_queries (bool): If True, patches are used as queries, otherwise as keys. cross_attn_k (int): Cross-attention multiplier for repeating patches. dtype (torch.dtype): Data type for the output mask. Returns: Tuple[torch.Tensor, torch.Tensor]: - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len] r+rr*rr,zCross attention mask shape z doesn't match expected g?) r2r)r.r1 unsqueezeexpandrepeat_interleave ValueErrorto masked_fillboolfinfomin)r^r_r`rarbr(rDrEr)q_lenkv_len q_patch_ids kv_patch_idscross_attention_mask repeat_dimexpected_shapeinverted_cross_attn_masks r8#_prepare_patch_cross_attention_maskrus4$/J  F l*  LV 4 4 4 Yq\\ Yr]] VJ W 5 5  !**1--44ZgVV |+))"--44Z+VV LV 4 4 4 > >q A A K KA N N U UV`bikv w w  ',6)0bJ/AA,T^A__!%0N!^33 n*>*D n n^l n n   099!<< #%9%<%z(process_patch_lengths..s(66&#f++666666r:r'r,r)sizeitemdivmodextendappendmaxr.rAr(r) enumerater/rUanyr3r2nonzero) rvrwrD processedseqr|length full_chunks remaindermax_lenpaddedi last_nonzeros r8process_patch_lengthsrs##A&&JI!!#'l ) )F[[]]F%+F4D%E%E "K MM+,{: ; ; ; ) i(((    66I66666G [*g.m6IR_Rf g g gFy))tt 6  t',|F-BU^k^r's's'sF1mF m# $ ! Q##%% Q77! ((Q(//7799==??DDFFJ =L=() Mr:ceZdZdS)BltMLPN__name__ __module__ __qualname__r:r8rrDr:rceZdZdS) BltRMSNormNrrr:r8rr rr:rceZdZdS)BltRotaryEmbeddingNrrr:r8rrrr:rc$eZdZdeffd ZxZS)BltTransformerLayer layer_idxctt|||_t ||_t |j|j|_ t |j|j|_ dS)N)configreps) super__init__BltSelfAttention self_attnrmlpr hidden_size rms_norm_epsinput_layernormpost_attention_layernormselfrr __class__s r8rzBltTransformerLayer.__init__sv )9MMM&>>)&*<&BUVVV(263E6K^(_(_(_%%%r:)rrrintr __classcell__rs@r8rrsO`#``````````r:rc jeZdZdedeffd Z d dejdejdejd effd Z xZ S) rrrcZt||d|_dS)NT)rr is_causalrs r8rzBltSelfAttention.__init__s( +++r:FN hidden_statesattention_maskposition_embeddings use_cachec Dtjd||||||d|S)N)rrrrpast_key_valuescache_positionr)rforward) rrrrrrrkwargsrs r8rzBltSelfAttention.forward#sEuww ') 3+)      r:)FNN) rrrrrrr.Tensorrjrrrs@r8rrsyS   |   #\             r:rceZdZdZddededeeffd Z ddej deej d ee d eej d eej d e e f d ZxZS)BltCrossAttentionz>L#b$2JDMZZddefhijjJ',,S"d6NPTP]^^hhijlmnnL*+:+A+A dn?OQ_>`,,( LA ! # #&t~6;&t~6=%JJ p )@ ; +w 6 6"9$+:Z"[ $7$7      % $}>CC$,L % %  % % ! \"))#ub99DDFF kk+.. !M1 L((r:rzNNNN)rrr__doc__rrrrr.rr LongTensorr rrrrs@r8rr8sFFLLyLSLxPS}LLLLLL:>+/1559 3)3)|3)!) 63)"% 3) !. 3) !!12 3)+,3)3)3)3)3)3)3)3)r:rcxeZdZUeed<dZdZdZdgZe e dde e dddZ d Z d Zd Zd S) BltPreTrainedModelrFrr local_decoderindex layer_namer)r attentionsc tdNzNo need to inherit it!AttributeErrorrmodules r8 _init_weightsz BltPreTrainedModel._init_weights5666r:c tdrrrs r8_update_causal_maskz&BltPreTrainedModel._update_causal_maskrr:c tdrrrs r85_prepare_4d_causal_attention_mask_with_cache_positionzHBltPreTrainedModel._prepare_4d_causal_attention_mask_with_cache_positionrr:N)rrrr__annotations___supports_attention_backend_supports_flash_attn_supports_flex_attn_no_split_modulesrrr_can_record_outputsrrrrr:r8rrws "' ./'(;1Q`aaa$n%5Q?[[[ 77777777777r:rcteZdZUeed<deeddiZdeffd Z dde e j d e e j d e e j d e e j d e e j d e e de e j de e j de ede e j deefdZdZxZS)BltLocalEncoderrencoder_attentionsrrVrctd|_|_t jfdt jD|_t|_ t j j j j zd|_t jjj |_t j|_jrjnd}t |D]1}|jt+|j 2|dS)NFc0g|]}t|Srrr{rrs r8 z,BltLocalEncoder.__init__..$ e e e  3 3 e e er:r in_features out_featuresbiasrrrr)rrgradient_checkpointingrnn ModuleListrTnum_hidden_layersrr rotary_embLinearrrbpatch_embedding_projection Embedding vocab_sizerScross_attn_layerscross_attn_all_layersrr post_initrr layers_to_addrrs ` r8rzBltLocalEncoder.__init__sI    &+# m e e e eU6KcEdEd e e e   -F;;;*,)*+f.AA+ + + ' L):F>|LL +33 21 5 8P PRVR]Ri   $(;#DKCC! ,MD,B9,M-".+8#9-- --)& ,.DD +222r:cB|jd}|jd}|ddd|jd}tj|||f|j|j}||d|dd}|ddd|ddf}|S) a Reduce variable length patches to single embedding per patch Note: this works with variable number of patches for different sequences in the batch It handles variable length patches by assuming that patch_lengths will be 0 for any extra patches on the *right*. Since there can be a variable number of patches this function also return the number of patches for each sequence in the batch. Any embeddings on the right that are not allocated to a patch (i.e. if the sum(patch_lengths[i]) < seq_len for any i) will be sent to a dummy patch, which is trimmed before returning. rr*r'ramaxF)srcr-rreduce include_selfN)r2rdrer.rAr(r)scatter_reduce)rrmax_num_patchesr^rD embedding_dimreduced_embeddingss r8r'zBltLocalEncoder.patch_reduces#(+ %+B/ ''++222r=;Nr;RSS "[ - 8 @S\i\p   0>> ?  03CO3CQQQ0FG!!r: NNNNNNNNNN)rrrr"rrrrrrr.rrrrr rrr'rrs@r8rrsq !!!!nn-=QSbccc421504/31537+/599=%),04343E,-43  -43u|, 43 !. 43 u/0 43"%43!!1243!) 643c]43EL)43+,43434343l"""""""r:rc0eZdZUeed<deffd Ze ddeej deej deej deej deej d ee d eej d eej d e e fd ZxZS)BltLocalDecoderrctd|_|_d|_t jfdtjD|_ t|_ t j j jjzd|_t#jj|_t j|_jrjnd}t|D]1}|jt/|j2|dS) NFTc0g|]}t|Srrrs r8rz,BltLocalDecoder.__init__.. rr:rr rrr )rrrrcross_attn_decoderrrrTrrrrrhidden_size_globalrrbrrrnormrrrrrrs ` r8rzBltLocalDecoder.__init__sQ    &+# "&m e e e eU6KcEdEd e e e   -F;;;*,)1+f.AA+ + + ' v1v7JKKK !#4:4PW00VW }--  I  " ) )!9RXRdeee     r:Nrrrrrrrr rc |jd} |} ||}|| |jd|jjz|jj}| |js| |z} |Mtj|jd|j  d | d}| | |} tj| |jj|j} t!|jD]D\} }| dks |jjr|j| d| ||d| \}}| |z} || f| |||d| } E|| }|S) Nrrr+r*r"r%r$r)r2rrrrbrr:r.r1r)rdrerr&rrrrrrr<)rrrrrrrrr rrDrrrr)r*rlogitss r8rzBltLocalDecoder.forwards#(+ % 66|DD #++  *1- 0HH$+Ja    #D,C #)L8M   ]03Mr) entropiesr`rPrQr')rgrSrget_seq_lengthr.r1r2r)rdr rrrrOr< distributions Categoricalentropypatch_lengths_from_entropiesonesr(r)rrrrrrrrrPrQrwrpast_seen_tokens causal_maskrrr)r>prediction_entropiesrDr`rvs r8rzBltPatcher.forwards  -t"; < [YZZ Z   --i88M  -0*nnO  !CRC^==???de "\ "2]5H5K"KTaThN  )33A66L(;&))+%    & "oom\JJ[ v vE!E-EXituuuMMdii 6677$2>>f>MMUUWW&3&9"1"&=# O  ! ==. /%# >MM"J_-]5HQ^QeM.m=MNN #]F::r:c|jd}tjddgtj|jd|d}|jd}|ddddf}||k}|jd}tj||jd|d} tj | |} tj | | gd} tj ||gd} | |  ||} | d }| ddd|f} tj || |zfd}tj |ddddf|dz }tj |ddddfdz |fd}||z dz}|S)z Computes patch lengths from token entropies. Depending on whether a threshold is provided, the function uses either: - Thresholding the entropy values (when `threshold` is set). rrr'Nr+r*r,)r2r.r/longr)rdrepeatr1re full_likerBrr3r)rUr`rPrQrD init_tokensoffset patch_maskrE token_indicessentinelpadded_indices padded_mask patch_startsmax_valid_patchespatch_start_ids last_token patch_endsrvs r8rZz'BltPatcher.patch_lengths_from_entropiess_Q'  L!Quz):J K K K U UVW X X _ _`jlm n n "1%aaae$ * "1% WY5EFFFPPQRSSZZ[egijj ?='::M8#?;!!12?;SM?;E??;#3-?;+,?;?;?;?;B 333\33333r:rMcReZdZdeffd Ze ddeejdeej deej deejdee d eej d ee d eejd e ed efdZdZdZdej ded ej fdZxZS)BltModelrct|d|_||_t |j|_t|j|_ t|j |_ |j t|jz}|j|z}t#j||jj|_|jjrVt-|j|_|j|jD] }d|_ nd|_|dS)NF)rrrrrencoder_configrVrA global_configrCr7decoder_configrrNrUrOrPrrrrMpatch_in_forwardrMpatcher_configpatchereval parameters requires_gradr)rrnum_embeddingstotal_vocab_sizeparamrs r8rzBltModel.__init__s#    &+# ,V-BCC"6v7K"L"L,V-BCCDs6KnGoGoo!?.P*,,7GI^Ij*k*k' ; ' %f&;<
~ !!12~ +,~  !~ ~ ~ ~ @///000 iU\ iC iTYT` i i i i i i i ir:rsceZdZUeed<dZdZdgZdeffd Z dde e j d e e j d e e j d e e j d e e j d e e e j e j fde eeee jfde e jde e j de ede e j deee j fdeedee effdZxZS)BltForCausalLMrFmodelzlm_head.weightct||j|_t||_t j|jj|jd|_ | dS)NFrF) rrrrsrrrrwrrOr)rrrs r8rzBltForCausalLM.__init__sl     +f%% y!6!BFDU\abbb  r:Nrrrrrrqfull_text_row_masked_out_maskrrlabelsrrlogits_to_keeprrQc n|jd|||||||| | d | }|j}t| trt | dn| }||dd|ddf}d}| |j|| |jfi| }t|||j |j |j S)N) rrrrqrrrrr)lossr>rrrr) rr isinstancerslicerOrp loss_functionrr rrr)rrrrrrqrrrrrrrroutputsr slice_indicesr>rs r8rzBltForCausalLM.forwards"$*  )%!5*G+')       1 8B>SV8W8Wk~ot444]k mAAA}aaa,?@AAGGII  %4%ffdoPPPPD%#3!/)     r:) NNNNNNNNNNNr)rrrrr_can_compile_fullgraphbase_model_prefix_tied_weights_keysrrr.rrtuplerrlistrorjrr rr rrrs@r8rrs "*+y151537=A;?UYKO59-1$(5934, , E,-, !., u/0 , !))9 : , 'u'78 , (0elEL6P0Q'R, "%tE4E/F(F"GH,   12, )*, D>, !!12, c5</0, +,,  u,, -, , , , , , , , r:r)rrsrMr)r$)rr$r;)Ortypingrrrr.torch.distributionstorch.nnrtorch.nn.functional functionalr& cache_utilsrr masking_utilsr modeling_outputsr r modeling_utilsr processing_utilsr utilsrrr utils.genericrrcohere2.modeling_cohere2rrmllama.modeling_mllamarrrrrrrrconfiguration_bltrr r!r"r# get_loggerrloggerrr9rrKrrr]float32rjr(rrurrrrrrrrrr7rArMrsr__all__rr:r8rsCB,,,,,,,,,, ........//////OOOOOOOO555555&&&&&&@@@@@@@@@@????????                      H % %::::::<\a|),9<UX$#,#!# #+. # #' # $' # \####T % K K |K K K  K  K ; K  5< %&K K K K \))RU)[`[g))))X     ]        "        /   `````9```     .   4<)<)<)<)<)0<)<)<)~77777.777*p"p"p"p"p"(p"p"p"fLLLLL(LLL^22222-222jFFFFF#FFFRfififififi!fififiR9 9 9 9 9 &9 9 9 x   r: