.`i] ddlZddlmZddlmZddlZddlmZddlmcm Z ddl m Z ddl mZddlmZddlmZddlmZd d lmZmZGd d ejZGd dejZGddejZGddejZGddejZdejdede eje eefffdZ!dejdede eefde eefdejf dZ"dededejdejfd Z#d!ejd"ejd#ejde eefde eefdejf d$Z$Gd%d&ejZ%d'Z&d(Z'Gd)d*eZ(Gd+d,ejZ)dS)-N)Iterable)partial)CLIPVisionConfig)MMEncoderAttention) Conv2dLayer)QuantizationConfig)default_weight_loader) CLIPEncoderCLIPVisionEmbeddingsc xeZdZejfdededeejddffd Zde j de j fdZ xZ S) MLPBlock embedding_dimmlp_dimactreturnNcttj|||_tj|||_||_dSN)super__init__nnLinearlin1lin2r)selfrrr __class__s z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/deepencoder.pyrzMLPBlock.__init__sR ImW55 Ig}55 355xcx||||Sr)rrrrrs rforwardzMLPBlock.forward)s*yy$))A,,//000r) __name__ __module__ __qualname__rGELUinttypeModulertorchTensorr" __classcell__rs@rrrs "w    ")_        11%,11111111rrcReZdZd dededdffd ZdejdejfdZxZ S) LayerNorm2dư> num_channelsepsrNcttjt j||_tjt j||_||_ dSr) rrr Parameterr*onesweightzerosbiasr2)rr1r2rs rrzLayerNorm2d.__init__0sZ l5:l#;#;<< L\!:!:;; rrc"|dd}||z ddd}||z tj||jzz }|jddddf|z|jddddfz}|S)Nr T)keepdim)meanpowr*sqrtr2r6r8)rruss rr"zLayerNorm2d.forward6s FF1dF # # UKKNN  4  0 0 UejTX.. . K4 & *TYqqq$}-E Er)r0) r#r$r%r'floatrr*r+r"r,r-s@rr/r//s}Su %,rr/c#eZdZdddddddddejejdd dd d fd ed edededededededede ej de ej dedededede edfddf"fd Z d e jd!efd"Zd#e jde jfd$ZxZS)%ImageEncoderViT @TFrimg_size patch_sizein_chans embed_dimdepth num_heads mlp_ratio out_chansqkv_bias norm_layer act_layer use_abs_pos use_rel_posrel_pos_zero_init window_sizeglobal_attn_indexes.rNct||_t||f||f|||_d|_| r4t jtj d||z||z||_t j |_ t|D]C}t|||| | | | |||vr|nd||z||zf }|j |Dt jt!||ddt#|t!||ddd t#||_t!d d dd dd |_t!d ddd dd |_dS)a Args: img_size (int): Input image size. patch_size (int): Patch size. in_chans (int): Number of input image channels. embed_dim (int): Patch embedding dimension. depth (int): Depth of ViT. num_heads (int): Number of attention heads in each ViT block. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool): If True, add a learnable bias to query, key, value. norm_layer (nn.Module): Normalization layer. act_layer (nn.Module): Activation layer. use_abs_pos (bool): If True, use absolute positional embeddings. use_rel_pos (bool): If True, add relative positional embeddings to the attention map. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. window_size (int): Window size for window attention blocks. global_attn_indexes (list): Indexes for blocks using global attention. ) kernel_sizestriderNrONr r) dimrQrRrTrUrVrXrYrZ input_sizeF)r]r8rF)r]paddingr8rJir;)r]r^rar8rD)rrrL PatchEmbed patch_embed pos_embedrr4r*r7 ModuleListblocksrangeBlockappend Sequentialrr/necknet_2net_3)rrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[iblockrs rrzImageEncoderViT.__init__@sJ   %#Z0 +    /3  \ x:-x:/EyDN moo u & &A##!%#'"3+,4G+G+GKKQ$ 2H 4JK   E K  u % % % %M       " "       " "   $! !Aqu    ! 1Q    rabs_postgt_sizecH|j}|d}||kr|dddd}|tj}t j|||fddd|}|dddd}|S|S) Nr rrFr;bicubicTFsizemode antialias align_corners)dtyperupermutetor*float32F interpolate)rrprqrysrc_size old_pos_embed new_pos_embeds r get_abs_poszImageEncoderViT.get_abs_poss <<?? x  #OOAq!Q77M),,U];;MM)#  bii *11!Q1==M Nrrct||}|j1|||j|dz}|jD] }||}||dddd}||}||}|S)Nr rrFr;) rcrdrrurfrkrzrlrm)rrblk neck_output conv2_output conv3_outputs rr"zImageEncoderViT.forwards   Q   > %D$$T^QVVAYY???A;  CAAAii !Q1 5 566 zz+.. zz,// r)r#r$r%r LayerNormr&r'rAboolr(r)tuplerr*r+rr"r,r-s@rrCrC?s&(l%'W !"&/1#_ _ _ _  _  _  _ _ _ _ _ O_  ?_ _ _  _ !_ "#38_#_ $ %_ _ _ _ _ _ B5<3(  %,        rrCceZdZdZddejejddddfdeded ed e d e ej d e ej d e de dede eefdzddffd Z dejdejfdZxZS)rhzWTransformer blocks with support of window attention and residual propagation blocksrITFrNr_rQrRrTrUrVrXrYrZr`rc ,t|||_t|||||| dkr| n| | f|_|||_t |t||z||_| |_ dS)ai Args: dim (int): Number of input channels. num_heads (int): Number of attention heads in each ViT block. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool): If True, add a learnable bias to query, key, value. norm_layer (nn.Module): Normalization layer. act_layer (nn.Module): Activation layer. use_rel_pos (bool): If True, add relative positional embeddings to the attention map. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. window_size (int): Window size for window attention blocks. If it equals 0, then use global attention. input_size (tuple(int, int) or None): Input resolution for calculating the relative positional parameter size. r)rQrTrXrYr`)rrrN) rrnorm1RelPosAttentionattnnorm2rr'mlprZ) rr_rQrRrTrUrVrXrYrZr`rs rrzBlock.__init__s8 Z__ # #/%0A%5%5zzK;U      Z__ s3?';';   'rrc|}||}|jdkr2|jd|jd}}t||j\}}||}|jdkrt ||j|||f}||z}||||z}|S)Nrr r;)rrZshapewindow_partitionrwindow_unpartitionrr)rrshortcutHWpad_hws rr"z Block.forwards JJqMM  a  71:qwqzqA(D,<==IAv IIaLL  a  "1d&6AGGA qL A'' 'r)r#r$r%__doc__rrr&r'rArr(r)rrr*r+r"r,r-s@rrhrhs&(l%'W!"&-1,',' ,',' ,'  ,' O ,' ?,',' ,','#s(Od*,' ,',',',',','\%,rrhceZdZdZ ddededed ed ed eeefdzd dffd Zdej d ej fdZ xZ S)rz=Multi-head Attention block with relative position embeddings.TFNr_rQrTrXrYr`rct||_||z}|dz|_t j||dz||_t j|||_||_|jr~| Jdt j tj d|dzdz ||_ t j tj d|dzdz ||_ dSdS) a Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. qkv_bias (bool): If True, add a learnable bias to query, key, value. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. input_size (tuple(int, int) or None): Input resolution for calculating the relative positional parameter size. grF)r8NzBInput size must be provided if using relative positional encoding.r;rr )rrrQscalerrqkvprojrXr4r*r7 rel_pos_h rel_pos_w) rr_rQrTrXrYr`head_dimrs rrzRelPosAttention.__init__ s$ ")#t^ 9S#'999Ic3'' &   X))T*)) \%+a*Q-6G!6KX*V*VWWDN\%+a*Q-6G!6KX*V*VWWDNNN  X Xrrc |j\}}}}|||||zd|jdddddd}|d||jz||zdd\}}} d\} } |jr$t||j|j ||f||f\} } | ||j||zd}| ||j||zd}| ||j||zd} |jr3| ||j| d| d| d} | ||j| d| d| d} | | z ||j| d| d| dz} tj j||| | }n&tj j||| }| ||j||dddddd|||d}||}|S) NrFr;rr )NN) attn_mask)rrreshaperQrzunbindrXadd_decomposed_rel_posrrviewrur*r functionalscaled_dot_product_attentionr) rrBrr_rqkvrel_hrel_w attn_biass rr"zRelPosAttention.forward/sW 1a HHQKK  1q5!T^R @ @ H HAqRSUV W W ++aT^!3QUB??FFqII1a! u   14>4>Aq6Aq6LE5 FF1dna!eR 0 0 FF1dna!eR 0 0 FF1dna!eR 0 0   JJJ4>5::a==%**Q--AEJJ4>5::a==%**Q--AE,,4>5::a==%**Q--%**Q--2OI#@@1a9AAA#@@AqIIA FF1dnaB / / WQ1a # # WQ1b ! ! IIaLLr)rTFTN) r#r$r%rr'rrrr*r+r"r,r-s@rrr sGG !"&-1!X!X !X!X !X  !X  !X#s(Od*!X !X!X!X!X!X!XF++%,++++++++rrrrZrc x|j\}}}}|||zz |z}|||zz |z}|dks|dkrtj|ddd|d|f}||z||z} }||||z|| |z||}|ddddddd|||} | || ffS)aU Partition into non-overlapping windows with padding if needed. Args: x (tensor): input tokens with [B, H, W, C]. window_size (int): window size. Returns: windows: windows after partition with [B * num_windows, window_size, window_size, C]. (Hp, Wp): padded height and width before partition rr rFr;rr)rr}padrrz contiguous) rrZrrrCpad_hpad_wHpWpwindowss rrr]sJAq!Q 1{? *k 9E 1{? *k 9E qyyEAII E!aAua/ 0 0 YE B q" #[" 2C[RSTTA !Q1a##..0055b+{TUVV  RH rrrhwct|\}}|\}}|jd||z|z|zz}||||z||z||d} | dddddd|||d} ||ks||kr&| ddd|d|ddf} | S) a Window unpartition into original sequences and removing padding. Args: windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. window_size (int): window size. pad_hw (Tuple): padded height and width (Hp, Wp). hw (Tuple): original height and width (H, W) before padding. Returns: x: unpartitioned sequences with [B, H, W, C]. rrr rFr;rrN)rrrzr) rrZrrrrrrrrs rrrys"FB DAq aR"W 3{BCA  2 bK/k2  A !Q1a##..0055aRDDA Avva aaa!RaRlO & & ( ( Hrq_sizek_sizerel_posc$tdt||zdz }|jd|kr|j}|t j}tj| d|jdd ddd|d|}| d| dd}n|}t j ||j dddft||z d z}t j ||j dddft||z d z}||z |dz t||z d zz}|| S) a\ Get relative positional embeddings according to the relative positions of query and key sizes. Args: q_size (int): size of query q. k_size (int): size of key k. rel_pos (Tensor): relative position embeddings (L, C). Returns: Extracted positional embeddings according to relative positions. r;r rrlinear)rurv)deviceNg?)r'maxrryr{r*r|r}r~rrzarangerlong) rrr max_rel_distryrel_pos_resizedq_coordsk_coordsrelative_coordss r get_rel_posrsq3vv...233L}Q<'' **U]++- OOAw}Q/ 4 4 < :::111d7CcGGH|F7>:::47CcGGH (*vzS&RU=V=V.VVO ?//11 22rrrrc|\}}|\}}t|||} t|||} |j\} } } || ||| }tjd|| }tjd|| }|d}|d}|| ||z|d}|| ||zd|}||fS)a Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py Args: q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. q_size (Tuple): spatial sequence size of query q with (q_h, q_w). k_size (Tuple): spatial sequence size of key k with (k_h, k_w). Returns: attn (Tensor): attention map with added relative positional embeddings. zbhwc,hkc->bhwkzbhwc,wkc->bhwkrr )rrrr*einsum unsqueeze)rrrrrq_hq_wk_hk_wRhRwrrr_r_qrrs rrrs(HCHC S#y ) )B S#y ) )BIAq# ))AsC % %C L)3 3 3E L)3 3 3E OOB  E OOB  E MM!S3YQ / /E MM!S3Y3 / /E %<rc eZdZdZ ddeeefdeeefdeeefd ed ed d f fd Zdejd ejfdZ xZ S)rbz# Image to Patch Embedding. rErErrrFrGr]r^rarNrOrNcztt||||||_dS)aP Args: kernel_size (Tuple): kernel size of the projection layer. stride (Tuple): stride of the projection layer. padding (Tuple): padding size of the projection layer. in_chans (int): Number of input image channels. embed_dim (int): Patch embedding dimension. )r]r^raN)rrrr)rr]r^rarNrOrs rrzPatchEmbed.__init__sA  i[QX    rrc`||}|dddd}|S)Nrr;rFr )rrzr!s rr"zPatchEmbed.forwards- IIaLL IIaAq ! !r)rrrrFrG) r#r$r%rrr'rr*r+r"r,r-s@rrbrbs (0"*#)   38_ c3h sCx             ,%,rrbc,tdddgdS)NrGrH)r;rr )encoder_embed_dim encoder_depthencoder_num_headsencoder_global_attn_indexes) _build_samrKrrbuild_sam_vit_brs) $1MM    rcd}d}d}t|||dttjjd||dd|d| }|S) NrJrDrErr0r2T) rPrOrLrRrUrQrMrTrXr[rZrS)rCrr*rr)rrrrprompt_embed_dim image_sizevit_patch_size image_encoders rrrsf JN##58-4888#!7"   M rcdeZdZdejdefdZ d dejdejdzdejfdZdS) DeepCLIPVisionEmbeddingsrprqc&|d}|d}|dd|dd}}ttj|jddz }ttj|}|j}||kr|d|||dddd }| tj }tj|||fddd  |} | dddd} | ||z|} tj|| gd } | d||zdz|} | S|S) Nrrr rFr;rsTFrtr_)rusqueezer'mathr>rryrrzrr{r*r|r}r~cat) rrprqr_ abs_pos_new cls_tokenrrryrvision_pos_embeds rrz$DeepCLIPVisionEmbeddings.get_abs_pos*sll2ooa(( #.rr?KO= ty!21!5!9::;;ty**++  x  ""1h#>>Aq!$$  *,,U];;MM)#  bii *11!Q1==M)..x(/BCHHM$y)])CKKK /44Q88Ka8OQTUU # #NrN pixel_values patch_embedsrc|jd}||}n||}|ddd}|j|dd}t j||gd}||| |j | dz}|S)Nrr;r rr) rpatch_embeddingflatten transposeclass_embeddingexpandr*rrposition_embedding position_idsru)rrr batch_size class_embeds embeddingss rr"z DeepCLIPVisionEmbeddings.forwardLs"'*  #'LL// ==L#++A..88A>> +22:q"EE Y l;CCC $"2"2  # #D$5 6 6 8J8J# #  rr)r#r$r%r*r+r'rr"rKrrrr)s| 5< 3    FOS!L8= t8K rrc eZdZ dddddededzdedzdeddf fd Zed Z ed Z ddd d e j de j dzde edzde j fdZdeeee j fdeefdZxZS)DeepCLIPVisionTransformerN)num_hidden_layers_overrideprefixconfig quant_configr r rct||_|j}t ||_t j||j|_ t||||dt|_ |j }t|j j|j kr-t!d|dt|j jddS)Nrz.encoder)r r r r attn_clszThe original encoder only has z layers, but you requested z layers.)rrr  hidden_sizerrrrlayer_norm_eps pre_layrnormr r transformernum_hidden_layerslenlayers ValueError)rr r r r rOrrs rrz"DeepCLIPVisionTransformer.__init___s  & 26::L8MNNN&%'A&&&'    #4 t& ' '&*B B BT1BTT-01A1H-I-ITTT  C BrcNt|jSr)next parametersryrs rryzDeepCLIPVisionTransformer.dtypesDOO%%&&,,rcNt|jSr)rrrrs rrz DeepCLIPVisionTransformer.devicesDOO%%&&--r) select_layersrrrc|||}||}|||du}|S)N) inputs_embedsreturn_all_hidden_states)rrr)rrrr hidden_statesencoder_outputss rr"z!DeepCLIPVisionTransformer.forwardsY lCC ))-88 **'%2$%>+  rweightsct|}t}|D]D\}}||}t|dt}|||||E|S)N weight_loader)dictnamed_parameterssetgetattrr add)rr# params_dict loaded_paramsname loaded_weightparamr%s r load_weightsz&DeepCLIPVisionTransformer.load_weightss4002233 "%%% #* $ $ D-%E#E?r>s $$$$$$ ))))))XXXXXX777777FFFFFFOOOOOO33333333 1 1 1 1 1ry 1 1 1$     ")    BBBBBbiBBBJBBBBBBIBBBJQQQQQbiQQQh |"% 5<sCx ()8 \   #s(O  c3h   \     <$3$3S$35<$3EL$3$3$3$3N" |"|"|" #s(O " #s(O "  \ """"JF4222223222jFFFFF FFFFFr