`ia[ddlmZmZmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZddlmZdd lmZmZd d lmZmZmZmZmZmZmZd d lmZd d l m!Z!m"Z"ej#e$Z%GddeZ&GddeZ'Gdde!Z(GddeZ)GddeZ*GddeZ+GddeZ,GddeZ-eGddeZ.Gd d!eZ/gd"Z0dS)#)CallableOptionalUnionN)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringlogging)CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayerCLIPVisionEmbeddingsCLIPVisionModelCLIPVisionTransformer)eager_attention_forward)VisionRotaryEmbeddingapply_rotary_pos_emb_visioncFeZdZdZdZdZ dfd ZxZS)MLCDVisionConfiga This is the configuration class to store the configuration of a [`MLCDVisionModel`]. It is used to instantiate a MLCD vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the MLCD [DeepGlint-AI/mlcd-vit-bigG-patch14-336](https://huggingface.co/DeepGlint-AI/mlcd-vit-bigG-patch14-336) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1664): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 8192): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. projection_dim (`int`, *optional*, defaults to 1024): Dimensionality of text and vision projection layers. num_hidden_layers (`int`, *optional*, defaults to 48): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): The number of input channels. image_size (`int`, *optional*, defaults to 336): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. initializer_factor (`float`, *optional*, defaults to 1.0): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). Example: ```python >>> from transformers import MLCDVisionConfig, MLCDVisionModel >>> # Initializing a MLCDVisionConfig with DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> configuration = MLCDVisionConfig() >>> # Initializing a MLCDVisionModel (with random weights) from the DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> model = MLCDVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```mlcd_vision_model vision_config 0rPgeluh㈵>{Gz??c tjdi|||_||_||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_dS)N)super__init__ hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_groups num_channels patch_size image_sizeinitializer_rangeinitializer_factorattention_dropoutlayer_norm_eps hidden_act)selfr/r0r1r2r3r4r6r5r;r:r9r7r8kwargs __class__s y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/mlcd/modular_mlcd.pyr.zMLCDVisionConfig.__init__ds" ""6"""&!2!2#6 $8!($$!2"4!2,$) rr r!r"r#rr$r%r&r'r(r)r*)__name__ __module__ __qualname____doc__ model_typebase_config_keyr. __classcell__r>s@r?rr*s}4 4 l%J%O%%%%%%%%%%r@rceZdZdS)MLCDMLPN)rArBrCr,r@r?rJrJsDr@rJc,eZdZdededejfdZdS)MLCDRotaryEmbeddingnum_patches_heightnum_patches_widthreturncztj||jjdd|}tj||jjd|d}tj||gd}t||}tj||jj|jj }tj ||j}||d} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicer#rdim)rQdtype) torcharangeinv_freqrQ unsqueezeexpandstackflattenmaxrUouter) r<rMrNhpos_idswpos_idspos_ids max_grid_sizeseqrotary_pos_emb_fullrotary_pos_embs r?forwardzMLCDRotaryEmbedding.forwards L+DM4H I I I S STU V V ] ]^`bs t t  L*4=3G H H H R RST U U \ \]oqs t t  +x//1183C3C3E3EFBOOO.0ABB l=1ET]M`aaa#k#t}==-W5==a@@r@N)rArBrCintrVTensorrfr,r@r?rLrLs?##%,r@rLcHeZdZdeffd ZdejdejfdZxZ S)MLCDVisionEmbeddingsconfigcNt||`dSN)r-r.position_embeddingr<rkr>s r?r.zMLCDVisionEmbeddings.__init__s'     # # #r@ pixel_valuesrOcN|jd}|jjj}|||}|ddd}|j|dd}tj ||gd}|S)Nr)rUrr#rRrS) shapepatch_embeddingweightrUtor\ transposeclass_embeddingrZrVcat)r<rp batch_size target_dtype patch_embeds class_embeds embeddingss r?rfzMLCDVisionEmbeddings.forwards!'* +28 ++LOO,O,O,OPP #++A..88A>> +22:q"EE Y l;CCC r@) rArBrCrr.rV FloatTensorrhrfrGrHs@r?rjrjsl$/$$$$$$ E$5 %,        r@rjceZdZdZdeffd Z d dejdeejejfde ejde e d eeje ejff d Z xZ S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rkcpt||j|_d|_dSNF)r-r.r3 is_causalros r?r.zMLCDAttention.__init__s1    $*$?!r@N hidden_statesposition_embeddingsattention_maskr=rOc |jdd\}}|||||j|jf}|||||j|jf}|||||j|jf} |dd} |dd} t||| | \}}| dddd }| dddd }| dddd } t} |j jdkrt|j j} | |||| |f|jsdn|j|j|jd|\} }| dddd } | ||d} || } | ddd } | |fS) NrRrr#rreagerr()dropoutscalingr)rrq_projreshape num_headshead_dimk_projv_projrYfloatrpermute contiguousrrk_attn_implementationr trainingrscalerviewout_proj)r<rrrr=ry seq_length query_states key_states value_statescossinattention_interface attn_output attn_weightss r?rfzMLCDAttention.forwardsz"/!4SbS!9 J{{=1199:zSWSacgcp:qrr [[//77ZQUQ_aean8opp {{=1199:zSWSacgcp:qrr "!$..q117799!!$..q117799#>|ZY\^a#b#b j$++Aq!Q77BBDD ''1a33>>@@ #++Aq!Q77BBDD (? ; +w 6 6"9$+:Z"[ $7$7      % $}>CC$,Jn % %  % % ! \"))!Q155@@BB !&&z:rBB mmK00 !))!Q22==?? L((r@rm)rArBrCrDrr.rVrhtuplerr rrfrGrHs@r?rrs/26 ,),)|,)#5<#=>,)!. ,) -. ,) u|Xel33 4 ,),),),),),),),)r@rc eZdZdeffd Z d dejdeejejfdeejdee d eej f d Z xZ S) MLCDEncoderLayerrkcrt|t||_dSrm)r-r.r self_attnros r?r.zMLCDEncoderLayer.__init__s.    &v..r@NFrrroutput_attentionsrOc|}||}|||||\}}||z}|}||}||}||z}|f}|r||fz }|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. rrrr) layer_norm1r layer_norm2mlp)r<rrrrresidualroutputss r?rfzMLCDEncoderLayer.forwards*!((77 &*nn' 3)/ '5' ' # | !=0  ((77 //  =0  "  '   &Gr@r) rArBrCrr.rVrhrrboolr~rfrGrHs@r?rrs////////26,1 **|*#5<#=>*!. * $D> * u ! ********r@rceZdZdZdeffd Z d dejdeej ej fde ej de e d e e d e e d e ee ffd ZxZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rkcJt|dS)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.N)r-r.ros r?r.zMLCDEncoder.__init__5s!      r@N inputs_embedsrrroutput_hidden_states return_dictrOc|||n |jj}||n |jj}||n |jj}|rdnd}|rdnd}|} t |jD]2\} } |r|| fz}| | |||} | d} |r || dfz}3|r|| fz}|st d| ||fDSt| ||S)aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr,rrr#c3K|]}||V dSrmr,).0vs r? z&MLCDEncoder.forward..ws(eeqWXWdWdWdWdWdeer@)last_hidden_stater attentions)rkruse_return_dictr enumeratelayersrr ) r<rrrrrrencoder_statesall_attentionsridx encoder_layer layer_outputss r?rfzMLCDEncoder.forward9sMD%9$D $+Jj &1%>N fee]NN$Seeeee e+(%    r@NNNN)rArBrCrDrr.rVr~rrhrrrr rfrGrHs@r?rr,s!/!!!!!!26,0/3&*C C (C #5<#=>C !. C $D> C 'tn C d^C  uo% &C C C C C C C C r@rceZdZdeffd Ze d deejdee dee dee de e e ff d Z xZS) MLCDVisionTransformerrkc t|t|j|jzdz|_t jtj d|j|jzdz|_ dS)Nrr#) r-r.rLr/r2vision_rotary_embeddingnn ParameterrVrandn class_pos_embros r?r.zMLCDVisionTransformer.__init__sr    ':6;MQWQk;kop;p'q'q$\%+a9KvOi9imn9n*o*oppr@NrprrrrOcL||n |jj}||n |jj}||n |jj}|t d|jd|jjz}|jd|jjz}|||}||j j }tj |j |gd}tj ||fd}| |f} ||} || } || | |||} | d} | dddddf} || } |s| | f| ddzSt'| | | j| jS) Nz You have to specify pixel_valuesrRrrS)rrrrrr#)r pooler_outputrr)rkrrr ValueErrorrrr5rrurrQrVrxrrr} pre_layrnormencoderpost_layernormr rr)r<rprrrrMrNreembrrencoder_outputsr pooled_outputs r?rfzMLCDVisionTransformer.forwards%9$D $+Jj &1%/ 'tn / d^ / u00 1 / / / ^/ / / / / r@rc0eZdZUeed<dZdZdZdZdZ dS)MLCDPreTrainedModelrkmlcdTc|jj}t|trx|jj}tj|jd|jdz|ztj|j j |jj |zdSt|tr|jj}|jdzd|jj zdzz|z}|jdz|z}tj|jj |tj|jj |tj|jj |tj|jj |dSt|t$r|jj}|jjdzd|jj zdzz|z}d|jjzdz|z}tj|jj |tj|jj |dSt|t,rW|jj}|jj|jjzdzdz|z}tj|jd|dSt|tjr?|jj|j jddSt|tjr'|j"|jjdSdSdS)zInitialize the weightsr(g)meanstd)rrr*N)rkr8 isinstancerjrinitnormal_rw embed_dimrsrtr7rr1rrrrrJr/fc1fc2rr2r LayerNormbiasdatazero_fill_Linear)r<modulefactor in_proj_std out_proj_stdfc_std pos_emb_stds r? _init_weightsz!MLCDPreTrainedModel._init_weightss/ f2 3 3 %[3F GOOF2&BRTXBX[aBaO b b b GOOF29v}?^ag?gO h h h h h  . . %[3F!+T1q6=;Z7Z_c6cdgmmK",d2f>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```N)rprrr)rkrrr vision_model)r<rprrrs r?rfzMLCDVisionModel.forwardsr>%9$D $+Jj &1%( 'tn ( d^ ( u00 1 ( ( ( ^( ( ( r@r)rrr)1typingrrrrVtorch.nnrconfiguration_utilsrmodeling_flash_attention_utilsrmodeling_outputsr r modeling_utilsr r processing_utilsr utilsrrclip.modeling_cliprrrrrrrllama.modeling_llamarqwen2_vl.modeling_qwen2_vlrr get_loggerrAloggerrrJrLrjrrrrrr__all__r,r@r?rs:-,,,,,,,,, 333333BBBBBBKKKKKKKKFFFFFFFF&&&&&&,,,,,,,,;:::::[[[[[[[[  H % %Y%Y%Y%Y%Y%'Y%Y%Y%x     g   /D/$9)9)9)9)9)M9)9)9)x/////'///dP P P P P +P P P f6 6 6 6 6 16 6 6 r$%$%$%$%$%/$%$%$%N* * * * * o* * * Z   r@