
    .`i              	       ~   d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ ddlmZmZmZ ddlmZmZmZm Z  ddl!m"Z"m#Z#m$Z$  ee%          Z& G d dej'                  Z( G d de          Z) ej*        e)e e           G d deeee                      Z+dS )    )IterableMappingN)BatchFeature)ModelConfig
VllmConfig)TokensPrompt)init_logger)ColumnParallelLinearRowParallelLinear)DispatchPooler)MULTIMODAL_REGISTRY)IntermediateTensors   )SupportsCrossEncodingSupportsMultiModalSupportsScoreTemplate)Qwen2VLDummyInputsBuilderQwen2VLForConditionalGenerationQwen2VLMultiModalProcessorQwen2VLProcessingInfo)AutoWeightsLoaderWeightsMappermaybe_prefixc                   0     e Zd Zddddef fdZd Z xZS )JinaVLScorer model_configr   prefixc                     t                                                       |j                                        }|j        }t          |j        |j        |d| d          | _        t          |j        |j	        |d| d          | _
        d S )NTz.dense)params_dtypebiasr   z	.out_proj)super__init__	hf_configget_text_config
head_dtyper
   hidden_sizedenser   
num_labelsout_proj)selfr   r   configr&   	__class__s        v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/jina_vl.pyr#   zJinaVLScorer.__init__   s    '7799!,
)#$$$
 
 

 *#'''
 
 
    c                     |                      |          \  }}t          j        |          }|                     |          \  }}|S N)r(   torchrelur*   )r+   xkwargs_s       r.   forwardzJinaVLScorer.forward1   s<    zz!}}1JqMM}}Q1r/   )r   )__name__
__module____qualname__strr#   r7   __classcell__r-   s   @r.   r   r      s_        
 
] 
C 
 
 
 
 
 
&      r/   r   c            
       d     e Zd Zdedeeef         deeef         deeef         def
 fdZ xZS )JinaVLMultiModalProcessorpromptmm_data	mm_kwargs
tok_kwargsreturnc                     |                                 D ]\  }}|                                 t                                          ||||          S r1   )itemsreverser"   _call_hf_processor)r+   r@   rA   rB   rC   r6   valuer-   s          r.   rH   z,JinaVLMultiModalProcessor._call_hf_processor9   sM       	 	HAuMMOOOOww))&'9jQQQr/   )	r8   r9   r:   r;   r   objectr   rH   r<   r=   s   @r.   r?   r?   8   s        RR f%R 3;'	R
 CK(R 
R R R R R R R R R Rr/   r?   )infodummy_inputsc                   \    e Zd ZdZ eddddddd          Zd	d
dedef fdZe	dede
dedz  fd            Ze	dedededz  fd            Ze	deddfd            Z	 	 d dej        dej        dedz  dej        dz  dedej        f fdZdeeeej        f                  fdZ xZS )!JinaVLForSequenceClassificationTzscore.dense.zscore.out_proj.zlanguage_model.model.visual.zlanguage_model.lm_head.)zscore.0.zscore.2.zmodel.language_model.rO   zlm_head.zmodel.)orig_to_new_prefixr   r   vllm_configr   c                   t                                          |t          |d                     |j        j        }|J t          |j        t          |d                    | _        t          j        || j                  | _	        d S )Nqwen2_vl)rR   r   scorerQ   )
classifier)
r"   r#   r   r   pooler_configr   rU   r   for_seq_clspooler)r+   rR   r   rW   r-   s       r.   r#   z(JinaVLForSequenceClassification.__init__b   s    #L,L,L 	 	
 	
 	
 $0>(((!$\&'-J-J
 
 

 %04:VVVr/   modalityirD   Nc                 N    |                     d          rdS t          d          )Nimagez+<|vision_start|><|image_pad|><|vision_end|>z Only image modality is supported)
startswith
ValueError)clsrZ   r[   s      r.   get_placeholder_strz3JinaVLForSequenceClassification.get_placeholder_strn   s.    w'' 	A@@;<<<r/   querydocumentc                     d| d| S )Nz**Document**:
z
**Query**:
 )r`   rb   rc   s      r.   get_score_templatez2JinaVLForSequenceClassification.get_score_templateu   s    @@@@@@r/   r@   c                 <    |d                              d           d S )Nprompt_token_idsd   )append)r`   r@   s     r.   post_process_tokensz3JinaVLForSequenceClassification.post_process_tokensy   s$     	!"))#.....r/   	input_ids	positionsintermediate_tensorsinputs_embedsr5   c                 D     t                      j        d||||d|}|S )N)rl   rm   rn   ro   re   )r"   r7   )r+   rl   rm   rn   ro   r5   hidden_statesr-   s          r.   r7   z'JinaVLForSequenceClassification.forward~   sD     ( 
!5'	
 

 
 
 r/   weightsc                 X    t          |           }|                    || j                  S )N)mapper)r   load_weightsweight_mapper)r+   rr   loaders      r.   ru   z,JinaVLForSequenceClassification.load_weights   s+    "4((""743E"FFFr/   )NN)r8   r9   r:   is_pooling_modelr   rv   r   r;   r#   classmethodintra   rf   r   rk   r2   Tensorr   rJ   r7   r   tupleru   r<   r=   s   @r.   rN   rN   I   s        !M&)%< 1-	
 	
  M BD 
W 
W 
Wz 
W3 
W 
W 
W 
W 
W 
W =3 =3 =3: = = = [= As Ac AcDj A A A [A / /$ / / / [/ <@-1 < < 2D8	
 |d*  
     "GHU33D-E$F G G G G G G G Gr/   rN   ),collections.abcr   r   r2   torch.nnnntransformersr   vllm.configr   r   vllm.inputsr   vllm.loggerr	   !vllm.model_executor.layers.linearr
   r   !vllm.model_executor.layers.poolerr   vllm.multimodalr   vllm.sequencer   
interfacesr   r   r   rT   r   r   r   r   utilsr   r   r   r8   loggerModuler   r?   register_processorrN   re   r/   r.   <module>r      s>   . - - - - - - -        % % % % % % / / / / / / / / $ $ $ $ $ $ # # # # # # U U U U U U U U < < < < < < / / / / / / - - - - - - X X X X X X X X X X            B A A A A A A A A A	X		    29   6R R R R R : R R R" ('	*  
CG CG CG CG CG#	CG CG 
CG CG CGr/   