
    .`i,              	          d Z ddlmZ ddlmZmZmZ ddlmZ ddl	Z	ddl
mZ ddlmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z<m=Z= ddl>m?Z?  ee@          ZAdefdZBdefdZC G d de4          ZD G d de2eD                   ZE G d d e0          ZF G d! d"e3          ZG e?d#           e jH        eGeDeE$           G d% d&ejI        e;e=                                  ZJdS )'z"Wrapper around `Terratorch` models    )OrderedDict)IterableMappingSequence)AnyN)DummyDataGeneratorInferenceRunnerInputDefinitionInputTypeEnum)BatchFeature)
VllmConfig)BaseDummyOptions)init_logger)IdentityPooler)default_weight_loader)AutoWeightsLoader)MULTIMODAL_REGISTRY)MultiModalProcessorOnlyCache)	ImageItemModalityDataMultiModalDataDictMultiModalFieldConfigMultiModalInputsMultiModalKwargsItemsMultiModalUUIDDictPlaceholderRange)DictEmbeddingItemsModalityDataItemsMultiModalDataItemsMultiModalDataParser)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptUpdate)IntermediateTensors)&length_from_prompt_token_ids_or_embeds   )IsAttentionFreeMultiModalEmbeddingsSupportsMultiModal)	attn_typeinput_definitionc                 N    t          | j                                                  S N)setdatakeys)r,   s    y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/terratorch.py_terratorch_field_namesr3   I   s     $))++,,,    c                      dt           t          t          j        f         dt           t          t          f         f fd}|S )N	hf_inputsreturnc                     t          t          t          f                     }j                                        D ]5\  }}d}|j        t          j        k    rt          j        |d          ||<   6|S )Nimager'   )
batch_size)	dictstrr   r0   itemstyper   tensorshared)r6   fieldsnameinputmodalityr,   s        r2   _terratorch_field_configz;_terratorch_field_factory.<locals>._terratorch_field_configN   st     c00133+06688 	T 	TKD%Hz]1114;HQRSSStr4   )r   r<   torchTensorr   )r,   rE   s   ` r2   _terratorch_field_factoryrH   M   sP    	3,-		++	,	 	 	 	 	 	 $#r4   c                   0    e Zd Zdeeedz  f         fdZdS )TerratorchProcessingInfor7   Nc                 
    dd iS Nr9    selfs    r2   get_supported_mm_limitsz0TerratorchProcessingInfo.get_supported_mm_limits]   s    r4   )__name__
__module____qualname__r   r<   intrP   rM   r4   r2   rJ   rJ   \   s<        cDj)A      r4   rJ   c            	            e Zd Zdef fdZdeeef         defdZ	 d
dedeeef         deee	f         dz  de
fd	Z xZS )TerratorchInputBuilderinfoc                     t                                          |           t          | j                                                                        d                   | _        d S Npretrained_cfg)super__init__r   rW   get_hf_configto_dictdummy_data_generator)rO   rW   	__class__s     r2   r\   zTerratorchInputBuilder.__init__b   sV    $6I##%%--//0@A%
 %
!!!r4   	mm_countsr7   c                     dS )N rM   )rO   ra   s     r2   get_dummy_textz%TerratorchInputBuilder.get_dummy_texth   s    rr4   Nseq_len
mm_optionsc                 l    |rt                               d           | j                                        S )NzeConfigurable multimodal profiling options are not supported for Terratorch. They are ignored for now.)loggerwarningr_   get_dummy_mm_data)rO   re   ra   rf   s       r2   rj   z(TerratorchInputBuilder.get_dummy_mm_datak   s>      	NN,   (::<<<r4   r.   )rQ   rR   rS   rJ   r\   r   r<   rT   rd   r   r   rj   __classcell__r`   s   @r2   rV   rV   a   s        
5 
 
 
 
 
 
S(9 c     =A	= == 38$= C!112T9	=
 
= = = = = = = =r4   rV   c                        e Zd Zdef fdZdeeej        f         e	e
         z  deeef         dz  f fdZdedef fdZ xZS )	TerratorchMultiModalDataParserr,   c                 H     t                      j        |i | || _        d S r.   )r[   r\   r,   )rO   r,   argskwargsr`   s       r2   r\   z'TerratorchMultiModalDataParser.__init__   s-    $)&))) 0r4   r0   r7   Nc                     t          |t                    r7t          |dt          | j                  t          | j                            S t                                          |          S )Nr9   )rD   required_fieldsfields_factory)
isinstancer;   r   r3   r,   rH   r[   _parse_image_data)rO   r0   r`   s     r2   rv   z0TerratorchMultiModalDataParser._parse_image_data   sj     dD!! 	%  78M N N89NOO	    ww((...r4   mm_datac                 V    d|vrd|i}t                                          |          S rL   )r[   parse_mm_data)rO   rw   r`   s     r2   ry   z,TerratorchMultiModalDataParser.parse_mm_data   s0    '!!(Gww$$W---r4   )rQ   rR   rS   r
   r\   r;   r<   rF   rG   r   r   r   r   rv   r   r   ry   rk   rl   s   @r2   rn   rn   ~   s        1 1 1 1 1 1 1
/3$%Y(??/ 
38	$t	+/ / / / / /.%7 .<O . . . . . . . . . .r4   rn   c                       e Zd Zdddedddedz  ddf fdZdefd	Zd
ede	e
ef         de	e
ef         fdZdede	e
ef         dedee         fdZ	 	 dde
ee         z  dede	e
ef         de	e
ef         dz  dedz  defdZ xZS )TerratorchMultiModalProcessorN)cacherW   dummy_inputsz0BaseDummyInputsBuilder[TerratorchProcessingInfo]r|   r7   c                    |                                                                 d         }t          di |d         | _        t	                                          |||           d S )NrZ   rC   )rW   r}   r|   rM   )r]   r^   r
   _input_definitionr[   r\   )rO   rW   r}   r|   rZ   r`   s        r2   r\   z&TerratorchMultiModalProcessor.__init__   sh     ++--55778HI!0!K!K>'3J!K!KdUKKKKKr4   c                 *    t          | j                  S r.   )rn   r   rN   s    r2   _get_data_parserz.TerratorchMultiModalProcessor._get_data_parser   s    -d.DEEEr4   r6   hf_processor_mm_kwargsc                 <     t          | j                  |          S r.   )rH   r   )rO   r6   r   s      r2   _get_mm_fields_configz3TerratorchMultiModalProcessor._get_mm_fields_config   s!    
 A()?@@KKKr4   mm_itemsout_mm_kwargsc                     g S r.   rM   )rO   r   r   r   s       r2   _get_prompt_updatesz1TerratorchMultiModalProcessor._get_prompt_updates   s	     	r4   promptrw   tokenization_kwargsmm_uuidsc                 T   |                      |          }|pi }|                     ||||          }t          |                    d|          d          }dt	          dd          gi}	t          j        ||                     ||                    }
t          ddg|
||		          S )
N)r   r9   pt)tensor_typer   )offsetlength
multimodalr'   )r>   prompt_token_ids	mm_kwargs	mm_hashesmm_placeholders)	_to_mm_items_hash_mm_itemsr   getr   r   from_hf_inputsr   r   )rO   r   rw   r   r   r   r   r   mm_processed_datar   r   s              r2   applyz#TerratorchMultiModalProcessor.apply   s     $$W--17R'',.AH ( 
 
	 )KK))t
 
 
 #%5Qq%I%I%I$JK)8&&'8:PQQ
 
	
  S+
 
 
 	
r4   NN)rQ   rR   rS   rJ   r   r\   r    r   r   r   r<   objectr   r   r   r   r   r$   r   listrT   r   r   r   r   rk   rl   s   @r2   r{   r{      s        6:
L 
L 
L&
L I
L
 ,d2
L 

L 
L 
L 
L 
L 
LF"6 F F F FLL !(V 4L 
++	,	L L L L% !(V 4 -	
 
,	    <@.2
 
d3i
 $
 !(V 4	

 %S&[1D8
 %t+
 

 
 
 
 
 
 
 
r4   r{   attention_free)rW   r}   c                   B    e Zd ZdZdZededededz  fd            Zdde	d	ef fd
Z
	 dddddej        dedz  dej        dz  dedej        f
dZ	 	 ddej        dz  dej        dedz  dej        dz  def
dZdeeeej        f                  dee         fdZ xZS )
TerratorchTrD   ir7   Nc                 N    |                     d          rd S t          d          )Nr9   z Only image modality is supported)
startswith
ValueError)clsrD   r   s      r2   get_placeholder_strzTerratorch.get_placeholder_str   s,    w'' 	4;<<<r4   rc   vllm_configprefixc                     t                                                       |j        j                                        d         }t          |          | _        | j        j        | _        |j        j        }|J t                      | _
        d S rY   )r[   r\   model_config	hf_configr^   r	   inference_runnermodelpooler_configr   pooler)rO   r   r   configr   r`   s        r2   r\   zTerratorch.__init__   s{    )3;;==>NO / 7 7*0
#0>((($&&r4   F)is_multimodalhandle_oov_mm_token	input_idsmultimodal_embeddingsr   r   c                D    t          j        |j        d         df          S )Nr   )rF   emptyshape)rO   r   r   r   r   s        r2   embed_input_idszTerratorch.embed_input_ids   s      {IOA.2333r4   	positionsintermediate_tensorsinputs_embedsrq   c                     t          ||          }d |                                D             } | j        j        di |j        } |j        |gd t          |j        dz
            D             R  S )Nc                 @    i | ]\  }}||                     d           S )r   )	unsqueeze).0kvs      r2   
<dictcomp>z&Terratorch.forward.<locals>.<dictcomp>  s(    GGG1!Q[[^^GGGr4   c              3      K   | ]}d V  dS )NrM   )r   _s     r2   	<genexpr>z%Terratorch.forward.<locals>.<genexpr>  s"      BBBBBBBBr4   r'   rM   )r&   r=   r   forwardoutputexpandrangendim)	rO   r   r   r   r   rq   	input_lenbatched_kwargsmodel_outputs	            r2   r   zTerratorch.forward  s     ;9mTT	GGGGG4t,4FF~FFM #|"
BBU<+<q+@%A%ABBB
 
 
 	
r4   weightsc                    g }t          |                                           }g }|D ]\  }}t          |t           t          f          r|dk    r|}|                                D ]\  }}	d| }d|v rd|v r|                    dd          }||v rZd|v r|                    dd          }||         }
t          |
dt                    } ||
|	           |                    |           |                    ||	f            n7t          |t          j
                  r|                    d| |f           t          |           }|                    |          }|                    t          |                    S )N
state_dictzinference_runner.	pos_embedz_timm_module.rc   weight_loaderzinference_runner.model.)r;   named_buffersru   r   r=   replacegetattrr   appendrF   rG   r   load_weightsunionr/   )rO   r   params_listmodel_buffersloaded_bufferskeyvalueweights_to_parserB   weightbufferr   loaderautoloaded_weightss                 r2   r   zTerratorch.load_weights  s   T//1122! 	M 	MJC%$!455 M,&&',$(8(>(>(@(@ ? ?f9499&$..$*d22#'<<#D#DD  =00.$66'+||OR'H'H%24%8F,3 &9N- -M *M&&999*11$7777'..f~>>>>E1 '4 E5<00 M""$Cc$C$CU#KLLL #4((#00==!''N(;(;<<<r4   )rc   r.   r   )rQ   rR   rS   "supports_multimodal_raw_input_onlyis_pooling_modelclassmethodr<   rT   r   r   r\   rF   rG   r)   boolr   r%   r   r   r   tupler/   r   rk   rl   s   @r2   r   r      s        *.&=3 =3 =3: = = = [=' 'J ' ' ' ' ' ' '  >B4
 .2$)4 4 4<4  4d:4
 |d*4 "4 
4 4 4 4$ <@-1
 
<$&
 <
 2D8	

 |d*
 
 
 
 
$'=HU33D-E$F '=3s8 '= '= '= '= '= '= '= '=r4   r   )K__doc__collectionsr   collections.abcr   r   r   typingr   rF   torch.nnnnterratorch.vllmr   r	   r
   r   transformersr   vllm.configr   vllm.config.multimodalr   vllm.loggerr   !vllm.model_executor.layers.poolerr   -vllm.model_executor.model_loader.weight_utilsr    vllm.model_executor.models.utilsr   vllm.multimodalr   vllm.multimodal.cacher   vllm.multimodal.inputsr   r   r   r   r   r   r   r   vllm.multimodal.parser   r   r   r    vllm.multimodal.processingr!   r"   r#   r$   vllm.sequencer%   
vllm.utilsr&   
interfacesr(   r)   r*   interfaces_baser+   rQ   rh   r3   rH   rJ   rV   rn   r{   register_processorModuler   rM   r4   r2   <module>r     s  $ ) ( # # # # # # 7 7 7 7 7 7 7 7 7 7                         & % % % % % " " " " " " 3 3 3 3 3 3 # # # # # # < < < < < < O O O O O O > > > > > > / / / / / / > > > > > >	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                       . - - - - - = = = = = = Q Q Q Q Q Q Q Q Q Q & & & & & &	X		-o - - - -$ $ $ $ $    1   
= = = = =34LM = = =:. . . . .%9 . . .6=
 =
 =
 =
 =
$; =
 =
 =
@ ''!	!'  
_= _= _= _= _=O-? _= _=  _= _= _=r4   