
    .`i|                     @   U d dl mZmZmZ d dlmZmZmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z; ddl<m=Z=m>Z>m?Z? ddl@mAZAmBZBmCZC dZDdZEdZFdZGdZH G d de:          ZI G d de:          ZJeIeJz  ZKeeLd <   d!eMfd"ZNd#eOd$ePeQeMeMf                  d%eMd&eMd'eMd(eQeMeMf         fd)ZRd*eMd+eMd,eSd-eSd(eQeMeMf         f
d.ZTd/eMd0eMd(ePeQeMeMf                  fd1ZUd2eMd3eMd$ePeQeMeMf                  d'eMd-eSd(eQeMeMeMf         fd4ZVd5ej        d$ePeQeMeMf                  d'eMd-eSd(ePej                 f
d6ZWd5ej        d!eMd/eMd0eMd-eSd(ejX        fd7ZY G d8 d9          ZZ G d: d;e1          Z[ G d< d=e/e[                   Z\ G d> d?e0e[                   Z] e"j^        e]e[e\@           G dA dBe
j_        e>e?                      Z`dS )C    )IterableMappingSequence)	AnnotatedLiteral	TypeAliasN)Image)BatchFeaturePretrainedConfig
TensorType)
VllmConfig)BaseDummyOptions)ReplicatedLinear)QuantizationConfig)	AWQConfig)InternVisionModelInternVisionPatchModel)MULTIMODAL_REGISTRYconvert_image_mode)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems)ImageEmbeddingItemsImageProcessorItems	ImageSizeMultiModalDataItems)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacementPromptUpdatePromptUpdateDetails)IntermediateTensors)TokenizerLike)TensorSchemaTensorShape   )MultiModalEmbeddingsSupportsMultiModal
SupportsPP)AutoWeightsLoaderinit_vllm_registered_modelmaybe_prefixz<img>z</img>z<IMG_CONTEXT>)g
ףp=
?gv/?gCl?)gZd;O?gy&1?g?c                       e Zd ZU dZdZed         ed<   eej	         e
dddd          f         ed<   eej	         e
d	          f         ed
<   dS )SkyworkR1VImagePixelInputsz
    Dimensions:
        - bnp: Batch size * number of images * (1 + num_patches)
        - c: Number of channels (3)
        - h: Height
        - w: Width
        - bn: Batch size * number of images
    pixel_valuestypebnp   hwpixel_values_flatbnnum_patchesN)__name__
__module____qualname____doc__r2   r   __annotations__r   torchTensorr'        y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/skyworkr1v.pyr0   r0   @   s           %3D'.
!222 E1c3''	)   
 D	     rB   r0   c                       e Zd ZU dZdZed         ed<   eej	        e
ej	                 z   eddd          f         ed<   dS )	SkyworkR1VImageEmbeddingInputsz
    Dimensions:
        - ni: Number of images
        - ifs: Image feature size
        - hs: Hidden size (must match the hidden size of language model
          backbone)
    image_embedsr2   niifshsdataN)r:   r;   r<   r=   r2   r   r>   r   r?   r@   listr'   rA   rB   rC   rE   rE   W   su           %3D'.
!222
tEL))D%&&	(     rB   rE   SkyworkR1VImageInputs
input_sizec           	         t           t          }}t          j        t          j        d           t          j        | | ft          j        j                  t          j                    t          j	        ||          g          S )Nc                 "    t          | d          S )NRGBr   )imgs    rC   <lambda>z!build_transform.<locals>.<lambda>r   s    !3C!?!? rB   )interpolation)meanstd)
IMAGENET_MEANIMAGENET_STDTComposeLambdaResizeInterpolationModeBICUBICToTensor	Normalize)rM   MEANSTDs      rC   build_transformrb   n   sv    |#D9H??@@HZ(8K8S   JLLKTs+++	
	 	 	rB   aspect_ratiotarget_ratioswidthheight
image_sizereturnc                    t          d          }d}||z  }|D ]V}|d         |d         z  }	t          | |	z
            }
|
|k     r|
}|}0|
|k    r |d|z  |z  |d         z  |d         z  k    r|}W|S )Ninf)r(   r(   r   r(         ?)floatabs)rc   rd   re   rf   rg   best_ratio_diff
best_ratioarearatiotarget_aspect_ratio
ratio_diffs              rC   find_closest_aspect_ratiort   }   s     EllOJ6>D # ##Ahq1(;;<<
''(OJJ?**cJ&3eAh>qIII"
rB   min_dynamic_patchmax_dynamic_patchdynamic_image_sizeuse_thumbnailc                 <    |r| nd} |r|nd}|r|dk    r|dz  }| |fS )Nr(   rA   ru   rv   rw   rx   s       rC   resolve_skyworkr1v_min_max_numr{      sQ     .@F))Q-?F))Q *a//Q///rB   min_nummax_numc                 j      fdt           dz             D             }t          |d           S )Nc                     h | ]E}t          d |d z             D ]/}t          d |d z             D ]}||z  cxk    rk    n n||f0FS )r(   )range).0nijr}   r|   s       rC   	<setcomp>z/get_skyworkr1v_target_ratios.<locals>.<setcomp>   s       q!a%  q!a%	  a!e&&&&w&&&&&	 
A '&&&&rB   r(   c                 $    | d         | d         z  S )Nr   r(   rA   )xs    rC   rR   z.get_skyworkr1v_target_ratios.<locals>.<lambda>   s    qtad{ rB   )key)r   sorted)r|   r}   rd   s   `` rC   get_skyworkr1v_target_ratiosr      sX        w!,,  M -%:%:;;;;rB   
orig_widthorig_heightc                     | |z  }t          ||| ||          }||d         z  }||d         z  }|d         |d         z  }	|r|	dk    r|	dz  }	|	||fS )N)re   rf   rg   r   r(   )rt   )
r   r   rd   rg   rx   rc   rr   target_widthtarget_heightblockss
             rC   calculate_skyworkr1v_targetsr      s     +L 4    3A 66L!4Q!77M #&9!&<<F  1!<..rB   imagec                    | j         \  }}t          ||||d          \  }}}|                     ||f          }	g }
t          |          D ]\}|||z  z  |z  |||z  z  |z  |||z  z  dz   |z  |||z  z  dz   |z  f}|	                    |          }|
                    |           ]t          |
          |k    sJ |r?t          |
          dk    r,|                     ||f          }|
                    |           |
S )NF)r   r   rd   rg   rx   r(   )sizer   resizer   cropappendlen)r   rd   rg   rx   r   r   r   r   r   resized_imgprocessed_imagesr   box	split_imgthumbnail_imgs                  rC   dynamic_preprocess_skyworkr1vr      sZ    $jJ +G#+ + +'FL- ,,m<==K6]] 	+ 	+,*,-;<:-.*<<:-.!3zALJ./14
B	
  $$S))		****  F**** /-..!33j*%=>>...rB   c                    t          ||          }t          |          t          | |||          }t          j        fd|D                       }|S )N)rM   )rd   rg   rx   c                 &    g | ]} |          S rA   rA   )r   r   	transforms     rC   
<listcomp>z4image_to_pixel_values_skyworkr1v.<locals>.<listcomp>  s#    EEEU		% 0 0EEErB   )r   rb   r   r?   stack)	r   rM   r|   r}   rx   rd   imagesr1   r   s	           @rC    image_to_pixel_values_skyworkr1vr      sp     1'BBM:666I*##	  F ;EEEEfEEEFFLrB   c                   P    e Zd ZdZdddddedededz  dedz  dedz  d	df fd
Ze	d	efd            Z
dededz  d	ee         fdZddddddedz  dedz  dedz  dedz  d	eeef         f
dZddddddedz  dedz  dedz  dedz  d	eeeef                  f
dZdeded	efdZ	 	 	 ddeej                 dedz  dedz  dedz  d	eej                 f
dZ	 	 	 	 	 	 ddeee         z  dz  dej        eej                 z  dz  dedz  dedz  dedz  deez  dz  d	efdZ xZS )SkyworkR1VProcessorz
    This model doesn't define its own HF processor,
    so we implement our own one here.

    The code to insert image tokens is based on:
    https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252
    Nru   rv   rw   config	tokenizerru   rv   rw   rh   c                   t                                                       || _        || _        |j        j        }|j        j        }||j        }t          |t                    sJ ||j
        }t          |t                    sJ ||j        }t          |t                    sJ t          ||z  dz  |j        dz  z            | _        || _        || _        || _
        || _        |j        | _        d S )N   )super__init__r   r   vision_configrg   
patch_sizeru   
isinstanceintrv   rw   booldownsample_rationum_image_tokenrx   )	selfr   r   ru   rv   rw   rg   r   	__class__s	           rC   r   zSkyworkR1VProcessor.__init__  s    	" .9
 .9
$ & 8+S11111$ & 8+S11111%!'!:,d33333":%!+v/F/IJ 
  
 %!2!2"4#)#7rB   c                 J    | j                                         t                   S N)r   	get_vocabIMG_CONTEXTr   s    rC   image_token_idz"SkyworkR1VProcessor.image_token_id@  s    ~''))+66rB   feature_sizer9   c                 n    t           |z  }t          |z   t          z   }t          j        |t                     S r   )r   	IMG_STARTIMG_ENDr#   select_text)r   r   r9   repl_features	repl_fulls        rC   get_image_replz"SkyworkR1VProcessor.get_image_replD  s0    
 $l2-7	".y+FFFrB   rz   rx   c                    || j         n|}|| j        n|}|| j        n|}|| j        n|}t	          ||||          S Nrz   )ru   rv   rw   rx   r{   )r   ru   rv   rw   rx   s        rC   resolve_min_max_numz'SkyworkR1VProcessor.resolve_min_max_numN  s     '8&?D""EV 	 '8&?D""EV 	
 ") ### 	
 /<.C**-//1'	
 
 
 	
rB   c                Z    |                      ||||          \  }}t          ||          S r   )r   r   )r   ru   rv   rw   rx   r|   r}   s          rC   resolve_target_ratiosz)SkyworkR1VProcessor.resolve_target_ratiosj  sA      33//1'	 4 
 
 ,GW===rB   image_widthimage_heightc                    |                      d          }t          ||| j        || j                  \  }}}|| j        z  S )NF)rx   )r   r   rg   rd   rx   )r   r   rg   rx   r   )r   r   r   rd   r9   _s         rC   get_num_image_tokensz(SkyworkR1VProcessor.get_num_image_tokens{  sb     22 3 
 
 9"$',
 
 
Q T111rB   r   c                 `                           |||d          \   fd|D             S )NFrz   c           	      L    g | ] }t          |j        j                   !S ))rM   r|   r}   rx   )r   rg   rx   )r   r   r}   r|   r   s     rC   r   zCSkyworkR1VProcessor._images_to_pixel_values_lst.<locals>.<listcomp>  sN     	
 	
 	
  -?"0  	
 	
 	
rB   )r   )r   r   ru   rv   rw   r}   r|   s   `    @@rC   _images_to_pixel_values_lstz/SkyworkR1VProcessor._images_to_pixel_values_lst  si      33//1	 4 
 
	
 	
 	
 	
 	
 	
  	
 	
 	
 		
rB   textreturn_tensorsc                    |g }t          |t                    s|g}|g }t          |t                    s|g}t          |          dk    ri }n|                     ||||          }t	          j        |          t	          j        d |D                       d}|D ]=}	|	j        d         }
|
| j        z  }| 	                    ||
          fd|D             }>| 
                    |          }i ||}t          ||          S )Nr   r   c                 ,    g | ]}t          |          S rA   )r   )r   items     rC   r   z0SkyworkR1VProcessor.__call__.<locals>.<listcomp>  s    <<<4SYY<<<rB   )r7   image_num_patchesc                 H    g | ]}|                     d j        d          S )<image>r(   )replacefull)r   t
image_repls     rC   r   z0SkyworkR1VProcessor.__call__.<locals>.<listcomp>  s+    OOOQ		)Z_a@@OOOrB   )tensor_type)r   rK   r   r   r?   cattensorshaper   r   r   r
   )r   r   r   ru   rv   rw   r   image_inputspixel_values_lstr1   r9   r   text_inputscombined_outputsr   s                 @rC   __call__zSkyworkR1VProcessor.__call__  sZ    <D$%% 	6D>F&$'' 	XFv;;!LL#??"3"3#5	  @     &+Y/?%@%@%*\<<+;<<<& & L !1 P P*03*T-AA!00{KK
OOOO$OOOnnT**:k:\:,.IIIIrB   )NNN)NNNNNN)r:   r;   r<   r=   r   r%   r   r   r   propertyr   r#   strr   tupler   rK   r   r   r	   r?   r@   r   r   r
   r   __classcell__r   s   @rC   r   r     sR         )-(,*.$8 $8 $8 $8 !$8
 :$8 :$8 !4K$8 
$8 $8 $8 $8 $8 $8L 7 7 7 7 X7GG 4ZG 
S	!	G G G G )-(,*.%)
 
 
 :
 :	

 !4K
 d{
 
sCx
 
 
 
> )-(,*.%)> > > :> :	>
 !4K> d{> 
eCHo	> > > >"2 2 	2
 
2 2 2 2. )-(,*.
 
U[!
 :
 :	

 !4K
 
el	
 
 
 
6 (,9=(,(,*.26.J .JDIo$.J d5;//$6.J :	.J
 :.J !4K.J j(4/.J 
.J .J .J .J .J .J .J .JrB   r   c                   j    e Zd ZdedefdZdeeedz  f         fdZ	dedededz  defd	Z
defd
ZdS )SkyworkR1VProcessingInfokwargsrh   c                      | j         j        t          f|                                 |                                 d|S )N)r   r   )ctxinit_processorr   get_hf_configget_tokenizer)r   r   s     rC   get_hf_processorz)SkyworkR1VProcessingInfo.get_hf_processor  sP    &tx&
%%''((**
 
 	
 
 	
rB   Nc                 
    dd iS )Nr   rA   r   s    rC   get_supported_mm_limitsz0SkyworkR1VProcessingInfo.get_supported_mm_limits  s    rB   r   r   	processorc                \    ||                                  }|                    ||          S )N)r   r   )r   r   )r   r   r   r   s       rC   r   z-SkyworkR1VProcessingInfo.get_num_image_tokens  s>     --//I--#% . 
 
 	
rB   c                 "   |                                  }|j        }|                                }d\  }}|D ]@\  }}||z  ||z  }	}|                     ||	|          }
|
|k    r|
}t	          ||	          }A|dk    s|t          d          |S )N)r   Nr   r   r   )re   rf   r   z(Cannot have a largest feature size of 0!)r   rg   r   r   r   
ValueError)r   r   	base_sizerd   largest_feature_sizelargest_feature_pinpointwrhrre   rf   	feat_sizes              rC   !get_image_size_with_most_featuresz:SkyworkR1VProcessingInfo.get_image_size_with_most_features  s    ))++	(	!77999@66# 
	Q 
	QFB%NIN6E11!## 2  I
 ///'0$+45+P+P+P(1$$(@(HGHHH''rB   )r:   r;   r<   objectr   r   r   r   r   r   r   r   r  rA   rB   rC   r   r     s        
 
4G 
 
 
 
cDj)A    
 
 	

 '-
 

 
 
 
(9 ( ( ( ( ( (rB   r   c            	       p    e Zd Zdeeef         defdZ	 ddedeeef         deeef         dz  defdZ	dS )	SkyworkR1VDummyInputsBuilder	mm_countsrh   c                 8    |                     dd          }d|z  S )Nr   r   r   )get)r   r  
num_imagess      rC   get_dummy_textz+SkyworkR1VDummyInputsBuilder.get_dummy_text  s     ]]7A..
:%%rB   Nseq_len
mm_optionsc                     | j                                         \  }}|                    dd          }|r|                    d          nd }d|                     ||||          iS )Nr   r   )re   rf   r
  	overrides)infor  r	  _get_dummy_images)r   r  r  r  r   r   r
  image_overridess           rC   get_dummy_mm_dataz.SkyworkR1VDummyInputsBuilder.get_dummy_mm_data  s|     '+i&Q&Q&S&S#m]]7A..
5?I*..111T T++"$%)	 ,  
 	
rB   r   )
r:   r;   r<   r   r   r   r  r   r   r  rA   rB   rC   r  r    s        &S(9 &c & & & & =A	
 

 38$
 C!112T9	

 

 
 
 
 
 
rB   r  c            
            e Zd Zdedeeef         deeef         deeef         def
 fdZdedeeef         deeef         fd	Z	d
e
deeef         dedee         fdZ xZS )SkyworkR1VMultiModalProcessorpromptmm_data	mm_kwargs
tok_kwargsrh   c                     t                                          ||||          } | j        j        di |}|j        }t          j        |          |d<   |S )N)r  r  r  r  r   rA   )r   _call_hf_processorr  r   r   r?   r   )	r   r  r  r  r  processed_outputshf_processorr   r   s	           rC   r  z0SkyworkR1VMultiModalProcessor._call_hf_processor)  sr     "GG66!	 7 
 
 2ty1>>I>>%4
 /4l>.J.J*+  rB   	hf_inputshf_processor_mm_kwargsc           	      *   |                     dt          j        d                    }t          |          }t	          t          j        d|          t          j        d          t          j        d          t          j        d|                    S )Nr   r   r   )r7   r   rF   r   )	r	  r?   emptyr   dictr   flat_from_sizesbatchedshared)r   r  r  r   r
  s        rC   _get_mm_fields_configz3SkyworkR1VMultiModalProcessor._get_mm_fields_configA  s    
 &MM*=u{1~~NN*++
3C*  4;GDD.6w??07LL
 
 
 	
rB   mm_itemsout_mm_kwargsc                 V      j         j        di ||                                }d|v r9|d         t          t          j                  sJ                                 n d|v rd gt          |d                   z  ng dt          f fd}t          dd|          gS )	Nr   rF   item_idxc                                         dt          t          f          }t          |t                    r|                    |           }n<|                    |           }j                            |j        |j	                  }|          }|t          |t                    sJ                     ||          S )Nr   r   )	get_itemsr   r   r   get_feature_sizeget_image_sizer  r   re   rf   r   r   )	r*  r   r   rg   r9   r  r   r'  r   s	        rC   get_replacement_skyworkr1vzUSkyworkR1VMultiModalProcessor._get_prompt_updates.<locals>.get_replacement_skyworkr1vf  s    ''-/BC F &"566 %66x@@#228<<
#y== * 0!+!2*  >     ,H5K&!+s33333..|[IIIrB   r   r   )modalitytargetreplacementrA   )
r  r   get_datar   r?   r@   tolistr   r   r!   )r   r'  r  r(  out_mm_datar/  r  r   s   ``    @@rC   _get_prompt_updatesz1SkyworkR1VMultiModalProcessor._get_prompt_updatesR  s    2ty1KK4JKK#,,..+-- +,? @/>>>>> 1 8 8 : :{** "&[-H)I)I I "	J 	J 	J 	J 	J 	J 	J 	J 	J 	J,   6  
 	
rB   )r:   r;   r<   r   r   r  r
   r  r   r&  r   r   r   r"   r6  r   r   s   @rC   r  r  (  s       !! f%! 3;'	!
 CK(! 
! ! ! ! ! !0

 !(V 4
 
++	,	
 
 
 
"/
%/
 !(V 4/
 -	/

 
,	/
 /
 /
 /
 /
 /
 /
 /
rB   r  )r  dummy_inputsc                       e Zd Zededededz  fd            Zddded	eddf fd
Zde	de
fdZde	de
dz  ded	efdZ	 d+de	de
d	edej        fdZd,dZdej        dej        fdZdededz  fdZdedej        eej                 z  eej        df         z  fdZdej        ddfdZdedefdZ	 d-ddddej        dedz  d ej        dz  d!edej        f
 fd"Z	 	 d.dej        d#ej        d$edz  d%ej        dz  dedefd&Z d'ej        dej        dz  fd(Z!d)e"eeej        f                  de#e         fd*Z$ xZ%S )/SkyworkR1VChatModelr0  r   rh   Nc                 N    |                     d          rdS t          d          )Nr   r   z Only image modality is supported)
startswithr   )clsr0  r   s      rC   get_placeholder_strz'SkyworkR1VChatModel.get_placeholder_str  s,    w'' 	9;<<<rB    prefixvllm_configr@  c          
         t                                                       |j        j        }|j        }|j        j        }|| _        || _        |                     ||           |j        p|j	        j
        }|j	        j        }|| _        t          ||z  dz  |j        dz  z            | _        |j        | _        |j        | _        |j        j        d         }|dk    | _        |                     |d          5  |                     ||| j        t+          |d                    | _        |                     ||t+          |d                    | _        d d d            n# 1 swxY w Y   |                     |          5  t5          ||j        t+          |d	          
          | _        d d d            n# 1 swxY w Y   d | _        d | _        | j        j        | _        d S )Nr   r   SkyworkLM2VEForCausalLMr   vision_model)quant_configis_monor@  mlp1r?  language_model)rA  	hf_configr@  )r   r   model_configrI  rE  multimodal_configr   _patch_quant_configforce_image_sizer   rg   r   r   r   r   
ps_versiontext_configarchitecturesrF  _mark_tower_model_init_vision_modelr.   rD  
_init_mlp1rG  _mark_language_modelr-   rH  img_context_token_idvisual_token_maskmake_empty_intermediate_tensors)
r   rA  r@  r   rE  rK  rg   r   llm_arch_namer   s
            rC   r   zSkyworkR1VChatModel.__init__  sq   )3"/'4F!2  666,O0D0O
)4
$":%!+v/F/IJ 
  
 !' 7 +*8;$(AA##K99 		 		 $ 7 7)#FN;;	 !8 ! !D \&&-I-I (  DI		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 &&{33 	 	"<' ,#F,<==# # #D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 %)!!%? 	,,,s%   ;AE$$E(+E(+F;;F?F?r   rE  c                     t          |t                    r=|j        }t          |dd           }|j        s | |j                            d           d S d S d S d S )Nquantization_configrD  )r   r   rO  getattrmodules_to_not_convertr   )r   r   rE  rO  llm_quant_configs        rC   rL  z'SkyworkR1VChatModel._patch_quant_config  s    
 lI.. 	K ,K&{4I4PP 7 K ,3::>JJJJJ	K 	KK K,,rB   rF  c                    |s=|j         }|dk     r|j        j        |z   dz   }n|dz   }t          |j        |||          S t	          |j                  S )Nr   r(   )rE  num_hidden_layers_overrider@  )select_layerr   num_hidden_layersr   r   )r   r   rE  rF  r@  vision_feature_layerra  s          rC   rR  z&SkyworkR1VChatModel._init_vision_model  s      	@#)#6 #a''(:=QQTUU "! %91$<!$$)+<	    *&*>???rB   c                 f   |j         j        }|j        j        }t          j        t          j        |t          d| j        z            dz  z            t          |t          d| j        z            dz  z  |d|| d          t          j	                    t          ||d|| d                    S )Nr(   r   Fz.1)return_biasrE  r@  z.3)
r   hidden_sizerO  nn
Sequential	LayerNormr   r   r   GELU)r   r   rE  r@  vit_hidden_sizellm_hidden_sizes         rC   rS  zSkyworkR1VChatModel._init_mlp1  s     !.: ,8}L3q43H/H+I+IQ+NNOO#a$*?&?"@"@A"EE!) }}}   GII!) }}}  
 
 	
rB   rk   c           
         |                                 \  }}}}|                    ||t          ||z            t          ||z                      }|                    dddd                                          }|                    |t          ||z            t          ||z            t          |||z  z                      }| j        dk    rn*|                    dddd                                          }|S )Nr   r   r(   r4   v1)r   viewr   permute
contiguousrN  )r   r   scale_factorr   r6   r5   cs          rC   pixel_shufflez!SkyworkR1VChatModel.pixel_shuffle  s    VVXX
1aFF1aQ-..A4D0E0EFFIIaAq!!,,..FFL !!L !!\L0122	
 
 ?d""		!Q1%%0022ArB   r1   c                    |                      |          }|d d dd d d f         }t          |j        d         dz            x}}|                    |j        d         ||d          }|                     || j                  }|                    |j        d         d|j        d                   }|                     |          }|S )N)r1   r(   rk   r   )rq  )rD  r   r   reshapers  r   rG  )r   r1   
vit_embedsr5   r6   s        rC   extract_featurez#SkyworkR1VChatModel.extract_feature  s    &&L&AA
122qqq)
J$Q'3.///A''
(8(;Q2FF
''
AV'WW
''
(8(;RAQRTAUVV
YYz**
rB   r   c                 2   |                     dd           }|                     dd           }|                     dd           }||d S |t          d|          S |d         }t          |t          j                  r8|                                                                                                }t          |t                    sJ || _	        |4t          d||| j        j        j        | j        j        j        d          S t          d	          )
Nr7   r   rF   )r2   rJ   r   r1   )r5   r6   )r2   r7   r9   resolve_bindingsz This line should be unreachable.)poprE   r   r?   r@   flattenuniquer   r   rU  r0   r   r   rg   AssertionError)r   r   r7   r   rF   r   s         rC   _parse_and_validate_image_inputz3SkyworkR1VChatModel._parse_and_validate_image_input   s4    #JJ':DAA"JJ':DAAzz.$77$)=4#1#!   
   01nel33 	F+3355<<>>CCEEN.#.....$2!(-#"3-2=2=" "	    ?@@@rB   image_input.c                    |d         dk    r|d         S |                      |d                   }|d         }t          |          dk    r8|                    d| j        j        j                                      d          S |j        d         |                    d| j        j        j                  }fd	|D             }|                    |          S )
Nr2   rF   rJ   r7   r9   r(   ru  r   c                     g | ]}|z  S rA   rA   )r   r9   r   s     rC   r   z<SkyworkR1VChatModel._process_image_input.<locals>.<listcomp>Y  s+     
 
 
+6K,&
 
 
rB   )	rx  r   rn  r   rO  re  	unsqueezer   split)r   r  rF   r9   image_feature_sizesr   s        @rC   _process_image_inputz(SkyworkR1VChatModel._process_image_inputD  s     v.00v&&++K8K,LMM!-0 {q  $$R)@)LMMWW   $)!,#((T[-D-PQQ
 
 
 
:E
 
 
 !!"5666rB   	input_idsc                 n    | j         r&|| j        k                        dd          | _        d S d | _        d S )Nru  r(   )rF  rU  rv  rV  )r   r  s     rC   _set_visual_token_maskz*SkyworkR1VChatModel._set_visual_token_mask^  sG    < 	*&/43L&L%U%UA& &D""" &*D"""rB   c                 N     | j         di |}|g S |                     |          S )NrA   )r  r  )r   r   r  s      rC   embed_multimodalz$SkyworkR1VChatModel.embed_multimodalf  s9    :d:DDVDDI((555rB   F)is_multimodalhandle_oov_mm_tokenmultimodal_embeddingsr  r  c                    |(t          |          dk    r|                     |           ||!t                                          |          S t                                          ||||          S )Nr   )r  r  r  )r   r  r   embed_input_ids)r   r  r  r  r  r   s        rC   r  z#SkyworkR1VChatModel.embed_input_idsm  s     !,5J1K1Ka1O1O''	222 !(M,A77**9555ww&&"7' 3	 ' 
 
 	
rB   	positionsintermediate_tensorsinputs_embedsc                     |d }||||d}| j         #|                    d| j         i           d | _          | j        j        di |}|S )N)r  r  r  r  rV  rA   )rV  updaterH  model)r   r  r  r  r  r   forward_kwargshidden_statess           rC   forwardzSkyworkR1VChatModel.forward  sz      + M #"$8*	
 
 !-!!#68N"OPPP%)D"1+1CCNCCrB   r  c                 6    | j                             |          S r   )rH  compute_logits)r   r  s     rC   r  z"SkyworkR1VChatModel.compute_logits  s     "11-@@@rB   weightsc                 V    g d}t          | |          }|                    |          S )N)action_embedtemporal_embedtrack_embedtrack_embed_decoder	box_tokencg_criterioncg_modelloc_encoderloc_decodersamtemporal_tokentrack_token)skip_prefixes)r,   load_weights)r   r  r  loaders       rC   r  z SkyworkR1VChatModel.load_weights  s;    
 
 
 #4}EEE""7+++rB   )r>  )rk   r   )NN)&r:   r;   r<   classmethodr   r   r=  r   r   r   r   rL  r   rR  rf  ModulerS  rs  r?   r@   rx  r  rL   r  rK   r   r  r  r)   r  r  r$   r  r  r   setr  r   r   s   @rC   r9  r9    s        =3 =3 =3: = = = [= BD -
 -
 -
z -
3 -
 -
 -
 -
 -
 -
 -
^K&K6HK K K K@ @ )4/@
 @ @ @ @ @< 	
 
 
 )
 	

 

 
 
 
8   $	EL 	U\ 	 	 	 	"A"A		%"A "A "A "AH7*7 
U\*	*U5<3D-E	E7 7 7 74* * * * * *6 64H 6 6 6 6 >B

 .2$)
 
 
<
  4d:

 |d*
 "
 

 
 
 
 
 
4 <@-1 < < 2D8	
 |d*  
   4A|A 
	A A A A,HU33D-E$F ,3s8 , , , , , , , ,rB   r9  )acollections.abcr   r   r   typingr   r   r   r?   torch.nnrf  torchvision.transforms
transformsrX   PILr	   transformersr
   r   r   vllm.configr   vllm.config.multimodalr   !vllm.model_executor.layers.linearr   'vllm.model_executor.layers.quantizationr   +vllm.model_executor.layers.quantization.awqr   %vllm.model_executor.models.intern_vitr   r   vllm.multimodalr   vllm.multimodal.imager   vllm.multimodal.inputsr   r   r   vllm.multimodal.parser   r   r   r   vllm.multimodal.processingr   r   r    r!   r"   r#   vllm.sequencer$   vllm.tokenizersr%   vllm.utils.tensor_schemar&   r'   
interfacesr)   r*   r+   utilsr,   r-   r.   r   r   r   rV   rW   r0   rE   rL   r>   r   rb   rl   rK   r   rt   r   r{   r   r   r   r@   r   r   r   r  r  register_processorr  r9  rA   rB   rC   <module>r     si   8 7 7 7 7 7 7 7 7 7 7 0 0 0 0 0 0 0 0 0 0        " " " " " "       C C C C C C C C C C " " " " " " 3 3 3 3 3 3 > > > > > > F F F F F F A A A A A A        0 / / / / / 4 4 4 4 4 4         
                           . - - - - - ) ) ) ) ) ) > > > > > > > > L L L L L L L L L L N N N N N N N N N N	
%$       .    \   $ !?? y       c3h( 	
   38_   .00 0 	0
 0 38_0 0 0 0 <<< 
%S/< < < <// / c3h(	/
 / / 3S=/ / / />&;& c3h(& 	&
 & 
%+& & & &T;  	
   \   ,EJ EJ EJ EJ EJ EJ EJ EJP1( 1( 1( 1( 1(1 1( 1( 1(h
 
 
 
 
#9:R#S 
 
 
6Y
 Y
 Y
 Y
 Y
$;<T$U Y
 Y
 Y
x ('!	!-  
j, j, j, j, j,")%7 j, j, 
j, j, j,rB   