
    .`iV                        U d dl mZmZ d dlmZmZmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/ d dl0m1Z1m2Z2m3Z3m4Z4 d dl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZB ddlCmDZDmEZEmFZFmGZG ddlHmIZImJZJmKZK dZLdZMdZNdZOdZP G d deA          ZQ G d d eA          ZReQeRz  ZSeeTd!<    G d" d#eA          ZU G d$ d%eA          ZVeUeVz  ZWeeTd&<   d'eXfd(ZYd)eZd*e[e\eXeXf                  d+eXd,eXd-eXd.e\eXeXf         fd/Z]d0eXd1eXd2e^d3e^d.e\eXeXf         f
d4Z_d5eXd6eXd.e[e\eXeXf                  fd7Z`d8eXd9eXd*e[e\eXeXf                  d-eXd3e^d.e\eXeXeXf         fd:Zad;ej        d*e[e\eXeXf                  d-eXd3e^d.e[ej                 f
d<Zbd;ej        d'eXd5eXd6eXd3e^d.ejc        fd=Zdd>eje        d'eXd5eXd6eXd3e^d.ejc        fd?Zf G d@ dAe          Zg G dB dCeg          Zh G dD dEe8          Zi edFeiG          Zj G dH dIe6ej                   Zk G dJ dKe7ej                   Zl G dL dMei          Zm G dN dOekem                   Zn G dP dQelem                   Zo e)jp        eoemenR           G dS dTejq        eFeGeE                      ZrdS )U    )ABCabstractmethod)IterableMappingSequence)	AnnotatedAnyLiteral	TypeAliasTypeVarN)Image)BatchFeaturePretrainedConfig
TensorType)
VllmConfig)BaseDummyOptions)QuantizationConfig)	AWQConfig)InternVisionModelInternVisionPatchModel)MultiModelKeys)MULTIMODAL_REGISTRYconvert_image_mode)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems)ImageEmbeddingItemsImageProcessorItems	ImageSizeMultiModalDataItems)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacementPromptUpdatePromptUpdateDetails)IntermediateTensors)TokenizerLike)TensorSchemaTensorShape   )MultiModalEmbeddingsSupportsLoRASupportsMultiModal
SupportsPP)AutoWeightsLoaderinit_vllm_registered_modelmaybe_prefixz<img>z</img>z<IMG_CONTEXT>)g
ףp=
?gv/?gCl?)gZd;O?gy&1?g?c                       e Zd ZU dZed         ed<   eej         e	dddd          f         ed<   eej         e	d	          f         ed
<   dS )InternVLImagePixelInputsz
    Dimensions:
        - bn: Batch size * number of images
        - bnp: Batch size * number of images * (1 + num_patches)
        - c: Number of channels (3)
        - h: Height of each image patch
        - w: Width of each image patch
    pixel_valuestypebnp   hwpixel_values_flatbnnum_patchesN
__name__
__module____qualname____doc__r
   __annotations__r   torchTensorr+        w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/internvl.pyr5   r5   G   sy           .
!!!! {{5!S#/N/N!NOOOO5<T):)::;;;;;;rH   r5   c                       e Zd ZU dZed         ed<   eej        e	ej                 z   e
ddd          f         ed<   dS )	InternVLImageEmbeddingInputsz
    Dimensions:
        - n: Number of images
        - f: Total image feature size
        - h: Hidden size (must match the hidden size of language model backbone)
    image_embedsr7   nfr:   dataNr@   rA   rB   rC   r
   rD   r   rE   rF   listr+   rG   rH   rI   rK   rK   V   a           .
!!!!
EL4#55{{3S7Q7QQ
RRRRRRrH   rK   InternVLImageInputsc                       e Zd ZU dZed         ed<   eej         e	dddd          f         ed<   eej         e	d	          f         ed
<   dS )InternVLVideoPixelInputsz
    Dimensions:
        - bvf: Batch size * number of videos * num_frames
        - bn: Batch size * number of images
        - c: Number of channels (3)
        - h: Height of each video frame
        - w: Width of each video frame
    pixel_values_videosr7   bvfr9   r:   r;   r<   r=   r>   Nr?   rG   rH   rI   rU   rU   e   sz           '
(((( {{5!S#/N/N!NOOOO5<T):)::;;;;;;rH   rU   c                       e Zd ZU dZed         ed<   eej        e	ej                 z   e
ddd          f         ed<   dS )	InternVLVideoEmbeddingInputsz
    Dimensions:
        - n: Number of videos
        - f: Total video feature size
        - h: Hidden size (must match the hidden size of language model backbone)
    video_embedsr7   rM   rN   r:   rO   NrP   rG   rH   rI   rY   rY   t   rR   rH   rY   InternVLVideoInputs
input_sizec           	         t           t          }}t          j        t          j        d           t          j        | | ft          j        j                  t          j                    t          j	        ||          g          }|S )Nc                 "    t          | d          S )NRGBr   )imgs    rI   <lambda>z!build_transform.<locals>.<lambda>   s    !3C!?!? rH   )interpolation)meanstd)
IMAGENET_MEANIMAGENET_STDTComposeLambdaResizeInterpolationModeBICUBICToTensor	Normalize)r\   MEANSTD	transforms       rI   build_transformrr      s{    |#D	H??@@HZ(8K8S   JLLKTs+++	
	 	I rH   aspect_ratiotarget_ratioswidthheight
image_sizereturnc                    t          d          }d}||z  }|D ]V}|d         |d         z  }	t          | |	z
            }
|
|k     r|
}|}0|
|k    r |d|z  |z  |d         z  |d         z  k    r|}W|S )Ninf)r,   r,   r   r,         ?)floatabs)rs   rt   ru   rv   rw   best_ratio_diff
best_ratioarearatiotarget_aspect_ratio
ratio_diffs              rI   find_closest_aspect_ratior      s     EllOJ6>D # ##Ahq1(;;<<
''(OJJ?**cJ&3eAh>qIII"
rH   min_dynamic_patchmax_dynamic_patchdynamic_image_sizeuse_thumbnailc                 <    |r| nd} |r|nd}|r|dk    r|dz  }| |fS )Nr,   rG   r   r   r   r   s       rI   resolve_internvl_min_max_numr      sQ     .@F))Q-?F))Q *a//Q///rH   min_nummax_numc                 j      fdt           dz             D             }t          |d           S )Nc                     h | ]E}t          d |d z             D ]/}t          d |d z             D ]}||z  cxk    rk    n n||f0FS )r,   )range).0rM   ijr   r   s       rI   	<setcomp>z-get_internvl_target_ratios.<locals>.<setcomp>   s       q!a%  q!a%	  a!e&&&&w&&&&&	 
A '&&&&rH   r,   c                 $    | d         | d         z  S Nr   r,   rG   )xs    rI   ra   z,get_internvl_target_ratios.<locals>.<lambda>   s    qtad{ rH   )key)r   sorted)r   r   rt   s   `` rI   get_internvl_target_ratiosr      sX        w!,,  M -%:%:;;;;rH   
orig_widthorig_heightc                     | |z  }t          ||| ||          }||d         z  }||d         z  }|d         |d         z  }	|r|	dk    r|	dz  }	|	||fS )N)ru   rv   rw   r   r,   )r   )
r   r   rt   rw   r   rs   r   target_widthtarget_heightblockss
             rI   calculate_internvl_targetsr      s     +L 4    3A 66L!4Q!77M #&9!&<<F  1!<..rH   imagec                    | j         \  }}t          ||||d          \  }}}|                     ||f          }	g }
t          |          D ]\}|||z  z  |z  |||z  z  |z  |||z  z  dz   |z  |||z  z  dz   |z  f}|	                    |          }|
                    |           ]t          |
          |k    sJ |r?t          |
          dk    r,|                     ||f          }|
                    |           |
S )NF)r   r   rt   rw   r   r,   )sizer   resizer   cropappendlen)r   rt   rw   r   r   r   r   r   r   resized_imgprocessed_imagesr   box	split_imgthumbnail_imgs                  rI   dynamic_preprocess_internvlr      sZ    $jJ +E#+ + +'FL- ,,m<==K6]] 	+ 	+,*,-;<:-.*<<:-.!3zALJ./14
B	
  $$S))		****  F**** /-..!33j*%=>>...rH   c                    t          ||          }t          |          t          | |||          }t          j        fd|D                       }|S )Nr\   rt   rw   r   c                 &    g | ]} |          S rG   rG   r   r   rq   s     rI   
<listcomp>z2image_to_pixel_values_internvl.<locals>.<listcomp>%  s#    EEEU		% 0 0EEErH   )r   rr   r   rE   stack)	r   r\   r   r   r   rt   imagesr6   rq   s	           @rI   image_to_pixel_values_internvlr     sp     /w@@M:666I(##	  F ;EEEEfEEEFFLrH   videoc                p  
 t          ||          }t          |          
t          t          j                             }| D ]S}t	          t          j        |d          |||          }t          |          dk    sJ |                    |           Tt          j	        
fd|D                       }	|	S )Nr   r_   )moder   r,   c                 &    g | ]} |          S rG   rG   r   s     rI   r   z2video_to_pixel_values_internvl.<locals>.<listcomp>@  s#    JJJU		% 0 0JJJrH   )
r   rr   rQ   r   r   	fromarrayr   extendrE   r   )r   r\   r   r   r   rt   frames_listframe	pil_framer6   rq   s             @rI   video_to_pixel_values_internvlr   *  s     /w@@M:666Iu{#%%K & &/OE...'!'	
 
 
	 9~~""""9%%%%;JJJJkJJJKKLrH   c                   (    e Zd ZdZdddddedededz  dedz  dedz  d	df fd
Ze	e
d	efd                        Ze
dededz  d	ee         fd            Zddddddedz  dedz  dedz  dedz  d	eeef         f
dZddddddedz  dedz  dedz  dedz  d	eeeef                  f
dZdeded	efdZ	 	 	 ddeej                 dedz  dedz  dedz  d	eej                 f
dZ	 	 	 ddee         deej                 dedz  dedz  dedz  d	eee         eeej        f         f         fdZddeee         z  dz  fdZ	 	 	 	 	 	 d deee         z  dz  dej        eej                 z  dz  dedz  dedz  dedz  deez  dz  d	efdZ xZS )!BaseInternVLProcessorz
    This model doesn't define its own HF processor,
    so we implement our own one here.

    The code to insert image tokens is based on:
    https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
    Nr   r   r   config	tokenizerr   r   r   rx   c                   t                                                       || _        || _        |j        j        }|j        j        }||j        }t          |t                    sJ ||j
        }t          |t                    sJ ||j        }t          |t                    sJ t          ||z  dz  |j        dz  z            | _        || _        || _        || _
        || _        |j        | _        d S )N   )super__init__r   r   vision_configrw   
patch_sizer   
isinstanceintr   r   booldownsample_rationum_image_tokenr   )	selfr   r   r   r   r   rw   r   	__class__s	           rI   r   zBaseInternVLProcessor.__init__M  s    	" .9
 .9
$ & 8+S11111$ & 8+S11111%!'!:,d33333":%!+v/F/IJ 
  
 %!2!2"4#)#7rH   c                     t           NNotImplementedErrorr   s    rI   image_token_idz$BaseInternVLProcessor.image_token_ids  s
     "!rH   feature_sizer>   c                     t           r   r   )r   r   r>   s      rI   get_image_replz$BaseInternVLProcessor.get_image_replx  s
     "!rH   r   r   c                    || j         n|}|| j        n|}|| j        n|}|| j        n|}t	          ||||          S Nr   )r   r   r   r   r   )r   r   r   r   r   s        rI   resolve_min_max_numz)BaseInternVLProcessor.resolve_min_max_num  s     '8&?D""EV 	 '8&?D""EV 	
 ") ### 	
 /<.C**+//1'	
 
 
 	
rH   c                Z    |                      ||||          \  }}t          ||          S r   )r   r   )r   r   r   r   r   r   r   s          rI   resolve_target_ratiosz+BaseInternVLProcessor.resolve_target_ratios  sA      33//1'	 4 
 
 *'7;;;rH   image_widthimage_heightc                    |                      d          }t          ||| j        || j                  \  }}}|| j        z  S )NF)r   )r   r   rw   rt   r   )r   r   rw   r   r   )r   r   r   rt   r>   _s         rI   get_num_image_tokensz*BaseInternVLProcessor.get_num_image_tokens  sb     22 3 
 
 7"$',
 
 
Q T111rH   r   c                 `                           |||d          \   fd|D             S )NFr   c           	      L    g | ] }t          |j        j                   !S )r\   r   r   r   )r   rw   r   )r   r   r   r   r   s     rI   r   zEBaseInternVLProcessor._images_to_pixel_values_lst.<locals>.<listcomp>  sN     	
 	
 	
  +?"0  	
 	
 	
rH   r   )r   r   r   r   r   r   r   s   `    @@rI   _images_to_pixel_values_lstz1BaseInternVLProcessor._images_to_pixel_values_lst  si      33//1	 4 
 
	
 	
 	
 	
 	
 	
  	
 	
 	
 		
rH   textc                 P   t          |          dk    ri }n|                     ||||          }t          j        |          t          j        d |D                       d}|D ]=}|j        d         }	|	| j        z  }
|                     |
|	          fd|D             }>||fS )Nr   r   c                 ,    g | ]}t          |          S rG   r   r   items     rI   r   z;BaseInternVLProcessor._preprocess_image.<locals>.<listcomp>  s    <<<4SYY<<<rH   )r<   image_num_patchesc                 H    g | ]}|                     d j        d          S )<image>r,   replacefull)r   t
image_repls     rI   r   z;BaseInternVLProcessor._preprocess_image.<locals>.<listcomp>  +    OOOQ		)Z_a@@OOOrH   )r   r   rE   cattensorshaper   r   )r   r   r   r   r   r   image_inputspixel_values_lstr6   r>   r   r   s              @rI   _preprocess_imagez'BaseInternVLProcessor._preprocess_image  s     v;;!LL#??"3"3#5	  @     &+Y/?%@%@%*\<<+;<<<& & L !1 P P*03*T-AA!00{KK
OOOO$OOO\!!rH   
input_itemc                 >    |g }t          |t                    s|g}|S r   )r   rQ   )r   r  s     rI   _make_batch_inputz'BaseInternVLProcessor._make_batch_input  s,    J*d++ 	&$JrH   return_tensorsc                       fd||fD             \  }}                      |||||          \  }}                     |          }i ||}	t          |	|          S )Nc                 :    g | ]}                     |          S rG   r  r   r   r   s     rI   r   z2BaseInternVLProcessor.__call__.<locals>.<listcomp>
  s'    JJJa..q11JJJrH   r   r   r   r   r   tensor_type)r   r   r   )
r   r   r   r   r   r   r  r   text_inputscombined_outputss
   `         rI   __call__zBaseInternVLProcessor.__call__  s     KJJJD&>JJJf!33//1 4 
 
l nnT**:k:\:,.IIIIrH   )NNNr   )NNNNNN) r@   rA   rB   rC   r   r)   r   r   r   propertyr   r   r'   strr   tupler   rQ   r   r   r   rE   rF   r   dictr   r	   r  r   r   r  __classcell__r   s   @rI   r   r   D  s         )-(,*.$8 $8 $8 $8 !$8
 :$8 :$8 !4K$8 
$8 $8 $8 $8 $8 $8L " " " " ^ X" "" 4Z" 
S	!	" " " ^" )-(,*.%)
 
 
 :
 :	

 !4K
 d{
 
sCx
 
 
 
> )-(,*.%)< < < :< :	<
 !4K< d{< 
eCHo	< < < <"2 2 	2
 
2 2 2 2. )-(,*.
 
U[!
 :
 :	

 !4K
 
el	
 
 
 
: )-(,*." "3i" U[!" :	"
 :" !4K" 
tCy$sEL011	2" " " "@ C$s)Od,B     (,9=(,(,*.26J JDIo$J d5;//$6J :	J
 :J !4KJ j(4/J 
J J J J J J J JrH   r   c                   d    e Zd ZdZddddddedededz  dedz  dedz  d	edz  d
df fdZ	e
d
efd            Ze
d
edz  fd            Ze
d
efd            Z	 ddeej                 dedz  d
eej                 fdZ	 ddee         deej                 dedz  fdZ	 	 	 	 	 	 	 ddeee         z  dz  dej        eej                 z  dz  dej        eej                 z  dz  dedz  dedz  dedz  deez  dz  d
efdZdededz  d
ee         fdZdefdededz  ded
ee         fdZ xZS )InternVLProcessorz
    HF Processor for InternVLChatModel with extended video processing logic.

    Code for video processing is adapted from video example:
    https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
    N)r   r   r   video_tokenr   r   r   r   r   r  rx   c                b    t                                          |||||           || _        d S )N)r   r   r   r   r   )r   r   r  )r   r   r   r   r   r   r  r   s          rI   r   zInternVLProcessor.__init__#  sE     	//1 	 	
 	
 	
 'rH   c                 J    | j                                         t                   S r   )r   	get_vocabIMG_CONTEXTr   s    rI   r   z InternVLProcessor.image_token_id7  s    ~''))+66rH   c                 x    | j         d S | j                                                            | j         d           S r   )r  r   r  getr   s    rI   video_token_idz InternVLProcessor.video_token_id;  s7    #4~''))--d.>EEErH   c                     | j         d uS r   )r  r   s    rI   supports_videoz InternVLProcessor.supports_videoA  s    "$..rH   videosc                 `                           dd|d          \   fd|D             S )Nr,   Fr   c           	      B    g | ]}t          |j        d           S )Fr   )r   rw   )r   r   r   r   r   s     rI   r   zAInternVLProcessor._videos_to_pixel_values_lst.<locals>.<listcomp>Q  sK     	
 	
 	
  +?#  	
 	
 	
rH   r   )r   r!  r   r   r   s   `  @@rI   _videos_to_pixel_values_lstz-InternVLProcessor._videos_to_pixel_values_lstE  si    
  331	 4 
 
	
 	
 	
 	
 	
 	
  	
 	
 	
 		
rH   r   c                 \   t          |          dk    s| j        si }n|                     ||          }t          j        |          t          j        d |D                       d}|D ]>}|j        d         }|                     | j        || j	                  fd|D             }?||fS )Nr   )r   c                 ,    g | ]}t          |          S rG   r   r   s     rI   r   z7InternVLProcessor._preprocess_video.<locals>.<listcomp>l  s    BBB4SYYBBBrH   )pixel_values_flat_videovideo_num_patchesc                 H    g | ]}|                     d j        d          S )<video>r,   r   )r   r   
video_repls     rI   r   z7InternVLProcessor._preprocess_video.<locals>.<listcomp>v  r   rH   )
r   r   r$  rE   r   r   r   get_video_replr   r  )	r   r   r!  r   video_inputspixel_values_lst_videor6   r>   r+  s	           @rI   _preprocess_videoz#InternVLProcessor._preprocess_video\  s     v;;!4#6LL%)%E%E#5 &F & &"
 ,195K+L+L%*\BB+ABBB& & L !7 P P*03!00(+t7G 
 POOO$OOO\!!rH   r   r  c                       fd|||fD             \  }}}                      |||||          \  }}                     |||          \  }}	                     |          }
i |
||	}t          ||          S )Nc                 :    g | ]}                     |          S rG   r  r  s     rI   r   z.InternVLProcessor.__call__.<locals>.<listcomp>  s4      
  
  
*+D""1%% 
  
  
rH   r	  )r   r!  r   r
  )r   r/  r   r   )r   r   r   r!  r   r   r   r  r   r-  r  r  s   `           rI   r  zInternVLProcessor.__call__y  s     
  
  
  
04ff/E 
  
  
ff "33//1 4 
 
l "331 4 
 
l nnT**JkJ\J\J,.IIIIrH   r   r>   c                 n    t           |z  }t          |z   t          z   }t          j        |t                     S r   )r  	IMG_STARTIMG_ENDr'   select_text)r   r   r>   repl_features	repl_fulls        rI   r   z InternVLProcessor.get_image_repl  s0    
 $l2-7	".y+FFFrH   video_context_tokenc                     || j         z  }t          |z   t          z   d                    fdt	          |          D                       }t          j        ||          S )N c                 &    g | ]}d |dz    d S )Framer,   z: rG   )r   r   repl_features_with_seps     rI   r   z4InternVLProcessor.get_video_repl.<locals>.<listcomp>  s/    SSS16QU66466SSSrH   )r   r3  r4  joinr   r'   r5  )r   r   r>   r8  r6  r7  r=  s         @rI   r,  z InternVLProcessor.get_video_repl  sj     ,d.BB!*]!:W!DGGSSSSk@R@RSSS
 
	 #.y:MNNNrH   r   )NNNNNNN)r@   rA   rB   rC   r   r)   r   r   r  r   r  r   r  r   rQ   nptNDArrayrE   rF   r$  r/  r   r   r   r  r'   r   r  r,  r  r  s   @rI   r  r    s8         )-(,*."&' ' ' ' !'
 :' :' !4K' 4Z' 
' ' ' ' ' '( 7 7 7 7 X7 Fd
 F F F XF
 / / / / X/ +/
 
S[!
 !4K
 
el		
 
 
 
6 +/	" "3i" S[!" !4K	" " " "> (,9=9=(,(,*.26 J  JDIo$ J d5;//$6 J d3;//$6	 J
 : J : J !4K J j(4/ J 
 J  J  J  JDGG 4ZG 
S	!	G G G G #'#.	O OO 4ZO !	O
 
S	!O O O O O O O OrH   r  c                       e Zd ZdZededefd            Zdee	e
dz  f         fdZde
de
d	edz  de
fd
ZdefdZde
fdZdS )BaseInternVLProcessingInfoz:Basic image-only ProcessingInfo for InternVL-style models.kwargsrx   c                     t           r   r   r   rC  s     rI   get_hf_processorz+BaseInternVLProcessingInfo.get_hf_processor  s    !!rH   Nc                 
    dd iS )Nr   rG   r   s    rI   get_supported_mm_limitsz2BaseInternVLProcessingInfo.get_supported_mm_limits  s    rH   r   r   	processorc                \    ||                                  }|                    ||          S )N)r   r   )rF  r   )r   r   r   rI  s       rI   r   z/BaseInternVLProcessingInfo.get_num_image_tokens  s>     --//I--#% . 
 
 	
rH   c                 "   |                                  }|j        }|                                }d\  }}|D ]@\  }}||z  ||z  }	}|                     ||	|          }
|
|k    r|
}t	          ||	          }A|dk    s|t          d          |S )N)r   Nr   r   rI  )ru   rv   r   z(Cannot have a largest feature size of 0!)rF  rw   r   r   r    
ValueError)r   rI  	base_sizert   largest_feature_sizelargest_feature_pinpointwrhrru   rv   	feat_sizes              rI   !get_image_size_with_most_featuresz<BaseInternVLProcessingInfo.get_image_size_with_most_features  s    ))++	(	!77999@66# 
	Q 
	QFB%NIN6E11!## 2  I
 ///'0$+45+P+P+P(1$$(@(HGHHH''rH   c                     |                                  }|                                 \  }}|                     |||          S )NrL  )rF  rT  r   )r   rI  r   r   s       rI   get_max_image_tokensz/BaseInternVLProcessingInfo.get_max_image_tokens  sO    ))++	&*&L&L&N&N#m(($& ) 
 
 	
rH   )r@   rA   rB   rC   r   objectr   rF  r   r  r   rH  r   r    rT  rV  rG   rH   rI   rB  rB    s        DD" "4I " " " ^"cDj)A    
 
 	

 )4/
 

 
 
 
(9 ( ( ( (0
c 
 
 
 
 
 
rH   rB  _I)boundc            	       t    e Zd ZdZdeeef         defdZ	 d	dedeeef         deeef         dz  de	fdZ
dS )
BaseInternVLDummyInputsBuilderz>Basic image-only DummyInputsBuilder for InternVL-style models.	mm_countsrx   c                 8    |                     dd          }d|z  S )Nr   r   r   )r  )r   r\  
num_imagess      rI   get_dummy_textz-BaseInternVLDummyInputsBuilder.get_dummy_text  s     ]]7A..
:%%rH   Nseq_len
mm_optionsc                     | j                                         \  }}|                    dd          }|r|                    d          nd }d|                     ||||          iS )Nr   r   )ru   rv   r^  	overrides)inforT  r  _get_dummy_images)r   r`  r\  ra  r   r   r^  image_overridess           rI   get_dummy_mm_dataz0BaseInternVLDummyInputsBuilder.get_dummy_mm_data  s|     '+i&Q&Q&S&S#m]]7A..
5?I*..111T T++"$%)	 ,  
 	
rH   r   )r@   rA   rB   rC   r   r  r   r_  r   r   rg  rG   rH   rI   r[  r[    s        HH&S(9 &c & & & & =A	
 

 38$
 C!112T9	

 

 
 
 
 
 
rH   r[  c            
            e Zd ZdZdedeeef         deeef         deeef         def
 fdZded	eeef         deee	f         fd
Z
ded	eeef         dedee         fdZ xZS )BaseInternVLMultiModalProcessorz?Basic image-only MultiModalProcessor for InternVL-style models.promptmm_data	mm_kwargs
tok_kwargsrx   c                     t                                          ||||          } | j        j        di |}|j        }t          j        |          |d<   |S )N)rj  rk  rl  rm  r   rG   )r   _call_hf_processorrd  rF  r   rE   r   )	r   rj  rk  rl  rm  processed_outputshf_processorr   r   s	           rI   ro  z2BaseInternVLMultiModalProcessor._call_hf_processor  sr     "GG66!	 7 
 
 2ty1>>I>>%4
 /4l>.J.J*+  rH   	hf_inputshf_processor_mm_kwargsc           	      *   |                     dt          j        d                    }t          |          }t	          t          j        d|          t          j        d          t          j        d          t          j        d|                    S )Nr   r   r   )r<   r   rL   r   )	r  rE   emptyr   r  r   flat_from_sizesbatchedshared)r   rr  rs  r   r^  s        rI   _get_mm_fields_configz5BaseInternVLMultiModalProcessor._get_mm_fields_config,  s    
 &MM*=u{1~~NN*++
3C*  4;GDD.6w??07LL
 
 
 	
rH   mm_itemsout_mm_kwargsc                 V      j         j        di ||                                }d|v r9|d         t          t          j                  sJ                                 n d|v rd gt          |d                   z  ng dt          f fd}t          dd|          gS )	Nr   rL   item_idxc                                         dt          t          f          }t          |t                    r|                    |           }n<|                    |           }j                            |j        |j	                  }|          }|t          |t                    sJ                     ||          S )Nr   rL  )	get_itemsr   r   r   get_feature_sizeget_image_sizerd  r   ru   rv   r   r   )	r}  r   r   rw   r>   rq  r   rz  r   s	        rI   get_replacement_internvlzUBaseInternVLMultiModalProcessor._get_prompt_updates.<locals>.get_replacement_internvlQ  s    ''-/BC F &"566 %66x@@#228<<
#y== * 0!+!2*  >     ,H5K&!+s33333..|[IIIrH   r   r   modalitytargetreplacementrG   )
rd  rF  get_datar   rE   rF   tolistr   r   r%   )r   rz  rs  r{  out_mm_datar  rq  r   s   ``    @@rI   _get_prompt_updatesz3BaseInternVLMultiModalProcessor._get_prompt_updates=  s    2ty1KK4JKK#,,..+-- +,? @/>>>>> 1 8 8 : :{** "&[-H)I)I I "	Js 	J 	J 	J 	J 	J 	J 	J 	J 	J,   4  
 	
rH   r@   rA   rB   rC   r  r   rW  r   ro  r   ry  r!   r   r   r&   r  r  r  s   @rI   ri  ri    s       II!! f%! 3;'	!
 CK(! 
! ! ! ! ! !0

 !(V 4
 
++	,	
 
 
 
"/
%/
 !(V 4/
 -	/

 
,	/
 /
 /
 /
 /
 /
 /
 /
rH   ri  c                   ~     e Zd ZdZed             Z fdZdedz  fdZde	de
ee	f         de	fd	Zd
edefdZ xZS )InternVLProcessingInfoz5InternVL ProcessingInfo extended for video processingc                 4    |                                  j        S r   )rF  r   r   s    rI   r   z%InternVLProcessingInfo.supports_videor  s    $$&&55rH   c                 f    | j         rdd ini }i t                                                      |S )Nr   )r   r   rH  )r   video_limitr   s     rI   rH  z.InternVLProcessingInfo.get_supported_mm_limitsv  s8    )-)<Dwoo"C%''1133C{CCrH   rx   Nc                     |                                                                  j        }ddddd}|                    |          S )Nz<|video_pad|>z<|reserved_200000|>)qwen2qwen3	qwen3_moegpt_oss)get_hf_configget_text_config
model_typer  )r   text_model_typevideo_token_maps      rI   get_video_tokenz&InternVLProcessingInfo.get_video_tokenz  sP    ,,..>>@@K$$(,	
 
 ""?333rH   r`  r\  c                    |                     dd          }|                     dd          }|                                 }|                                 |z  }||z
  |j        z  }|t	          |d          z  }t	          |d          S )Nr   r   r   r,   )r  rF  rV  r   max)	r   r`  r\  
max_images
max_videosrI  max_image_tokensmax_total_framesmax_frames_per_videos	            rI   !get_num_frames_with_most_featuresz8InternVLProcessingInfo.get_num_frames_with_most_features  s    
 ]]7A..
]]7A..
))++	4466C#&669;TT/3z13E3EE'+++rH   rC  c                      | j         j        t          f|                                 |                                 |                                 d|S )N)r   r   r  )ctxinit_processorr  r  get_tokenizerr  rE  s     rI   rF  z'InternVLProcessingInfo.get_hf_processor  s_    &tx&
%%''((**,,..	
 

 
 
 	
rH   )r@   rA   rB   rC   r  r   rH  r  r  r   r   r  rW  r  rF  r  r  s   @rI   r  r  o  s        ??6 6 X6D D D D D4t 4 4 4 4,, 38$, 
	, , , , 
 
4E 
 
 
 
 
 
 
 
rH   r  c            	            e Zd ZdZdeeef         def fdZ	 d	dedeeef         deeef         dz  de	f fdZ
 xZS )
InternVLDummyInputsBuilderz6InternVL DummyInputsBuilder extended for video supportr\  rx   c                 ~    |                     dd          }t                                          |          d|z  z   S )Nr   r   r*  )r  r   r_  )r   r\  
num_videosr   s      rI   r_  z)InternVLDummyInputsBuilder.get_dummy_text  s8    ]]7A..
ww%%i009z3IIIrH   Nr`  ra  c                    t                                          |||          }| j        j        r| j                                        }|j        j        }| j                            ||          }|                    dd          }|r|                    d          nd }	d| 	                    |||||	          i}
ni }
i ||
S )N)r`  r\  ra  r   r   )ru   rv   
num_framesr  rc  )
r   rg  rd  r   r  r   rw   r  r  _get_dummy_videos)r   r`  r\  ra  dummy_imager   rw   target_num_framesr  video_overridesdummy_videor   s              rI   rg  z,InternVLDummyInputsBuilder.get_dummy_mm_data  s     gg//yZ 0 
 
 9# 	Y,,..F$2=J $	 K K! ! #w22J9CMjnnW555O//$%0)- 0  KK K-+---rH   r   )r@   rA   rB   rC   r   r  r   r_  r   r   rg  r  r  s   @rI   r  r    s         A@JS(9 Jc J J J J J J =A	. .. 38$. C!112T9	.
 
. . . . . . . . . .rH   r  c            
            e Zd ZdZdedeeef         deeef         deeef         def
 fdZded	eeef         deee	f         f fd
Z
ded	eeef         dedee         f fdZ xZS )InternVLMultiModalProcessorz7InternVL MultiModalProcessor extended for video supportrj  rk  rl  rm  rx   c                     t                                          ||||          } | j        j        di |}| j        j        r |j        x}t          j        |          |d<   |S )Nr  rG   )r   ro  rd  rF  r   r  rE   r   )	r   rj  rk  rl  rm  rp  rq  r  r   s	           rI   ro  z.InternVLMultiModalProcessor._call_hf_processor  s|     "GG66GY

 
 2ty1>>I>>I$	O#/#>>K27,~2N2N./  rH   rr  rs  c                 r   t                                          ||          }| j        j        r|                    dt          j        d                    }t          |          }t          t          j
        d|          t          j        d          t          j        d|                    }ni }||z  S )Nr(  r   r   )r'  r(  r  )r   ry  rd  r   r  rE   ru  r   r  r   rv  rw  rx  )r   rr  rs  image_fieldsr(  r  video_fieldsr   s          rI   ry  z1InternVLMultiModalProcessor._get_mm_fields_config  s    
 ww44Y@VWW9# 	 ).A5;q>> R R.//J(=(M.) ) #8"?"H"H4;GZPP  LL Ll**rH   rz  r{  c                 ~   t                                          |||          } | j        j        di ||                                }d|v r9|d         t          t          j                  sJ                                 ng dt          ffd}| j        j
        rg |t          dd|          }|S )	N)rz  rs  r{  r(  r}  c                     j         }|          }|t          |t                    sJ                     ||j                  S )N)r8  )r   r   r   r,  r  )r}  r   r>   rq  r(  s      rI   get_video_replacement_internvlzWInternVLMultiModalProcessor._get_prompt_updates.<locals>.get_video_replacement_internvl
  sX    '7L+H5K&!+s33333..k|?W /   rH   r   r*  r  rG   )r   r  rd  rF  r  r   rE   rF   r  r   r   r%   )
r   rz  rs  r{  prompt_replr  r  rq  r(  r   s
          @@rI   r  z/InternVLMultiModalProcessor._get_prompt_updates  s    gg11#9' 2 
 
 2ty1KK4JKK#,,..+-- +,? @/>>>>> 1 8 8 : : "	S 	 	 	 	 	 	 	 9# 	!$$ >  K rH   r  r  s   @rI   r  r    s1        BA!! f%! 3;'	!
 CK(! 
! ! ! ! ! !&++ !(V 4+ 
++	,	+ + + + + +**%* !(V 4* -	*
 
,	* * * * * * * * * *rH   r  )rd  dummy_inputsc                       e Zd ZdZededededz  fd            Zddd	ed
eddf fdZ	de
defdZde
dedz  ded
efdZde
dej        fdZd3dZdej        dej        fdZdededz  fdZdededz  fdZdeez  deej        df         fdZdedefdZdej        ddfdZ dede!fdZ"	 d4dd d!dej        d"e!dz  d#ej        dz  d$edej        f
 fd%Z#	 	 d5dej        d&ej        d'e$dz  d(ej        dz  dede$fd)Z%d*ej        dej        dz  fd+Z&d,e'eeej        f                  de(e         fd-Z)de*fd.Z+d/edefd0Z,d1edefd2Z- xZ.S )6InternVLChatModelTr  r   rx   Nc                 |    |                     d          rdS |                     d          rdS t          d          )Nr   r   r   r*  z)Only image or video modality is supported)
startswithrM  )clsr  r   s      rI   get_placeholder_strz%InternVLChatModel.get_placeholder_str)  sG    w'' 	9w'' 	9DEEErH   r:  )prefixvllm_configr  c          
         t                                                       |j        j        }|j        }|j        j        }|| _        || _        |j        dk    | _        | 	                    ||           |j
        p|j        j        }|j        j        }|| _        ||z  dz  | _        t          | j        |j        dz  z            | _        |j        | _        |j        | _        |j        j        d         }|dk    | _        |                     |ddh          5  |                     ||| j        t1          |d                    | _        |                     |          | _        d d d            n# 1 swxY w Y   |                     |          5  t;          ||j        t1          |d	          
          | _        d d d            n# 1 swxY w Y   d | _        d | _         d | _!        | j        j"        | _"        d S )NrO   r   r   InternLM2VEForCausalLMr   r   vision_model)quant_configis_monor  language_model)r  	hf_configr  )#r   r   model_configr  r  multimodal_configr   mm_encoder_tp_modeuse_data_parallel_patch_quant_configforce_image_sizer   rw   r   patch_tokensr   r   r   
ps_versiontext_configarchitecturesr  _mark_tower_model_init_vision_modelr3   r  
_init_mlp1mlp1_mark_language_modelr2   r  img_context_token_idvideo_context_token_idvisual_token_maskmake_empty_intermediate_tensors)
r   r  r  r   r  r  rw   r   llm_arch_namer   s
            rI   r   zInternVLChatModel.__init__2  s{   )3"/'4F!2!2!E!O  666,O0D0O
)4
$':5!;"4#48OQR8R#STT & 7 +*8;$(@@##K'71CDD 	0 	0 $ 7 7)#FN;;	 !8 ! !D //DI	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 &&{33 	 	"<' ,#F,<==# # #D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 %)!&*#!%? 	,,,s%   AE11E58E5+GGGr   r  c                     t          |t                    r=|j        }t          |dd           }|j        s | |j                            d           d S d S d S d S )Nquantization_configr  )r   r   r  getattrmodules_to_not_convertr   )r   r   r  r  llm_quant_configs        rI   r  z%InternVLChatModel._patch_quant_configa  s    
 lI.. 	K ,K&{4I4PP 7 K ,3::>JJJJJ	K 	KK K,,rH   r  c                    |sC|j         }|dk     r|j        j        |z   dz   }n|dz   }t          |j        |||| j                  S t          |j                  S )Nr   r,   )r  num_hidden_layers_overrider  r  )select_layerr   num_hidden_layersr   r  r   )r   r   r  r  r  vision_feature_layerr  s          rI   r  z$InternVLChatModel._init_vision_modeln  s      	@#)#6 #a''(:=QQTUU "! %91$<!$$)+<"&"8    *&*>???rH   c           
      ^   |j         j        }|j        j        }t          j        t          j        |t          d| j        z            dz  z            t          j        |t          d| j        z            dz  z  |          t          j	                    t          j        ||                    S )Nr,   r   )
r   hidden_sizer  nn
Sequential	LayerNormr   r   LinearGELU)r   r   vit_hidden_sizellm_hidden_sizes       rI   r  zInternVLChatModel._init_mlp1  s     .: ,8}L3q43H/H+I+IQ+NNOOI#a$*?&?"@"@A"EE  GIIIo77
 
 	
rH   r{   c           
         |                                 \  }}}}|                    ||t          ||z            t          ||z                      }|                    dddd                                          }|                    |t          ||z            t          ||z            t          |||z  z                      }| j        dk    rn*|                    dddd                                          }|S )Nr   r   r,   r9   v1)r   viewr   permute
contiguousr  )r   r   scale_factorrM   r;   r:   cs          rI   pixel_shufflezInternVLChatModel.pixel_shuffle  s    VVXX
1aFF1aQ-..A4D0E0EFFIIaAq!!,,..FFL !!L !!\L0122	
 
 ?d""		!Q1%%0022ArH   r6   c                    |                      |          }|d d dd d d f         }t          |j        d         dz            x}}|                    |j        d         ||d          }|                     || j                  }|                    |j        d         d|j        d                   }|                     |          }|S )N)r6   r,   r{   r   )r  )r  r   r   reshaper  r   r  )r   r6   
vit_embedsr:   r;   s        rI   extract_featurez!InternVLChatModel.extract_feature  s    &&L&AA
122qqq)
J$Q'3.///A''
(8(;Q2FF
''
AV'WW
''
(8(;RAQRTAUVV
YYz**
rH   rC  c                     |                     dd           }|                     dd           }|                     dd           }||d S |t          d|          S |d         }t          |t          j                  r8|                                                                                                }t          |t                    sJ || _	        |+| j
        j        j        x}}||d}t          d|||          S t          d	          )
Nr<   r   rL   r7   rO   r   r:   r;   r6   r7   r<   r>   resolve_bindings This line should be unreachable.)poprK   r   rE   rF   flattenuniquer   r   r  r   r   rw   r5   AssertionError)	r   rC  r<   r   rL   r   
expected_h
expected_wr  s	            rI   _parse_and_validate_image_inputz1InternVLChatModel._parse_and_validate_image_input  s7    #JJ':DAA"JJ':DAAzz.$77$)=4#/#!   
   01nel33 	F+3355<<>>CCEEN.#.....$2!(&*k&?&JJJ%/jAA+#"3-!1	    ?@@@rH   c                     |                     dd           }|                     dd           }|                     dd           }||d S |t          d|          S |d         }t          |t          j                  r8|                                                                                                }t          |t                    sJ || _	        |+| j
        j        j        x}}||d}t          d|||	          S t          d
          )Nr'  r(  rL   rZ   r  r  r  rV   r  r  )r  rY   r   rE   rF   r  r	  r   r   r  r   r   rw   rU   r
  )	r   rC  r'  r(  rZ   r  r  r  r  s	            rI   _parse_and_validate_video_inputz1InternVLChatModel._parse_and_validate_video_input  s7    #)**-F"M"M"JJ':DAAzz.$77"*|/C4#/#!   
   01nel33 	F+3355<<>>CCEEN.#.....&4#".&*k&?&JJJ%/jAA+*"9-!1	    ?@@@rH   image_input.c                    |d         dk    s|d         dk    r|d         S |                      |d                   }|d         }t          |          dk    r&|                    d| j        j        j                  fS |j        d         |                    d| j        j        j                  }fd	|D             }|                    |          S )
Nr7   rL   rZ   rO   r<   r>   r,   r  c                     g | ]}|z  S rG   rG   )r   r>   r   s     rI   r   z;InternVLChatModel._process_vision_input.<locals>.<listcomp>  s+     
 
 
+6K,&
 
 
rH   )r   r   r  r   r  r  r   split)r   r  rL   r>   image_feature_sizesr   s        @rI   _process_vision_inputz'InternVLChatModel._process_vision_input  s    
 >116"n44v&&++K8K,LMM!-0 {q   %%b$+*A*MNNPP $)!,#((T[-D-PQQ
 
 
 
:E
 
 
 !!"5666rH   c                 t    i }|D ]2}|dv rd|vr | j         di ||d<   |dv rd|vr | j        di ||d<   3|S )N)r<   rL   r   )r'  r!  rG   )r  r  )r   rC  
modalities	input_keys       rI   %_parse_and_validate_multimodal_inputsz7InternVLChatModel._parse_and_validate_multimodal_inputs  s    
   	V 	VIBBBJ..'Kt'K'U'Uf'U'U
8$888XZ=W=W'Kt'K'U'Uf'U'U
8$rH   	input_idsc                     | j         r/| j        J || j        k                        dd          | _        d S d | _        d S )Nr  r,   )r  r  r  r  )r   r  s     rI   _set_visual_token_maskz(InternVLChatModel._set_visual_token_mask&  sV    < 	*,888&/43L&L%U%UA& &D""" &*D"""rH   c                 
    | j         di |}|sg S d}|D ]l}|dk    r/|d         }|                     |          }|t          |          z  }|dk    r/|d         }|                     |          }|t          |          z  }m|S )NrG   r   r!  )r  r  r  )	r   rC  r  multimodal_embeddingsr  r  image_embeddingsvideo_inputvideo_embeddingss	            rI   embed_multimodalz"InternVLChatModel.embed_multimodal/  s    ?T?II&II
 	I ;= # 	A 	AH8##(2#'#=#=k#J#J %/?)@)@@%8##(2#'#=#=k#J#J %/?)@)@@%$$rH   F)is_multimodalhandle_oov_mm_tokenr  r#  r$  c                    |(t          |          dk    r|                     |           ||!t                                          |          S t                                          ||||          S )Nr   )r  r#  r$  )r   r  r   embed_input_ids)r   r  r  r#  r$  r   s        rI   r&  z!InternVLChatModel.embed_input_idsF  s     !,5J1K1Ka1O1O''	222 !(M,A77**9555ww&&"7' 3	 ' 
 
 	
rH   	positionsintermediate_tensorsinputs_embedsc                     |d }||||d}| j         #|                    d| j         i           d | _          | j        j        di |}|S )N)r  r'  r(  r)  r  rG   )r  updater  model)r   r  r'  r(  r)  rC  forward_kwargshidden_statess           rI   forwardzInternVLChatModel.forward\  sz      + M #"$8*	
 
 !-!!#68N"OPPP%)D"1+1CCNCCrH   r.  c                 6    | j                             |          S r   )r  compute_logits)r   r.  s     rI   r1  z InternVLChatModel.compute_logitsv  s     "11-@@@rH   weightsc                 V    g d}t          | |          }|                    |          S )N)action_embedtemporal_embedtrack_embedtrack_embed_decoder	box_tokencg_criterioncg_modelloc_encoderloc_decodersamtemporal_tokentrack_token)skip_prefixes)r1   load_weights)r   r2  r@  loaders       rI   rA  zInternVLChatModel.load_weights|  s;    
 
 
 #4}EEE""7+++rH   c                 0    t          j        ddd          S )z<
        Get the module prefix in multimodal models
        r  r  r  )r  	connectortower_model)r   from_string_fieldr   s    rI   get_mm_mappingz InternVLChatModel.get_mm_mapping  s'     /+&
 
 
 	
rH   num_image_tokensc                 V    |dk    s| j         dk    rdS || j         z  }|| j        dz   z  S r   r   r  )r   rH  r>   s      rI   get_num_mm_encoder_tokensz+InternVLChatModel.get_num_mm_encoder_tokens  s?    q  D$8A$=$=1&$*>>d/!344rH   num_vision_tokensc                 V    |dk    s| j         dk    rdS || j        dz   z  }|| j         z  S r   rJ  )r   rL  r>   s      rI   get_num_mm_connector_tokensz-InternVLChatModel.get_num_mm_connector_tokens  s?    !!T%9Q%>%>1'D,=,ABT111rH   )r{   r   )NN)/r@   rA   rB   supports_encoder_tp_dataclassmethodr  r   r  r   r   r   r   r  r   r  r  Moduler  r  rE   rF   r   rW  rS   r  rU   r  r[   r  r  r  r  r  r-   r"  r&  r(   r/  r1  r   setrA  r   rG  rK  rN  r  r  s   @rI   r  r  !  s-         $F3 F3 F3: F F F [F BD -
 -
 -
z -
3 -
 -
 -
 -
 -
 -
 -
^K&K6HK K K K@ @ )4/@
 @ @ @ @ @6
!1 
bi 
 
 
 
   $	EL 	U\ 	 	 	 	"A"A	t	#"A "A "A "AH"A"A	!D	("A "A "A "AH7(+>>7 
u|S 	!7 7 7 76f      * * * * * *% %4H % % % %4 >B

 .2$)
 
 
<
  4d:

 |d*
 "
 

 
 
 
 
 
4 <@-1 < < 2D8	
 |d*  
   4A|A 
	A A A A,HU33D-E$F ,3s8 , , , ,&
 
 
 
 
5# 5# 5 5 5 52S 2S 2 2 2 2 2 2 2 2rH   r  )sabcr   r   collections.abcr   r   r   typingr   r	   r
   r   r   numpy.typingr?  rE   torch.nnr  torchvision.transforms
transformsrg   PILr   transformersr   r   r   vllm.configr   vllm.config.multimodalr   'vllm.model_executor.layers.quantizationr   +vllm.model_executor.layers.quantization.awqr   %vllm.model_executor.models.intern_vitr   r   )vllm.model_executor.models.module_mappingr   vllm.multimodalr   vllm.multimodal.imager   vllm.multimodal.inputsr   r   r   vllm.multimodal.parser   r   r    r!   vllm.multimodal.processingr"   r#   r$   r%   r&   r'   vllm.sequencer(   vllm.tokenizersr)   vllm.utils.tensor_schemar*   r+   
interfacesr-   r.   r/   r0   utilsr1   r2   r3   r3  r4  r  re   rf   r5   rK   rS   rD   rU   rY   r[   r   rr   r|   rQ   r  r   r   r   r   r   r   rF   r   r@  r   r   r  rB  rX  r[  ri  r  r  r  register_processorrQ  r  rG   rH   rI   <module>rm     sv   $ # # # # # # # # 7 7 7 7 7 7 7 7 7 7 > > > > > > > > > > > > > >              " " " " " "       C C C C C C C C C C " " " " " " 3 3 3 3 3 3 F F F F F F A A A A A A        E D D D D D / / / / / / 4 4 4 4 4 4         
                           . - - - - - ) ) ) ) ) ) > > > > > > > >            O N N N N N N N N N	
%$< < < < <| < < <	S 	S 	S 	S 	S< 	S 	S 	S ":<X!X Y X X X< < < < <| < < <	S 	S 	S 	S 	S< 	S 	S 	S ":<X!X Y X X X     c3h( 	
   38_   .00 0 	0
 0 38_0 0 0 0 <<< 
%S/< < < <// / c3h(	/
 / / 3S=/ / / /@&;& c3h(& 	&
 & 
%+& & & &T;  	
   \   .;  	
   \   4TJ TJ TJ TJ TJC TJ TJ TJnWO WO WO WO WO- WO WO WOt9
 9
 9
 9
 9
!3 9
 9
 9
x WT3444
 
 
 
 
%;B%? 
 
 
:[
 [
 [
 [
 [
&=b&A [
 [
 [
|,
 ,
 ,
 ,
 ,
7 ,
 ,
 ,
^&. &. &. &. &."#9:&. &. &.RW W W W W#$:;W W Wt ('	+  
2 2 2 2 2	#5z< 2 2 
2 2 2rH   