.`i< @ddlZddlmZmZmZddlmZddlmZm Z ddl Z ddl Z ddl mZddlmZddlmZmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lm Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9m:Z:ddl;mZ>m?Z?m@Z@ddlAmBZBddlCmDZDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLmMZMddlNmOZOddl mPZPmQZQmRZRmSZSmTZTdd lUmVZV dId$eWd%eWd&eWd'eWd(eWf d)ZXGd*d+e>ZYGd,d-ed?ej\ZdGd@dAej\ZeGdBdCej\ZfGdDdEej\Zge1jhe[eYeZFGdGdHej\eMeLZidS)JN)IterableMappingSequence)partial) AnnotatedLiteral rearrange) BatchFeaturePretrainedConfig)GELUActivation)BaseModelOutputWithPooling) torch_int) VllmConfig)BaseDummyOptions)parallel_state)utils)MMEncoderAttention) Conv2dLayer)QKVParallelLinearRowParallelLinear)QuantizationConfig)ApplyRotaryEmb)default_weight_loadermaybe_remap_kv_scale_name)MULTIMODAL_REGISTRY)MultiModalDataDictMultiModalFeatureSpecMultiModalFieldConfigMultiModalKwargsItems)ImageProcessorItems ImageSizeMultiModalDataItems)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacement PromptUpdate)IntermediateTensors) TensorSchema TensorShape)AttentionBackendEnum)Ernie4_5ForCausalLM)MultiModalEmbeddings SupportsMRoPESupportsMultiModal) SiglipMLP)AutoWeightsLoaderPPMissingLayer WeightsMapperis_pp_missing_parameter maybe_prefix)get_vit_attn_backend Pheightwidthfactor min_pixels max_pixelsc||krt||z|z }|}||krt||z|z }|}t||t||z dkr1tdt||t||z t||z |z}t||z |z}||z|krUt j||z|z }t j||z |z |z}t j||z |z |z}n]||z|krTt j|||zz }t j||z|z |z}t j||z|z |z}||fS)a)Rescales the image so that the following conditions are met: 1. Both dimensions (height and width) are divisible by 'factor'. 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. 3. The aspect ratio of the image is maintained as closely as possible. z4absolute aspect ratio must be smaller than 200, got )roundmaxmin ValueErrormathsqrtfloorceil)r<r=r>r?r@h_barw_barbetas {/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/paddleocr_vl.py smart_resizerOYs"uv~/00 v~~5011 65C...44 =vu%%FE(:(:: = =    &6/ " "V +E %&. ! !F *E u}z!!y&5.J677 6D=6122V; 54<&011F:  # #yv~677 &4-&011F: %$,/0069 %<cReZdZdZdefdZdefdZdZdededefd Z de fd Z d S) PaddleOCRVLProcessingInfoc4|jSN)ctx get_hf_configselfs rNrVz'PaddleOCRVLProcessingInfo.get_hf_configsx%%'''rPkwargsc &|jjdi|SN)rUget_hf_processorrXrYs rNr]z*PaddleOCRVLProcessingInfo.get_hf_processors(tx(226222rPc &|jdi|jSr[)r]image_processorr^s rNget_image_processorz-PaddleOCRVLProcessingInfo.get_image_processors$t$..v..>>rPc ddiS)Nimager\rWs rNget_supported_mm_limitsz1PaddleOCRVLProcessingInfo.get_supported_mm_limitss rP image_width image_heightreturnc:||}|}|j}|j}|j}t ||||z|j|j\}} t| |} d} | j |z} | j |z} | | z| z}||dzz}|S)N)r<r=r>r?r@)r=r<r-) rarV vision_config patch_sizespatial_merge_sizerOr?r@r"r<r=)rXrerfr` hf_configrjrk merge_sizeresized_height resized_widthpreprocessed_sizegrid_tgrid_hgrid_w num_patchesnum_image_tokenss rNget_num_image_tokensz.PaddleOCRVLProcessingInfo.get_num_image_tokenss  ""6688O&&(( !/ "- "5 (4 *&1&1 ) ) ) % &M.QQQ")Z7"(J6vo. &:q=9rPc,|}|jj}|jj}||z}|j|dzz}t tj|}|||zz}||z}t||z||zS)Nri)r<r=) rVrjrlrkrar@intrGrHr")rXrmrnrkr>max_num_tokens h_patches w_patchess rN!get_image_size_with_most_featuresz;PaddleOCRVLProcessingInfo.get_image_size_with_most_featuress&&(( ,? ,7 j(1133>619M .1122 .944"i/  F 2)f:LMMMMrPN) __name__ __module__ __qualname__rVobjectr]rardryrwr"r}r\rPrNrRrRs(((33333?F????        @ N9 N N N N N NrPrRc peZdZdeeefdefdZ ddedeeefdeeefdzdefdZ dS) PaddleOCRVLDummyInputsBuilder mm_countsrgcx|dd}|j}|j}||zS)Nrcr)getinfor] image_token)rXr num_images processorrs rNget_dummy_textz,PaddleOCRVLDummyInputsBuilder.get_dummy_texts;]]7A.. I..00 + Z''rPNseq_len mm_optionsc|dd}|j}|r|dnd}d||j|j||iS)Nrcr)r=r<r overrides)rrr}_get_dummy_imagesr=r<)rXrrrrmax_image_sizeimage_overridess rNget_dummy_mm_dataz/PaddleOCRVLDummyInputsBuilder.get_dummy_mm_datas} ]]7A.. DDFF5?I*..111T T++$*%,%) ,  rPrT) r~rrrstrryrrrrr\rPrNrrs(S(9(c((((=A    38$ C!112T9         rPrc eZdZdedeeefdeeefdeeefdef dZdedeeefdeeeffd Z d e deeefd e de e fd Zd S)PaddleOCRVLMultiModalProcessorpromptmm_data mm_kwargs tok_kwargsrgc |r|jj|jjdi|t dd|i|t di||}|dd}|d||d<n'|j}||dd}|S) Ntextimage_grid_thw pixel_valuesTpt)add_special_tokensreturn_tensorsr\) rrUcall_hf_processorr]dictprodsplittolist get_tokenizer)rXrrrrprocessed_outputsnum_patches_per_image tokenizers rN_call_hf_processorz1PaddleOCRVLMultiModalProcessor._call_hf_processors   $ ? ?* *77Y77,,&,G,,//y/J//!!  %66F$G$L$LR$P$P !0A.0Q0W0W%,,..11 n - - //11I ) 4!!! ! rP hf_inputshf_processor_mm_kwargsclttjdtjdS)Nrc)rr)rrbatched)rXrrs rN_get_mm_fields_configz4PaddleOCRVLMultiModalProcessor._get_mm_fields_configs7 .6w??08AA    rPmm_items out_mm_kwargscjjdi|}j}|jdtffd }t dgt ||gS)Nitem_idxcdt}||}j|j|j|}g|zS)Nrc)rerfr`) get_itemsr!get_image_sizerrwr=r<)rr`images image_sizervimage_token_idrrXs rNget_replacementzKPaddleOCRVLMultiModalProcessor._get_prompt_updates..get_replacementsi''1DEEF..x88J#y==&,'. / >    ##&66 6rPrc)r`)modalitytarget replacementr\)rrarVrryr'r)rXrrrr`rmrrs`` @rN_get_prompt_updatesz2PaddleOCRVLMultiModalProcessor._get_prompt_updatess 8$)7QQ:PQQI++-- "1 7c 7 7 7 7 7 7 7 7  &'#O_UUU     rPN)r~rrrrrr rrrr#r rr(rr\rPrNrrs!!f%!3;' ! CK( !  !!!!0  !(V 4  ++ ,      % !(V 4 -  ,        rPrcbeZdZ d dededeffd Zdejdejdejfd ZxZ S) Projector text_configrjprefixct||_||_d|_|jj|jdz|jdz|_t j|jjd|_ tj |j|jd|_ t|_ tj |j|jjd|_dS)N)ririrr-gh㈵>epsT)bias)super__init__rrjmerge_kernel_size hidden_sizetorchnn LayerNormpre_normLinearlinear_1r actlinear_2)rXrrjr __class__s rNrzProjector.__init__$s &*!'   *$Q' ($Q' (  **4+=+Iu*UU  $"2D4D4PPP !##  d.:    rPimage_featuresrrgc 0|j\}}t|ttfrt}t ||D]\}}||}|\}} } t |d|| |z|| |z|}||} || } | | } | | |S|j dd} |j d} | tj| | }|| d|j} || } || } | | } | j g| dRS)Nz$(t h p1 w p2) d -> (t h w) (p1 p2 d))thp1wp2r)r isinstancelisttupleziprr rrrappendshapeviewnprr)rXrrm1m2processed_features image_feature image_gridrrr hidden_statesdimsdims rNforwardzProjector.forward<s 'B ntUm 4 4 &!% -0-P-P 9 9) z $ m < < $1a )!:2g2g!!! !% m < < $ 7 7 $ m < < "))-8888% %#CRC("2&',,RWT]]C@@ n55::2t?OPP  m44 //  m44 !}!,4,,,,,rP)r) r~rrr rrrTensorr __classcell__rs@rNrr#s    % (        0$- $- $-  $-$-$-$-$-$-$-$-rPrc eZdZUeded<eejeddddddhfed<eejeddfed<d S) PaddleOCRImagePixelInputsrtypebnprk) dynamic_dimsrN) r~rrr__annotations__rrrr+r\rPrNrrcs . !!!!  D#q, C5QQQ S  D! rPrc eZdZdeffd Z ddejdedededejf d Z ddejd ed ed efdZ ddej dejdzde e eeefe e eeefzdzdejfdZxZS)SiglipVisionEmbeddingsconfigct||_|j|_|j|_|j|_t|j|j|j|jd|_ |j|jzdz|_ |j |_ t|_ t|_tj|j |j|_tjd|j|_|dt)j|j dddS) Nvalid) in_channels out_channels kernel_sizestridepaddingrii position_ids)r-rF persistent)rrrr embed_dimrrkr num_channelspatch_embeddingru num_positionsrcache_position_embeddingcache_position_countr Embeddingposition_embeddingpacking_position_embeddingregister_bufferrarangeexpand)rXrrs rNrzSiglipVisionEmbeddings.__init__ps"  + + +*+?    !Ot>1D!-(,%$(FF!"$,t/A4>"R"R*,,udn*M*M'   L+ , , 3 3G < <      rPF embeddingsr<r=is_after_patchifyrgc|jjjd}|jjd}|jd}|r|}|} n||jz}||jz} t |dz} |d| | |}|dddd}tj ||| fdd }|dddd dd|}|S) Nrrg?r-rribilinearF)sizemode align_corners) rweightr unsqueezerkrreshapepermuter functional interpolater) rXrr<r=rr patch_pos_embedr new_height new_widthsqrt_num_positionss rNinterpolate_pos_encodingz/SiglipVisionEmbeddings.interpolate_pos_encodings/6 (b l) c h wdtyperTr-rz$Unsupported pixel_values dimension: z. Expected 4 or 5.)rrrFrr rr2r toflattensqueezerr$repeatrrconcatr)rXrrrr$ batch_size squence_lenchannelr<r= target_dtype patch_embedsrstarttmp_embeddingsrrrrendimage_embeddingsrs rNrzSiglipVisionEmbeddings.forwards      " "'11!44L      " "# O"  /6z)all_gather_interleave..s$OOO1(66OOOrP)groupcBg|]}tj|zdS)r)rr)rJtensorrrFs rNrLz)all_gather_interleave..s9.s9dz,SiglipAttention.split_qkv..Ss,99!6169%999999rP) rrFr^rurchunkrrqsplit_tensor_along_last_dimrprtrs) rXryrbsrKqkvsplitterr~s @rN split_qkvzSiglipAttention.split_qkv>sQ  s b dc36K|]}t|dVdS)zs b h d -> b s h dNr )rJrs rNrz*SiglipAttention.forward..cs-II!9Q 455IIIIIIrPrr4ri)queryr(valuerrzb s h d -> b s (h d)) rr rurrrXrxcossinrrwrv)rXrrrrr:rKr}rrr qk_concat qk_rotated context_layeroutputs rNrzSiglipAttention.forwardVs1). Aq m%5 6 6}}Q1..##1aII1ayIII1a  % 1a&a000I..""$$""$$J ;z1!444DAq !! "  "-1GHH MM-00  rP)r~rr__doc__ryrrrrrrrrrrs@rNr`r` sGG37, , , ,  ,  , )4/ , ,  , , , , , , \U\eEL#4E.F0"|"L "  t+ " L4' " """"""""rPr`cNeZdZd dededdffd ZdZdedejfd Z xZ S) SigLIPRotaryEmbedding@rthetargNct||_||_|dSrT)rrrr rope_init)rXrrrs rNrzSigLIPRotaryEmbedding.__init__|s=   rPcd|jtjd|jdtj|jz zz }|d|ddS)N?rrir1inv_freqFr)rrrrfloatr)rXrs rNrzSigLIPRotaryEmbedding.rope_initsW J5<48QekJJJTXU V  ZeDDDDDrPseqlenctj||jj|jj}tj||j}|S)N)devicer2)rrrrr2outer)rXrseqfreqss rNrzSigLIPRotaryEmbedding.forwardsEl ='-%     C// rP)r) r~rrryrrrrrrrrs@rNrr{sCD EEE celrPrc eZdZ d dededzdeffd Zdejdejd ejdzd ejdzd ejf d Z xZ S)SiglipEncoderLayerNrrrbrc~t|j|_t j|j|j|_t|j|j |j||d|_ t j|j|j|_ t|||d|_ dS)Nrz .self_attn)rrcrdrbrz.mlpra)rrrrrrlayer_norm_eps layer_norm1r`num_attention_heads self_attn layer_norm2r2mlprXrrbrrs rNrzSiglipEncoderLayer.__init__s +<Fg|]}td|S)z.layers.ra)r)rJ layer_idxrrrbs rNrLz*SiglipEncoder.__init__..sQ     #!-$99i99   rPri)rrrrrr8rget_default_dtype attn_backendr, FLASH_ATTN TORCH_SDPA ROCM_AITER_FA RuntimeErrorr ModuleListrTnum_hidden_layerslayersrr)rXrrbrrrchead_dimrs ``` rNrzSiglipEncoder.__init__s!  & .  )0)++      + + .%   Q1BQQQ m       "'v'?!@!@        4HMBBrPct}|D]B}t|tr||-||C|SrT)rrextendr)rtmp_image_grid_thwrs rN flatten_listzSiglipEncoder.flatten_listsa!VV( 6 6J*d++ 6"))*5555"))*5555!!rPrrheight_position_idswidth_position_idsrgc|j}|}||}||t} t} |D]\\} } } tj| | z| z|| | zz}|| z}|| z}| || |]tj| d}tj| d}tj||gd}|dz}| |}|| d}|tdt|tj s"tj|tj|}n||}d}|jt$jt$jhvr'|dd|ddz }|}|jD]}|||||}|S) N)rrr4rr-z,cu_seqlens cannot be None for SiglipEncoder.)r2r)rrr)rrrrrrr9stackrDrr6rFrrrOint32r5rr,rrr)rX inputs_embedsrrrrrrflatten_image_grid_thw split_hids split_widsrrr image_pids sample_hids sample_widspids max_grid_sizerope_emb_max_gridrr encoder_layers rNrzSiglipEncoder.forwards+%% !%!2!2>!B!B  %)<)DJJ1 / /1a"\!a%!)FCCCq1uM (Ao (1n !!+...!!+....!&ja!@!@!@ "',zq"A"A"A { "4 5    Q  // >>*4088;;  KLL L*el33 6j FSSSJJ#f55J   + .!   %QRR.:crc?:??AAJ% ![  M)M%-% MM rPr)NNNN)r~rrr rrr staticmethodrrrrrryrrrs@rNrrs.37 "C"C "C)4/"C "C"C"C"C"C"CH""\"+/372688L4'8U3S=1DsC}9M4NNO  8 #\D0 8"L4/8 88888888rPrceZdZ ddededzdeffd Z ddejd e dzd ejdzd ejdzd ejdzd ejdzdejdzdejfdZ xZ S)SiglipVisionTransformerNrrrbrct||_|j}t ||_t |||d|_tj ||j |_ dS)Nz.encoderrar) rrrrrrrencoderrrrpost_layernorm)rXrrbrrrs rNrz SiglipVisionTransformer.__init__2s  & 088$ %&&&    !l9&:OPPPrPFrr$rrrrrrgc|||||}||||||} || } | S)N)r$rr)rrrrr)rrr) rXrr$rrrrrrlast_hidden_states rNrzSiglipVisionTransformer.forwardDso %=%) (  !LL'!) 31 )  !//0ABB  rPr)FNNNNN) r~rrr rrrrrrCrrrs@rNrr1s37 QQ Q)4/Q QQQQQQ*16,03726*..2!!l!#'+!lT) ! #\D0 ! "L4/ !L4'! t+! !!!!!!!!rPrc feZdZ ddedzdeffd ZedejfdZedej fdZ de j fd Z dd e d ejdzd eeeeefeeeeefzdzdejdzdef dZdeeeejfdeefdZxZS)SiglipVisionModelNrrbrctt|||d|_||_dS)Nz .vision_modelra)rrr vision_modelrbrs rNrzSiglipVisionModel.__init__bsT 3 %+++    )rPrgc8|jjjjjSrT)rrr rr2rWs rNr2zSiglipVisionModel.dtypeqs +;BHHrPc8|jjjjjSrT)rrr rrrWs rNrzSiglipVisionModel.deviceus +;BIIrPc$|jjjSrT)rrr rWs rNget_input_embeddingsz&SiglipVisionModel.get_input_embeddingsys +;;rPFr$rrrc6||||||S)N)rr$rrr)r)rXrr$rrrs rNrzSiglipVisionModel.forward|s0  %%=%)! !   rPweightscgd}t|d}t}|D]\}}d|vr d|vsd|vrd|vsd|vr|j~|j|x}rb||}t |d t } |d kr|n|d }| |||||D]i\} } } | |vr | | | }| d r||vr;t||rL||}|j } | ||| nk| d r||vr)t||}|=t||rO||}t |d t } | |||||S) N))ruq_projr)ruk_projr)ruv_projrF)remove_duplicatezrotary_emb.inv_freqzhead.attentionzhead.layernormzhead.mlpz head.probe weight_loaderrz.bias)rnamed_parameterssetrbget_cache_scalegetattrrraddreplaceendswithr6rr) rXrstacked_params_mapping params_dict loaded_paramsname loaded_weight scale_nameparamr param_name weight_nameshard_ids rN load_weightszSiglipVisionModel.load_weightsso" " "  400%0HHII "%%% #*5 $5 $ D-$,,4''+;t+C+CT!!\T%9%9 ,"/??EEE -$J/ '#)!! &3%6%6%8%8A%=%=MM=QRCS e]333!!*--- (  4 4d**||K<<==))d+.E.E*466#D) % 3  e]H=====))d+.E.E0{CC<*466#D) '#)!!  e]333   d # # # #rPr)FNNN)r~rrrrrpropertyrr2rrModulerrCrrrryrrrrrrrs@rNrras37 ) ))4/ ) ) ) ) ) ) )Iu{IIIXIJ JJJXJHU3 3D-E$F>3s8>>>>>>>>rPr)r dummy_inputsc eZdZedddZededededzfd Zd d d e d effdZ de j de j dzfdZ deedeedee j effdZdededzfdZ d#de j de j dedzde j dzfdZde j de j de j fdZdedefdZdefd Zd!eeee j fdeefd"ZxZS)$#PaddleOCRVLForConditionalGenerationzlanguage_model.model.zlanguage_model.lm_head.)zmodel.zlm_head.)orig_to_new_prefixrirgNcN|drdStd)Nrcz1<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>z Only image modality is supported) startswithrF)clsrrs rNget_placeholder_strz7PaddleOCRVLForConditionalGeneration.get_placeholder_strs.   w ' ' GFF;<<rvideo_grid_thwsecond_per_grid_tsc6g|]}|Sr\rrJitems rNrLzQPaddleOCRVLForConditionalGeneration.get_mrope_input_positions.. UUUD$++--UUUrPrc6g|]}|Sr\r r!s rNrLzQPaddleOCRVLForConditionalGeneration.get_mrope_input_positions..r#rPrrtokens_per_secondrr-r)rrgrrr4)r gather_kwargsrrrvideo_token_idvision_start_token_idrjrlrrrOargwherer7sumrTindexrFr)rDrrrrlongr6rrXrr")*rXrrrYrrrrmrr'r(rlr%input_tokens_tensorvision_start_indices vision_tokens image_nums video_numsllm_pos_ids_listst remain_images remain_videos image_index video_indexrKvideo_second_per_grid_ted_imageed_videorrred llm_grid_t llm_grid_h llm_grid_wtext_lenst_idxt_indexh_indexw_index llm_positionsmrope_position_deltas* rNget_mrope_input_positionsz=PaddleOCRVLForConditionalGeneration.get_mrope_input_positionss '4  F F F  VUFJJ?OQS4T4TUUUUUFJJ?OQS4T4TUUU#ZZ(AFFFII    \*EK@@@CCLDWXX %),%)! %  rP image_inputc|j}|j}tfdt||D}||}|S)Nc3pK|]0\}}||dV1dS)rN)rTr7)rJpixelr+rXs rNrzKPaddleOCRVLForConditionalGeneration._process_image_input..sY  t   eT * * 2 21 5 5      rP)rrrrr)rXrUrrrS image_embedss` rN_process_image_inputz8PaddleOCRVLForConditionalGeneration._process_image_inputso#/ $3    "<@@     {{>>BB rPc z|jdi|}|dSd}||}|t|z }|Sr[)rHrZr)rXrYrUmultimodal_embeddingsrYs rNembed_multimodalz4PaddleOCRVLForConditionalGeneration.embed_multimodalsW:d:DDVDD  2:<00== |!4!44$$rPrc\t|}|||j}|S)N)mapper)r3rhf_to_vllm_mapper)rXrloaderautoloaded_weightss rNrz0PaddleOCRVLForConditionalGeneration.load_weightss1"4((#00AW0XX!!rP)NN)r~rrr5r` classmethodrryr rrrrrrrrrFrrrHr)rrTr/rZr]rrrrrs@rNrrsk & -1  =3=3=3:===[= BD   z 3      :A|A  AAAA r33ir3/0r3 u|S ! r3r3r3r3h    "T )     &<@-1   <  <  2D8  |d*     !L:?, : 4       %,@ % % % %"HU3 3D-E$F"3s8""""""""rPr)r9r:r;)jrGcollections.abcrrr functoolsrtypingrrnumpyrrtorch.nnreinopsr transformersr r transformers.activationsr transformers.modeling_outputsrtransformers.utilsr vllm.configrvllm.config.multimodalrvllm.distributedrrrq9vllm.model_executor.layers.attention.mm_encoder_attentionrvllm.model_executor.layers.convr!vllm.model_executor.layers.linearrr'vllm.model_executor.layers.quantizationr2vllm.model_executor.layers.rotary_embedding.commonr-vllm.model_executor.model_loader.weight_utilsrrvllm.multimodalrvllm.multimodal.inputsrrrr vllm.multimodal.parser!r"r#vllm.multimodal.processingr$r%r&r'r( vllm.sequencer)vllm.utils.tensor_schemar*r+#vllm.v1.attention.backends.registryr,ernie45r. interfacesr/r0r1siglipr2r3r4r5r6r7visionr8ryrOrRrrrrrrrr^r`rrrrrregister_processorrr\rPrNrs" 7777777777%%%%%%%% 77777777333333)(((((""""""333333++++++000000877777GFFFFF0/////  .----->>>>>>>>DDDDDD((((((OOOOOOOOOO)((((( #$ (( ( ( ( (  ((((V:N:N:N:N:N 2:N:N:Nz     $:;T$U