
     `it>                         d dl mZmZ d dlZddlmZ ddlmZm	Z	m
Z
 ddlmZmZmZmZmZ ddlmZmZ ddlmZ  G d	 d
ed          Z G d ded          Z G d de          ZdgZdS )    )OptionalUnionN   )BatchFeature)
ImageInputconcatenate_listmake_flat_list_of_images)ImagesKwargsMultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)
VideoInputc                   R    e Zd ZU ee         ed<   ee         ed<   ee         ed<   dS )InternVLImagesKwargscrop_to_patchesmin_patchesmax_patchesN)__name__
__module____qualname__r   bool__annotations__int     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/internvl/processing_internvl.pyr   r      sB         d^#####r   r   F)totalc                   4    e Zd ZU eed<   dddddiddid	Zd
S )InternVLProcessorKwargsimages_kwargsleftF)padding_sidereturn_mm_token_type_idsr   Treturn_tensorspt)text_kwargsr#   videos_kwargsN)r   r   r   r   r   	_defaultsr   r   r   r"   r"   !   sR         '''' #(-
 

 t
 d
 IIIr   r"   c                   J    e Zd ZdZg dZdZdZdZ	 	 	 	 	 ddef fd	Z	d
e
e         de
e         de
e         dej        dej        dej        fdZ	 	 	 	 ddee         d
eeeee
e         e
e         f                  dee         dee         def
dZddZed             Z xZS )InternVLProcessoraM  
    Constructs a InternVL processor which wraps a [`AutoImageProcessor`] and
    [`PretrainedTokenizerFast`] tokenizer into a single processor that inherits both the image processor and
    tokenizer functionalities. See the [`~InternVLProcessor.__call__`] and [`~InternVLProcessor.decode`] for more information.
    Args:
        image_processor ([`AutoImageProcessor`], *optional*):
            The image processor is a required input.
        tokenizer ([`PreTrainedTokenizer`, `PreTrainedTokenizerFast`], *optional*):
            The tokenizer is a required input.
        video_processor ([`AutoVideoProcessor`], *optional*):
            The video processor is a required input.
        image_seq_length (`int`, *optional*, defaults to 256):
            The number of image token to use per image patch. it should be set so that:
            image_seq_length = (config.image_size // config.patch_size) ** 2 * (config.scale_factor**2)
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    )image_processor	tokenizervideo_processorAutoImageProcessorAutoVideoProcessorAutoTokenizerN   image_seq_lengthc                 ,   || _         |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _	        | j	        | j        | j        g| _
         t                      j        |||fd|i| d S )Nchat_template)r5   start_image_tokenend_image_tokenstart_image_token_idend_image_token_idcontext_image_tokenimage_tokenvideo_tokencontext_image_token_idimage_token_id	image_idssuper__init__)selfr.   r/   r0   r5   r7   kwargs	__class__s          r   rC   zInternVLProcessor.__init__I   s     !1!*!<(8$-$B!"+">$8$0'>-t/H$Jab)_llTaleklllllr   textimage_num_patchesvideo_num_patchesimage_num_patches_indicesvideo_num_patches_indicesvideo_patch_indicesc	           	      d    d}	d}
g }g }g }|D ]}|} j         |v s
 j        |v r j         |v rԉ j        |vs6|                     j                   |                     j                  k     r|	dk    r||	dz
           nd}||	         }|                    |||                    |                     j         dd          }|                     j          j          j        z  ||	         z    j                    |	dz  }	n||
         }||
dz            }||         }||         }|                    |||                    t          |||                   d	                     fdt          t                              D                       }|                    |           |                     j        dd          }|
dz  }
 j         |v  j        |v d|v r0|                    d          }|                    d|d          }d|v 0|                    |           |||	|
fS )z
        Processes interleaved text with <image> and <video> placeholders, replacing them with appropriate
        image and video tokens while keeping track of the patches used.
        r      z<placeholder>
c              3   v   K   | ]3}d |dz    dj          j        j        z  |         z   j         V  4dS )FramerN   z: N)r8   r=   r5   r9   ).0inum_patchesrD   s     r   	<genexpr>z?InternVLProcessor._insert_media_placeholders.<locals>.<genexpr>   s       - - SA  S  S)?  SAQTXTiAilwxylzAz  S  }A  }Q  S  S- - - - - -r   )r=   r>   indexappendreplacer8   r5   r9   listjoinrangelenpop)rD   rG   image_pixel_valuesvideo_pixel_valuesrH   rI   rJ   rK   rL   image_indexvideo_indexprocessed_textimage_video_patchesreplace_stringsprompt
new_promptstart_index	end_indexcurrent_patch_indexend_patch_indexvideo_promptreplace_strrT   s   `                     @r   _insert_media_placeholdersz,InternVLProcessor._insert_media_placeholders^   s       &	. &	.FJ"j00D4D
4R4R#z11$J66!''(899J<L<LTM]<^<^^^ Q\^_P_P_";K!O"L"LefK 9+ FI'../A+iBW/XYYY!+!3!3D4DoWX!Y!YJ#**1  U43CdF[3[^op{^|3|  U  C  S  U  U    1$KK
 +>k*J'&9+/&JO";<O"PK 9/ JI'../A+iBW/XYYY"&'89L_9\']"^"^K#'99 - - - - -!&s;'7'7!8!8- - - $ $L $**<888!+!3!3D4DoWX!Y!YJ1$KA "j00D4D
4R4RB "Z//-11!44'//aPP
 "Z// !!*----2KLLr   imagesvideosrE   returnc           
         |t          d           | j        t          fd| j        j        i|}t          |t          t          f          s|g}g }d}t          j	        dg          }	||| j
                            |          }t          |          } | j
        dd|i|d         }
|
                    d          }|
                    d          }t          j        |          }	g }d}t          j	        dg          }t          j	        dg          }||d	         } | j        dd
|i|}|                    d          }|j        ^}}}t          j        ||          }t%          |          }t          j        |dz   t(                    }d|d<   t          j        |          |dd<   dg|z  }t          j        |dz   t(                    }d|d<   t          j        |          |dd<   |                    dd          }i }||z|                     ||||||	||          \  }}}}|"|t/          |          k    rt          d          |"|t/          |          k    rt          d          dt1          |          i}|d                             dd          }|d                             dd          } | j        |fi |d         }|                     ||dg           |rht          j	        |d                   }t          j        |d                   }d|t          j        || j                  <   |                                |d<   t=          i |||          S )a	  
        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
        and `kwargs` arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] to encode the text if `text`
        is not `None`, otherwise encode default OCR queries which depends on the `format`, `box`, `color`, `multi_page` and
        `crop_to_patches` arguments. To prepare the vision inputs, this method forwards the `images` and `kwargs` arguments to
        GotOcr2ImageProcessor's [`~GotOcr2ImageProcessor.__call__`] if `images` is not `None`.

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. Both channels-first and channels-last formats are supported.
            text (`str`, `list[str]`, `list[list[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors of a particular framework. Acceptable values are:
                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return NumPy `np.ndarray` objects.
                - `'jax'`: Return JAX `jnp.ndarray` objects.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
        NzYou have to specify text.tokenizer_init_kwargsr   rn   r#   rT   pixel_valuesr*   ro   pixel_values_videosrN   zONumber of image placeholders in the prompt does not match the number of images.zONumber of video placeholders in the prompt does not match the number of videos.r)   r'   r&   image)
modalities	input_idsmm_token_type_ids)datatensor_typer   )
ValueError_merge_kwargsr"   r/   init_kwargs
isinstancerY   tuplenparrayr.   fetch_imagesr	   r]   cumsumr0   shapefullsumemptyr   flattenrm   r\   r   _check_special_mm_tokens
zeros_likeisinrA   tolistr   )rD   rn   rG   audioro   rE   output_kwargsrH   r^   rJ   image_inputsrI   r_   rL   rK   video_kwargsvideo_inputs
batch_size
num_frames_num_frames_per_videoimage_videos_inputsrc   r`   ra   r'   r&   text_inputs	array_idsrx   s                                 r   __call__zInternVLProcessor.__call__   s   R <8999**#
 
"&."<
 
 
 $u.. 	6D !$&HaSMM!)66v>>F-f55F/4/``v`A_``L , 0 0 ? ?!-!1!1.!A!A(*	2C(D(D%! hsmm$&HaSMM!(9L/4/NNvNNNL!-!1!12G!H!H);)A&J
Q#%7:z#B#B 122J"$(:>3"?"?%&"&(i0D&E&E#!"j 0(*a(E(E%+,%a(,.I6G,H,H%abb)!3!;!;Aq!A!A !3BFBaBa""!!))#	C 	C?D%{K !kS[[&@&@ !rsss!kS9M5N5N&N&N !rsss $23CDW3X3X"Y&}599:JDQQ#0#?#C#CD^`d#e#e $dnTJJ]=-IJJ%%dKWI%NNN# 	J[!9::I "k+.F G GDEbgi@@A/@/G/G/I/IK+,!GK!G3F!GUcddddr   c                      i }|jt           j                            di                               |            fd|D             } fd|D             }|                    ||d           t	          di |S )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr#   c                 8    g | ]} j         j        g |R  S r   )r.   get_number_of_image_patches)rR   
image_sizer#   rD   s     r   
<listcomp>z@InternVLProcessor._get_num_multimodal_tokens.<locals>.<listcomp>'  sE     ! ! ! A$@\*\m\\\! ! !r   c                 *    g | ]}d j         |z  z   S )   )r5   )rR   rT   rD   s     r   r   z@InternVLProcessor._get_num_multimodal_tokens.<locals>.<listcomp>,  s&    oookT%:[%H Iooor   )num_image_tokensnum_image_patchesr   )r"   r+   getupdater   )rD   image_sizesrE   vision_datar   r   r#   s   `     @r   _get_num_multimodal_tokensz,InternVLProcessor._get_num_multimodal_tokens  s     "3=AA/SUVVM  (((! ! ! ! !"-! ! !
  pooo]nooo4D[lmmnnn,,,,,r   c                 <    | j         j        }| j        j        }||z   S N)r/   model_input_namesr.   )rD   tokenizer_input_namesimage_processor_input_namess      r   r   z#InternVLProcessor.model_input_names1  s'     !% @&*&:&L#$'BBBr   )NNNr4   N)NNNNr   )r   r   r   __doc__
attributesimage_processor_classvideo_processor_classtokenizer_classr   rC   rY   strr   ndarrayrm   r   r   r   r   r   r   r   r"   r   r   r   propertyr   __classcell__)rF   s   @r   r-   r-   1   s        $ EDDJ00%O  #m m
 m m m m m m*>M3i>M
  9>M  9>M $&:>M $&:>M  Z>M >M >M >MD (,hl'+ue ue$ue uY(94	?DQbLccdeue
 $ue 01ue 
ue ue ue uen- - - -8 C C XC C C C Cr   r-   )typingr   r   numpyr   image_processing_utilsr   image_utilsr   r   r	   processing_utilsr
   r   r   r   r   tokenization_utils_baser   r   video_utilsr   r   r"   r-   __all__r   r   r   <module>r      s[    # " " " " " " "     2 2 2 2 2 2 Q Q Q Q Q Q Q Q Q Q f f f f f f f f f f f f f f C C C C C C C C % % % % % %    <u        .e     EC EC EC EC EC EC EC ECP 
r   