
     `i                     4   d dl mZmZ d dlZd dlmZ d dlZd dlm	Z
 d dl	Zd dlmZmZmZ d dlmZ d dlmZmZ ddlmZmZmZmZ ddlmZmZmZmZ dd	lm Z m!Z! d
dl"m#Z# ej$        j%         G d de                      Z&dZ'dZ(de)e*e*f         de
j+        fdZ,e
j-        fdZ. G d dej/                  Z0 G d dej/                  Z1 G d dej/                  Z2 G d dej/                  Z3 G d dej/                  Z4 G d dej/                  Z5 G d  d!ej/                  Z6 G d" d#ej/                  Z7 G d$ d%ej/                  Z8 G d& d'ej/                  Z9 G d( d)ej/                  Z: G d* d+ej/                  Z; G d, d-e          Z< G d. d/ej/                  Z= G d0 d1ej/                  Z> e d2e'           G d3 d4e<                      Z?d5Z@ ee?e@            ee?e&e#6            G d7 d8ej/                  ZA e d9e'           G d: d;e<                      ZBd<ZC eeBeC            eeBee#6            G d= d>ej/                  ZD e d?e'           G d@ dAe<                      ZEdBZF eeEeF            eeEee#6           g dCZGdS )D    )CallableOptionalN)
FrozenDictfreezeunfreeze)dot_product_attention_weights)flatten_dictunflatten_dict   )FlaxBaseModelOutputFlaxBaseModelOutputWithPoolingFlaxMaskedLMOutputFlaxSequenceClassifierOutput)ACT2FNFlaxPreTrainedModel append_replace_return_docstringsoverwrite_call_docstring)add_start_docstrings%add_start_docstrings_to_model_forward   )
BeitConfigc                       e Zd ZdZdS )FlaxBeitModelOutputWithPoolinga  
    Class for outputs of [`FlaxBeitModel`].

    Args:
        last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`jnp.ndarray` of shape `(batch_size, hidden_size)`):
            Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
            *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
            will be returned.
        hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
            the initial embedding outputs.
        attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
            the self-attention heads.
    N)__name__
__module____qualname____doc__     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/beit/modeling_flax_beit.pyr   r   ,   s           r   r   a  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`BeitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].
a  
    Args:
        pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`AutoImageProcessor.__call__`] for details.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
window_sizereturnc                 Z   d| d         z  dz
  d| d         z  dz
  z  dz   }t          j        | d                   }t          j        | d                   }t          j        t          j        ||d                    }t          j        |d          }|dddddf         |dddddf         z
  }t          j        |d	          }|dddddfxx         | d         dz
  z  cc<   |dddddfxx         | d         dz
  z  cc<   |dddddfxx         d| d         z  dz
  z  cc<   t          j        | d         | d         z  dz   fdz  |j        
          }|                    d          |ddddf<   |dz
  |dddf<   |dz
  |dddf<   |dz
  |d<   t          j
        |          S )zP
    get pair-wise relative position index for each token inside the window
       r   r   r   ij)indexing)r$   N)r   r$   r   shapedtyper'   )r   r   )nparangestackmeshgridreshape	transposezerosr*   sumjnparray)r!   num_relative_distancecoords_hcoords_wcoordscoords_flattenrelative_coordsrelative_position_indexs           r    relative_position_index_initr<   w   s    Q/!3KN8JQ8NORSSyQ((HyQ((HXbk(HtDDDEEFZ00N$QQQ4Z0>!!!T111*3MMOl?I>>OAAAqqq!GA 22AAAqqq!GA 22AAAqqq!GKN 2Q 66 hk!n{1~.MPQ.Q-SVW-W_n_tuuu&5&9&9"&=&=ABBF#%:Q%>AqrrE"%:Q%>ABBE"$9A$=D!9,---r   c                 2    t          j        ||          |z  S N)r3   ones)keyr)   scaler*   s       r    ones_with_scalerB      s    8E5!!E))r   c                   \    e Zd ZU dZeed<   ej        j        dde	e
         fd            ZdS )FlaxBeitDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).rateTdeterministicc                 *   | j         dk    r|S d| j         z
  }|r|S |j        d         fd|j        dz
  z  z   }|                     d          }|t          j                            |||j                  z   }t          j	        |          }||z  |z  }|S )N        g      ?r   )r   r   droppathr(   )
rE   r)   ndimmake_rngjaxrandomuniformr*   r3   floor)	selfinputsrF   	keep_probr)   rngrandom_tensorbinary_tensoroutputs	            r    __call__zFlaxBeitDropPath.__call__   s    9M$)O	 	M\!_&q)AAE--
++C%
(:(:3eSYS_(:(`(``MIm44Mi'-7FMr   NT)r   r   r   r   float__annotations__nnmodulecompactr   boolrW   r   r   r    rD   rD      sY         bb
KKKY htn      r   rD   c                   H    e Zd ZU eed<   ej        Zej        ed<   d Zd Z	dS )FlaxBeitPatchEmbeddingsconfigr*   c           
      Z   | j         j        | _        | j         j        }| j         j        }||z  ||z  z  }||z  ||z  f}|| _        || _        t          j        | j         j        ||f||fd| j	        t          j        j                            | j         j                            | _        d S )NVALID)kernel_sizestridespaddingr*   kernel_init)ra   num_channels
image_size
patch_sizenum_patchespatch_shaper[   Convhidden_sizer*   rL   initializersnormalinitializer_range
projection)rP   ri   rj   rk   rl   s        r    setupzFlaxBeitPatchEmbeddings.setup   s     K4[+
[+
!Z/J*4LM!Z/z1IJ&&'K##Z0,*+224;3PQQ
 
 
r   c                     |j         d         }|| j        k    rt          d          |                     |          }|j         \  }}}}t	          j        ||d|f          S )Nr'   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)r)   rh   
ValueErrorrr   r3   r/   )rP   pixel_valuesrh   
embeddings
batch_size_channelss          r    rW   z FlaxBeitPatchEmbeddings.__call__   so    #)"-4,,,w   __\22
%/%5"
Aq({:
B'ABBBr   N
r   r   r   r   rZ   r3   float32r*   rs   rW   r   r   r    r`   r`      sY         {E39"""
 
 
"C C C C Cr   r`   c                   N    e Zd ZU dZeed<   ej        Zej        ed<   d Z	ddZ
dS )	FlaxBeitEmbeddingsz7Construct the CLS token, position and patch embeddings.ra   r*   c                 ,   |                      dt          j        j        dd| j        j        f          | _        | j        j        r8|                      dt          j        j        dd| j        j        f          | _        t          | j        | j
                  | _        | j        j        }| j        j        r;|                      dt          j        j        d|dz   | j        j        f          | _        t          j        | j        j                  | _        d S )N	cls_tokenr   
mask_tokenr*   position_embeddingsrE   )paramr[   ro   r1   ra   rn   r   use_mask_tokenr   r`   r*   patch_embeddingsrk    use_absolute_position_embeddingsr   Dropouthidden_dropout_probdropout)rP   rk   s     r    rs   zFlaxBeitEmbeddings.setup   s    K1FAt{OfHghh;% 	o"jjr7LqRSUYU`UlNmnnDO 74: V V V+7;7 	'+zz%r'<q+PQ/SWS^Sj>k( (D$ zt{'FGGGr   NTc                 P   |                      |          }|j        \  }}}t          j        | j        |d| j        j        f          }|                    |j                  }|et          j        | j	        ||| j        j        f          }	|	                    |j                  }	t          j
        |d          }
|d|
z
  z  |	|
z  z   }t          j        ||fd          }| j        j        r"|| j                            |j                  z   }|                     ||          }|S )Nr   r'   axisrF   )r   r)   r3   broadcast_tor   ra   rn   astyper*   r   expand_dimsconcatenater   r   r   )rP   rv   bool_masked_posrF   rw   rx   seq_lenry   
cls_tokensmask_tokensws              r    rW   zFlaxBeitEmbeddings.__call__   s#   **<88
!+!1
GQ%dnz1dkF]6^__
&&z'788
&*4?ZRVR]Ri<jkkK%,,Z-=>>Kb999A#q1u-a?J_j*%=AFFF
;7 	X#d&>&E&EjFV&W&WWJ\\*M\JJ
r   )NT)r   r   r   r   r   rZ   r3   r|   r*   rs   rW   r   r   r    r~   r~      sb         AA{E39"""
H 
H 
H     r   r~   c                   b    e Zd ZU eed<   eeef         ed<   ej        Z	ej	        ed<   d Z
d ZdS )FlaxBeitRelativePositionBiasra   r!   r*   c                     d| j         d         z  dz
  d| j         d         z  dz
  z  dz   }|                     dt          j        j        || j        j        f          | _        t          | j                   | _	        d S )Nr$   r   r   r   relative_position_bias_table)
r!   r   r[   ro   r1   ra   num_attention_headsr   r<   r;   )rP   r5   s     r    rs   z"FlaxBeitRelativePositionBias.setup   s    !"T%5a%8!81!<TEUVWEXAX[\A\ ]`a a,0JJ*O!"DK$CD-
 -
) (DDDT'U'U$$$r   c                    | j                             d          }| j        d         | j        d         z  dz   | j        d         | j        d         z  dz   df}| j        |                             |          }t	          j        |d          S )Nr'   r   r   )r$   r   r   )r;   r/   r!   r   r3   r0   )rP   indexr)   relative_position_biass       r    rW   z%FlaxBeitRelativePositionBias.__call__  s    ,44R88!!$t'7'::Q>@PQR@SVZVfghVi@ilm@moqr!%!B5!I!Q!QRW!X!X}3Y???r   N)r   r   r   r   rZ   tupleintr3   r|   r*   rs   rW   r   r   r    r   r      sp         sCx   {E39"""	V 	V 	V@ @ @ @ @r   r   c                   p    e Zd ZU eed<   eeef         ed<   ej        Z	ej	        ed<   d Z
	 dded	efd
ZdS )FlaxBeitSelfAttentionra   r!   r*   c                 (   | j         j        | j         j        z  dk    r?t          | j         d          s*t	          d| j         j         d| j         j         d          t          j        | j         j        | j        t          j        j	        
                    | j         j                            | _        t          j        | j         j        | j        t          j        j	        
                    | j         j                  d          | _        t          j        | j         j        | j        t          j        j	        
                    | j         j                            | _        | j        r!t!          | j         | j        | j        	          nd | _        d S )
Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .)r*   rg   F)r*   rg   use_biasr!   r*   )ra   rn   r   hasattrru   r[   Denser*   rL   ro   rp   rq   queryr@   valuer!   r   r   rP   s    r    rs   zFlaxBeitSelfAttention.setup  s|   ;"T[%DDIIRYK)S
 S
I <4;#: < <8< < <  
 XK#*+224;3PQQ
 
 


 8K#*+224;3PQQ	
 
 
 XK#*+224;3PQQ
 
 

 ($BRZ^Zdeeee 	###r   NTFrF   output_attentionsc                    | j         j        | j         j        z  }|                     |                              |j        d d         | j         j        |fz             }|                     |                              |j        d d         | j         j        |fz             }|                     |                              |j        d d         | j         j        |fz             }d }	|s%| j         j        dk    r| 	                    d          }	t          j        d| j                  }
| j        At          j        |                                 d          }
|
                    |j                  }
||
|                    |
j                  z   }
t!          |||
|	| j         j        d|| j        d 	  	        }t          j        d||          }|                    |j        d d         d	z             }|r||fn|f}|S )
Nr$   rH   r   r   r   T)biasdropout_rngdropout_ratebroadcast_dropoutrF   r*   	precisionz...hqk,...khd->...qhd)r'   )ra   rn   r   r   r/   r)   r   r@   attention_probs_dropout_probrK   r3   r4   r*   r   r   r   r   einsum)rP   hidden_statesr   rF   r   head_dimquery_statesvalue_states
key_statesr   attention_biasattn_weightsattn_outputoutputss                 r    rW   zFlaxBeitSelfAttention.__call__-  s	    ;*dk.MMzz-0088#t{'F&QQ
 
 zz-0088#t{'F&QQ
 
 XXm,,44#t{'F&QQ
 

  	3!IC!O!O--	22K3dj999&2 _T-H-H-J-JANNN+22<3EFFN "-+.D.K.KNL`.a.aaN4#A"'*

 

 

 j!8,UU!))+*;BQB*?%*GHH1BV;--r   NTFr   r   r   r   rZ   r   r   r3   r|   r*   rs   r^   rW   r   r   r    r   r     s         sCx   {E39"""
 
 
B qv- -IM-im- - - - - -r   r   c                   P    e Zd ZU eed<   ej        Zej        ed<   d Zdde	fdZ
dS )	FlaxBeitSelfOutputra   r*   c                     t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          j
        | j        j                  | _        d S Nrg   r*   r   r[   r   ra   rn   rL   ro   rp   rq   r*   denser   r   r   r   s    r    rs   zFlaxBeitSelfOutput.setupa  a    XK#+224;3PQQ*
 
 


 zt{'FGGGr   TrF   c                 ^    |                      |          }|                     ||          }|S Nr   r   r   rP   r   rF   s      r    rW   zFlaxBeitSelfOutput.__call__i  s.    

=11]-PPr   NrX   r   r   r   r   rZ   r3   r|   r*   rs   r^   rW   r   r   r    r   r   ]  sh         {E39"""H H H T      r   r   c                   l    e Zd ZU eed<   eeef         ed<   ej        Z	ej	        ed<   d Z
	 d
defd	ZdS )FlaxBeitAttentionra   r!   r*   c                     t          | j        | j        | j                  | _        t          | j        | j                  | _        d S )Nr   )r   ra   r!   r*   	attentionr   rV   r   s    r    rs   zFlaxBeitAttention.setupt  s=    .t{D<LTXT^___(DJGGGr   NTFr   c                     |                      ||||          }|d         }|                     ||          }|f}|r||d         fz  }|S NrF   r   r   r   r   )r   rV   )rP   r   r   rF   r   attn_outputsr   r   s           r    rW   zFlaxBeitAttention.__call__x  sl     ~~1bs & 
 
 #1okk+]kKK. 	*Q))Gr   r   r   r   r   r    r   r   o  s         sCx   {E39"""H H H
 in ae     r   r   c                   H    e Zd ZU eed<   ej        Zej        ed<   d Zd Z	dS )FlaxBeitIntermediatera   r*   c                     t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          | j        j                 | _        d S )Nr   )r[   r   ra   intermediate_sizerL   ro   rp   rq   r*   r   r   
hidden_act
activationr   s    r    rs   zFlaxBeitIntermediate.setup  sY    XK)+224;3PQQ*
 
 


 !!78r   c                 Z    |                      |          }|                     |          }|S r>   )r   r   )rP   r   s     r    rW   zFlaxBeitIntermediate.__call__  s*    

=1166r   Nr{   r   r   r    r   r     sT         {E39"""9 9 9    r   r   c                   P    e Zd ZU eed<   ej        Zej        ed<   d Zdde	fdZ
dS )	FlaxBeitOutputra   r*   c                     t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          j
        | j        j                  | _        d S r   r   r   s    r    rs   zFlaxBeitOutput.setup  r   r   TrF   c                 ^    |                      |          }|                     ||          }|S r   r   r   s      r    rW   zFlaxBeitOutput.__call__  s.    

=11]-PPr   NrX   r   r   r   r    r   r     sh         {E39"""H H H T      r   r   c                   z    e Zd ZU eed<   eeef         ed<   eed<   ej	        Z
ej
        ed<   d Z	 dd	ed
efdZdS )FlaxBeitLayerra   r!   drop_path_rater*   c                    t          | j        | j        | j                  | _        t          | j        | j                  | _        t          | j        | j                  | _        t          j
        | j        j        | j                  | _        t          | j                  | _        t          j
        | j        j        | j                  | _        | j        j        | _        | j        dk    rd|                     dt(          | j        j        | j                  | _        |                     dt(          | j        j        | j                  | _        d S d | _        d | _        d S )Nr   epsilonr*   r   r   lambda_1lambda_2)r   ra   r!   r*   r   r   intermediater   rV   r[   	LayerNormlayer_norm_epslayernorm_beforerD   r   	drop_pathlayernorm_afterlayer_scale_init_valueinit_valuesr   rB   rn   r   r   r   s    r    rs   zFlaxBeitLayer.setup  s   *4;8HPTPZ[[[0DJOOO$T[
CCC "T[5OW[Wa b b b)t/BCCC!|DK4NVZV`aaa;=a JJz?T[E\_c_oppDM JJz?T[E\_c_oppDMMM DM DMMMr   NTFrF   r   c                    |                      |                     |          |||          }|d         }| j        "| j                            |j                  |z  }|                     ||          |z   }|                     |          }|                     |          }|                     ||          }| j	        "| j	                            |j                  |z  }|                     ||          |z   }|f}|r||d         fz  }|S r   )
r   r   r   r   r*   r   r   r   rV   r   )	rP   r   r   rF   r   self_attention_outputsattention_outputlayer_outputr   s	            r    rW   zFlaxBeitLayer.__call__  s4    "&!!-00"'/	 "0 "
 "
 2!4 =$#}334D4JKKN^^ '7}UUXee ++M::((66{{<}{MM =$=//0BCClRL ~~l-~PPS``/ 	4.q133Gr   r   )r   r   r   r   rZ   r   r   rY   r3   r|   r*   rs   r^   rW   r   r   r    r   r     s         sCx   {E39"""! ! !" qv$ $IM$im$ $ $ $ $ $r   r   c            	           e Zd ZU eed<   eeef         ed<   ee         ed<   e	g e
j        f         ed<   e
j        Ze
j        ed<   d Z	 	 	 	 dd	ed
ededefdZdS )FlaxBeitLayerCollectionra   r!   drop_path_ratesr   r*   c                 \      fdt           j        j                  D              _        d S )Nc           
          g | ]K}t          j        j        j        rj        nd j        |         t          |          j                  LS )N)r!   r   namer*   )r   ra   use_relative_position_biasr!   r   strr*   ).0irP   s     r    
<listcomp>z1FlaxBeitLayerCollection.setup.<locals>.<listcomp>  sn     	
 	
 	
  040V`D,,\`#3A6VVj  	
 	
 	
r   )rangera   num_hidden_layerslayersr   s   `r    rs   zFlaxBeitLayerCollection.setup  s@    	
 	
 	
 	
 4;899	
 	
 	
r   TFrF   r   output_hidden_statesreturn_dictc                 R   |rdnd }|rdnd }t          | j                  D ]O\  }}	|r||fz  }| j        |                                 nd }
 |	||
||          }|d         }|r||d         fz  }P|r||fz  }|f}|st          d |D                       S t	          |||          S )Nr   r   r   r   c              3      K   | ]}||V  	d S r>   r   )r   vs     r    	<genexpr>z3FlaxBeitLayerCollection.__call__.<locals>.<genexpr>  s"      ==qq}}}}}==r   )last_hidden_stater   
attentions)	enumerater  r   r   r   )rP   r   rF   r   r  r  all_attentionsall_hidden_statesr   layerr   layer_outputsr   s                r    rW   z FlaxBeitLayerCollection.__call__  s     1:d"6@BBD!$+.. 	6 	6HAu# 6!m%55!FJFaFmT%@%@%B%B%Bsw"!E5]fw  M *!,M  6=#3"55 	2-!11 " 	>==G======"+;LYg
 
 
 	
r   NTFFT)r   r   r   r   rZ   r   r   listrY   r   r3   ndarrayr|   r*   rs   r^   rW   r   r   r    r   r     s         sCx   %[   $R_5555{E39"""

 

 

 #"'%* !
 !
 !
  	!

 #!
 !
 !
 !
 !
 !
 !
r   r   c            	       ~    e Zd ZU eed<   eeef         ed<   ej        Z	ej	        ed<   d Z
	 	 	 	 ddeded	ed
efdZdS )FlaxBeitEncoderra   r!   r*   c                 N   | j         j        r&t          | j         | j        | j                  | _        t          t          j        d| j         j	        | j         j
                            }t          | j         | j        || j         j        r| j        nd | j                  | _        d S )N)ra   r!   r*   r   )r!   r   r   r*   )ra   !use_shared_relative_position_biasr   r!   r*   r   r  r+   linspacer   r  r   r  )rP   r   s     r    rs   zFlaxBeitEncoder.setup(  s    ;8 	*F{0@
+ + +D'
 r{1dk.H$+Jghhii,K(+{<$4#>#>*
 
 



r   TFrF   r   r  r  c                 6    |                      |||||          S )NrF   r   r  r  )r  )rP   r   rF   r   r  r  s         r    rW   zFlaxBeitEncoder.__call__:  s.     zz'/!5#  
 
 	
r   Nr  r   r   r   r    r  r  #  s         sCx   {E39"""
 
 
* #"'%* 
 
 
  	

 #
 
 
 
 
 
 
r   r  c                   b    e Zd ZU dZeZdZdZdZe	j
        ed<   ddej        dfded	ed
ej        def fdZddej        j        dededefdZ ee                    d                    	 	 	 	 	 	 	 ddee         dej        j        dedee         dee         dee         fd            Z xZS )FlaxBeitPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    beitrv   Nmodule_classr   Tra   seedr*   _do_initc                      | j         d||d|}|d|j        |j        |j        f}t                                          ||||||           d S )N)ra   r*   r   )input_shaper   r*   r!  r   )r  ri   rh   super__init__)	rP   ra   r#  r   r*   r!  kwargsr\   	__class__s	           r    r%  z FlaxBeitPreTrainedModel.__init__V  sn     #"H&HHHHf/1BFDWXK[tSXcklllllr   rS   r#  paramsr"   c                    t          j        || j                  }t          j                            |          \  }}t          j                            |          \  }}|||d}| j                            ||d          d         }	||t          t          |	                    }	t          t          |                    }| j
        D ]}
|	|
         ||
<   t                      | _
        t          t          |                    S |	S )Nr   )r(  r   rI   F)r  r(  )r3   r1   r*   rL   rM   splitr\   initr	   r   _missing_keyssetr   r
   )rP   rS   r#  r(  rv   
params_rngr   droppath_rngrngsrandom_paramsmissing_keys              r    init_weightsz$FlaxBeitPreTrainedModel.init_weightsd  s    yDJ???"%*"2"23"7"7
K$'J$4$4[$A$A!\$,WW((|(OOPXY(-)@)@AAM!(6"2"233F#1 A A&3K&@{##!$D.00111  r   zbatch_size, sequence_lengthFr   trainr   r  r  c	           
         ||n| j         j        }||n| j         j        }||n| j         j        }t	          j        |d          }i }	|,t          j                            |          \  }}
||	d<   |
|	d<   | j	        
                    d|p| j        it	          j        |t          j                  || ||||	          S )N)r   r$   r   r   r   rI   r(  r   )r0  )ra   r   r  r  r3   r0   rL   rM   r*  r\   applyr(  r4   r|   )rP   rv   r   r(  r   r4  r   r  r  r0  r/  s              r    rW   z FlaxBeitPreTrainedModel.__call__x  s     2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+BY}\<@@"(+
(8(8(E(E%K)DO+D{  v,-Il#+666I  ! 	
 	
 		
r   r>   )NNNFNNN)r   r   r   r   r   config_classbase_model_prefixmain_input_namer  r[   ModulerZ   r3   r|   r   r*   r^   r%  rL   rM   PRNGKeyr   r   r3  r   BEIT_INPUTS_DOCSTRINGformatr   dictrW   __classcell__)r'  s   @r    r  r  K  s         
 L$O"L")"""
 ;m mm 	m
 ym m m m m m m! !
 2 ! !PZ !fp ! ! ! !( +*+@+G+GHe+f+fgg !%*.,0/3&*"
 "
 	"

 Z'"
 "
 $D>"
 'tn"
 d^"
 "
 "
 hg"
 "
 "
 "
 "
r   r  c                   H    e Zd ZU eed<   ej        Zej        ed<   d Zd Z	dS )FlaxBeitPoolerra   r*   c                 v    | j         j        r,t          j        | j         j        | j                  | _        d S d S )Nr   )ra   use_mean_poolingr[   r   r   r*   	layernormr   s    r    rs   zFlaxBeitPooler.setup  s?    ;' 	`\$+2LTXT^___DNNN	` 	`r   c                     | j         j        r;|d d dd d d f         }|                     t          j        |d                    }n|d d df         }|S )Nr   r   r   )ra   rC  rD  r3   mean)rP   r   patch_tokenspooled_outputs       r    rW   zFlaxBeitPooler.__call__  sg    ;' 	0(ABB2L NN38Lq+I+I+IJJMM *!!!Q$/Mr   Nr{   r   r   r    rA  rA    sW         {E39"""` ` `	 	 	 	 	r   rA  c            	       t    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 	 dde	d	e	d
e	de	fdZdS )FlaxBeitModulera   r*   Tadd_pooling_layerc                 d   t          | j        | j                  | _        t	          | j        | j        j        j        | j                  | _        | j        j        s*t          j
        | j        j        | j                  | _        | j        rt          | j        | j                  nd | _        d S )Nr   r   r   )r~   ra   r*   rw   r  r   rl   encoderrC  r[   r   r   rD  rK  rA  poolerr   s    r    rs   zFlaxBeitModule.setup  s    ,T[
KKK&KT_%E%QY]Yc
 
 
 {+ 	`\$+2LTXT^___DNGKG]gnT[
CCCCcgr   NFrF   r   r  r  c                 p   |                      |||          }|                     |||||          }|d         }| j        j        s|                     |          }| j        r|                     |          nd }	|s|	|f|dd          z   S ||	f|dd          z   S t          ||	|j        |j	                  S )Nr   r  r   r   )r  pooler_outputr   r  )
rw   rM  ra   rC  rD  rK  rN  r   r   r  )
rP   rv   r   rF   r   r  r  r   r   pooleds
             r    rW   zFlaxBeitModule.__call__  s     oUbcc,,'/!5#  
 
  
{+ 	: NN=99M/3/EO]+++4 	9~%''!""+55!6*WQRR[88-+ !/)	
 
 
 	
r   )NTFFT)r   r   r   r   rZ   r3   r|   r*   rK  r^   rs   rW   r   r   r    rJ  rJ    s         {E39""""t"""h h h ""'%* "
 "
 	"

  "
 #"
 "
 "
 "
 "
 "
 "
r   rJ  z^The bare Beit Model transformer outputting raw hidden-states without any specific head on top.c                       e Zd ZeZdS )FlaxBeitModelN)r   r   r   rJ  r  r   r   r    rS  rS    s        
 "LLLr   rS  a  
    Returns:

    Examples:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxBeitModel
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k")
    >>> model = FlaxBeitModel.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> last_hidden_states = outputs.last_hidden_state
    ```
)output_typer7  c                   \    e Zd ZU eed<   ej        Zej        ed<   d Z	 	 	 	 	 	 dde	fdZ
dS )	$FlaxBeitForMaskedImageModelingModulera   r*   c                 J   t          | j        d| j                  | _        t	          j        | j        j        | j                  | _        t	          j        | j        j	        t          j        j                            | j        j                  | j                  | _        d S )NF)rK  r*   r   r   )rJ  ra   r*   r  r[   r   r   rD  r   
vocab_sizerL   ro   rp   rq   lm_headr   s    r    rs   z*FlaxBeitForMaskedImageModelingModule.setup  s    "4;%tzZZZ	 dk.HPTPZ[[[xK"+224;3PQQ*
 
 
r   NTrF   c                 0   ||n| j         j        }|                     ||||||          }|d         }|                     |          }|                     |d d dd f                   }	|s|	f|dd          z   }
|
S t          |	|j        |j                  S )Nr  r   r   r$   logitsr   r  )ra   use_return_dictr  rD  rY  r   r   r  )rP   rv   r   rF   r   r  r  r   sequence_outputprediction_scoresrV   s              r    rW   z-FlaxBeitForMaskedImageModelingModule.__call__  s     &1%<kk$+B]))'/!5#  
 
 "!*..99 LLABB)?@@ 	')GABBK7FM!$!/)
 
 
 	
r   NNTNNNr   r   r   r    rV  rV    s|         {E39"""	
 	
 	
 "! 
  
 	 
  
  
  
  
  
r   rV  zYBeit Model transformer with a 'language' modeling head on top (to predict visual tokens).c                       e Zd ZeZdS )FlaxBeitForMaskedImageModelingN)r   r   r   rV  r  r   r   r    rb  rb  9  s        
 8LLLr   rb  a?  
    bool_masked_pos (`numpy.ndarray` of shape `(batch_size, num_patches)`):
        Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

    Returns:

    Examples:

    ```python
    >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
    >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> logits = outputs.logits
    ```
c                   \    e Zd ZU eed<   ej        Zej        ed<   d Z	 	 	 	 	 	 dde	fdZ
dS )	$FlaxBeitForImageClassificationModulera   r*   c                     t          | j        | j        d          | _        t	          j        | j        j        t          j        j        	                    | j        j
                  | j                  | _        d S )NT)ra   r*   rK  r   )rJ  ra   r*   r  r[   r   
num_labelsrL   ro   rp   rq   
classifierr   s    r    rs   z*FlaxBeitForImageClassificationModule.setupd  sa    "$+TZ[_```	(K"+224;3PQQ*
 
 
r   NTrF   c                     ||n| j         j        }|                     |||||          }|d         }|                     |          }	|s|	f|dd          z   }
|
S t	          |	|j        |j                  S )Nr  r   r$   r[  )ra   r]  r  rg  r   r   r  )rP   rv   r   rF   r   r  r  r   rH  r\  rV   s              r    rW   z-FlaxBeitForImageClassificationModule.__call__l  s     &1%<kk$+B]))'/!5#  
 
  
// 	Y,FM+!/)
 
 
 	
r   r`  r   r   r   r    rd  rd  `  s|         {E39"""
 
 
 "!
 
 	
 
 
 
 
 
r   rd  z
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                       e Zd ZeZdS )FlaxBeitForImageClassificationN)r   r   r   rd  r  r   r   r    rj  rj    s         8LLLr   rj  aM  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxBeitForImageClassification
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
    >>> model = FlaxBeitForImageClassification.from_pretrained("microsoft/beit-base-patch16-224")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> logits = outputs.logits
    >>> # model predicts one of the 1000 ImageNet classes
    >>> predicted_class_idx = logits.argmax(-1).item()
    >>> print("Predicted class:", model.config.id2label[predicted_class_idx])
    ```
)rj  rb  rS  r  )Htypingr   r   flax
flax.linenlinenr[   rL   	jax.numpynumpyr3   r+   flax.core.frozen_dictr   r   r   flax.linen.attentionr   flax.traverse_utilr	   r
   modeling_flax_outputsr   r   r   r   modeling_flax_utilsr   r   r   r   utilsr   r   configuration_beitr   struct	dataclassr   BEIT_START_DOCSTRINGr<  r   r   r  r<   r|   rB   r:  rD   r`   r~   r   r   r   r   r   r   r   r   r  r  rA  rJ  rS  FLAX_BEIT_MODEL_DOCSTRINGrV  rb  FLAX_BEIT_MLM_DOCSTRINGrd  rj  FLAX_BEIT_CLASSIF_DOCSTRING__all__r   r   r    <module>r     s  " & % % % % % % %        



           > > > > > > > > > > > > > > > > ; ; ; ; ; ; ; ;                       Q P P P P P P P * * * * * *     %C   ,! F ".eCHo .#+ . . . .0 .1[ * * * *    ry   *C C C C Cbi C C C@& & & & & & & &R@ @ @ @ @29 @ @ @.R R R R RBI R R Rj       $    	   4    29   &    RY   &: : : : :BI : : :z4
 4
 4
 4
 4
bi 4
 4
 4
n%
 %
 %
 %
 %
bi %
 %
 %
PP
 P
 P
 P
 P
1 P
 P
 P
f    RY   (0
 0
 0
 0
 0
RY 0
 0
 0
f d " " " " "+ " "	 " ,  (A B B B    <Zis t t t t/
 /
 /
 /
 /
29 /
 /
 /
d _ 8 8 8 8 8%< 8 8	 8 2  79P Q Q Q    "0BQ[   
*
 *
 *
 *
 *
29 *
 *
 *
Z   8 8 8 8 8%< 8 8 8 2  79T U U U    "0L[e   
  r   