
     `i                        d dl mZmZ d dlmZ d dlZd dlmZ	 d dlZ
d dlmZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d	d
lmZmZmZmZmZmZm Z m!Z!m"Z" d	dl#m$Z$m%Z%m&Z&m'Z' d	dl(m)Z)m*Z*m+Z+ ddl,m-Z-  e+j.        e/          Z0dZ1dZ2ej3        Z3d Z4dZ5dZ6 G d dej7                  Z8 G d dej7                  Z9 G d dej7                  Z: G d dej7                  Z; G d dej7                  Z< G d dej7                  Z= G d  d!ej7                  Z> G d" d#ej7                  Z? G d$ d%ej7                  Z@ G d& d'ej7                  ZA G d( d)ej7                  ZB G d* d+ej7                  ZC G d, d-e%          ZD G d. d/ej7                  ZE e)d0e5           G d1 d2eD                      ZF e&eFe1ee2            G d3 d4ej7                  ZG e)d5e5           G d6 d7eD                      ZH e&eHe1ee2d89            G d: d;ej7                  ZI e)d<e5           G d= d>eD                      ZJ e&eJe1e!e2            G d? d@ej7                  ZK e)dAe5           G dB dCeD                      ZL e'eLe6M                    dD                      e&eLe1ee2            G dE dFej7                  ZN e)dGe5           G dH dIeD                      ZO e&eOe1e"e2            G dJ dKej7                  ZP e)dLe5           G dM dNeD                      ZQ e&eQe1e e2            G dO dPej7                  ZR e)dQe5           G dR dSeD                      ZS e&eSe1ee2           g dTZTdS )U    )CallableOptionalN)
FrozenDictfreezeunfreeze)combine_masksmake_causal_mask)partitioning)dot_product_attention_weights)flatten_dictunflatten_dict)lax   )	-FlaxBaseModelOutputWithPastAndCrossAttentionsFlaxBaseModelOutputWithPooling0FlaxBaseModelOutputWithPoolingAndCrossAttentions%FlaxCausalLMOutputWithCrossAttentionsFlaxMaskedLMOutputFlaxMultipleChoiceModelOutput FlaxQuestionAnsweringModelOutputFlaxSequenceClassifierOutputFlaxTokenClassifierOutput)ACT2FNFlaxPreTrainedModelappend_call_sample_docstringoverwrite_call_docstring)add_start_docstrings%add_start_docstrings_to_model_forwardlogging   )RobertaConfigzFacebookAI/roberta-baser!   c                    | |k                         d          }|j        dk    ri|                    d|j        d         f          }t	          j        |d                               d          |z  }|                    | j                  }n,t	          j        |d                               d          |z  }|                     d          |z   S )a!  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        input_ids: jnp.ndarray
        padding_idx: int

    Returns: jnp.ndarray
    i4   r    axis)astypendimreshapeshapejnpcumsum)	input_idspadding_idxmaskincremental_indicess       /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/roberta/modeling_flax_roberta.py"create_position_ids_from_input_idsr3   4   s     $,,T22Dy1}}||RB011!jA666==dCCdJ199)/JJ!jA666==dCCdJ%%d++k99    a   

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`RobertaConfig`]): Model configuration class with all the parameters of the
            model. Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`numpy.ndarray` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        head_mask (`numpy.ndarray` of shape `({0})`, `optional):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   T    e Zd ZU dZeed<   ej        Zej        ed<   d Z	d	de
fdZdS )
FlaxRobertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.configdtypec                    t          j        | j        j        | j        j        t
          j         j                            | j        j                  | j	                  | _
        t          j        | j        j        | j        j        t
          j         j                            | j        j                  | j	                  | _        t          j        | j        j        | j        j        t
          j         j                            | j        j                  | j	                  | _        t          j        | j        j        | j	                  | _        t          j        | j        j                  | _        d S )N)stddev)embedding_initr8   epsilonr8   rate)nnEmbedr7   
vocab_sizehidden_sizejaxinitializersnormalinitializer_ranger8   word_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutselfs    r2   setupzFlaxRobertaEmbeddings.setup   s(   !xK"K#6.55T[=Z5[[*	 
  
  
 $&8K/K#6.55T[=Z5[[*	$
 $
 $
  &(XK'K#6.55T[=Z5[[*	&
 &
 &
" dk.HPTPZ[[[zt{'FGGGr4   Tdeterministicc                 ^   |                      |                    d                    }|                     |                    d                    }|                     |                    d                    }||z   |z   }	|                     |	          }	|                     |	|          }	|	S )Nr#   rU   )rH   r(   rJ   rL   rM   rQ   )
rS   r.   token_type_idsposition_idsattention_maskrU   inputs_embedsposition_embedsrL   hidden_statess
             r2   __call__zFlaxRobertaEmbeddings.__call__   s    ,,Y-=-=d-C-CDD22<3F3Ft3L3LMM $ : :>;P;PQU;V;V W W &(==O }55]-PPr4   NT)__name__
__module____qualname____doc__r!   __annotations__r,   float32r8   rT   boolr^    r4   r2   r6   r6      so         QQ{E39"""H H H, _c      r4   r6   c                       e Zd ZU eed<   dZeed<   ej        Z	ej	        ed<   d Z
d Zd Zej        d             Z	 	 	 	 ddeej                 dedefdZd	S )FlaxRobertaSelfAttentionr7   Fcausalr8   c                 "   | j         j        | j         j        z  | _        | j         j        | j         j        z  dk    rt	          d          t          j        | j         j        | j        t          j        j	        
                    | j         j                            | _        t          j        | j         j        | j        t          j        j	        
                    | j         j                            | _        t          j        | j         j        | j        t          j        j	        
                    | j         j                            | _        | j        r8t!          t#          j        d| j         j        fd          d          | _        d S d S )Nr   z`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`                    : {self.config.num_attention_heads}r8   kernel_initr    rf   r8   )r7   rC   num_attention_headshead_dim
ValueErrorr@   Denser8   rD   rE   rF   rG   querykeyvaluerj   r	   r,   onesrI   causal_maskrR   s    r2   rT   zFlaxRobertaSelfAttention.setup   sb   /4;3RR;"T[%DDIII  
 XK#*+224;3PQQ
 
 


 8K#*+224;3PQQ
 
 

 XK#*+224;3PQQ
 
 

 ; 	/!T[@APPPX^     D	 	r4   c                 n    |                     |j        d d         | j        j        | j        fz             S Nr$   )r*   r+   r7   ro   rp   rS   r]   s     r2   _split_headsz%FlaxRobertaSelfAttention._split_heads   s5    $$]%8!%<@_aean?o%opppr4   c                 b    |                     |j        d d         | j        j        fz             S ry   )r*   r+   r7   rC   rz   s     r2   _merge_headsz%FlaxRobertaSelfAttention._merge_heads   s/    $$]%8!%<@W?Y%YZZZr4   c                    |                      dd          }|                     ddt          j        |j        |j                  }|                     ddt          j        |j        |j                  }|                     ddd           }|r|j        j        ^ }	}
}}|j        }dt          |	          z  |ddfz   }t          j	        |j        ||          }t          j	        |j        ||          }||_        ||_        |j        d         }|j        |z   |_        t          j
        t          j        |
          ||z   k     t          |	          d||
fz             }t          ||          }|||fS )	a\  
        This function takes projected key, value states from a single input token and concatenates the states to cached
        states from previous steps. This function is slightly adapted from the official Flax repository:
        https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252
        cache
cached_keycached_valuecache_indexc                  B    t          j        dt           j                  S )Nr   rn   )r,   arrayint32rg   r4   r2   <lambda>z@FlaxRobertaSelfAttention._concatenate_to_cache.<locals>.<lambda>   s    CIaWZW`DaDaDa r4   )r   r   r    )has_variablevariabler,   zerosr+   r8   ru   lenr   dynamic_update_slicebroadcast_toarangetupler   )rS   rt   ru   rs   rZ   is_initializedr   r   r   
batch_dims
max_length	num_headsdepth_per_head	cur_indexindicesnum_updated_cache_vectorspad_masks                    r2   _concatenate_to_cachez.FlaxRobertaSelfAttention._concatenate_to_cache   sp    **7LAA]]7L#)SYPSPYZZ
}}WnciV[VabbmmG]<a<abb 	EAKAQAW>ZY#)IS__,	1a/@@G*:+;S'JJC,\-?PPE"J!&L(-A% + 14M MK'
:&&5N)NNj!!Q(A:$NN H +8^DDNE>))r4   NTkey_value_states
init_cacheoutput_attentionsc                    |d u}|j         d         }	|                     |          }
|r+|                     |          }|                     |          }n*|                     |          }|                     |          }|                     |
          }
|                     |          }|                     |          }| j        r|
j         d         |j         d         }}|                     dd          rU| j        d         d         }| j        d         d         j         d         }t          j	        | j
        dd|dfdd||f          }n| j
        d d d d d |d |f         }t          j        ||	f|j         dd          z             }|F| j        r?t          j        t          j        |d          |j                   }t          ||          }n"| j        r|}n|t          j        |d          }| j        r4|                     dd          s|r|                     |||
|          \  }}}|t          j        |dk    t          j        |j         d                              | j                  t          j        |j         t          j        | j                  j                                      | j                            }nd }d }|s%| j        j        dk    r|                     d	          }t3          |
|||| j        j        d
|| j        d 	  	        }|t          j        d||          }t          j        d||          }|                    |j         d d         dz             }|r||fn|f}|S )Nr   r    r   r   r   )r&   g        rQ   T)biasdropout_rngdropout_ratebroadcast_dropoutrU   r8   	precisionz...hqk,h->...hqkz...hqk,...khd->...qhdr$   )r%   )r+   rs   rt   ru   r{   rj   r   	variablesr   dynamic_slicerw   r,   r   expand_dimsr   r   selectfullr(   r8   finfominr7   attention_probs_dropout_probmake_rngr   einsumr*   )rS   r]   rZ   layer_head_maskr   r   rU   r   is_cross_attention
batch_sizequery_states
key_statesvalue_statesquery_length
key_length
mask_shiftmax_decoder_lengthrw   attention_biasr   attn_weightsattn_outputoutputss                          r2   r^   z!FlaxRobertaSelfAttention.__call__   s    .T9"(+
 zz-00 	5"233J::&677LL -00J::m44L((66&&z22
((66 ; 
	_'3'9!'<j>Nq>Q*L  ,77 Q!^G4]C
%)^G%<\%J%PQR%S"!/$q!Z&;aLRd=e  #.qqq!!!]l]KZK/OP*;HYZ[Z\Z\H]8]^^K %$+% -conS[.\.\.\^i^oppN*>;GGNN[ 	L(NN' _^(KKKN ; 	D--g|DD 	
 	7;7Q7QL,8 84Jn
 % Z"-s33::4:FF-sy/D/D/HIIPPQUQ[\\ NN "N 	3!IC!O!O--	22K4#A"'*

 

 

 &:&8,XXLj!8,UU!))+*;BQB*?%*GHH1BV;--r4   NFTF)r`   ra   rb   r!   rd   rj   rf   r,   re   r8   rT   r{   r}   r@   compactr   r   ndarrayr^   rg   r4   r2   ri   ri      s         FD{E39"""  :q q q[ [ [ Z* * Z*H 37 "'_ _
 #3;/_ _  _ _ _ _ _ _r4   ri   c                   P    e Zd ZU eed<   ej        Zej        ed<   d Zdde	fdZ
dS )	FlaxRobertaSelfOutputr7   r8   c                 P   t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          j
        | j        j        | j                  | _
        t          j        | j        j                  | _        d S )Nrm   r8   r<   r>   )r@   rr   r7   rC   rD   rE   rF   rG   r8   denserM   rN   rO   rP   rQ   rR   s    r2   rT   zFlaxRobertaSelfOutput.setupf  s    XK#+224;3PQQ*
 
 


 dk.HPTPZ[[[zt{'FGGGr4   TrU   c                     |                      |          }|                     ||          }|                     ||z             }|S NrW   r   rQ   rM   )rS   r]   input_tensorrU   s       r2   r^   zFlaxRobertaSelfOutput.__call__o  sD    

=11]-PP}|'CDDr4   Nr_   r`   ra   rb   r!   rd   r,   re   r8   rT   rf   r^   rg   r4   r2   r   r   b  sh         {E39"""H H H 4      r4   r   c                   f    e Zd ZU eed<   dZeed<   ej        Z	ej	        ed<   d Z
	 	 	 	 d
defd	ZdS )FlaxRobertaAttentionr7   Frj   r8   c                     t          | j        | j        | j                  | _        t          | j        | j                  | _        d S )Nrj   r8   rn   )ri   r7   rj   r8   rS   r   outputrR   s    r2   rT   zFlaxRobertaAttention.setup|  s<    ,T[TXT^___	+DKtzJJJr4   NTr   c           	          |                      |||||||          }|d         }	|                     |	||          }|f}
|r|
|d         fz  }
|
S )N)r   r   r   rU   r   r   rW   r    )rS   r   )rS   r]   rZ   r   r   r   rU   r   attn_outputsr   r   s              r2   r^   zFlaxRobertaAttention.__call__  sy     yy+-!'/ ! 
 
 #1oKm\\ " 	*Q))Gr4   r   )r`   ra   rb   r!   rd   rj   rf   r,   re   r8   rT   r^   rg   r4   r2   r   r   w  s         FD{E39"""K K K "'        r4   r   c                   H    e Zd ZU eed<   ej        Zej        ed<   d Zd Z	dS )FlaxRobertaIntermediater7   r8   c                     t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          | j        j                 | _        d S Nr   )r@   rr   r7   intermediate_sizerD   rE   rF   rG   r8   r   r   
hidden_act
activationrR   s    r2   rT   zFlaxRobertaIntermediate.setup  sY    XK)+224;3PQQ*
 
 


 !!78r4   c                 Z    |                      |          }|                     |          }|S N)r   r   rz   s     r2   r^   z FlaxRobertaIntermediate.__call__  s*    

=1166r4   N
r`   ra   rb   r!   rd   r,   re   r8   rT   r^   rg   r4   r2   r   r     sT         {E39"""9 9 9    r4   r   c                   P    e Zd ZU eed<   ej        Zej        ed<   d Zdde	fdZ
dS )	FlaxRobertaOutputr7   r8   c                 P   t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        t          j
        | j        j                  | _        t          j        | j        j        | j                  | _        d S )Nr   r>   r<   )r@   rr   r7   rC   rD   rE   rF   rG   r8   r   rO   rP   rQ   rM   rN   rR   s    r2   rT   zFlaxRobertaOutput.setup  s    XK#+224;3PQQ*
 
 


 zt{'FGGGdk.HPTPZ[[[r4   TrU   c                     |                      |          }|                     ||          }|                     ||z             }|S r   r   )rS   r]   attention_outputrU   s       r2   r^   zFlaxRobertaOutput.__call__  sE    

=11]-PP}7G'GHHr4   Nr_   r   rg   r4   r2   r   r     sh         {E39"""\ \ \ t      r4   r   c                       e Zd ZU eed<   ej        Zej        ed<   d Z	 	 	 	 	 dde	ej
                 de	ej
                 d	ed
edef
dZdS )FlaxRobertaLayerr7   r8   c                 :   t          | j        | j        j        | j                  | _        t          | j        | j                  | _        t          | j        | j                  | _        | j        j	        r#t          | j        d| j                  | _
        d S d S )Nr   rn   F)r   r7   
is_decoderr8   	attentionr   intermediater   r   add_cross_attentioncrossattentionrR   s    r2   rT   zFlaxRobertaLayer.setup  s    -dk$+BX`d`jkkk3DKtzRRR'4:FFF;* 	d"6t{5X\Xb"c"c"cD	d 	dr4   NFTencoder_hidden_statesencoder_attention_maskr   rU   r   c	                 .   |                      ||||||          }	|	d         }
|#|                     |
|||||          }|d         }
|                     |
          }|                     ||
|          }|f}|r||	d         fz  }|||d         fz  }|S )N)r   r   rU   r   r   )rZ   r   r   rU   r   rW   r    )r   r   r   r   )rS   r]   rZ   r   r   r   r   rU   r   attention_outputsr   cross_attention_outputsr   s                r2   r^   zFlaxRobertaLayer.__call__  s     !NN+!'/ + 
 
 -Q/ !,&*&9&9 5 /!6+"3 ': ' '#  7q9))*:;;M3CS`aa " 	9)!,..G$03A688r4   )NNFTF)r`   ra   rb   r!   rd   r,   re   r8   rT   r   r   rf   r^   rg   r4   r2   r   r     s         {E39"""d d d 8<8< ""'+ +
  (4+ !) 5+ + +  + + + + + +r4   r   c                       e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 	 	 	 ddeej                 d	eej                 d
e	de	de	de	de	fdZdS )FlaxRobertaLayerCollectionr7   r8   Fgradient_checkpointingc                       j         rCt          t          d           fdt           j        j                  D              _        d S  fdt           j        j                  D              _        d S )N)         )static_argnumsc                 Z    g | ]'} j         t          |          j                   (S )namer8   )r7   strr8   ).0iFlaxRobertaCheckpointLayerrS   s     r2   
<listcomp>z4FlaxRobertaLayerCollection.setup.<locals>.<listcomp>  sE        +*4;SVV4:VVV  r4   c                 b    g | ]+}t          j        t          |          j                   ,S r   )r   r7   r   r8   )r   r   rS   s     r2   r   z4FlaxRobertaLayerCollection.setup.<locals>.<listcomp>  sB        !3q66LLL  r4   )r   rematr   ranger7   num_hidden_layerslayers)rS   r   s   `@r2   rT   z FlaxRobertaLayerCollection.setup	  s    & 
	)./?PY)Z)Z)Z&    t{<==  DKKK
   t{<==  DKKKr4   NTr   r   r   rU   r   output_hidden_statesreturn_dictc                    |rdnd }|	rdnd }|r|dnd }|V|j         d         t          | j                  k    r3t          dt          | j                   d|j         d          d          t	          | j                  D ]M\  }}|	r||fz  } ||||||         nd |||||          }|d         }|r||d         fz  }|||d         fz  }N|	r||fz  }||||f}|
st          d |D                       S t          ||||	          S )
Nrg   r   z&The head_mask should be specified for z/ layers, but it is for                         .r    r$   c              3      K   | ]}||V  	d S r   rg   )r   vs     r2   	<genexpr>z6FlaxRobertaLayerCollection.__call__.<locals>.<genexpr>L  s"      ==qq}}}}}==r4   )last_hidden_stater]   
attentionscross_attentions)r+   r   r   rq   	enumerater   r   )rS   r]   rZ   	head_maskr   r   r   rU   r   r  r  all_attentionsall_hidden_statesall_cross_attentionsr   layerlayer_outputsr   s                     r2   r^   z#FlaxRobertaLayerCollection.__call__  s     1:d"6@BBD&7h<Q<]rrdh  q!c$+&6&677 4S=M=M 4 4'oa04 4 4  
 "$+.. 	@ 	@HAu# 6!m%55!!E ) 5	!4%&!	 	M *!,M  @=#3"55(4(]1-=,??( 	2-!11 "3^EYZ 	>==G======<++%1	
 
 
 	
r4   NNFTFFTr`   ra   rb   r!   rd   r,   re   r8   r   rf   rT   r   r   r^   rg   r4   r2   r   r     s         {E39"""#(D(((  $ 8<8< ""'%* =
 =

  (4=
 !) 5=
 =
 =
  =
 #=
 =
 =
 =
 =
 =
 =
r4   r   c                       e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 	 	 	 ddeej                 d	eej                 d
e	de	de	de	de	fdZdS )FlaxRobertaEncoderr7   r8   Fr   c                 R    t          | j        | j        | j                  | _        d S )Nr8   r   )r   r7   r8   r   r  rR   s    r2   rT   zFlaxRobertaEncoder.setup\  s,    /K*#'#>
 
 



r4   NTr   r   r   rU   r   r  r  c                 @    |                      |||||||||	|

  
        S )N)r  r   r   r   rU   r   r  r  )r  )rS   r]   rZ   r  r   r   r   rU   r   r  r  s              r2   r^   zFlaxRobertaEncoder.__call__c  s=     zz"7#9!'/!5#  
 
 	
r4   r  r  rg   r4   r2   r  r  W  s         {E39"""#(D(((
 
 
 8<8< ""'%* 
 

  (4
 !) 5
 
 
  
 #
 
 
 
 
 
 
r4   r  c                   H    e Zd ZU eed<   ej        Zej        ed<   d Zd Z	dS )FlaxRobertaPoolerr7   r8   c                     t          j        | j        j        t          j         j                            | j        j                  | j                  | _	        d S r   )
r@   rr   r7   rC   rD   rE   rF   rG   r8   r   rR   s    r2   rT   zFlaxRobertaPooler.setup  sF    XK#+224;3PQQ*
 
 



r4   c                 l    |d d df         }|                      |          }t          j        |          S )Nr   )r   r@   tanh)rS   r]   cls_hidden_states      r2   r^   zFlaxRobertaPooler.__call__  s9    (A.::&677w'(((r4   Nr   rg   r4   r2   r  r    sT         {E39"""
 
 
) ) ) ) )r4   r  c                       e Zd ZU eed<   ej        Zej        ed<   ej	        j
        j        Zedej        f         ed<   d ZddZdS )	FlaxRobertaLMHeadr7   r8   .	bias_initc                    t          j        | j        j        | j        t
          j         j                            | j        j                            | _	        t          j
        | j        j        | j                  | _        t          j        | j        j        | j        dt
          j         j                            | j        j                            | _        |                     d| j        | j        j        f          | _        d S )Nrl   r<   F)r8   use_biasrm   r   )r@   rr   r7   rC   r8   rD   rE   rF   rG   r   rM   rN   
layer_normrB   decoderparamr!  r   rR   s    r2   rT   zFlaxRobertaLMHead.setup  s    XK#*+224;3PQQ
 
 


 ,t{/IQUQ[\\\xK"*+224;3PQQ	
 
 
 JJvt~8N7PQQ			r4   Nc                 D   |                      |          }t          d         |          }|                     |          }|%| j                            dd|j        ii|          }n|                     |          }t          j        | j        | j	                  }||z  }|S )Ngeluparamskernel)
r   r   r$  r%  applyTr,   asarrayr   r8   )rS   r]   shared_embeddingr   s       r2   r^   zFlaxRobertaLMHead.__call__  s    

=11v}5566' L..8EUEW:X/Y[hiiMM LL77M{49dj11r4   r   )r`   ra   rb   r!   rd   r,   re   r8   rD   r@   rE   r   r!  r   npr   rT   r^   rg   r4   r2   r   r     s         {E39"""+.6+>+DIxRZ(DDDR R R     r4   r   c                   J    e Zd ZU eed<   ej        Zej        ed<   d ZddZ	dS )FlaxRobertaClassificationHeadr7   r8   c                    t          j        | j        j        | j        t
          j         j                            | j        j                            | _	        | j        j
        | j        j
        n| j        j        }t          j        |          | _        t          j        | j        j        | j        t
          j         j                            | j        j                            | _        d S )Nrl   r>   )r@   rr   r7   rC   r8   rD   rE   rF   rG   r   classifier_dropoutrP   rO   rQ   
num_labelsout_projrS   r3  s     r2   rT   z#FlaxRobertaClassificationHead.setup  s    XK#*+224;3PQQ
 
 

 {-9 K**0 	
 z'9:::K"*+224;3PQQ
 
 
r4   Tc                     |d d dd d f         }|                      ||          }|                     |          }t          j        |          }|                      ||          }|                     |          }|S )Nr   rW   )rQ   r   r@   r  r5  )rS   r]   rU   s      r2   r^   z&FlaxRobertaClassificationHead.__call__  s{    %aaaAAAg.]-PP

=11..]-PPm44r4   Nr_   r   rg   r4   r2   r1  r1    sY         {E39"""
 
 
$     r4   r1  c                       e Zd ZU dZeZdZdZej	        e
d<   ddej        ddfd	ed
ededej        dedef fdZd Zddej        j        d
ededefdZd Z ee                    d                    	 	 	 	 	 	 	 	 	 	 	 	 	 ddee         dej        j        dedee         dee         dee         dee         fd            Z xZS ) FlaxRobertaPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    robertaNmodule_class)r    r    r   TFr7   input_shapeseedr8   _do_initr   c                 x     | j         d|||d|}t                                          ||||||           d S )Nr7   r8   r   )r<  r=  r8   r>  rg   )r;  super__init__)
rS   r7   r<  r=  r8   r>  r   kwargsmodule	__class__s
            r2   rB  z#FlaxRobertaPreTrainedModel.__init__  sU     #"w&Vlwwpvww[tSXcklllllr4   c                 T    |                      | j        | j        d          | _        d S )NTr@  )r;  r7   r8   _modulerR   s    r2   enable_gradient_checkpointingz8FlaxRobertaPreTrainedModel.enable_gradient_checkpointing  s/    ((;*#' ) 
 
r4   rngr)  returnc                 N   t          j        |d          }t          j        |          }t          || j        j                  }t          j        |          }t          j        | j        j        | j        j        f          }t          j
                            |          \  }	}
|	|
d}| j        j        rHt          j        || j        j        fz             }|}| j                            ||||||||d	  	        }n!| j                            ||||||d          }|d         }||t!          t#          |                    }t!          t#          |                    }| j        D ]}||         ||<   t'                      | _        t)          t+          |                    S |S )Nr#   rn   )r)  rQ   F)r  r)  )r,   r   	ones_liker3   r7   pad_token_idrv   r   ro   rD   randomsplitr   rC   rD  initr   r   _missing_keyssetr   r   )rS   rI  r<  r)  r.   rX   rY   rZ   r  
params_rngr   rngsr   r   module_init_outputsrandom_paramsmissing_keys                    r2   init_weightsz'FlaxRobertaPreTrainedModel.init_weights  s   Ik666	y119)T[E]^^y11Hdk;T[=\]^^	"%*"2"23"7"7
K$==;* 	$'IkT[=T<V.V$W$W!%3""&+"2"2%&! #3 
# 
# #'+"2"2iyfk #3 # # ,H5(-)@)@AAM!(6"2"233F#1 A A&3K&@{##!$D.00111  r4   c                    t          j        ||fd          }t          j        |d          }t          j        t          j        t          j        |          j        d                   |j                  }| j                            t          j
                            d          |||dd          }t          |d                   S )	aW  
        Args:
            batch_size (`int`):
                batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache.
            max_length (`int`):
                maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
                cache.
        r#   rn   r%   r   FT)r  r   r   )r,   rv   rL  r   r   
atleast_2dr+   rD  rP  rD   rN  PRNGKeyr   )rS   r   r   r.   rZ   rY   init_variabless          r2   r   z%FlaxRobertaPreTrainedModel.init_cache  s     Hj*5TBBB	y==='
3>)3L3L3RSU3V(W(WYbYhii))Jq!!9nlX]jn * 
 
 w/000r4   zbatch_size, sequence_lengthr   trainr   r  r  past_key_valuesc                    ||n| j         j        }||n| j         j        }||n| j         j        }|t	          j        |          }|t          || j         j                  }|t	          j        |          }|*t	          j	        | j         j
        | j         j        f          }i }|	|	|d<   d|p| j        i}| j         j        r|r	||d<   dg}nd}| j                            |t	          j        |d          t	          j        |d          t	          j        |d          t	          j        |d          t	          j        |d          |||
 |||||          }|!|r|\  }}t#          |d                   |d<   |S |3|s1|\  }}|d d	         t#          |d                   fz   |d	d          z   }n| j                            |t	          j        |d          t	          j        |d          t	          j        |d          t	          j        |d          t	          j        |d          |
 ||||
          }|S )NrQ   r)  r   Fr#   rn   )rX   rY   r  r   r   rU   r   r  r  rT  mutabler^  r    )rX   rY   r  rU   r   r  r  rT  )r7   r   r  r  r,   
zeros_liker3   rM  rL  rv   r   ro   r)  r   rD  r+  r   r   )rS   r.   rZ   rX   rY   r  r   r   r)  r   r]  r   r  r  r^  rT  inputsr`  r   s                      r2   r^   z#FlaxRobertaPreTrainedModel.__call__1  s   $ 2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+BY ! ^I66N=iIabbL! ]955N$+"?A`!abbI ")DOF1dk2;* 1	   "1w")k''	)4000	.555"ytDDD Y|4@@@)IT:::&;'="'i"3%9' (  G$ *{*+2(-5og6N-O-O)* ,[,+2(!"1"+/'2J)K)K(MMPWXYXZXZP[[ k''	)4000	.555"ytDDD Y|4@@@)IT:::"'i"3%9' (  G r4   r   )NNNNNNNNFNNNN) r`   ra   rb   rc   r!   config_classbase_model_prefixr;  r@   Modulerd   r,   re   r   intr8   rf   rB  rH  rD   rN  r[  r   rX  r   r   ROBERTA_INPUTS_DOCSTRINGformatr   dictr^   __classcell__)rE  s   @r2   r9  r9    s         
 !L!"L")"""
 $;',m mm m 	m
 ym m !%m m m m m m
 
 
(! (!
 2 (! (!PZ (!fp (! (! (! (!V1 1 1& +*+C+J+JKh+i+ijj "#!%*.,0/3&**.^ ^ ^ Z'^ ^ $D>^ 'tn^ d^^ "$^ ^ ^ kj^ ^ ^ ^ ^r4   r9  c                      e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   dZ
e	ed<   d Z	 	 	 	 	 	 	 	 	 	 dd	eej                 d
eej                 deej                 deej                 deej                 de	de	de	de	de	fdZdS )FlaxRobertaModuler7   r8   Tadd_pooling_layerFr   c                     t          | j        | j                  | _        t	          | j        | j        | j                  | _        t          | j        | j                  | _        d S )Nrn   r  )	r6   r7   r8   
embeddingsr  r   encoderr  poolerrR   s    r2   rT   zFlaxRobertaModule.setup  s^    /4:NNN)K*#'#>
 
 

 (4:FFFr4   NrX   rY   r  r   r   r   rU   r   r  r  c                 
   |t          j        |          }|It          j        t          j        t          j        |          j        d                   |j                  }|                     |||||	          }|                     ||||	||||
||
  
        }|d         }| j        r| 	                    |          nd }|s||f|dd          z   S ||f|dd          z   S t          |||j        |j        |j                  S )Nr%   rW   )r  rU   r   r   r   r   r  r  r   r    )r  pooler_outputr]   r	  r
  )r,   ra  r   r   rZ  r+   ro  rp  rm  rq  r   r]   r	  r
  )rS   r.   rZ   rX   rY   r  r   r   r   rU   r   r  r  r]   r   pooleds                   r2   r^   zFlaxRobertaModule.__call__  sC     ! ^I66N +CJs~i7P7P7VWY7Z,[,[]f]lmmL~|^S` ( 
 
 ,,'"7#9!/!5#  
 
  
/3/EO]+++4 	9~%''!""+55!6*WQRR[88?+ !/)$5
 
 
 	
r4   )
NNNNNFTFFT)r`   ra   rb   r!   rd   r,   re   r8   rm  rf   r   rT   r   r   r^   rg   r4   r2   rl  rl    s*        {E39""""t"""#(D(((G G G 15.2+/7;8< ""'%* 5
 5
 !-	5

 s{+5
 CK(5
  (45
 !) 55
 5
 5
  5
 #5
 5
 5
 5
 5
 5
 5
r4   rl  zaThe bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.c                       e Zd ZeZdS )FlaxRobertaModelN)r`   ra   rb   rl  r;  rg   r4   r2   rv  rv    s        
 %LLLr4   rv  c            	       r    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 dde	de	d	e	d
e	fdZdS )FlaxRobertaForMaskedLMModuler7   r8   Fr   c                     t          | j        d| j        | j                  | _        t          | j        | j                  | _        d S NF)r7   rm  r8   r   r7   r8   rl  r7   r8   r   r:  r   lm_headrR   s    r2   rT   z"FlaxRobertaForMaskedLMModule.setup  I    (;#*#'#>	
 
 
 )4:NNNr4   TrU   r   r  r  c
                 :   |                      |||||||||		  	        }
|
d         }| j        j        r%| j         j        d         d         d         d         }nd }|                     ||          }|	s|f|
dd          z   S t          ||
j        |
j        	          S )
NrU   r   r  r  r   r)  ro  rH   	embeddingr.  r    logitsr]   r	  )r:  r7   tie_word_embeddingsr   r}  r   r]   r	  )rS   r.   rZ   rX   rY   r  rU   r   r  r  r   r]   r.  r  s                 r2   r^   z%FlaxRobertaForMaskedLMModule.__call__  s     ,,'/!5#  

 

  
;* 	$#|5h?MN_`alm# m>NOO 	+9wqrr{**!!/)
 
 
 	
r4   NTFFTr`   ra   rb   r!   rd   r,   re   r8   r   rf   rT   r^   rg   r4   r2   rx  rx    s         {E39"""#(D(((O O O  #"'%* )
 )
 )
  )
 #)
 )
 )
 )
 )
 )
 )
r4   rx  z5RoBERTa Model with a `language modeling` head on top.c                       e Zd ZeZdS )FlaxRobertaForMaskedLMN)r`   ra   rb   rx  r;  rg   r4   r2   r  r     s        /LLLr4   r  z<mask>)r0   c            	       r    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 dde	de	d	e	d
e	fdZdS )*FlaxRobertaForSequenceClassificationModuler7   r8   Fr   c                     t          | j        | j        d| j                  | _        t          | j        | j                  | _        d S )NFr7   r8   rm  r   r{  )rl  r7   r8   r   r:  r1  
classifierrR   s    r2   rT   z0FlaxRobertaForSequenceClassificationModule.setup3  sK    (;*##'#>	
 
 
 8t{RVR\]]]r4   TrU   r   r  r  c
                     |                      |||||||||		  	        }
|
d         }|                     ||          }|	s|f|
dd          z   S t          ||
j        |
j                  S Nr  r   rW   r    r  )r:  r  r   r]   r	  )rS   r.   rZ   rX   rY   r  rU   r   r  r  r   sequence_outputr  s                r2   r^   z3FlaxRobertaForSequenceClassificationModule.__call__<  s     ,,'/!5#  

 

 "!*NN 	+9wqrr{**+!/)
 
 
 	
r4   Nr  r  rg   r4   r2   r  r  .  s         {E39"""#(D(((^ ^ ^  #"'%* #
 #
 #
  #
 ##
 #
 #
 #
 #
 #
 #
r4   r  z
    Roberta Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                       e Zd ZeZdS )$FlaxRobertaForSequenceClassificationN)r`   ra   rb   r  r;  rg   r4   r2   r  r  b  s         >LLLr4   r  c            	       r    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 dde	de	d	e	d
e	fdZdS )"FlaxRobertaForMultipleChoiceModuler7   r8   Fr   c                     t          | j        | j        | j                  | _        t          j        | j        j                  | _        t          j	        d| j                  | _
        d S )Nr@  r>   r    rn   )rl  r7   r8   r   r:  r@   rO   rP   rQ   rr   r  rR   s    r2   rT   z(FlaxRobertaForMultipleChoiceModule.setup{  s_    (;*#'#>
 
 

 zt{'FGGG(1DJ777r4   TrU   r   r  r  c
                 l   |j         d         }
|!|                    d|j         d                   nd }|!|                    d|j         d                   nd }|!|                    d|j         d                   nd }|!|                    d|j         d                   nd }|                     |||||||||		  	        }|d         }|                     ||          }|                     |          }|                    d|
          }|	s|f|dd          z   S t          ||j        |j                  S )Nr    r%   r  rW   r$   r  )r+   r*   r:  rQ   r  r   r]   r	  )rS   r.   rZ   rX   rY   r  rU   r   r  r  num_choicesr   pooled_outputr  reshaped_logitss                  r2   r^   z+FlaxRobertaForMultipleChoiceModule.__call__  sl     oa(BKBWI%%b)/"*=>>>]a	Q_Qk//N4H4LMMMquQ_Qk//N4H4LMMMquKWKc|++B0B20FGGGim ,,'/!5#  

 

  
]-PP// ..[99 	4#%33,"!/)
 
 
 	
r4   Nr  r  rg   r4   r2   r  r  v  s         {E39"""#(D(((8 8 8  #"'%* ,
 ,
 ,
  ,
 #,
 ,
 ,
 ,
 ,
 ,
 ,
r4   r  z
    Roberta Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                       e Zd ZeZdS )FlaxRobertaForMultipleChoiceN)r`   ra   rb   r  r;  rg   r4   r2   r  r    s         6LLLr4   r  z(batch_size, num_choices, sequence_lengthc            	       r    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 dde	de	d	e	d
e	fdZdS )'FlaxRobertaForTokenClassificationModuler7   r8   Fr   c                 $   t          | j        | j        d| j                  | _        | j        j        | j        j        n| j        j        }t          j        |          | _	        t          j
        | j        j        | j                  | _        d S )NFr  r>   rn   )rl  r7   r8   r   r:  r3  rP   r@   rO   rQ   rr   r4  r  r6  s     r2   rT   z-FlaxRobertaForTokenClassificationModule.setup  s    (;*##'#>	
 
 
 {-9 K**0 	
 z'9:::(4;#9LLLr4   TrU   r   r  r  c
                     |                      |||||||||		  	        }
|
d         }|                     ||          }|                     |          }|	s|f|
dd          z   S t          ||
j        |
j                  S r  )r:  rQ   r  r   r]   r	  )rS   r.   rZ   rX   rY   r  rU   r   r  r  r   r]   r  s                r2   r^   z0FlaxRobertaForTokenClassificationModule.__call__  s     ,,'/!5#  

 

  
]-PP// 	+9wqrr{**(!/)
 
 
 	
r4   Nr  r  rg   r4   r2   r  r    s         {E39"""#(D(((M M M, #"'%* $
 $
 $
  $
 #$
 $
 $
 $
 $
 $
 $
r4   r  z
    Roberta Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       e Zd ZeZdS )!FlaxRobertaForTokenClassificationN)r`   ra   rb   r  r;  rg   r4   r2   r  r    s         ;LLLr4   r  c            	       r    e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 dde	de	d	e	d
e	fdZdS )%FlaxRobertaForQuestionAnsweringModuler7   r8   Fr   c                     t          | j        | j        d| j                  | _        t          j        | j        j        | j                  | _        d S )NFr  rn   )	rl  r7   r8   r   r:  r@   rr   r4  
qa_outputsrR   s    r2   rT   z+FlaxRobertaForQuestionAnsweringModule.setup  sN    (;*##'#>	
 
 
 (4;#9LLLr4   TrU   r   r  r  c
                 p   |                      |||||||||		  	        }
|
d         }|                     |          }t          j        || j        j        d          \  }}|                    d          }|                    d          }|	s||f|
dd          z   S t          |||
j        |
j	                  S )Nr  r   r%   r&   r    )start_logits
end_logitsr]   r	  )
r:  r  r,   rO  r7   r4  squeezer   r]   r	  )rS   r.   rZ   rX   rY   r  rU   r   r  r  r   r]   r  r  r  s                  r2   r^   z.FlaxRobertaForQuestionAnsweringModule.__call__'  s     ,,'/!5#  

 

  
//#&9VT[5KRT#U#U#U j#++B//''++
 	< *-;;/%!!/)	
 
 
 	
r4   Nr  r  rg   r4   r2   r  r    s         {E39"""#(D(((M M M  #"'%* (
 (
 (
  (
 #(
 (
 (
 (
 (
 (
 (
r4   r  z
    Roberta Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       e Zd ZeZdS )FlaxRobertaForQuestionAnsweringN)r`   ra   rb   r  r;  rg   r4   r2   r  r  R  s         9LLLr4   r  c                       e Zd ZU eed<   ej        Zej        ed<   dZe	ed<   d Z
	 	 	 	 	 	 	 	 	 ddeej                 d	eej                 d
eej                 deej                 de	de	de	de	de	fdZdS )FlaxRobertaForCausalLMModuler7   r8   Fr   c                     t          | j        d| j        | j                  | _        t          | j        | j                  | _        d S rz  r|  rR   s    r2   rT   z"FlaxRobertaForCausalLMModule.setupj  r~  r4   NTrX   r  r   r   r   rU   r   r  r  c                 L   |                      |||||||||	|
||          }|d         }| j        j        r%| j         j        d         d         d         d         }nd }|                     ||          }|s|f|dd          z   S t          ||j        |j        |j        	          S )
N)r   r   r   rU   r   r  r  r   r)  ro  rH   r  r  r    )r  r]   r	  r
  )	r:  r7   r  r   r}  r   r]   r	  r
  )rS   r.   rZ   rY   rX   r  r   r   r   rU   r   r  r  r   r]   r.  r  s                    r2   r^   z%FlaxRobertaForCausalLMModule.__call__s  s      ,,"7#9!'/!5#  
 
  
;* 	$#|5h?MN_`alm# m>NOO 	+9wqrr{**4!/)$5	
 
 
 	
r4   )	NNNNFTFFTr  rg   r4   r2   r  r  e  s        {E39"""#(D(((O O O 15+/7;8< ""'%* 0
 0

 !-0
 CK(0
  (40
 !) 50
 0
 0
  0
 #0
 0
 0
 0
 0
 0
 0
r4   r  z
    Roberta Model with a language modeling head on top (a linear layer on top of the hidden-states output) e.g for
    autoregressive tasks.
    c                   <    e Zd ZeZddeej                 fdZd Z	dS )FlaxRobertaForCausalLMNrZ   c                 L   |j         \  }}|                     ||          }t          j        ||fd          }|0|                    d          dz
  }t          j        ||d          }n5t          j        t          j        |d          d d d f         ||f          }|||dS )Nr#   rn   r%   r&   r    )r   r   )r^  rZ   rY   )	r+   r   r,   rv   r-   r   r   r   r   )	rS   r.   r   rZ   r   
seq_lengthr^  extended_attention_maskrY   s	            r2   prepare_inputs_for_generationz4FlaxRobertaForCausalLM.prepare_inputs_for_generation  s    !*
J//*jAA #&(J
+C4"P"P"P%)00b099A=L&)&>?VXfhn&o&o##+CJz,N,N,NtUVUVUVw,WZdfpYqrrL  /5(
 
 	
r4   c                 N    |j         |d<   |d         d d dd f         dz   |d<   |S )Nr^  rY   r%   r    )r^  )rS   model_outputsmodel_kwargss      r2   update_inputs_for_generationz3FlaxRobertaForCausalLM.update_inputs_for_generation  s;    *7*G&''3N'CAAArssF'Ka'O^$r4   r   )
r`   ra   rb   r  r;  r   rD   Arrayr  r  rg   r4   r2   r  r    sT         0L
 
S[\_\eSf 
 
 
 
*    r4   r  )r  r  r  r  r  r  rv  r9  )Utypingr   r   
flax.linenlinenr@   rD   	jax.numpynumpyr,   r/  flax.core.frozen_dictr   r   r   r   r	   r
   nn_partitioningflax.linen.attentionr   flax.traverse_utilr   r   r   modeling_flax_outputsr   r   r   r   r   r   r   r   r   modeling_flax_utilsr   r   r   r   utilsr   r   r   configuration_robertar!   
get_loggerr`   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr   r3   ROBERTA_START_DOCSTRINGrg  re  r6   ri   r   r   r   r   r   r   r  r  r   r1  r9  rl  rv  rx  r  r  r  r  r  rh  r  r  r  r  r  r  __all__rg   r4   r2   <module>r     s   & % % % % % % %       



           > > > > > > > > > > 6 6 6 6 6 6 6 6 6 6 6 6 6 6 > > > > > > ; ; ; ; ; ; ; ;      
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 w v v v v v v v v v v v Y Y Y Y Y Y Y Y Y Y 0 0 0 0 0 0 
	H	%	%/ !: : :0 .# N( ( ( ( (BI ( ( (Xh h h h hry h h hX    BI   *' ' ' ' '29 ' ' 'V    bi   &    	   *6 6 6 6 6ry 6 6 6tO
 O
 O
 O
 O
 O
 O
 O
f$
 $
 $
 $
 $
 $
 $
 $
P) ) ) ) )	 ) ) )"         	      F    BI   @} } } } }!4 } } }BD
 D
 D
 D
 D
	 D
 D
 D
N g % % % % %1 % %	 %  -/BDbds t t t7
 7
 7
 7
 7
29 7
 7
 7
t QSjkk0 0 0 0 07 0 0 lk0  "	   1
 1
 1
 1
 1
 1
 1
 1
h   > > > > >+E > > >  ( 	  :
 :
 :
 :
 :
 :
 :
 :
z   6 6 6 6 6#= 6 6 6   ":"A"ABl"m"m     !	  8
 8
 8
 8
 8
bi 8
 8
 8
v   ; ; ; ; ;(B ; ; ;  %	  6
 6
 6
 6
 6
BI 6
 6
 6
r   9 9 9 9 9&@ 9 9 9  #$	  >
 >
 >
 >
 >
29 >
 >
 >
B       7   <  )	  	 	 	r4   