
     `i-                        d Z ddlZddlZddlmZmZ ddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+  e'j,        e-          Z.d Z/ G d dej0                  Z1 G d dej0                  Z2 G d dej0                  Z3 G d dej0                  Z4 G d dej0                  Z5 G d dej0                  Z6 G d d ej0                  Z7 G d! d"e          Z8 G d# d$ej0                  Z9 G d% d&ej0                  Z: G d' d(ej0                  Z; G d) d*ej0                  Z<e& G d+ d,e                       Z= e&d-.           G d/ d0e=                      Z>e& G d1 d2e=                      Z? e&d3.           G d4 d5e=e                      Z@ e&d6.           G d7 d8e=                      ZAe& G d9 d:e=                      ZBe& G d; d<e=                      ZCe& G d= d>e=                      ZDg d?ZEdS )@zPyTorch RemBERT model.    N)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging)deprecate_kwarg   )RemBertConfigc           
         	 ddl }ddl}ddl}n)# t          $ r t                              d            w xY wt          j                            |          }t          	                    d|            |j
                            |          }g }g }	|D ]\  }
t          fddD                       r!t          	                    d d|
            |j
                            |          }|                               |	                    |           t          ||	          D ]\  }                    d	d
                              d          t          d D                       r1t          	                    dd                                          {| }D ]X}|                    d|          r|                    d|          }n|g}|d         dk    s|d         dk    rt)          |d          }n|d         dk    s|d         dk    rt)          |d          }n|d         dk    rt)          |d          }n|d         dk    rt)          |d          }ni	 t)          ||d                   }nQ# t*          $ rD t          	                    d                    d                                                   Y $w xY wt/          |          dk    rt1          |d                   }||         }Z|dd         dk    rt)          |d          }n|dk    r|                    |          }	 |j        |j        k    r t7          d|j         d|j         d           n/# t8          $ r"}|xj        |j        |j        fz  c_         d}~ww xY wt          	                    d!            t=          j        |          |_         | S )"z'Load tf checkpoints in a pytorch model.r   NzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from c              3       K   | ]}|v V  	d S N ).0denynames     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/rembert/modeling_rembert.py	<genexpr>z-load_tf_weights_in_rembert.<locals>.<genexpr>G   s'      XXtt|XXXXXX    )adam_vadam_moutput_embeddingclszLoading TF weight z with shape zbert/zrembert//c              3      K   | ]}|d v V  	dS ))r+   r,   AdamWeightDecayOptimizerAdamWeightDecayOptimizer_1global_stepNr$   )r%   ns     r(   r)   z-load_tf_weights_in_rembert.<locals>.<genexpr>X   s<       
 
 nn
 
 
 
 
 
r*   z	Skipping z[A-Za-z]+_\d+z_(\d+)kernelgammaweightoutput_biasbetabiasoutput_weightssquad
classifierzSkipping {}   r   i_embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )!renumpy
tensorflowImportErrorloggererrorospathabspathinfotrainlist_variablesanyload_variableappendzipreplacesplitjoin	fullmatchgetattrAttributeErrorformatlenint	transposeshape
ValueErrorAssertionErrorargstorch
from_numpydata)modelconfigtf_checkpoint_pathr@   nptftf_path	init_varsnamesarraysrZ   arraypointerm_namescope_namesnumer'   s                    @r(   load_tf_weights_in_rembertrp   1   s\   
			   Q	
 	
 	
 	 goo011G
KKBBBCCC''00IEF  	 	e XXXX(WXXXXX 	BBB5BBCCC&&w55Te5&)) 1/ 1/e||GZ00 zz#  
 

 
 
 
 
 	 KK4CHHTNN44555 	' 	'F||,f55 ' hhy&99%h1~))[^w-F-F!'844Q=00KNf4L4L!'622Q#333!'844Q7**!'<88%g{1~>>GG%   KK 4 4SXXd^^ D DEEEH ;1$$+a.))!#,#$$<=((gx00GGxLL''E	}++ !j'-!j!jRWR]!j!j!jkkk , 	 	 	FFw}ek22FF	 	777888'..Ls3    &6+JA
KK 0M11
N;NNc                        e Zd ZdZ fdZ	 	 	 	 	 ddeej                 deej                 deej                 deej                 d	e	d
ej
        fdZ xZS )RemBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 8   t                                                       t          j        |j        |j        |j                  | _        t          j        |j        |j                  | _	        t          j        |j
        |j                  | _        t          j        |j        |j                  | _        t          j        |j                  | _        |                     dt%          j        |j                                      d          d           d S )N)padding_idxepsposition_ids)r   F)
persistent)super__init__r   	Embedding
vocab_sizeinput_embedding_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferr^   arangeexpandselfrb   	__class__s     r(   r{   zRemBertEmbeddings.__init__   s    !|v:H[ 
  
  
 $&<0NPVPk#l#l %'\&2H&Je%f%f" f&AvG\]]]z&"<== 	EL)GHHOOPWXXej 	 	
 	
 	
 	
 	
r*   Nr   	input_idstoken_type_idsrw   inputs_embedspast_key_values_lengthreturnc                    ||                                 }n|                                 d d         }|d         }|| j        d d |||z   f         }|+t          j        |t          j        | j        j                  }||                     |          }|                     |          }||z   }	|                     |          }
|	|
z  }	| 	                    |	          }	| 
                    |	          }	|	S )Nrx   r   dtypedevice)sizerw   r^   zeroslongr   r   r   r   r   r   )r   r   r   rw   r   r   input_shape
seq_lengthr   
embeddingsr   s              r(   forwardzRemBertEmbeddings.forward   s	     #..**KK',,..ss3K ^
,QQQ0FVlIl0l-lmL!"[EJtO`OghhhN  00;;M $ : :> J J"%::
"66|DD))
^^J//
\\*--
r*   )NNNNr   )__name__
__module____qualname____doc__r{   r   r^   
LongTensorFloatTensorrX   Tensorr   __classcell__r   s   @r(   rr   rr      s        QQ
 
 
 
 
( 15593759&' E,- !!12 u/0	
   12 !$ 
       r*   rr   c                   B     e Zd Z fdZdej        dej        fdZ xZS )RemBertPoolerc                     t                                                       t          j        |j        |j                  | _        t          j                    | _        d S r#   )rz   r{   r   Linearhidden_sizedenseTanh
activationr   s     r(   r{   zRemBertPooler.__init__   sC    Yv163EFF
'))r*   hidden_statesr   c                 r    |d d df         }|                      |          }|                     |          }|S )Nr   )r   r   )r   r   first_token_tensorpooled_outputs       r(   r   zRemBertPooler.forward   s@     +111a40

#56666r*   r   r   r   r{   r^   r   r   r   r   s   @r(   r   r      s^        $ $ $ $ $
U\ el        r*   r   c                        e Zd Zd fd	Z eddd          	 	 	 	 	 	 ddej        d	eej                 d
eej                 deej                 dee	         de
deej                 defd            Z xZS )RemBertSelfAttentionNc                    t                                                       |j        |j        z  dk    r0t	          |d          s t          d|j         d|j         d          |j        | _        t          |j        |j        z            | _        | j        | j        z  | _        t          j
        |j        | j                  | _        t          j
        |j        | j                  | _        t          j
        |j        | j                  | _        t          j        |j                  | _        |j        | _        || _        d S )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ())rz   r{   r   num_attention_headshasattrr[   rX   attention_head_sizeall_head_sizer   r   querykeyvaluer   attention_probs_dropout_probr   
is_decoder	layer_idxr   rb   r   r   s      r(   r{   zRemBertSelfAttention.__init__   s*    ::a??PVXhHiHi?8F$6 8 8 48 8 8  
 $*#= #&v'9F<V'V#W#W !58PPYv143EFF
9V/1CDDYv143EFF
z&"EFF +"r*   past_key_valuepast_key_valuesz4.58)new_nameversionFr   attention_mask	head_maskencoder_hidden_statesoutput_attentionscache_positionr   c                 z   |j         \  }}	}
|                     |                              |d| j        | j                                      dd          }d}|d u}|Ht          |t                    r1|j        	                    | j
                  }|r|j        }n
|j        }n|}|r|n|}|r3|1|r/|j        | j
                 j        }|j        | j
                 j        }n|                     |                              |d| j        | j                                      dd          }|                     |                              |d| j        | j                                      dd          }|N|s|nd }|                    ||| j
        d|i          \  }}|r$t          |t                    rd|j        | j
        <   t'          j        ||                    dd                    }|t+          j        | j                  z  }|||z   }t.          j                            |d          }|                     |          }|||z  }t'          j        ||          }|                    d	ddd
                                          }|                                d d         | j        fz   } |j        | }||fS )Nrx   r   r>   Fr   Tdimr   r	   )rZ   r   viewr   r   rY   
isinstancer   
is_updatedgetr   cross_attention_cacheself_attention_cachelayerskeysvaluesr   r   updater^   matmulmathsqrtr   
functionalsoftmaxr   permute
contiguousr   r   )r   r   r   r   r   r   r   r   
batch_sizer   _query_layerr   is_cross_attentioncurr_past_key_valuecurrent_states	key_layervalue_layerattention_scoresattention_probscontext_layernew_context_layer_shapes                         r(   r   zRemBertSelfAttention.forward   s    %2$7!
JJJ}%%T*b$":D<TUUYq!__ 	 
2$>&/+>?? 6,7;;DNKK
% O*9*O''*9*N''&5#2DW..- 	F/"=*"=+24>BGI-4T^DKKK ((j"d&>@XYY1a  

>**j"d&>@XYY1a  *7I!St)<)C)C{DN=M~<^* *&	; & F*_FY*Z*Z FAEO.t~> !<Y5H5HR5P5PQQ+di8P.Q.QQ%/.@ -//0@b/II ,,77  -	9O_kBB%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S**,CDo--r*   r#   NNNNFN)r   r   r   r{   r   r^   r   r   r   r   booltupler   r   r   s   @r(   r   r      s       # # # # # #* _%0A6RRR 7;15=A+/"'15R. R.|R. !!23R. E-.	R.
  ((9:R. "%R.  R. !.R. 
R. R. R. SRR. R. R. R. R.r*   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )RemBertSelfOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j        |j	                  | _
        d S Nru   )rz   r{   r   r   r   r   r   r   r   r   r   r   s     r(   r{   zRemBertSelfOutput.__init__8  sf    Yv163EFF
f&8f>STTTz&"<==r*   r   input_tensorr   c                     |                      |          }|                     |          }|                     ||z             }|S r#   r   r   r   r   r   r   s      r(   r   zRemBertSelfOutput.forward>  @    

=11]33}|'CDDr*   r   r   s   @r(   r   r   7  i        > > > > >U\  RWR^        r*   r   c                        e Zd Zd fd	Zd Z	 	 	 	 	 	 ddej        deej                 deej                 deej                 d	ee	         d
ee
         deej                 deej                 fdZ xZS )RemBertAttentionNc                     t                                                       t          ||          | _        t	          |          | _        t                      | _        d S )Nr   )rz   r{   r   r   r   outputsetpruned_headsr   s      r(   r{   zRemBertAttention.__init__F  sN    (9EEE	'//EEr*   c                    t          |          dk    rd S t          || j        j        | j        j        | j                  \  }}t          | j        j        |          | j        _        t          | j        j        |          | j        _        t          | j        j	        |          | j        _	        t          | j
        j        |d          | j
        _        | j        j        t          |          z
  | j        _        | j        j        | j        j        z  | j        _        | j                            |          | _        d S )Nr   r   r   )rW   r   r   r   r   r  r   r   r   r   r  r   r   union)r   headsindexs      r(   prune_headszRemBertAttention.prune_headsM  s    u::??F7490$)2OQUQb
 
u
 -TY_eDD	*49=%@@	,TY_eDD	.t{/@%QOOO )-	(EE

(R	%"&)"?$)B_"_	 -33E::r*   Fr   r   r   r   r   r   r   r   c           	          |                      |||||||          }|                     |d         |          }	|	f|dd          z   }
|
S )Nr   r   r   r   r   r   r   r   )r   r  )r   r   r   r   r   r   r   r   self_outputsattention_outputoutputss              r(   r   zRemBertAttention.forward`  sf     yy)"7+/) ! 
 
  ;;|AFF#%QRR(88r*   r#   r   )r   r   r   r{   r  r^   r   r   r   r   r   r   r   r   r   s   @r(   r   r   E  s        " " " " " "; ; ;, 7;15=A+/,115 | !!23 E-.	
  ((9: "% $D> !. 
u|	       r*   r   c                   B     e Zd Z fdZdej        dej        fdZ xZS )RemBertIntermediatec                    t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _        d S |j        | _        d S r#   )rz   r{   r   r   r   intermediate_sizer   r   
hidden_actstrr
   intermediate_act_fnr   s     r(   r{   zRemBertIntermediate.__init__z  sn    Yv163KLL
f'-- 	9'-f.?'@D$$$'-'8D$$$r*   r   r   c                 Z    |                      |          }|                     |          }|S r#   )r   r  r   r   s     r(   r   zRemBertIntermediate.forward  s,    

=1100??r*   r   r   s   @r(   r  r  y  s^        9 9 9 9 9U\ el        r*   r  c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )RemBertOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j	        |j
                  | _        d S r   )rz   r{   r   r   r  r   r   r   r   r   r   r   r   s     r(   r{   zRemBertOutput.__init__  sf    Yv79KLL
f&8f>STTTz&"<==r*   r   r   r   c                     |                      |          }|                     |          }|                     ||z             }|S r#   r   r   s      r(   r   zRemBertOutput.forward  r   r*   r   r   s   @r(   r  r    r   r*   r  c                       e Zd Zd fd	Z	 	 	 	 	 	 	 ddej        deej                 deej                 deej                 deej                 d	ee         d
ee	         deej                 de
ej                 fdZd Z xZS )RemBertLayerNc                    t                                                       |j        | _        d| _        t	          ||          | _        |j        | _        |j        | _        | j        r/| j        st          |  d          t	          ||          | _	        t          |          | _        t          |          | _        d S )Nr   z> should be used as a decoder model if cross attention is addedr   )rz   r{   chunk_size_feed_forwardseq_len_dimr   	attentionr   add_cross_attentionr[   crossattentionr  intermediater  r  r   s      r(   r{   zRemBertLayer.__init__  s    '-'E$)&)<< +#)#= # 	P? j D!h!h!hiii"26Y"O"O"OD/77#F++r*   Fr   r   r   r   encoder_attention_maskr   r   r   r   c	           	      h   |                      ||||||          }	|	d         }
|	dd          }| j        rV|Tt          | d          st          d|  d          |                     |
||||||          }|d         }
||dd          z   }t          | j        | j        | j        |
          }|f|z   }|S )N)r   r   r   r   r   r   r   r"  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r
  )	r   r   r   r[   r"  r   feed_forward_chunkr  r  )r   r   r   r   r   r$  r   r   r   self_attention_outputsr  r  cross_attention_outputslayer_outputs                 r(   r   zRemBertLayer.forward  s!    "&)/+) "0 "
 "
 2!4(,? 	<4@4!122  Dd D D D  
 '+&9&9 5#&; /"3- ': ' '#  7q9 7 ;;G0#T%A4CSUe
 
  /G+r*   c                 \    |                      |          }|                     ||          }|S r#   )r#  r  )r   r  intermediate_outputr)  s       r(   r&  zRemBertLayer.feed_forward_chunk  s2    "//0@AA{{#68HIIr*   r#   )NNNNNFN)r   r   r   r{   r^   r   r   r   r   r   r   r   r&  r   r   s   @r(   r  r    s       , , , , , ,$ 7;15=A>B+/,115. .|. !!23. E-.	.
  ((9:. !)): ;. "%. $D>. !.. 
u|	. . . .b      r*   r  c                       e Zd Z fdZ	 	 	 	 	 	 	 	 	 	 ddej        deej                 deej                 deej                 d	eej                 d
ee         dee	         de	de	de	deej                 de
eef         fdZ xZS )RemBertEncoderc                    t                                                       | _        t          j        j        j                  | _        t          j        fdt          j
                  D                       | _        d| _        d S )Nc                 2    g | ]}t          |           S )r   )r  )r%   irb   s     r(   
<listcomp>z+RemBertEncoder.__init__.<locals>.<listcomp>  s&    #o#o#o!L1$E$E$E#o#o#or*   F)rz   r{   rb   r   r   r~   r   embedding_hidden_mapping_in
ModuleListrangenum_hidden_layerslayergradient_checkpointingr   s    `r(   r{   zRemBertEncoder.__init__  s|    +-9V5PRXRd+e+e(]#o#o#o#ouU[UmOnOn#o#o#opp
&+###r*   NFTr   r   r   r   r$  r   	use_cacher   output_hidden_statesreturn_dictr   r   c           
         | j         r%| j        r|rt                              d           d}|r8|6t	          t          | j                  t          | j                            }|rCt          |t                    r.t                              d           t	          j	        |          }| 
                    |          }|	rdnd }|rdnd }|r| j        j        rdnd }t          | j                  D ]X\  }}|	r||fz   }|||         nd } ||||||||          }|d         }|r$||d         fz   }| j        j        r||d         fz   }Y|	r||fz   }|
st          d	 |||||fD                       S t          |||||
          S )NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rb   zPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.r$   r   r   r>   c              3      K   | ]}||V  	d S r#   r$   )r%   vs     r(   r)   z)RemBertEncoder.forward.<locals>.<genexpr>%  s4       
 
 =  !===
 
r*   )last_hidden_stater   r   
attentionscross_attentions)r7  trainingrD   warning_oncer   r   rb   r   r   from_legacy_cacher2  r!  	enumerater6  r   )r   r   r   r   r   r$  r   r8  r   r9  r:  r   all_hidden_statesall_self_attentionsall_cross_attentionsr0  layer_modulelayer_head_masklayer_outputss                      r(   r   zRemBertEncoder.forward  sS    & 	"4= 	" "##p   "	 	v01,dk2R2R2RT`hlhsTtTtTtuuO 	UOU;; 	U\  
 2COTTO88GG"6@BBD$5?bb4%6d4;;Zdrr`d(44 	V 	VOA|# I$58H$H!.7.CillO(L%&! M *!,M  V&9]1=M<O&O#;2 V+?=QRCSBU+U( 	E 1]4D D 	 
 
 "#%'(
 
 
 
 
 
 9+++*1
 
 
 	
r*   )
NNNNNNFFTN)r   r   r   r{   r^   r   r   r   r   r   r   r   r   r   r   r   s   @r(   r-  r-    s8       , , , , , 7;15=A>B+/$("'%* 15O
 O
|O
 !!23O
 E-.	O

  ((9:O
 !)): ;O
 "%O
 D>O
  O
 #O
 O
 !.O
 
u??	@O
 O
 O
 O
 O
 O
 O
 O
r*   r-  c                   B     e Zd Z fdZdej        dej        fdZ xZS )RemBertPredictionHeadTransformc                 V   t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _
        n|j        | _
        t          j        |j        |j                  | _        d S r   )rz   r{   r   r   r   r   r   r  r  r
   transform_act_fnr   r   r   s     r(   r{   z'RemBertPredictionHeadTransform.__init__;  s    Yv163EFF
f'-- 	6$*6+<$=D!!$*$5D!f&8f>STTTr*   r   r   c                     |                      |          }|                     |          }|                     |          }|S r#   )r   rN  r   r  s     r(   r   z&RemBertPredictionHeadTransform.forwardD  s=    

=11--m<<}55r*   r   r   s   @r(   rL  rL  :  sc        U U U U UU\ el        r*   rL  c                   B     e Zd Z fdZdej        dej        fdZ xZS )RemBertLMPredictionHeadc                 P   t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          |j
                 | _        t          j        |j        |j                  | _        d S r   )rz   r{   r   r   r   output_embedding_sizer   r}   decoderr
   r  r   r   r   r   s     r(   r{   z RemBertLMPredictionHead.__init__L  sz    Yv163OPP
y!=v?PQQ !23f&BH]^^^r*   r   r   c                     |                      |          }|                     |          }|                     |          }|                     |          }|S r#   )r   r   r   rT  r  s     r(   r   zRemBertLMPredictionHead.forwardS  sL    

=1166}55]33r*   r   r   s   @r(   rQ  rQ  K  sc        _ _ _ _ _U\ el        r*   rQ  c                   B     e Zd Z fdZdej        dej        fdZ xZS )RemBertOnlyMLMHeadc                 p    t                                                       t          |          | _        d S r#   )rz   r{   rQ  predictionsr   s     r(   r{   zRemBertOnlyMLMHead.__init__]  s/    26::r*   sequence_outputr   c                 0    |                      |          }|S r#   )rY  )r   rZ  prediction_scoress      r(   r   zRemBertOnlyMLMHead.forwarda  s     ,,_==  r*   r   r   s   @r(   rW  rW  \  s^        ; ; ; ; ;!u| ! ! ! ! ! ! ! ! !r*   rW  c                   ,    e Zd ZU eed<   eZdZdZd Z	dS )RemBertPreTrainedModelrb   rembertTc                    t          |t          j                  rT|j        j                            d| j        j                   |j         |j        j        	                                 dS dS t          |t          j
                  r_|j        j                            d| j        j                   |j        +|j        j        |j                 	                                 dS dS t          |t          j                  r?|j        j        	                                 |j        j                            d           dS dS )zInitialize the weightsg        )meanstdNg      ?)r   r   r   r7   r`   normal_rb   initializer_ranger:   zero_r|   rt   r   fill_)r   modules     r(   _init_weightsz$RemBertPreTrainedModel._init_weightsm  s)   fbi(( 	* M&&CT[5R&SSS{& &&((((( '&-- 	*M&&CT[5R&SSS!-"6#56<<>>>>> .--- 	*K""$$$M$$S)))))	* 	*r*   N)
r   r   r   r    __annotations__rp   load_tf_weightsbase_model_prefixsupports_gradient_checkpointingrh  r$   r*   r(   r^  r^  f  sB         0O!&*#* * * * *r*   r^  a
  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    )custom_introc            "           e Zd Zd fd	Zd Zd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j
                 dee	j
                 d	ee	j
                 d
ee	j
                 dee	j                 dee	j                 dee	j                 dee	j                 dee         dee         dee         dee         dee         dee	j                 deeef         fd            Z xZS )RemBertModelTc                     t                                          |           || _        t          |          | _        t          |          | _        |rt          |          nd| _        | 	                                 dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
rz   r{   rb   rr   r   r-  encoderr   pooler	post_init)r   rb   add_pooling_layerr   s      r(   r{   zRemBertModel.__init__  ss    
 	   +F33%f--/@JmF+++d 	r*   c                     | j         j        S r#   r   r   r   s    r(   get_input_embeddingsz!RemBertModel.get_input_embeddings  s    ..r*   c                     || j         _        d S r#   rv  )r   r   s     r(   set_input_embeddingsz!RemBertModel.set_input_embeddings  s    */'''r*   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrq  r6  r   r  )r   heads_to_pruner6  r  s       r(   _prune_headszRemBertModel._prune_heads  sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr*   Nr   r   r   rw   r   r   r   r$  r   r8  r   r9  r:  r   r   c                    ||n| j         j        }||n| j         j        }||n| j         j        }| j         j        r|
|
n| j         j        }
nd}
||t          d          |+|                     ||           |                                }n.||                                d d         }nt          d          |\  }}||j	        n|j	        }d}|	Bt          |	t                    s|	d         d         j        d         n|	                                }|t          j        |||z   f|          }|!t          j        |t          j        |          }|                     ||          }| j         j        rL|J|                                \  }}}||f}|t          j        ||          }|                     |          }nd }|                     || j         j                  }|                     |||||	          }|                     ||||||	|
||||
          }|d         }| j        |                     |          nd }|s||f|dd          z   S t3          |||j        |j        |j        |j                  S )NFzDYou cannot specify both input_ids and inputs_embeds at the same timerx   z5You have to specify either input_ids or inputs_embedsr   r   )r   r   )r   rw   r   r   r   )
r   r   r   r$  r   r8  r   r9  r:  r   r   )r>  pooler_outputr   r   r?  r@  )rb   r   r9  use_return_dictr   r8  r[   %warn_if_padding_and_no_attention_maskr   r   r   r   rZ   get_seq_lengthr^   onesr   r   get_extended_attention_maskinvert_attention_maskget_head_maskr5  r   rq  rr  r   r   r   r?  r@  )r   r   r   r   rw   r   r   r   r$  r   r8  r   r9  r:  r   r   r   r   r   r   extended_attention_maskencoder_batch_sizeencoder_sequence_lengthr   encoder_hidden_shapeencoder_extended_attention_maskembedding_outputencoder_outputsrZ  r   s                                 r(   r   zRemBertModel.forward  s)   $ 2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B];! 	%.%:		@UIII ]%>cddd"66y.QQQ#..**KK&',,..ss3KKTUUU!,
J%.%:!!@T!"& "/5996"1%+B//$3355 # !"Z*jCY6Y)ZdjkkkN!"[EJvVVVN 150P0PQ_al0m0m ;! 	3&;&G=R=W=W=Y=Y: 7$68O#P %-).4HQW)X)X)X&.2.H.HI_.`.`++.2+ &&y$+2OPP	??%)'#9 + 
 
 ,,2"7#B+/!5#) ' 
 
 *!,8<8OO444UY 	J#]3oabb6III;-'+;)7&1,=
 
 
 	
r*   )TNNNNNNNNNNNNNN)r   r   r   r{   rx  rz  r~  r   r   r^   r   r   r   r   r   r   r   r   r   r   r   s   @r(   ro  ro  ~  s             / / /0 0 0C C C  155959371559=A>B+/$(,0/3&*15m
 m
E,-m
 !!12m
 !!12	m

 u/0m
 E-.m
   12m
  ((9:m
 !)): ;m
 "%m
 D>m
 $D>m
 'tnm
 d^m
 !.m
  
uBB	C!m
 m
 m
 ^m
 m
 m
 m
 m
r*   ro  c                       e Zd ZdgZ fdZd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 ddee	j
                 dee	j
                 dee	j
                 d	ee	j
                 d
ee	j                 dee	j                 dee	j                 dee	j                 dee	j
                 dee         dee         dee         deeef         fd            ZddZedefd            Z xZS )RemBertForMaskedLMcls.predictions.decoder.weightc                    t                                          |           |j        rt                              d           t          |d          | _        t          |          | _        | 	                                 d S )NznIf you want to use `RemBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Frt  
rz   r{   r   rD   warningro  r_  rW  r.   rs  r   s     r(   r{   zRemBertForMaskedLM.__init__  s~        	NN1  
 $FeDDD%f-- 	r*   c                 $    | j         j        j        S r#   r.   rY  rT  rw  s    r(   get_output_embeddingsz(RemBertForMaskedLM.get_output_embeddings-      x#++r*   c                 (    || j         j        _        d S r#   r  r   new_embeddingss     r(   set_output_embeddingsz(RemBertForMaskedLM.set_output_embeddings0      '5$$$r*   Nr   r   r   rw   r   r   r   r$  labelsr   r9  r:  r   c                    ||n| j         j        }|                     |||||||||
||          }|d         }|                     |          }d}|	Kt	                      } ||                    d| j         j                  |	                    d                    }|s|f|dd         z   }||f|z   n|S t          |||j        |j	                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)
r   r   rw   r   r   r   r$  r   r9  r:  r   rx   r>   losslogitsr   r?  )
rb   r  r_  r.   r   r   r}   r   r   r?  )r   r   r   r   rw   r   r   r   r$  r  r   r9  r:  r  rZ  r\  masked_lm_lossloss_fctr  s                      r(   r   zRemBertForMaskedLM.forward3  s   , &1%<kk$+B],,))%'"7#9/!5#  
 
 "!* HH_55'))H%X&7&<&<RAW&X&XZ`ZeZefhZiZijjN 	Z')GABBK7F3A3M^%..SYY$!/)	
 
 
 	
r*   c                 \   |j         }|d         }| j        j        
J d            t          j        ||                    |j         d         df          gd          }t          j        |df| j        j        t          j        |j                  }t          j        ||gd          }||dS )Nr   z.The PAD token should be defined for generationr   rx   r   r   )r   r   )	rZ   rb   r   r^   cat	new_zerosfullr   r   )r   r   r   model_kwargsr   effective_batch_sizedummy_tokens          r(   prepare_inputs_for_generationz0RemBertForMaskedLM.prepare_inputs_for_generationl  s    o*1~ {'335e333NN4L4LnNbcdNeghMi4j4j#kqstttj!1%t{'?uzZcZj
 
 
 Iy+6A>>>	&.IIIr*   c                     dS )z
        Legacy correction: RemBertForMaskedLM can't call `generate()` from `GenerationMixin`, even though it has a
        `prepare_inputs_for_generation` method.
        Fr$   )r.   s    r(   can_generatezRemBertForMaskedLM.can_generatez  s	     ur*   )NNNNNNNNNNNNr#   )r   r   r   _tied_weights_keysr{   r  r  r   r   r^   r   r   r   r   r   r   r   r  classmethodr  r   r   s   @r(   r  r    s       :;    , , ,6 6 6  155959371559=A>B-1,0/3&*6
 6
E,-6
 !!126
 !!12	6

 u/06
 E-.6
   126
  ((9:6
 !)): ;6
 )*6
 $D>6
 'tn6
 d^6
 
un$	%6
 6
 6
 ^6
pJ J J J T    [    r*   r  zS
    RemBERT Model with a `language modeling` head on top for CLM fine-tuning.
    c            "           e Zd ZdgZ fdZd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j
                 dee	j
                 dee	j
                 d	ee	j
                 d
ee	j                 dee	j                 dee	j                 dee	j                 dee         dee	j
                 dee         dee         dee         dee         deeef         fd            Z xZS )RemBertForCausalLMr  c                    t                                          |           |j        st                              d           t          |d          | _        t          |          | _        | 	                                 d S )NzOIf you want to use `RemBertForCausalLM` as a standalone, add `is_decoder=True.`Fr  r  r   s     r(   r{   zRemBertForCausalLM.__init__  su         	nNNlmmm#FeDDD%f-- 	r*   c                 $    | j         j        j        S r#   r  rw  s    r(   r  z(RemBertForCausalLM.get_output_embeddings  r  r*   c                 (    || j         j        _        d S r#   r  r  s     r(   r  z(RemBertForCausalLM.set_output_embeddings  r  r*   Nr   r   r   rw   r   r   r   r$  r   r  r8  r   r9  r:  r   c                 d   ||n| j         j        }|                     |||||||||	||||          }|d         }|                     |          }d}|
 | j        ||
fd| j         j        i|}|s|f|dd         z   }||f|z   n|S t          |||j        |j        |j	        |j
                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert")
        >>> config = RemBertConfig.from_pretrained("google/rembert")
        >>> config.is_decoder = True
        >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```N)r   r   rw   r   r   r   r$  r   r8  r   r9  r:  r   r}   r>   )r  r  r   r   r?  r@  )rb   r  r_  r.   loss_functionr}   r   r   r   r?  r@  )r   r   r   r   rw   r   r   r   r$  r   r  r8  r   r9  r:  kwargsr  rZ  r\  lm_lossr  s                        r(   r   zRemBertForCausalLM.forward  s#   R &1%<kk$+B],,))%'"7#9+/!5#  
 
  "!* HH_55(d(!   ;1 	 G  	L')GABBK7F,3,?WJ''VK0$#3!/)$5
 
 
 	
r*   r  )r   r   r   r  r{   r  r  r   r   r^   r   r   r   r   r   r   r   r   r   r   s   @r(   r  r    s        ;;
 
 
 
 
, , ,6 6 6  155959371559=A>B+/-1$(,0/3&*Q
 Q
E,-Q
 !!12Q
 !!12	Q

 u/0Q
 E-.Q
   12Q
  ((9:Q
 !)): ;Q
 "%Q
 )*Q
 D>Q
 $D>Q
 'tnQ
 d^Q
" 
u77	8#Q
 Q
 Q
 ^Q
 Q
 Q
 Q
 Q
r*   r  z
    RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee	         dee	         dee	         de
eef         fd            Z xZS ) RemBertForSequenceClassificationc                 6   t                                          |           |j        | _        t          |          | _        t          j        |j                  | _        t          j	        |j
        |j                  | _        |                                  d S r#   rz   r{   
num_labelsro  r_  r   r   classifier_dropout_probr   r   r   r=   rs  r   s     r(   r{   z)RemBertForSequenceClassification.__init__  sy        +#F++z&"@AA)F$68IJJ 	r*   Nr   r   r   rw   r   r   r  r   r9  r:  r   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }|                     |          }d}|Z| j         j        f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j         _        nd| j         _        | j         j        dk    rWt                      }| j        dk    r1 ||                                |                                          }n |||          }n| j         j        dk    rGt                      } ||                    d| j                  |                    d                    }n*| j         j        dk    rt                      } |||          }|
s|f|dd         z   }||f|z   n|S t!          |||j        |j        	          S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r   rw   r   r   r   r9  r:  r   
regressionsingle_label_classificationmulti_label_classificationrx   r>   r  )rb   r  r_  r   r=   problem_typer  r   r^   r   rX   r   squeezer   r   r   r   r   r?  )r   r   r   r   rw   r   r   r  r   r9  r:  r  r   r  r  r  r  s                    r(   r   z(RemBertForSequenceClassification.forward  s   ( &1%<kk$+B],,))%'/!5#  

 

  
]33//{'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8FNN$4$4fnn6F6FGGDD#8FF33DD)-JJJ+--xB @ @&++b//RR)-III,..x// 	FY,F)-)9TGf$$vE'!/)	
 
 
 	
r*   
NNNNNNNNNN)r   r   r   r{   r   r   r^   r   r   r   r   r   r   r   r   r   s   @r(   r  r    sL             266:59481559-1,0/3&*E
 E
E-.E
 !!23E
 !!12	E

 u01E
 E-.E
   12E
 )*E
 $D>E
 'tnE
 d^E
 
u..	/E
 E
 E
 ^E
 E
 E
 E
 E
r*   r  c                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee	         dee	         dee	         de
eef         fd            Z xZS )RemBertForMultipleChoicec                    t                                          |           t          |          | _        t	          j        |j                  | _        t	          j        |j	        d          | _
        |                                  d S )Nr   )rz   r{   ro  r_  r   r   r  r   r   r   r=   rs  r   s     r(   r{   z!RemBertForMultipleChoice.__init__N  sl       #F++z&"@AA)F$6:: 	r*   Nr   r   r   rw   r   r   r  r   r9  r:  r   c                    |
|
n| j         j        }
||j        d         n|j        d         }|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|=|                    d|                    d          |                    d                    nd}|                     ||||||||	|
	  	        }|d         }|                     |          }|                     |          }|                    d|          }d}|t                      } |||          }|
s|f|dd         z   }||f|z   n|S t          |||j
        |j                  S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rx   r   r  r>   r  )rb   r  rZ   r   r   r_  r   r=   r   r   r   r?  )r   r   r   r   rw   r   r   r  r   r9  r:  num_choicesr  r   r  reshaped_logitsr  r  r  s                      r(   r   z RemBertForMultipleChoice.forwardX  s+   X &1%<kk$+B],5,Aioa((}GZ[\G]>G>SINN2y~~b'9'9:::Y]	M[Mg,,R1D1DR1H1HIIImqM[Mg,,R1D1DR1H1HIIImqGSG_|((\->->r-B-BCCCei ( r=#5#5b#9#9=;M;Mb;Q;QRRR 	 ,,))%'/!5#  

 

  
]33// ++b+66'))H8OV44D 	F%''!""+5F)-)9TGf$$vE("!/)	
 
 
 	
r*   r  )r   r   r   r{   r   r   r^   r   r   r   r   r   r   r   r   r   s   @r(   r  r  L  sL             266:59481559-1,0/3&*X
 X
E-.X
 !!23X
 !!12	X

 u01X
 E-.X
   12X
 )*X
 $D>X
 'tnX
 d^X
 
u//	0X
 X
 X
 ^X
 X
 X
 X
 X
r*   r  c                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee	         dee	         dee	         de
eef         fd            Z xZS )RemBertForTokenClassificationc                 :   t                                          |           |j        | _        t          |d          | _        t          j        |j                  | _        t          j	        |j
        |j                  | _        |                                  d S NFr  r  r   s     r(   r{   z&RemBertForTokenClassification.__init__  s~        +#FeDDDz&"@AA)F$68IJJ 	r*   Nr   r   r   rw   r   r   r  r   r9  r:  r   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }|                     |          }d}|Ft                      } ||                    d| j                  |                    d                    }|
s|f|dd         z   }||f|z   n|S t          |||j	        |j
                  S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   rx   r>   r  )rb   r  r_  r   r=   r   r   r  r   r   r?  )r   r   r   r   rw   r   r   r  r   r9  r:  r  rZ  r  r  r  r  s                    r(   r   z%RemBertForTokenClassification.forward  s   $ &1%<kk$+B],,))%'/!5#  

 

 "!*,,7711'))H8FKKDO<<fkk"ooNND 	FY,F)-)9TGf$$vE$!/)	
 
 
 	
r*   r  )r   r   r   r{   r   r   r^   r   r   r   r   r   r   r   r   r   s   @r(   r  r    s8       	 	 	 	 	  266:59481559-1,0/3&*2
 2
E-.2
 !!232
 !!12	2

 u012
 E-.2
   122
 )*2
 $D>2
 'tn2
 d^2
 
u++	,2
 2
 2
 ^2
 2
 2
 2
 2
r*   r  c                   b    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
eej                 dee	         dee	         dee	         de
eef         fd            Z xZS )RemBertForQuestionAnsweringc                     t                                          |           |j        | _        t          |d          | _        t          j        |j        |j                  | _        | 	                                 d S r  )
rz   r{   r  ro  r_  r   r   r   
qa_outputsrs  r   s     r(   r{   z$RemBertForQuestionAnswering.__init__  sj        +#FeDDD)F$68IJJ 	r*   Nr   r   r   rw   r   r   start_positionsend_positionsr   r9  r:  r   c                 h   ||n| j         j        }|                     |||||||	|
|	  	        }|d         }|                     |          }|                    dd          \  }}|                    d          }|                    d          }d }||t          |                                          dk    r|                    d          }t          |                                          dk    r|                    d          }|                    d          }|                    d|           |                    d|           t          |          } |||          } |||          }||z   dz  }|s||f|dd          z   }||f|z   n|S t          ||||j        |j                  S )	Nr  r   r   rx   r   )ignore_indexr>   )r  start_logits
end_logitsr   r?  )rb   r  r_  r  rQ   r  rW   r   clamp_r   r   r   r?  )r   r   r   r   rw   r   r   r  r  r   r9  r:  r  rZ  r  r  r  
total_lossignored_indexr  
start_lossend_lossr  s                          r(   r   z#RemBertForQuestionAnswering.forward  s    &1%<kk$+B],,))%'/!5#  

 

 "!*11#)<<r<#:#: j#++B//''++

&=+D?''))**Q.."1"9"9""="==%%''((1,, - 5 5b 9 9(--a00M""1m444  M222']CCCH!,@@Jx
M::H$x/14J 	R"J/'!""+=F/9/EZMF**6Q+%!!/)
 
 
 	
r*   )NNNNNNNNNNN)r   r   r   r{   r   r   r^   r   r   r   r   r   r   r   r   r   s   @r(   r  r    sM       	 	 	 	 	  266:594815596:48,0/3&*>
 >
E-.>
 !!23>
 !!12	>

 u01>
 E-.>
   12>
 "%"23>
   01>
 $D>>
 'tn>
 d^>
 
u22	3>
 >
 >
 ^>
 >
 >
 >
 >
r*   r  )
r  r  r  r  r  r  r  ro  r^  rp   )Fr   r   rF   typingr   r   r^   r   torch.nnr   r   r   activationsr
   cache_utilsr   r   r   
generationr   modeling_layersr   modeling_outputsr   r   r   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   utils.deprecationr   configuration_rembertr    
get_loggerr   rD   rp   Modulerr   r   r   r   r   r  r  r  r-  rL  rQ  rW  r^  ro  r  r  r  r  r  r  __all__r$   r*   r(   <module>r     s9      				 " " " " " " " "        A A A A A A A A A A ! ! ! ! ! ! C C C C C C C C C C ) ) ) ) ) ) 9 9 9 9 9 9	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 . - - - - - l l l l l l l l l l , , , , , , , , 0 0 0 0 0 0 0 0 0 0 0 0 
	H	%	%P P Pf3 3 3 3 3	 3 3 3n    BI   i. i. i. i. i.29 i. i. i.Z    	   0 0 0 0 0ry 0 0 0h    ")        BI   D D D D D- D D DNX
 X
 X
 X
 X
RY X
 X
 X
x    RY   "    bi   "! ! ! ! ! ! ! ! * * * * *_ * * *. 	  M
 M
 M
 M
 M
) M
 M
 M
` e e e e e/ e e eP   
g
 g
 g
 g
 g
/ g
 g
 
g
T   Q
 Q
 Q
 Q
 Q
'= Q
 Q
 Q
h d
 d
 d
 d
 d
5 d
 d
 d
N ?
 ?
 ?
 ?
 ?
$: ?
 ?
 ?
D K
 K
 K
 K
 K
"8 K
 K
 K
\  r*   