
     `i              
          d Z ddlZddlmZmZ ddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,  e            rddlm-Z-  e*j.        e/          Z0 G d dej1                  Z2d Z3d>dZ4 G d dej5                  Z6dej7        de8dej9        dej7        fdZ:d ej7        d!ej7        d"e;d#e<dej7        f
d$Z= G d% d&ej5                  Z> G d' d(e>          Z? G d) d*ej5                  Z@e>e>e?d+ZA G d, d-e          ZBe) G d. d/e'                      ZCe) G d0 d1eC                      ZD e)d23           G d4 d5eCe                      ZE e)d63           G d7 d8eC                      ZFe) G d9 d:eC                      ZGe) G d; d<eC                      ZHg d=ZIdS )?zPyTorch Falcon model.    N)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLoss	LayerNormMSELoss)
functional   )get_activation)CacheDynamicCacheStaticCache)GenerationMixin)AttentionMaskConverter)!flash_attn_supports_top_left_maskis_flash_attn_available)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)auto_docstringlogging   )FalconConfig)_flash_attention_forwardc                   2    e Zd Zdej        dej        fdZdS )FalconLinearinputreturnc                 F    || j         j        z  }| j        |S || j        z   S N)weightTbias)selfr$   hidden_statess      ~/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/falcon/modeling_falcon.pyforwardzFalconLinear.forward:   s+    -9  ty((    N)__name__
__module____qualname__torchTensorr.    r/   r-   r#   r#   9   s:        )U\ )el ) ) ) ) ) )r/   r#   c                     | dd| j         d         dz  f         }| d| j         d         dz  df         }t          j        | |fd          S )z*Rotates half the hidden dims of the input..N   dim)shaper3   cat)xx1x2s      r-   rotate_halfr@   B   s]    	
3"!'"+"""	#B	
3q """	#B9rc2YB''''r/   c                     |                     |          }|                     |          }| |z  t          |           |z  z   }||z  t          |          |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer@   )qkcossinposition_idsunsqueeze_dimq_embedk_embeds           r-   apply_rotary_pos_embrK   J   sc    ( --
&
&C
--
&
&C3w;q>>C/0G3w;q>>C/0GGr/   c                   |     e Zd ZU ej        ed<   ddef fdZ ej                    e	d                         Z
 xZS )FalconRotaryEmbeddinginv_freqNconfigc                    t                                                       t          |d          rSt          |j        t
                    r9|j                            d|j                            d                    | _        nd| _        |j        | _	        |j        | _
        || _        t          | j                 | _        |                     | j        |          \  }| _        |                     d|d           | j        | _        d S )Nrope_scaling	rope_typetypedefaultrN   F)
persistent)super__init__hasattr
isinstancerQ   dictgetrR   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrO   r   rope_init_fnattention_scalingregister_bufferrN   original_inv_freq)r+   rO   devicerN   	__class__s       r-   rW   zFalconRotaryEmbedding.__init__i   s    6>** 	'z&:Mt/T/T 	'#044[&BUBYBYZ`BaBabbDNN&DN"("@$*$B!/?+/+<+<T[&+Q+Q($(ZeDDD!%r/   c                 X   | j         d d d d f                                                             |j        d         dd                              |j                  }|d d d d d f                                         }t          |j        j        t                    r|j        j        dk    r|j        j        nd}t          j
        |d          5  |                                |                                z                      dd          }t          j        ||fd	          }|                                | j        z  }|                                | j        z  }	d d d            n# 1 swxY w Y   |                    |j        
          |	                    |j        
          fS )Nr   r7   r   mpscpuF)device_typeenabledr8   r9   dtype)rN   floatexpandr;   torc   rY   rS   strr3   autocast	transposer<   rE   r`   rF   rk   )
r+   r=   rG   inv_freq_expandedposition_ids_expandedrh   freqsembrE   rF   s
             r-   r.   zFalconRotaryEmbedding.forwardz   s    !M$4-8>>@@GGHZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk^UCCC 	5 	5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C		5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 vvAGv$$cff17f&;&;;;s   BE++E/2E/r'   )r0   r1   r2   r3   r4   __annotations__r    rW   no_gradr   r.   __classcell__rd   s   @r-   rM   rM   f   s         l/ /| / / / / / /" U]__< <  _< < < < <r/   rM   attention_mask	num_headsrk   r%   c                    | j         \  }}dt          j        t          j        |                    z  }t	          j        ddt          j        |          dz
   z   z  | j        t          j                  }t	          j        dd|z   | j        t          j	                  }t	          j
        ||          }||k    rt	          j        ddt          j        d|z            dz
   z   z  | j        t          j                  }	t          |||z
            }
t	          j        ddd|
z  z   d| j        t          j	                  }t	          j        |t	          j
        |	|          gd          }|                     d          dz
  | z  d d d d d f         }|d                                         |z  }|                    ||z  d|                              |          S )	Nr8   r   rc   rk   r   r   r9   r7   ).N)r;   mathfloorlog2r3   tensorrc   float32arangeint32powminr<   cumsumbfloat16reshapern   )rz   r{   rk   
batch_size
seq_lengthclosest_power_of_2basepowersslopes
extra_basenum_remaining_headsextra_powersarange_tensoralibis                 r-   build_alibi_tensorr      s   +1J
dj9)=)=>>><	ty!344q899:;NDYafan  D \!Q!33N<QY^YdeeeFYtV$$FY&&\A49Q);%;<<q@AABCNLainiv
 
 

 ""4iBT6TUU|Aq1/B+B'BAnNckpkvwwwFEIj,$G$GHaPPP %+++33a7>I111dTUTUTU:VM9&&((=8E==i/J??BB5IIIr/   r=   residualprobtrainingc                 >    t          j        | ||          }||z   }|S )a
  
    Dropout add function

    Args:
        x (`torch.tensor`):
            input tensor
        residual (`torch.tensor`):
            residual tensor
        prob (`float`):
            dropout probability
        training (`bool`):
            training mode
    )pr   )Fdropout)r=   r   r   r   outs        r-   dropout_addr      s(     )A
1
1
1C
S.CJr/   c                   |    e Zd Zddef fdZdej        deej        ej        ej        f         fdZdej        dej        fdZ		 	 	 	 	 	 	 dd
ej        de
ej                 dej        de
ej                 de
e         de
ej                 dedede
ej                 de
eej        ej        f                  fdZ xZS )FalconAttentionNrO   c                    t                                                       || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | _        |j        | _        |j	        | _	        |j
        | _
        d| _        || _        |(t                              d| j        j         d           | j        | j        z  | j        k    r t#          d| j         d| j         d          dt%          j        | j                  z  | _        | j        | _        |j        r|j        dz  |j        z   | j        z  }n$|j        r| j        d| j        z  z   }n
d	| j        z  }t3          | j        ||j        
          | _        |j        | _        |j        | _        t3          | j        | j        |j        
          | _        t;          j        |j                  | _        | j        s| j        s|j        nd| _        d S )NTzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zA`hidden_size` must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).      ?r8   r   r*   r   ) rV   rW   rO   hidden_sizenum_attention_headsr{   head_dim
split_sizehidden_dropoutr\   
rope_theta	is_causal	layer_idxloggerwarning_oncerd   r0   
ValueErrorr~   sqrtinv_norm_factorbetanew_decoder_architecturenum_kv_headsmulti_queryr#   r*   query_key_valuedenser   Dropoutattention_dropout)r+   rO   r   qkv_out_dimrd   s       r-   rW   zFalconAttention.__init__   s   !-3(DN:*$3'-'E$ +",!8 , , ,   =4>)T-==='TXTd ' 'N' ' '    #TYt}%=%==(	* 	/!.2V5OOSWS``KK 	/*Q->>KKd..K+D,<kPVP[\\\(.(G%!-!$"2D4D6;WWW
!#F,D!E!E484QqY]YiqF//pqr/   	fused_qkvr%   c                    | j         r|j        \  }}}|                    ||d| j        | j        z  dz   | j                  }|ddddddddf         }|dddddddgf         }|dddddddgf         }t          j        ||j                  }t          j        ||j                  }d |||fD             \  }}}|||fS | j        sT|j        \  }	}
}|                    |	|
| j        d| j                  }|ddddf         |dd	ddf         |ddddf         fS |j        \  }	}
}|                    |	|
| j        dz   | j                  }|dddddf         |ddgddf         |ddgddf         fS )
a  
        Split the last dimension into (num_heads, head_dim), results share same memory storage as `fused_qkv`

        Args:
            fused_qkv (`torch.tensor`): [batch_size, seq_length, num_heads * 3 * head_dim]

        Returns:
            query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
            value: [batch_size, seq_length, num_heads, head_dim]
        r7   r8   Nc                 :    g | ]}|                     d d          S )r8   r   )flatten).0r=   s     r-   
<listcomp>z0FalconAttention._split_heads.<locals>.<listcomp>   s$     N N NQ1a N N Nr/   r   .r   r   )	r   r;   viewr{   r   r   r3   broadcast_tor   )r+   r   batchseq_len_qkvquerykeyvaluer   r   three_times_hidden_sizes               r-   _split_headszFalconAttention._split_heads   s    ( 	\ )E7A..T^tGX5X[\5\^b^kllC111aaa"%EaaaAAAtm$C111aaa"&E$S%+66C&uek::E N N5#u:M N N NE3#u$$! 	\>Go;J
$;!z:t~qRVR_``IS!QQQY'31119)=yaQRQRQR?SSS>Go;J
$;!z:t~PQ?QSWS`aaIS#2#qqq[)9S2$\+BIcTVSWYZYZYZlD[[[r/   r=   c                     |j         \  }}}|| j        z  }|                    || j        || j                  }|                    dddd          }|                    ||| j        | j        z            S )z
        Merge heads together over the last dimension

        Args:
            x (`torch.tensor`): [batch_size * num_heads, seq_length, head_dim]

        Returns:
            torch.tensor: [batch_size, seq_length, num_heads * head_dim]
        r   r8   r   r   )r;   r{   r   r   permuter   )r+   r=   batch_size_and_num_headsr   r   r   s         r-   _merge_headszFalconAttention._merge_heads  sv     34'/ *a-?
 FF:t~z4=II IIaAq!! yyZ$-1OPPPr/   Fr,   r   rz   rG   
layer_past	head_mask	use_cacheoutput_attentionscache_positionposition_embeddingsc                 	   |                      |          }| j        r| j        n| j        }|                     |          \  }}}|j        \  }}}}|                    dd                              || j        || j                  }|                    dd                              |||| j                  }|                    dd                              |||| j                  }||
\  }}t          ||||          \  }}|>d|	i}||
                    ||d           |
                    ||| j        |          \  }}|j        d         }| j        j        dk    rN|j        j        dk    r>|<|                                }|                                }|                                }||d d d d d d d |j        d         f         }| | j        j        dk    r@|s>| j        o	|d u o|dk    }t$          j        j                            ||||d|	          }d }nY||                    d
d          z  }|t-          j        | j                  z  }t1          j        ||z   d
|j                  }||z  }|                    || j        || j                  }|                    dddd          }|                    ||| j        | j        z            }|                     |          }||fS | j        j        dk    r|s|| j        o	|d u o|dk    }t$          j        j                            ||||| j        r| j        j         nd|	          }d }|                    dd          }|                    ||| j        | j        z            }|                     |          }nA||                    d
d          z  }|                    || j        ||          }|j        }|t$          j!        k    s|t$          j"        k    r|#                    t$          j$                  }||                    || j        dd
          z   }|| j%        z  }t1          j        ||z   d
|j                  }|                     |          }|||z  }|                    || j        ||          }||z  &                    dd          }| '                    |          }|                     |          }||fS )Nr   r8   r   rF   rE   r   sdpacuda        )	attn_mask	dropout_pr   r7   )r:   rk   r   r   )(r   r   r{   r   r   r;   rq   r   r   rK   updater   rO   _attn_implementationrc   rS   
contiguousr   r3   r   r
   scaled_dot_product_attentionr~   r   r   softmaxrk   r   r   r   r   r   r   float16r   rn   r   r   r   r   )r+   r,   r   rz   rG   r   r   r   r   r   r   r   r   query_layer	key_layervalue_layerr   query_lengthr   rE   rF   cache_kwargs	kv_lengthr   attn_outputattention_scoresattention_probsmatmul_resultinput_dtypeattention_logitsattention_probs_reshapeds                                  r-   r.   zFalconAttention.forward  sh    ((77	)-)F]t~~DL]040A0A)0L0L-i)4):&
L!Q!++Aq1199*dnVbdhdqrr''1--55j,P\^b^kll	!++Aq1199*lT`bfbopp=*HC%9+yRUWZ%[%["K!,n=L}##C$<$<===%/%6%6y+t~_k%l%l"I{OB'	K,66"'611* &0022K!,,..I%0022K%+AAAqqq!!!5Jyr7J5J,JKN={/699BS9
 !NZ~/EZ,YZJZ	#h1NN,!' O   $(  #.1D1DR1L1L#L  DIdm$<$<< #$9-=-NTV^k^q#r#r#r .<%**:t~|UYUbccK%--aAq99K%--j,Y]YfHfggK**[11K 000 {/699BS9XaXi !NZ~/EZ,YZJZ	#h1NN,:>-Pd466S' O   #')33Aq99)11*lDN]a]jLjkk"jj55 +i.A.A"b.I.I I $1#5#5j$.R^`i#j#j  /4%-//;%.3P3P'7':':5='I'I$#3ejjT^]^`b6c6c#c  D$88 "#),<~,MSU]j]p"q"q"q"&"8"8"I"I(&5	&AO ,;+?+?
DN\hjs+t+t(  8+ENNqRSTT #//<<"jj55//r/   r'   NNNFFNN)r0   r1   r2   r    rW   r3   r4   tupler   r   r   
LongTensorr   boolr.   rx   ry   s   @r-   r   r      s       (r (r| (r (r (r (r (r (rT\el \uU\5<Y^Ye=e7f \ \ \ \@Qel Qu| Q Q Q Q< 48&*,0"'59KOC0 C0|C0 %C0 	C0
 u/0C0 UOC0 EL)C0 C0  C0 !!12C0 &eEL%,,F&GHC0 C0 C0 C0 C0 C0 C0 C0r/   r   c                   
    e Zd ZdZ fdZ	 	 	 	 	 	 	 ddej        deej                 dej        deej                 d	ee	         d
eej                 de
de
deej                 deeej        ej        f                  fdZ xZS )FalconFlashAttention2aH  
    Falcon flash attention module. This module inherits from `FalconAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 `     t                      j        |i | t                      | _        d S r'   )rV   rW   r   _flash_attn_uses_top_left_mask)r+   argskwargsrd   s      r-   rW   zFalconFlashAttention2.__init__  s6    $)&)))
 /P.Q.Q+++r/   NFr,   r   rz   rG   r   r   r   r   r   r   c                 H   |                      |          }| j        r| j        n| j        }|                     |          \  }}}|j        \  }}}}|                    dd                              || j        || j                  }|                    dd                              |||| j                  }|                    dd                              |||| j                  }||
\  }}t          ||||          \  }}|>d|	i}||
                    ||d           |
                    ||| j        |          \  }}|                    dd          }|                    dd          }|                    dd          }|t          d          | j        r| j        j        nd}|j        }|j        j        dk    r|j        j        nd}|t&          j        k    rt'          j                    r=t-          t&          d	          rt'          j        |          nt'          j                    }n3t-          | j        d
          r| j        j        }n| j         j        j        }t6                              d| d           |                    |          }|                    |          }|                    |          }t=          |||||||| j        | j         	  	        }|                    ||| j        | j        z            }| !                    |          }|sd }||fS )Nr   r8   r   r   z6`alibi` is not supported when `use_flash_attn` is Truer   rf   rg   get_autocast_dtype_pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .)rG   r   r   use_top_left_mask)"r   r   r{   r   r   r;   rq   r   r   rK   r   r   r   r   rO   r   rk   rc   rS   r3   r   is_autocast_enabledrX   r   get_autocast_gpu_dtyper   r(   r   r   rn   r!   r   r   r   )r+   r,   r   rz   rG   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rE   rF   r   attn_dropoutr   rh   target_dtyper   attn_weightss                               r-   r.   zFalconFlashAttention2.forward  sc    ((77	)-)F]t~~DL]040A0A)0L0L-i)4):&
L!Q!++Aq1199*dnVbdhdqrr''1--55j,P\^b^kll	!++Aq1199*lT`bfbopp=*HC%9+yRUWZ%[%["K!,n=L}##C$<$<===%/%6%6y+t~_k%l%l"I{ "++Aq11''1--	!++Aq11UVVV8<Nt{443
 "'1<1C1HE1Q1Qk(--W\%-''(** 
A u&:;;8E,[999577  &?@@ A#{B#3:@$ $ $ $   &..66K!\22I%..66K.% n"A

 

 

 #**:|T^VZVcEcddjj..  	 LL((r/   r   )r0   r1   r2   __doc__rW   r3   r4   r   r   r   r   r   r.   rx   ry   s   @r-   r   r     s%        R R R R R 48&*,0"'59KO[) [)|[) %[) 	[)
 u/0[) UO[) EL)[) [)  [) !!12[) &eEL%,,F&GH[) [) [) [) [) [) [) [)r/   r   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )	FalconMLPrO   c                 $   t                                                       |j        }t          ||j        |j                  | _        t          |j                  | _	        t          |j        ||j                  | _
        |j        | _        d S )Nr   )rV   rW   r   r#   ffn_hidden_sizer*   dense_h_to_4hr   
activationactdense_4h_to_hr   )r+   rO   r   rd   s      r-   rW   zFalconMLP.__init__  s    ()+v7MTZT_```!&"344)&*@+TZT_```$3r/   r=   r%   c                     |                      |                     |                    }|                     |          }|S r'   )r  r	  r  )r+   r=   s     r-   r.   zFalconMLP.forward  s9    HHT''**++q!!r/   )	r0   r1   r2   r    rW   r3   r4   r.   rx   ry   s   @r-   r  r    sj        4| 4 4 4 4 4 4 %,        r/   r  )eagerr   flash_attention_2c                   B    e Zd Zddef fdZ	 	 	 	 	 	 	 ddej        deej                 dej        deej                 d	ee	e
eej        ej        f         f                  d
eej                 dededeej                 deeej        ej        f                  fdZ xZS )FalconDecoderLayerNrO   c                 `   t                                                       |j        }|j        | _        t          |j                 ||          | _        t          |          | _	        |j
        | _
        || _        |j        |j        rd|_        |j        s8t          ||j                  | _        t          ||j                  | _        d S |j        dk    r8t          ||j                  | _        t          ||j                  | _        d S t          ||j                  | _        d S )Nr8   eps)rV   rW   r   r   r{   FALCON_ATTENTION_CLASSESr   self_attentionr  mlpr   rO   num_ln_in_parallel_attnr   parallel_attnr   layer_norm_epsilonpost_attention_layernorminput_layernormln_attnln_mlp)r+   rO   r   r   rd   s       r-   rW   zFalconDecoderLayer.__init__)  s!   (36v7RSTZ\effV$$$3)1f6U1-.F*# 
	],5kvG`,a,a,aD)#,[f>W#X#X#XD   -22(&:STTT'9RSSS'0&B['\'\'\$$$r/   Fr,   r   rz   rG   r   r   r   r   r   r   c                    |}| j         j        r;| j         j        dk    r+|                     |          }|                     |          }n|                     |          }|                     |||||||||	|

  
        \  }}| j         j        sF| j         j        r|}n7t          ||| j         j	        | j
                  }|                     |          }| j         j        r| j         j        r| j         j        dk    r|}|                     |          }| j         j        s| j         j        r||z  }t          ||| j         j        | j
                  }||fS )Nr8   )	r   rz   rG   r   r   r   r   r   r   )r   r   )rO   r   r  r  r  r  r  r  r   r   r   r  r  r   )r+   r,   r   rz   rG   r   r   r   r   r   r   r   r   attention_layernorm_outmlp_layernorm_outattention_outputr  
mlp_outputoutputs                      r-   r.   zFalconDecoderLayer.forwardB  s    !;/ 	JDK4W[\4\4\&*ll=&A&A# $M : :&*&:&:=&I&I# *.)<)<#!)%/) 3 *= *
 *
&, {3 	L{( L$;!!&$h0MX\Xe   %)$A$A($K$K! K0	8)	8 3q88 7 XX/00
;/ 	+4;3L 	+**JZ4;3MX\Xefff|##r/   r'   r   )r0   r1   r2   r    rW   r3   r4   r   r   r   r   r   r   r.   rx   ry   s   @r-   r  r  (  s5       ] ]| ] ] ] ] ] ]< 48PT,0"'59KO<$ <$|<$ %<$ 	<$
 u/0<$ U5%el0J*K#KLM<$ EL)<$ <$  <$ !!12<$ &eEL%,,F&GH<$ <$ <$ <$ <$ <$ <$ <$r/   r  c                   x     e Zd ZU eed<   dZdZdgZdZdZ	dZ
 fdZdej        fdZedd	efd
            Z xZS )FalconPreTrainedModelrO   transformerTr  c                 :     t                      j        |i | d S r'   )rV   rW   )r+   inputsr   rd   s      r-   rW   zFalconPreTrainedModel.__init__  s%    &+F+++++r/   modulec                    t          |t          j        t          f          rT|j        j                            d| j        j                   |j	         |j	        j        
                                 dS dS t          |t          j                  r_|j        j                            d| j        j                   |j        +|j        j        |j                 
                                 dS dS t          |t                    r?|j	        j        
                                 |j        j                            d           dS dS )zInitialize the weights.r   )meanstdNr   )rY   r   Linearr#   r(   datanormal_rO   initializer_ranger*   zero_	Embeddingpadding_idxr   fill_)r+   r*  s     r-   _init_weightsz#FalconPreTrainedModel._init_weights  s,   fry,788 	* M&&CT[5R&SSS{& &&((((( '&-- 	*M&&CT[5R&SSS!-"6#56<<>>>>> .-	** 	*K""$$$M$$S)))))	* 	*r/   Fhard_check_onlyc                 B    t          | dd          }|r|S |sd|_        |S )Nuse_bettertransformerFr   )getattrr   )clsrO   r7  _is_bettertransformers       r-   _check_and_enable_sdpaz,FalconPreTrainedModel._check_and_enable_sdpa  s7     '-De L L  	M 	1*0F'r/   )F)r0   r1   r2   r    rv   base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_can_compile_fullgraphrW   r   Moduler6  classmethodr   r=  rx   ry   s   @r-   r&  r&    s         %&*#-.N!, , , , ,*BI * * * *"  T    [    r/   r&  c                   r    e Zd Zdef fdZd Zdej        fdZe		 	 	 	 	 	 	 	 	 	 	 dde
ej                 de
eeeeej        ej        f         d	f         f                  d
e
ej                 de
ej                 de
ej                 de
ej                 de
e         de
e         de
e         de
e         de
ej                 deeej        d	f         ef         fd            Zd
ej        dej        dej        dededej        dej        fdZed
ej        dededej        dej        defd            Z xZS )FalconModelrO   c                    t                                                     j        | _        j        | _        j        | _        t          j	        j
        | j                  | _        t          j        fdt          j                  D                       | _        t!          | j        j                  | _        t'                    | _        d| _        |                                  d S )Nc                 2    g | ]}t          |           S ))r   )r  )r   irO   s     r-   r   z(FalconModel.__init__.<locals>.<listcomp>  s'    qqqA 26Q G G Gqqqr/   r  rO   F)rV   rW   r   	embed_dimr   r{   r   	use_alibir   r3  
vocab_sizeword_embeddings
ModuleListrangenum_hidden_layershr   r  ln_frM   
rotary_embgradient_checkpointing	post_initr+   rO   rd   s    `r-   rW   zFalconModel.__init__  s       +3  "|F,=t~NN qqqqQVW]WoQpQpqqqrr dn&2KLLL	/v>>>&+# 	r/   c                     | j         S r'   rO  )r+   s    r-   get_input_embeddingsz FalconModel.get_input_embeddings  s    ##r/   new_embeddingsc                     || _         d S r'   rZ  r+   r\  s     r-   set_input_embeddingsz FalconModel.set_input_embeddings  s    -r/   N	input_idspast_key_values.rz   rG   r   inputs_embedsr   r   output_hidden_statesreturn_dictr   r%   c                    ||n| j         j        }|	|	n| j         j        }	||n| j         j        }|
|
n| j         j        }
|du |duz  rt          d          | j        r%| j        r|rt          	                    d           d}|| 
                    |          }|r|t          | j                   }d}||                                nd}|j        \  }}}| j        rK|+t          j        |||z   f|j        t          j                  n|}t'          || j        |j                  }|t          j        |||z   |j        	          }||                    d          }|                     |||||||          }|                     || j         j                  }|}|                     ||          }|rd
nd}|	rd
nd}t9          | j                  D ]>\  }}|	r||fz   } |||||||         |||||
  
        }|d         }|r||d         fz   }?|                     |          }|	r||fz   }|
st?          d ||||fD                       S tA          ||||          S )  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FrK  r   r}   rj   rc   r5   )	r   rz   rG   r   r   r   r   r   r   r   c              3      K   | ]}||V  	d S r'   r5   )r   vs     r-   	<genexpr>z&FalconModel.forward.<locals>.<genexpr>@  s1        ghgtgtgtgtgt r/   )last_hidden_statera  r,   
attentions)!rO   r   rc  r   use_return_dictr   rV  r   r   r   rO  r   get_seq_lengthr;   rM  r3   onesrc   longr   r{   rk   r   rB   _update_causal_maskget_head_maskrR  rU  	enumeraterS  rT  r   r   )r+   r`  ra  rz   rG   r   rb  r   r   rc  rd  r   r   past_key_values_lengthr   r   r   maskcausal_maskr,   r   all_self_attentionsall_hidden_statesrJ  blockoutputss                             r-   r.   zFalconModel.forward  s   8 2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]-t";< 	[YZZZ& 	"4= 	" "##p   "	  00;;M 	?0*$+>>>O ETE`!?!?!A!A!Afg$1$7!
J> 	X
 ") 
.D!DEmNbjojt    $  'tT^=CVWWWE!"\&(>(KTaTh  N )33A66L..M>?L]_hjo
 
 &&y$+2OPP	% #oom\JJ$5?bb4"6@BBD!$&)) 	J 	JHAu# I$58H$H!e**)#A,#"3-$7  G $AJM  J&9WQZM&I# 		-00 	E 1]4D D 	  )?<MObc      9+++*	
 
 
 	
r/   input_tensorr   c           
      j   | j         j        dk    r
|d|v r|S d S ||                                nd}t          |t                    }	| j         j        dk    r'|	s%|s#|!|t          j        |||| j                  rd S |j        |j	        }}
t          j        |
          j        }|j        \  }}}|	r|                                }n,t          |t          j                  r|j        d         n||z   }|                     ||||
|||j        d                   }|_|] |j        |dg|j        dd          R  }t          j        |t'          j        | j         j        | j        z            z  |dk     |          }| j         j        dk    r'|%|j	        j        d	v r|st          j        ||          }|S )
Nr  r   r   r   )rb  rt  is_trainingr7   )sequence_lengthtarget_lengthrk   rc   r   r   r   )r   xpunpu)rO   r   rn  rY   r   r   _ignore_causal_mask_sdpar   rk   rc   r3   finfor   r;   get_max_cache_shaper4   5_prepare_4d_causal_attention_mask_with_cache_positionr   masked_fillr~   r   r   r{   rS   _unmask_unattended)r+   rz   r{  r   ra  r   r   r   past_seen_tokensusing_static_cacherk   rc   	min_dtyper   r~  r   r  rv  s                     r-   rq  zFalconModel._update_causal_maskK  s/    ;+/BBB)c^.C.C%%4
 @O?Z?99;;;`a'EE K,66& 7% 7 !%>*'7 M	    t$*L,?vK&&*	)5);&
OQ 	+??AAMM nel;;8$R((%7  PP+')#)!, Q 
 
 !2!EM*bC5;qrr?CCCE+	$+"9T^"KLLLb  K K,66*%*.DDD% E 1CKQZ[[Kr/   r~  r  rk   r   c                    | |                                  dk    r| }nMt          j        |          j        }t          j        ||f|||j                  }|dk    rt          j        |d          }|t          j        ||j                  |                    dd          k    z  }|ddddddf         	                    |ddd          }| |
                                }| j        d         }	|ddddddd|	f         | ddddddf                             |j                  z   }
|
dk    }
|ddddddd|	f                             |
|          |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuerk   rc   r   )diagonalrg  r7   r   )r:   r3   r  r   fullrc   triur   r   rm   cloner;   rn   r  )rz   r~  r  rk   r   r   r   rv  r  mask_lengthpadding_masks              r-   r  zAFalconModel._prepare_4d_causal_attention_mask_with_cache_position  s   > %.*<*<*>*>!*C*C(KKE**.I* -0Ye\j\q  K !###jqAAA5<n>STTTWeWmWmnprsWtWtttK%dD!!!QQQ&67>>z1bRTUUK))//11,226*111aaaL[L+@ANSTSTSTVZ\`bcbcbcScDdDgDg&E E    ,q05@AAAqqq,;,AV5W5c5c )6 6AAAqqq!!!\k\12 r/   )NNNNNNNNNNN)r0   r1   r2   r    rW   r[  r3   r4   r_  r   r   r   r   r   r   r   r   r.   rq  staticmethodintrk   r  rx   ry   s   @r-   rG  rG    s       |      .$ $ $.5< . . . .  15ae15370448$(,0/3&*59~
 ~
E,-~
 "%uU5<;U5VX[5[/\(\"]^~
 !.	~

 u/0~
 E,-~
   01~
 D>~
 $D>~
 'tn~
 d^~
 !!12~
 
uU\3&')RR	S~
 ~
 ~
 ^~
@UU lU 	U
 U  U <U |U U U Un 444 4 {	4
 4 4 4 4 \4 4 4 4 4r/   rG  z
    The Falcon Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).
    )custom_introc                        e Zd ZdgZdef fdZdej        fdZe		 	 	 	 	 	 	 	 	 	 	 	 	 dde
ej                 d	e
eeeeej        ej        f         d
f         f                  de
ej                 de
ej                 de
ej                 de
ej                 de
ej                 de
e         de
e         de
e         de
e         de
ej                 deeej        f         deeej                 ef         fd            Z xZS )FalconForCausalLMzlm_head.weightrO   c                     t                                          |           t          |          | _        t	          j        |j        |j        d          | _        | 	                                 d S NFr   )
rV   rW   rG  r'  r   r.  r   rN  lm_headrW  rX  s     r-   rW   zFalconForCausalLM.__init__  sa       &v..y!3V5FUSSS 	r/   r\  c                     || _         d S r'   )r  r^  s     r-   set_output_embeddingsz'FalconForCausalLM.set_output_embeddings  s    %r/   Nr   r`  ra  .rz   rG   r   rb  labelsr   r   rc  rd  r   logits_to_keepr%   c                    ||n| j         j        }|                     ||||||||	|
||          }|d         }t          |t                    rt          | d          n|}|                     |dd|ddf                   }d}| | j        ||fd| j         j        i|}|s|f|dd         z   }||f|z   n|S t          |||j
        |j        |j                  S )a\  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)
ra  rz   rG   r   rb  r   r   rc  rd  r   r   rN  r   losslogitsra  r,   rl  )rO   rm  r'  rY   r  slicer  loss_functionrN  r   ra  r,   rl  )r+   r`  ra  rz   rG   r   rb  r  r   r   rc  rd  r   r  r   transformer_outputsr,   slice_indices	lm_logitsr  r$  s                        r-   r.   zFalconForCausalLM.forward  sV   H &1%<kk$+B]"..+)%'/!5#) / 
 
 ,A.8B>SV8W8Wk~ot444]kLLqqq-/B!CDD	%4%   ;1 	 D  	F\$7$;;F)-)9TGf$$vE0/?-;*5
 
 
 	
r/   )NNNNNNNNNNNNr   )r0   r1   r2   _tied_weights_keysr    rW   r3   r4   r  r   r   r   r   r   r   r   r  r   r.   rx   ry   s   @r-   r  r    s        ++|      &EL & & & &  15ae1537,004)-$(,0/3&*5934J
 J
E,-J
 "%uU5<;U5VX[5[/\(\"]^J
 !.	J

 u/0J
 EL)J
  -J
 &J
 D>J
 $D>J
 'tnJ
 d^J
 !!12J
 c5</0J
  
uU\"$EE	F!J
 J
 J
 ^J
 J
 J
 J
 J
r/   r  a  
    The Falcon Model transformer with a sequence classification head on top (linear layer).

    [`FalconForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   N    e Zd Zdef fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 dee	         deej
                 deej
                 deej
                 d	eej
                 d
ee         dee         dee         dee         deeej
                 ef         fd            Z xZS )FalconForSequenceClassificationrO   c                     t                                          |           |j        | _        t          |          | _        t          j        |j        |j        d          | _        | 	                                 d S r  )
rV   rW   
num_labelsrG  r'  r   r.  r   scorerW  rX  s     r-   rW   z(FalconForSequenceClassification.__init__K  sk        +&v..Yv163D5QQQ
 	r/   Nr`  ra  rz   r   rb  r  r   r   rc  rd  r%   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }||j        d         }n|j        d         }| j         j        |dk    rt          d          | j         j        d}n|}|| j         j        k                        |j        t          j
                  }t          j        |j        d         |j        t          j
                  }||z                      d          }n)d}t                              | j        j         d           |t          j        ||j        	          |f         }d}|.| j         j        f| j        dk    rd
| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j         _        nd| j         _        | j         j        d
k    rWt-                      }| j        dk    r1 ||                                |                                          }nb |||          }nU| j         j        dk    rt1                      } |||          }n*| j         j        dk    rt3                      } |||          }|
s|f|dd         z   }||f|z   n|S t5          |||j        |j        |j                  S )6  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nra  rz   r   rb  r   r   rc  rd  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r7   r}   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rg  
regressionsingle_label_classificationmulti_label_classificationr  )rO   rm  r'  r  r;   pad_token_idr   rn   rc   r3   r   r   argmaxr   r   rd   r0   problem_typer  rk   rp  r  r	   squeezer   r   r   ra  r,   rl  )r+   r`  ra  rz   r   rb  r  r   r   rc  rd  r  r,   r  r   last_non_pad_tokennon_pad_masktoken_indicespooled_logitsr  loss_fctr$  s                         r-   r.   z'FalconForSequenceClassification.forwardT  s   @ &1%<kk$+B]"..+)'/!5# / 

 

 ,A.M** "+JJ&,Q/J;#+
a\]]];#+!#"%)AAEEfmUZU`aaL!L)<V]Z_ZefffM"/,">!F!Fr!J!J!#>* Z Z Z  
 u|Jv}MMMOaab{'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8M$9$9$;$;V^^=M=MNNDD#8M6::DD)-JJJ+--xv66)-III,..xv66 	F#%(;ABB(??F)-)9TGf$$vE/ /?-;*5
 
 
 	
r/   
NNNNNNNNNN)r0   r1   r2   r    rW   r   r   r3   r   r   r4   r   r   r   r   r.   rx   ry   s   @r-   r  r  <  sS       |        15+/15,004)-$(,0/3&*g
 g
E,-g
 "%g
 !.	g

 EL)g
  -g
 &g
 D>g
 $D>g
 'tng
 d^g
 
uU\"$DD	Eg
 g
 g
 ^g
 g
 g
 g
 g
r/   r  c                   N    e Zd Zdef fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 dee	         deej
                 deej
                 deej
                 d	eej
                 d
ee         dee         dee         dee         deeej
                 ef         fd            Z xZS )FalconForTokenClassificationrO   c                    t                                          |           |j        | _        t          |          | _        t          |dd           |j        }nt          |dd           |j        }nd}t          j	        |          | _
        t          j        |j        |j                  | _        |                                  d S )Nclassifier_dropoutr   g?)rV   rW   r  rG  r'  r:  r  r   r   r   r   r.  r   
classifierrW  )r+   rO   r  rd   s      r-   rW   z%FalconForTokenClassification.__init__  s        +&v..6/66B!'!:V-t44@!'!6!$z"455)F$68IJJ 	r/   Nr`  ra  rz   r   rb  r  r   r   rc  rd  r%   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }|                     |          }d}|V|j        \  }}t                      } ||                    ||z  | j                  |                    ||z                      }|
s|f|dd         z   }||f|z   n|S t          |||j
        |j                  S )r  Nr  r   r8   )r  r  r,   rl  )rO   rm  r'  r   r  r;   r   r   r  r   r,   rl  )r+   r`  ra  rz   r   rb  r  r   r   rc  rd  r  r,   r  r  r   r   r  r$  s                      r-   r.   z$FalconForTokenClassification.forward  s=   @ &1%<kk$+B]"..+)'/!5# / 

 

 ,A.]33//%+\"J
'))H8J3T_EEv{{S]`jSjGkGk D  	FY!4QRR!88F)-)9TGf$$vE$-;*5	
 
 
 	
r/   r  )r0   r1   r2   r    rW   r   r   r3   r   r   r4   r   r   r   r   r.   rx   ry   s   @r-   r  r    sS       |      "  15+/15,004)-$(,0/3&*B
 B
E,-B
 "%B
 !.	B

 EL)B
  -B
 &B
 D>B
 $D>B
 'tnB
 d^B
 
uU\"$99	:B
 B
 B
 ^B
 B
 B
 B
 B
r/   r  c                   *    e Zd Z fdZe	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	ee	         d
ee	         dee	         de
eef         fd            Z xZS )FalconForQuestionAnsweringc                     t                                          |           t          |          | _        t	          j        |j        d          | _        |                                  d S )Nr8   )	rV   rW   rG  r'  r   r.  r   
qa_outputsrW  rX  s     r-   rW   z#FalconForQuestionAnswering.__init__  sY       &v..)F$6:: 	r/   Nr`  rz   r   rb  start_positionsend_positionsr   rc  rd  r%   c
           	         |	|	n| j         j        }	|                     |||||||	          }
|
d         }|                     |          }|                    dd          \  }}|                    d                                          }|                    d                                          }d}||t          |                                          dk    r|                    d          }t          |                                          dk    r|                    d          }|                    d          }|	                    d|          }|	                    d|          }t          |          } |||          } |||          }||z   dz  }|	s||f|
dd         z   }||f|z   n|S t          ||||
j        |
j        	          S )
rf  N)rz   r   rb  r   rc  rd  r   r   r7   r9   )ignore_indexr8   )r  start_logits
end_logitsr,   rl  )rO   rm  r'  r  splitr  r   lensizeclampr   r   r,   rl  )r+   r`  rz   r   rb  r  r  r   rc  rd  rz  sequence_outputr  r  r  
total_lossignored_indexr  
start_lossend_lossr$  s                        r-   r.   z"FalconForQuestionAnswering.forward"  s   4 &1%<kk$+B]"")'/!5# # 
 
 "!*11#)<<r<#:#: j#++B//::<<''++6688

&=+D?''))**Q.."1"9"9""="==%%''((1,, - 5 5b 9 9(--a00M-33A}EEO)//=AAM']CCCH!,@@Jx
M::H$x/14J 	R"J/'!""+=F/9/EZMF**6Q+%!!/)
 
 
 	
r/   )	NNNNNNNNN)r0   r1   r2   rW   r   r   r3   r   FloatTensorr   r   r   r   r.   rx   ry   s   @r-   r  r    s6             156:15596:48,0/3&*G
 G
E,-G
 !!23G
 E-.	G

   12G
 "%"23G
   01G
 $D>G
 'tnG
 d^G
 
u22	3G
 G
 G
 ^G
 G
 G
 G
 G
r/   r  )r  rG  r&  r  r  r  )Nr   )Jr  r~   typingr   r   r3   r   torch.nnr   r   r   r	   r
   r   activationsr   cache_utilsr   r   r   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   utilsr   r   configuration_falconr    r!   
get_loggerr0   r   r.  r#   r@   rK   rD  rM   r4   r  rk   r   rl   r   r   r   r   r  r  r  r&  rG  r  r  r  r  __all__r5   r/   r-   <module>r     s      " " " " " " " "        L L L L L L L L L L L L $ $ $ $ $ $ ) ) ) ) ) ) ; ; ; ; ; ; ; ; ; ; ) ) ) ) ) )      i h h h h h h h 9 9 9 9 9 9              L K K K K K K K - - - - - -        / . . . . .  KJJJJJJ		H	%	%
) ) ) ) )29 ) ) )( ( (   8!< !< !< !< !<BI !< !< !<HJu| J JEK J\a\h J J J J:5< 5< u PT Y^Ye    &g0 g0 g0 g0 g0bi g0 g0 g0Tj) j) j) j) j)O j) j) j)Z    	   " .  V$ V$ V$ V$ V$3 V$ V$ V$r & & & & &O & & &R l l l l l' l l l^	   
Y
 Y
 Y
 Y
 Y
- Y
 Y
 
Y
x   r
 r
 r
 r
 r
&; r
 r
 r
j U
 U
 U
 U
 U
#8 U
 U
 U
p Q
 Q
 Q
 Q
 Q
!6 Q
 Q
 Q
h  r/   