
     `i                        d Z ddlmZ ddlmZmZmZ ddlZddlm	c m
Z ddlm	Z	 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*  e%j+        e,          Z- G d de	j.                  Z/ G d de	j.                  Z0 G d de0          Z1 G d de0          Z2d Z3dNdZ4 G d de	j.                  Z5 G d  d!e	j6                  Z7d"ej8        d#e9d$ej8        fd%Z:	 dOd'e	j.        d(ej8        d)ej8        d*ej8        d+eej8                 d,e;d-e;d.e e"         fd/Z< G d0 d1e	j.                  Z= G d2 d3e          Z> G d4 d5e          Z? G d6 d7e	j.                  Z@ G d8 d9e	j.                  ZA G d: d;e	j.                  ZB G d< d=e	j.                  ZC G d> d?e	j.                  ZD G d@ dA          ZEe# G dB dCe                      ZF e#dDE           G dF dGeF                      ZGe# G dH dIeF                      ZH e#dJE           G dK dLeFe                      ZIg dMZJdS )PzPyTorch Chameleon model.    )cached_property)CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)create_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg   )ChameleonConfigChameleonVQVAEConfigc                   ,     e Zd Zd fd	Zd Zd Z xZS )ChameleonRMSNormư>c                     t                                                       t          j        t	          j        |                    | _        || _        dS )z?
        ChameleonRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__s      /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/chameleon/modeling_chameleon.pyr"   zChameleonRMSNorm.__init__0   sD     	l5:k#:#:;; #    c                    |j         }|                    t          j                  }|                    d                              dd          }|t          j        || j        z             z  }| j        |                    |          z  S )N   T)keepdim)	dtypetor$   float32powmeanrsqrtr'   r&   )r(   hidden_statesinput_dtypevariances       r,   forwardzChameleonRMSNorm.forward8   s|    #)%((77 $$Q'',,R,>>%Ht?T4T(U(UU{]--k::::r-   c                 H    t          | j        j                   d| j         S )Nz, eps=)tupler&   shaper'   r(   s    r,   
extra_reprzChameleonRMSNorm.extra_repr?   s&    )**II$2GIIIr-   )r   )__name__
__module____qualname__r"   r;   r@   __classcell__r+   s   @r,   r   r   /   sb        $ $ $ $ $ $; ; ;J J J J J J Jr-   r   c                   f     e Zd ZU ej        ed<   d fd	Z ej                    d             Z xZ	S )	ChameleonRotaryEmbeddinginv_freq   '  N      ?c                 p   t                                                       || _        || _        || _        || _        d| j        t          j        d| j        dt          j                  	                    |t          j
                  | j        z  z  z  }|                     d|d           || _        d S )	NrK   r   r/   r2   devicer2   rH   F
persistent)r!   r"   scaling_factordimmax_position_embeddingsbaser$   arangeint64r3   floatregister_buffermax_seq_len_cached)r(   rS   rT   rU   rO   rR   rH   r+   s          r,   r"   z!ChameleonRotaryEmbedding.__init__H   s    ,'>$	IQ!5;???BB&X]XcBddgkgooq
 	ZeDDD"9r-   c                    | j         d d d d f                                                             |j        d         dd          }|d d d d d f                                         }|j        j        }|dk    r|nd}t          j        |d          5  |                                |                                z                      dd          }t          j	        ||fd	          }|
                                }|                                }	d d d            n# 1 swxY w Y   |                    |j        
          |	                    |j        
          fS )Nr   r0   r   mpscpuF)device_typeenabledr/   rS   rM   )rH   rX   expandr>   rO   typer$   autocast	transposecatcossinr3   r2   )
r(   xposition_idsinv_freq_expandedposition_ids_expandedr^   freqsembrf   rg   s
             r,   r;   z ChameleonRotaryEmbedding.forwardV   s    !M$4-8>>@@GGHZ[\H]_acdee ,QQQaaaZ 8 > > @ @ hm%0E%9%9kku^UCCC 	 	&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))C''))C		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 vvAGv$$cff17f&;&;;;s   A>DD #D )rI   rJ   NrK   )
rA   rB   rC   r$   Tensor__annotations__r"   no_gradr;   rD   rE   s   @r,   rG   rG   E   sp         l: : : : : : U]__< < _< < < < <r-   rG   c                   "     e Zd ZdZ fdZ xZS )%ChameleonLinearScalingRotaryEmbeddingz_ChameleonRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendevc                     |                                 | j        z  }t                                          ||          \  }}||fS N)rX   rR   r!   r;   )r(   rh   ri   rf   rg   r+   s        r,   r;   z-ChameleonLinearScalingRotaryEmbedding.forwardj   s@    #))++d.AA77??1l33SCxr-   rA   rB   rC   __doc__r;   rD   rE   s   @r,   rr   rr   g   s>        ii        r-   rr   c                   "     e Zd ZdZ fdZ xZS ))ChameleonDynamicNTKScalingRotaryEmbeddingzqChameleonRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozillac                    t          j        |          dz   }|| j        k    r| j        | j        |z  | j        z  | j        dz
  z
  | j        | j        dz
  z  z  z  }d|t          j        d| j        dt           j                                      |j	        t           j
                  | j        z  z  z  }|                     d|d	           t                                          ||          \  }}||fS )
Nr   r/   rK   r   rM   rN   rH   FrP   )r$   maxrT   rU   rR   rS   rV   rW   r3   rO   rX   rY   r!   r;   )	r(   rh   ri   seq_lenrU   rH   rf   rg   r+   s	           r,   r;   z1ChameleonDynamicNTKScalingRotaryEmbedding.forwardt   s    )L))A-T1119$w.1MMRVRehiRij(dhl+ - -D LDHau{CCCFFah^c^iFjjmqmuuwH   X% HHH77??1l33SCxr-   ru   rE   s   @r,   rx   rx   q   s>        {{        r-   rx   c                     | dd| j         d         dz  f         }| d| j         d         dz  df         }t          j        | |fd          S )z*Rotates half the hidden dims of the input..Nr0   r/   r`   )r>   r$   re   )rh   x1x2s      r,   rotate_halfr      s]    	
3"!'"+"""	#B	
3q """	#B9rc2YB''''r-   c                     |                     |          }|                     |          }| |z  t          |           |z  z   }||z  t          |          |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkrf   rg   ri   unsqueeze_dimq_embedk_embeds           r,   apply_rotary_pos_embr      sc    ( --
&
&C
--
&
&C3w;q>>C/0G3w;q>>C/0GGr-   c                   $     e Zd Z fdZd Z xZS )ChameleonMLPc                    t                                                       || _        |j        | _        |j        | _        t          j        | j        | j        |j                  | _        t          j        | j        | j        |j                  | _	        t          j        | j        | j        |j                  | _
        t          |j                 | _        d S )Nbias)r!   r"   configr)   intermediate_sizer   Linearmlp_bias	gate_projup_proj	down_projr	   
hidden_actact_fnr(   r   r+   s     r,   r"   zChameleonMLP.__init__   s    !-!'!94#3T5KRXRabbby!143IPVP_```4#94;KRXRabbbV./r-   c                     |                      |                     |                     |                    |                     |          z            }|S rt   )r   r   r   r   )r(   rh   r   s      r,   r;   zChameleonMLP.forward   sA    NN4;;t~~a/@/@#A#ADLLQROO#STT	r-   rA   rB   rC   r"   r;   rD   rE   s   @r,   r   r      sG        0 0 0 0 0      r-   r   c                   (     e Zd ZdZ fdZd Z xZS )ChameleonLayerNorma  
    LayerNorm but computes stats only over the last dim because Chameleon applies gamma and beta
    from each shard separately to each head, instead of reducing. We can apply each head's own
    gamma/beta by repeat-interleaving weights from each shard, but the stats have to be computed
    in the last dimension. This module applies gamma/beta manually to fulfill this requirement.
    c                 ^     t                      j        |g|R i | |d         f| _        d S )Nr0   )r!   r"   normalized_shape)r(   r)   argskwargsr+   s       r,   r"   zChameleonLayerNorm.__init__   s?    6t666v666!,R 2r-   c                 f    t          j        || j        d d d          }|| j        z  | j        z   }|S )Ngh㈵>r*   )F
layer_normr   r&   r   r(   r8   s     r,   r;   zChameleonLayerNorm.forward   s:    ]D4I4QU[_```%3di?r-   )rA   rB   rC   rv   r"   r;   rD   rE   s   @r,   r   r      sQ         3 3 3 3 3      r-   r   r8   n_repreturnc                     | j         \  }}}}|dk    r| S | dddddddddf                             |||||          } |                     |||z  ||          S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r>   ra   reshape)r8   r   batchnum_key_value_headsslenhead_dims         r,   	repeat_kvr      s    
 2?1D.Ehzz!!!!QQQaaa"23::5BUW\^bdlmmM  (;e(CT8TTTr-           modulequerykeyvalueattention_maskscalingdropoutr   c                 R   t          || j                  }t          || j                  }	t          j        ||                    dd                    |z  }
|$|d d d d d d d |j        d         f         }|
|z   }
t          j                            |
dt          j	                  
                    |j                  }
t          j                            |
|| j                  }
t          j        |
|	          }|                    dd                                          }||
fS )Nr/   r   r0   )rS   r2   )ptrainingr   )r   num_key_value_groupsr$   matmulrd   r>   r   
functionalsoftmaxr4   r3   r2   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r,   eager_attention_forwardr      s    3 ;<<JUF$?@@L<z';';Aq'A'ABBWLL!$QQQ111.D
0@0D.D%DE#k1=((2U](SSVVW\WbccL=((6?([[L,|\::K''1--88::K$$r-   c                   J    e Zd ZdZddedee         f fdZd Z e	ddd	
          	 	 	 	 	 	 dde
j        dee
j                 dee
j                 dee         dededee
j                 dee
j        ee
j                 eee
j                          f         fd            Z xZS )ChameleonAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   	layer_idxc                 |   t                                                       || _        || _        |(t                              d| j        j         d           |j        | _        |j	        | _	        |j
        | _        | j	        | j        z  | _        |j        | _        | j        | j        z  | _        |j        | _        |j        | _        d| _        |j        | _        | j        dz  | _        | j        | j        z  | j	        k    r t)          d| j	         d| j         d          t+          j        | j	        | j        | j        z  |j                  | _        t+          j        | j	        | j        | j        z  |j                  | _        t+          j        | j	        | j        | j        z  |j                  | _        t+          j        | j	        | j	        |j                  | _        t9          | j        | j        f          | _        t9          | j        | j        f          | _        |                                  d S )	NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.T      z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r   ) r!   r"   r   r   loggerwarning_oncer+   rA   attention_dropoutr)   num_attention_heads	num_headsr   r   r   rT   
rope_theta	is_causalmodel_parallel_sizer   
ValueErrorr   r   attention_biasq_projk_projv_projo_projr   q_normk_norm
_init_roper(   r   r   r+   s      r,   r"   zChameleonAttention.__init__   s   ",!8 , , ,   "(!9!-3(DN:#)#= $(Nd6N$N!'-'E$ +#)#= }d*MDN*t/???8RVRb 8 8%)^8 8 8  
 i 0$.4=2PW]Wlmmmi 0$2JT]2Zagavwwwi 0$2JT]2Zagavwwwi 0$2BI^___($.$-)HII($*BDM)RSSr-   c                    | j         j        (t          | j        | j        | j                  | _        d S | j         j        d         }| j         j        d         }|dk    r)t          | j        | j        || j                  | _        d S |dk    r)t          | j        | j        || j                  | _        d S t          d|           )N)rT   rU   rb   factorlinear)rT   rR   rU   dynamiczUnknown RoPE scaling type )
r   rope_scalingrG   r   rT   r   
rotary_embrr   rx   r   )r(   scaling_typerR   s      r,   r   zChameleonAttention._init_rope  s    ;#+6(,(D_  DOOO  ;3F;L![5h?Nx''"GM,0,H#1	# # # **"KM,0,H#1	# # # !!Ll!L!LMMMr-   past_key_valuepast_key_values4.58new_nameversionFr8   r   ri   output_attentions	use_cachecache_positionr   c                    |                                 \  }	}
}|                     |          }|                     |          }|                     |          }|                    d| j        | j                  }|                     |          }|                    d| j        | j                  }| 	                    |          }|                    |	|
| j        | j                  
                    dd          }|                    |	|
| j        | j                  
                    dd          }|                    |	|
| j        | j                  
                    dd          }|                     ||          \  }}t          ||||          \  }}|&|||d}|                    ||| j        |          \  }}t           }| j        j        dk    rt&          | j        j                 } || ||||f| j        sdn| j        | j        d|\  }}|                    |	|
d                                          }|                     |          }||fS )Nr0   r   r/   )rg   rf   r   eagerr   )r   r   )sizer   r   r   r   r   r   r   r   r   rd   viewr   r   updater   r   r   _attn_implementationr   r   r   r   r   r   )r(   r8   r   ri   r   r   r   r   r   bszq_len_query_statesr   r   rf   rg   cache_kwargsattention_interfacer   r   s                        r,   r;   zChameleonAttention.forward:  sX    &**,,UA{{=11[[//
{{=11#++BNN{{<00''D,DdmTT
[[,,
#++CVV``abdeff''UD4Ldm\\ffghjkll
#((eT5Mt}]]gghiklmm??<>>S#7jRUWZ#[#[ j&#&snUUL'6'='=j,X\Xfht'u'u$J(?;+w66"9$+:Z"[$7$7	%
  $}HCC$2HL	%
 	%
 	%
 	%
!\ "))#ub99DDFFkk+..L((r-   rt   NNNFFN)rA   rB   rC   rv   r   r   intr"   r   r   r$   rn   
LongTensorr
   boolr=   r;   rD   rE   s   @r,   r   r      sG       GG# # #8C= # # # # # #NN N N6 _%0A6RRR 2637+/"'595) 5)|5) !.5) u/0	5)
 "%5)  5) 5) !!125) 
u|Xel3XeEL>Q5RR	S5) 5) 5) SR5) 5) 5) 5) 5)r-   r   c                   @    e Zd Zdedef fdZ eddd          	 	 	 	 	 	 dd
ej        de	ej                 de	ej
                 de	e         de	e         de	e         de	ej
                 deej        e	eej        ej        f                  f         fd            Z xZS )ChameleonDecoderLayerr   r   c                 4   t                                                       |j        | _        t          ||          | _        t          |          | _        t          |j        |j                  | _	        t          |j        |j                  | _
        d S N)r   r   r   r!   r"   r)   r   	self_attnr   mlpr   rms_norm_epsinput_layernormpost_attention_layernormr   s      r,   r"   zChameleonDecoderLayer.__init__u      !-+6YOOO''/0BH[\\\(89KQWQd(e(e(e%%%r-   r   r   r   r   NFr8   r   ri   r   r   r   r   c                     |}	|                      |          } | j        d|||||||d|\  }}
|	|z   }|}	|                     |          }|                     |          }|	|z   }|f}|r||
fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        r8   r   ri   r   r   r   r    )r  r   r  r  r(   r8   r   ri   r   r   r   r   r   residualself_attn_weightsoutputss               r,   r;   zChameleonDecoderLayer.forward  s    > !,,];; ,:4> 	,
')%+/)	,
 	,
 	,
 	,
(( !=0 !55mDD// =0 " 	,)++Gr-   r   rA   rB   rC   r   r   r"   r   r$   rn   r   r   r
   r   r=   FloatTensorr;   rD   rE   s   @r,   r   r   t  s6       f f3 f f f f f f _%0A6RRR 2637+/,1$)59: :|: !.: u/0	:
 "%: $D>: D>: !!12: 
u (51BEDU1U+V"WW	X: : : SR: : : : :r-   r   c                   @    e Zd Zdedef fdZ eddd          	 	 	 	 	 	 dd
ej        de	ej                 de	ej
                 de	e         de	e         de	e         de	ej
                 deej        e	eej        ej        f                  f         fd            Z xZS )ChameleonSwinDecoderLayerr   r   c                 4   t                                                       |j        | _        t          ||          | _        t          |          | _        t          |j        |j                  | _	        t          |j        |j                  | _
        d S r   r   r   s      r,   r"   z"ChameleonSwinDecoderLayer.__init__  r  r-   r   r   r   r   NFr8   r   ri   r   r   r   r   c                     |}	 | j         d|||||||d|\  }}
|                     |          }|	|z   }|}	|                     |          }|                     |          }|	|z   }|f}|r||
fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
        r  r  )r   r  r  r  r	  s               r,   r;   z!ChameleonSwinDecoderLayer.forward  s    @ ! ,:4> 	,
')%+/)	,
 	,
 	,
 	,
(( ,,];; =0 //55mDD =0 " 	,)++Gr-   r   r  rE   s   @r,   r  r    s6       f f3 f f f f f f _%0A6RRR 2637+/,1$)598 8|8 !.8 u/0	8
 "%8 $D>8 D>8 !!128 
u (51BEDU1U+V"WW	X8 8 8 SR8 8 8 8 8r-   r  c                   8     e Zd ZdZ fdZdej        fdZ xZS )ChameleonVQVAEVectorQuantizera  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    c                     t                                                       |j        | _        |j        | _        t          |dd          | _        t          j        | j        | j                  | _	        d S )Nbetag      ?)
r!   r"   num_embeddings	embed_dimembedding_dimgetattrr  r   	Embedding	embeddingr   s     r,   r"   z&ChameleonVQVAEVectorQuantizer.__init__  s_    $3#-FFD11	d&94;MNNr-   hidden_statec           
      R   |                     dddd                                          }|                    d| j                  }t	          j        |dz  dd          t	          j        | j        j        dz  d          z   dt	          j        d	|| j        j        	                    dd                    z  z
  }t	          j
        |d          }|                     |                              |j                  }t	          j        |                                |z
  dz            | j        t	          j        ||                                z
  dz            z  z   }|||z
                                  z   }|                     dddd                                          }|||fS )
Nr   r/   r   r   r0   T)rS   r1   r`   z	bd,dn->bn)permuter   r   r  r$   sumr  r&   einsumrd   argminr>   r6   detachr  )r(   r  hidden_state_flattened	distancesmin_encoding_indiceshidden_state_quantlosss          r,   r;   z%ChameleonVQVAEVectorQuantizer.forward  s   #++Aq!Q77BBDD!-!2!22t7I!J!J I,a/QEEEi-q0a8889%,{,BDNDYDcDcdeghDiDijjjk 	  %|I1===!^^,@AAFF|GYZZ z-4466E!KLLty[`[e,"5"5"7"77A=\
 \
 P
 

 *-?,-N,V,V,X,XX 0771aCCNNPP!4)===r-   )	rA   rB   rC   rv   r"   r$   rn   r;   rD   rE   s   @r,   r  r    sd         O O O O O>EL > > > > > > > >r-   r  c                   $     e Zd Z fdZd Z xZS )#ChameleonVQVAEEncoderConvDownsamplec                     t                                                       t          j        ||ddd          | _        d S )Nr   r/   r   kernel_sizestridepadding)r!   r"   r   Conv2dconvr(   in_channelsr+   s     r,   r"   z,ChameleonVQVAEEncoderConvDownsample.__init__4  s:    Ik;AaYZ[[[			r-   c                 `    t          j        |ddd          }|                     |          }|S )N)r   r   r   r   constantr   )padmoder   )r   r6  r1  r   s     r,   r;   z+ChameleonVQVAEEncoderConvDownsample.forward8  s2    mJVWXXX		-00r-   r   rE   s   @r,   r*  r*  3  sL        \ \ \ \ \      r-   r*  c                   *     e Zd Z	 	 d fd	Zd Z xZS ) ChameleonVQVAEEncoderResnetBlockNFc                    t                                                       || _        ||n|| _        || _        t
          j                            d|dd          | _        t
          j        	                    ||ddd          | _
        t
          j                            d|dd          | _        t
          j                            |j                  | _        t
          j        	                    ||ddd          | _        | j        | j        k    r]| j        r+t
          j        	                    ||ddd          | _        d S t
          j        	                    ||ddd          | _        d S d S )	N    r   T
num_groupsnum_channelsr*   affiner   r   r,  r   )r!   r"   r3  out_channelsuse_conv_shortcutr$   r   	GroupNormnorm1r0  conv1norm2Dropoutr   conv2conv_shortcutnin_shortcut)r(   r   r3  r@  rH  r+   s        r,   r"   z)ChameleonVQVAEEncoderResnetBlock.__init__@  sR    	&+7+?KK\!.X''2KUYbf'gg
X__[,AVWab_cc
X''2LVZcg'hh
x''77X__\<QWXbc_dd
t000% s%*X__[,\]fgqr_%s%s"""$)HOOK[\efpqO$r$r!!!	 10r-   c                    |}|                      |          }|t          j        |          z  }|                     |          }|                     |          }|t          j        |          z  }|                     |          }|                     |          }| j        | j        k    r2| j	        r| 
                    |          }n|                     |          }||z   S rt   )rC  r$   sigmoidrD  rE  r   rG  r3  r@  rA  rH  rI  )r(   r8   r
  s      r,   r;   z(ChameleonVQVAEEncoderResnetBlock.forwardW  s     

=11}555

=11

=11}555]33

=11t000% 7--h77,,X66-''r-   )NFr   rE   s   @r,   r9  r9  ?  sZ        
 s s s s s s.( ( ( ( ( ( (r-   r9  c                   $     e Zd Z fdZd Z xZS )ChameleonVQVAEEncoderAttnBlockc                    t                                                       || _        t          j                            d|dd          | _        t          j                            ||ddd          | _        t          j                            ||ddd          | _	        t          j                            ||ddd          | _
        t          j                            ||ddd          | _        d S )Nr;  r   Tr<  r   r   r,  )r!   r"   r3  r$   r   rB  normr0  r   r   vproj_outr2  s     r,   r"   z'ChameleonVQVAEEncoderAttnBlock.__init__l  s    &H&&";TXae&ff	kqQR\]^^kqQR\]^^kqQR\]^^[aXYcdeer-   c                    |}|                      |          }|                     |          }|                     |          }|                     |          }|j        \  }}}}	|                    ||||	z                                ddd          }|                    ||||	z            }t          j        ||          }
|
t          |          dz  z  }
t          j        |
d          }
|                    ||||	z            }|
                    ddd          }
t          j        ||
                              ||||	          }|                     |          }||z   S )Nr   r/   r   r   r`   )rO  r   r   rP  r>   r   r  r$   bmmr   r   r   rQ  )r(   r8   r
  r   r   r   
batch_sizechannelsheightwidthr   r   s               r,   r;   z&ChameleonVQVAEEncoderAttnBlock.forwardv  s[    		-00vvm,,VVM**
vvm,, /;.@+
Hfe#++J&5.QQYYZ[]^`abb''
HfunMM
yz::#s8}}'>?y1555 $++J&5.QQ#++Aq!44il;;CCJPXZ`bghhmmK00+%%r-   r   rE   s   @r,   rM  rM  k  sL        f f f f f& & & & & & &r-   rM  c                   4     e Zd Z fdZdej        fdZ xZS )ChameleonVQVAEEncoderc           	         t                                                       t          |j                  | _        |j        | _        |j        }|j        }|j        }|j	        }|j
        }|j        }t          j                            ||ddd          | _        |}dt          |          z   }	|	| _        t          j                    | _        t'          | j                  D ]}
t          j                    }t          j                    }||	|
         z  }|||
         z  }t'          | j                  D ]f}|                    t+          |||                     |}|j        6||j        v r-|j        dk    r"|                    t1          |                     gt          j                    }||_        ||_        |
| j        dz
  k    rt9          |          |_        |dz  }| j                            |           t          j                    | _        t+          |||          | j        _        |j        dk    rt1          |          nt          j                     | j        _!        t+          |||          | j        _"        t          j        #                    d|d	d
          | _$        t          j                            ||rd|z  n|ddd          | _%        d S )Nr   r   r,  )r   )r   r3  r@  vanillar/   r;  r   Tr<  )&r!   r"   lenchannel_multipliernum_resolutionsnum_res_blocksbase_channels
resolutionr3  double_latentlatent_channelsr$   r   r0  conv_inr=   in_channel_multiplier
ModuleListdownrangeappendr9  attn_resolutions	attn_typerM  Moduleblockattnr*  
downsamplemidblock_1Identityattn_1block_2rB  norm_outconv_out)r(   r   r`  ra  r3  rb  rc  r]  curr_resre  i_levelrm  rn  block_in	block_outi_blockrg  r+   s                    r,   r"   zChameleonVQVAEEncoder.__init__  s   "6#<==$3,&
(, 0#6x{MqYZdeff $u-?'@'@ @%:"MOO	T122 	# 	#GMOOE=??D$'<W'EEH%(:7(CCI !455 J J4%$,%.     %+7 F$;;;(I55KK >x H HIII9;;DDJDI$.222"Eh"O"O#q=IT""""9;;; !
 
 

 GMFVZcFcFc8BBBikitiviv; !
 
 
 **bxUYbf*gg#0EAo ( 
 
r-   pixel_valuesc                 J   |                      |          g}t          | j                  D ]}t          | j                  D ]} | j        |         j        |         |d                   }t          | j        |         j                  dk    r! | j        |         j        |         |          }|                    |           || j        dz
  k    r9|                    | j        |         	                    |d                              |d         }| j
                            |          }| j
                            |          }| j
                            |          }|                     |          }|t          j        |          z  }|                     |          }|S )Nr0   r   r   )rd  rh  r^  r_  rg  rm  r\  rn  ri  ro  rp  rq  rs  rt  ru  r$   rK  rv  )r(   r|  r8   rx  r{  r  last_hidden_states          r,   r;   zChameleonVQVAEEncoder.forward  s   l334T122 		W 		WG !455 3 3@ty17@!"%    ty).//!33#C49W#5#:7#CL#Q#QL$$\2222$.222$$TYw%7%B%B=QSCT%U%UVVV *"- H,,->?? HOO,=>> H,,->?? !MM*;<<U]+<=== MM*;<<  r-   )rA   rB   rC   r"   r$   r   r;   rD   rE   s   @r,   rY  rY    s\        C
 C
 C
 C
 C
J!E$4 ! ! ! ! ! ! ! !r-   rY  c                       e Zd ZdZd Zed             Zed             Zed             Zed             Z	ed             Z
ed             Zd	ej        d
ej        fdZdS )ChameleonImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 H    || _         |                    d          | _        d S )Nz<image>)	vocab_mapgetimage_token_id)r(   r  s     r,   r"   z(ChameleonImageVocabularyMapping.__init__  s#    "'mmI66r-   c                 H    d | j                                         D             S )Nc                     i | ]\  }}||	S r  r  .0r   rP  s      r,   
<dictcomp>z<ChameleonImageVocabularyMapping.val2name.<locals>.<dictcomp>  s    888A1888r-   )r  itemsr?   s    r,   val2namez(ChameleonImageVocabularyMapping.val2name  s$    88!5!5!7!78888r-   c                 b    t          d | j                                        D                       S )Nc                 B    g | ]\  }}|                     d           |S )IMGIMG)
startswith)r  namevals      r,   
<listcomp>z@ChameleonImageVocabularyMapping.image_tokens.<locals>.<listcomp>  s.    ```ytSdooV^F_F_`s```r-   )sortedr  r  r?   s    r,   image_tokensz,ChameleonImageVocabularyMapping.image_tokens  s-    ``DN,@,@,B,B```aaar-   c                      d t          d          D             dt          dt          ffd fd j        D             S )Nc                 h    i | ]/}t          t          d           |z             t          |          0S )A)chrordstr)r  is     r,   r  z;ChameleonImageVocabularyMapping.bpe2img.<locals>.<dictcomp>  s2    LLLQs3s88a<00#a&&LLLr-   
   old_namer   c                 p    d                     fd| t          d          d         D                       S )N c              3   D   K   | ]}                     ||          V  d S rt   )r  )r  cimg_tkn_chr_mappings     r,   	<genexpr>zIChameleonImageVocabularyMapping.bpe2img.<locals>.remap.<locals>.<genexpr>  s4      __Q.221a88______r-   r  r0   )joinr\  )r  r  s    r,   remapz6ChameleonImageVocabularyMapping.bpe2img.<locals>.remap  s;    77____(3x==[]K]B^______r-   c           	      X    i | ]&}|t           j        |                             'S r  )r   r  )r  tokr  r(   s     r,   r  z;ChameleonImageVocabularyMapping.bpe2img.<locals>.<dictcomp>  s4    QQQSt}S12233QQQr-   )rh  r  r  )r(   r  r  s   `@@r,   bpe2imgz'ChameleonImageVocabularyMapping.bpe2img   sv    LL%))LLL	`C 	`C 	` 	` 	` 	` 	` 	` RQQQQt?PQQQQr-   c                 H    d | j                                         D             S )Nc                     i | ]\  }}||	S r  r  r  s      r,   r  z;ChameleonImageVocabularyMapping.img2bpe.<locals>.<dictcomp>  s    666A1666r-   )r  r  r?   s    r,   img2bpez'ChameleonImageVocabularyMapping.img2bpe	  s$    66!3!3!5!56666r-   c                     t          j        t          | j                                                            t          j        t          | j                                                            fS rt   )r$   tensorr  r  keysvaluesr?   s    r,   bpe2img_search_tensorsz6ChameleonImageVocabularyMapping.bpe2img_search_tensors  sM    |F4<#4#4#6#67788%,vdlNaNaNcNcGdGd:e:eeer-   c                     t          j        t          | j                                                  dz   t           j                  }| j                                        D ]
\  }}|||<   |S )Nr   rM   )r$   zerosrz   r  r  r   r  )r(   mappingr   rP  s       r,   img2bpe_mapping_tensorz6ChameleonImageVocabularyMapping.img2bpe_mapping_tensor  sd    +c$,"3"3"5"566:%)LLLL&&(( 	 	DAqGAJJr-   	img_batchr   c                 z    |j         }| j        |                    d                   }|                    |          S )Nr]   )rO   r  r3   )r(   r  rO   
img_tokenss       r,   convert_img2bpez/ChameleonImageVocabularyMapping.convert_img2bpe  s5    !0e1D1DE
}}V$$$r-   N)rA   rB   rC   rv   r"   r   r  r  r  r  r  r  r$   rn   r  r  r-   r,   r  r    s         7 7 7 9 9 _9 b b _b R R _R 7 7 _7 f f _f   _% %%, % % % % % %r-   r  c                   J    e Zd ZU eed<   dZdZddgZddgZdZ	dZ
dZdZdZdZd	S )
ChameleonPreTrainedModelr   modelTr   r  r   r   FN)rA   rB   rC   r   ro   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph!_supports_param_buffer_assignment_supports_flex_attn_supports_attention_backendr  r-   r,   r  r    sd         &*#02MN#4m"DN!(-%"&r-   r  aW  
    The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    )custom_introc                   N     e Zd ZU eed<   g dZdef fdZdej        fdZ	 xZ
S )ChameleonVQVAEr   )r  rM  r9  c                    t                                          |           t          |          | _        t	          |          | _        t          j                            |j	        |j
        d          | _        t          j                            |j
        |j	        d          | _        |                                  d S Nr   )r!   r"   rY  encoderr  quantizer$   r   r0  rc  r  
quant_convpost_quant_convevalr   s     r,   r"   zChameleonVQVAE.__init__>  s       ,V445f==(//&*@&BRTUVV$xv/?AWYZ[[		r-   r|  c                     |                      |          }|                     |          }|                     |          \  }}}|||fS rt   )r  r  r  )r(   r|  r8   quantemb_lossindicess         r,   encodezChameleonVQVAE.encodeG  sI    \2266#'==#?#? xh''r-   )rA   rB   rC   r   ro   r  r"   r$   r   r  rD   rE   s   @r,   r  r  .  s          !     3      (5#3 ( ( ( ( ( ( ( (r-   r  c                       e Zd Zdef fdZdej        fdZdej        fdZdej	        dej        dej        fd	Z
e	 	 	 	 	 	 	 	 	 	 	 ddeej	                 deej                 deej                 deej	                 dee         deej                 dee         dee         dee         dee         deej	                 dee         deeef         fd            Z xZS )ChameleonModelr   c                 f   t                                                     j        | _        j        | _        t          j        j        j        | j                  | _        t          j
                  | _        | j        j        st          nt          t          j        fdt#          j                  D                       | _        t)          j        j                  | _        t.                              j                  | _        d| _        |                                  d S )Nc                 (    g | ]} |          S r  r  )r  r   r   decoder_layers     r,   r  z+ChameleonModel.__init__.<locals>.<listcomp>Y  s%    ___)]]69--___r-   r   F)r!   r"   pad_token_idpadding_idx
vocab_sizer   r  r)   embed_tokensr  vocabulary_mapvocabulary_mappingr   	swin_normr   r  rf  rh  num_hidden_layerslayersr   r  rO  r  _from_config	vq_configvqmodelgradient_checkpointing	post_init)r(   r   r  r+   s    `@r,   r"   zChameleonModel.__init__P  s      !. +L):F<NPTP`aa"A&BW"X"X59[5Ji--Pim_____uVE]?^?^___
 
 %V%7V=PQQQ	%2263CDD&+# 	r-   r|  c                     |j         d         }| j                            |          \  }}}| j                            |          }|                    |d          }|S )as  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
                The tensors corresponding to the input images.
        r   r0   )r>   r  r  r  r  r   )r(   r|  rT  r   
image_toksbpe_tokss         r,   get_image_tokenszChameleonModel.get_image_tokensb  s[     "'*
<..|<<1j*:::FF==R00r-   c                 j    |                      |          } |                                 |          }|S )a7  
        Tokenizes images into discrete tokens with VQGAN module and embeds
        them with text embeddings layer

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
                The tensors corresponding to the input images.
        )r  get_input_embeddings)r(   r|  r  vision_embeddingss       r,   get_image_featuresz!ChameleonModel.get_image_featuresr  s9     ,,\::7D5577EE  r-   	input_idsinputs_embedsimage_featuresc                 \   |e| |                                  t          j        | j        j        t          j        |j                            k    }|                    d          }n|| j        j        k    }|                                }|	                    d          
                    |                              |j                  }|j        d         |j        d         z  }||                                         |                                k    rt          d| d|           |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N)r2   rO   r0   r   r   z6Image features and image tokens do not match: tokens: z, features )r  r$   r  r  r  longrO   allr   r   	expand_asr3   r>   numelr   )r(   r  r  r  special_image_maskn_image_tokensn_image_featuress          r,   get_placeholder_maskz#ChameleonModel.get_placeholder_mask  s1    !.2M$2K2K2M2MT4C5:^k^rsss3 3 " "4!7!7!;!;!*d.E.T!T+//11/99"==GGVVYYZgZnoo)/2^5I!5LL+,22448L8L8N8NNNvvvdtvv   "!r-   Nr   ri   r   r   r   output_hidden_statesreturn_dictr   r   r   c                 D   ||n| j         j        }|	|	n| j         j        }	||n| j         j        }|
|
n| j         j        }
| j        r%| j        r|rt                              d           d}|d u |d uz  rt          d          || 
                    |          }|C|                     |          }|                     |||          }|                    ||          }|r5|3t          j                                        st#          | j                   }|B||                                nd}t          j        |||j        d         z   |j                  }||                    d          }t/          | j         |||||	          }|}|	rd
nd }|rd
nd }| j        D ]0}|	r||fz  } ||f||||||d|}|d         }|r||d         fz  }1|                     |          }|	r||fz  }|
st5          d ||||fD                       S t7          ||||          S )NzX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fz:You must specify exactly one of input_ids or inputs_embeds)r  r  )r   r   r   )rO   )r   input_embedsr   r   r   ri   r  )r   ri   r   r   r   r   c              3      K   | ]}||V  	d S rt   r  )r  rP  s     r,   r  z)ChameleonModel.forward.<locals>.<genexpr>  s1        bcbobobobobo r-   )r~  r   r8   
attentions)r   r   r  r   use_return_dictr  r   r   r   r   r  r  r  masked_scatterr$   jit
is_tracingr   get_seq_lengthrV   r>   rO   r   r   r  rO  r=   r   )r(   r  r|  r   ri   r   r  r   r   r  r  r   r   image_embedsr  past_seen_tokensr   r8   all_hidden_statesall_self_attnsr  layer_outputss                         r,   r;   zChameleonModel.forward  s2     2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]& 	4= 	Y 	j   I-t";< 	[YZZZ  --i88M#22<@@L!%!:!:| "; " " *889K\ZZM  	?09M9M9O9O0*$+>>>O!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L(;&))+%
 
 
 & #7@BBD0:d![ 	6 	6M# 6!m%55!)M	*) /"3#-	 	 	 	M *!,M  6=#3"55		-00   	2-!11 	  )?<M~^      '+++%	
 
 
 	
r-   NNNNNNNNNNN)rA   rB   rC   r   r"   r$   r  r  r  r   r  r   r   rn   r
   r   r   r   r   r=   r   r;   rD   rE   s   @r,   r  r  N  s             $U->     !u/@ ! ! ! !")":?:K"]b]n" " " "0  15481537+/59$(,0/3&*59j
 j
E,-j
 u01j
 !.	j

 u/0j
 "%j
   12j
 D>j
 $D>j
 'tnj
 d^j
 !!12j
 -.j
 
u--	.j
 j
 j
 ^j
 j
 j
 j
 j
r-   r  zb
    Chameleon Model with a head on top used for outputting logits for next token prediction.
    c                       e Zd ZdgZ fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 dde	e
j                 de	e
j                 de	e
j                 d	e	e
j                 d
e	e         de	e
j                 de	e
j                 de	e         de	e         de	e         de	e
j                 dee         deeef         fd                        Z	 	 	 	 	 	 	 d fd	Z xZS )!ChameleonForConditionalGenerationzlm_head.weightc                     t                                          |           t          |          | _        |j        | _        t          j        |j        |j        d          | _        | 	                                 d S )NFr   )
r!   r"   r  r  r  r   r   r)   lm_headr  r   s     r,   r"   z*ChameleonForConditionalGeneration.__init__  sj       #F++
 +y!3V5FUSSS 	r-   c                 6    | j                             |          S rt   )r  r  r(   r|  s     r,   r  z2ChameleonForConditionalGeneration.get_image_tokens  s    z**<888r-   c                 6    | j                             |          S rt   )r  r  r  s     r,   r  z4ChameleonForConditionalGeneration.get_image_features  s    z,,\:::r-   Nr  r|  r   ri   r   r  labelsr   r   r  r   r   r   c                    |	|	n| j         j        }	|
|
n| j         j        }
 | j        d||||||||	|
d|d|}|d         }|                     |          }| j        j        j        }t          j        |j	                  j
        |dddd|f<   d}| | j        d||| j         j        d|}t          |||j        |j        |j                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import ChameleonProcessor, ChameleonForConditionalGeneration
        >>> import torch
        >>> import requests
        >>> from PIL import Image

        >>> model = ChameleonForConditionalGeneration.from_pretrained("facebook/chameleon-7b", dtype=torch.bfloat16)
        >>> processor = ChameleonProcessor.from_pretrained("facebook/chameleon-7b")

        >>> prompt = "I used to know a lot about constellations when I was younger, but as I grew older, I forgot most of what I knew. These are the only two constellations that I really remember now.<image><image>I would like for you to tell me about 3 more constellations and give me a little bit of history about the constellation."
        >>> image = Image.open(requests.get("https://nineplanets.org/wp-content/uploads/2020/12/the-big-dipper-1.jpg", stream=True).raw)
        >>> image_2 = Image.open(requests.get("https://www.kxan.com/wp-content/uploads/sites/40/2020/10/ORION.jpg", stream=True).raw)

        >>> inputs = processor(images=[image, image_2], text=prompt, return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```NT)r  r|  r   ri   r   r  r   r   r  r  r   r   )logitsr  r  )r(  r  r   r8   r  r  )r   r   r  r  r  r  r  r$   finfor2   minloss_functionr  r   r   r8   r  )r(   r  r|  r   ri   r   r  r  r   r   r  r   r   r  r8   r  r  r(  s                     r,   r;   z)ChameleonForConditionalGeneration.forward  s4   V 2C1N--TXT_Tq$8$D  $+Jj 	
 $* 
%)%+'/!5)
 
 
 
  
m,, z4A%*[%>%>%Bqqq!!!\!"%4%pVFt{OeppioppD%#3!/)
 
 
 	
r-   Tc	                 n     t                      j        |f|||||||d|	}
|d         dk    rd |
d<   |
S )N)r|  r   r   r  r   ri   r   r   r|  )r!   prepare_inputs_for_generation)r(   r  r|  r   r   r  r   ri   r   r   model_inputsr+   s              r,   r  z?ChameleonForConditionalGeneration.prepare_inputs_for_generationo  sm     =uww<

%+)')%

 

 

 

 !!! ,0L(r-   r  )NNNNNNT)rA   rB   rC   _tied_weights_keysr"   r  r  r   r   r   r$   r   r  rn   r
   r   r   r   r   r=   r   r;   r  rD   rE   s   @r,   r  r    s        ++    9 9 9; ; ;  15481537+/59-1$(,0/359O
 O
E,-O
 u01O
 !.	O

 u/0O
 "%O
   12O
 )*O
 D>O
 $D>O
 'tnO
 !!12O
 +,O
 
u,,	-O
 O
 O
 ^ O
h          r-   r  )r  r  r  r  r  )r   )Krv   	functoolsr   typingr   r   r   r$   torch.nn.functionalr   r   r   activationsr	   cache_utilsr
   r   
generationr   masking_utilsr   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.deprecationr   configuration_chameleonr   r   
get_loggerrA   r   rl  r   rG   rr   rx   r   r   r   	LayerNormr   rn   r   r   rX   r   r   r   r  r  r*  r9  rM  rY  r  r  r  r  r  __all__r  r-   r,   <module>r0     si     % % % % % % , , , , , , , , , ,                 ! ! ! ! ! ! . . . . . . . . ) ) ) ) ) ) / / / / / / B B B B B B 9 9 9 9 9 9 O O O O O O O O F F F F F F F F & & & & & &            1 0 0 0 0 0 J J J J J J J J 
	H	%	%J J J J Jry J J J,< < < < <ry < < <D    ,D       0H   *( ( (   8    29   "       &	UU\ 	U# 	U%, 	U 	U 	U 	U( % %I%<% 
% <	%
 U\*% % % '(% % % %4{) {) {) {) {) {) {) {)~F F F F F6 F F FRD D D D D : D D DN,> ,> ,> ,> ,>BI ,> ,> ,>^	 	 	 	 	") 	 	 	)( )( )( )( )(ry )( )( )(X &  &  &  &  &RY  &  &  &F^! ^! ^! ^! ^!BI ^! ^! ^!B,% ,% ,% ,% ,% ,% ,% ,%^ ' ' ' ' ' ' ' '   ( ( ( ( (- ( ( (0 s
 s
 s
 s
 s
- s
 s
 s
l   
D D D D D(@/ D D 
DN p
o
or-   