
    .`i/r                     N   U d Z ddlZddlmZ ddlmZ ddlmZ ddlZddl	m
c mZ ddlm
Z
 ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z: dZ;dZ<dZ=dZ> ee?          Z@deAfdZBdeAfdZCdeDfd ZEdeFeDef         dz  fd!ZGdeHfd"ZI G d# d$e
jJ                  ZK G d% d&e
jJ                  ZL G d' d(e
jJ                  ZM G d) d*e
jJ                  ZNe G d+ d,e
jJ                              ZO G d- d.e
jJ        e3e4          ZP G d/ d0eP          ZQ G d1 d2eP          ZReQeRd3ZSeFeDeTeP         f         eUd4<    G d5 d6eP          ZVdS )7z(Inference-only Grok (Grok1/Grok2) model.    N)Iterable)islice)Any)nn)	Attention)support_torch_compile)CacheConfig
VllmConfig)get_pp_group$get_tensor_model_parallel_world_size)init_logger)
GeluAndMul)FusedMoE)RMSNorm)MergedColumnParallelLinearQKVParallelLinearReplicatedLinearRowParallelLinear)LogitsProcessor)QuantizationConfig)get_rope)ParallelLMHeadVocabParallelEmbedding)default_weight_loadermaybe_remap_kv_scale_name)IntermediateTensors   )SupportsLoRA
SupportsPP)AutoWeightsLoaderis_pp_missing_parameter'make_empty_intermediate_tensors_factorymake_layersmaybe_prefixg;f?g3Ey?g.!	S@      >@returnc           	      B    t          | dt          | dd                    S )Nnum_expertsnum_local_experts   )getattrconfigs    t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/grok1.py_get_num_expertsr/   P   s!    6='&:Mq*Q*QRRR    c                 .    t          | d| j                  S )Nmoe_intermediate_size)r+   intermediate_sizer,   s    r.   _get_moe_intermediate_sizer4   T   s    62F4LMMMr0   c                 T    t          | dd          }t          | d          }|s|rdS dS )z=Detect Grok version from HF config using multiple heuristics.residual_moeFr2   grok2grok1)r+   hasattr)r-   has_residual_moehas_moe_intermediate_sizes      r.   _get_grok_versionr<   X   sA     v~u== '0G H H 4 w7r0   c                 R   t          | dd           }|ft          | dd           }|d S d|i}t          | dd           }|||d<   t          | dd           }|||d<   dD ]}t          | |d           }||||<   |                    d          dk    rt          |          }d|d<   |S )	Nrope_parameters	rope_type
rope_thetascaling_factorfactor) original_max_position_embeddingsextrapolation_factorattn_factor	beta_fast	beta_sloworiginaldefault)r+   getdict)r-   r>   r?   r@   rA   namevalues          r.   _get_rope_parametersrN   d   s    f&7>>OFK66	4&	2V\488
!,6OL) )94@@%(6OH%
 		. 		.D FD$//E (-%;'':55//'0$r0   c           	          t          | dt          | dd                     }|t          |          S t          | dd           S )Nmoe_router_renormalizemoe_renormalizer6   F)r+   bool)r-   explicit_values     r.   _get_moe_renormalizerT      sQ    ('&:KT*R*R N !N###v~u5555r0   c                   d     e Zd Z	 	 ddedededz  deddf
 fdZd	ej        dej        fd
Z	 xZ
S )Grok1MLPN hidden_sizer3   quant_configprefixr&   c                     t                                                       t          ||gdz  d|| d          | _        t	          ||d|| d          | _        t                      | _        d S )N   Fz.gate_up_proj)
input_sizeoutput_sizesbiasrY   rZ   z
.down_proj)r]   output_sizer_   rY   rZ   )super__init__r   gate_up_projr   	down_projr   act_fn)selfrX   r3   rY   rZ   	__class__s        r.   rb   zGrok1MLP.__init__   s     	6"+,q0%+++
 
 
 +(#%(((
 
 
 !llr0   xc                     |                      |          \  }}|                     |          }|                     |          \  }}|S N)rc   re   rd   )rf   rh   _s      r.   forwardzGrok1MLP.forward   sB      ##1KKNN~~a  1r0   )NrW   )__name__
__module____qualname__intr   strrb   torchTensorrl   __classcell__rg   s   @r.   rV   rV      s        
 37# ## # )4/	#
 # 
# # # # # #0 %,        r0   rV   c                        e Zd ZdZ	 	 	 	 	 	 ddededed	ed
edej        dz  dedz  dedz  de	de
f fdZdej        dej        fdZ xZS )Grok1MoEa  A tensor-parallel MoE implementation for Grok1 that shards each expert
    across all ranks.

    Each expert's weights are sharded across all ranks and a fused MoE
    kernel is used for the forward pass, and finally we reduce the outputs
    across ranks.
            NFrW   r(   top_krX   r3   router_logit_soft_capparams_dtyperY   tp_sizerenormalizerZ   c                     t                                                       || _        t          ||d|d |
 d          | _        t          |||||d|	||d|
 d          | _        || _        d S )NFz.gate)r_   r{   rY   rZ   Tgeluz.experts)r(   ry   rX   r3   r{   reduce_resultsr}   rY   r|   
activationrZ   )ra   rb   rX   r   gater   expertsrz   )rf   r(   ry   rX   r3   rz   r{   rY   r|   r}   rZ   rg   s              r.   rb   zGrok1MoE.__init__   s     	& %%###
 
 
	  ##/%#%&&&
 
 
 &;"""r0   hidden_statesr&   c                 *   |j         }|                    d| j                  }|                     |          \  }}| j        dk    r$| j        t          j        || j        z            z  }|                     ||          }|                    |          S )Nr   )shapeviewrX   r   rz   Ftanhr   )rf   r   
orig_shaperouter_logitsrk   final_hidden_statess         r.   rl   zGrok1MoE.forward   s    "(
%**2t/?@@99]33q%)) 6 ::: : M #ll=-HH"''
333r0   )rx   NNNFrW   )rm   rn   ro   __doc__rp   floatrr   dtyper   rR   rq   rb   rs   rl   rt   ru   s   @r.   rw   rw      s          (++/26"!'; ';'; '; 	';
 ';  %'; kD('; )4/'; t'; '; '; '; '; '; '; ';R4U\ 4el 4 4 4 4 4 4 4 4r0   rw   c                        e Zd Z	 	 	 	 	 	 ddededededeeef         dz  d	edz  d
edz  deddf fdZ	de
j        de
j        de
j        fdZ xZS )Grok1Attention   NrW   rX   	num_headsnum_kv_headsmax_positionr>   cache_configrY   rZ   r&   c
                 *   t                                                       || _        |	| _        t	                      }
|| _        | j        |
z  dk    sJ | j        |
z  | _        || _        | j        |
k    r| j        |
z  dk    sJ n|
| j        z  dk    sJ t          d| j        |
z            | _	        || j        z  | _
        | j        | j
        z  | _        | j	        | j
        z  | _        | j
        dz  | _        t          || j
        | j        | j        d|| d          | _        t!          | j        | j
        z  |d|| d          | _        t%          | j
        ||d	          | _        t          t)          |	d
d          d          }t)          |	dd           }|dvrt*                              d|           t/          | j        | j
        | j        | j	        |||| d          | _        | j        rt)          | j        dd          nd| _        d S )Nr   r   g      Fz	.qkv_proj)r_   rY   rZ   z.o_projT)r   r>   is_neox_styleattn_logit_softcappingr%   rx   attn_logit_softcapping_method)Nr   z`Grok attention logit softcapping method '%s' is not supported; falling back to default behavior..attn)r   r   rY   logits_soft_caprZ   attn_output_multiplier      ?)ra   rb   rX   r-   r   total_num_headsr   total_num_kv_headsmaxr   head_dimq_sizekv_sizescalingr   qkv_projr   o_projr   
rotary_embr+   loggerwarning_oncer   attnattn_multiplier)rf   rX   r   r   r   r>   r   rY   rZ   r-   r|   attn_logits_soft_capr   rg   s                r.   rb   zGrok1Attention.__init__   s    	&688(#g-2222-8"."g-- *W499999 T4499994#:g#EFF#t';;nt}4(4=8}d*)M #%'''
 
 
 ( 4=0%%%%
 
 
 #M%+	
 
 
  #763KT#R#RTWXX(/3T)
 )
% )>>?-   NML*%%0###	
 	
 	
	 DH;WGDK!93???TW 	r0   	positionsr   c                 6   |                      |          \  }}|                    | j        | j        | j        gd          \  }}}|                     |||          \  }}|                     |||          }|                     |          \  }	}|	| j        z  }	|	S )Nr   )dim)r   splitr   r   r   r   r   r   )
rf   r   r   qkvrk   qkvattn_outputoutputs
             r.   rl   zGrok1Attention.forward>  s    
 }--Q))T[$,E2)NN1ay!Q//1ii1a((KK,,	$&&r0   )r   NNNrW   N)rm   rn   ro   rp   rK   rq   r   r	   r   rb   rr   rs   rl   rt   ru   s   @r.   r   r      s        &15+/26P
 P
P
 P
 	P

 P
 c3h$.P
 "D(P
 )4/P
 P
 
P
 P
 P
 P
 P
 P
d< | 
	       r0   r   c            
            e Zd Z	 	 	 ddedz  dedz  deddf fdZdej        d	ej        d
ej        dz  de	ej        ej        f         fdZ
 xZS )Grok1DecoderLayerNrW   r   rY   rZ   r&   c                 N   t                                                       |j        | _        d| _        |B t	          |dd                       | _        | j        st          |d          r|j        | _        t          | j        |j        |j	        |j
        t          |          ||| d|	  	        | _        t          |          }t	          |dd          }t          |          }t          |          }t!          |||j        |t#          t	          |d	t$                    d
          ||| d          | _        t	          |dd          | _        dt+          j        d          z  | _        t1          |j        |j                  | _        t1          |j        |j                  | _        t1          |j        |j                  | _        t1          |j        |j                  | _        d | _        | j        r't?          |j        |j         || d          | _        d S d S )NFis_fp8_w8a8c                      dS )NF r   r0   r.   <lambda>z,Grok1DecoderLayer.__init__.<locals>.<lambda>Y  s     r0   is_fp8r   )	rX   r   r   r   r>   r   rY   rZ   r-   num_experts_per_tokr\   router_logit_softcappingrx   z
.moe_block)r(   ry   rX   r3   rz   rY   r}   rZ   r6   r   g       @epsz.mlp)rX   r3   rY   rZ   )!ra   rb   rX   use_fp8r+   r9   r   r   num_attention_headsmax_position_embeddingsnum_key_value_headsrN   r   r/   r4   rT   rw   r   DEFAULT_ROUTER_LOGIT_SOFTCAP	moe_blockr6   mathsqrtresidual_moe_scaler   rms_norm_epspre_attn_normpost_attn_normpre_moe_normpost_moe_normmlprV   r3   )
rf   r-   r   rY   rZ   r(   r   r2   rQ   rg   s
            r.   rb   zGrok1DecoderLayer.__init__M  sF    	!-#N7<NNPPDL< 3GL($C$C 3+2"(073088%%###

 

 

	 'v..%f.CQGG :6 B B.v66!#%*3"%.0 
 # # &'(((
 
 
" $FNEBB"%	#"6$V%7V=PQQQ%f&8f>QRRR#F$6F<OPPP$V%7V=PQQQ 	"."(":) 	  DHHH	 	r0   r   r   residualc                    ||}|                      |          }n|                      ||          \  }}|                     ||          }|                     |          }|                     ||          \  }}| j        r=| j        J |                     |          |                     |          z   | j        z  }n|                     |          }|                     |          }||fS )N)r   r   )	r   r   r   r   r6   r   r   r   r   )rf   r   r   r   s       r.   rl   zGrok1DecoderLayer.forward  s    $H ..}==MM&*&8&8&Q&Q#M8		' " 
 
 ++M:: #'"3"3M8"L"Lx 	:8'''}--0G0GG'(MM !NN=99M**=99h&&r0   )NNrW   )rm   rn   ro   r	   r   rq   rb   rr   rs   tuplerl   rt   ru   s   @r.   r   r   L  s         ,026@ @ "D(@ )4/	@
 @ 
@ @ @ @ @ @D '< ' | ' ,%	 '
 
u|U\)	* '  '  '  '  '  '  '  'r0   r   c                   H    e Zd Zdddddddeded	ed
ededeeef         dz  f fdZdej        dej        fdZ		 ddej        dej        de
dz  dej        dz  dej        e
z  f
dZdeeeeeef                  fdZdeeeej        f                  dee         fdZ xZS )
Grok1ModelrW   linearlinear_1linear_vN)rZ   ckpt_gate_proj_nameckpt_down_proj_nameckpt_up_proj_nameweight_name_remappingvllm_configrZ   r   r   r   r   c                ^  	 t                                                       |j        j        |j        |j        	| _        	| _        j        | _        || _	        || _
        || _        |pi | _        j        | _        t          dt                    | _        t#          | j        j        	          | _        t)          j        	fd| d          \  | _        | _        | _        t3          j        j                  | _        t9          ddgj                  | _        d S )	Nembedding_multiplier_scale)rY   c                 *    t          |           S )NrY   rZ   )r   )rZ   r   r-   rY   s    r.   r   z%Grok1Model.__init__.<locals>.<lambda>  s     ,<   r0   z.layersrZ   r   r   r   )ra   rb   model_config	hf_configr   rY   r-   pad_token_idpadding_idxr   r   r   r   
vocab_sizer+   "DEFAULT_EMBEDDING_MULTIPLIER_SCALEr   r   rX   embed_tokensr#   num_hidden_layersstart_layer	end_layerlayersr   r   normr"   make_empty_intermediate_tensors)rf   r   rZ   r   r   r   r   r   r-   rY   rg   s          @@@r.   rb   zGrok1Model.__init__  s^    	)3"/"/(!. $7 #6 !2%:%@b" +*102T+
 +
' 3O%
 
 
 9D$      %%%9
 9
 9
5$.$+ F.F4GHHH	/Vj)6+=0
 0
,,,r0   	input_idsr&   c                 D    |                      |          }|| j        z  }|S rj   )r   r   )rf   r   r   s      r.   embed_input_idszGrok1Model.embed_input_ids  s(    )))44%(GGr0   r   intermediate_tensorsinputs_embedsc                 p   t                      j        r||}n|                     |          }d }n|J |d         }|d         }t          | j        | j        | j                  D ]} ||||          \  }}t                      j        st          ||d          S | 	                    ||          \  }}|S )Nr   r   )r   r   )
r   is_first_rankr   r   r   r   r   is_last_rankr   r   )	rf   r   r   r   r   r   r   layerrk   s	            r.   rl   zGrok1Model.forward  s     >>' 		8( - $ 4 4Y ? ?HH'3330AM+J7HDK)94>JJ 	P 	PE&+eI}h&O&O#M88~~* 	&"/XFF    99]H==qr0   c                 z    t          | j                  }t          j        | | j        | j        | j        |          S )N)r   r   r   r(   )r/   r-   r   make_expert_params_mappingr   r   r   )rf   r(   s     r.   get_expert_mappingzGrok1Model.get_expert_mapping	  sC    &t{332 $ 8 $ 8"4#
 
 
 	
r0   weightsc           	         g d}t          |                                           }t                      }|                                 }|D ]\  }}| j                                        D ]\  }}	||v r|                    ||	          } | j        ~| j                            |          x}
rb||
         }t          |dt                    }|                                dk    r|n|d         } |||           |                    |
           |D ]\  }}}||vr|                    ||          }|                    d          s|                    d          r||vrPt          ||           ra|                    d          rt          ||          }|||vr||         }|j        } ||||            n'|D ]}|\  }}}}||vr|                    ||          }t          ||           r5|                    d          s|                    d          r||vrd||         }|j        } ||||||            n|                    d          s|                    d          r||vr*t          ||           r<t          ||          }|Pd|v r|                    dd	          }||vrp||         }t          |dt                    } |||           |                    |           |S )
N))r   q_projr   )r   k_projr   )r   v_projr   )mlp.gate_up_projzmlp.gate_projr   )r  zmlp.up_projr   weight_loaderr   z.bias_biasscale)shard_id	expert_idz
norm.scaleweight)rK   named_parameterssetr   r   itemsreplacerY   get_cache_scaler+   r   r   addendswithr!   r   r  )rf   r   stacked_params_mappingparams_dictloaded_paramsexpert_params_mappingrL   loaded_weightold_patternnew_pattern
scale_nameparamr  
param_nameweight_namer  mappingr	  s                     r.   load_weightszGrok1Model.load_weights  s   "
 "
 "
 4002233"%%% $ 7 7 9 9#* Z	$ Z	$D-,0,F,L,L,N,N B B([$&&<<[AAD ,"/??EEE
 - $J/ '@U V V%2%6%6%8%8A%=%=MM=QRCS  e]333!!*---5K G8 G81
Kd**||K<< MM'**.2mmG.D.D+--*466 ==)) !4T;GGD| {**#D) % 3e]H===4 /8 /8GCJ@JY"$.. <<Z@@D.tT:: ! g..!26--2H2H!k11 '-E$)$7M!M%!)"+    E g..!26--2H2H!k11 .tT:: !  5T;GGD|  $t++#||GX>>;.. '-E$+0E% %M "M%777d####r0   rj   )rm   rn   ro   r
   rq   rK   rb   rr   rs   r   r   rl   listr   rp   r   r   r  r  rt   ru   s   @r.   r   r     s        #+#-!+7;1
 1
 1
  1
 	1

 !1
 !1
 1
  $CH~41
 1
 1
 1
 1
 1
f %,     .2 < < 2D8	
 |d* 
+	+   :	
DsCc/A)B$C 	
 	
 	
 	
hHU33D-E$F h3s8 h h h h h h h hr0   r   c                       e Zd ZU dZdZdg diZdZeed<   dZ	eed<   d	Z
eed
<   deeef         fdZdddedef fdZdej        dej        fdZ	 	 ddej        dej        dedz  dej        dz  dej        ez  f
dZdej        dej        dz  fdZdeeeej        f                  dee         fdZdeeeeeef                  fdZ xZS )GrokBaseForCausalLMz-Base class for Grok models with shared logic.Fr   r  r  r  r   r   r   r   r   r   r&   c                     i S )zFReturn weight name remapping for this version. Override in subclasses.r   rf   s    r.   get_weight_name_remappingz-GrokBaseForCausalLM.get_weight_name_remapping      	r0   rW   r   r   rZ   c          	         t                                                       |j        j        }|j        }|| _        || _        t          |t          |d          | j        | j	        | j
        |                                           | _        t          |j        |j        |t          |d                    | _        | j        j        r| j        j        j        | j        _        t)          |dt*                    | _        t/          |j        | j        t)          |dd                     | _        | j        j        | _        d S )Nmodel)r   rZ   r   r   r   r   lm_headr   output_multiplier_scalefinal_logit_softcapping)r  soft_cap)ra   rb   r   r   rY   r-   r   r$   r   r   r   r%  r(  r   r   rX   r)  tie_word_embeddingsr   r
  r+   DEFAULT_OUTPUT_MULTIPLIER_SCALEr*  r   logits_processorr   )rf   r   rZ   r-   rY   rg   s        r.   rb   zGrokBaseForCausalLM.__init__  s@   )3"/(#00 $ 8 $ 8"4"&"@"@"B"B
 
 

 &%	22	
 
 
 ;* 	A"&*"9"@DL'.-/N(
 (
$ !0.V%>EE!
 !
 !
 J6 	,,,r0   r   c                 6    | j                             |          S rj   )r(  r   )rf   r   s     r.   r   z#GrokBaseForCausalLM.embed_input_ids  s    z)))444r0   Nr   r   r   c                 6    |                      ||||          }|S rj   )r(  )rf   r   r   r   r   r   s         r.   rl   zGrokBaseForCausalLM.forward  s)     

y"6
 
 r0   r   c                 <    |                      | j        |          }|S rj   )r/  r)  )rf   r   logitss      r.   compute_logitsz"GrokBaseForCausalLM.compute_logits  s      &&t|]CCr0   r   c                 p    | j         j        rdgnd }t          | |          }|                    |          S )Nr)  )skip_prefixes)r-   r-  r    r  )rf   r   r6  loaders       r.   r  z GrokBaseForCausalLM.load_weights  sG    '+{'FPD"'
 
 
 ""7+++r0   c                 4    | j                                         S rj   )r(  r   r$  s    r.   r   z&GrokBaseForCausalLM.get_expert_mapping  s    z,,...r0   )NN)rm   rn   ro   r   fall_back_to_pt_during_loadpacked_modules_mappingr   rq   __annotations__r   r   rK   r%  r
   rb   rr   rs   r   r   rl   r4  r   r   r  r  r  rp   r   rt   ru   s   @r.   r!  r!    s        77"' 	 
 
 
  ('''))))'s'''4S>     BD '
 '
 '
z '
3 '
 '
 '
 '
 '
 '
R5 5%, 5 5 5 5 <@-1
 
<
 <
 2D8	

 |d*
 
+	+
 
 
 
| 
	   ,HU33D-E$F ,3s8 , , , ,/DsCc/A)B$C / / / / / / / /r0   r!  c                   :    e Zd ZdZdZdZdZdeeef         fdZ	dS )Grok1ForCausalLMzGrok1-specific implementation.r   r   r   r&   c                     i S rj   r   r$  s    r.   r%  z*Grok1ForCausalLM.get_weight_name_remapping  r&  r0   N)
rm   rn   ro   r   r   r   r   rK   rq   r%  r   r0   r.   r=  r=    sQ        (( #$"4S>      r0   r=  c                   L    e Zd ZdZg dddgdZdZdZdZd	ee	e	f         fd
Z
dS )Grok2ForCausalLMzGrok2-specific implementation.r"  	gate_projup_proj)r   rc   w1w2w3r&   c                     dddS )Nz.attn.z.moe_block.)z.self_attn.z.block_sparse_moe.r   r$  s    r.   r%  z*Grok2ForCausalLM.get_weight_name_remapping  s     $"/
 
 	
r0   N)rm   rn   ro   r   r:  r   r   r   rK   rq   r%  r   r0   r.   r@  r@    sz        ((
 
 
 

 
 
4S> 
 
 
 
 
 
r0   r@  )r8   r7   _GROK_VERSIONSc                   (    e Zd ZdZdddedefdZdS )GrokForCausalLMzAFactory class that dispatches to version-specific implementation.rW   r   r   rZ   c                   |j         j        }t          |          }t                              |          }|t          d|           t          | j                  | _        | j                            |j                    |||          S )NzUnsupported Grok version: )r   rZ   )	r   r   r<   rG  rJ   
ValueErrorrK   r:  update)clsr   rZ   r-   versioninstance_clss         r.   __new__zGrokForCausalLM.__new__  s    )3#F++%))'22C'CCDDD &*#*D%E%E"")),*MNNN|FCCCCr0   N)rm   rn   ro   r   r
   rq   rP  r   r0   r.   rI  rI    sS        KK?A D D DZ D D D D D D Dr0   rI  )Wr   r   collections.abcr   	itertoolsr   typingr   rr   torch.nn.functionalr   
functionalr   vllm.attention.layerr   vllm.compilation.decoratorsr   vllm.configr	   r
   vllm.distributedr   r   vllm.loggerr   %vllm.model_executor.layers.activationr   $vllm.model_executor.layers.fused_moer   $vllm.model_executor.layers.layernormr   !vllm.model_executor.layers.linearr   r   r   r   +vllm.model_executor.layers.logits_processorr   'vllm.model_executor.layers.quantizationr   +vllm.model_executor.layers.rotary_embeddingr   3vllm.model_executor.layers.vocab_parallel_embeddingr   r   -vllm.model_executor.model_loader.weight_utilsr   r   vllm.sequencer   
interfacesr   r   utilsr    r!   r"   r#   r$   DEFAULT_ATTN_OUTPUT_MULTIPLIERr.  r   r   rm   r   rp   r/   r4   rq   r<   rK   rN   rR   rT   ModulerV   rw   r   r   r   r!  r=  r@  rG  typer;  rI  r   r0   r.   <module>rj     s(  0 / . .  $ $ $ $ $ $                             * * * * * * = = = = = = / / / / / / / / O O O O O O O O # # # # # # < < < < < < 9 9 9 9 9 9 8 8 8 8 8 8            H G G G G G F F F F F F @ @ @ @ @ @               . - - - - - 0 0 0 0 0 0 0 0              "5 "4 %6 "# 	X		S S S S SN# N N N N	 	 	 	 	DcNT$9    <6D 6 6 6 6    ry   @=4 =4 =4 =4 =4ry =4 =4 =4@^ ^ ^ ^ ^RY ^ ^ ^Bc' c' c' c' c'	 c' c' c'L I I I I I I I IXa/ a/ a/ a/ a/")\: a/ a/ a/H
 
 
 
 
* 
 
 

 
 
 
 
* 
 
 
> 8 8S$2334   D D D D D) D D D D Dr0   