
    .`isy              	          d Z ddlZddlmZmZ ddlmZ ddlmZ ddlZddl	m
c mZ ddlm
Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC  eeD          ZE G d de
jF                  ZG G d de
jF                  ZH G d de
jF                  ZI G d  d!e
jF                  ZJe G d" d#e
jF                              ZK G d$ d%e
jF        e;e:e9e8          ZLdS )&zBInference-only Qwen3MoE model compatible with HuggingFace weights.    N)CallableIterable)islice)Any)nn)	Attention)support_torch_compile)CacheConfig
VllmConfigget_current_vllm_config)get_ep_groupget_pp_group$get_tensor_model_parallel_world_size tensor_model_parallel_all_gather)init_logger)
SiluAndMul)SharedFusedMoE)RMSNorm)MergedColumnParallelLinearQKVParallelLinearReplicatedLinearRowParallelLinear)LogitsProcessor)QuantizationConfig)get_rope)ParallelLMHeadVocabParallelEmbedding)default_weight_loadermaybe_remap_kv_scale_name)sequence_parallel_chunk)IntermediateTensors   )MixtureOfExpertsSupportsEagle3SupportsLoRA
SupportsPP)AutoWeightsLoaderPPMissingLayerextract_layer_indexis_pp_missing_parameter'make_empty_intermediate_tensors_factorymake_layersmaybe_prefixc                   p     e Zd Z	 	 	 	 ddededededz  ded	ej        j	        dz  d
eddf fdZ
d Z xZS )Qwen3MoeMLPNT hidden_sizeintermediate_size
hidden_actquant_configreduce_resultsexpert_gateprefixreturnc           	      (   t                                                       t          ||gdz  d|| d          | _        t	          ||d||| d          | _        |dk    rt          d| d	          t                      | _        || _	        d S )
N   Fz.gate_up_projbiasr4   r7   z
.down_proj)r<   r4   r5   r7   siluzUnsupported activation: z!. Only silu is supported for now.)
super__init__r   gate_up_projr   	down_proj
ValueErrorr   act_fnr6   )	selfr1   r2   r3   r4   r5   r6   r7   	__class__s	           x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/qwen3_moe.pyr?   zQwen3MoeMLP.__init__S   s     	6!#%+++
 
 
 +%)(((
 
 
 X:XXX   !ll&    c                     |                      |          \  }}|                     |          }|                     |          \  }}| j        0t	          j        |                     |          d                   |z  }|S )Nr   )r@   rC   rA   r6   Fsigmoid)rD   xgate_up_outs        rF   forwardzQwen3MoeMLP.forwardt   st    &&q))
kk'""$$Q')D,,Q//233c9C
rG   )NTNr0   )__name__
__module____qualname__intstrr   booltorchr   Linearr?   rO   __classcell__rE   s   @rF   r/   r/   R   s         37#.2' '' ' 	'
 )4/' ' X_t+' ' 
' ' ' ' ' 'B      rG   r/   c                   P     e Zd Z	 ddedef fdZdej        dej        fdZ xZ	S )	Qwen3MoeSparseMoeBlockr0   vllm_configr7   c                    t                                                       |j        j        }|j        }|j        }t                      | _        t                      j	        | _
        t                      j        | _        | j
                                        | _        |j        | _        |j        | _        | j        |j        k    r t'          d| j         d|j         d          t)                      }|j        j        }|j        | _        | j        | _        |j        | _        | j        | j        z   | _        | j        | j        z  | _        | j        | j        z  | _        | j        | j        z   | _        t=          |j        |j        d|| d          | _         tC          |dd          }|dk    rOt=          |j        d	dd | d
          | _"        tG          |j        ||j$        |d| j"        | d          | _%        nd | _"        d | _%        tM          | j%        | j         | j        |j'        |j        |j(        d|j)        || d| j        | j        | j                  | _*        d S )NzTensor parallel size z' is greater than the number of experts .Fz.gater;   shared_expert_intermediate_sizer   r"   z.shared_expert_gatez.shared_expert)r1   r2   r3   r4   r5   r6   r7   z.experts)shared_expertsgatenum_expertstop_kr1   r2   r5   renormalizer4   r7   enable_eplbnum_redundant_expertsis_sequence_parallel)+r>   r?   model_confighf_text_configparallel_configr4   r   tp_sizer   device_groupep_grouprank_in_groupep_ranksizeep_sizerb   n_routed_expertsuse_sequence_parallel_moerg   rB   r   eplb_configre   n_logical_expertsrf   n_redundant_expertsn_physical_expertsn_local_physical_expertsphysical_expert_startphysical_expert_endr   r1   ra   getattrshared_expert_gater/   r3   shared_expertr   num_experts_per_tokmoe_intermediate_sizenorm_topk_probexperts)	rD   r\   r7   configrj   r4   rt   r_   rE   s	           rF   r?   zQwen3MoeSparseMoeBlock.__init__   s   
 	)8%5"/;==$3#~~3}))++ & 2$3$M!<&,,,? ? ?)/);? ? ?   .//!1=*6!%!6#.#D "&"84;S"S(,(?4<(O%%)\D4Q%Q"&)FF 	  %%###
 
 
	 +25q+
 +
' +Q..&6"! 555' ' 'D# "-"."A!,)$ 3 000" " "D '+D#!%D%--,*$: -%&&&("&":!%!:
 
 
rG   hidden_statesr8   c                    |                                 dk    s
J d            |                                 dk    }|j        \  }}|                    d|          }| j        rt	          |          }|                     |          \  }}|                     ||          \  }}|||z   n|}	| j        rt          |	d          }	|	d |         }	n%| j        dk    r| j        	                    |	          }	|r|	
                    d          n|	S )Nr:   z4Qwen3MoeSparseMoeBlock only supports 1D or 2D inputsr"   )r   router_logitsr   )dimshapeviewrg   r    ra   r   r   rk   &maybe_all_reduce_tensor_model_parallelsqueeze)
rD   r   is_input_1d
num_tokens
hidden_dimr   rM   
shared_out	fused_outfinal_hidden_statess
             rF   rO   zQwen3MoeSparseMoeBlock.forward   sP     ""a'''B ('' $''))Q.!.!4
J%**2z::$ 	C3MBBM  99]33q $'} !- !
 !

I '1&<J"") 	 $ 	"B#Q# # #6kzk"B\A"&,"U"U## #
 2=U"**1---BUUrG   r0   )
rP   rQ   rR   r   rT   r?   rV   TensorrO   rX   rY   s   @rF   r[   r[      s         W
 W
W
 W
 W
 W
 W
 W
 W
rVU\ Vel V V V V V V V VrG   r[   c                        e Zd Z	 	 	 	 	 	 	 	 ddededed	eeef         d
ededz  dedede	dz  de
dz  dedeeef         dz  ddf fdZdej        dej        dej        fdZ xZS )Qwen3MoeAttention    Nư>Fr0   r1   	num_headsnum_kv_headsrope_parametersmax_position_embeddingshead_dimrms_norm_epsqkv_biascache_configr4   r7   dual_chunk_attention_configr8   c           
         t                                                       || _        t                      }|| _        | j        |z  dk    sJ | j        |z  | _        || _        | j        |k    r| j        |z  dk    sJ n|| j        z  dk    sJ t          d| j        |z            | _        |p	|| j        z  | _	        | j        | j	        z  | _
        | j        | j	        z  | _        | j	        dz  | _        || _        || _        t          || j	        | j        | j        ||
| d          | _        t#          | j        | j	        z  |d|
| d          | _        t'          | j	        |||          | _        t+          | j        | j	        | j        f| j        |	|
| d	d
|rt-          |          |dni | _        t1          | j	        |          | _        t1          | j	        |          | _        d S )Nr   r"   g      z	.qkv_projr;   Fz.o_proj)max_positionr   r   z.attn)r   r   r4   r7   )	layer_idxr   eps)r>   r?   r1   r   total_num_headsr   total_num_kv_headsmaxr   r   q_sizekv_sizescalingr   r   r   qkv_projr   o_projr   
rotary_embr   r)   attnr   q_normk_norm)rD   r1   r   r   r   r   r   r   r   r   r4   r7   r   rk   rE   s                 rF   r?   zQwen3MoeAttention.__init__   s_    	&688(#g-2222-8"."g-- *W499999 T4499994#:g#EFF I[D4H%Hnt}4(4=8}d*'>$+F()M #%'''
 
 
 ( 4=0%%%%
 
 
 #M0+(C	
 
 
 NML
 *%%###
 
 +	088/J  
 
 
	  dm>>>dm>>>rG   	positionsr   c                    |                      |          \  }}|                    | j        | j        | j        gd          \  }}} |j        g |j        d d         |j        d         | j        z  | j        R  }|                     |          }|                    |j                  } |j        g |j        d d         |j        d         | j        z  | j        R  }	|                     |	          }	|	                    |j                  }| 	                    |||          \  }}| 
                    |||          }
|                     |
          \  }}|S )Nr   )r   )r   splitr   r   r   r   r   r   r   r   r   r   )rD   r   r   qkvrM   qkv	q_by_head	k_by_headattn_outputoutputs               rF   rO   zQwen3MoeAttention.forwardM  sE   
 }--Q))T[$,E2)NN1aAFVAGCRCLV!'"+*FVVVV	KK	**	NN17##AFVAGCRCLV!'"+*FVVVV	KK	**	NN17##y!Q//1ii1a((KK,,	rG   )r   Nr   FNNr0   N)rP   rQ   rR   rS   dictrT   r   floatrU   r
   r   r?   rV   r   rO   rX   rY   s   @rF   r   r      sJ        (,##+/26=AO? O?O? O? 	O?
 c3hO? "%O? *O? O? O? "D(O? )4/O? O? &*#s(^d%:O? 
O? O? O? O? O? O?b< | 
	       rG   r   c            
            e Zd Zddededdf fdZdej        dej        d	ej        dz  deej        ej        f         fd
Z	 xZ
S )Qwen3MoeDecoderLayerr0   r\   r7   r8   Nc                 &   t                                                       |j        j        }|j        }|j        }|j        | _        t          |dd          }t          |dd           }t          | j        |j	        |j
        |j        ||j        t          |dd          t          |dd           ||| d|          | _        t          |          }t          |d	          sg n|j        }	||	vr6|j        d
k    r+|dz   |j        z  d
k    rt'          || d          | _        n+t+          |j        |j        |j        || d          | _        t1          |j        |j                  | _        t1          |j        |j                  | _        d S )Nr   r   r   attention_biasFr   z
.self_attn)r1   r   r   r   r   r   r   r   r   r4   r7   r   mlp_only_layersr   r"   z.mlpr\   r7   )r1   r2   r3   r4   r7   r   )r>   r?   rh   ri   r   r4   r1   r{   r   num_attention_headsnum_key_value_headsr   r   	self_attnr)   hasattrr   rb   decoder_sparse_stepr[   mlpr/   r2   r3   r   input_layernormpost_attention_layernorm)rD   r\   r7   r   r   r4   r   r   r   r   rE   s             rF   r?   zQwen3MoeDecoderLayer.__init__c  s   )8"/"/!-")&2KT"R"R&-14'
 '
# +(03"2$;,V%5u==VZ66%%((((C
 
 
  (//	f&788TBBf>T 	 _,,""	A9S'SWX'X'X-'6  DHH #"."(":!,)   DH  'v'9v?RSSS(/F$7)
 )
 )
%%%rG   r   r   residualc                     ||}|                      |          }n|                      ||          \  }}|                     ||          }|                     ||          \  }}|                     |          }||fS )N)r   r   )r   r   r   r   )rD   r   r   r   s       rF   rO   zQwen3MoeDecoderLayer.forward  s     $H 00??MM&*&:&:=(&S&S#M8' ' 
 
 #'"?"?x"X"Xx//h&&rG   r   )rP   rQ   rR   r   rT   r?   rV   r   tuplerO   rX   rY   s   @rF   r   r   b  s        1
 1
J 1
 1
T 1
 1
 1
 1
 1
 1
f'<' |' ,%	'
 
u|U\)	*' ' ' ' ' ' ' 'rG   r   c                   R    e Zd Zdddedef fdZdej        dej        fdZ	 	 ddej        d
ej        de	d	z  dej        d	z  dej        e	z  e
ej        eej                 f         z  f
dZdee
eeeef                  fdZdee
eej        f                  dee         fdZ xZS )Qwen3MoeModelr0   r7   r\   r7   c                    t                                                       j        j        }j        }j        }|j        }|j        | _        |j        | _	        |j
        | _
        || _        || _        t          |j
        |j        || d          | _        t          |j        fd| d          \  | _        | _        | _        t)          |j        |j                  | _        t/          ddg|j                  | _        d	| _        d S )
Nz.embed_tokensr4   r7   c                 &    t          |           S )Nr   )r   )r7   r\   s    rF   <lambda>z(Qwen3MoeModel.__init__.<locals>.<lambda>  s    /KPVWWW rG   z.layersr   r   r   r    )r>   r?   rh   ri   r4   rj   rt   rf   pad_token_idpadding_idx
vocab_sizer   r   r1   embed_tokensr,   num_hidden_layersstart_layer	end_layerlayersr   r   normr+   make_empty_intermediate_tensorsaux_hidden_state_layers)rD   r\   r7   r   r4   rj   rt   rE   s    `     rF   r?   zQwen3MoeModel.__init__  s,   )8"/%5%1%0%F"!. +(2%+++	
 
 
 9D$WWWW%%%9
 9
 9
5$.$+
 F.F4GHHH	/Vj)6+=0
 0
, 9;$$$rG   	input_idsr8   c                 ,    |                      |          S N)r   rD   r   s     rF   embed_input_idszQwen3MoeModel.embed_input_ids  s      +++rG   Nr   intermediate_tensorsinputs_embedsc                    t                      j        r||}n|                     |          }d }n|J |d         }|d         }g }t          t	          | j        | j        | j                  | j                  D ]<\  }}	|| j        v r|||z   n|}
|	                    |
            |	|||          \  }}=t                      j
        st          ||d          S |                     ||          \  }}t          |          dk    r||fS |S )Nr   r   )start)r   r   r   )r   is_first_rankr   	enumerater   r   r   r   r   appendis_last_rankr!   r   len)rD   r   r   r   r   r   r   aux_hidden_statesr   layeraux_hidden_staterM   s               rF   rO   zQwen3MoeModel.forward  sg    >>' 		8( - $ 4 4Y ? ?HH'3330AM+J7H )4; 0$.AA"!
 !
 !
 
	P 
	PIu
 D888080DMH,,- ! "(()9:::&+eI}h&O&O#M88~~* 	&"/XFF    99]H==q  !!A%% "333rG   c                 T    t          j        | ddd| j        j        | j                  S )N	gate_projrA   up_proj)ckpt_gate_proj_nameckpt_down_proj_nameckpt_up_proj_namerb   rf   )r   make_expert_params_mappingr   rb   rf   rD   s    rF   get_expert_mappingz Qwen3MoeModel.get_expert_mapping  s8     8 + +'/"&"<
 
 
 	
rG   weightsc           
      V   g d}d}t          |                                           }t                      }|                                 }|D ]\  }}| j        | j                            |          x}	r||	         }
t          |
dt                    }|                                dk    s J d|                                 d            |	                                } ||
|           |
                    |	           |D ]\  }}}||vrd|v r|                    ||          }|                    |          r||vr@t          ||           rQ|                    d          rt          ||          }|y||vr~||         }
t          |
dt                    }|t          k    r ||
|           n ||
||            nKd	}|D ]}|\  }}}}||vrd
}|                    ||          }t          ||           r7|                    |          r||vrQ||         }
t          j        t"          dt$          f         |
j                  } ||
||||d
          }|r|} n|r |                    |          r||vr;t          ||           rM|                    d          r:|                    dd          }||vrt(                              d||           |}||         }
t          |
dt                    } ||
|           |
                    |           |S )N))r   q_projr   )r   k_projr   )r   v_projr   )r@   r   r   )r@   r   r"   )
z.bias_biasz.k_scale_k_scalez.v_scale_v_scalez.weight_scale_weight_scalez.input_scale_input_scaleweight_loaderr"   zKV scale numel z != 1zmlp.expertsscaleFT.)shard_id	expert_idreturn_successkv_scalez	.kv_scalez.attn.kv_scalez{Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.)r   named_parameterssetr   r4   get_cache_scaler{   r   numelr   addreplaceendswithr*   r   typingcastr   rU   r  loggerwarning_once)rD   r   stacked_params_mappingignore_suffixesparams_dictloaded_paramsexpert_params_mappingnameloaded_weight
scale_nameparamr  
param_nameweight_namer	  is_expert_weightmappingr
  name_mappedsuccessremapped_kv_scale_names                        rF   load_weightszQwen3MoeModel.load_weights  s7   "
 "
 "

 4002233"%%% $ 7 7 9 9#* }	$ }	$D- ,"/??EEE
 - $J/ '@U V V$**,,111Bm&9&9&;&;BBB 211 !. 5 5 7 7e]333!!*---5K n8 n81
Kd** !D((||K<< ==11 d+6M6M +466 ==)) !4T;GGD| {**#D) '@U V V $999!M%7777!M%AAA#( 4 H8 H8GCJ@JY"$..  (,$ #',,{J"G"GK.{DAA !  $,,_==!'{:: '4E %+K d+U-@% %M ,m%#!)"+'+  G  * ( ! ! }}_55 !$k:Q:Q .tT:: ! }}Z00 :15')92 2. 2DD"// !^ $ 6  
 %#9D'-E$+0E% %M "M%777d####rG   NN)rP   rQ   rR   r   rT   r?   rV   r   r   r!   r   listrO   rS   r   r   r  r(  rX   rY   s   @rF   r   r     sp       AC ; ; ;z ;3 ; ; ; ; ; ;>, ,%, , , , , <@-1( (<( <( 2D8	(
 |d*( 
+	+eEL$u|BT4T.U	U( ( ( (T

DsCc/A)B$C 

 

 

 

YHU33D-E$F Y3s8 Y Y Y Y Y Y Y YrG   r   c                       e Zd Zdg diZdZdddedef fdZd	ed
eddfdZ	de
edf         ddfdZde
edf         fdZdej        dej        fdZ	 	 ddej        dej        dedz  dej        dz  dej        ez  f
dZdej        dej        dz  fdZdee
eej        f                  dee         fdZdee
eeeef                  fdZ xZS )Qwen3MoeForCausalLMr   )r   r   r  Fr0   r   r\   r7   c          	         t                                                       |j        j        }|j        }|| _        || _        t          |dg           rddg| j        d<   t          |t          |d                    | _
        t          |j        |j        |t          |d                    | _        | j        j        r| j
        j        j        | j        _        t%          |j                  | _        | j
        j        | _        g | _        g | _        d }| j
        j        D ]t}t1          |t2                    rt1          |t4                    sJ t1          |j        t8                    r+|j        }| j                            |j        j                   u|t?          d	          tA          | j                  | _!        d
| _"        d| _#        |j$        | _%        |j&        | _'        |j(        | _)        |j*        | _+        |j,        | _-        d S )Nr   r   r   r@   modelr   lm_headr   z,No Qwen3MoE layer found in the model.layers.r"   r   ).r>   r?   rh   ri   r4   r   r{   packed_modules_mappingr   r-   r.  r   r   r1   r/  tie_word_embeddingsr   weightr   logits_processorr   expert_weights
moe_layersr   
isinstancer(   r   r   r[   r   r   RuntimeErrorr   num_moe_layersnum_expert_groupsnum_shared_expertsru   num_logical_expertsrw   num_physical_expertsrx   num_local_physical_expertsrr   num_routed_expertsrv   rf   )rD   r\   r7   r   r4   example_layerr   rE   s          rF   r?   zQwen3MoeForCausalLM.__init__  s   )8"/(6,b11 	S;F	:RD'7"#L,I,I
 
 

 &%	22	
 
 
 ;* 	A"&*"9"@DL /0A B BJ6 	,
 !Z& 	: 	:E%00 e%9:::::%)%;<< : %	&&uy'8999 MNNN!$/22!""##0#B $1$D!*7*P'"/"@%2%F"""rG   r<  r=  r8   Nc                     | j         |k    sJ || _        || _         || j        z
  | _        | j        j        D ]V}t          |j        t                    r:|j        }||_	        ||_
        | j        |_        |j                                         Wd S r   )r=  r<  r;  rf   r.  r   r6  r   r[   rx   rw   rv   r   update_expert_map)rD   r<  r=  r   moes        rF    update_physical_experts_metadataz4Qwen3MoeForCausalLM.update_physical_experts_metadata  s    
 .2LLLLL$8!*D'%9D<T%T"Z& 	0 	0E%)%;<< 0i/I,)=&*.*D'--///	0 	0rG   r   .c                     || j         _        d S r   )r.  r   )rD   r   s     rF   set_aux_hidden_state_layersz/Qwen3MoeForCausalLM.set_aux_hidden_state_layers  s    -3
***rG   c                 J    t          | j        j                  }d|dz  |dz
  fS )Nr:      )r   r.  r   )rD   
num_layerss     rF   "get_eagle3_aux_hidden_state_layersz6Qwen3MoeForCausalLM.get_eagle3_aux_hidden_state_layers  s)    *++
:?JN33rG   r   c                 6    | j                             |          S r   )r.  r   r   s     rF   r   z#Qwen3MoeForCausalLM.embed_input_ids  s    z)))444rG   r   r   r   c                 6    |                      ||||          }|S r   )r.  )rD   r   r   r   r   r   s         rF   rO   zQwen3MoeForCausalLM.forward  s)     

y"6
 
 rG   r   c                 <    |                      | j        |          }|S r   )r3  r/  )rD   r   logitss      rF   compute_logitsz"Qwen3MoeForCausalLM.compute_logits
  s      &&t|]CCrG   r   c                 J    t          |           }|                    |          S r   )r'   r(  )rD   r   loaders      rF   r(  z Qwen3MoeForCausalLM.load_weights  s#    "4((""7+++rG   c                 4    | j                                         S r   )r.  r   r   s    rF   r   z&Qwen3MoeForCausalLM.get_expert_mapping  s    z,,...rG   r)  )rP   rQ   rR   r0  fall_back_to_pt_during_loadr   rT   r?   rS   rC  r   rE  rI  rV   r   r   r!   rO   rN  r   r  r(  r*  r   rX   rY   s   @rF   r,  r,    s(        	 
 
 
 #(AC 1G 1G 1Gz 1G3 1G 1G 1G 1G 1G 1Gf0!0 %(0 
	0 0 0 0"4%S/ 4d 4 4 4 44E#s(O 4 4 4 45 5%, 5 5 5 5 <@-1
 
<
 <
 2D8	

 |d*
 
+	+
 
 
 
| 
	   ,HU33D-E$F ,3s8 , , , ,/DsCc/A)B$C / / / / / / / /rG   r,  )M__doc__r  collections.abcr   r   	itertoolsr   r   rV   torch.nn.functionalr   
functionalrI   vllm.attention.layerr   vllm.compilation.decoratorsr	   vllm.configr
   r   r   vllm.distributedr   r   r   r   vllm.loggerr   %vllm.model_executor.layers.activationr   $vllm.model_executor.layers.fused_moer   $vllm.model_executor.layers.layernormr   !vllm.model_executor.layers.linearr   r   r   r   +vllm.model_executor.layers.logits_processorr   'vllm.model_executor.layers.quantizationr   +vllm.model_executor.layers.rotary_embeddingr   3vllm.model_executor.layers.vocab_parallel_embeddingr   r   -vllm.model_executor.model_loader.weight_utilsr   r    vllm.model_executor.models.utilsr    vllm.sequencer!   
interfacesr#   r$   r%   r&   utilsr'   r(   r)   r*   r+   r,   r-   rP   r  Moduler/   r[   r   r   r   r,  r   rG   rF   <module>rk     s,  0 I H  . . . . . . . .                             * * * * * * = = = = = = H H H H H H H H H H            $ # # # # # < < < < < < ? ? ? ? ? ? 8 8 8 8 8 8            H G G G G G F F F F F F @ @ @ @ @ @               E D D D D D - - - - - - R R R R R R R R R R R R                  
X		* * * * *") * * *ZyV yV yV yV yVRY yV yV yVxd d d d d	 d d dNH' H' H' H' H'29 H' H' H'V r r r r rBI r r rjs/ s/ s/ s/ s/Iz<9Is/ s/ s/ s/ s/rG   