
    .`im                     "   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z" ddl#m$Z$m%Z%  G d de          Z& G d de&          Z'dS )    N)PretrainedConfig)envs)
LoRAConfig)get_tensor_model_parallel_rank$get_tensor_model_parallel_world_size)divide)BaseLayerWithLoRA)get_lora_op_configs)FusedMoE)_get_config_dtype_str)MarlinExperts)TritonExperts)FusedMoEModularMethod)UnfusedOAITritonExperts)FusedMoEModularKernel)MoEPrepareAndFinalizeNoEP   )_get_lora_devicetry_get_optimal_moe_lora_configc                   V    e Zd Zdeddf fdZdeeedz  f         deeedz  f         fdZdeded	ed
ededededefdZ	d Z
dedefdZdedefdZ	 d,dedededz  ddfdZdej        dej        fdZdej        fdZdej        dej        fdZdej        dej        fdZdefd Zded!ej        eej                 z  d"ej        eej                 z  fd#Zd$ Zd% Zed&             Zed'             Zedefd(            Ze 	 d,d)e!j"        ded*ededz  def
d+            Z# xZ$S )-FusedMoEWithLoRA
base_layerreturnNc                 H   t                                                       || _        | j        j        r
J d            t	                      | _        t                      | _        t          |          | _	        |j
        j        rdnd| _        |                                  d S )Nz5EP support for Fused MoE LoRA is not implemented yet.   r   )super__init__r   use_epr   tp_sizer   tp_rankr   device
moe_configis_act_and_mul_w13_slices_inject_lora_into_fused_moeselfr   	__class__s     n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/lora/layers/fused_moe.pyr   zFusedMoEWithLoRA.__init__-   s    $?) 	
 	
C	
 	
) <==577&z22 !+ 5 DK11!((*****    configc                 *   i }|                                 D ]{\  }}|                                r[|                    d          r1d|                    d          d                                         z   }n|                                }n|}|||<   ||S )Nblock_BLOCK_SIZE__)itemsislower
startswithsplitupper)r'   r+   normalized_configkeyvaluenormalized_keys         r)   _normalize_keysz FusedMoEWithLoRA._normalize_keys<   s     ,,.. 	6 	6JC{{}} %>>(++ 1%2SYYs^^B5G5M5M5O5O%ONN%(YY[[NN!$05n--  r*   	op_prefix	num_lorasrank
num_slicesMlayertop_kconfig_dtypec	           
         t           j        rC|j        }	|j        }
t	          d| d|||	|||
          }t	          d| d|||	|||
          }n~t          j        t          |j        	                                |j
        	                                |||||j        j        j                  } |d| d          } |d| d          }|                     |          }|                     |          }||fS )Nfused_moe_lora__shrink)op_type	max_lorasbatchhidden_sizer=   r>   moe_intermediate_size_expand)w1_shapew2_shaper=   rA   dtyper?   block_shape)rF   )r   VLLM_TUNED_CONFIG_FOLDERrI   intermediate_size_per_partitionr
   	functoolspartialr   
w13_weightsize	w2_weightquant_methodmoe_quant_configrO   r:   )r'   r;   r<   r=   r>   r?   r@   rA   rB   rI   intermediate_sizeshrink_configexpand_configget_config_funcs                 r)   _get_lora_moe_configsz&FusedMoEWithLoRA._get_lora_moe_configsI   sb    ( %	+K % E/<)<<<#'%&7  M 0<)<<<#'%&7  MM (//)..00--//"!.?K	 	 	O ,O<)<<<  M ,O<)<<<  M ,,];;,,];;m++r*   c                    	 i  j         j        	 j                                           j         j        j        }t                      }t          | j         j                            | j                    j         j                  }|j	        r$t          |j        t          t          f          sJ n#t          |j        t          t          f          sJ fd} 	fd} 	fd}|j        } | j         |j                  |_         | j         |j                  |_         | j         |j                  |_        t%           j         j        |           j         _        d S )Nc                     fd}|S )Nc                      |d         d<   |d         d<   |d         d<   |d         d<   |d         d<    | i |}|S )Nhidden_statestopk_idstopk_weights
expert_mapapply_router_weight_on_input )argskwargsresultfuncmoe_state_dicts      r)   wrapperzTFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.fwd_decorator.<locals>.wrapper   sn    282I/-3J-?z*171G~./5l/C|,AG2B=> t.v..r*   rf   )r@   rj   rl   rk   s    ` r)   fwd_decoratorzCFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.fwd_decorator   s)    	 	 	 	 	 	 Nr*   c                       fd}|S )Nc                  B   | \  }}}d         }d         }d         }d         }t          |j        ddd          }	t          j        }
|                    d          }t          ||
          }j        d         j        d         }                    d	j	        |j
        ||	
          \  }}j                            |||d         j        j        j	        j        |          \  }}}|d<   |d<   |d<   |                    j	        d          }|                    j	        d          }j                            |                    d|j        d                   |j        j        |||||||j        j                    | i |}|d<   |S )Nra   rc   rb   rd   FrN   use_fp8_w8a8use_int8_w8a16use_int4_w4a16r   w13r;   r<   r=   r>   r?   r@   rA   rB   BLOCK_SIZE_Msorted_token_ids_loraexpert_ids_loranum_tokens_post_padded_lorar0   )fully_shardedintermediate_cache2)r   rN   r   VLLM_FUSED_MOE_CHUNK_SIZErU   minw13_lora_a_stackedshaper]   rG   r$   punica_wrappermoe_lora_align_block_sizer   local_num_expertsadapter_enabledviewadd_lora_fused_moew13_lora_b_stackedr{   )rg   rh   r/   outputinputra   rc   curr_topk_idsrd   rB   
CHUNK_SIZE
num_tokensr?   max_lora_rankrZ   r[   rx   ry   rz   ri   rj   r@   rk   r'   rA   s                       r)   rl   zTFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.act_decorator.<locals>.wrapper   s   #' 65 . ?-n= .z :+L9
4'-!&#(#(	      ";
*//22

J// $ 7 : @ D/3/I/I#"n&#/!- 0J 	0 	0,}  'AA!!.1O5N( 	)#/ ;P674C01/ <= #2"6"6t~r"J"J(=(B(B4>SU(V(V% #66JJr5%+b/::!++ )#/!!!("&"4 7   " t.v..8>45r*   rf   r@   rj   rl   rk   r'   rA   s   `` r)   act_decoratorzCFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.act_decorator   sC    K K K K K K K K KZ Nr*   c                       fd}|S )Nc                     d         }d         }t          |j        ddd          }t          j        }|                    d          }t          ||          }j        d         j        d         }                    dj	        |d||	          \  }	}
d
         }d         }d         }|
                    j	        d          }|
                    j	        d          }d         }| d         }t          j        j        j                  }j                            ||j        j        ||||||	|
j        dj        j        r
|j        z  nd            | i |}|S )Nra   rc   Frp   r   rt   w2r   rv   rx   ry   rz   r0   r|   T)r{   offset)r   rN   r   r}   rU   r~   w2_lora_a_stackedr   r]   rG   r   r   r   rI   r   r   r   w2_lora_b_stackedr   r{   r    )rg   rh   ra   rc   rB   r   r   r?   r   rZ   r[   rx   ry   rz   r|   intermediate_cache3shard_size_w2ri   rj   r@   rk   r'   rA   s                     r)   rl   zXFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.moe_sum_decorator.<locals>.wrapper   s    . ?-n=4'-!&#(#(	      ";
*//22

J// $ 6q 9 ? C/3/I/I""n& !- 0J 	0 	0,} )77N(O%"01B"C.<1/+ #2"6"6t~r"J"J(=(B(B4>SU(V(V%&45J&K#&*1g# &t'BDL Q Q#66''** )#/!!!("&"4;?;MT=4<77ST! 7   & t.v..r*   rf   r   s   `` r)   moe_sum_decoratorzGFusedMoEWithLoRA._inject_lora_into_fused_moe.<locals>.moe_sum_decorator   s:    : : : : : : : : :x Nr*   )r   rA   ensure_moe_quant_config_initrW   rX   r   r   select_gemm_implshared_expertsuse_mxfp4_w4a16
isinstancefused_expertsr   r   r   forward
activationmoe_sumr   )
r'   quant_configprepare_finalizem_fused_moe_fnrm   r   r   r   rk   rA   s
   `       @@r)   r%   z,FusedMoEWithLoRA._inject_lora_into_fused_moe~   s   %446663D466.O(99 $/  O*
 
 ' 	,}>U.V      ,}m.L    	 	 	 	 	N	 N	 N	 N	 N	 N	 N	`=	 =	 =	 =	 =	 =	 =	~ '4!.t@V!W!W#0=O]5$
 $
  !2 1O]2!
 !
 (=O(.(
 (
$$$r*   rG   lora_configc                      t           fdt           j                  D                        _        t	          j         j        j        j         j        j	        fj
         j                  f _        d S )Nc           	   3      K   | ]c}t          j        j        j        j        sj        nt          j        j                  j        j        fj	        j
                   V  ddS rN   r!   N)torchzerosr   r   r{   r   r   r   rI   
lora_dtyper!   .0r/   r   rG   r'   s     r)   	<genexpr>z:FusedMoEWithLoRA._create_lora_a_weights.<locals>.<genexpr>E  s       B
 B
  KO5-IK-- 94<HHO/ ",{  B
 B
 B
 B
 B
 B
r*   r   )tupleranger$   r   r   r   r   r   r   rQ   r   r!   r   r'   rG   r   s   ```r)   _create_lora_a_weightsz'FusedMoEWithLoRA._create_lora_a_weights@  s    
 =B B
 B
 B
 B
 B
 B
 4+,,B
 B
 B
 =
 =
  KO5-OC	 ",{	 	 	<
r*   c                 H    t           fdt           j                  D                        _        t	          j         j        j         j        s j        j	        nt           j        j	         j                  j        fj         j                  f _        d S )Nc              3      K   | ]B}t          j        j        j        j        j        j        fj        j                   V  CdS r   r   r   r   r   rQ   r   r   r!   r   s     r)   r   z:FusedMoEWithLoRA._create_lora_b_weights.<locals>.<genexpr>b  s}       B
 B
  KO5OC-	 ",{	 	 	B
 B
 B
 B
 B
 B
r*   r   r   r   r$   r   r   r   r   r   r{   rI   r   r   r   r   r!   r   r   s   ```r)   _create_lora_b_weightsz'FusedMoEWithLoRA._create_lora_b_weightsa  s    <A B
 B
 B
 B
 B
 B
 4+,,B
 B
 B
 =
 =
 KO5-KDO// ;T\JJ- ",{  <
r*   model_configc                    |j         | _         |j        | _        t          j        dg|dz   z  t          j        | j                  | _        |                     ||           | 	                    ||           g | _
        g | _        t          |          D ]R}t          | j        j                  D ]4}| j
                            | j        d         |         |                    | j
                            | j        d         |         |                    | j                            | j        d         |         |                    | j                            | j        d         |         |                    | j        dk    rb| j
                            | j        d         |         |                    | j                            | j        d         |         |                    6TdS )Initializes lora matrices.r   r   r   r   N)rG   fully_sharded_lorasr{   r   tensorintr!   r   r   r   lora_a_stackedlora_b_stackedr   r   r   appendr   r   r   r   r$   )r'   rG   r   r   lora_id
experts_ids         r)   create_lora_weightsz$FusedMoEWithLoRA.create_lora_weights~  s    %.(<$|C9q=!4; 
  
  
 	##I{;;;##I{;;; ! Y'' 	 	G#DO$EFF  
 #**+A.w7
C   #***1-g6zB   #**+A.w7
C   #***1-g6zB  
 #q(('../27;JG   '../27;JG  -	 	r*   
w13_lora_ac                     | j         dk    s| j        s|S |j        d         }|| j         z  dk    sJ | j        d         j        d         }| j        |z  }| j        dz   |z  }|dd||ddf         S D
        Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
        r   r   r   N)r   r{   r   r   r    )r'   r   current_lora_rank
shard_size	start_idxend_idxs         r)   _slice_w13_azFusedMoEWithLoRA._slice_w13_a  s     <1D$6 ',Q/ 4</14444,Q/5a8
L:-	<!#z1!!!Yw.122r*   
w13_lora_bc                     | j         dk    r|S | j        j        }| j        |z  }| j        dz   |z  }|d d ||d d f         S Nr   r   r   rQ   r    )r'   r   r   r   r   s        r)   _slice_w13_bzFusedMoEWithLoRA._slice_w13_b  s]    <1 _D
L:-	<!#z1!!!Yw.122r*   	w2_lora_ac                     | j         dk    r|S | j        j        }| j        |z  }| j        dz   |z  }|dddd||f         S )r   r   Nr   )r'   r   r   r   r   s        r)   _slice_w2_azFusedMoEWithLoRA._slice_w2_a  s]     <1_D
L:-	<!#z1AAAy0011r*   	w2_lora_bc                     | j         dk    s| j        s|S | j        d         j        d         }| j        |z  }| j        dz   |z  }|dd||ddf         S r   )r   r{   r   r   r    )r'   r   r   r   r   s        r)   _slice_w2_bzFusedMoEWithLoRA._slice_w2_b  sr     <1D$6 +A.4Q7
L:-	<!#z1Ig-qqq011r*   indexc                     t          | j                  D ]"}d| j        |         |<   d| j        |         |<   #d| j        d         |<   d| j        d         |<   d| j        |<   dS )z+Resets the lora weights at index back to 0.r   N)r   r$   r   r   r   r   r   )r'   r   poss      r)   
reset_lorazFusedMoEWithLoRA.reset_lora  sz    )** 	4 	4C23D#C(/23D#C(//+,q!%(+,q!%(&'U###r*   lora_alora_bc                    t          |t                    sJ t          |t                    sJ |                     |           d| j        |<   | j        d         j        d         }|\  }}}|\  }}	}
||j        d         cxk    r#|j        d         cxk    r|j        d         k    sn J |                     |          }|                     |          }|                     |          }| 	                    |	          }| j        d         |ddd|j        d         d|j        d         f         
                    |d           | j        d         |ddd|j        d         d|j        d         f         
                    |d           | j        dk    r|                     |          }|                     |
          }| j        d         |ddd|j        d         d|j        d         f         
                    |d           | j        d         |ddd|j        d         d|j        d         f         
                    |d           | j        d         |ddd|j        d         d|j        d         f         
                    |d           | j        d         |ddd|j        d         d|j        d         f         
                    |d           dS )!Overwrites lora tensors at index.r   r   Nr   Tnon_blocking)r   listr   r   r   r   r   r   r   r   copy_r   r$   r   r   )r'   r   r   r   num_experts	w1_lora_ar   	w3_lora_a	w1_lora_br   	w3_lora_bslliced_w1_lora_aslliced_w1_lora_bsliced_w2_lora_asliced_w2_lora_bslliced_w3_lora_aslliced_w3_lora_bs                    r)   set_lorazFusedMoEWithLoRA.set_lora  si    &$'''''&$'''''&'U#-a06q9*0'	9i*0'	9iq!" " " "q!" " " " q!" " " " " " !--i88 --i88++I66++I66"1112)/224P6G6Ma6P4PP	

%!%
5
5
5"1112)/224P6G6Ma6P4PP	

%!%
5
5
5 q   $ 1 1) < < $ 1 1) < <#A&qqq6-3A668T:K:QRS:T8TTe%De999#A&qqq6-3A668T:K:QRS:T8TTe%De999q!1111(.q113N5E5KA5N3NN	

% t%
4
4
4q!1111(.q113N5E5KA5N3NN	

% t%
4
4
4
4
4r*   c                 &     | j         j        |i |S N)r   r   r'   rg   rh   s      r)   r   zFusedMoEWithLoRA.forward,  s    &t&7777r*   c                 &     | j         j        |i |S r   )r   &maybe_all_reduce_tensor_model_parallelr   s      r)   r   z7FusedMoEWithLoRA.maybe_all_reduce_tensor_model_parallel/  s    EtEtVvVVVr*   c                     | j         j        S r   )r   _shared_expertsr'   s    r)   r   z FusedMoEWithLoRA._shared_experts2  s    ..r*   c                     | j         j        S r   )r   rW   r   s    r)   rW   zFusedMoEWithLoRA.quant_method6  s    ++r*   c                     | j         j        S r   )r   is_internal_routerr   s    r)   r   z#FusedMoEWithLoRA.is_internal_router:  s    11r*   source_layerpacked_modules_listc                 R    t          |t                    ot          |          dk    S )=Returns True if the layer can be replaced by this LoRA layer.r   r   r   lenclsr   r   r   r   s        r)   can_replace_layerz"FusedMoEWithLoRA.can_replace_layer>  s(     ,11Sc:M6N6NRS6SSr*   r   )%__name__
__module____qualname__r   r   dictstrr   r:   r]   r%   r   r   r   r   r   r   Tensorr   r   r   r   r   r   r   r   r   propertyr   rW   boolr   classmethodnnModuler  __classcell__r(   s   @r)   r   r   ,   sv       +8 + + + + + + +!d3d
?&; !S#PT*_@U ! ! ! !3,3, 3, 	3,
 3, 3, 3, 3, 3, 3, 3, 3,j@
 @
 @
D

  
 
 
 
B
 
* 
 
 
 
B 15	. ..  . '-	.
 
. . . .`3u| 3 3 3 3 3 	3u| 	3 	3 	3 	32U\ 2el 2 2 2 22U\ 2el 2 2 2 2( ( ( ( (:5:5 tEL11:5 tEL11	:5 :5 :5 :5x8 8 8W W W / / X/ , , X, 2D 2 2 2 X2  15
T 
Ti
T  
T "	
T
 '-
T 

T 
T 
T [
T 
T 
T 
T 
Tr*   r   c                   T    e Zd Z fdZd Z	 ddedededz  ddfdZd	e	j
        fd
Zdede	j
        ee	j
                 z  de	j
        ee	j
                 z  fdZed             Zed             Zed             Zed             Ze	 ddej        dedededz  def
d            Z xZS )FusedMoE3DWithLoRAc                 X    t                                          |           d| _        d S r   )r   r   r$   r&   s     r)   r   zFusedMoE3DWithLoRA.__init__M  s)    $$$r*   c                 H    t           fdt           j                  D                        _        t	          j         j        j         j        s j        j	        nt           j        j	         j                  j        fj         j                  f _        d S )Nc              3      K   | ]E}t          j        j        j        j        j        d z  j        fj        j                  V  FdS )r   r   Nr   r   s     r)   r   z<FusedMoE3DWithLoRA._create_lora_b_weights.<locals>.<genexpr>R  sz       =
 =
  KO5OCaG-	 ",{	 	 	=
 =
 =
 =
 =
 =
r*   r   r   r   s   ```r)   r   z)FusedMoE3DWithLoRA._create_lora_b_weightsQ  s    7< =
 =
 =
 =
 =
 =
 4+,,=
 =
 =
 8
 8
 KO5-KDO// ;T\JJ- ",{  7
r*   NrG   r   r   r   c                 D   t          |t                    sJ |j        d         | _        |j        | _        |j        | _        t          j        dg|dz   z  t          j	        | j
                  | _        |                     ||           |                     ||           dS )r   r   r   r   N)r   r   architectures_base_modelrG   r   r{   r   r   r   r!   r   r   r   )r'   rG   r   r   s       r)   r   z&FusedMoE3DWithLoRA.create_lora_weightsn  s     ,(899999'5a8$.(<$|C9q=!4; 
  
  
 	##I{;;;##I{;;;;;r*   r   c                 4   | j         dk    r|S | j        j        }| j        |z  }| j        dz   |z  }| j        dk    rr|d d d d dd d f         }|d d dd dd d f         }|d d ||d d f         }|d d ||d d f         }t          j        ||gd                              dd          S |j        d         dz  }	|d d d |	d d f         }|d d |	d d d f         }|d d ||d d f         }|d d ||d d f         }t          j	        ||gd          S )Nr   GptOssForCausalLMr   )dim)
r   r   rQ   r    r  r   stackflattenr   cat)
r'   r   r   r   r   r   r   sliced_w1_lora_bsliced_w3_lora_b
slice_sizes
             r)   r   zFusedMoE3DWithLoRA._slice_w13_b  s   <1 _D
L:-	<!#z1222 #111ccc1119-I"111addAAA:.I(Ig,=qqq)@A(Ig,=qqq)@A; 02BCKKKSS1   $)!,1J"111kzk111#45I"111jkk111#45I(Ig,=qqq)@A(Ig,=qqq)@A9.0@AqIIIIr*   r   r   r   c                    t          |t                    sJ t          |t                    sJ t          |          t          |          cxk    rdk    sn J |                     |           d| j        |<   |\  }}|\  }}|                     |          }|                     |          }	|                     |          }
|                     |          }| j	        d         |ddd|j
        d         d|j
        d         f                             |d           | j        d         |ddd|
j
        d         d|
j
        d         f                             |
d           | j        d         |ddd|	j
        d         d|	j
        d         f                             |	d           | j        d         |ddd|j
        d         d|j
        d         f                             |d           dS )r   r   r   r   NTr   )r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   )r'   r   r   r   r   r   r   r   sliced_w13_lora_asliced_w13_lora_br   r   s               r)   r   zFusedMoE3DWithLoRA.set_lora  s6    &$'''''&$'''''6{{c&kk....Q......&'U# &
I &
I --j99 --j99++I66++I66"1112)/224P6G6Ma6P4PP	

%!%
5
5
5q!1111(.q113N5E5KA5N3NN	

% t%
4
4
4"1112)/224P6G6Ma6P4PP	

%!%
5
5
5q!1111(.q113N5E5KA5N3NN	

% t%
4
4
4
4
4r*   c                 2    | j         d         j        d         S 
        Full size
        r   r0   )r   r   r   s    r)   w13_input_sizez!FusedMoE3DWithLoRA.w13_input_size  s    
 &q)/33r*   c                 B    | j         d         j        d         | j        z  S )r(  r   rt   )r   r   r   r   s    r)   w13_output_sizez"FusedMoE3DWithLoRA.w13_output_size  s!    
 &q)/3dlBBr*   c                 B    | j         d         j        d         | j        z  S r'  )r   r   r   r   s    r)   w2_input_sizez FusedMoE3DWithLoRA.w2_input_size  s!    
 %a(.r2T\AAr*   c                     | j         j        S )r(  )r   rI   r   s    r)   w2_output_sizez!FusedMoE3DWithLoRA.w2_output_size  s    
 **r*   r   r   c                 R    t          |t                    ot          |          dk    S )r   r   r   r  s        r)   r  z$FusedMoE3DWithLoRA.can_replace_layer  s(     ,11Sc:M6N6NRS6SSr*   r   )r  r  r  r   r   r   r   r   r   r   r
  r   r   r   r  r)  r+  r-  r/  r  r  r  r  r  r  r  s   @r)   r  r  L  s           
 
 
B 15	< <<  < '-	<
 
< < < <(Ju| J J J J:$5$5 tEL11$5 tEL11	$5 $5 $5 $5L 4 4 X4 C C XC B B XB + + X+  15	T 	Ti	T  	T "		T
 '-	T 
	T 	T 	T [	T 	T 	T 	T 	Tr*   r  )(rR   r   torch.nnr  transformersr   vllmr   vllm.config.lorar   vllm.distributed.parallel_stater   r   vllm.distributed.utilsr   vllm.lora.layers.baser	   vllm.lora.ops.triton_ops.utilsr
   $vllm.model_executor.layers.fused_moer   +vllm.model_executor.layers.fused_moe.configr   5vllm.model_executor.layers.fused_moe.fused_marlin_moer   .vllm.model_executor.layers.fused_moe.fused_moer   =vllm.model_executor.layers.fused_moe.fused_moe_modular_methodr   ?vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moer   3vllm.model_executor.layers.fused_moe.modular_kernelr   5vllm.model_executor.layers.fused_moe.prepare_finalizer   utilsr   r   r   r  rf   r*   r)   <module>rB     sH              ) ) ) ) ) )       ' ' ' ' ' '        * ) ) ) ) ) 3 3 3 3 3 3 > > > > > > 9 9 9 9 9 9                                    E D D D D D D D]T ]T ]T ]T ]T( ]T ]T ]T@_T _T _T _T _T) _T _T _T _T _Tr*   