
    -`iY                     t   d Z ddlZddlmZ ddlmZ ddlZddlZddl	m
Z
mZ ddlmZmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlm Z m!Z!m"Z"  ee#          Z$e G d d                      Z% G d d          Z&e G d d                      Z'de
ez  de(e)e)f         de)fdZ*dS )a  
Expert parallelism load balancer (EPLB) metrics and states.

# Glossary

- **Logical Expert**: An expert that is part of the model's logical structure.
  It holds a set of weights and is replicated across multiple physical
  experts.
- **Redundant Expert**: To achieve load balancing, for some popular logical
  experts, we create additional copies of the expert weights. During inference,
  each of these copies can be routed to by the same set of tokens.
- **Physical Expert**: An expert that is instantiated on a specific device.
  It is a replica of a logical expert and can be rearranged across devices.
  I.e., one logical expert may have multiple sets of weights initialized on
  different devices, and each of these sets is a physical expert.
- **Local Physical Expert**: A physical expert that is instantiated on the
  current device.

For example: DeepSeek-R1 has 256 logical experts, so each MoE layer
has 256 sets of linear layer weights in the model parameters. If we add 32
redundant experts, DeepSeek-R1 will have 256 + 32 = 288 physical experts in
total. And when deploying, we'll have 288 sets of linear layer weights for each
MoE layer. If we have 32 EP ranks, then each GPU will hold 288 / 32 = 9 local
physical experts.
    N)Sequence)	dataclass)ProcessGroup
all_reduce)ModelConfigParallelConfig)get_ep_groupget_node_countin_the_same_node_as)StatelessProcessGroup)init_logger)MixtureOfExperts   )start_async_worker)EPLB_POLICIESAbstractEplbPolicyDefaultEplbPolicy)RecvMetadatamove_from_buffer rearrange_expert_weights_inplacec                      e Zd ZU dZej        ed<   	 ej        ed<   	 ej        ed<   	 ej        ed<   	 ej        ed<   	 eed<   eed<   e	ej                 ed	<   	 e
j        ed
<   	 ej        j        dz  ed<   	 eed<   	 eed<   	 eed<   	 eed<   	 ej        ed<   	 ej        ed<   	 eed<   	 edz  ed<   	 dZej        dz  ed<   	 dZej        dz  ed<   	 dZej        dz  ed<   dS )EplbModelStatezEPLB metrics.physical_to_logical_maplogical_to_physical_maplogical_replica_countexpert_load_passexpert_load_window
model_namemodelexpert_bufferbuffer_lockNbuffer_ready_eventep_buffer_readylayer_to_transfer
rebalancedpending_global_ready_checkis_unchangedis_received_locallyrecv_metadatacuda_device_indexnew_physical_to_logical_mapnew_logical_to_physical_mapnew_logical_replica_count)__name__
__module____qualname____doc__torchTensor__annotations__strr   list	threadingLockcudaEventintboolnpndarrayr   r+   r,   r-        t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/distributed/eplb/eplb_state.pyr   r   :   s        "\))) #\))). !<''' l""" $$$ OOO%%%%  
(4////    !%$$$ * ###   Tz!!! 8<!4;;; 8<!4;;; 6:u|d2999 r@   r   c                      e Zd ZdZdedej        fdZede	de	de
e	         fd            Zd	efd
Z	 	 	 d(dededej        dz  dej        dz  dee	e	f         dz  f
dZ	 	 	 d)dedededdfdZ	 	 	 	 d*dededeej                 dz  dee	e	f         dz  dej        dz  f
dZ	 	 d+dee	e	f         dz  defdZdede	ddfdZdedefdZ	 d,ded edefd!Zd,dededdfd"Zedeeej                 eej                 f         fd#            Zededeeej                 dz  eej                 dz  ee	e	f         dz  f         fd$            Z d%eej                 deej                 fd&Z!deej                 fd'Z"dS )-	EplbStatezP
    EplbState of each expert parallel model. Key is the model config hash.
    parallel_configdevicec                    || _         || _        i | _        t          | _        	 d| _        	 d| _        	 d| _        	 d| _        	 d| _	        	 t          j                    | _        	 d | _        	 d | _        	 | j        j        dk    r[| j        j        | _        | j        Et"          j                                        r)t"          j                                        | _        d S d S d S d S )Nr   Fr9   )rD   rE   model_statesr   policyexpert_load_window_stepexpert_load_window_sizeexpert_rearrangement_step"expert_rearrangement_step_intervalis_asyncr7   r:   rearrange_eventasync_workerr*   typeindexr2   r9   is_availablecurrent_device)selfrD   rE   s      rA   __init__zEplbState.__init__   s   .790A	 -.$	 -.$	 /0&	 89/	 $	  )00	 6:	 .2	 ;v%%%)[%6D"%-%*2I2I2K2K-).)B)B)D)D&&& &%----r@   num_routed_expertsnum_redundant_expertsreturnc                 |     t          t                               }| fdt          |          D             z  }|S )aj  
        Build an initial expert arrangement using the following structure:
        [original routed experts, redundant experts]

        Returns:
            physical_to_logical_map (Sequence[int]): A list of integers,
                where each integer is the index of the logical expert
                that the corresponding physical expert maps to.
        c                     g | ]}|z  S r?   r?   ).0irV   s     rA   
<listcomp>zJEplbState.build_initial_global_physical_to_logical_map.<locals>.<listcomp>  s,     +
 +
 +
'(A""+
 +
 +
r@   )r6   range)rV   rW   global_physical_to_logical_maps   `  rA   ,build_initial_global_physical_to_logical_mapz6EplbState.build_initial_global_physical_to_logical_map  s^     *.e4F.G.G)H)H&& +
 +
 +
 +
,12G,H,H+
 +
 +
 	
& .-r@   	new_modelc                 >   t          | j                  dk    rt          t          | j                                                            j        }|j        |j        k    s@|j        |j        k    s0|j        |j        k    s |j	        |j	        k    s|j
        |j
        k    r{t          d                    t          |          |j        |j        |j        |j	        |j
        t          |          |j        |j        |j        |j	        |j
                            dS dS )z~
        Validate that the expert parallel configuration of
        the new model is the same as the existing models.
        r   zZModel: {} with config {} {} {} {} {} mismatch with new model {} with config {} {} {} {} {}N)lenrG   nextitervaluesr   rV   rW   num_physical_expertsnum_logical_expertsnum_expert_groupsRuntimeErrorformatrP   )rT   ra   r   s      rA   validate_ep_configurationz#EplbState.validate_ep_configuration  s   
 t !!A%%d/668899::@E(I,HHH.)2QQQ-1OOO,	0MMM*i.III""
 #)&U0321/Y!4!7!6!5!3# #   &% JIr@   Nr   model_configglobal_expert_loadold_global_expert_indicesrank_mappingc                 
   |                      |           | j        j        j        | _        t
                              |j        |j                  }t          j
        || j                  }d}|j        |k    sJ d|j         d|             |dz   }	t          j        |j        |	fd| j                  }
t          j        |j        f| j        t          j                  }t!          |j                  D ]'}||         }||
|||         f<   ||xx         dz  cc<   (|                    d                              |j        d                                          }|
                    d                              |j        dd                                          }
|                    d                              |j        d                                          }t          j        |j        |j        ft          j        | j        	          }| j        j        j        | _        t          j        | j        |j        |j        ft          j        | j        	          }| j        j        j        }t5          d||d
z  z
            | _        || _        | j        j        j        }t<          |         | _        t>                               d|           |\tC                      j"        }|j#        |j        |j        fk    sJ |j$        t          j%        k    sJ |j        }|j&        }tO                      }|(                                }||z  dk    r"d}t>          )                    d|d|           | j        *                    |||||          \  }}}|j#        d         }||
j#        d         k    sJ t          j+        j,        -                    |d|
j#        d         |z
  fd          }|.                    | j                  }|
/                    |           |/                    |           nd}d}d}|0                    ||
|           | tc          |||j2        |d|           d| _        d |j2        d         D             }tg          d(i d|d|
d|d|d|d|j4        d|d|dtk          j6                    ddddddddddd to          j8        g           d!to          j8        g           d"ts          to          j8        g           dto          j8        g           to          j8        g           #          d$| j:        d%|d&|d'|}|| j;        |<                                <   dS ))z/
        Build the initial EPLB state.
        rE   i  znum_redundant_experts z must be less than or equal to r   )rE   dtyper   rt   rE      zSelected EPLB policy: %sNTnum_gpus % num_nodes != 0, not using hierarchical rearrangement algorithm.
num_gpus=, num_nodes=valueFc                 6    g | ]}t          j        |          S r?   )r2   
empty_like)r[   ws     rA   r]   z'EplbState.add_model.<locals>.<listcomp>  s#    NNN)!,,NNNr@   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   )recv_primary_mask
recv_countrecv_expert_idsrecv_dst_rowsr*   r+   r,   r-   r?   )=rl   rD   eplb_config	use_asyncrM   rC   r`   rV   rW   r2   tensorrE   fullrh   zeroslongr^   rg   	unsqueezeexpandnum_moe_layers
contiguousint32window_sizerJ   step_intervalmaxrK   rL   rH   r   loggerdebugr	   device_groupshapert   int64ri   r
   sizewarning_oncerebalance_expertsnn
functionalpadtocopy_set_eplb_stater   expert_weightsr   r   r7   r8   r=   arrayr   r*   rG   compute_hash)rT   r   rm   rn   ro   rp   physical_to_logical_map_listr   MAX_EXPERT_REDUNDANCYmax_slots_per_logical_expertr   r   r\   logical_idxr   r   eplb_step_intervalpolicy_typeep_groupnum_replicas
num_groups	num_nodesnum_gpusr+   r,   r-   max_physical_slotsr    model_states                                rA   	add_modelzEplbState.add_modelC  s    	&&u---,8B BB(+  	% #(,(;#
 #
 #
 !%*.CCCCEU%@ E E-BE E DCC (=q'@$"'*&(DE;#
 #
 #

 !&&(;*!
 !
 !
 u122 	4 	4A1!4KWX#K1F{1S$ST!+...!3.... $--a00V$  Z\\ 	  $--a00V$ 
 Z\\ 	  "++A..V$  Z\\ 	 !;!5#=>+;
 
 

 (,';'G'S$"[,$*
 +;
 
 
 "1=K),!$6!$;;*
 *
& 3E/ *6=#K0/===)#~~2H%+$)0     &+u{:::: 5L0J&((I}}H)#q((	##11 1$-1 1   --" 	++) "=!B2!F%)@)Fr)JJJJJ*/(*=*A*A++1"58JJK +B + +'
 'B&D&DT[&Q&Q##))*EFFF!''(ABBBB*.'*.'(,%#!	
 	
 	

 ),)+$   ./D*NNe6J16MNNN$ 
 
 
$;$;
$;$;
 #8"7
 .-	

  21
 $))
 %
 (-
 "(((
  $t
 A
  a
 u
 (-u
 "
  !#!
" '"$(2,, " hrll	   #
. #44/
0 )D(C1
2 )D(C3
4 '@&?5
8 :E,3355666r@   Fis_dummy
is_profile	log_statsc                    t                      j        }|r|                     d           dS |r5| j                                        D ]}|j                                         |r| j        | j        j	        j
        z  dk    r|                                 }t                      j        }t          || j                                                  D ]B\  }}|                    |j        d         |                                d                              d                                          }|                    d                              d          }	|                    d          j                            d          }
t)          j        |	|
g                                          }|\  }}|dk    r||z  nd}|                                dk    r7t0                              d| j        |j        |||| j        | j        z
             D|s| j                                        D ]A}|j                                        |j        | j        <   |j                                         B| xj        d	z  c_        | j        | j        k    rd| _        | xj        d	z  c_        | j         r| j                                        D ]}d
}|j!        r| "                    |          }|j#        r|r| $                    |||           |j%        |j&        j'        k    ri| (                    ||           d
|_)        d|_%        d
|_!        t0                              d|j        |                                |j&        j'                   | j        | j        k    rV| j         r2tU          d | j                                        D                       rdS d| _        |                                  dS dS )a'  
        Step the EPLB state.

        Args:
            is_dummy (bool): If `True`, this is a dummy step and the load
                metrics recorded in this forward pass will not count.
                Defaults to `False`.
            is_profile (bool): If `True`, perform a dummy rearrangement
                with maximum communication cost. This is used in
                `profile_run` to reserve enough memory
                for the communication buffer.
            log_stats (bool): If `True`, log the expert load metrics.

        # Stats
            The metrics are all summed up across layers.
            - `avg_tokens`: The average load across ranks.
            - `max_tokens`: The maximum load across ranks.
            - `balancedness`: The ratio of average load to maximum load.
        T)r   Nr   rs   dimg        zuEPLB step: %d for model %s: avg_tokens=%.2f, max_tokens=%d, balancedness=%.4f, steps until the next rearrangement: %dr   F)r   r   r   z3finish async transfer for model %s rank %d layer %dc              3   $   K   | ]}|j         V  d S )N)r%   )r[   eplb_model_states     rA   	<genexpr>z!EplbState.step.<locals>.<genexpr>  s9       % %$ !+% % % % % %r@   )+r	   r   	rearrangerG   rf   r   zero_rK   rD   r   log_balancedness_interval_sync_load_passzipreshaper   r   sumfloatmeanr   r2   stacktolistrankr   infor   rL   cloner   rI   rJ   rM   r&   _all_ranks_buffer_readyr#   move_to_workspacer$   r   r   	post_eplbr%   any)rT   r   r   r   r   r   expert_load_pass_listr   num_tokens_per_rankavg_tokens_tensormax_tokens_tensortokens_tensors
avg_tokens
max_tokensbalancednessall_ranks_buffer_readys                   rA   stepzEplbState.step  su   2  >>. 	NNdN+++F 	:$($5$<$<$>$> : :  1779999 /	.".HI  %)$8$8$:$:!#~~2H69%t'8'?'?'A'A7 7 % %2 "2
 %,,(.q18==??B  SRS[[UWW $ %8$<$<$<$C$C$G$GA$G$N$N!$7$;$;$;$B$B$I$M$MRS$M$T$T! /4k&(9:/ /&((  *8&
J:Dq..zJ66c==??a''KKA 6(3""$?89    		1$($5$<$<$>$> : : $5;;== !3D4PQ !1779999((A-((+t/KKK/0, 	&&!+&&= 	$($5$<$<$>$>   ).&#> -1-I-I(. .* $3 8N **$4!)#- +    ):+1@A A '7DDD6;(3=>(:FK(CQ,7$MMOO,2A	   )T-TTT}  % %(,(9(@(@(B(B% % % " " 
 -.D*NN UTr@   Texecute_shuffleglobal_expert_loadsc                    t                      j        }|                                }d}d}|dk    }	|	r| j        r|rTt          j                            d          }t          j                            d          }|                                 t          	                    d| j        rdnd|rdnd	           |Hg }
|sgt	          j
        t          | j                  gt          j        d
          }t          j                            |t                      j        d           | j                                        D ]P}t	          j        | j        |j        j        |j        j        |j        j        |j        j                  }|                    d|j                            d                              |j                                                  |j                   |s{t	          j
        |j        j        |j        j        |j        j        d         gt          j        d
          }t          j                            |t                      j        d           |                     d          }|
!                    |           R| "                    |
          }
|sVtG          | j                                        |
          D ].\  }}|j        }t          j                            ||d           /|s|
S n|sJ |}
tI          tK          | j                                                            }|j        }|j&        }|j'        }|t          |          |(                                k    rit                      j        }tS          ||          }tA          d |                                D                       }||(                                z  |z  }n"tU                      }|(                                }||z  dk    r"d}t          +                    d|d|           tG          | j                                        |
          D ]\  }}| j,        -                    ||||||j                  \  }}}| j        r|rt]          |j        ||j        j/        |||           |s|j        j        d         |j        d         k    r%|0                    |j        j                  |_        n|j        1                    |           |j        d         }||j2        j        d         k    sJ t          j3        j4        5                    |d|j2        j        d         |z
  fd          }|j2        1                    |           |j6        1                    |           |	rh|J |J |                                 |7                                 |8                    |          dz  }t          	                    d|rdnd|           |j2        j        d         }t          j3        j4        5                    |dts          d||j        d         z
            fd          0                    |j2        j                  }|0                    |j6        j                  }|:                                |_;        ||_<        ||_=        d|_>        d|_?        d|_@        | j        r|s| jA        B                                 dS )a  
        Rearrange the experts according to the current load.

        Args:
            is_profile (bool): If `True`, perform a dummy rearrangement.
                This is used in `profile_run` to reserve enough memory,
                no memory movement will be performed. Default is False.
            execute_shuffle (bool): If `True`, execute the shuffle
                in elastic expert parallel (EEP). Default is True.
            global_expert_loads (list[torch.Tensor] | None): The global expert
                loads when scaling is done in EEP.
                List of expert loads for the main and drafter
                (when spec decode is used) models.
            rank_mapping (dict[int, int] | None): The rank mapping
                when scaling is done in EEP.
        Nr   T)enable_timingzRearranging experts %s %s...z(async mode)z	sync modez	(profile) cpuru   group	group_srcrs   )r   rQ   srcr   r   c              3   "   K   | ]
}|d k    V  dS )rs   Nr?   )r[   new_ranks     rA   r   z&EplbState.rearrange.<locals>.<genexpr>  s&      PPh8r>PPPPPPr@   rw   rx   ry   g     @@z Rearranged experts %s in %.2f s.z (profile)  )Cr	   r   r   rM   r2   r9   r:   recordr   r   r   rc   rG   r   distributed	broadcast	cpu_grouprf   r   rJ   r   r   rh   r   rt   rE   scatter_add_r   r   	expand_asr   r   r   append_allreduce_listr   rd   re   rg   ri   r   _node_count_with_rank_mappingr
   r   rH   r   r   r   r   r   r   r   r   r   r   synchronizeelapsed_timer   r   r+   r,   r-   r%   r$   r&   rN   set)rT   r   r   r   rp   r   ep_rankstart_event	end_eventis_main_rankglobal_expert_load_windows
num_modelsr   logical_expert_load_windowmetadataglobal_expert_load_windowro   r   r   r   r   r   r   r+   r,   r-   r   gpu_elapsed	max_slotspadded_logicalnew_replicas                                  rA   r   zEplbState.rearrange  s   0  >>.--//	!| 		= %J %#j..T.BB!J,,4,@@	""$$$KK."&-@[)1r   &)+&" "\*++,EK  
 !++lnn&>! ,    %)$5$<$<$>$> M M -2[0$*9$*>*=C+>E. . .* +77*BLLQOOY/BCCTVV(; 8    ' $|,2A,2F,DJ1M
 $k$     H %// (@A 0    -G,J,Jq,J,Q,Q)*112KLLLL)-)=)=** *& # CF%,,..0JD D  ?$&? 1A0X-%//1Q 0     # 2112 #""?)<&  T%6%=%=%?%? @ @AA &1,
#L(9(9X]]__(L(L %0I5iNNIPP,:M:M:O:OPPPPPH/(: L '((I}}Hi1$$I-- - )- -   <?$$&&(B<
 <
 ]	C ]	C77 --) 8 	++) = LCJ LC0$</$*9    " !(@FqI6<Q?@ @ 8:: 0 H O  )@@ )@FF7   *E)J2)N&*+CI"MN N N N 38(2E2I2I3,DJ2N01
 ! 3J 3 3/ %<BB3   %:@@1     
&222$000$$&&&))+++"-":":9"E"E"NKKK:)3<#   -DJ2N	!&!4!8!8/Ay+F+LR+PPQQR "9 " " "%=DEE	 
 8::$:A  03355 !< @N <=H :.2 +56 2>B ;; = 	'* 	' $$&&&tr@   c                 X    | j         sd S | j        t          | ||          | _        d S d S )N)rp   r   )rM   rO   r   )rT   rp   r   s      rA   start_async_loopzEplbState.start_async_loop  sJ    
 } 	F$ 2)%! ! !D %$r@   r   layerc                    |j         |j        |j        d S |j        j        }|j         }|j        j        d         |j        d         k    r|                    |          |_        n9|j        |                             ||                             |                     |j        j        }|j        |                             |          }|j        j        d         }||j        d         z
  }|dk    r)t          j
        j                            |d|fd          }|j        |                             |           |j        j        }	|j        |                             |j        |                             |	                     d S )Nr   rs   r   ry   )r+   r,   r-   r   rE   r   r   r   r   r2   r   r   r   r   )
rT   r   r   target_devicenew_physicallogical_devicenew_logicalr   
slot_deltareplica_devices
             rA   _update_layer_mapping_from_newz(EplbState._update_layer_mapping_from_new  s    3;6>4<F#;B">.4Q7<;Ma;PPP2>//-2P2PK///6<<U#&&}55   %<C!=eDGGWW7=bA	!22!66
>>(-11a_B 2  K 	+E288EEE$:A)%0661%8;;NKK	
 	
 	
 	
 	
r@   c                     t                      }t          |dd           }||                                dk    r|t          j        t          |j                  ft          j        d          }t          ||           t          |	                                          |                                k    S |j
        }|                                dk    rt          |j                  S t          |d|j        j                  }t          j        t          |j                  ft          j        |          }t          ||           t          |	                                          |                                k    S )Nr   r   r   ru   r   rE   )r	   getattrr   r2   r   r;   r#   r   r   itemr   r<   r   rE   )rT   r   parallel_stater   flagr   rE   s          rA   r   z!EplbState._all_ranks_buffer_ready  sT   %NK>>	 Y^^%5%5%9%9<[01135;u  D t9----tyy{{##y~~'7'777%2!##3444Hk&I&P
 
 |,--/u{6
 
 
 	4|,,,,499;;<#4#4#6#666r@   r   c           
      |   d}d}|j                             dd          s{|dz  }||k    r%t          d|                                 d          t                              d	|                                ||           |j                             dd          {	 |j        J |j        p| j        }|j        C|At          j
                            |
          }|                    |j                   d |_        |j        j        |j                 }|j        }	|j        |j                                                                                 }
t'          ||	|j        |j        |j        |
|                                           |j        }|                     ||           |xj        dz  c_        d|_        t                              d|j        |           	 |j                                          d S # t8          $ rF}t                              d|                                t=          |                     Y d }~d S d }~ww xY w# 	 |j                                          w # t8          $ rE}t                              d|                                t=          |                     Y d }~w d }~ww xY wxY w)N   r   Tg      $@)blockingtimeoutr   zRank z/: buffer_lock timeout after {max_retries * 10}sz:Rank %d: EPLB buffer_lock acquire failed, retrying (%d/%d)rr   )r   expert_weights_buffersr'   r(   r)   new_indicesr   z0model %s successfully move_to_workspace layer %dz<Rank %d: buffer_lock release failed in move_to_workspace: %s)r!   acquirerj   r   r   warningr+   r*   r"   r2   r9   current_stream
wait_eventr   r   r$   r    r   numpyr   r'   r(   r)   r  r#   r   r   release	Exceptionerrorr5   )rT   r   r   r   max_retriesretriesdevice_indexstreamr   expert_weights_bufferr  transferred_layeres                rA   r   zEplbState.move_to_workspace  s    )1141NN 	qLG+%%"*HMMOO * * *   NNL	   )1141NN 	+	:FFF&8RD<RL-9l>V22,2GG!!+"@AAA15.(.=-N %0$=!78UV 
 -'<(5$/$C)7'     !, =//=NOOO))Q.))*+K'LLB&!  '//11111   RMMOOFF        '//1111   RMMOOFF       sO   EI G: :
I
;II
J;I)(J;)
J83;J3.J;3J88J;c                     |j         J |j        J |j        J |s8t          |j        j        d                   D ]}|                     ||           d |_         d |_        d |_        d S )Nr   )r+   r,   r-   r^   r   r   r  )rT   r   r   	layer_idxs       rA   r   zEplbState.post_eplb  s    6BBB6BBB4@@@ 	L";#F#LQ#OPP L L	33KKKKK26/26/04---r@   c                  <   t                      } t          j        dt          j        d          }t          j                            || j        d           |                                }g }g }t          |          D ]}t          j        dt          j        d          }t          j                            || j        d           |	                                \  }}}t          j
        ||ft          j        | j                  }	t          |	| j                   t          j        ||ft          j        | j                  }
t          j                            |
| j        d           |                    |	           |                    |
           ||fS )zQ
        Receive the expert load and old placement from the master rank.
        r   r   ru   r   r      r  )r	   r2   emptyr   r   r   r   r  r^   r   r   r   rE   r   r   r   )r   r   r   #old_global_expert_indices_per_model_r   r   rh   num_old_physical_expertsrn   ro   s              rA   
recv_statezEplbState.recv_state  s   
  >>[%+eDDD
##Jh6HTU#VVV__&&
 .0+z"" 	R 	RA{1EKFFFH''8JVW'XXX!! JN/1I "'!45k" " "
 )1FGGGG(-!9:k) ) )%
 '')+ (   
  &&'9:::/667PQQQQ"$GGGr@   c                 4   t          j        dt           j        d          }t           j                            |t                      j        d           t          |                                          }t                      j	        }t                                          \  }}|d         j        d         }||z  |z
  |j        _        |d         j        d         |z  dk    sJ |d         j        d         |z  }d t          |          D             }|||fS )Nr   r   ru   r   r   c                     i | ]}||S r?   r?   )r[   old_ep_ranks     rA   
<dictcomp>z+EplbState.get_eep_state.<locals>.<dictcomp>\  s    WWW[[WWWr@   )r2   r$  r   r   r   r	   r   r;   r  
world_sizerC   r(  r   r   rW   r^   )	clsrD   num_local_physical_expertsnew_ep_sizer   r%  rh   old_ep_sizerp   s	            rA   get_eep_statezEplbState.get_eep_state;  s>    &+[%+e%T%T%T"##&..* 	$ 	
 	
 	

 &))C)H)H)J)J%K%K""nn/  "" 	A@
 2!4:1=&47JJ 	#9 028;>XX    028;)* 	 XWE+DVDVWWW/
 	
r@   tensor_listc                    t                    dk    r*t          d         t                      j                   S t	          d D                       s
J d            t	          fdD                       s
J d            d D             }t          j        d	          }t                      j        }t          ||           g }d}|D ]7}|                    ||||d         z   d
d
f                    ||d         z  }8|S )z/
        All-reduce a list of tensors.
        r   r   r  c              3   F   K   | ]}|                                 d k    V  dS )   Nr   r[   ts     rA   r   z,EplbState._allreduce_list.<locals>.<genexpr>j  s.      55A15577a<555555r@   zAll tensors must be 2D.c              3   \   K   | ]&}|j         d          d         j         d          k    V  'dS )r   r   Nr   )r[   r8  r3  s     rA   r   z,EplbState._allreduce_list.<locals>.<genexpr>k  s:      NNQ171:Q!5a!88NNNNNNr@   z(All tensors must have the same shape[1].c                     g | ]	}|j         
S r?   r:  r7  s     rA   r]   z-EplbState._allreduce_list.<locals>.<listcomp>q  s    ///a!'///r@   r   N)rc   r   r	   r   allr2   catr   )rT   r3  shapesconcat_tensorr   all_reduce_listoffsetr   s    `      rA   r   zEplbState._allreduce_listc  sI    {q  {1~\^^-HIIII5555555PP7PPP5NNNN+NNNNN 	
 	
6	
 	
N 0/;///	+1555>>.=1111 	 	E""=&58:K1KQQQ1N#OPPPeAhFFr@   c                     g }| j                                         D ].}|                    |j                                                   /|                     |          S )z
        Sync the expert load pass across all ranks for log stats.
        Doesn't update the expert load pass in eplb_model_state.
        )rG   rf   r   r   r   r   )rT   load_pass_listr   s      rA   r   zEplbState._sync_load_pass~  se    
  $ 1 8 8 : : 	M 	M!!"2"C"I"I"K"KLLLL##N333r@   )NNN)FFF)FTNN)NF)F)#r.   r/   r0   r1   r   r2   rE   rU   staticmethodr;   r   r`   r   rl   r   r3   dictr   r<   r   r6   r   r   r   r  r   r   r   r   tupler(  classmethodr2  r   r   r?   r@   rA   rC   rC      s        6E 6E 6E 6E 6E 6Ep ..". 
#. . . \.&"3C " " " "P 379=.2GE GEGE "GE "L4/	GE
 $)<$#6GE 38nt+GE GE GE GEV  	J JJ J 	J
 
J J J J\ ! $9=.2f ff f "%,/$6	f
 38nt+f 
	f f f fT /3  38nt+     
) 
25 
	 
  
  
  
D7> 7d 7 7 7 76 !	B B#B B 	B B B BH	5 	5^ 	5 	5RV 	5 	5 	5 	5 "Hd5<0$u|2DDE "H "H "H \"HH %
,%
	U\T!U\T!S#X	
%
 %
 %
 [%
N4+= $u|BT    64el!3 4 4 4 4 4 4r@   rC   c                   n    e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )EplbLayerStatez*Runtime EPLB data stored in the MoE layer.Nexpert_load_viewr   r   )
r.   r/   r0   r1   rJ  r2   r3   r4   r   r   r?   r@   rA   rI  rI    s`         44,0elT)00037U\D0777155<$.55555r@   rI  pgrp   rX   c                    t          | t                    r!t          j                            |           }n| j        }|dk    rdS dg|z  }d}t          |          D ]d}||         dk    r||v sJ ||         dk    r"|dz  }|||<   t          | |          }t          |          D ]\  }}|r||         dk    r|||<   e|S )Nr  r   r   rs   )	
isinstancer   r2   r   get_world_sizer-  r^   r   	enumerate)	rK  rp   r-  node_assignmentnext_node_idcurrent_ranksame_node_flags
other_rankis_same_nodes	            rA   r   r     s    "l## #&55B5??

]
Qq cJ&OLj)) ; ;<(A--|++++%++ 	(4% .b,??(1/(B(B 	; 	;$J ;
 ;q @ @.:
+	; r@   )+r1   r7   collections.abcr   dataclassesr   r  r=   r2   torch.distributedr   r   vllm.configr   r   vllm.distributed.parallel_stater	   r
   r   vllm.distributed.utilsr   vllm.loggerr   %vllm.model_executor.models.interfacesr   rO   r   rH   r   r   r   rebalance_executer   r   r   r.   r   r   rC   rI  rE  r;   r   r?   r@   rA   <module>r_     sK   4     $ $ $ $ $ $ ! ! ! ! ! !      6 6 6 6 6 6 6 6 3 3 3 3 3 3 3 3         
 9 8 8 8 8 8 # # # # # # B B B B B B , , , , , , H H H H H H H H H H          
X		 P P P P P P P Pfx4 x4 x4 x4 x4 x4 x4 x4v 6 6 6 6 6 6 6 6",,"sCx." 	" " " " " "r@   