)`i-0 UdZddlZddlZddlmZmZddlmZddlm Z m Z m Z m Z ddl mZddlmZmZmZddlZddlmZddlmZd d lmZd d lmZmZd d lmZmZm Z GddZ!GddZ"GddZ#GddZ$GddZ%ej&dZ'dZ(dZ)dZ*ed dwde+de+de+d e ed!e e e+f d"Z, dwd#e e e+d e ed!dfd$Z-d%Z.d&Z/ed' dxd)e+de+de+d*e0d e ed+e0d,e ed-e0d!e e e e e+ej1fe e e e+ej1e2fe e e e+ej1e ee2fffd.Z3 dwd#e e e+d e ed!dfd/Z4d0Z5d1e+d2e+d3ej6d!dfd4Z7d5e+d6e+d7e+d2e+d3ej6d!df d8Z8ed9d:ej1d;ej1de+d)e+de!d?e"d@e0dAe+dBej1dCej1dDej1dEe ej1dFe ej1dGe ej1dHe ej1dIe e9dJe ej1dKe ej1dLe ej1dMe ej1d!df.dNZ:dOdPdQdRZ;e2e+e+feedY dwdZej1dUe+d[e+dr>As5J!"!"$%!$%!!!r(r>ceZdZdZdZdZdS)QuantizationSFLayoutrrrN)rrrSWIZZLED_128x4 SWIZZLED_8x4LINEARr'r(r)rFrFQs!NLFFFr(rFc0 ttddgdtdtdtjddffd }td gd d td td tdtdtjddf fd }t dtdgddtjdtjdtdtdtdtdtdtdtdtdtjdtjdtjdttjd ttjd!ttjd"ttjd#ttd$ttjd%ttjd&ttjd'ttjddf.fd( }td)gd*d+tjd,td-tdtd.td/tjdtd0td1td2td3td4ttjd5ttjd6ttjd7ttjd8ttjd9ttjd:ttjd;ttd }td?gd@d,td-tdtd.td/tjdtd5tjd:tjd;td.trtllm_lamport_initializegs# ((T5AAAAAr(z)flashinfer::trtllm_lamport_initialize_all buffer_0_ptr buffer_1_ptr buffer_2_ptrrNrOrVrWrXc:|||||dSrR)trtllm_lamport_initialize_all)rVrWrXrNrOrTs r)rZz=get_trtllm_comm_module..trtllm_lamport_initialize_allos2 ,, , dE     r(trtllm_create_ipc_workspace_for_all_reduce and trtllm_custom_all_reduce are deprecated and will be removed in the next major bump, use allreduce.py instead.z$flashinfer::trtllm_custom_all_reduceinpouttp_sizetp_rank token_numfusion_op_code strategy_code config_codelaunch_with_pdl flag_valuepeer_comm_buffer_ptrspeer_barrier_ptrs_inpeer_barrier_ptrs_outbiasresidualweightweight_pre_residual_normepsintermediate_bufferlamport_peer_comm_buffer_ptrs_0lamport_peer_comm_buffer_ptrs_1lamport_peer_comm_buffer_ptrs_2r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrc\|||||||||| | | | | ||||||||dSrR)trtllm_custom_all_reduce)r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrTs r)rtz8get_trtllm_comm_module..trtllm_custom_all_reduce~sjl ''           ! !    $   + + +-     r(z#flashinfer::trtllm_allreduce_fusion allreduce_in world_size world_rankra hidden_dimworkspace_ptrsre use_oneshottrigger_completion_at_endfp32_acc pattern_code allreduce_out residual_in residual_outnorm_out quant_out scale_out rms_gammarms_eps scale_factor layout_codervrwrxryrzr{r|r}r~rrrrrrrrrrcZ|||||||||| | | | | |||||||dSrR)trtllm_allreduce_fusion)rvrwrxraryrzrer{r|r}r~rrrrrrrrrrrTs r)rz7get_trtllm_comm_module..trtllm_allreduce_fusionsgb &&         %            +     r(z'flashinfer::trtllm_moe_allreduce_fusion)r^r_r`raryrzrerrrr moe_reduction_device_num_expertsmoe_reduction_scale_input(moe_reduction_active_experts_token_inputmoe_reduction_token_inputrrrrrrrrrrmoe_allreduce_outcX|||||||||| | | | | ||||||dSrR)trtllm_moe_allreduce_fusion)rwrxraryrzrerrrrrrrrrrrrrrrTs r)rz;get_trtllm_comm_module..trtllm_moe_allreduce_fusionsd` **           , % 4 %      )     r(z0flashinfer::trtllm_moe_finalize_allreduce_fusion norm_weightexpanded_idx_to_permuted_idx workspaceshared_expert_outputexpert_scale_factorc J |||||||||| | | | dSrR)$trtllm_moe_finalize_allreduce_fusion)rvrrrrrrerrxrwrnrrrTs r)rzDget_trtllm_comm_module..trtllm_moe_finalize_allreduce_fusion]sN& 33    (             r()rSrZrtrrr)rbuild_and_loadrinttorchrOr Tensorr/rr+boolrfloatr>r rFr)rSrZrtrrrrTs @r)get_trtllm_comm_modulercs~ # % % 4 4 6 6F/xjBB"B+0;B BBBBBB 3VVV          {           g.   6/ \/ \/ /  /  / * / -/ -/ / /  %|/ $l/  %|/ u|$/ 5<(/ &!/ "#+5<"8#/ $e_%/ &&el3'/ (*2%,)?)/ **2%,)?+/ ,*2%,)?-/ . // / / / / 7 ";&C!9 7$?-Q    r(r[rankr_ max_token_numgrouprPc||z|zdz}tdzdz}||zdz}|tz|z|zdz}t} |||||||fD]5} t| d} | t | |6t d|dd| Dt| d|| d|| d ||dztj tj | | S) ax Parameters: - rank: the rank of the current process. - tp_size: the size of the process group. - max_token_num: the maximum number of tokens in a sequence. - hidden_dim: the dimension of the hidden states. - group: the process group to use. Note: This function is used to create a workspace for all reduce. The workspace is a list of IPC handles. The workspace should be initialized before calling trtllm_custom_all_reduce. The workspace should be destroyed after calling trtllm_custom_all_reduce. The workspace can be reused for multiple all reduce calls under the same configuration. We would init 7 IPC buffers for trtllm_custom_all_reduce. They are sized as follows: [buffer_size, buffer_size, flag_size, flag_size, lamport_buffer_size, lamport_buffer_size, lamport_buffer_size] where: - buffer_size: tp_size * max_token_num * hidden_dim * sizeof(float) * (maxBeamWidth) - flag_size: (MAX_ALL_REDUCE_BLOCKS + 1) * sizeof(uint32_t) * tp_size * 2 - lamport_buffer_size: tp_size * LamportTokenNumThreshold * tp_size * hidden_dim * sizeof(half) They are for: ipcHandles[0] - peer_comm_buffer_ptrs ipcHandles[2] - peer_barrier_ptrs_in ipcHandles[3] - peer_barrier_ptrs_out ipcHandles[4] - lamport_peer_comm_buffer_ptrs[0:tp_size] ipcHandles[5] - lamport_peer_comm_buffer_ptrs[tp_size:tp_size * 2] ipcHandles[6] - lamport_peer_comm_buffer_ptrs[tp_size * 2:tp_size * 3] We use tp_size and world_size here interchangeably (customAllReduce). Reference: trtllm, cpp/tests/unit_tests/kernels/allReduce/allReduceKernelTest.cu, Workspace init rrr rank  allocated ipc_handles: c&g|]}d|DS)c,g|]}t|Sr'hex.0handles r) zItrtllm_create_ipc_workspace_for_all_reduce...s/R/R/RF /R/R/Rr(r'rsublists r)rz>trtllm_create_ipc_workspace_for_all_reduce..s(.n.n.nW^/R/R'/R/R/R.n.n.nr(rrr) MAX_ALL_REDUCE_BLOCKSLamportTokenNumThresholdlistrappendrprintrZrfloat16distbarrier) rr_rryr buffer_size FLAG_SIZE flag_sizelamport_buffer_size ipc_handlesrN aligned_sizes r)*trtllm_create_ipc_workspace_for_all_reducersN\M)J6:K&*a/IG#a'I!$<