.`icQ dZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl Z ddl Z ddlmZddlmZddlmZddlmZeeZejdd Zejd efd Zejd efd Zd e de d e fdZde d e dzfdZ!efde de dede ffdZ"e"ddZ#e"ddZ$e"ddZ%e"ddZ&e"ddZ'e"ddZ(e"ddZ)e"dd Z*e"dd!Z+e"dd"Z,e"d#d$d%&Z-ejd efd'Z.ejd efd(Z/ejd efd)Z0ejd efd*Z1ejd efd+Z2ejd efd,Z3ejd efd-Z4ejd efd.Z5ejd efd/Z6d edzfd0Z7d1e8d2e8d efd3Z9 did1e8d2e8d5e8d6e8d7e8d8e d9e j:d:ed;edzdZ;ere j<=d?gd@AdBe j>dCe j>dDe j>dEe j>dFe j>dGe j:dHedIe d e j>fdJZ?e j<@d?dBe j>dCe j>dDe j>dEe j>dFe j>dGe j:dHedIe d e j>fdKZAe j<=dLgd@AdBe j>dCe j>dDe j>dEe j>dGe j:dIe d e j>fdMZBe j<@dLdBe j>dCe j>dDe j>dEe j>dGe j:dIe d e j>fdNZCe j<=dOgd@AdPe j>dQe j>d eDe j>e j>ffdRZEe j<@dOdPe j>dQe j>d eDe j>e j>ffdSZFdPe j>dTe j>dUe j>dVe j>dWe j>dXe j:dIe d e j>fdYZG djdPe j>dTe j>dZe j>d[e j>dXe j:d\e j>dzd e j>fd]ZHdPe j>dQe j>d eDe j>e j>ffd^ZIe"d_d`ZJejd efdaZKejd efdbZLdcedde j:dee j>dfe j>fdgZMgdhZNdS)kzoCompatibility wrapper for FlashInfer API changes. Users of vLLM should always import **only** these wrappers. N)Callable)AnyNoReturn) init_logger)vllm_is_batch_invariant)current_platformFLASHINFER_CUBINS_REPOSITORYzWhttps://edge.urm.nvidia.com/artifactory/sw-kernelinferencelibrary-public-generic-local/returnctjrdStjddSt ddS)z7Return `True` if flashinfer-cubin package is available.Tflashinfer_cubinNz&flashinfer-cubin package was not foundF)envsVLLM_HAS_FLASHINFER_CUBIN importlibutil find_speclogger debug_oncei/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/utils/flashinfer.pyhas_flashinfer_cubinr&sJ %t~ 233?t >??? 5rctjdtddSt s0t jdtddSdS)z8Return `True` if flashinfer-python package is available. flashinferNz2FlashInfer unavailable since package was not foundFnvcczSFlashInfer unavailable since nvcc was not found and not using pre-downloaded cubinsT)rrrrrrshutilwhichrrrhas_flashinferr1s| ~ --5NOOOu ! !fl6&:&:&B 2   u 4r___c td)z/Placeholder for unavailable FlashInfer backend.zFlashInfer backend is not available. Please install the package to enable FlashInfer kernels: https://github.com/flashinfer-ai/flashinfer) RuntimeError)rrs r_missingr"Ds  6  r module_namec\ tj|S#ttf$rYdSwxYw)zBSafely import a submodule and return it, or None if not available.N)r import_module ImportErrorModuleNotFoundError)r#s r_get_submoduler(Ms@&{333 , -tts ++ attr_name fallback_fn.cJtjfdfd}|S)z5Create a lazy import wrapper for a specific function.cltsdSt}|rt|dndSN)rr(getattr)modr)r#s r _get_implz'_lazy_import_wrapper.._get_impl[s? 4[))03=wsIt,,,=rc<}||i|S||i|Sr-r)argskwargsimplr0r*s rwrapperz%_lazy_import_wrapper..wrapperbs=y{{ <;/// /tT$V$$$r) functoolscache)r#r)r*r5r0s``` @r_lazy_import_wrapperr8VsW _>>>>>_> %%%%%% Nrflashinfer.fused_moetrtllm_fp8_block_scale_moetrtllm_fp8_per_tensor_scale_moecutlass_fused_moe$flashinfer.cute_dsl.blockscaled_gemmgrouped_gemm_nt_maskedr fp4_quantizenvfp4_batched_quantize*silu_and_mul_scaled_nvfp4_experts_quantizescaled_fp4_grouped_quantizenvfp4_block_scale_interleavetrtllm_fp4_block_scale_moezflashinfer.autotunerautotunec(tjSr-) contextlib nullcontext)r2r3s rrIs (>(@(@r)r*c`to tjdduS)z5Return `True` if FlashInfer comm module is available.flashinfer.commNrrrrrrrhas_flashinfer_commrMs,    W  8 89J K KSW WWrctsdSgd}|D])\}}t|}|rt||sdS*dS)z7Return `True` if FlashInfer mnnvl all2all is available.F))rKMapping)zflashinfer.comm.mnnvl MnnvlMemory)flashinfer.comm.trtllm_alltoallMnnvlMoe)rQMoEAlltoallInfoT)rMr(hasattrrequired_functionsr#r)r/s rhas_flashinfer_all2allrWsv  u#5 Y[)) '#y11 55  4rc`to tjdduS)z4Return `True` if FlashInfer MoE module is available.r9NrLrrrhas_flashinfer_moerYs1  I N $ $%; < )rrB)r&silu_and_scaled_nvfp4_experts_quantizeT)r[r(rTrUs r-has_flashinfer_cutedsl_grouped_gemm_nt_maskedrdsv " # #u #5 Y[)) '#y11 55  4rcTtrdS tjtd}|jdk}|rt dn t d|j|S#t$r&}t d|Yd}~d Sd}~wwxYw) zReturn `True` if NVIDIA's artifactory is accessible. This checks connectivity to the kernel inference library artifactory which is required for downloading certain cubin kernels like TRTLLM FHMA. T)timeoutz NVIDIA artifactory is accessiblez2NVIDIA artifactory returned failed status code: %dz+Failed to connect to NVIDIA artifactory: %sNF) rrequestsgetr status_coderr warning_once Exception)response accessiblees rhas_nvidia_artifactoryrqst< T>V>Vrc:ddlm}|}|jjS)a, This function should only be called during initialization stage when vllm config is set. Return `None` if --attention-config.use_trtllm_attention is not set, return `True` if TRTLLM attention is forced to be used, return `False` if TRTLLM attention is forced to be not used. r)get_current_vllm_config) vllm.configrwattention_configuse_trtllm_attention)rw vllm_configs rforce_use_trtllm_attentionr|!s0433333))++K  ' <.sQq 01 4rrrr)rrruint8r)rrmnround_up rounded_mscale_n rounded_ns rflashinfer_nvfp4_quantize_fakers w144HQNN r'HWa(( {1a1fEKIII5; y AHL L L   rb block_scale_a block_scale_balpha out_dtypec :|jdkr |jdksJ|jdkr |jdksJ|ddkr|ddksJ|jd|jdksJ|dkr>|tj}|tj}|dkr|jddkrdnd }t |||||||| S) Nrrcutlasstrtllmr TF)rr)ndimstriderviewrrrt)rrrrrrrrs rflashinfer_scaled_fp4_mmrs% 6Q;;16Q;;; &   " "}'9Q'>'>'> > 88B<<1  "!2!2!2 2 71: # # # #)%**5;77 %**5;77 '8 3 3 b8H8He    +    rscale_ascale_bbiasc|jdkr |jdksJ|jd|jdksJ|dkr|dksJ|jtjkr|jtjksJ|jjdkr|jjdksJ|jtjkr|jtjksJ|jjdkr|jjdksJt| d| d|||d |jd|jd}|||z}|S)Nrrrrr) rrnumelrr float8_e4m3fnrtypefloat32r unsqueezer)rrrrrroutputs rflashinfer_scaled_fp8_mmrsa 6Q;;16Q;;; & 71: # # # # ==??a  GMMOOq$8$8$8 8 7e) ) )ag9L.L.L.L L 8=F " "qx}'>'>'> > =EM ) )gmu}.L.L.L L > & ( (W^-@F-J-J-J J  A A  d171:qwqz""  $ Mrc"t||Sr-)r)rrs r$flashinfer_quant_nvfp4_8x4_sf_layoutr9s %Q 4 44rflashinfer.gemmfp8_blockscale_gemm_sm90cto0tjdott ddS)z>Return `True` if FlashInfer block-scale FP8 GEMM is available.Zrr)rris_device_capabilityrTr(rrr"has_flashinfer_fp8_blockscale_gemmrDsC  S  1" 5 5 S N#4557Q R Rrc6tjo tS)z>Return `True` if FlashInfer block-scale FP8 GEMM is supported.)r #VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFERrrrr+is_flashinfer_fp8_blockscale_gemm_supportedrNs 0 1 . 0 0ris_flashinfer_supported output_dtypeinputweightc|sdSd}d}|j}|j}|tjkoG|tjko7|tjko'|jd|zdko|jd|zdk}|S)NF@rr)rrbfloat16rr) rrrr N_MULTIPLE K_MULTIPLE weight_dtype input_dtypeshould_use_flashinfers r-should_use_flashinfer_for_blockscale_fp8_gemmrWs #u JJ ) . E/ / . LOj (A - . LOj (A -  ! r)r%flashinfer_trtllm_fp8_block_scale_moeflashinfer_cutlass_fused_moe)flashinfer_cutedsl_grouped_gemm_nt_maskedflashinfer_fp4_quantizerArBrCrDrErYrMrWrardrrqrurrzrrrflashinfer_fp8_blockscale_gemmrr)NFFr-)O__doc__rGr6rimportlib.utilosrcollections.abcrtypingrrrir vllm.envsr vllm.loggerr*vllm.model_executor.layers.batch_invariantrvllm.platformsr__name__renvironrjr r7boolrrr"strr(r8r*flashinfer_trtllm_fp8_per_tensor_scale_moerrrr@rArBrCrDrErMrWrYr[r_rardrqrur|intrrrzlibrary custom_opTensorr register_fakerrrtuplerrrrrrrrr__all__rrrrs   $$$$$$  ######,+++++ X   "z~~"]    d $CHd IQ!$3;CH3E,)=(<8))%.B-A=..* 43/  -A,@*,D--)/.|^LL--l..*32/ 430  21. @@    XTXXXX  ( D         $( t& 6  4     =D4K = = = ==3=c=d====$%)QQQQQ Q  Q  Q[QQTkQQQ QQQQh>m  ]!  < <      {           0 ]  ! Q < Q < Q Q Q  Q { Q  Q Q  Q Q Q Q ] F < F < F F F { F  F  F F F  F ]    <  <      {            ])  < &+l u|U\) *      ]  )  <  &+l  u|U\) *       | |<<  <  {  \L!%  | |\\  {  ,   \<5 |5"',5 5< %&5555 "6!51""  D T!!!+! <! L !!!!:   r