.`iZddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z ddl ZddlZddlmZddlmZddlmZe r/ddlmZmZddlmZdd lmZmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%ne&Z#ee'Z(de)fdZ*Gddej+Z,Gddej+Z-Gdde Z.GddZ/Gdde/Z0dS)N) timedelta) TYPE_CHECKINGAny NamedTupleOptional) deprecated) init_logger)AttentionBackendEnum) PrefixStore ProcessGroup) VllmConfig)ProcessorInputs PromptType) PoolingParams)SamplingParams)FlexibleArgumentParser)AttentionSelectorConfigreturncvddtjvS)N microsoft )joinplatformunamelowerl/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/platforms/interface.pyin_wslr"s- #((8>#3#344::<< <rAs r__le__zDeviceCapability.__le__D8%!122 "! ! DJ'EK+EEErcvt|tstS|j|jf|j|jfkSr=r>rAs r__eq__zDeviceCapability.__eq__IrGrcvt|tstS|j|jf|j|jfkSr=r>rAs r__ge__zDeviceCapability.__ge__NrGrcvt|tstS|j|jf|j|jfkSr=r>rAs r__gt__zDeviceCapability.__gt__SrDrc$|jd|jS)N.)r9r:rBs ras_version_strzDeviceCapability.as_version_strXs*++tz+++rcRd|jcxkrdksnJ|jdz|jzS)z Express device capability as an integer ``. It is assumed that the minor version is always a single digit. r )r:r9rPs rto_intzDeviceCapability.to_int[s> DJ##########zB++rN)r"r#r$int__annotations__rboolrCrFrIrKrMstrrQrTrrrr8r8;s& JJJ JJJECEDEEEE FCFDFFFF FCFDFFFF FCFDFFFF ECEDEEEE ,,,,,,,,,,,,rr8c teZdZUeed<eed<eed<dZeed<dZeed<dZeed <d Z eed <dZ eed <gZ e eed <gZ e eed<dZedzed<edefdZede ejfdZdefdZdefdZdefdZdefdZdefdZdefdZdefdZdedefdZdefdZdefdZ e!defdZ"e!defd Z#e!d!efd"Z$e!drd#Z%e!d$d%d&d'defd(Z&e!de d%fd)Z'e! dsd*ed+ejd,e(d%dd%fd-Z)e! dtd!ede*dzfd/Z+e! dtd0e,eefezd!edefd1Z-e! dtd0e,eefezd!edefd2Z.e! dtd0ed!edefd3Z/e!dtd!edefd4Z0e!dtd!edefd5Z1e!dtd!edefd6Z2e!d7Z3e!e4d8dsd9edzddfd:Z5e!d;ej6ddfd<Z7e! dsd=e8dzddfd>Z9e!dudAZ:e!dBeddfdCZ;e!dDeddfdEZe!defdGZ?e! dsd;ej@jAdzdeBfdHZCe!defdIZDe!d+ejde,eBeBffdJZEe!defdKZFe!defdLZGe!defdMZHe!defdNZIe!defdOZJe!defdPZKe!dejfdQZLe!defdRZMe!defdSZNe!defdTZOe! dvd[ZPd\efd]ZQdefd^ZRe!defd_ZSe!d,ed`dadbedceddeTddef dfZUe!d+ejfdgZVe!defdhZWe!defdiZXe!defdjZYe!dkZZe!de[ee,edlfffdmZ\e!dedzfdnZ]e!doedefdpZ^e!de[eeffdqZ_dS)wPlatform_enum device_name device_typer+ dispatch_keyray_device_key'VLLM_DEVICE_CONTROL_ENV_VAR_PLACEHOLDERdevice_control_env_varinductorsimple_compile_backend dist_backendsupported_quantizationadditional_env_varsN_global_graph_poolrcdS)z3Inductor config key for the PassManager custom passpost_grad_custom_post_passrrPs rpass_keyzPlatform.pass_keys ,+rcHtjtjtjgS)z6Returns the supported dtypes for the current platform.)torchbfloat16float16float32rPs rsupported_dtypeszPlatform.supported_dtypess  u}==rc,|jtjkSr=)r[r!r'rPs ris_cudazPlatform.is_cudaz\...rc,|jtjkSr=)r[r!r(rPs ris_rocmzPlatform.is_rocmrtrc,|jtjkSr=)r[r!r)rPs ris_tpuzPlatform.is_tpuz\---rc,|jtjkSr=)r[r!r*rPs ris_xpuzPlatform.is_xpuryrc,|jtjkSr=)r[r!r+rPs ris_cpuzPlatform.is_cpuryrc,|jtjkSr=)r[r!r,rPs ris_out_of_treezPlatform.is_out_of_treeryrc,|jtjkSr=)r[r!r-rPs ris_unspecifiedzPlatform.is_unspecifiedsz\555r prompt_lenctjSr=)sysmaxsize)rBrs rget_max_output_tokenszPlatform.get_max_output_tokenss {rc@|jtjtjfvS)z1Stateless version of [torch.cuda.is_available][].r[r!r'r(rPs r is_cuda_alikezPlatform.is_cuda_alikeszl/1BCCCrc@|jtjtjfvSr=rrPs ris_sleep_mode_availablez Platform.is_sleep_mode_availables zl/1BCCCrcdS)z Get the pass manager class for this platform. It will be registered as a custom pass under the current_platform.pass_key. z1vllm.compilation.pass_manager.PostGradPassManagerrclss rget_pass_manager_clszPlatform.get_pass_manager_clss CBrc|jS)zF Get the custom compile backend for current platform. )rdrs rget_compile_backendzPlatform.get_compile_backends ))r device_idc|jtjvr\tj|jdkrAtj|jd}||}t |S|S)Nr_,)rbosenvironsplitrU)rr device_idsphysical_device_ids rdevice_id_to_physical_device_idz(Platform.device_id_to_physical_device_idsf  &"* 4 4 356"<<C$>?EEcJJJ!+I!6 )** * rc ddl}n2#t$r%}td|Yd}~nd}~wwxYwt jt5ddl}ddddS#1swxYwYdS)z'Import any platform-specific C kernels.rNz!Failed to import from vllm._C: %r)vllm._C ImportErrorloggerwarning contextlibsuppress vllm._moe_C)rvllmes rimport_kernelszPlatform.import_kernelss C NNNN C C C NN> B B B B B B B B C   - -                        s" 616A$$A(+A(selected_backendr attn_selector_configrcdS)z,Get the attention backend class of a device.r_r)rrrs rget_attn_backend_clszPlatform.get_attn_backend_clss rrctjgSr=)r TORCH_SDPArs rget_supported_vit_attn_backendsz(Platform.get_supported_vit_attn_backendss ! +  r head_sizedtypebackendc|X||vs"Jd|d|td|d|StdtjdtjS)a Get the vision attention backend class of a device. NOTE: ViT Attention should be checked and override in the platform-specific implementation. we should not override this in any other places, like the model_executor/models/.py. We check if the backend is None or not: 1. If not, check if the backend is supported by the platform. 2. If None, continue to the default selection logic. NzBackend z; is not supported for vit attentionSupported backends are: zUsing backend z for vit attentionzUsing default backend )rr info_oncer r)rrrrs rget_vit_attn_backendzPlatform.get_vit_attn_backends$  cAACCCCCS7SS+.+N+N+P+PSSDCC   IgIII J J JN X%9%D X X X   $..rrcdS)z:Stateless version of [torch.cuda.get_device_capability][].Nrrrs rget_device_capabilityzPlatform.get_device_capabilitys tr capabilityc||}|dSt|tr||kS||kS)a. Test whether this platform is compatible with a device capability. The `capability` argument can either be: - A tuple `(major, minor)`. - An integer ``. (See [`DeviceCapability.to_int`][vllm.platforms.interface.DeviceCapability.to_int]) rNFrr?tuplerTrrrcurrent_capabilitys rhas_device_capabilityzPlatform.has_device_capabilityZ!666KK  %5 j% ( ( 4%3 3!((**j88rc||}|dSt|tr||kS||kS)a3 Test whether this platform has exactly the specified device capability. The `capability` argument can either be: - A tuple `(major, minor)`. - An integer ``. (See [`DeviceCapability.to_int`][vllm.platforms.interface.DeviceCapability.to_int]) rNFrrs ris_device_capabilityzPlatform.is_device_capability0rrcr||}|dS|dz|dzkS)z Returns True if the device capability is any .x. Mirrors CUDA 13 'family' architecture semantics (e.g. 10.x, 11.x, 12.x). rNFrS)rrTrs ris_device_capability_familyz$Platform.is_device_capability_familyHsG!666KK  %5"))++r1zR7GHHrct)zGet the name of a device.NotImplementedErrorrs rget_device_namezPlatform.get_device_nameW "!rct)z.Get the uuid of a device, e.g. the PCI bus ID.rrs rget_device_uuidzPlatform.get_device_uuid\rrct)z*Get the total memory of a device in bytes.rrs rget_device_total_memoryz Platform.get_device_total_memoryarrc,tjdS)aA device-specific wrapper of `torch.inference_mode`. This wrapper is recommended because some hardware backends such as TPU do not support `torch.inference_mode`. In such a case, they will fall back to `torch.no_grad` by overriding this method. T)mode)rminference_moders rrzPlatform.inference_modefs#....rz`seed_everything` is deprecated. It will be removed in v0.15.0 or later. Please use `vllm.utils.torch_utils.set_random_seed` instead.seedc|Itj|tj|tj|dSdS)z Set the seed of each random module. `torch.manual_seed` will set seed on all devices. Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 N)randomrnprm manual_seed)rrs rseed_everythingzPlatform.seed_everythingpsM   K    INN4  d # # # # #  rdevicect)z: Set the device for the current platform. rrrs r set_devicezPlatform.set_device "!rparsercdS)a Do some pre-registration or update action for the current platform. This function is called before global VllmConfig is initialized or cli arguments are parsed. It's used for out-of-tree platforms to register or update the configuration. For example, the out-of-tree quantization config can be imported and registered here dynamically. Nr)rrs rpre_register_and_updatez Platform.pre_register_and_updates  r vllm_configr cdS)ac Check and update the configuration for the current platform. It can raise an exception if the configuration is not compatible with the current platform, or it can update the configuration to make it compatible with the current platform. The config is passed by reference, so it can be modified in place. Nr)rrs rcheck_and_update_configz Platform.check_and_update_configs  r model_archcdS)a  Verify whether the current platform supports the specified model architecture. - This will raise an Error or Warning based on the model support on the current platform. - By default all models are considered supported. Nr)rrs rverify_model_archzPlatform.verify_model_archs  rquantc^|jr#||jvrt|d|jddSdS)zW Verify whether the quantization is supported by the current platform. z, quantization is currently not supported in rON)rf ValueErrorr\)rrs rverify_quantizationzPlatform.verify_quantizationsT  % %s7Q*Q*QXXcoXXX   *Q*Qrctj}|dvr tjS|ds|dr tjS|dr tjS|dkr tjS|dr tj S|r tj n tj S)z Determine the CPU architecture of the current system. Returns CpuArchEnum indicating the architecture type. )x86_64amd64i386i686armaarchppcs390xriscv) rmachinerr/r0 startswithr1r2r3r4r5r6)rrs rget_cpu_architecturezPlatform.get_cpu_architectures "$$**,, 9 9 9? "    & & %'*<*tdtdfS)zE Return the platform specific values for (-inf, inf) z-infinf)floatrrs rget_infinity_valueszPlatform.get_infinity_valuess V}}eEll**rcdS)zF Checks if the platform allows inplace memory updates Trrs rcan_update_inplacezPlatform.can_update_inplace trcdS)zK Returns how much padding the LoRA logits need for kernels rrs rget_lora_vocab_padding_sizez$Platform.get_lora_vocab_padding_sizes srcdS)zW Get device specific communicator class for distributed communication. zUvllm.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBaserrs rget_device_communicator_clsz$Platform.get_device_communicator_clss gfrcdS)zI Returns whether the current platform supports MX types. Frrs r supports_mxzPlatform.supports_mx  urcdS)zJ Returns whether the current platform supports FP8 types. Frrs r supports_fp8zPlatform.supports_fp8r rcdS)a Returns whether the preferred FP8 type is FNUZ on the current platform. There are two representations of FP8, OCP FP8 and FNUZ FP8. The OCP specification can be found at https://tinyurl.com/b7jvwpft. The FNUZ specification can be found at https://tinyurl.com/5n6hwwu5. AMD's MI300 and MI325 have native hardware support for FNUZ. All other hardware has converged on the OCP FP8 standard. Frrs r is_fp8_fnuzzPlatform.is_fp8_fnuzs urctjS)z Returns the preferred FP8 type on the current platform. See the documentation for is_fp8_fnuz for details. )rm float8_e4m3fnrs r fp8_dtypezPlatform.fp8_dtype%s ""rcdS)zS Whether to use allgather in LogitsProcessor to gather the logits. Trrs ruse_all_gatherzPlatform.use_all_gather.rrcdS)zR Returns if custom allreduce is supported on the current platform Frrs ruse_custom_allreducezPlatform.use_custom_allreduce5r rcdS)zu Returns True if we register attention as one giant opaque custom op on the current platform Frrs ropaque_attention_opzPlatform.opaque_attention_op<s urpromptrparamsSamplingParams | PoolingParamsprocessed_inputsrcdS)z6Raises if this request is unsupported on this platformNr)rrrrs rvalidate_requestzPlatform.validate_requestDsrkeyctt|jd}|$t||rt||}||Std|j|dS)Nz1Current platform %s does not have '%s' attribute.)getattrrmr]hasattrrr)rBrrattrs r __getattr__zPlatform.__getattr__Msp 0$77  '&#"6"6 63''D  ?      trc^|j}|j||_|jS)zA Return the global graph pool for this platform. ) __class__rhgraph_pool_handle)rBrs rget_global_graph_poolzPlatform.get_global_graph_pool]s1n  ! )%)%;%;%=%=C "%%rcdS)zB Get static graph wrapper class for static graph. z=vllm.compilation.base_static_graph.AbstractStaticGraphWrapperrrs rget_static_graph_wrapper_clsz%Platform.get_static_graph_wrapper_clsfs ONr prefix_storer group_rank group_sizetimeoutr ct)zI Init platform-specific torch distributed process group. r)rrr)r*r+r,s r#stateless_init_device_torch_dist_pgz,Platform.stateless_init_device_torch_dist_pgms "!rct)zJ Check if the dtype is supported by the current platform. rrs rcheck_if_supports_dtypez Platform.check_if_supports_dtype{rrcdS)zV Returns if the hybrid kv cache is supported by the current platform. Frrs rsupport_hybrid_kv_cachez Platform.support_hybrid_kv_cacher rcdS)zQ Returns if the graph mode is supported by the current platform. Frrs rsupport_static_graph_modez"Platform.support_static_graph_moder rcdS)zN Returns if the current platform needs to sync weight loader. Frrs ruse_sync_weight_loaderzPlatform.use_sync_weight_loaderr rc>|sSfd}|S)zD Wrap the original weight loader to make it synced. c|g|Ri|}|jtjdkrtj||S)Ncpu)rrm_sync)paramargskwargsoutoriginal_weight_loaders r_synced_weight_loaderzAPlatform.make_synced_weight_loader.._synced_weight_loadersO((@@@@@@C|u|E2222 E"""Jr)r6)rr?r@s ` rmake_synced_weight_loaderz"Platform.make_synced_weight_loaders@ ))++ *) )      %$r.ciS)zo Returns a mapping from device_type to a tuple of supported kv_buffer_device for nixl. rrs rget_nixl_supported_devicesz#Platform.get_nixl_supported_devicess  rcdS)zH Returns the nixl memory type for the current platform. Nrrs rget_nixl_memory_typezPlatform.get_nixl_memory_typerr max_model_lenc|S)z? Check max_model_len for the current platform. r)rrFs rcheck_max_model_lenzPlatform.check_max_model_lens rciS)zX Set some additional forward context for the current platform if needs. r)rr<r=s rset_additional_forward_contextz'Platform.set_additional_forward_contexts  r)rNr=)r)rr rN)rrrrrrrN)`r"r#r$r!rVrXr^r`rbrdrerflistrgrhrpropertyrkrmrrqrWrsrvrxr{r}rrrUrrr classmethodrrrrrrrrr8rrrrrrrrrrrrrrrrrrr/rrtypesDevicerrrrrrrrr r rrrrrr"r&r(rr.r0r2r4r6rAdictrCrErHrJrrrrZrZes  L# NC #LCKKK#-C,,,L#(*DI***%'c'''%)d ))) ,#,,,X,>$u{"3>>>X>//////////....................66666DtDDDDDDDDDCSCCC[C*C***[*     [ [08 [ 5K0L   [  59 ///{/01 / ///[/< D [99#s(Oc)99  999[9.99#s(Oc)99  999[9. I I I I  I I I[ I"""C"""[""""C"""[""""C"""["//[/Z G $ $3: $ $ $ $ [ $" """"[" 59   +d2      [      [    3  4    [  [E[EEE[E(    [ 15""['$." """[""3"""[" + +eUl8K+++[+ 4[ C[ gCggg[g D[ T[  D   [ #%+###[#t[ T[ D[EE1E, E  EEE[Es &s&&&&OSOOO[O  " "$ " "  "  "  " " "[ ""EK"""[" [ $[ t[  % %[ %4U38_0D+E[S4Z[ [ S#X[rrZc eZdZejZdZdS)UnspecifiedPlatformr_N)r"r#r$r!r-r[r]rrrrRrRs  $EKKKrrR)1rr%rrrrdatetimertypingrrrrnumpyrrmtyping_extensionsr vllm.loggerr #vllm.v1.attention.backends.registryr torch.distributedr r vllm.configr vllm.inputsrrvllm.pooling_paramsrvllm.sampling_paramsrvllm.utils.argparse_utilsrvllm.v1.attention.selectorrobjectr"rrWrEnumr!r/r8rZrRrrrrbsn  ;;;;;;;;;;;; ((((((######DDDDDD $;;;;;;;;&&&&&&77777777111111333333@@@@@@BBBBBBB# X  ===== 49$)',',',',',z',',',T] ] ] ] ] ] ] ] @(r