
    .`ik_                        U d dl Z d dlmZmZmZ d dlmZmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ ddlmZmZmZ erd d	lmZ d d
lmZ  ee          Z	 d dlmZmZmZmZm Z m!Z! n(# e"$ r Z#e$                    de#           Y dZ#[#ndZ#[#ww xY w	 d dl%Z&n(# e"$ r Z#e$                    de#           Y dZ#[#ndZ#[#ww xY w	 d dl'Z&n(# e"$ r Z#e$                    de#           Y dZ#[#ndZ#[#ww xY wg Z(e)e*         e+d<   i Z,e-e*e*f         e+d<   dddddddddd	Z.e-e*e*f         e+d<   de j/        v r=e j/        d         Z0e j/        1                    dd          xZ2r	e0e2k    sJ n
e0e j/        d<   d Z3ede4fd            Z5ede4fd            Z6ede4fd            Z7ede4fd             Z8ede4fd!            Z9e	 	 d/d"ej:        d#e;d$e;d%e;d&e;d'e;d(e*d)ej<        dz  d*ej<        dz  de4fd+            Z=ede4fd,            Z> G d- d.e          Z?dS )0    N)cache	lru_cachewraps)TYPE_CHECKINGOptional)init_loggercuda_device_count_stateless)AttentionBackendEnum   )DeviceCapabilityPlatformPlatformEnum)
VllmConfig)AttentionSelectorConfig)AmdSmiExceptionamdsmi_get_gpu_asic_infoamdsmi_get_processor_handlesamdsmi_initamdsmi_shut_downamdsmi_topo_get_link_typez$Failed to import from amdsmi with %rz%Failed to import from vllm._C with %rz*Failed to import from vllm._rocm_C with %r_ROCM_UNSUPPORTED_MODELS _ROCM_PARTIALLY_SUPPORTED_MODELSAMD_Instinct_MI300AAMD_Instinct_MI300XAMD_Instinct_MI308XAMD_Instinct_MI325XAMD_Instinct_MI300X_HFAMD_Radeon_RX7900XTX)	0x74a00x74a10x74b50x74a20x74a50x74b90x74a90x74bd0x744c_ROCM_DEVICE_ID_NAME_MAPHIP_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESc                 <     t                      fd            }|S )Nc                  t    t                       	  | i |t                       S # t                       w xY wN)r   r   )argskwargsfns     g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/platforms/rocm.pywrapperz$with_amdsmi_context.<locals>.wrapperO   sE    	2t&v&&s   ' 7)r   )r1   r3   s   ` r2   with_amdsmi_contextr4   N   s3    
2YY    Y N    returnc                       t           j                            d          j         t	           fddD                       S )Ncudac              3       K   | ]}|v V  	d S r.    .0archGPU_ARCHs     r2   	<genexpr>zon_gfx1x.<locals>.<genexpr>]   s(      ??Dtx??????r5   gfx11gfx12torchr8   get_device_propertiesgcnArchNameanyr>   s   @r2   on_gfx1xrI   Z   s?    z//77CH????,>??????r5   c                       t           j                            d          j         t	           fddD                       S )Nr8   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   zon_mi3xx.<locals>.<genexpr>c   s(      AADtxAAAAAAr5   )gfx942gfx950rC   rH   s   @r2   on_mi3xxrN   `   s?    z//77CHAAAA,@AAAAAAr5   c                       t           j                            d          j         t	           fddD                       S )Nr8   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   zon_gfx9.<locals>.<genexpr>i   s(      KKDtxKKKKKKr5   gfx90arL   rM   rC   rH   s   @r2   on_gfx9rS   f   s?    z//77CHKKKK,JKKKKKKr5   c                       t           j                            d          j         t	           fddD                       S )Nr8   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   zon_gfx942.<locals>.<genexpr>o   (      77Dtx777777r5   )rL   rC   rH   s   @r2   	on_gfx942rW   l   >    z//77CH7777J777777r5   c                       t           j                            d          j         t	           fddD                       S )Nr8   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   zon_gfx950.<locals>.<genexpr>u   rV   r5   )rM   rC   rH   s   @r2   	on_gfx950r[   r   rX   r5   qtype	head_size
block_size	gqa_ratiomax_seq_lensliding_windowkv_cache_dtypealibi_slopessinksc	                 T   t           j                            d          j        t	          fddD                       }	t	          fddD                       }
|	rf|dk    s|dk    oY| t           j        k    p| t           j        k    o9|dk    p|d	k    o-|d
k    p|dk    o!|dk    o|d
k    o|dk    ot          j        o|d u S |
oc|dk    p|dk    oW| t           j        k    p| t           j        k    o7|d	k    o1|d
k    o+|dk    o|d
k    o|dk    o|d u o|dk    ot          j        o|d u S )Nr8   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   z2use_rocm_custom_paged_attention.<locals>.<genexpr>   s(      NNt$("NNNNNNr5   rQ   c              3       K   | ]}|v V  	d S r.   r:   r;   s     r2   r?   z2use_rocm_custom_paged_attention.<locals>.<genexpr>   s(      IId)IIIIIIr5   r@   r   )rh   @             r   i      auto)	rD   r8   rE   rF   rG   halfbfloat16envsVLLM_ROCM_CUSTOM_PAGED_ATTN)r\   r]   r^   r_   r`   ra   rb   rc   rd   ON_GFX9ON_GFX11_GFX12r>   s              @r2   use_rocm_custom_paged_attentionru   x   s    z//77CHNNNN/MNNNNNGIIII6HIIIIIN  
q >Nh$> %*$?(?b4I$4 r!5Z2%5 a3IO	
 z) 1 		
  
1$B((B
%*$?(?
 S 
 b 	

 a3IO
 z)
 $
 &(
 0
 	
r5   c                     t                      sdS 	 ddlm}   | d          dS  | d          dS t          j                            d          dk    rt                              d           dS d	S # t          $ r Y dS w xY w)
NFr   	find_spec
flash_attnz flash_attn.flash_attn_triton_amd!FLASH_ATTENTION_TRITON_AMD_ENABLETRUEz\Set FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE to enable Flash Attention Triton backend on RDNA.T)	rI   importlib.utilrx   osenvirongetlogger	info_onceImportErrorrw   s    r2   flash_attn_triton_availabler      s    :: u,,,,,,9\""*59788@5:>>=>>&HH:   5t   uus   A3 A3 =A3 3
B Bc            
           e Zd ZU ej        ZdZeed<   dZ	eed<   dZ
eed<   dZeed<   d	Zeed
<   dZeed<   g dZee         ed<    e            sedgz  Zed> fd            Zedddddefd            Zeded         fd            Ze	 d?dedej        ded         ddfd            Zedej        ddfd            Ze ed           d@d"ededz  fd#                        Zeed$ee         de fd%                        Z!ee ed           d@d"edefd&                                    Z"ed@d"edefd'            Z#edAd*            Z$ed+eddfd,            Z%ed-eddf fd.            Z&edefd/            Z'e	 d?dej(        j)        dz  de*fd0            Z+edefd1            Z,ede fd2            Z-ede fd3            Z.ede fd4            Z/edej        fd5            Z0ede fd6            Z1ede fd7            Z2ede fd8            Z3edefd9            Z4edefd:            Z5edej        fd;            Z6ede fd<            Z7ede fd=            Z8 xZ9S )BRocmPlatformrocmdevice_namer8   device_typeCUDAdispatch_keyGPUray_device_keynccldist_backendr+   device_control_env_var)awq
awq_marlingptqgptq_marlinfp8zcompressed-tensors
fbgemm_fp8ggufquarkptpc_fp8mxfp4petit_nvfp4torchaosupported_quantizationbitsandbytesr6   Nc                     t                                                       ddl}|                    t                    5  ddl}ddd           dS # 1 swxY w Y   dS )zImport ROCm-specific kernels.r   N)superimport_kernels
contextlibsuppressr   vllm._rocm_C)clsr   vllm	__class__s      r2   r   zRocmPlatform.import_kernels   s     	      -- 	  	 	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	 s    AAAselected_backendr   attn_selector_configr   c                 8
   ddl m} |j        }|j        }|j        rn|r$|                    d          rt          d          |dk    s
J d            t                              d           t          j
                                        S |j        rE|2|                                s|dk    rt          j        nt          j        }|t          j        k    rY|dk    r8t                              d           t          j                                        S t          d	|j         d
| d          |t          j        k    r8t                              d           t          j                                        S |t          j        k    r8t                              d           t          j                                        S t          d	|j         d          |t          j        k    r8t                              d           t          j                                        S |t          j        k    r8t                              d           t          j                                        S |t          j        k    r8t                              d           t          j                                        S |t          j        k    r^t/                      r8t                              d           t          j                                        S t          d|j         d          |t          j        k    r8t                              d           t          j                                        S |t2          j        rDt2          j        r8t                              d           t          j                                        S t2          j        rRt2          j        rFt/                      r8t                              d           t          j                                        S ddlm}  |            }|D|j        j         r8t                              d           t          j                                        S t2          j        rTt/                      rFt2          j        dur8t                              d           t          j                                        S t                              d           t          j                                        S tC          d|j         d          )Nr   rocm_aiter_opsr   z=ROCMAiterMLASparseBackend doesn't support fp8 kv_cache_dtype.r   z>Sparse MLA backend on ROCm only supports block size 1 for now.zUsing Sparse MLA backend.zUsing Triton MLA backend.z The selected backend, z,does not support block size .zUsing AITER MLA backend.zUsing AITER TRITON MLA backend.z1,is not MLA type while requested for MLA backend.zUsing FlexAttention backend.zUsing Triton Attention backend.zUsing Rocm Attention backend.z$Using Aiter Flash Attention backend.zThe selected backend, z*, is only supported on gfx9 architectures.z&Using Aiter Unified Attention backend.)get_current_vllm_config_or_noneFzAttention backend zM is not supported on ROCm. Note that V0 attention backends have been removed.)"vllm._aiter_opsr   r^   rb   
use_sparse
startswith
ValueErrorr   r   r   ROCM_AITER_MLA_SPARSEget_pathuse_mlais_mla_enabledROCM_AITER_MLA
TRITON_MLAnameinfoROCM_AITER_TRITON_MLAFLEX_ATTENTIONTRITON_ATTN	ROCM_ATTNROCM_AITER_FArS   ROCM_AITER_UNIFIED_ATTNrq   VLLM_ROCM_USE_AITER%VLLM_ROCM_USE_AITER_UNIFIED_ATTENTIONVLLM_ROCM_USE_AITER_MHAvllm.configr   attention_configuse_prefill_decode_attentionRuntimeError)r   r   r   r   r^   rb   r   vllm_configs           r2   get_attn_backend_clsz!RocmPlatform.get_attn_backend_cls   s    	322222)4
-<* 		I .";";E"B"B  S   ???P #?? 8999'=FFHHH' 	' &44669:D// )77-8 !
  #7#BBB??$$%@AAA/:CCEEE A.>.C A A3=A A A    #7#FFF6777+:CCEEE#7#MMM=>>>+AJJLLLD*:*? D D D  
 3BBBKK6777'6??AAA3???KK9:::'3<<>>>3===KK7888'1::<<<3AAAyy BCCC+9BBDDD ?-=-B ? ? ?  
 3KKKKK@AAA'?HHJJJ #' OD,V ODEEE+CLLNNN ' ED,H EWYY EBCCC+9BBDDD DCCCCC99;;K'0M ( ;<<<+5>>@@@
 (EIIE 0==BCCC+9BBDDD KK9:::'3<<>>>G!1!6 G G G
 
 	
r5   c                 H    t           j        t           j        t           j        gS r.   )r   
FLASH_ATTNr   
TORCH_SDPAr   s    r2   get_supported_vit_attn_backendsz,RocmPlatform.get_supported_vit_attn_backendsZ  s      !+ . +
 	
r5   r]   dtypebackendc                 ,   |X||                                  v s"J d| d|                                               t                              d| d           |S ddlm} ddlm} |                                r4t                      r&t                              d           t          j
        S t                      rQ |d	          F|t          j        k    s|t          j        k    r&t                              d
           t          j        S t                      rTt!                      rF|t          j        k    s|t          j        k    r&t                              d           t          j        S t                              d           t          j        S )NzBackend z= is not supported for vit attention. Supported backends are: zUsing backend z for vit attentionr   rw   r   z2Using AITER Flash Attention backend for ViT model.ry   z,Using Flash Attention backend for ViT model.z=Using Flash Attention (Triton backend) for ViT model on RDNA.z'Using Torch SDPA backend for ViT model.)r   r   r   r|   rx   r   r   
is_enabledrS   r   r   rD   float16rp   r   rI   r   r   )r   r]   r   r   rx   r   s         r2   get_vit_attn_backendz!RocmPlatform.get_vit_attn_backendb  s    cAACCCCCS7 S S+.+N+N+P+PS S DCC IgIIIJJJN,,,,,,222222$$&& 	6799 	6QRRR'55 II	3	,''3%-''5EN+B+BKLLL'22 JJ	3+--	3 %-''5EN+B+BO   (22BCCC#..r5   devicec                 D    t           j                            |           dS )z:
        Set the device for the current platform.
        N)rD   r8   
set_devicer   r   s     r2   r   zRocmPlatform.set_device  s     
 	
f%%%%%r5      )maxsizer   	device_idc                 h    t           j                            |          \  }}t          ||          S )N)majorminor)rD   r8   get_device_capabilityr   )r   r   r   r   s       r2   r   z"RocmPlatform.get_device_capability  s0     z77	BBue59999r5   physical_device_idsc                 F   d |D             }t          |          D ]\  }}t          |          D ]o\  }}||k     rd	 t          ||          }|d         dk    s|d         dk    r  dS 9# t          $ r)}t                              d|           Y d	}~  dS d	}~ww xY wpd
S )zN
        Query if the set of gpus are fully connected by xgmi (1 hop)
        c                 6    g | ]}t                      |         S r:   )r   )r<   is     r2   
<listcomp>z3RocmPlatform.is_fully_connected.<locals>.<listcomp>  s$    RRR/11!4RRRr5   hopsr   type   Fz AMD 1 hop XGMI detection failed.)exc_infoNT)	enumerater   r   r   error)	r   r   handlesr   handlejpeer_handle	link_typer   s	            r2   is_fully_connectedzRocmPlatform.is_fully_connected  s     SR>QRRR"7++ 
	% 
	%IAv"+G"4"4 	% 	%;q55%$=fk$R$R	$V,11Yv5F!5K5K#(555 6L* % % %%GRWXXX$uuuuuuu% 	% ts   (A))
B3BBc                     |                      |          }t                      |         }t          |          }|d         }|t          v rt          |         S |d         S )Nr   market_name)device_id_to_physical_device_idr   r   r)   )r   r   physical_device_idr   	asic_infor   s         r2   get_device_namezRocmPlatform.get_device_name  sb     !@@KK-//0BC,V44	$[1222+K88''r5   c                 N    t           j                            |          }|j        S r.   )rD   r8   rE   total_memory)r   r   device_propss      r2   get_device_total_memoryz$RocmPlatform.get_device_total_memory  s     z77	BB((r5   r   r   c                 T   ddl m} ddlm} |j        }|j        }|j        }||j        k    }|                                }|	                                }	|
                                }
|                                }|j                                        rc|j        dk    r't                              d           |j        |_        n1|j        dk    r&t                              d           |j        |_        |rH|j        At(          j        r.t(          j        r"d|_        t                              d           nd	|_        |j        d
k    rd|_        |	r%|s#d|j        vr|j                            d           |
r#d|j        vr|j                            d           |r=d|j        v r4t                              d           |j                            d           |r,d|j        vr#d|j        vr|j                            d           |j                            d           d S )Nr   r   )CUDAGraphModer   z~Decode context parallel (DCP) is enabled, which is incompatible with full CUDA graphs. Overriding cudagraph_mode to PIECEWISE.zPrefill context parallel (PCP) is enabled, which is incompatible with full CUDA graphs. Overriding cudagraph_mode to PIECEWISE.ri   z=[ROCM_AITER_UNIFIED_ATTN]: Setting kv cache block size to 64.rk   rn   z vllm.v1.worker.gpu_worker.Workerz	-rms_normz	+rms_normz
-quant_fp8z
+quant_fp8z-grouped_topkzVLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled, which requires the 'grouped_topk' custom op. Overriding the user-provided '-grouped_topk'.z+grouped_topkz+sparse_attn_indexer)r   r   vllm.config.compilationr   cache_configcompilation_configparallel_configNONEis_fused_moe_enabledis_rmsnorm_enabledis_linear_fp8_enabled$is_fusion_moe_shared_experts_enabledcudagraph_modehas_full_cudagraphsdecode_context_parallel_sizer   warning_once	PIECEWISEprefill_context_parallel_sizer^   rq   r   r   warning
worker_cls
custom_opsappendremove)r   r   r   r   r   r   r   is_eager_executionuse_aiter_fused_moeuse_aiter_rms_normuse_aiter_fp8_linearuse_aiter_fused_ses               r2   check_and_update_configz$RocmPlatform.check_and_update_config  s   222222999999"/(;%5/=3EE,AACC+>>@@-CCEE+PPRR,@@BB 	L;a??##>  
 5B4K"11 >BB##>  
 5B4K"1 	-L3;:-?C?W- +-'S    +-'%//)KO& 	>&	> #5#@@@)00=== 	?L8J8U$U$U)00>>> 	B/5G5R"R"R1  
 )00AAA  	B'9'DDD'9'DDD)00AAA 	%,,-CDDDDDr5   
model_archc                     |t           v rt          d| d          |t          v r+t          |         }t                              d||           d S d S )NzModel architecture 'z#' is not supported by ROCm for now.z:Model architecture '%s' is partially supported by ROCm: %s)r   r   r   r   r  )r   r  msgs      r2   verify_model_archzRocmPlatform.verify_model_arch  sw    111VzVVV   9992:>CNNL     :9r5   quantc                     t                                          |           |dk    r&t          j        st                              d           dt          j        d<   d S )Nr   zcUsing AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ is not set, enabling VLLM_USE_TRITON_AWQ.1VLLM_USE_TRITON_AWQ)r   verify_quantizationrq   r  r   r  r}   r~   )r   r  r   s     r2   r  z RocmPlatform.verify_quantization!  s[    ##E***E>>$":>NN=   -0
()))r5   c                     dS )Nz4vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPUr:   r   s    r2   get_punica_wrapperzRocmPlatform.get_punica_wrapper+  s    EEr5   c                     t           j                            |           t           j                            |          d         t           j                            |          d         z
  S )Nr   r   )rD   r8   reset_peak_memory_statsmem_get_infor   s     r2   get_current_memory_usagez%RocmPlatform.get_current_memory_usage/  sP     	
**6222z&&v..q1EJ4K4KF4S4STU4VVVr5   c                     	 dS )NzHvllm.distributed.device_communicators.cuda_communicator.CudaCommunicatorr:   r   s    r2   get_device_communicator_clsz(RocmPlatform.get_device_communicator_cls6  s     W	
 	
r5   c                     t           j                            d          j        t	          fddD                       S )Nr   c              3       K   | ]}|v V  	d S r.   r:   r<   gfxgcn_archs     r2   r?   z+RocmPlatform.supports_mx.<locals>.<genexpr>?  s'      88s3(?888888r5   )gfx95rC   r   r+  s    @r2   supports_mxzRocmPlatform.supports_mx<  s>    :33A66B8888i888888r5   c                     t           j                            d          j        t	          fddD                       S )Nr   c              3       K   | ]}|v V  	d S r.   r:   r)  s     r2   r?   z,RocmPlatform.supports_fp8.<locals>.<genexpr>D  s'      JJs3(?JJJJJJr5   )gfx94r,  rB   rC   r-  s    @r2   supports_fp8zRocmPlatform.supports_fp8A  s?    :33A66BJJJJ.IJJJJJJr5   c                 N    dt           j                            d          j        v S )Nr1  r   rD   r8   rE   rF   r   s    r2   is_fp8_fnuzzRocmPlatform.is_fp8_fnuzF  s"     %*::1==IIIr5   c                 Z    |                                  rt          j        S t          j        S r.   )r5  rD   float8_e4m3fnuzfloat8_e4m3fnr   s    r2   	fp8_dtypezRocmPlatform.fp8_dtypeK  s&    ?? 	'((&&r5   c                     t           j                            d          j        ddg}t	          fd|D                       S )Nr   r1  r,  c              3       K   | ]}|v V  	d S r.   r:   r)  s     r2   r?   z4RocmPlatform.use_custom_allreduce.<locals>.<genexpr>W  s'      >>s3(?>>>>>>r5   rC   )r   supported_archsr+  s     @r2   use_custom_allreducez!RocmPlatform.use_custom_allreduceR  sJ     :33A66B"G,>>>>o>>>>>>r5   c                     dS NTr:   r   s    r2   opaque_attention_opz RocmPlatform.opaque_attention_opY      tr5   c                 N    dt           j                            d          j        v S )Ngfx1r   r4  r   s    r2   is_navizRocmPlatform.is_navi]  s     99!<<HHHr5   c                     dS )Nz,vllm.compilation.cuda_graph.CUDAGraphWrapperr:   r   s    r2   get_static_graph_wrapper_clsz)RocmPlatform.get_static_graph_wrapper_clsa  s    ==r5   c                     t                      S r.   r	   r   s    r2   device_countzRocmPlatform.device_counte  s    *,,,r5   c                    |t           j        k    rq|                     d          s^|                                 }|                                 }|d}n|                                }d| }t          d| d| d          d S d S )NP   z"does not have a compute capabilityzhas compute capability zQBfloat16 is only supported on GPUs with compute capability of at least 8.0. Your z GPU zg. You can use float16 instead by explicitly setting the `dtype` flag in CLI, for example: --dtype=half.)rD   rp   has_device_capabilityr   r   as_version_strr   )r   r   
capabilitygpu_namecompute_strversion_strs         r2   check_if_supports_dtypez$RocmPlatform.check_if_supports_dtypei  s    EN"",,R00  6688
..00%"FKK",";";"="=K"IK"I"IK F$F F+6F F F   #" r5   c                     dS r?  r:   r   s    r2   support_hybrid_kv_cachez$RocmPlatform.support_hybrid_kv_cache~  rA  r5   c                     dS r?  r:   r   s    r2   support_static_graph_modez&RocmPlatform.support_static_graph_mode  rA  r5   )r6   Nr.   )r   )r   r   r6   N):__name__
__module____qualname__r   ROCM_enumr   str__annotations__r   r   r   r   r   r   listrS   classmethodr   r   r   intrD   r   r   r   r   r   r   r   r   r4   boolr   r   r   r  r  r  r   typesDevicefloatr$  r&  r.  r2  r5  r9  r=  r@  rD  rF  rH  rQ  rS  rU  __classcell__)r   s   @r2   r   r      s        EKKL#NCL#"8C888) ) )DI     799 3>"22          [  s
0s
 8s
 
	s
 s
 s
 [s
j 
5K0L 
 
 
 [
 
 59	*/ */*/ {*/ 01	*/
 
 */ */ */ [*/X & & & & & [& Yq: :c ::JT:Q : : :  [: T#Y 4     [$ Yq( ( (C ( ( (   [( ) ) )C ) ) ) [) NE NE NE [NE` 3 4    [ 0 0 0 0 0 0 0 [0 F3 F F F [F 15W W['$.W	W W W [W 
C 
 
 
 [

 9D 9 9 9 [9 KT K K K [K JD J J J [J '%+ ' ' ' [' ?T ? ? ? [? D    [ I I I I [I >S > > > [> -S - - - [- EK    [(     [ $    [    r5   r   )NN)@r}   	functoolsr   r   r   typingr   r   rD   	vllm.envsrq   vllm.loggerr   vllm.utils.torch_utilsr
   #vllm.v1.attention.backends.registryr   	interfacer   r   r   r   r   vllm.v1.attention.selectorr   rV  r   amdsmir   r   r   r   r   r   r   er  vllm._Cr   r   r   r]  r[  r\  r   dictr)   r~   valr   cuda_valr4   r`  rI   rN   rS   rW   r[   r   r_  Tensorru   r   r   r:   r5   r2   <module>rt     s   
				 - - - - - - - - - - * * * * * * * *        # # # # # # > > > > > > D D D D D D ? ? ? ? ? ? ? ? ? ? C&&&&&&BBBBBB	X		
>                  > > >
NN91========>?NNNN ? ? ?
NN:A>>>>>>>>?D D D D
NN?CCCCCCCCD ') $s) ( ( ( 46  $sCx. 5 5 5######&&$
, 
, $sCx. 
 
 
 BJ&&
**
+C:>>"8$???x 1h-0
)*	 	 	 @$ @ @ @ @
 B$ B B B B
 L L L L L
 84 8 8 8 8
 84 8 8 8 8
  )-!%*
 *
;*
*
 *
 	*

 *
 *
 *
 ,%*
 <$*
 
*
 *
 *
 *
Z T    *H H H H H8 H H H H HsH   A) )B.B		BB B<B77B< C C*
C%%C*