
    PiO              	          U d dl Z d dlmZmZmZmZmZmZ d dlZd dl	m
Z
 d dlmZ  e
            Zej        ej        ej        ej        dZeeej        f         ed<   dded	dfd
Zd	efdZ	 ddee         deej                 d	ej        fdZe j        dej        d	ed         fd            Z	 ddeeeej        j        f                  dej        deee                  d	dfdZdS )    N)Dict	GeneratorIterableListOptionalTuple)
get_logger)is_npu_available)fp16bf16fp32fp64PRECISION_STR_TO_DTYPEhigh	precisionreturnc                     t           j                                        s	t          sdS t          j        |            | dk    rdt           j        j        _        dS dt           j        j        _        dS )a  Sets the precision of float32 matrix multiplications and convolution operations.

    For more information, see the PyTorch docs:
    - https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html
    - https://pytorch.org/docs/stable/backends.html#torch.backends.cudnn.allow_tf32

    Args:
        precision (str): The setting to determine which datatypes to use for matrix multiplication and convolution operations.
    NhighestFT)torchcudais_availabler
   set_float32_matmul_precisionbackendscudnn
allow_tf32)r   s    p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/training/precision.py_set_float32_precisionr      si     J##%% )9 	&y111I*/'''*.'''    c                  b   t           j                                        obt           j                                        oDt           j                                        o&t           j        j                                        dk    } t           j        j	                                        o"t           j        j	        
                                }t          ot           j                                        }t           j                                        ot           j                                        }| p|p|p|S )a  
    Check that bf16 is available on this hardware. Requirements:
        - CUDA is available and supports bf16
            - CUDA version >= 11
            - CUDA compute capability >= 8
        - NCCL is available and version >= 2.10
        - MPS is available and torch was built with MPS
        - NPU is available and supports bf16
        - XPU is available and supports bf16

    Returns:
        bool: True if bf16 is available, False otherwise.

    )   
   )r   r   r   is_bf16_supporteddistributedis_nccl_availablencclversionr   mpsis_builtr
   npuxpu)cuda_supportmps_supportnpu_supportxpu_supports       r   verify_bf16_supportr/   0   s      	
!! 	1J((**	1//11	1 JO##%%0	  .$1133U8J8S8S8U8UK"Duy'B'B'D'DK)((**Luy/J/J/L/LKD;D+DDr   dtypedevicec                    | t           j        S t                              | |           }|t                                          vrMt          d| dd                    t          t                                                               d          |t           j	        k    r5|t          j
        d          k    rt                      st          d          |S )a  Get the torch.dtype corresponding to the given precision string. If no string is passed,
    we will default to torch.float32.

    Note:
        If bf16 precision is requested with a CUDA device, we verify whether the device indeed supports
        bf16 kernels. If not, a ``RuntimeError`` is raised.

    Args:
        dtype (Optional[str]): The precision dtype. Default: ``None``, in which we default to torch.float32
        device (Optional[torch.device]): Device in use for training. Only CUDA and CPU
            devices are supported. If a CUDA device is passed in, additional checking is done
            to ensure that the device supports the requested precision. Default: ``None``, in which case
            a CUDA device is assumed.
    Raises:
        ValueError: if precision isn't supported by the library
        RuntimeError: if bf16 precision is requested but not available on this hardware.

    Returns:
        torch.dtype: The corresponding torch.dtype.

    NzDtype z must be one of z, z for finetuning.cpuzcbf16 precision was requested but not available on this hardware. Please use fp32 precision instead.)r   float32r   getvalues
ValueErrorjoinlistkeysbfloat16r1   r/   RuntimeError)r0   r1   torch_dtypes      r   	get_dtyper>   K   s    4 }} ),,UE::K 0779999r[rr$))DAWA\A\A^A^<_<_2`2`rrr
 
 	

 	u~%%el5))))#%% * q
 
 	
 r   )NNNc              #      K   t          j                    }t          j        |            	 dV  t          j        |           dS # t          j        |           w xY w)a  
    Context manager to set torch's default dtype.

    Args:
        dtype (torch.dtype): The desired default dtype inside the context manager.

    Returns:
        ContextManager: context manager for setting default dtype.

    Example:
        >>> with set_default_dtype(torch.bfloat16):
        >>>     x = torch.tensor([1, 2, 3])
        >>>     x.dtype
        torch.bfloat16


    N)r   get_default_dtypeset_default_dtype)r0   	old_dtypes     r   rA   rA   }   sc      & '))I	E"""+	*****	****s   A Anamed_paramsexclude_param_namesc           	          | D ]K\  }|t          fd|D                       r#|j        |k    rt          d d|j         d|           LdS )a  
    Validates that all input parameters have the expected dtype.

    Args:
        named_params (Iterable[Tuple[str, torch.nn.Parameter]]): Iterable of named parameters.
        dtype (torch.dtype): Expected dtype.
        exclude_param_names (Optional[List[str]]): Optional list of parameter names to exclude from dtype checking

    Raises:
        ValueError: If any parameter has a different dtype than `dtype`.
    Nc              3       K   | ]}|v V  	d S N ).0nnames     r   	<genexpr>z0validate_expected_param_dtype.<locals>.<genexpr>   s'      ::19::::::r   z
Parameter z has dtype z, but expected )anyr0   r7   )rC   r0   rD   paramrK   s       @r   validate_expected_param_dtyperO      s      $  e*::::&9::::: ;%QTQQekQQ%QQ    	 r   )r   )NNrG   ) 
contextlibtypingr   r   r   r   r   r   r   torchtune.utilsr	   torchtune.utils._devicer
   logfloat16r;   r4   float64r   strr0   __annotations__r   boolr/   r1   r>   contextmanagerrA   nn	ParameterrO   rH   r   r   <module>r]      s        C C C C C C C C C C C C C C C C  & & & & & & 4 4 4 4 4 4jll MNMM	2 2 S%+-.   / /c /t / / / /,ET E E E E8 CG/ /C=/)1%,)?/
[/ / / /d +U[ +Y7G-H + + + +: 04 5eh&8!89:; "$s), 
	     r   