o
    ̳iO                  	   @   s  U d dl Z d dlmZmZmZmZmZmZ d dlZd dl	m
Z
 d dlmZ e
 ZejejejejdZeeejf ed< dded	dfd
dZd	efddZ	ddee deej d	ejfddZe jdejd	ed fddZ	ddeeeejjf  dejdeee  d	dfddZdS )    N)Dict	GeneratorIterableListOptionalTuple)
get_logger)is_npu_available)fp16bf16fp32fp64PRECISION_STR_TO_DTYPEhigh	precisionreturnc                 C   s@   t j s	ts	dS t |  | dkrdt jj_dS dt jj_dS )a  Sets the precision of float32 matrix multiplications and convolution operations.

    For more information, see the PyTorch docs:
    - https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html
    - https://pytorch.org/docs/stable/backends.html#torch.backends.cudnn.allow_tf32

    Args:
        precision (str): The setting to determine which datatypes to use for matrix multiplication and convolution operations.
    NhighestFT)torchcudais_availabler	   set_float32_matmul_precisionbackendscudnn
allow_tf32)r    r   P/home/ubuntu/.local/lib/python3.10/site-packages/torchtune/training/precision.py_set_float32_precision   s   
r   c                  C   sx   t j ot j ot j ot jj dk} t jj	 o"t jj	
 }to)t j }t j o3t j }| p;|p;|p;|S )a  
    Check that bf16 is available on this hardware. Requirements:
        - CUDA is available and supports bf16
            - CUDA version >= 11
            - CUDA compute capability >= 8
        - NCCL is available and version >= 2.10
        - MPS is available and torch was built with MPS
        - NPU is available and supports bf16
        - XPU is available and supports bf16

    Returns:
        bool: True if bf16 is available, False otherwise.

    )   
   )r   r   r   is_bf16_supporteddistributedis_nccl_availablencclversionr   mpsis_builtr	   npuxpu)cuda_supportmps_supportnpu_supportxpu_supportr   r   r   verify_bf16_support0   s   
r,   dtypedevicec              	   C   st   | du rt jS t| | }|t vr%td| ddtt  d|t j	kr8|t 
dkr8t s8td|S )a  Get the torch.dtype corresponding to the given precision string. If no string is passed,
    we will default to torch.float32.

    Note:
        If bf16 precision is requested with a CUDA device, we verify whether the device indeed supports
        bf16 kernels. If not, a ``RuntimeError`` is raised.

    Args:
        dtype (Optional[str]): The precision dtype. Default: ``None``, in which we default to torch.float32
        device (Optional[torch.device]): Device in use for training. Only CUDA and CPU
            devices are supported. If a CUDA device is passed in, additional checking is done
            to ensure that the device supports the requested precision. Default: ``None``, in which case
            a CUDA device is assumed.
    Raises:
        ValueError: if precision isn't supported by the library
        RuntimeError: if bf16 precision is requested but not available on this hardware.

    Returns:
        torch.dtype: The corresponding torch.dtype.

    NzDtype z must be one of z, z for finetuning.cpuzcbf16 precision was requested but not available on this hardware. Please use fp32 precision instead.)r   float32r   getvalues
ValueErrorjoinlistkeysbfloat16r.   r,   RuntimeError)r-   r.   torch_dtyper   r   r   	get_dtypeK   s   
r:   )NNNc              	   c   s8    t  }t |  zdV  W t | dS t | w )a  
    Context manager to set torch's default dtype.

    Args:
        dtype (torch.dtype): The desired default dtype inside the context manager.

    Returns:
        ContextManager: context manager for setting default dtype.

    Example:
        >>> with set_default_dtype(torch.bfloat16):
        >>>     x = torch.tensor([1, 2, 3])
        >>>     x.dtype
        torch.bfloat16


    N)r   get_default_dtypeset_default_dtype)r-   	old_dtyper   r   r   r<   }   s   
r<   named_paramsexclude_param_namesc                    sX   | D ]'\ }|durt  fdd|D rq|j|kr)td  d|j d| qdS )a  
    Validates that all input parameters have the expected dtype.

    Args:
        named_params (Iterable[Tuple[str, torch.nn.Parameter]]): Iterable of named parameters.
        dtype (torch.dtype): Expected dtype.
        exclude_param_names (Optional[List[str]]): Optional list of parameter names to exclude from dtype checking

    Raises:
        ValueError: If any parameter has a different dtype than `dtype`.
    Nc                 3   s    | ]}| v V  qd S Nr   ).0nnamer   r   	<genexpr>   s    z0validate_expected_param_dtype.<locals>.<genexpr>z
Parameter z has dtype z, but expected )anyr-   r3   )r>   r-   r?   paramr   rC   r   validate_expected_param_dtype   s   
rH   )r   )NNr@   ) 
contextlibtypingr   r   r   r   r   r   r   torchtune.utilsr   torchtune.utils._devicer	   logfloat16r7   r0   float64r   strr-   __annotations__r   boolr,   r.   r:   contextmanagerr<   nn	ParameterrH   r   r   r   r   <module>   sB   
 
2
