o
    Ni+                     @  s   d dl mZ d dlZd dlmZ d dlmZmZ d dlZ	d dl
Z
d dlZd dl
mZ e	jeje	jdkZd,d	d
Zed-d.ddZd/ddZd0d1ddZd2ddZdd Zd3dd Zed0d4d"d#Zed0d5d&d'Zed(d) Zd*d+ ZdS )6    )annotationsN)contextmanager)LiteralOptional)nnz
5.0.0.dev0returnboolc                  C  s>   t jtjt jdkrddlm}  |  S ddlm}  |  S )Nz4.33.0r   is_deepspeed_zero3_enabled)	packagingversionparsetransformers__version__transformers.integrationsr
   transformers.deepspeedr	    r   K/home/ubuntu/.local/lib/python3.10/site-packages/peft/utils/integrations.pycheck_deepspeed_zero3_enabled   s
   r   modifier_rankOptional[int]
fwd_moduletorch.nn.Modulec                 c  sX    t  s	dV  dS ddl}|jj| ||d dV  W d   dS 1 s%w   Y  dS )z`Call DeepSpeed GatheredParameters context manager if DeepSpeed is enabled, otherwise do nothing.Nr   )r   r   )r   	deepspeedzeroGatheredParameters)paramr   r   r   r   r   r   gather_params_ctx&   s   
r   moduletorch.nn.Parameterc                 C  s   t | dr|  }|S t| jjdr| j }|S | j}t|tjj	s7t|tj
r-|S tdt| d|jj}|dvrA|S t| dd}|j}|jtdjk}t||d	}|rb| j|| _|S )
z
    Helper function to dequantize a quantized weight.

    This function should be extended if more quantization schemes are added to the library.

    If the weight is not quantized, it will be returned as is.
    W_qztorchao.z1Input weight should be of type nn.Parameter, got z instead)
Params4bit
Int8ParamsstateNcpu)r#   )hasattr
dequantizetypeweight
__module__
startswith
isinstancetorchr   	ParameterTensor	TypeError	__class____name__getattrdevicedequantize_bnb_weightto)r   r(   cls_namequant_stater3   is_cpur   r   r   dequantize_module_weight5   s*   

r9   r(   c                 C  s   ddl }| j}| jj}|dkr|j| j| j}|S |du r"td|j	du r+| j	|_	t
|jdr<|j| j|j	}|S | j|j	dd d }|S )	z7Helper function to dequantize 4bit or 8bit bnb weights.r   Nr!   zNo `state` was passed for bnb 8bit quantized weights. Please open an issue on the PEFT repository and report the error: https://github.com/huggingface/peft/issuesint8_vectorwise_dequant   g   @ ?)bitsandbytesr3   r0   r1   
functionaldequantize_4bitdatar7   
ValueErrorSCBr%   r:   view)r(   r#   bnbr3   r6   dequantizedr   r   r   r4   Z   s"   
r4   r   Literal[False, '4bit', '8bit']c                 C  s$   | j jdkrdS | j jdkrdS dS )z>Returns '4bit' or '8bit' if bitsandbytes parameter, else Falser!   4bitr"   8bitF)r0   r1   )r   r   r   r   get_bnb_param_typey   s
   rI   c                   s   dd | j  D d fdd| j  D   du rdS t dkr5d v r5 fd	dt| jjD S i } D ]}t| jjD ]}d
| d
| d
v rV | ||<  nqAq9t| jjD ]}||vrltd| dq^|S )z<
    Derive the device map for the layers of the model.
    c                 S  s   g | ]}|d vr|qS )r$   diskr   ).0dr   r   r   
<listcomp>   s    z(get_layer_device_map.<locals>.<listcomp>r   c                   s"   i | ]\}}||d v r n|qS rJ   r   )rL   namer3   )main_devicer   r   
<dictcomp>   s    z(get_layer_device_map.<locals>.<dictcomp>Nr<    c                   s   i | ]}| d  qS )rR   r   )rL   idx)execution_device_mapr   r   rQ      s    .zlayer z! has not been mapped to a device.)hf_device_mapvaluesitemslenrangeconfignum_hidden_layersRuntimeError)modellayer_device_maplayerrS   r   )rT   rP   r   get_layer_device_map   s*   
ra   Nonec                 C  s   t |tjrt| dsdS t |tjrt| |j dS t| }t| j	j
D ]5}|| }t|drD|j| }|j||_|j||_q%|j| ||j|< |j| ||j|< q%dS )zp
    Ensure that the key and value cache of the model are on the same device as their corresponding layers.
    rV   Nlayers)r+   r   Cacher%   EncoderDecoderCachemap_cache_to_layer_device_mapself_attention_cachera   rZ   r[   r\   rc   keysr5   rW   	key_cachevalue_cache)r^   cacher_   rS   layer_devicer`   r   r   r   rf      s   

rf   include_buffersc                 c  s@    t td| d}|V  W d    d S 1 sw   Y  d S )Nmeta)rm   )_init_on_devicer,   r3   )rm   fr   r   r   init_empty_weights   s   "rq   r3   torch.devicec                 #  s   t jj|rt jj fdd}d fdd	}|r$dd dD }ni } fd	d
}z;|t j_|r7|t j_| D ]}tt||tt| q;d V  W t j_|rWt j_| D ]
\}}tt|| q[d S t j_|rrt j_| D ]
\}}tt|| qvw )Nc                   sv   | || |d ur7t tdddur9t| j| }| j| j}|j|d< || j|  fi || j|< d S d S d S )N_skipFTrequires_grad)r2   ro   r'   _parameters__dict__rt   r5   )r   rO   r   	param_clskwargs)r3   old_register_parameterr   r   register_empty_parameter   s   
&z1_init_on_device.<locals>.register_empty_parameterTc                   s6   | |||d |d ur| j |  | j |< d S d S )N)
persistent)_buffersr5   )r   rO   bufferr{   )r3   old_register_bufferr   r   register_empty_buffer   s   z._init_on_device.<locals>.register_empty_bufferc                 S  s   i | ]}|t t|qS r   )r2   r,   )rL   torch_function_namer   r   r   rQ      s    
z#_init_on_device.<locals>.<dictcomp>)emptyzerosonesfullc                   s    fdd}|S )Nc                    s    |d< | i |S )Nr3   r   argsrx   )r3   fnr   r   wrapper   s   zB_init_on_device.<locals>.patch_tensor_constructor.<locals>.wrapperr   )r   r   )r3   )r   r   patch_tensor_constructor   s   z1_init_on_device.<locals>.patch_tensor_constructor)T)	r   Moduleregister_parameterregister_bufferrh   setattrr,   r2   rX   )r3   rm   rz   r   tensor_constructors_to_patchr   r   old_torch_functionr   )r3   r~   ry   r   ro      s>   ro   c                  c  s0    t tdd} zdt_d V  W | t_d S | t_w )Nrs   FT)r2   ro   rs   )old_valr   r   r   _skip_init_on_device  s   r   c                   s   t   fdd}|S )z
    Ignore the init_on_device context manager when calling the decorated function.

    This is a narrow use decorator that allows us to avoid initializing on meta device even when we're inside the
    init_empty_weights context.

    c                    s8   t    | i |W  d    S 1 sw   Y  d S N)r   r   funcr   r   r     s   $z$skip_init_on_device.<locals>.wrapper)	functoolswraps)r   r   r   r   r   skip_init_on_device  s   r   )r   r   )r   N)r   r   r   r   )r   r   r   r   r   )r(   r   )r   r   r   rF   )r   rb   )rm   r   )r3   rr   rm   r   )
__future__r   r   
contextlibr   typingr   r   packaging.versionr   r,   r   r   r   r   r   is_transformers_ge_v5r   r   r9   r4   rI   ra   rf   rq   ro   r   r   r   r   r   r   <module>   s0   

%

"5

