o
    NÆÏi	>  ã                   @  s¼   d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZmZ d dlmZ ddlmZ e
ƒ rGG d	d
„ d
ejjeƒZddd„Ze	ƒ r\G dd„ dejjeƒZddd„ZdS dS )é    )ÚannotationsN)ÚOptional)Úis_bnb_4bit_availableÚis_bnb_available)ÚBaseTunerLayerÚcheck_adapters_to_merge)Údequantize_bnb_weighté   )ÚOFTLayerc                      sh   e Zd Z									d,d-‡ fdd„Zd.d/d d!„Zd0d"d#„Zd$d%„ Zd1d(d)„Zd2‡ fd*d+„Z‡  Z	S )3ÚLinear8bitLté   r   ç        TFçiUMu?é   Ú
base_layerútorch.nn.ModuleÚadapter_nameÚstrÚrÚintÚoft_block_sizeÚmodule_dropoutÚfloatÚinit_weightsÚboolÚcoftÚepsÚblock_shareÚuse_cayley_neumannÚnum_cayley_neumann_termsÚreturnÚNonec                   sD   t ƒ  ¡  t | |¡ d| _|| _| j|||||||	||
|d
 d S ©NF)r   r   r   r   r   r   r   r   ©ÚsuperÚ__init__r
   Úfan_in_fan_outÚ_active_adapterÚupdate_layer)Úselfr   r   r   r   r   r   r   r   r   r   r   Úkwargs©Ú	__class__© úG/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/oft/bnb.pyr%   !   ó    

özLinear8bitLt.__init__NÚ
safe_mergeÚadapter_namesúOptional[list[str]]c           	      C  s  t | |ƒ}|s	dS |D ]~}|| j ¡ vrqt d¡ |  ¡ j}|  ¡ j}|jdu r-|j|_t	||d}|  
|¡}t |dd¡}t || |j¡¡}t |dd¡}| |j¡ |j¡}|rkt |¡ ¡ sktd|› dƒ‚tjj| d¡d	|jd
 |j¡|  ¡ _| ¡  | j |¡ qdS )á†  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NzVMerge oft module to 8-bit linear may get different generations due to rounding errors.©Ústater   r	   ú1NaNs detected in the merged weights. The adapter ú seems to be brokenÚcpuF©Úrequires_gradÚhas_fp16_weights)r   Úoft_RÚkeysÚwarningsÚwarnÚget_base_layerÚweightr5   ÚSCBr   Úget_delta_weightÚtorchÚ	transposeÚmmÚtoÚdtypeÚdeviceÚisfiniteÚallÚ
ValueErrorÚbnbÚnnÚ
Int8Paramsr;   Úreset_gradsÚmerged_adaptersÚappend)	r)   r0   r1   Úactive_adapterrA   r5   ÚoutputÚoft_dataÚw_datar-   r-   r.   ÚmergeB   s:   






ÿÿþázLinear8bitLt.mergec                 C  s  | j s
t d¡ dS t| jƒdkr†| j ¡ }|| j ¡ vrq
t d¡ |  ¡ j	}|  ¡ j
}|jdu r6|j|_t||d}|  |¡}t |dd¡}t | ¡ | |j¡¡}t |dd¡}| |j¡ |j¡}tjj| d¡d|jd	 |j¡|  ¡ _	| ¡  t| jƒdksdS dS )
ú_
            This method unmerges all merged adapter layers from the base weights.
            ú Already unmerged. Nothing to do.Nr   zXUnmerge oft module to 8-bit linear may get different generations due to rounding errors.r4   r	   r8   Fr9   )Úmergedr>   r?   ÚlenrQ   Úpopr<   r=   r@   rA   r5   rB   r   rC   rD   rE   rF   ÚtrG   rH   rI   rM   rN   rO   r;   rP   )r)   rS   rA   r5   rT   rU   rV   r-   r-   r.   Úunmergeu   s6   

ÿ



ÿþçzLinear8bitLt.unmergec                 C  ó   | j |  ¡ S ©N©r<   Ú
get_weight©r)   Úadapterr-   r-   r.   rC   ˜   ó   zLinear8bitLt.get_delta_weightÚxútorch.Tensorc           	      O  óÎ   | j r| jr
|  ¡  | j|g|¢R i |¤Ž}|S | jr)| j|g|¢R i |¤Ž}|S | jD ],}|| j ¡ vr6q,| j| }t ¡  }|rM|j	}|  
||jj	¡}||ƒ}|rX| |¡}q,| j|g|¢R i |¤Ž}|S r`   ©Údisable_adaptersrZ   r^   r   Úactive_adaptersr<   r=   rD   Úis_autocast_enabledrH   Ú_cast_input_dtyperA   rG   ©	r)   rf   Úargsr*   ÚresultrS   r<   Úrequires_conversionÚexpected_dtyper-   r-   r.   Úforward›   s,   í
ð


€zLinear8bitLt.forwardc                   ó   t ƒ  ¡ }d| S ©Nzoft.©r$   Ú__repr__©r)   Úrepr+   r-   r.   rw   µ   ó   
zLinear8bitLt.__repr__)	r   r   r   TFr   FFr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   ©FN©r0   r   r1   r2   r    r!   ©r    r!   ©rf   rg   r    rg   ©r    r   ©
Ú__name__Ú
__module__Ú__qualname__r%   rW   r^   rC   rs   rw   Ú__classcell__r-   r-   r+   r.   r      s     ô!
3#
r   Útargetr   r   r   c                 K  st   d }t | tƒr|  ¡ }n| }| dd¡}|r8t |tjjƒr8| ¡ }| | j	j
| j	j| jdœ¡ t| |fi |¤Ž}|S )NÚloaded_in_8bitF)r;   Ú	thresholdÚindex)Ú
isinstancer   r@   ÚgetrM   rN   r   ÚcopyÚupdater5   r;   r‡   rˆ   )r…   r   r*   Ú
new_moduleÚtarget_base_layerr†   Úeightbit_kwargsr-   r-   r.   Údispatch_bnb_8bit¹   s   

ýÿr   c                      sh   e Zd Z									d,d-‡ fdd„Zd.d/d d!„Zd0d"d#„Zd$d%„ Zd1d(d)„Zd2‡ fd*d+„Z‡  Z	S )3Ú
Linear4bitr   r   r   Fr   Tr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   c                   sD   t ƒ  ¡  t | |¡ d| _|| _| j||||||||	|
|d
 d S r"   r#   )r)   r   r   r   r   r   r   r   r   r   r   r   r*   r+   r-   r.   r%   Ô   r/   zLinear4bit.__init__Nr0   r1   r2   c           	      C  s2  t | |ƒ}|s	dS |D ]‹}|| j ¡ vrqt d¡ |  ¡ j}|j}t||j	d}|  
|¡}t |dd¡}t || |j¡¡}t |dd¡}| |j¡ |j¡}|rat |¡ ¡ satd|› dƒ‚d|v rid	|d< d	|d
< | dd¡ dd„ | ¡ D ƒ}tjj| d¡fi |¤Ž |j¡|  ¡ _| j |¡ qdS )r3   NzVMerge oft module to 4-bit linear may get different generations due to rounding errors.r4   r   r	   r6   r7   Úbnb_quantizedFr:   Údatac                 S  s    i | ]\}}|  d ¡s||“qS )Ú_)Ú
startswith)Ú.0ÚkÚvr-   r-   r.   Ú
<dictcomp>"  s     z$Linear4bit.merge.<locals>.<dictcomp>r8   )r   r<   r=   r>   r?   r@   rA   Ú__dict__r   Úquant_staterC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r\   ÚitemsrM   rN   Ú
Params4bitrQ   rR   )	r)   r0   r1   rS   rA   r*   rT   rU   rV   r-   r-   r.   rW   õ   s6   




ÿ(âzLinear4bit.mergec                 C  s  | j s
t d¡ dS t| jƒdkrŠ| j ¡ }|| j ¡ vrq
t d¡ |  ¡ j	}|j
}t||jd}|  |¡}t |dd¡}t | ¡ | |j¡¡}t |dd¡}| |j¡ |j¡}d|v rcd|d< d|d	< | d
d¡ tjj| d¡fi |¤Ž |j¡|  ¡ _	t| jƒdksdS dS )rX   rY   Nr   zXUnmerge oft module to 4-bit linear may get different generations due to rounding errors.r4   r	   r’   Fr:   r“   r8   )rZ   r>   r?   r[   rQ   r\   r<   r=   r@   rA   rš   r   r›   rC   rD   rE   rF   r]   rG   rH   rI   rM   rN   r   )r)   rS   rA   r*   rT   rU   rV   r-   r-   r.   r^   '  s0   

ÿ

(ézLinear4bit.unmergec                 C  r_   r`   ra   rc   r-   r-   r.   rC   H  re   zLinear4bit.get_delta_weightrf   rg   c           	      O  rh   r`   ri   rn   r-   r-   r.   rs   K  s,   æ
ð


€zLinear4bit.forwardc                   rt   ru   rv   rx   r+   r-   r.   rw   l  rz   zLinear4bit.__repr__)	r   r   r   Fr   FTFr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r{   r|   r}   r~   r   r€   r-   r-   r+   r.   r‘   Ò   s     ô!
2!
!r‘   c                 K  sz   d }t | tƒr|  ¡ }n| }| dd¡}|r;tƒ r;t |tjjƒr;| ¡ }| 	|j
|jj|jjdœ¡ t| |fi |¤Ž}|S )NÚloaded_in_4bitF)Úcompute_dtypeÚcompress_statisticsÚ
quant_type)r‰   r   r@   rŠ   r   rM   rN   r‘   r‹   rŒ   rŸ   rA   r    r¡   )r…   r   r*   r   rŽ   rž   Úfourbit_kwargsr-   r-   r.   Údispatch_bnb_4bitp  s   

ýÿr£   )r…   r   r   r   )Ú
__future__r   r>   Útypingr   ÚbitsandbytesrM   rD   Úpeft.import_utilsr   r   Úpeft.tuners.tuners_utilsr   r   Úpeft.utils.integrationsr   Úlayerr
   rN   ÚModuler   r   r‘   r£   r-   r-   r-   r.   Ú<module>   s&    
  ß