o
    NÆÏiõ?  ã                   @  s°   d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ dd	lmZ e
ƒ rFG d
d„ dejjeƒZe	ƒ rVG dd„ dejjeƒZdS dS )é    )ÚannotationsN)ÚOptional)Úis_bnb_4bit_availableÚis_bnb_available)Úcheck_adapters_to_merge)Údequantize_bnb_weight)Ú	transposeé   )Ú	VeraLayerc                      ób   e Zd Z					d&d'‡ fdd„Zd(d)dd„Zd*dd„Zd+dd „Zd,d"d#„Zd-‡ fd$d%„Z‡  Z	S ).ÚLinear8bitLtr   ç        FTçš™™™™™¹?Ú
base_layerútorch.nn.ModuleÚadapter_nameÚstrÚrÚintÚvera_dropoutÚfloatÚfan_in_fan_outÚboolÚinit_weightsÚ	d_initialÚreturnÚNonec
              	     ó>   t ƒ  ¡  t | |¡ || _|| _| j|||||||	d d S ©N)r   r   r   ©ÚsuperÚ__init__r
   r   Ú_active_adapterÚupdate_layer©Úselfr   r   Úvera_AÚvera_Br   r   r   r   r   Úkwargs©Ú	__class__© úH/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/vera/bnb.pyr!   !   ó   

ùzLinear8bitLt.__init__NÚ
safe_mergeÚadapter_namesúOptional[list[str]]c           	      C  s  | j rt dd | j¡› dd | j¡› d¡ t| |ƒ}|s d S |D ]g}|| j ¡ vr,q"t d¡ |  	|¡}|  
¡ j}|  
¡ j}|jd u rI|j|_t||ƒ}| |j¡ |j¡| }|rkt |¡ ¡ sktd|› dƒ‚tjj| d¡d	|jd
 |j¡|  
¡ _| ¡  | j |¡ q"d S )Nú'Already following adapters were merged ú,ú#. You are now additionally merging Ú.zWMerge vera module to 8-bit linear may get different generations due to rounding errors.ú1NaNs detected in the merged weights. The adapter ú seems to be brokenÚcpuF©Úrequires_gradÚhas_fp16_weights)ÚmergedÚwarningsÚwarnÚjoinÚmerged_adaptersÚactive_adaptersr   Úvera_lambda_dÚkeysÚget_delta_weightÚget_base_layerÚweightÚstateÚSCBr   ÚtoÚdtypeÚdeviceÚtorchÚisfiniteÚallÚ
ValueErrorÚbnbÚnnÚ
Int8Paramsr:   Úreset_gradsÚappend)	r%   r.   r/   Úactive_adapterÚ	vera_datarE   rF   ÚoutputÚw_datar+   r+   r,   Úmerge=   sD   
ÿÿ
ÿ





ÿÿþæzLinear8bitLt.mergec                 C  sà   | j s
t d¡ d S t| jƒdkrn| j ¡ }|| j ¡ vrq
t d¡ |  |¡}|  	¡ j
}|  	¡ j}|jd u r;|j|_t||d}| |j¡ |j¡| }tjj| d¡d|jd |j¡|  	¡ _
| ¡  t| jƒdksd S d S )NúAlready unmerged. Nothing to dor   zYUnmerge vera module to 8-bit linear may get different generations due to rounding errors.)rF   r7   Fr8   )r;   r<   r=   Úlenr?   ÚpoprA   rB   rC   rD   rE   rF   rG   r   rH   rI   rJ   rO   rP   rQ   r:   rR   )r%   rT   rU   rE   rF   rV   rW   r+   r+   r,   Úunmerged   s0   

ÿ



ÿþìzLinear8bitLt.unmergeútorch.Tensorc                 C  sò   | j | }| j| }|j}|j}|jdko|tjkp|tjk}| j| }| j	| }|r;| 
¡ }| 
¡ }| 
¡ }| 
¡ }|dd…d| j…f  |j¡}	|d| j…dd…f  |j¡}
| d¡}| d¡}t||
 ||	  | jƒ}|rw|j|d}|S )a=  
            Compute the delta weight for the given adapter.

            Args:
                adapter (str): The name of the adapter for which the delta weight should be computed.

            Returns:
                torch.Tensor: The computed delta weight for the VeRA adapter.

            Note:
                This method implements the VeRA-specific weight update. Unlike LoRA, VeRA uses shared projection
                matrices (vera_A and vera_B) across all layers, along with per-layer trainable parameters (lambda_d and
                lambda_b).
            r7   Néÿÿÿÿ©rI   ©r&   r'   rJ   rI   ÚtyperK   Úfloat16Úbfloat16rA   Úvera_lambda_br   Úin_featuresrH   Úout_featuresÚ	unsqueezer   r   ©r%   Úadapterr&   r'   rJ   rI   Úcast_to_fp32Úlambda_dÚlambda_bÚsliced_AÚsliced_BÚoutput_tensorr+   r+   r,   rC      s(   





zLinear8bitLt.get_delta_weightÚxc              	   O  st  | j r| jr
|  ¡  | j|g|¢R i |¤Ž}n| jr'| j|g|¢R i |¤Ž}n| j|g|¢R i |¤Ž}| jD ]}}|| j ¡ vr@q6| j| }| j| }| j| }| j	| }	| j
| }
t ¡  }|rp|j}|j}|j|krp| |¡}|dd…d| j…f  |j¡}|	d| j…dd…f  |j¡}|
| |j¡ƒ}|tjj |tjj ||¡ |¡ }|r¯| |¡}|| }q6| |j¡S )a  
            Perform the forward pass using the VeRA adapter.

            Args:
                x (torch.Tensor): Input tensor.

            Returns:
                torch.Tensor: Output tensor after applying the VeRA adaptation.

            Note:
                This method implements the VeRA-specific forward pass. It applies the shared projections (vera_A and
                vera_B) along with the per-layer trainable parameters (lambda_d and lambda_b) to compute the adapter
                output.
            N)Údisable_adaptersr;   r\   r   r@   rA   rB   rd   r&   r'   r   rK   Úis_autocast_enabledrI   rH   re   rJ   rf   rP   Ú
functionalÚlinear©r%   rp   Úargsr(   ÚresultrT   rk   rl   r&   r'   ÚdropoutÚrequires_conversionÚexpected_dtypeÚcompute_dtyperm   rn   Úx_tempÚadapter_outputr+   r+   r,   Úforward´   s>   









ÿ

zLinear8bitLt.forwardc                   ó   t ƒ  ¡ }d| S ©Nzvera.©r    Ú__repr__©r%   Úrepr)   r+   r,   r‚   ï   ó   
zLinear8bitLt.__repr__©r   r   FTr   ©r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ©FN©r.   r   r/   r0   r   r   ©r   r   ©r   r]   ©rp   r]   r   r]   ©r   r   ©
Ú__name__Ú
__module__Ú__qualname__r!   rX   r\   rC   r~   r‚   Ú__classcell__r+   r+   r)   r,   r       s    ö
'

5;r   c                      r   ).Ú
Linear4bitr   r   FTr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   c
              	     r   r   r   r$   r)   r+   r,   r!   ÷   r-   zLinear4bit.__init__Nr.   r/   r0   c                 C  s  | j rt dd | j¡› dd | j¡› d¡ t| |ƒ}|s d S |D ]]}|| j ¡ vr,q"t d¡ |  	|¡}|  
¡ j}|j}dd„ | ¡ D ƒ}tj |j|j¡| }|rct |¡ ¡ sctd|› d	ƒ‚tjj| d
¡fddi|¤Ž |j¡|  
¡ _| j |¡ q"d S )Nr1   r2   r3   r4   zWMerge vera module to 4-bit linear may get different generations due to rounding errors.c                 S  s    i | ]\}}|  d ¡s||“qS )Ú_)Ú
startswith)Ú.0ÚkÚvr+   r+   r,   Ú
<dictcomp>*  s     z$Linear4bit.merge.<locals>.<dictcomp>r5   r6   r7   r9   F)r;   r<   r=   r>   r?   r@   r   rA   rB   rC   rD   rE   Ú__dict__ÚitemsrO   rs   Údequantize_4bitÚdataÚquant_staterK   rL   rM   rN   rP   Ú
Params4bitrH   rJ   rS   )r%   r.   r/   rT   rU   rE   r(   rW   r+   r+   r,   rX     s:   
ÿÿ
ÿ


ÿ
ÿézLinear4bit.mergec                 C  s¸   | j s
t d¡ d S t| jƒdkrZ| j ¡ }|| j ¡ vrq
t d¡ |  |¡}|  	¡ j
}|j}tj |j|j¡| }tjj| d¡fddi|¤Ž |j¡|  	¡ _
t| jƒdksd S d S )NrY   r   zYUnmerge vera module to 4-bit linear may get different generations due to rounding errors.r7   r9   F)r;   r<   r=   rZ   r?   r[   rA   rB   rC   rD   rE   rš   rO   rs   rœ   r   rž   rP   rŸ   rH   rJ   )r%   rT   rU   rE   r(   rW   r+   r+   r,   r\   7  s$   

ÿ


ÿózLinear4bit.unmerger]   c                 C  sò   | j | }| j| }|j}|j}|jdko|tjkp|tjk}| j| }| j	| }|r;| 
¡ }| 
¡ }| 
¡ }| 
¡ }|d d …d | j…f  |j¡}	|d | j…d d …f  |j¡}
| d¡}| d¡}t||
 ||	  | jƒ}|rw|j|d}|S )Nr7   r^   r_   r`   rh   r+   r+   r,   rC   M  s(   





zLinear4bit.get_delta_weightrp   c              	   O  s|  | j r| jr
|  ¡  | j|g|¢R i |¤Ž}n¡| jr'| j|g|¢R i |¤Ž}n‘| j|g|¢R i |¤Ž}| ¡ }| jD ]}}|| j ¡ vrDq:| j| }| j| }| j	| }| j
| }	| j| }
t ¡  }|rt|j}|j}|j|krt| |¡}|d d …d | j…f  |j¡}|	d | j…d d …f  |j¡}|
| |j¡ƒ}|tjj |tjj ||¡ |¡ }|r³| |¡}|| }q:| |j¡S )N)rq   r;   r\   r   Úcloner@   rA   rB   rd   r&   r'   r   rK   rr   rI   rH   re   rJ   rf   rP   rs   rt   ru   r+   r+   r,   r~   k  s@   









ÿ

zLinear4bit.forwardc                   r   r€   r   rƒ   r)   r+   r,   r‚   ™  r…   zLinear4bit.__repr__r†   r‡   rˆ   r‰   rŠ   r‹   rŒ   r   rŽ   r+   r+   r)   r,   r“   ö   s    ö
$

.r“   )Ú
__future__r   r<   Útypingr   ÚbitsandbytesrO   rK   Úpeft.import_utilsr   r   Úpeft.tuners.tuners_utilsr   Úpeft.utils.integrationsr   Úpeft.utils.otherr   Úlayerr
   rP   ÚModuler   r“   r+   r+   r+   r,   Ú<module>   s     Uþ