o
    8wi[                     @  s   d dl mZ d dlZd dlmZmZ d dlZd dlZd dl	m
Z
mZ d dlmZmZ d dlmZ d dlmZ dd	lmZmZ e rQG d
d dejjeZdddZe
 rfG dd dejjeZdddZdS dS )    )annotationsN)AnyOptional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight)	transpose   )	LoraLayerLoraVariantc                      x   e Zd Z							d1d2 fddZd3ddZd4d5dd Zd6d!d"Zd#d$ Zd7d+d,Zd8d-d.Z	d9 fd/d0Z
  ZS ):Linear8bitLtr   r           TF
base_layertorch.nn.Moduleadapter_namestrrint
lora_alphalora_dropoutfloatinit_lora_weightsbool
use_rslorause_dora	lora_biasreturnNonec
              
     @   t    t| | d| _|| _| j||||||||	d d S NF)r   r   r   r   r   r   super__init__r   fan_in_fan_out_active_adapterupdate_layerselfr   r   r   r   r   r   r   r   r   kwargs	__class__ Q/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/lora/bnb.pyr%   "      

zLinear8bitLt.__init__Optional[LoraVariant]c                K     |sd S ddl m} | S Nr   )DoraLinearVariantvariantsr4   r*   r   r+   r4   r.   r.   r/   resolve_lora_variant?      z!Linear8bitLt.resolve_lora_variantN
safe_mergeadapter_namesOptional[list[str]]c           
      C  sX  t | |}|s	dS |D ]}|| j vrqtd |  j}|  j}|jdu r-|j|_t	||d}|| j
vrJ| |}||j|j| }n
| j
| | ||}|ret| setd| dtjj|dd|jd|j|  _| j| r|  jj| j| j }	|rt|	std| d|	|  j_|  | j| qdS )	  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NzWMerge lora module to 8-bit linear may get different generations due to rounding errors.state1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r   lora_Akeyswarningswarnget_base_layerweightr?   SCBr	   lora_variantget_delta_weighttodtypedevice
merge_safetorchisfiniteall
ValueErrorbnbnn
Int8ParamsrE   r   biasdatalora_Breset_gradsmerged_adaptersappend)
r*   r:   r;   active_adapterrK   r?   output	lora_dataw_data	bias_datar.   r.   r/   mergeG   sJ   








zLinear8bitLt.mergec                 C  s&  | j s
td dS t| jdkr| j }|| j vrq
td |  j	}|  j
}|jdu r6|j|_t||d}|| jvrS| |}||j|j| }n
| j| | ||}tjj|dd|jd|j|  _	| j| r|  j j| j| j8  _|  t| jdksdS dS )	_
            This method unmerges all merged adapter layers from the base weights.
             Already unmerged. Nothing to do.Nr   zYUnmerge lora module to 8-bit linear may get different generations due to rounding errors.r>   rB   FrC   )mergedrH   rI   lenr^   poprF   rG   rJ   rK   r?   rL   r	   rM   rN   rO   rP   rQ   unmergerW   rX   rY   rE   r   rZ   r[   r\   r]   )r*   r`   rK   r?   ra   rb   rc   r.   r.   r/   rk      s8   







zLinear8bitLt.unmergec                 C  (   t | j| j| j| j d| j|  S NFr
   r\   rK   rF   scalingr*   adapterr.   r.   r/   rN         zLinear8bitLt.get_delta_weightxtorch.Tensorargsr   	list[str]r+   c                    | j |g|R i |}t|}g }|D ] | fddt|D  qt|D ]X\}}	|	dkr2q)|	| j vr:q)| j|	 }
| j|	 }| j|	 }| j|	 }t	
  }|r`|j}| ||
jj}|||  }||
||| }|rw||}|||   |7  < q)|S )Nc                      g | ]
\}}| kr|qS r.   r.   .0indexitemrq   r.   r/   
<listcomp>       z5Linear8bitLt._mixed_batch_forward.<locals>.<listcomp>__base__r   setr_   	enumeraterF   rG   r\   r   ro   rS   is_autocast_enabledrP   _cast_input_dtyperK   rO   r*   rs   r;   ru   r+   resultunique_adapterssub_batch_indices_listir`   rF   r\   dropoutro   requires_conversionexpected_dtype	sub_batchra   r.   r}   r/   _mixed_batch_forward   0   





z!Linear8bitLt._mixed_batch_forwardc                 O  s~  | j |g|R i | |dd }| jr*| jr|   | j|g|R i |}|S |d ur>| j|g|R d|i|}|S | jrO| j|g|R i |}|S | j|g|R i |}| jD ]^}|| j	 vrhq^| j| }| j
| }| j| }	| j| }
t  }|r|j}| ||jj}|| jvr|||	||
 }|r||}|| }q^| j| j| |||d}|r||}q^|S Nr;   )r`   rs   r   )_check_forward_argsrj   disable_adaptersrh   rk   r   r   active_adaptersrF   rG   r\   r   ro   rS   r   rP   r   rK   rM   rO   forwardr*   rs   ru   r+   r;   r   r`   rF   r\   r   ro   r   r   ra   r.   r.   r/   r      sP   #!










zLinear8bitLt.forwardc                      t   }d| S Nzlora.r$   __repr__r*   repr,   r.   r/   r        
zLinear8bitLt.__repr__r   r   r   TFFFr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   r   r1   FNr:   r   r;   r<   r   r    r   r    
rs   rt   ru   r   r;   rv   r+   r   r   rt   rs   rt   r   rt   r   r   __name__
__module____qualname__r%   r8   re   rk   rN   r   r   r   __classcell__r.   r.   r,   r/   r       s     

<$
	
&,r   targetr   r   r   c                 K  st   d }t | tr|  }n| }|dd}|r8t |tjjr8| }|| j	j
| j	j| jd t| |fi |}|S )Nloaded_in_8bitF)rE   	thresholdr{   )
isinstancer   rJ   getrW   rX   r   copyupdater?   rE   r   r{   )r   r   r+   
new_moduletarget_base_layerr   eightbit_kwargsr.   r.   r/   dispatch_bnb_8bit  s   

r   c                      r   ):
Linear4bitr   r   r   TFr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    c
              
     r!   r"   r#   r)   r,   r.   r/   r%   !  r0   zLinear4bit.__init__r1   c                K  r2   r3   r5   r7   r.   r.   r/   r8   >  r9   zLinear4bit.resolve_lora_variantNr:   r;   r<   c           
      C  sb  t | |}|s	dS |D ]}|| j vrqtd |  j}|j}t||j	d}|| j
vr8| |}|| }n
| j
| | ||}|rSt| sStd| dd|v r[d|d< d|d< |d	d d
d | D }tjj|dfi ||j|  _| j| r|  jj| j| j }	|rt|	std| d|	|  j_| j| qdS )r=   NzWMerge lora module to 4-bit linear may get different generations due to rounding errors.r>   r@   rA   bnb_quantizedFrD   r[   c                 S  s    i | ]\}}| d s||qS )_)
startswith)rz   kvr.   r.   r/   
<dictcomp>t  s     z$Linear4bit.merge.<locals>.<dictcomp>rB   )r   rF   rG   rH   rI   rJ   rK   __dict__r	   quant_staterM   rN   rR   rS   rT   rU   rV   rj   itemsrW   rX   
Params4bitrO   rQ   r   rZ   r[   r\   r^   r_   )
r*   r:   r;   r`   rK   r+   ra   rb   rc   rd   r.   r.   r/   re   F  sF   





(

zLinear4bit.mergec                 C  s  | j s
td dS t| jdkr| j }|| j vrq
td |  j	}|j
}t||jd}|| jvrA| |}|| }n
| j| | ||}d|v rSd|d< d|d< |d	d tjj|d
fi ||j|  _	| j| r|  j j| j| j8  _t| jdksdS dS )rf   rg   Nr   zYUnmerge lora module to 4-bit linear may get different generations due to rounding errors.r>   r   FrD   r[   rB   )rh   rH   rI   ri   r^   rj   rF   rG   rJ   rK   r   r	   r   rM   rN   rk   rW   rX   r   rO   rQ   r   rZ   r[   r\   )r*   r`   rK   r+   ra   rb   rc   r.   r.   r/   rk     s2   





(
zLinear4bit.unmergec                 C  rl   rm   rn   rp   r.   r.   r/   rN     rr   zLinear4bit.get_delta_weightrs   rt   ru   r   rv   r+   c                  rw   )Nc                   rx   r.   r.   ry   r}   r.   r/   r~     r   z3Linear4bit._mixed_batch_forward.<locals>.<listcomp>r   r   r   r.   r}   r/   r     r   zLinear4bit._mixed_batch_forwardc                 O  s  | j |g|R i | |dd }| jr*| jr|   | j|g|R i |}|S |d ur>| j|g|R d|i|}|S | jrO| j|g|R i |}|S | j|g|R i |}| }| jD ]^}|| j	
 vrlqb| j	| }| j| }| j| }	| j| }
t  }|r|j}| ||jj}|| jvr|||	||
 }|r||}|| }qb| j| j| |||d}|r||}qb|S r   )r   rj   r   rh   rk   r   r   cloner   rF   rG   r\   r   ro   rS   r   rP   r   rK   rM   rO   r   r   r.   r.   r/   r     sR   *(&










zLinear4bit.forwardc                   r   r   r   r   r,   r.   r/   r     r   zLinear4bit.__repr__r   r   r   r   r   r   r   r   r   r   r.   r.   r,   r/   r     s     

;#
	
&3r   c                 K  sz   d }t | tr|  }n| }|dd}|r;t r;t |tjjr;| }|	|j
|jj|jjd t| |fi |}|S )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   rJ   r   r   rW   rX   r   r   r   r   rK   r   r   )r   r   r+   r   r   r   fourbit_kwargsr.   r.   r/   dispatch_bnb_4bit
  s   

r   )r   r   r   r   )
__future__r   rH   typingr   r   bitsandbytesrW   rS   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r   peft.utils.integrationsr	   peft.utils.otherr
   layerr   r   rX   Moduler   r   r   r   r.   r.   r.   r/   <module>   s(    
g l 