o
    NiE                     @  s   d dl mZ d dlZd dlmZmZ d dlZd dlZd dl	m
Z
mZ d dlmZmZ d dlmZ ddlmZ dd	lmZmZmZ e rSG d
d dejjeZdddZe
 rhG dd dejjeZdddZdS dS )    )annotationsN)AnyOptional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight   )RoadVariant)	RoadLayer_apply_road_get_delta_weightc                      sT   e Zd Z			d$d% fddZd&d'ddZd(ddZd)d d!Zd* fd"d#Z  ZS )+Linear8bitLtroad_1@   T
base_layertorch.nn.Moduleadapter_namestrvariantr   
group_sizeintinit_weightsboolreturnNonec                   2   t    t| | || _| j||||d d S N)r   r   r   super__init__r   _active_adapterupdate_layerselfr   r   r   r   r   kwargs	__class__ H/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/road/bnb.pyr!   "      
	
zLinear8bitLt.__init__FN
safe_mergeadapter_namesOptional[list[str]]c                 C  s^  t | |}|s	dS |D ]}|| jv rtd |  j}|  j}|jdu r*|j|_t||d}t	| j
| | j| | j| j| j| j}t|||j}||j|j }|rlt| sltd| dtjj|dd|jd|j|  _|  jdur|  j}	|	j}
|	j}t|||j}||
|	_|  | j| qdS )	  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NzWMerge road module to 8-bit linear may get different generations due to rounding errors.state1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights) r   _available_adapterswarningswarnget_base_layerweightr1   SCBr	   r   r   r   
road_thetadata
road_alphatorchmatmultodtypedevice
contiguousisfiniteall
ValueErrorbnbnn
Int8Paramsr7   biasreset_gradsmerged_adaptersappend)r%   r,   r-   active_adapterr<   r1   outputroad_Rw_datarM   
orig_dtype	bias_datanew_biasr)   r)   r*   merge6   sP   








zLinear8bitLt.mergec                 C  sf  | j s
td dS t| jdkr| j }|| jv rtd |  j}|  j	}|j
du r3|j
|_
t||d}t| j| | j| | j| j| j| j}tj|tj|j}t|||j}||j|j }tjj|dd|jd|j|  _|  jdur|  j}|j}	|j}
t||
}||	|_|   t| jdksdS dS )	_
            This method unmerges all merged adapter layers from the base weights.
             Already unmerged. Nothing to do.Nr   zYUnmerge road module to 8-bit linear may get different generations due to rounding errors.r0   r4   Fr5   )!mergedr9   r:   lenrO   popr8   r;   r<   r1   r=   r	   r   r   r   r>   r?   r@   rA   linalginvrC   float32rD   rB   rE   rF   rJ   rK   rL   r7   rM   rN   )r%   rQ   r<   r1   rR   rS   
inv_road_RrT   rM   rU   rV   rW   r)   r)   r*   unmerges   sH   








zLinear8bitLt.unmergextorch.Tensorargsr   r&   c                 O     | j r| jr
|   | j|g|R i |}|S | jr)| j|g|R i |}|S | j|g|R i |}| jD ]7}|| jvr@q8t  }|rT|j}| 	|| j
| j}t| j| | j| | j
| | j| |}|ro||}q8|S Ndisable_adaptersr[   rb   r   active_adaptersr8   rA   is_autocast_enabledrD   _cast_input_dtyper>   r   r   r   r@   rC   r%   rc   re   r&   resultrQ   requires_conversionexpected_dtyper)   r)   r*   forward   s6   



zLinear8bitLt.forwardc                      t   }d| S )Nzroad.r    __repr__r%   repr'   r)   r*   rt         
zLinear8bitLt.__repr__r   r   Tr   r   r   r   r   r   r   r   r   r   r   r   FNr,   r   r-   r.   r   r   r   r   )rc   rd   re   r   r&   r   r   rd   r   r   	__name__
__module____qualname__r!   rX   rb   rq   rt   __classcell__r)   r)   r'   r*   r       s    
=
- r   targetr   r   r   c                 K  st   d }t | tr|  }n| }|dd}|r8t |tjjr8| }|| j	j
| j	j| jd t| |fi |}|S )Nloaded_in_8bitF)r7   	thresholdindex)
isinstancer   r;   getrJ   rK   r   copyupdater1   r7   r   r   )r   r   r&   
new_moduletarget_base_layerr   eightbit_kwargsr)   r)   r*   dispatch_bnb_8bit   s   

r   c                      sT   e Zd Z			d!d" fddZd#d$ddZd%ddZd&ddZd' fdd Z  ZS )(
Linear4bitr   r   Tr   r   r   r   r   r   r   r   r   r   r   r   c                   r   r   r   r$   r'   r)   r*   r!      r+   zLinear4bit.__init__FNr,   r-   r.   c                 C  st  t | |}|s	dS |D ]}|| jv rtd |  j}|j}t||jd}t	| j
| | j| | j| j| j| j}t|||j}||j|j}|r`t| s`td| dd|v rhd|d< d|d< |d	d d
d | D }tjj|dfi ||j|  _|  jdur|  j}	|	j}
|	j}t|||j}||
|	_| j| qdS )r/   NzVMerge oft module to 4-bit linear may get different generations due to rounding errors.r0   r2   r3   bnb_quantizedFr6   r?   c                 S  s    i | ]\}}| d s||qS )_)
startswith).0kvr)   r)   r*   
<dictcomp>#  s     z$Linear4bit.merge.<locals>.<dictcomp>r4   )r   r8   r9   r:   r;   r<   __dict__r	   quant_stater   r   r   r>   r?   r@   rA   rB   rC   rD   rE   rG   rH   rI   r]   itemsrJ   rK   
Params4bitrM   rO   rP   )r%   r,   r-   rQ   r<   r&   rR   rS   rT   rM   rU   rV   rW   r)   r)   r*   rX      sL   





(
zLinear4bit.mergec                 C  sj  | j s
td dS t| jdkr| j }|| jv rtd |  j}|j	}t
||jd}t| j| | j| | j| j| j| j}tj|tj|j}t|||j}||j|j}d|v rnd|d< d|d< |d	d tjj|d
fi ||j|  _|  jdur|  j}|j}	|j}
t||
}||	|_t| jdksdS dS )rY   rZ   Nr   zXUnmerge oft module to 4-bit linear may get different generations due to rounding errors.r0   r   Fr6   r?   r4   )r[   r9   r:   r\   rO   r]   r8   r;   r<   r   r	   r   r   r   r   r>   r?   r@   rA   r^   r_   rC   r`   rD   rB   rE   rJ   rK   r   rM   )r%   rQ   r<   r&   rR   rS   ra   rT   rM   rU   rV   rW   r)   r)   r*   rb   /  sB   





(
zLinear4bit.unmergerc   rd   c                 O  rf   rg   rh   rm   r)   r)   r*   rq   Z  s6   



zLinear4bit.forwardc                   rr   )Nzoft.rs   ru   r'   r)   r*   rt     rw   zLinear4bit.__repr__rx   ry   rz   r{   r|   )rc   rd   r   rd   r}   r~   r)   r)   r'   r*   r      s    
<
+%r   c                 K  sz   d }t | tr|  }n| }|dd}|r;t r;t |tjjr;| }|	|j
|jj|jjd t| |fi |}|S )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   r;   r   r   rJ   rK   r   r   r   r   r<   r   r   )r   r   r&   r   r   r   fourbit_kwargsr)   r)   r*   dispatch_bnb_4bit  s   

r   )r   r   r   r   )
__future__r   r9   typingr   r   bitsandbytesrJ   rA   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r   peft.utils.integrationsr	   configr   layerr   r   r   rK   Moduler   r   r   r   r)   r)   r)   r*   <module>   s(    
% ' 