o
    Nip'                     @  s   d dl mZ d dlZd dlmZ d dlmZmZ d dlm	Z	m
Z
mZ d dlZd dlmZ d dlmZ ddlmZmZmZmZ eG d	d
 d
eZG dd deZG dd deZdS )    )annotationsN)abstractmethod)	dataclassfield)AnyOptionalUnion)
PeftConfig   )	BaseTunerBaseTunerLayer_get_in_out_featurescheck_adapters_to_mergec                   @  sB   e Zd ZU dZeeddidZded< eeddidZded< d	S )
LycorisConfigz1
    A base config for LyCORIS like adapters
    helpzThe mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. For example, `{'^model.decoder.layers.0.encoder_attn.k_proj': 16}`.)default_factorymetadatazOptional[dict]rank_patternzThe mapping from layer names or regexp expression to alphas which are different from the default alpha specified by `alpha`. For example, `{'^model.decoder.layers.0.encoder_attn.k_proj': 16}`.alpha_patternN)	__name__
__module____qualname____doc__r   dictr   __annotations__r    r   r   M/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lycoris_utils.pyr   "   s   
 	r   c                   @  s   e Zd ZdZdZd4ddZeed5d
dZd6ddZ	ed7ddZ
ed8ddZed9ddZd:d;d#d$Zed<d%d&Zd'd( Zd=d+d,Zd6d-d.Zd>d6d/d0Zed?d2d3ZdS )@LycorisLayerz0
    A base layer for LyCORIS like adapters
    )ralphascalingrank_dropoutmodule_dropout
base_layer	nn.ModulereturnNonec                 C  s\   || _ i | _i | _i | _i | _i | _i | _d| _g | _d| _	t
|  \}}|| _|| _d S )NFT)r#   r   r   r    r!   rank_dropout_scaler"   _disable_adaptersmerged_adapterscast_input_dtype_enabledr   get_base_layerin_featuresout_features)selfr#   r,   r-   r   r   r   __init__D   s   
zLycorisLayer.__init__set[str]c                 C     d S Nr   )r.   r   r   r   _available_adaptersW   s   z LycorisLayer._available_adaptersc                 O  s@   |  }|dd}|j| g|R ddi| | j|d d S )Ndevicecpumeta)r4   )copypopr/   to_empty)r.   clsargskwargsfinal_devicer   r   r   _init_empty_weights[   s   z LycorisLayer._init_empty_weightsadapter_namestrr   intc                 K  r1   r2   r   )r.   r?   r   r<   r   r   r   create_adapter_parametersg      z&LycorisLayer.create_adapter_parametersxtorch.Tensorr;   r   r<   c                 O  s   dS )zZActivations added on top of the base layer output (i.e. after the base layer forward pass)Nr   )r.   r?   rD   r;   r<   r   r   r   _get_delta_activationsk   s    z#LycorisLayer._get_delta_activationsc                 C  r1   r2   r   r.   r?   r   r   r   get_delta_weighto   rC   zLycorisLayer.get_delta_weightFN
safe_mergebooladapter_namesOptional[list[str]]c                 C  s   t | |}|s	dS |D ]?}|| jv rJ|  }|r9|jj }|| |7 }t|	 s4t
d| d||j_n|j j| |7  _| j| qdS )ab  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
        Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   r3   r+   weightdataclonerH   torchisfiniteall
ValueErrorr)   append)r.   rI   rK   active_adapterr#   orig_weightsr   r   r   merger   s$   



zLycorisLayer.mergec                 C  r1   r2   r   rG   r   r   r   reset_adapter_parameters   rC   z%LycorisLayer.reset_adapter_parametersc                 C  s0   || j vrd S || j|  | j|  | j|< d S r2   )r3   r   r   r    )r.   adapterscaler   r   r   	set_scale   s   
"zLycorisLayer.set_scalerZ   floatc                 C  s:   |dkrd S | j D ]}|| jvrq	| j|  |9  < q	d S )Nr
   )active_adaptersr3   r    r.   rZ   rU   r   r   r   scale_layer   s   

zLycorisLayer.scale_layerc                 C  sf   | j s
td dS t| jdkr1| j }|| jv r(|  j j	| 
|8  _	t| jdksdS dS )zW
        This method unmerges all merged adapter layers from the base weights.
        z Already unmerged. Nothing to do.Nr   )mergedwarningswarnlenr)   r8   r3   r+   rM   rN   rH   )r.   rU   r   r   r   unmerge   s   


zLycorisLayer.unmergec                 C  sR   | j D ]#}|| jvrq|d u r| j| | j|  | j|< q| j|  |  < qd S r2   )r]   r3   r   r   r    r^   r   r   r   unscale_layer   s   

zLycorisLayer.unscale_layerr   c                 K  r1   r2   r   )r.   r?   r   r   r<   r   r   r   update_layer   rC   zLycorisLayer.update_layer)r#   r$   r%   r&   )r%   r0   )r%   r&   )r?   r@   r   rA   )
r?   r@   rD   rE   r;   r   r<   r   r%   rE   )r?   r@   r%   rE   )FN)rI   rJ   rK   rL   r%   r&   )r?   r@   )rZ   r\   r%   r&   r2   )r?   r@   r   rA   r   r\   )r   r   r   r   other_param_namesr/   propertyr   r3   r>   rB   rF   rH   rW   rX   r[   r_   rd   re   rf   r   r   r   r   r   <   s.    

#



r   c                   @  sB   e Zd ZU dZded< eZded< edddZe	dddZ
dS )LycorisTunera  
    A base tuner for LyCORIS like adapters

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`LoraConfig`]): The configuration of the Lora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    r@   prefixz/dict[type[torch.nn.Module], type[LycorisLayer]]layers_mappingconfigr   r?   targetUnion[LycorisLayer, nn.Module]c                 C  r1   r2   r   )r.   rl   r?   rm   target_nameparentcurrent_keyr   r   r   _create_and_replace   s   	z LycorisTuner._create_and_replacer$   r%   r   c                 K  s0  d }| j  D ]"\}}t|dr t| |r t|tr |} n
t||r)|} nq|d u rHddd | j  D }tdt	| d| dt|trR| }	n|}	t|	t
jjt
jjfrk||fd|i|}
|
S t|	t
jjr~||fd|i|}
|
S dd	d | j  D }tdt	| d| d)
Nr#   z, c                 s      | ]}|j V  qd S r2   r   .0layerr   r   r   	<genexpr>       z2LycorisTuner._create_new_module.<locals>.<genexpr>zTarget module of type z, not supported, currently only adapters for z are supportedr?   c                 s  rs   r2   rt   ru   r   r   r   rx     ry   )rk   itemshasattr
isinstancer+   r   joinkeysrS   typerP   nnConv2dConv1dLinear)r:   rl   r?   rm   r<   new_module_clssubtype
target_clssupported_modulestarget_base_layer
new_moduler   r   r   _create_new_module   sH   



zLycorisTuner._create_new_moduleN)rl   r   r?   r@   rm   rn   )rl   r   r?   r@   rm   r$   r%   r   )r   r   r   r   r   r   tuner_layer_clsr   rr   classmethodr   r   r   r   r   ri      s   
 
ri   )
__future__r   ra   abcr   dataclassesr   r   typingr   r   r   rP   torch.nnr   peft.configr	   tuners_utilsr   r   r   r   r   r   ri   r   r   r   r   <module>   s    