o
    8wi                     @  s  d dl mZ d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZ d dlmZmZ d dlZd dlmZ d d	lmZ d d
lmZmZ d dlmZmZmZmZmZ d dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z? ddl@mAZA ddlBmCZC dd ZDG dd deZEdS )    )annotationsN)contextmanager)asdictreplace)Enum)partialreduce)LiteralOptional)nn)tqdm)is_bnb_4bit_availableis_bnb_available)	BaseTunerBaseTunerLayercheck_target_module_existsonload_layerreplicate_layers)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGAuxiliaryTrainingWrapperModulesToSaveWrapper_freeze_adapter_get_submodulesget_peft_model_state_dictget_quantization_config)dare_linear	dare_tiesmagnitude_prunetask_arithmeticties)get_pattern_key   )dispatch_aqlm)dispatch_awq)
LoraConfig)dispatch_eetq)dispatch_gptq)dispatch_hqq)dispatch_inc)Conv2d	LoraLayerdispatch_default)dispatch_torchao)dispatch_megatronc                 C  s   ||d< ||fS )Nadapter_names )targetargskwargsr.   r/   r/   S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/lora/model.py_adapter_names_pre_forward_hook=   s   r4   c                      s^  e Zd ZU dZdZded< dcdd fd
dZdeddZedd Z	dfddZ
dd Zdd ZdgddZedd Zdh fd d!Zdcdid#d$Zdjdkd'd(Zdld)d*Zdld+d,Zdmd/d0Zed1d2 Z fd3d4Zed5d6 Z	%			7dndod<d=ZdpdDdEZ	F	7	7	%	7	7	GdqdrdRdSZ	7	%	7dsdTdUZdVdW ZdtdXdYZ	7dudvd[d\Zdwd]d^Z dxdydadbZ!  Z"S )z	LoraModela   
    Creates Low Rank Adapter (LoRA) model from a pretrained transformers model.

    The method is described in detail in https://huggingface.co/papers/2106.09685.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`LoraConfig`]): The configuration of the Lora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The Lora model.

    Example:

        ```py
        >>> from transformers import AutoModelForSeq2SeqLM
        >>> from peft import LoraModel, LoraConfig

        >>> config = LoraConfig(
        ...     task_type="SEQ_2_SEQ_LM",
        ...     r=8,
        ...     lora_alpha=32,
        ...     target_modules=["q", "v"],
        ...     lora_dropout=0.01,
        ... )

        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
        >>> lora_model = LoraModel(model, config, "default")
        ```

        ```py
        >>> import torch
        >>> import transformers
        >>> from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training

        >>> rank = ...
        >>> target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"]
        >>> config = LoraConfig(
        ...     r=4, lora_alpha=16, target_modules=target_modules, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM"
        ... )
        >>> quantization_config = transformers.BitsAndBytesConfig(load_in_8bit=True)

        >>> tokenizer = transformers.AutoTokenizer.from_pretrained(
        ...     "kakaobrain/kogpt",
        ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
        ...     bos_token="[BOS]",
        ...     eos_token="[EOS]",
        ...     unk_token="[UNK]",
        ...     pad_token="[PAD]",
        ...     mask_token="[MASK]",
        ... )
        >>> model = transformers.GPTJForCausalLM.from_pretrained(
        ...     "kakaobrain/kogpt",
        ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
        ...     pad_token_id=tokenizer.eos_token_id,
        ...     use_cache=False,
        ...     device_map={"": rank},
        ...     torch_dtype=torch.float16,
        ...     quantization_config=quantization_config,
        ... )
        >>> model = prepare_model_for_kbit_training(model)
        >>> lora_model = get_peft_model(model, config)
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`LoraConfig`]): The configuration of the Lora model.
    lora_strprefixFlow_cpu_mem_usageboolreturnNonec                   s   t  j||||d d S )N)r9   )super__init__)selfmodelconfigadapter_namer9   	__class__r/   r3   r>      s   zLoraModel.__init__rA   r$   c                 C  s2   t | jdkr|jdkrt| jj ddS dS )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r!   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueErrorrD   __name__)r?   rA   r/   r/   r3   _check_new_adapter_config   s
   	z#LoraModel._check_new_adapter_configc                 C  s
   t | |S N)r   )lora_configkeyr/   r/   r3   _check_target_module_exists   s   
z%LoraModel._check_target_module_existsrG   r@   	nn.Modulec                 C  s   |j rt||j  dS dS )a  
        A private method to modify the model structure before adapter is applied.

        Args:
            peft_config (`PeftConfig`):
                The prepared adapter config.
            model (`nn.Module`):
                The model that is going to be adapted.
        N)layer_replicationr   )r?   rG   r@   r/   r/   r3   _prepare_model   s   
zLoraModel._prepare_modelc                 C  s  |d u rt dt|j |}t|j |}|j||j}	|j||j}
|	|
|j|j	|j
|j|j|j|j|jj|jt| jddt| jddd}ztd| j|d< W n	 tya   Y nw g d}|D ]}t| j|d	}|d ur|||| d
< qhddlm} t|trt||s|j||	|
|j|j
|j|j|jd d S t| jdr| jjnd }| j|||fd|i|}|| j vr|!d | "|||| d S )NzCurrent Key shouldn't be `None`is_loaded_in_8bitFis_loaded_in_4bit)r
lora_alphalora_dropoutfan_in_fan_outinit_lora_weights
use_rslorause_dora
use_qaloraqalora_group_sizeephemeral_gpu_offload	lora_biasloaded_in_8bitloaded_in_4bitz:hf_quantizer.quantization_config.get_apply_tensor_subclassget_apply_tensor_subclass)gptqaqlmawq)method_quantization_configr   )AdaLoraLayer)rV   rW   rY   rZ   r[   r_   hf_device_map
device_map)#rI   r    rank_patternkeysalpha_patterngetrU   rV   rW   rX   rY   rZ   r[   r\   r]   runtime_configr^   r_   getattrr@   operator
attrgetterAttributeErrorr   peft.tuners.adalorarh   
isinstancer*   update_layerhasattrri   _create_new_moduleactive_adaptersrequires_grad__replace_module)r?   rM   rB   r0   target_nameparentcurrent_keyr_key	alpha_keyrU   alphar2   quant_methodsquant_methodquantization_configrh   rj   
new_moduler/   r/   r3   _create_and_replace   sh   	


zLoraModel._create_and_replacec                   s   t ||| t|dr|j}td | D ]M\}}| j|v s$d|v rdt|dr-|j}n$t|dr6|j}nt|dr?|j	}nt
|dd d urK|j}nt| }t fdd	| D sd||j qd S )
N
base_layermetaranknumqweightW_qweightin_proj_weightc                 3  s    | ]}|j  kV  qd S rL   )device).0pr   r/   r3   	<genexpr>      z,LoraModel._replace_module.<locals>.<genexpr>)setattrrw   r   torchr   named_modulesr8   r   r   r   rp   r   next
parametersanyto)r?   r}   
child_namer   childnamemoduler   r/   r   r3   r{      s&   




zLoraModel._replace_modulec                 C  s   |  D ]\}}| j|vrd|_q| jD ]H}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrU| D ]}t|trSt	|drS|jd urSd|j_q>qt
d| dd S )	NFrE   allrH   T	lora_onlyzRequested bias: z, is not implemented.)named_parametersr8   requires_gradry   rG   rH   modulesru   r*   rw   NotImplementedError)r?   r@   nr   active_adapterrH   mr/   r/   r3    _mark_only_adapters_as_trainable  s,   

z*LoraModel._mark_only_adapters_as_trainablec           
      K  s   g }| j rdd }|| t rddlm} || t r*ddlm} || |tt	t
ttttttg	 d }|D ]}	|	||fd| i|}|d urO nq<|d u r\td| d|S )	Nc                 [  sV   d }t | tr|  }n| }|j D ]\}}t ||r(|| |fi |} |S q|S rL   )ru   r   get_base_layer_custom_modulesitems)r0   rB   rM   r2   r   target_base_layerrN   
custom_clsr/   r/   r3   dynamic_dispatch_func.  s   


z;LoraModel._create_new_module.<locals>.dynamic_dispatch_funcr!   )dispatch_bnb_8bit)dispatch_bnb_4bitrM   zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `torch.nn.Embedding`, `torch.nn.Conv1d`, `torch.nn.Conv2d`, `torch.nn.Conv3d`, `transformers.pytorch_utils.Conv1D`, `torch.nn.MultiheadAttention.`.)r   appendr   bnbr   r   r   extendr%   r"   r#   r&   r'   r(   r,   r-   r+   rI   )
rM   rB   r0   r2   dispatchersr   r   r   r   
dispatcherr/   r/   r3   rx   %  sB   



zLoraModel._create_new_moduler   c                   s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r@   )r=   __getattr__rs   rp   r@   )r?   r   rC   r/   r3   r   h  s   zLoraModel.__getattr__	inferencec                 C  sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S  s&   i | ]\}}|t |tr|jn|qS r/   )ru   r   value)r   kvr/   r/   r3   
<dictcomp>t  s   & z5LoraModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)rG   r   r   )r?   r   config_dictrN   r   rA   r/   r/   r3   get_peft_config_as_dictq  s   z!LoraModel.get_peft_config_as_dictTenabledc                 C  s,   | j  D ]}t|ttfr|| qd S rL   )r@   r   ru   r   r   enable_adapters)r?   r   r   r/   r/   r3   _set_adapter_layersz  s
   
zLoraModel._set_adapter_layersc                 C  s   | j dd dS )zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        Tr   N)r   r?   r/   r/   r3   enable_adapter_layers  s   zLoraModel.enable_adapter_layersc                 C  sF   | j D ]}| j| j}|dkrd| d}t| q| jdd dS )zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        rE   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   N)ry   rG   rH   warningswarnr   )r?   r   valmsgr/   r/   r3   disable_adapter_layers  s   


z LoraModel.disable_adapter_layersrB   str | list[str]c                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	dS )a   Set the active adapter(s).

        Additionally, this function will set the specified adapters to trainable (i.e., requires_grad=True). If this is
        not desired, use the following code.

        ```py
        >>> for name, param in model_peft.named_parameters():
        ...     if ...:  # some check on name (ex. if 'lora' in name)
        ...         param.requires_grad = False
        ```

        Args:
            adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated.
        zJAdapter cannot be set when the model is merged. Unmerging the model first.N)
r@   r   ru   r*   mergedr   r   unmergeset_adapterr   )r?   rB   r   r/   r/   r3   r     s   



zLoraModel.set_adapterc                 /  s     dd }|d u rd V  d S | jrtdt }|  D ]}t|tr3||j O }||j	 O }qdd |D }|| }|rMtdd
t|  dd }t|to[|dk}	|d d  }
|	rt|ttfsutd	t| d
t fdd|D g }g }|  D ]}t|tst|trtt|d}|j|dd}|| q|	rt| jdr| j  D ]}t|tst|trtt|
d}|j|dd}|| qd V  |D ]}|  qd S )Nr.   z?Cannot pass `adapter_names` when the model is in training mode.c                 S  s   h | ]}|d kr|qS )__base__r/   )r   r   r/   r/   r3   	<setcomp>  s    z7LoraModel._enable_peft_forward_hooks.<locals>.<setcomp>z.Trying to infer with non-existing adapter(s): z, 	num_beamsr!   zGot adapter names of type z, expected a list of str.c                 3  s    | ]
}|g d   V  qdS )r   Nr/   )r   r   r2   r/   r3   r     s    z7LoraModel._enable_peft_forward_hooks.<locals>.<genexpr>)r.   T)with_kwargsget_encoder)poptrainingrI   setr   ru   r*   lora_Arl   lora_embedding_Ajoinsortedrn   intlisttuple	TypeErrortypesumr   r   r4   register_forward_pre_hookr   rw   r@   r   remove)r?   r1   r2   r.   expected_adapterslayerunique_adaptersunexpected_adaptersr   uses_beam_searchoriginal_adapter_nameshook_handlesr   pre_forwardhandler/   r   r3   _enable_peft_forward_hooks  sT   



z$LoraModel._enable_peft_forward_hooksc                   s<   t    t| jdddkrtd| jdrtddS )zVerify that the configuration supports merging.

        Currently gptq quantization and replicated layers do not support merging.
        quantization_methodNrc   z9Cannot merge LORA layers when the model is gptq quantizedrQ   z>Cannot merge LORA layers when base model layers are replicated)r=   _check_merge_allowedrp   r@   rI   rG   rn   r   rC   r/   r3   r     s   
zLoraModel._check_merge_allowedc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   rI   r   )rG   model_configr/   r/   r3   _prepare_adapter_config  s   

z!LoraModel._prepare_adapter_configNprogressbar
safe_merger.   Optional[list[str]]c              	     s  |r     fdd j D }d|rdnd d }t|| |dD ]Y}zt j|\}}	}
W n	 ty:   Y q$w t|	6 t|	drV|	j|||d	} 	||
||	 nt|	d
rn|rd|	j
||d  	||
|	 |	 W d    n1 sxw   Y  q$ jS )Nc                      g | ]\}} j |vr|qS r/   r8   r   rN   _r   r/   r3   
<listcomp>      z:LoraModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r@   )disabledesc"unload_and_optionally_merge_module)merger   r.   r   )r   r.   )r   r@   r   r   r   rs   r   rw   r   r{   r   r   )r?   r   r   r   r.   key_listr   rN   r}   r0   r|   unloaded_moduler/   r   r3   _unload_and_optionally_merge  s0   


z&LoraModel._unload_and_optionally_mergeadapters	list[str]combination_typesvd_rank
int | Nonetuple[str, int, str]c                   s   D ]}|t j vrtd| dqdd  D } fdd|D }|r4tdt| dt dkr<d	n|}d
d fdd D D }|dv ratt|dkr\td|d }n|dkrjt|}n|drv|ptt	|}ntd| fdd D }	|	std  tt|	dkrtd|	d t
u rdfdd D }
n|	d tu rttjfdd D }
n
td|	d  d|||
fS )z
        Helper function to check if the arguments to add_weighted_adapter are valid and compatible with the underlying
        model.
        Adapter  does not existc                 S  s   g | ]	}t |tr|qS r/   )ru   r   )r   r   r/   r/   r3   r   *  s    z9LoraModel._check_add_weighted_adapter.<locals>.<listcomp>c                   s*   g | ] t  fd dD dkr qS )c                 3      | ]}| j v V  qd S rL   )modules_to_saver   adapterwrapperr/   r3   r   .  r   zCLoraModel._check_add_weighted_adapter.<locals>.<listcomp>.<genexpr>r!   )r   )r   )r  r  r3   r   +  s
    z\Cannot add weighted adapters if they target the same module with modules_to_save, but found z such instance(s).r!   linearc                 S  s0   g | ]}|j s
|jnt|jg|j  R  qS r/   )rk   rU   maxvalues)r   rA   r/   r/   r3   r   9  s    "c                 3  s    | ]} j | V  qd S rL   )rG   r  r   r/   r3   r   <  r   z8LoraModel._check_add_weighted_adapter.<locals>.<genexpr>)r  r   r   r   r   zkAll adapters must have the same r value when using combination_type linear, ties, dare_ties or dare_linear.r   catsvdzInvalid combination_type: c                   s   g | ]
}t  j| jqS r/   )r   rG   r   r  r   r/   r3   r   Q  s    z'Found no adapter matching the names in zall adapter configs should follow the same target modules type. Combining adapters with `target_modules` type being a mix of list/set and string is not supported.|c                 3  s$    | ]}d  j | j dV  qdS )()NrG   r   r  r   r/   r3   r   [  s   " c                 3  s    | ]	} j | jV  qd S rL   r  r  r   r/   r3   r   ^  s    zInvalid type z found in target_modules)r   rG   rl   rI   r   rF   r   r   endswithr  r7   r   r   rq   or_r   )r?   r  r  r  r  modules_to_save_wrappersproblematic_wrappersadapters_ranksnew_ranktarget_module_typesnew_target_modulesr/   )r  r?   r3   _check_add_weighted_adapter  sX   




z%LoraModel._check_add_weighted_adapterr  totalweightslist[float]	svd_clampsvd_full_matrices
svd_driver
str | Nonedensityfloat | Nonemajority_sign_methodLiteral['total', 'frequency']c                   sf  |t  j v rdS  j|||d\}}}t j|d  |||i i d j|<   j| t j|  fdd j D }|D ]}t	 j|\}}}t
|tr0||jv rf|j| j}|j| j}n||jv rv|j| }|j| }nqC|jd |_|jd |_|dkrg g }}t||D ]:\}}||jv r|j| j}|j| j}n||jv r|j| }|j| }nq||j| |j|   ||j qt|dkrtd	tj|dd
}tj|dd
}||jd|jd ddf< ||jddd|jd f< qC|dv r j||||||||	|
|||d\|_|_qC|dv r0 |||||	|
\|_|_qCdS )a	  
        This method adds a new adapter by merging the given adapters with the given weights.

        When using the `cat` combination_type you should be aware that rank of the resulting adapter will be equal to
        the sum of all adapters ranks. So it's possible that the mixed adapter may become too big and result in OOM
        errors.

        Args:
            adapters (`list`):
                List of adapter names to be merged.
            weights (`list`):
                List of weights for each adapter.
            adapter_name (`str`):
                Name of the new adapter.
            combination_type (`str`):
                The merging type can be one of [`svd`, `linear`, `cat`, `ties`, `ties_svd`, `dare_ties`, `dare_linear`,
                `dare_ties_svd`, `dare_linear_svd`, `magnitude_prune`, `magnitude_prune_svd`]. When using the `cat`
                combination_type, the rank of the resulting adapter is equal to the sum of all adapters ranks (the
                mixed adapter may be too big and result in OOM errors).
            svd_rank (`int`, *optional*):
                Rank of output adapter for svd. If None provided, will use max rank of merging adapters.
            svd_clamp (`float`, *optional*):
                A quantile threshold for clamping SVD decomposition output. If None is provided, do not perform
                clamping. Defaults to None.
            svd_full_matrices (`bool`, *optional*):
                Controls whether to compute the full or reduced SVD, and consequently, the shape of the returned
                tensors U and Vh. Defaults to True.
            svd_driver (`str`, *optional*):
                Name of the cuSOLVER method to be used. This keyword argument only works when merging on CUDA. Can be
                one of [None, `gesvd`, `gesvdj`, `gesvda`]. For more info please refer to `torch.linalg.svd`
                documentation. Defaults to None.
            density (`float`, *optional*):
                Value between 0 and 1. 0 means all values are pruned and 1 means no values are pruned. Should be used
                with [`ties`, `ties_svd`, `dare_ties`, `dare_linear`, `dare_ties_svd`, `dare_linear_svd`,
                `magnintude_prune`, `magnitude_prune_svd`]
            majority_sign_method (`str`):
                The method, should be one of ["total", "frequency"], to use to get the magnitude of the sign values.
                Should be used with [`ties`, `ties_svd`, `dare_ties`, `dare_ties_svd`]
        N)r  r  r  r   )rU   rV   r   rm   rk   c                   r   r/   r   r   r   r/   r3   r     r   z2LoraModel.add_weighted_adapter.<locals>.<listcomp>g        r  z9No matching LoRAs found. Please raise an issue on GitHub.dimr!   )r  ties_svddare_linear_svddare_ties_svdmagnitude_prune_svdfull_matricesdriver)r  r   r   r   r   )r   rG   rl   r#  r   inject_adapterr@   r   r   r   ru   r*   r   r   lora_Br   lora_embedding_Bdatazipr   scalingrF   rI   r   r  shape1_svd_generalized_task_arithmetic_weighted_adapter-_generalized_task_arithmetic_weighted_adapter)r?   r  r%  rB   r  r  r'  r(  r)  r+  r-  r   r"  r  rN   r   r0   target_lora_Atarget_lora_Bloras_Aloras_Br  r   current_adapter_lora_Acurrent_adapter_lora_Br/   r   r3   add_weighted_adaptere  s   5









zLoraModel.add_weighted_adapterc                   sD  g }g }t  fdd|D }t||D ]\}}| jv s"| jv r1|| || j|   qt|dkr<td fdd|D }t	|
|d j}|dkrZt||}n5|dkrft||||	}n)|d	krqt|||}n|d
kr}t||||	}n|dkrt|||}ntd| t t}|r j dd dk}|s|jdd}n| }t dr js|r|j}tjj|||d\}}}|d d d |f }|d | }|t| }|d |d d f }|
d urt| | g}t||
}| }|||}|||}|r| |j!j"}| |j!j"}||fS )Nc                 3  r  rL   )r   r  r0   r/   r3   r     r   zNLoraModel._svd_generalized_task_arithmetic_weighted_adapter.<locals>.<genexpr>r   z9No matching LoRAs found. Please raise an issue on Github.c                   s   g | ]}  |qS r/   )get_delta_weightr  rH  r/   r3   r         zOLoraModel._svd_generalized_task_arithmetic_weighted_adapter.<locals>.<listcomp>r  r1  r2  r3  r4  z*Invalid value passed to combination type:       )r!   r!   r!   )	start_dimrX   r5  )#r   r<  r   r   r   r=  rF   rI   r   tensorr   r   r   r   r   r   r   ru   r)   r   sizeflattensqueezerw   rX   Tlinalgr  diagr  quantileclampreshaper;  r>  )r?   r  r  r%  r   r0   rA  rB  r+  r-  rV  r6  r7  valid_adaptersvalid_weightsis_embeddingr  r   delta_weightconv2d
conv2d_1x1USVhdisthi_vallow_valr/   rH  r3   r?    s\   


z;LoraModel._svd_generalized_task_arithmetic_weighted_adapterc                   sr  g }g }g }	t ||D ]@\}
}|
|jv r!|j|
 j}|j|
 j}n|
|jv r1|j|
 }|j|
 }nq|t||j	|
   ||j
 |	|j
 qt||d j}||	g}|d j t|D ]I\}}|dkrtt||||< qd|dkrt||||||< qd|dkrt|||||< qd|dkrt||||||< qd|dkrt|||||< qdtd fdd	|D }|S )
Nr   r  r   r   r   r   zInvalid combination typec                   s   g | ]}|  qS r/   )r   )r   deltadtyper/   r3   r   Z  rJ  zKLoraModel._generalized_task_arithmetic_weighted_adapter.<locals>.<listcomp>)r<  r   r   r9  r   r:  r   mathsqrtr=  r;  r   rN  r   r   rf  	enumerater   r   r   r   r   rI   )r?   r  r  r%  r0   r+  r-  rY  lora_A_deltaslora_B_deltasr  r   rE  rF  lora_deltasitask_tensorsr/   re  r3   r@  1  s>   




z7LoraModel._generalized_task_arithmetic_weighted_adapterc                   s   |t  j vrtd| d j|=  fdd j D }d}|D ] }t j|\}}}t|trE|	| |du rE|j
dd }q%|pIg  _ j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        r
  r  c                   r   r/   r   r   r   r/   r3   r   h  r   z,LoraModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)r   rG   rl   rI   r@   r   r   ru   r*   delete_adapterry   r   _delete_auxiliary_adapter)r?   rB   r  new_adapterrN   r   r0   r/   r   r3   rp  ]  s   


zLoraModel.delete_adaptertorch.nn.Modulec                 C  s   | j |||dS )aG  
        This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model
        as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import PeftModel

        >>> base_model = AutoModelForCausalLM.from_pretrained("tiiuae/falcon-40b")
        >>> peft_model_id = "smangrul/falcon-40B-int4-peft-lora-sfttrainer-sample"
        >>> model = PeftModel.from_pretrained(base_model, peft_model_id)
        >>> merged_model = model.merge_and_unload()
        ```
        )r   r   r.   r  )r?   r   r   r.   r/   r/   r3   merge_and_unloadt  s   zLoraModel.merge_and_unloadc                 C  s   | j ddS )z
        Gets back the base model by removing all the lora modules without merging. This gives back the original base
        model.
        F)r   rt  r   r/   r/   r3   unload  s   zLoraModel.unloadoutput_state_dictdict[str, torch.Tensor]c              	   C  s   | j  D ]#\}}|jjtjkr(|jjtjkr(|jjtjkr(|dr(t	
d qt| |dd|d}i }| D ]>}d|v rZtj|| |d|ddd  gd	d
||< q:d|v rxtj|| |d|ddd   gdd
||< q:|S )a!  
        This function can calculate the updates of the PiSSA/CorDA/OLoRA by comparing the parameters of the
        PiSSA/CorDA/OLoRA adapter in `output_state_dict` with the initial values of PiSSA/CorDA/OLoRA in
        `adapter_name`, thus converting PiSSA/CorDA/OLoRA to LoRA.
        pissaa   Note that Quant(W_res) + AB != Quant(W) + \Delta(AB); the converted LoRA, when combined with W or Quant(W), may introduce a certain gap in the fine-tuned model. Therefore, we recommend directly using the Quant(W_res) in conjunction with the PiSSA adapter. 
state_dictN)rz  rB   r   .r!   r   r/  r9  )r@   r   r;  rf  r   float32float16bfloat16
startswithr   r   r   rn   rl   r  r   split)r?   rw  rB   r2   r   parammutated_init_state_dicttensors_lorar/   r/   r3   subtract_mutated_init  s6   
$&
zLoraModel.subtract_mutated_init)F)r9   r:   r;   r<   )rA   r$   r;   r<   )rG   r$   r@   rP   )r@   rP   r;   r<   )r   r7   )r   r:   )T)r   r:   r;   r<   )r;   r<   )rB   r   r;   r<   )TFFN)r   r:   r   r:   r.   r   )r  r  r  r7   r  r  r;   r	  )r  NNTNNr$  )r  r  r%  r&  rB   r7   r  r7   r  r  r'  r  r(  r:   r)  r*  r+  r,  r-  r.  r;   r<   )NTN)rB   r7   r;   r<   )FFN)r   r:   r   r:   r.   r   r;   rs  )r;   rs  rL   )rw  rx  rB   r7   )#rJ   
__module____qualname____doc__r8   __annotations__r>   rK   staticmethodrO   rR   r   r{   r   rx   r   r   r   r   r   r   r   r   r   r   r  r#  rG  r?  r@  rp  ru  rv  r  __classcell__r/   r/   rC   r3   r5   C   sb   
 H


E

B		



;

N 
D
,
 r5   )F
__future__r   rg  rq   r   
contextlibr   dataclassesr   r   enumr   	functoolsr   r   typingr	   r
   r   r   r   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r   r   r   r   
peft.utilsr   r   r   r   r   r   r   peft.utils.merge_utilsr   r   r   r   r   peft.utils.otherr    rd   r"   re   r#   rA   r$   eetqr%   rc   r&   hqqr'   incr(   r   r)   r*   r+   torchaor,   tp_layerr-   r4   r5   r/   r/   r/   r3   <module>   s:   $	