o
    Ni                     @  sx   d dl mZ d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
mZ d dlmZ dd	lmZ G d
d de
ZdS )    )annotations)Optional)Module)tqdm)
PeftConfig)	BaseTuner_get_submodules)6TRANSFORMERS_MODELS_TO_LNTUNING_TARGET_MODULES_MAPPING   )LNTuningLayerc                   @  sd   e Zd ZU dZdZded< eZeZ	d"ddZ
d#ddZd$ddZ				d%d&ddZd'd(d d!ZdS ))LNTuningModela  
    Creates LayerNorm tuning from a pretrained transformer model.

    The method is described in detail in https://huggingface.co/papers/2312.11420.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`LNTuningConfig`]): The configuration of the Lora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            This option has no effect on LN tuning but exists for consistency with other PEFT methods.

    Returns:
        'torch.nn.Module': The adapted model with LayerNorm tuned on.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import get_peft_model, TaskType, LNTuningConfig

        >>> peft_config = LNTuningConfig(
        ...     task_type=TaskType.CAUSAL_LM,
        ... )

        >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
        >>> model = get_peft_model(model, peft_config)
        >>> model.print_trainable_parameters()
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`LNTuningConfig`]): The configuration of the Lora model.
    
ln_tuning_strprefixpeft_configr   adapter_nametargetr   target_nameparentcurrent_keyreturnNonec                 C  s6   |  |||}|| jkr|d | |||| d S )NF)_create_new_moduleactive_adapterrequires_grad__replace_module)selfr   r   r   r   r   r   
new_module r   O/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/ln_tuning/model.py_create_and_replaceD   s   


z!LNTuningModel._create_and_replacec                 C  s.   t |tst||}|S |}||j| |S N)
isinstancer   update_layer
base_layer)r   r   r   r   r   r   r   r   r   S   s   

z LNTuningModel._create_new_moduleadapter_namesOptional[list[str]]c                   s@   |p j }t fdd|D }|rt|dkrtdd S d S )Nc                 3  s    | ]	} j | jV  qd S r!   )r   modules_to_save).0adapterr   r   r   	<genexpr>b   s    
z2LNTuningModel._unloading_checks.<locals>.<genexpr>r
   z?Cannot unload multiple adapters that specify `modules_to_save`.)active_adaptersanylen
ValueError)r   r%   adapters_to_consideris_modules_to_save_availabler   r*   r   _unloading_checks`   s   
zLNTuningModel._unloading_checksTFNprogressbarbool
safe_mergec              	     s     |  fdd j D }d|rdnd d }t|| |dD ]-}zt j|\}}	}
W n	 ty9   Y q#w t|	drP|rF|	|  ||
|		 |	 q# jS )	Nc                   s   g | ]\}} j |vr|qS r   )r   )r(   key_r*   r   r   
<listcomp>p   s    z>LNTuningModel._unload_and_optionally_merge.<locals>.<listcomp>zUnloading adapters zand merging  model)disabledescr$   )
r2   r:   named_modulesr   r   AttributeErrorhasattrmerger   get_base_layer)r   r@   r3   r5   r%   key_listr<   r6   r   r   r   r   r*   r   _unload_and_optionally_mergeh   s   


z*LNTuningModel._unload_and_optionally_mergeautocast_adapter_dtypec                 C  s   d S r!   r   )r   r   rD   r   r   r   _cast_adapter_dtype   s   z!LNTuningModel._cast_adapter_dtype)r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   )r%   r&   )TFFN)r3   r4   r5   r4   r%   r&   )T)r   r   rD   r4   r   r   )__name__
__module____qualname____doc__r   __annotations__r   tuner_layer_clsr	   target_module_mappingr    r   r2   rC   rE   r   r   r   r   r      s   
 #



r   N)
__future__r   typingr   torch.nn.modulesr   r   peft.configr   peft.tuners.tuners_utilsr   r   
peft.utilsr	   layerr   r   r   r   r   r   <module>   s   