o
    8wi                      @  s   d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	m	Z	 d dl
mZ d dlmZmZmZ d d	lmZmZ d
dlmZ G dd deZdS )    )annotationsN)Optional)nn)Module)tqdm)
PeftConfig)	BaseTuner_get_submodulescheck_target_module_exists)6TRANSFORMERS_MODELS_TO_LNTUNING_TARGET_MODULES_MAPPINGModulesToSaveWrapper   )LNTuningLayerc                      s   e Zd ZU dZdZded< dCdD fd
dZdE fddZedFddZ	dGddZ
dHddZdId"d#ZdJd%d&ZdKd(d)ZdLd+d,ZdMd-d.ZdMd/d0ZdNd1d2Z	3			4dOdPd9d:Zd;d< Z	4dQdRd>d?ZdSdTdAdBZ  ZS )ULNTuningModela  
    Creates LayerNorm tuning from a pretrained transformer model.

    The method is described in detail in https://huggingface.co/papers/2312.11420.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`LNTuningConfig`]): The configuration of the Lora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            This option has no effect on LN tuning but exists for consistency with other PEFT methods.

    Returns:
        'torch.nn.Module': The adapted model with LayerNorm tuned on.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import get_peft_model, TaskType, LNTuningConfig

        >>> peft_config = LNTuningConfig(
        ...     task_type=TaskType.CAUSAL_LM,
        ... )

        >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
        >>> model = get_peft_model(model, peft_config)
        >>> model.print_trainable_parameters()
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`LNTuningConfig`]): The configuration of the Lora model.
    
ln_tuning_strprefixFlow_cpu_mem_usageboolreturnNonec                   s   t  j||||d d S )N)r   )super__init__)selfmodelconfigadapter_namer   	__class__ X/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/ln_tuning/model.pyr   D   s   zLNTuningModel.__init__namec                   s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r   )r   __getattr__AttributeErrorgetattrr   )r   r!   r   r   r    r"   H   s   zLNTuningModel.__getattr__peft_configr   model_configdictc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   
ValueErrorset)r%   r&   r   r   r    _prepare_adapter_configR   s   

z%LNTuningModel._prepare_adapter_configr   targetr   target_nameparentcurrent_keyc                 C  s6   |  |||}|| jkr|d | |||| d S )NF)_create_new_moduleactive_adapterrequires_grad__replace_module)r   r%   r   r-   r.   r/   r0   
new_moduler   r   r    _create_and_replace\   s   


z!LNTuningModel._create_and_replacec                 C  s.   t |tst||}|S |}||j| |S N)
isinstancer   update_layer
base_layer)r   r%   r-   r   r5   r   r   r    r1   k   s   

z LNTuningModel._create_new_module
child_namer5   childc                 C  s   t ||| t|dr|j}t|dd d ur,t|dr!|j|j_n|j|_||jj | D ]\}}t|dr<|j	n|j}||j q0d S )Nr:   stateqweight)
setattrhasattrr:   r$   r=   toweightdevicenamed_modulesr>   )r   r/   r;   r5   r<   r!   modulerB   r   r   r    r4   x   s   

zLNTuningModel._replace_moduler   c                 C  s.   |  D ]\}}| j|vrd|_qd|_qd S )NFT)named_parametersr   requires_grad)r   r   npr   r   r     _mark_only_adapters_as_trainable   s
   
z.LNTuningModel._mark_only_adapters_as_trainablekeyc                 C  s
   t ||S r7   )r
   )r   r%   rK   r   r   r    _check_target_module_exists   s   
z)LNTuningModel._check_target_module_existsenabledc                 C  s,   | j  D ]}t|ttfr|| qd S r7   )r   modulesr8   r   r   enable_adapters)r   rM   rE   r   r   r    _set_adapter_layers   s
   
z!LNTuningModel._set_adapter_layersc                 C     | j dd dS )zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        TrM   NrP   r   r   r   r    enable_adapter_layers      z#LNTuningModel.enable_adapter_layersc                 C  rQ   )zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        FrR   NrS   rT   r   r   r    disable_adapter_layers   rV   z$LNTuningModel.disable_adapter_layersc                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	d S )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r   rN   r8   r   mergedwarningswarnunmergeset_adapterr2   )r   r   rE   r   r   r    r\      s   



zLNTuningModel.set_adapterTNprogressbar
safe_mergeadapter_namesOptional[list[str]]c              	     s     |  fdd j D }d|rdnd d }t|| |dD ]-}zt j|\}}	}
W n	 ty9   Y q#w t|	drP|rF|	|  ||
|		 |	 q# jS )	Nc                   s   g | ]\}} j |vr|qS r   )r   ).0rK   _rT   r   r    
<listcomp>   s    z>LNTuningModel._unload_and_optionally_merge.<locals>.<listcomp>zUnloading adapters zand merging  r   )disabledescr:   )
_unloading_checksr   rD   r   r	   r#   r@   merger4   get_base_layer)r   rh   r]   r^   r_   key_listrf   rK   r/   r-   r.   r   rT   r    _unload_and_optionally_merge   s   


z*LNTuningModel._unload_and_optionally_mergec                 C     | j ddS )NFrh   rk   rT   r   r   r    unload   s   zLNTuningModel.unload	nn.Modulec                 C  rl   )NTrm   rn   )r   r]   r^   r_   r   r   r    merge_and_unload   s   zLNTuningModel.merge_and_unloadautocast_adapter_dtypec                 C  s   d S r7   r   )r   r   rr   r   r   r    _cast_adapter_dtype   s   z!LNTuningModel._cast_adapter_dtype)F)r   r   r   r   )r!   r   )r%   r   r&   r'   r   r   )r%   r   r   r   r-   r   r.   r   r/   r   r0   r   r   r   )r%   r   r-   r   r   r   r   r   )
r/   r   r;   r   r5   r   r<   r   r   r   )r   r   )r%   r   rK   r   r   r   )rM   r   r   r   )r   r   )r   r   r   r   )TFFN)r]   r   r^   r   r_   r`   )FFN)r]   r   r^   r   r_   r`   r   rp   )T)r   r   rr   r   r   r   )__name__
__module____qualname____doc__r   __annotations__r   r"   staticmethodr,   r6   r1   r4   rJ   rL   rP   rU   rW   r\   rk   ro   rq   rs   __classcell__r   r   r   r    r      s2   
 #

	







r   )
__future__r   rY   typingr   torchr   torch.nn.modulesr   r   peft.configr   peft.tuners.tuners_utilsr   r	   r
   
peft.utilsr   r   layerr   r   r   r   r   r    <module>   s   