o
    8wi4                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dl	m	Z	 d dl
mZmZmZ d dlmZmZmZ d	d
lmZ d	dlmZmZmZ G dd deZdS )    N)asdict)Enum)Optional)nn)tqdm)	BaseTunerBaseTunerLayercheck_target_module_exists)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )	HRAConfig)	HRAConv2dHRALayer	HRALinearc                	       s6  e Zd ZU dZdZeed< deddfddZe	d	d
 Z
dd Zdd ZdejddfddZe	dd Zdef fddZd2defddZd3ddZdd Zd d! Zd"d# Ze	d$d% Z				d4d&ed'ed(eee  fd)d*Zd+eddfd,d-Z	d5d&ed'ed(eee  dejjfd.d/Zdejjfd0d1Z  Z S )6HRAModela  
    Creates Householder reflection adaptation (HRA) model from a pretrained model. The method is described in
    https://huggingface.co/papers/2405.17484

    Args:
        model (`torch.nn.Module`): The model to which the adapter tuner layers will be attached.
        config ([`HRAConfig`]): The configuration of the HRA model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The HRA model.

    Example:
        ```py
        >>> from diffusers import StableDiffusionPipeline
        >>> from peft import HRAModel, HRAConfig

        >>> config_te = HRAConfig(
        ...     r=8,
        ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
        ...     init_weights=True,
        ... )
        >>> config_unet = HRAConfig(
        ...     r=8,
        ...     target_modules=[
        ...         "proj_in",
        ...         "proj_out",
        ...         "to_k",
        ...         "to_q",
        ...         "to_v",
        ...         "to_out.0",
        ...         "ff.net.0.proj",
        ...         "ff.net.2",
        ...     ],
        ...     init_weights=True,
        ... )

        >>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
        >>> model.text_encoder = HRAModel(model.text_encoder, config_te, "default")
        >>> model.unet = HRAModel(model.unet, config_unet, "default")
        ```

    **Attributes**:
        - **model** ([`~torch.nn.Module`]) -- The model to be adapted.
        - **peft_config** ([`HRAConfig`]): The configuration of the HRA model.
    hra_prefixconfigreturnNc                 C   s2   t | jdkr|jdkrt| jj ddS dS )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueError	__class____name__)selfr    r   R/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/hra/model.py_check_new_adapter_configW   s
   	z"HRAModel._check_new_adapter_configc                 C   s
   t | |S N)r	   )
hra_configkeyr   r   r    _check_target_module_existsf   s   
z$HRAModel._check_target_module_existsc                 K   s   |d u rt dt|do|jd u}|j|j|jd}	||	d< t|tsC| j|||fi |	}
|| j	vr9|

d | |||
| d S |j||j|j|jd d S )NzCurrent Key shouldn't be `None`r   )rapply_GSinit_weightsF)r   hasattrr   r&   r'   r(   
isinstancer   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)r   r#   adapter_nametargettarget_nameparentcurrent_keyoptional_kwargsr   kwargs
new_moduler   r   r    _create_and_replacej   s&   




zHRAModel._create_and_replacec                    s   t ||| t|dr|j}t|ds |j|_t|dr |j|_t|dd d ur>t|dr3|j|j_n|j|_||jj t	d |
 D ]\}}| j|v rdt fdd| D sd||jj qGd S )N
base_layerr   statemetac                 3   s    | ]}|j  kV  qd S r"   )device).0pr;   r   r    	<genexpr>   s    z+HRAModel._replace_module.<locals>.<genexpr>)setattrr)   r9   weightr   getattrr:   tor<   torchnamed_modulesr   any
parameters)r   r3   
child_namer7   childnamemoduler   r?   r    r.      s&   





zHRAModel._replace_modulemodelc                 C   s   |  D ]\}}| j|vrd|_q| jD ]J}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrW| D ]\}}t|trUt	|drU|jd urUd|j_q>qt
d| dd S )	NFr   allr   Thra_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradr,   r   r   rF   r*   r   r)   NotImplementedError)r   rM   nr>   active_adapterr   rK   mr   r   r     _mark_only_adapters_as_trainable   s,   

z)HRAModel._mark_only_adapters_as_trainablec                 K   sp   t |tr
| }n|}t |tjjrt||fi |}|S t |tjjr0t||fi |}|S t	d| d)NzTarget module zY is not supported. Currently, only `torch.nn.Linear` and `torch.nn.Conv2d` are supported.)
r*   r   get_base_layerrE   r   Linearr   Conv2dr   r   )r#   r0   r1   r6   target_base_layerr7   r   r   r    r+      s   

	
zHRAModel._create_new_modulerK   c                    s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.
base_model)super__getattr__AttributeErrorrC   rM   )r   rK   r   r   r    r]      s   zHRAModel.__getattr__F	inferencec                 C   sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S   s&   i | ]\}}|t |tr|jn|qS r   )r*   r   value)r=   kvr   r   r    
<dictcomp>   s   & z4HRAModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)r   itemsr   )r   r`   config_dictr$   ra   r   r   r   r    get_peft_config_as_dict   s   z HRAModel.get_peft_config_as_dictTc                 C   s,   | j  D ]}t|ttfr|| qd S r"   )rM   modulesr*   r   r   enable_adapters)r   enabledrL   r   r   r    _set_adapter_layers   s
   
zHRAModel._set_adapter_layersc                 C   s   | j dd d S )NTrk   )rl   r   r   r   r    enable_adapter_layers   s   zHRAModel.enable_adapter_layersc                 C   sF   | j D ]}| j| j}|dkrd| d}t| q| jdd d S )Nr   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Frm   )r,   r   r   warningswarnrl   )r   rT   valmsgr   r   r    disable_adapter_layers   s   


zHRAModel.disable_adapter_layersc                 C   sF   | j  D ]}t|tr|jrtd |  || q|| _	d S )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
rM   ri   r*   r   mergedrp   rq   unmergeset_adapterrT   )r   r0   rL   r   r   r    rw      s   



zHRAModel.set_adapterc                 C   s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr
   r   set)r   model_configr   r   r    _prepare_adapter_config   s   

z HRAModel._prepare_adapter_configprogressbar
safe_mergeadapter_namesc              	      s     |  fdd j D }d|rdnd d }t|| |dD ]?}zt j|\}}	}
W n	 ty9   Y q#w t|	drS|rH|	j||d	  ||
|		 |	 q#t
|	trbt||
|	j|	j  q# jS )
Nc                       g | ]\}} j |vr|qS r   r   r=   r$   _rn   r   r    
<listcomp>      z9HRAModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  rM   )disabledescr9   )r~   r   )_unloading_checksrM   rF   r   r   r^   r)   merger.   rW   r*   r   rA   modules_to_saverT   )r   r   r}   r~   r   key_listr   r$   r3   r1   r2   r   rn   r    _unload_and_optionally_merge
  s"   


z%HRAModel._unload_and_optionally_merger0   c                    s   |t  j vrtd| d j|=  fdd j D }d}|D ] }t j|\}}}t|trE|	| |du rE|j
dd }q%|pIg  _ j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existc                    r   r   r   r   rn   r   r    r   /  r   z+HRAModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)listr   keysr   rM   rF   r   r*   r   delete_adapterr,   rT   _delete_auxiliary_adapter)r   r0   r   new_adapterr$   r   r1   r   rn   r    r   $  s   


zHRAModel.delete_adapterc                 C   s   | j |||dS )a  
        This method merges the HRA layers into the base model. This is needed if someone wants to use the base model as
        a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        )r}   r~   r   r   )r   r}   r~   r   r   r   r    merge_and_unload;  s   zHRAModel.merge_and_unloadc                 C   s   | j ddS )z
        Gets back the base model by removing all the hra modules without merging. This gives back the original base
        model.
        F)r   r   rn   r   r   r    unloadQ  s   zHRAModel.unload)F)T)TFFN)FFN)!r   
__module____qualname____doc__r   str__annotations__r   r!   staticmethodr%   r8   r.   r   ModulerV   r+   r]   boolrh   rl   ro   rt   rw   r|   r   r   r   r   rE   r   r   __classcell__r   r   r_   r    r   #   sT   
 1
$
	
		




r   )rp   dataclassesr   enumr   typingr   rE   r   r   peft.tuners.tuners_utilsr   r   r	   
peft.utilsr
   r   r   r   r   layerr   r   r   r   r   r   r   r    <module>   s   