o
    8wi9                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dl	m	Z	 d dl
mZmZmZmZ d dlmZmZmZ d	d
lmZ d	dlmZmZmZ G dd deZdS )    N)asdict)Enum)Optional)nn)tqdm)	BaseTunerBaseTunerLayercheck_target_module_existsonload_layer)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )
BOFTConfig)	BOFTLayerConv2dLinearc                	       sN  e Zd ZU dZdZeed< d5deddf fdd	Zd
e	ddfddZ
edd Zdd Zdd ZdejddfddZedd Zdef fddZd5defddZd6dd Zd!d" Zd#d$ Zd%d& Zed'd( Z				d7d)ed*ed+eee  fd,d-Zd.eddfd/d0Z	d8d)ed*ed+eee  dejjfd1d2Zdejjfd3d4Z   Z!S )9	BOFTModela  
    Creates BOFT and OFT model from a pretrained transformers model. Paper: https://huggingface.co/papers/2311.06243
    https://huggingface.co/papers/2306.07280

    Args:
        model ([`transformers.PreTrainedModel`]): The model to be adapted.
        config ([`BOFTConfig`]): The configuration of the BOFT model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The BOFT model.

    Example::

        >>> import transformers >>> from transformers import AutoModelForSeq2SeqLM, BOFTConfig >>> from peft import
        BOFTConfig, get_peft_model

        >>> config = BOFTConfig( ... boft_block_size=8, ... boft_n_butterfly_factor=1, ... target_modules=["query",
        "value", "key", "output.dense", "mlp.fc1", "mlp.fc2"], ... boft_dropout=0.1, ... bias="boft_only", ...
        modules_to_save=["classifier"], ... )

        >>> model = transformers.Dinov2ForImageClassification.from_pretrained( ... "facebook/dinov2-large", ...
        num_labels=100, ... ) >>> boft_model = get_peft_model(model, config)

    **Attributes**:
        - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`BOFTConfig`]): The configuration of the BOFT model.
    boft_prefixFlow_cpu_mem_usagereturnNc                    s   t  j||||d d S )N)r   )super__init__)selfmodelconfigadapter_namer   	__class__ S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/boft/model.pyr   M   s   zBOFTModel.__init__r   c                 C   s2   t | jdkr|jdkrt| jj ddS dS )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueErrorr   __name__)r   r   r    r    r!   _check_new_adapter_configP   s
   	z#BOFTModel._check_new_adapter_configc                 C   s
   t | |S N)r	   )boft_configkeyr    r    r!   _check_target_module_exists_   s   
z%BOFTModel._check_target_module_existsc                 K   s   |d u rt dt|do|jd u}|j|j|j|j|j|jd}	||	d< t	|t
sI| j|||fi |	}
|| jvr?|
d | |||
| d S |j||j|j|j|j|jd d S )NzCurrent Key shouldn't be `None`r%   )boft_block_sizeboft_block_numboft_n_butterfly_factorboft_dropoutfan_in_fan_outinit_weightsF)r-   r.   r/   r0   r2   )r&   hasattrr%   r-   r.   r/   r0   r1   r2   
isinstancer   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)r   r*   r   targettarget_nameparentcurrent_keyoptional_kwargsr%   kwargs
new_moduler    r    r!   _create_and_replacec   s0   




zBOFTModel._create_and_replacec                    s   t ||| t|dr|j}t|ds |j|_t|dr |j|_t|dd d ur>t|dr3|j|j_n|j|_||jj t	d |
 D ]\}}| j|v rdt fdd| D sd||jj qGd S )N
base_layerr%   statemetac                 3   s    | ]}|j  kV  qd S r)   )device).0prD   r    r!   	<genexpr>   s    z,BOFTModel._replace_module.<locals>.<genexpr>)setattrr3   rB   weightr%   getattrrC   torE   torchnamed_modulesr   any
parameters)r   r<   
child_namer@   childnamemoduler    rH   r!   r8      s&   





zBOFTModel._replace_moduler   c                 C   s   |  D ]\}}| j|vrd|_q| jD ]J}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrW| D ]\}}t|trUt	|drU|jd urUd|j_q>qt
d| dd S )	NFr"   allr%   T	boft_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradr6   r$   r%   rO   r4   r   r3   NotImplementedError)r   r   nrG   active_adapterr%   rT   mr    r    r!    _mark_only_adapters_as_trainable   s,   

z*BOFTModel._mark_only_adapters_as_trainablec                 K   s   t |tr
| }n|}t |tjjr.|d r#td d |d< | _t||fi |}|S t |tjj	r@t	||fi |}|S t
d| d)Nr1   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FzTarget module zY is not supported. Currently, only `torch.nn.Linear` and `torch.nn.Conv2d` are supported.)r4   r   get_base_layerrN   r   r   warningswarnr1   r   r&   )r*   r   r:   r?   target_base_layerr@   r    r    r!   r5      s"   

	
zBOFTModel._create_new_modulerT   c                    s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r   )r   __getattr__AttributeErrorrL   r   )r   rT   r   r    r!   rc      s   zBOFTModel.__getattr__	inferencec                 C   sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S   s&   i | ]\}}|t |tr|jn|qS r    )r4   r   value)rF   kvr    r    r!   
<dictcomp>   s   & z5BOFTModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)r$   itemsr   )r   re   config_dictr+   rf   r   r    r    r!   get_peft_config_as_dict   s   z!BOFTModel.get_peft_config_as_dictTc                 C   s,   | j  D ]}t|ttfr|| qd S r)   )r   modulesr4   r   r   enable_adapters)r   enabledrU   r    r    r!   _set_adapter_layers   s
   
zBOFTModel._set_adapter_layersc                 C   s   | j dd d S )NTrp   )rq   r   r    r    r!   enable_adapter_layers   s   zBOFTModel.enable_adapter_layersc                 C   sF   | j D ]}| j| j}|dkrd| d}t| q| jdd d S )Nr"   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Frr   )r6   r$   r%   r`   ra   rq   )r   r\   valmsgr    r    r!   disable_adapter_layers   s   


z BOFTModel.disable_adapter_layersc                 C   sF   | j  D ]}t|tr|jrtd |  || q|| _	d S )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r   rn   r4   r   mergedr`   ra   unmergeset_adapterr\   )r   r   rU   r    r    r!   rz      s   



zBOFTModel.set_adapterc                 C   s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r&   set)r$   model_configr    r    r!   _prepare_adapter_config  s   

z!BOFTModel._prepare_adapter_configprogressbar
safe_mergeadapter_namesc              	      s  |r     fdd j D }d|rdnd d }t|| |dD ]g}zt j|\}}	}
W n	 ty:   Y q$w t|	D t|	drY|rN|	j||d	  	||
|	
 |	 n#t|	tr||	j|	j }t|drv|rr|j||d	 |
 }t||
| W d    n1 sw   Y  q$ jS )
Nc                       g | ]\}} j |vr|qS r    r   rF   r+   _rs   r    r!   
<listcomp>      z:BOFTModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r   )disabledescrB   )r   r   )_check_merge_allowedr   rO   r   r   rd   r
   r3   merger8   r_   r4   r   modules_to_saver\   rJ   )r   r   r   r   r   key_listr   r+   r<   r:   r;   r@   r    rs   r!   _unload_and_optionally_merge  s4   



z&BOFTModel._unload_and_optionally_merger   c                    s   |t  j vrtd| d j|=  fdd j D }d}|D ] }t j|\}}}t|trE|	| |du rE|j
dd }q%|pIg  _ j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existc                    r   r    r   r   rs   r    r!   r   ;  r   z,BOFTModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)listr$   keysr&   r   rO   r   r4   r   delete_adapterr6   r\   _delete_auxiliary_adapter)r   r   r   new_adapterr+   r   r:   r    rs   r!   r   0  s   


zBOFTModel.delete_adapterc                 C   s   | j |||dS )a  
        This method merges the BOFT layers into the base model. This is needed if someone wants to use the base model
        as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        )r   r   r   r   )r   r   r   r   r    r    r!   merge_and_unloadG  s   zBOFTModel.merge_and_unloadc                 C   s   | j ddS )z
        Gets back the base model by removing all the boft modules without merging. This gives back the original base
        model.
        F)r   r   rs   r    r    r!   unload]  s   zBOFTModel.unload)F)T)TFFN)FFN)"r'   
__module____qualname____doc__r   str__annotations__boolr   r   r(   staticmethodr,   rA   r8   r   Moduler^   r5   rc   rm   rq   rt   rw   rz   r   r   r   r   r   rN   r   r   __classcell__r    r    r   r!   r   +   sV   
 
)
	
		


"

r   )r`   dataclassesr   enumr   typingr   rN   r   r   peft.tuners.tuners_utilsr   r   r	   r
   
peft.utilsr   r   r   r   r   layerr   r   r   r   r    r    r    r!   <module>   s   