o
    8wi8                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZ ddlmZ ddlmZmZ G dd deZdS )    )annotationsN)asdict)Enum)chain)Optional)tqdm)Conv1D)	BaseTunerBaseTunerLayercheck_target_module_exists)7TRANSFORMERS_MODELS_TO_FOURIERFT_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )FourierFTConfig)FourierFTLayerFourierFTLinearc                      s   e Zd ZU dZdZded< d=d> fd
dZd?ddZedd Z	dd Z
dd Zd@ddZedd ZdA fddZd=dBd d!ZdCdDd$d%ZdEd&d'ZdEd(d)ZdFd,d-Zed.d/ Z	"			0dGdHd5d6ZdId7d8Z	0dJdKd9d:ZdLd;d<Z  ZS )MFourierFTModela*  
    Creates FourierFT model from a pretrained transformers model.

    The method is described in detail in https://huggingface.co/papers/2405.03003.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`FourierFTConfig`]): The configuration of the FourierFT model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The FourierFT model.

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`FourierFTConfig`]): The configuration of the Fourier model.
    
fourierft_strprefixFlow_cpu_mem_usageboolreturnNonec                   s   t  j||||d d S )N)r   )super__init__)selfmodelconfigadapter_namer   	__class__ X/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/fourierft/model.pyr   =   s   zFourierFTModel.__init__r   r   c                 C  s2   t | jdkr|jdkrt| jj ddS dS )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueErrorr"   __name__)r   r   r#   r#   r$   _check_new_adapter_config@   s
   	z(FourierFTModel._check_new_adapter_configc                 C  s
   t | |S N)r   )fourierft_configkeyr#   r#   r$   _check_target_module_existsO   s   
z*FourierFTModel._check_target_module_existsc                   s    d u rt dtt|j }tt fdd| }	|j|	|j}
|j	}|j
}t|do4|jd u}|
||j|j|j
d}||d< t|trU|||
||j| d S | j|||fi |}|| jkrj|d | |||| d S )NzCurrent Key shouldn't be `None`c                   s   t d|  d S )Nz.*\.$)rematch)r.   current_keyr#   r$   <lambda>a   s    z4FourierFTModel._create_and_replace.<locals>.<lambda>r(   )n_frequencyscalingfan_in_fan_outinit_weightsrandom_loc_seedF)r)   listr   n_frequency_patternkeysnextfiltergetr6   r7   r:   hasattrr(   r8   r9   
isinstancer   update_layer_create_new_moduleactive_adapterrequires_grad__replace_module)r   r-   r    targettarget_nameparentr4   optional_kwargspattern_keystarget_name_keyr6   r7   r:   r(   kwargs
new_moduler#   r3   r$   _create_and_replaceS   s6   



z"FourierFTModel._create_and_replacec                   s   t ||| t|dr|j}t|ds |j|_t|dr |j|_t|dd d ur>t|dr3|j|j_n|j|_||jj t	d |
 D ]\}}d|v rct fdd| D sc||jj qGd S )N
base_layerr(   statemetar   c                 3  s    | ]}|j  kV  qd S r,   )device).0prS   r#   r$   	<genexpr>   s    z1FourierFTModel._replace_module.<locals>.<genexpr>)setattrrA   rQ   weightr(   getattrrR   torT   torchnamed_modulesany
parameters)r   rJ   
child_namerO   childnamemoduler#   rW   r$   rG   ~   s&   




zFourierFTModel._replace_moduler   torch.nn.Modulec                 C  s   |  D ]\}}| j|vrd|_q| jD ]H}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrU| D ]}t|trSt	|drS|jd urSd|j_q>qt
d| dd S )	NFr%   allr(   Tfourier_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradactive_adaptersr'   r(   modulesrB   r   rA   NotImplementedError)r   r   nrV   rE   r(   mr#   r#   r$    _mark_only_adapters_as_trainable   s,   

z/FourierFTModel._mark_only_adapters_as_trainablec                 K  s   t |tr
| }n|}t |tjjr$|d r#td d |d< | _n"t |t	r>d|d< |d s=td d |d< | _nt
d| dt||fi |}|S )	Nr8   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FTis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module zZ is not supported. Currently, only the following modules are supported: `torch.nn.Linear`.)rB   r
   get_base_layerr]   nnLinearwarningswarnr8   r   r)   r   )r-   r    rH   rN   target_base_layerrO   r#   r#   r$   rD      s.   



z!FourierFTModel._create_new_modulerc   c                   s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r   )r   __getattr__AttributeErrorr[   r   )r   rc   r!   r#   r$   rw      s   zFourierFTModel.__getattr__	inferencec                 C  sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S  s&   i | ]\}}|t |tr|jn|qS r#   )rB   r   value)rU   kvr#   r#   r$   
<dictcomp>   s   & z:FourierFTModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)r'   itemsr   )r   ry   config_dictr.   rz   r   r#   r#   r$   get_peft_config_as_dict   s   z&FourierFTModel.get_peft_config_as_dictTenabledc                 C  s,   | j  D ]}t|ttfr|| qd S r,   )r   rk   rB   r
   r   enable_adapters)r   r   rd   r#   r#   r$   _set_adapter_layers   s
   
z"FourierFTModel._set_adapter_layersc                 C  s   | j dd dS )zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        Tr   N)r   r   r#   r#   r$   enable_adapter_layers   s   z$FourierFTModel.enable_adapter_layersc                 C  sF   | j D ]}| j| j}|dkrd| d}t| q| jdd dS )zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        r%   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   N)rj   r'   r(   rt   ru   r   )r   rE   valmsgr#   r#   r$   disable_adapter_layers   s   


z%FourierFTModel.disable_adapter_layersr    str | list[str]c                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	dS )zSet the active adapter(s).

        Args:
            adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated.
        zJAdapter cannot be set when the model is merged. Unmerging the model first.N)
r   rk   rB   r   mergedrt   ru   unmergeset_adapterrE   )r   r    rd   r#   r#   r$   r      s   



zFourierFTModel.set_adapterc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r)   set)r'   model_configr#   r#   r$   _prepare_adapter_config	  s   

z&FourierFTModel._prepare_adapter_configNprogressbar
safe_mergeadapter_namesOptional[list[str]]c              	     s    fdd j  D }d|rdnd d }t|| |dD ]?}zt j |\}}	}
W n	 ty4   Y qw t|	drN|rC|	j||d	  ||
|	 |	 qt	|	t
r]t||
|	j|	j  q j S )
Nc                   s   g | ]\}} j |vr|qS r#   )r   rU   r.   _r   r#   r$   
<listcomp>  s    z?FourierFTModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r   )disabledescrQ   )r   r   )r   r^   r   r   rx   rA   mergerG   rq   rB   r   rY   modules_to_saverE   )r   r   r   r   r   key_listr   r.   rJ   rH   rI   r#   r   r$   _unload_and_optionally_merge  s    

z+FourierFTModel._unload_and_optionally_mergec                 C  s   |t | j vrtd| d| j|= dd | j D }d}|D ] }t| j|\}}}t|trC|	| |du rC|j
dd }q#|pGg | _
| j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existc                 S  s   g | ]
\}}d |vr|qS )	fourierftr#   r   r#   r#   r$   r   8  s    z1FourierFTModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)r;   r'   r=   r)   r   r^   r   rB   r   delete_adapterrE   _delete_auxiliary_adapter)r   r    r   new_adapterr.   r   rH   r#   r#   r$   r   ,  s   


zFourierFTModel.delete_adapterc                 C  s   | j |||dS )a  
        This method merges the Fourier layers into the base model. This is needed if someone wants to use the base
        model as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        )r   r   r   r   )r   r   r   r   r#   r#   r$   merge_and_unloadD  s   zFourierFTModel.merge_and_unloadc                 C  s   | j ddS )z
        Gets back the base model by removing all the Fourier modules without merging. This gives back the original base
        model.
        F)r   r   r   r#   r#   r$   unloadY  s   zFourierFTModel.unload)F)r   r   r   r   )r   r   r   r   )r   re   r   r   )rc   r   )ry   r   )T)r   r   r   r   )r   r   )r    r   r   r   )TFFN)r   r   r   r   r   r   )r    r   )FFN)r   r   r   r   r   r   r   re   )r   re   )r*   
__module____qualname____doc__r   __annotations__r   r+   staticmethodr/   rP   rG   ro   rD   rw   r   r   r   r   r   r   r   r   r   r   __classcell__r#   r#   r!   r$   r   &   s:   
 

+

		




r   )
__future__r   r1   rt   dataclassesr   enumr   	itertoolsr   typingr   r]   r   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr	   r
   r   
peft.utilsr   r   r   r   r   layerr   r   r   r#   r#   r#   r$   <module>   s   