o
    Ni                      @  sx   d dl mZ d dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
mZ d dlmZ ddlmZmZ G d	d
 d
eZdS )    )annotationsN)Conv1D)	BaseTunerBaseTunerLayercheck_target_module_exists)4TRANSFORMERS_MODELS_TO_WAVEFT_TARGET_MODULES_MAPPING)get_pattern_key   )WaveFTLayerWaveFTLinearc                      sZ   e Zd ZU dZded< eZded< eZddd	Z	d
d Z
edd Zd fddZ  ZS )WaveFTModelwaveft_strprefixztype[BaseTunerLayer]tuner_layer_clsmodeltorch.nn.Modulec                 C  s4  g }|  D ]_\}}t||ret|tr:|j}t|tjjr&|j|j	}}n7t|t
r9|jjd |jjd }}n$qt|tjjrI|j|j	}}nt|t
r\|jjd |jjd }}nq||||f q|sltdtdd |D }	t|}
|j|
 }i }|D ]\}}}|| |	 }t|| }|||< q|S )zCCalculate proportional parameter allocation for all target modules.r	   r   z>No target modules found for proportional parameter allocation.c                 s  s    | ]
\}}}|| V  qd S )N ).0_	input_dim
output_dimr   r   L/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/waveft/model.py	<genexpr>=   s    zAWaveFTModel._calculate_proportional_parameters.<locals>.<genexpr>)named_modulesr   
isinstancer
   
base_layertorchnnLinearin_featuresout_featuresr   weightshapeappend
ValueErrorsumlenn_frequencyround)selfr   waveft_configtarget_modules_infonamemodulebase_moduler   r   	total_sum
num_layerstotal_budgetn_frequency_dictlayer_ration_freqr   r   r   "_calculate_proportional_parameters#   s8   





z.WaveFTModel._calculate_proportional_parametersc              	   K  s  |d u rt d|jr$t| dsi | _|| jvr$| | j|}|| j|< d }	|jr;t| dr;|| jv r;| j| |}	|	d u rGd|v rG|d }	|	d u r_t|j	 }
t
|
|}|j||j}	d }d|v ri|d }|d u rp|j}|j}|j}t|do|jd u}|	||j|j|j|d}||d< t|tr|j||	||j|||jd d S | j|||fi |}|| jkr|d | |||| d S )	NzCurrent Key shouldn't be `None`_proportional_params_cacher(   wavelet_familybias)r(   scalingfan_in_fan_outinit_weightsrandom_loc_seedr8   )r8   use_idwtF)r%   proportional_parametershasattrr7   r6   r   getlistn_frequency_patternkeysr   r(   r8   r:   r=   r9   r;   r<   r   r
   update_layerr>   _create_new_moduleactive_adapterrequires_grad__replace_module)r*   r+   adapter_nametargettarget_nameparentcurrent_keyoptional_kwargsr3   r(   pattern_keystarget_name_keyr8   r:   r=   r9   kwargs
new_moduler   r   r   _create_and_replaceI   sf   










zWaveFTModel._create_and_replacec                 K  s   t |tr
| }n|}t |tjjr$|d r#td d |d< | _n"t |t	r>d|d< |d s=td d |d< | _nt
d| d| j|d	< | j|d
< t||fi |}|S )Nr;   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FTis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module zZ is not supported. Currently, only the following modules are supported: `torch.nn.Linear`.r8   r>   )r   r   get_base_layerr   r   r   warningswarnr;   r   r%   r8   r>   r   )r+   rJ   rK   rR   target_base_layerrS   r   r   r   rF      s2   





zWaveFTModel._create_new_modulerJ   returnNonec                   s4   t  | t| dr|| jv r| j|= dS dS dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        r7   N)superdelete_adapterr@   r7   )r*   rJ   	__class__r   r   r]      s   zWaveFTModel.delete_adapter)r   r   )rJ   r   rZ   r[   )__name__
__module____qualname__r   __annotations__r
   r   r   target_module_mappingr6   rT   staticmethodrF   r]   __classcell__r   r   r^   r   r      s   
 
&O
 r   )
__future__r   rW   r   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr   r   r   
peft.utilsr   peft.utils.otherr   layerr
   r   r   r   r   r   r   <module>   s   