o Ni2@sddlZddlmZmZmZddlZddlmZddlmm Z ddl m Z ddl mZmZddlmZddlmZGdd d eZGd d d ejeZdS) N)AnyOptionalUnion)Conv1D)BaseTunerLayercheck_adapters_to_merge)WAVELET_REDUCTIONS) waverec2dc@sTeZdZdZdZdejddfddZ dd d Ze d d Z de j fddZ dS) WaveFTLayer)waveft_spectrum)waveft_n_frequencywaveft_scalingwaveft_random_loc_seedwaveft_wavelet_familywaveft_indiceswaveft_use_idwt base_layerreturnNcKs||_i|_i|_ti|_i|_i|_i|_i|_ d|_ g|_ ||_ | }t|tjr9|j|j|_|_dSt|trRt|jdrH|jjn|jj\|_|_dStdt|)NFds_shapezUnsupported layer type )rr rnn ParameterDictr rrrr_disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshape ValueErrortype)selfrrr&L/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/waveft/layer.py__init__*s$   zWaveFTLayer.__init__db1Tc Cs<|dkr td|||j|jkr!td|d|j|j||j|<||j|<||j|<||j|<t|\}} t |j|} tj |j|j| dd|} tj | |j| |jgdd|j |<||j|<|rtjt|dd|j|<||nd } tjt|| dd|j|<||||jdS) NrzI`n_frequency` should be a positive integer value but the value passed is zu`n_frequency` should be less than or equal to the product of the input and output dimensions but the value passed is z and the product is ) generator)dimT) requires_gradg{Gz?)r#rrr rrrr torch Generator manual_seedrandpermstackrrr Parameteremptyr reset_wave_parametersrandn%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) r% adapter_name n_frequencyscaling init_weightsrandom_loc_seedwavelet_familyuse_idwtreduction_rowsreduction_colsr*indicesstd_devr&r&r' update_layerBs8         zWaveFTLayer.update_layercCs(||jvrtj|j|dSdSN)r keysrinitzeros_)r%r9r&r&r'r4osz!WaveFTLayer.reset_wave_parameterscCs|j|}|j||j}|j|}|j|r"t|\}}|j|}|j|}|ddkr3|d7}|ddkr=|d7}t j |||j|j d} ||jd} ||jd} | } | dddf| 7<| dddf| 7<| dddf|k| dddf|k@} | dd| f}|| }|| |dddf|dddff<| j \}}|d|d}}| d|d|f}| d||df}| |dd|f}| |d|df}||||ff}t|||j|}|j d|jks|j d|jkr |j d|jd}|j d|jd}||||j|||jf}|St j |j|j|j|j d} || |dddf|dddff<| |j|}|S)Nrr)devicedtype)r rtorJrrr rrr-zerosrKcloner"r r)r%adapterspectrumrBr>r@rApadded_out_featurespadded_in_featuresdense_spectrum row_offset col_offsetpadded_indices valid_mask valid_indicesvalid_spectrumHWH2W2cAcHcVcDcoeffs delta_weight start_row start_colr&r&r'get_delta_weighttsX         ($ " $zWaveFTLayer.get_delta_weight)r)T)__name__ __module__ __qualname__adapter_layer_namesother_param_namesrModuler(rDr-no_gradr4Tensorrfr&r&r&r'r s  - r cseZdZ       d"deded ed ed eeefd ed ededdffdd Zd#dede e eddfddZ d$ddZ de jffdd Zde jdedede jfddZdeffd d! ZZS)% WaveFTLinearb@F r)Tr9r:r;fan_in_fan_outr<r=r>r?rNc sDttj||fi| ||_||_|||||||| dSrE)superr(r rs_active_adapterrD) r%rr9r:r;rsr<r=r>r?r __class__r&r'r(s zWaveFTLinear.__init__ safe_merge adapter_namescCst||}|s dS|D]A}||jvrL|}|r;|jj}|||7}t | s6t d|d||j_n |jj||7_|j |q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr rFrr!datarNrfr-isfiniteallr#rappend)r%rxryactive_adapterr orig_weightsr&r&r'merges$     zWaveFTLinear.mergecCsj|js tddSt|jdkr3|j}||jvr*|j j | |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) mergedwarningswarnlenrpopr rFrr!rzrf)r%r~r&r&r'unmerges  zWaveFTLinear.unmergecs t|SrE)rtrf)r%rOrvr&r'rfs zWaveFTLinear.get_delta_weightxargsrcOs|j}|jr|jr ||j|g|Ri|}n>|jr*|j|g|Ri|}n.|j|g|Ri|}|jD]}||jvrCq9||}| ||j}|t ||}q9| |}|SrE) rKdisable_adaptersrrrr8r rFrf_cast_input_dtypeFlinearrL)r%rrrprevious_dtyperesultr~delta_wr&r&r'forward s    zWaveFTLinear.forwardcst}d|S)Nzwaveft.)rt__repr__)r%reprvr&r'r!s zWaveFTLinear.__repr__)rprqFFrrr)T)FN)rN)rgrhristrintfloatboolrr(rlistrrr-rnrfrrr __classcell__r&r&rvr'ros@      % ro)rtypingrrrr-torch.nnrtorch.nn.functional functionalrtransformers.pytorch_utilsrpeft.tuners.tuners_utilsrr constantsr r r rlror&r&r&r's    &