o 8wi !@sddlZddlmZmZmZddlZddlmZddlmm Z ddl m Z ddl mZmZGdddeZGdddejeZdS) N)AnyOptionalUnion)Conv1D)BaseTunerLayercheck_adapters_to_mergec@sPeZdZdZdZdejddfddZdd Ze d d Z de j fd d Z dS)FourierFTLayer)fourierft_spectrum)fourierft_n_frequencyfourierft_scalingfourierft_random_loc_seed base_layerreturnNcKs||_i|_i|_ti|_i|_i|_d|_g|_ ||_ | }t |tj r3|j|j|_|_dSt |trLt|jdrB|jjn|jj\|_|_dStdt|)NFds_shapezUnsupported layer type )r r r nn ParameterDictr indicesr _disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshape ValueErrortype)selfr rr!X/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/fourierft/layer.py__init__ s    zFourierFTLayer.__init__cCs|dkr td|||j|jkr!td|d|j|j||j|<||j|<tj|j|jt|j|dd||j |<tj |j ||j|j ||jgdd|j |<||j |<t j t|dd|j|<|rv||||||jdS) NrzI`n_frequency` should be a positive integer value but the value passed is zu`n_frequency` should be less than or equal to the product of the input and output dimensions but the value passed is z and the product is ) generator)dimT) requires_grad)rrrr r torchrandperm Generator manual_seedrstackr r Parameterrandnr reset_fourier_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r adapter_name n_frequencyscaling init_weightsrandom_loc_seedr!r!r" update_layer6s6         zFourierFTLayer.update_layercCs(||jvrtj|j|dSdSN)r keysrinitzeros_)r r2r!r!r"r.Qsz'FourierFTLayer.reset_fourier_parameterscCs~|j|}|j||j}tj|j|j|jd}|||dddf|dddff<tj |j |j |}||j S)N)devicer)r rtor<r'zerosrrfloatfftifft2realr dtype)r adapterspectrumrdense_spectrum delta_weightr!r!r"get_delta_weightVs ( zFourierFTLayer.get_delta_weight)__name__ __module__ __qualname__adapter_layer_namesother_param_namesrModuler#r7r'no_gradr.TensorrIr!r!r!r"rs rcseZdZ     ddedededed eeefd ed d ffd d Zddede e ed d fddZ d ddZ d e jffdd Zde jdeded e jfddZd effdd ZZS)!FourierFTLinearb@F r2r3r4fan_in_fan_outr5r6rNc  s@ttj||fi|||_||_||||||dSr8)superr#rrV_active_adapterr7) r r r2r3r4rVr5r6r __class__r!r"r#bs zFourierFTLinear.__init__ safe_merge adapter_namescCst||}|s dS|D]A}||jvrL|}|r;|jj}|||7}t | s6t d|d||j_n |jj||7_|j |q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr r9rrdataclonerIr'isfiniteallrrappend)r r[r\active_adapterr orig_weightsr!r!r"mergess$     zFourierFTLinear.mergecCsj|js tddSt|jdkr3|j}||jvr*|j j | |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) mergedwarningswarnlenrpopr r9rrr]rI)r rbr!r!r"unmerges  zFourierFTLinear.unmergecs t|Sr8)rWrI)r rErYr!r"rIs z FourierFTLinear.get_delta_weightxargsrcOs|j}|jr|jr ||j|g|Ri|}n=|jr*|j|g|Ri|}n-|j|g|Ri|}|jD]}||jvrCq9||}| |j}|t ||}q9| |}|Sr8) rDdisable_adaptersrerjr r1r r9rIr>Flinear)r rkrlrprevious_dtyperesultrbdelta_wr!r!r"forwards     zFourierFTLinear.forwardcst}d|S)Nz fourierft.)rW__repr__)r reprYr!r"rts zFourierFTLinear.__repr__)rSrTFFrU)FN)rN)rJrKrLstrintr@boolrr#rlistrdrjr'rQrIrrsrt __classcell__r!r!rYr"rR`s4    % rR)rftypingrrrr'torch.nnrtorch.nn.functional functionalrntransformers.pytorch_utilsrpeft.tuners.tuners_utilsrrrrOrRr!r!r!r"s  F