o Ni)<@sddlZddlmZddlZddlmZddlmmZddl m Z ddl m Z m Z ddlmZddlmZGdd d ejjZGd d d e ZGd d d ejeZdS)N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transpose) BufferDictc@s$eZdZeddZeddZdS)UniqueBaseGradcCs4|dddddf||d}|||||S)NN)save_for_backward)ctx randlora_Arandlora_lambdarandlora_gammaoutrN/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/randlora/layer.pyforwards"zUniqueBaseGrad.forwardcCs^|j\}}}||j||j||j}}}td|||}td|||}d||fS)Nzkbj,kvj,bj->kbzkbj,kvj,kb->bj) saved_tensorstodtypetorcheinsum)r grad_outputr rrgrad_randlora_lambdagrad_randlora_gammarrrbackward%s      zUniqueBaseGrad.backwardN)__name__ __module__ __qualname__ staticmethodrrrrrrr s  r c@sXeZdZdZdZdejfddZede fddZ dd e d e d e fd dZ ddZ dS) RandLoraLayer)rr)r randlora_B base_layercKs||_i|_i|_ti|_ti|_ti|_d|_ d|_ d|_ g|_ d|_ |}t|tjr<|j|j}}nt|trQt|jdrK|jjn|jj\}}||_||_||_dS)NFTds_shape)r#rscalingnn ModuleDictrandlora_dropout ParameterDictrrr r"_disable_adaptersmerged_adapterscast_input_dtype_enabledget_base_layer isinstanceLinear in_features out_featuresrhasattrweightr$shapekwargs)selfr#r6r1r2rrr__init__7s(      zRandLoraLayer.__init__returncCs t|jSr )boolr,)r7rrrmergedYs zRandLoraLayer.mergedFr r"inference_modec KsP|dkr td|||j|<|dkrtj|d} nt} |jt|| it|j |j |} | r;t | nt | d|_ tjt||j dd|j|<tjt|j t|j |j t|j |j dd|j|<|||j|<||_||_||vrt|jdkrtdt|jd} t|jd} d }t|j |j t|j |j }}| jd|krt|d | jd|| jd |krt|d | jd|d }| jd|j|krt|d | jd|j|| jd |j|krt|d | jd |j|| |j|<| |j|<|r|||||j|j |ddS)Nrz?`r` should be a positive integer value but the value passed is )pT) requires_gradzfThe `randlora_A` and `randlora_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter was added after the first one with incompatible shapes.r"r z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.)r<)! ValueErrorr%r'DropoutIdentityr)updater(minr1r2 is_integerint num_bases Parameterrrandnronesmaxrr&r r"lenlistvaluesr5formatreset_randlora_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r7 adapter_namer r"r%randlora_alphar) init_weightsr<r6randlora_dropout_layerrIrandlora_A_paramrandlora_B_param error_tmplmax_dimmin_dimrrr update_layer]sZ         zRandLoraLayer.update_layercCsv||jvr9t$tj|j|tj|j|dt |j|j WddS1s2wYdSdS)Nr?) rkeysrno_gradr'initzeros_ constant_rrMr5)r7rVrrrrRs  &"z'RandLoraLayer.reset_randlora_parametersN)F)rrradapter_layer_namesother_param_namesr'Moduler8propertyr:r;rr_rRrrrrr!2s"  Pr!cseZdZ      d!dedededed ed ed ed ed eddffdd Zd"dede e eddfddZ d#ddZ d$de ejejffddZdejfddZdejdejfddZdeffdd ZZS)%r0rr=FTr r"rVr%rWr)fan_in_fan_outis_target_conv_1d_layerrXr9Nc sPttj|tj||fi| ||_||_|||||||| | |_dSr ) superr'r0r8r!ri_active_adapterr_rj) r7r#r r"rVr%rWr)rirjrXr6 __class__rrr8s  zLinear.__init__ safe_merge adapter_namescCst||}|s dS|D]M}||jvrX|}|jj}|rB|jj}|||7}t | s:t d|d| ||j_n||}|jj| |7_|j|q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rrr`r.r4rdatacloneget_delta_weightrisfiniteallrBrr,append)r7rorpactive_adapterr# orig_dtype orig_weights delta_weightrrrmerges(     z Linear.mergecCs|js tddSt|jdkr>|}|jj}|j}||j vr5| |}|jj | |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr)r;warningswarnrNr,r.r4rpoprr`rsrqr)r7r#rxrwrzrrrunmerges   zLinear.unmergecCs,|j|}|j|}|dur|j}|j}|jdko"|tjkp"|tjk}|j| |}|j | |}|rE| }| }| }| }t |j |jt|j |j} } |ddd|jd| f |} |d| d|jddf |} | jdd} t| ||jdd}| |jkr|| fS| j|jfS)a4 Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the correct order to fit the target layers' dimensions Args: adapter (str): The name of the adapter for which the delta weight should be computed. Ncpur?) start_dim)end_dim)r r"devicertyperfloat16bfloat16rrrfloatrFr2r1rMrIflattenr applyT)r7adapterrr r"r cast_to_fp32rrr^r]sliced_Asliced_Bupdate_Bupdate_Arrrget_scaled_basess* ""   zLinear.get_scaled_basescCs:||\}}|j|jj}t||j}|j|}||S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. )rrrrir&)r7rrrrE output_tensorr&rrrrs6s    zLinear.get_delta_weightxc Os|j}|jr|jr ||j|g|Ri|}nT|jr*|j|g|Ri|}nD|j|g|Ri|}|jD]4}||jvrCq9|j|}|j ||j d\}} | | j}|j |} |t t |||| | }q9| |}|S)N)r)rdisable_adaptersr;rr#rUrr`r)rrrr&Flinear) r7rargsr6previous_dtyperesultrwdropoutrrr&rrrrGs$    " zLinear.forwardcst}d|S)Nz randlora.)rk__repr__)r7reprmrrr\s zLinear.__repr__)rrr=FFT)FN)r9Nr )rrrrstrrHrr:r8rrOr{rtuplerTensorrrsrr __classcell__rrrmrr0sD      *0r0)r|typingrrtorch.nnr'torch.nn.functional functionalrtransformers.pytorch_utilsrpeft.tuners.tuners_utilsrrpeft.utils.otherr _buffer_dictrautogradFunctionr r!r0rrrrs