o 8wi;@sddlZddlmZddlZddlmZddlmmZddl m Z ddl m Z m Z ddlmZddlmZGdd d ejjZGd d d e ZGd d d ejeZdS)N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transpose) BufferDictc@s$eZdZeddZeddZdS)UniqueBaseGradcCs4|dddddf||d}|||||S)NN)save_for_backward)ctx randlora_Arandlora_lambdarandlora_gammaoutrW/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/randlora/layer.pyforwards"zUniqueBaseGrad.forwardcCs^|j\}}}||j||j||j}}}td|||}td|||}d||fS)Nzkbj,kvj,bj->kbzkbj,kvj,kb->bj) saved_tensorstodtypetorcheinsum)r grad_outputr rrgrad_randlora_lambdagrad_randlora_gammarrrbackward%s      zUniqueBaseGrad.backwardN)__name__ __module__ __qualname__ staticmethodrrrrrrr s  r c@sPeZdZdZdZdejfddZede fddZ d e d e fd d Z d dZ dS) RandLoraLayer)rr)r randlora_B base_layercKs||_i|_i|_ti|_ti|_ti|_d|_ d|_ d|_ g|_ d|_ |}t|tjr<|j|j}}nt|trQt|jdrK|jjn|jj\}}||_||_||_dS)NFTds_shape)r#rscalingnn ModuleDictrandlora_dropout ParameterDictrrr r"_disable_adaptersmerged_adapterscast_input_dtype_enabledget_base_layer isinstanceLinear in_features out_featuresrhasattrweightr$shapekwargs)selfr#r6r1r2rrr__init__7s(      zRandLoraLayer.__init__returncCs t|jSr )boolr,)r7rrrmergedYs zRandLoraLayer.mergedr r"cCsL|dkr td|||j|<|dkrtj|d}nt}|jt||it|j |j |} | r;t | nt | d|_ tjt||j dd|j|<tjt|j t|j |j t|j |j dd|j|<|||j|<||_||_||vrt|jdkrtdt|jd} t|jd} d } t|j |j t|j |j } }| jd| krt| d | jd| | jd |krt| d | jd|d } | jd|j|krt| d | jd|j|| jd |j|krt| d | jd |j|| |j|<| |j|<|r||||||j dS)Nrz?`r` should be a positive integer value but the value passed is )pT) requires_gradzfThe `randlora_A` and `randlora_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter was added after the first one with incompatible shapes.r"r z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.)! ValueErrorr%r'DropoutIdentityr)updater(minr1r2 is_integerint num_bases Parameterrrandnronesmaxrr&r r"lenlistvaluesr5formatreset_randlora_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r7 adapter_namer r"r%randlora_alphar) init_weightsrandlora_dropout_layerrHrandlora_A_paramrandlora_B_param error_tmplmax_dimmin_dimrrr update_layer]sZ         zRandLoraLayer.update_layercCsv||jvr9t$tj|j|tj|j|dt |j|j WddS1s2wYdSdS)Nr>) rkeysrno_gradr'initzeros_ constant_rrLr5)r7rUrrrrQs  &"z'RandLoraLayer.reset_randlora_parametersN)rrradapter_layer_namesother_param_namesr'Moduler8propertyr:r;rr^rQrrrrr!2s" Nr!cseZdZ      d!dedededed ed ed ed ed eddffdd Zd"dede e eddfddZ d#ddZ d$de ejejffddZdejfddZdejdejfddZdeffdd ZZS)%r0rr<FTr r"rUr%rVr)fan_in_fan_outis_target_conv_1d_layerrWr9Nc sPttj|tj||fi| ||_||_|||||||| | |_dSr ) superr'r0r8r!rh_active_adapterr^ri) r7r#r r"rUr%rVr)rhrirWr6 __class__rrr8s  zLinear.__init__ safe_merge adapter_namescCst||}|s dS|D]M}||jvrX|}|jj}|rB|jj}|||7}t | s:t d|d| ||j_n||}|jj| |7_|j|q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rrr_r.r4rdatacloneget_delta_weightrisfiniteallrArr,append)r7rnroactive_adapterr# orig_dtype orig_weights delta_weightrrrmerges(     z Linear.mergecCs|js tddSt|jdkr>|}|jj}|j}||j vr5| |}|jj | |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr)r;warningswarnrMr,r.r4rpoprr_rrrpr)r7r#rwrvryrrrunmerges   zLinear.unmergecCs,|j|}|j|}|dur|j}|j}|jdko"|tjkp"|tjk}|j| |}|j | |}|rE| }| }| }| }t |j |jt|j |j} } |ddd|jd| f |} |d| d|jddf |} | jdd} t| ||jdd}| |jkr|| fS| j|jfS)a4 Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the correct order to fit the target layers' dimensions Args: adapter (str): The name of the adapter for which the delta weight should be computed. Ncpur>) start_dim)end_dim)r r"devicertyperfloat16bfloat16rrrfloatrEr2r1rLrHflattenr applyT)r7adapterrr r"r cast_to_fp32rrr]r\sliced_Asliced_Bupdate_Bupdate_Arrrget_scaled_basess* ""   zLinear.get_scaled_basescCs:||\}}|j|jj}t||j}|j|}||S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. )rrrrhr&)r7rrrrD output_tensorr&rrrrr4s    zLinear.get_delta_weightxc Os|j}|jr|jr ||j|g|Ri|}nT|jr*|j|g|Ri|}nD|j|g|Ri|}|jD]4}||jvrCq9|j|}|j ||j d\}} | | j}|j |} |t t |||| | }q9| |}|S)N)r)rdisable_adaptersr;r~r#rTrr_r)rrrr&Flinear) r7rargsr6previous_dtyperesultrvdropoutrrr&rrrrEs$    " zLinear.forwardcst}d|S)Nz randlora.)rj__repr__)r7reprlrrrZs zLinear.__repr__)rrr<FFT)FN)r9Nr )rrrrstrrGrr:r8rrNrzr~tuplerTensorrrrrr __classcell__rrrlrr0sD      *0r0)r{typingrrtorch.nnr'torch.nn.functional functionalrtransformers.pytorch_utilsrpeft.tuners.tuners_utilsrrpeft.utils.otherr _buffer_dictrautogradFunctionr r!r0rrrrs