o NÆÏi¸(ã@s„ddlmZddlZddlZddlmZmZddlZddlm Z ddl m Z ddl m Z mZGdd„de ƒZGdd „d e jeƒZdS) é)Ú annotationsN)ÚAnyÚOptional)Ú BufferDict)ÚBaseTunerLayerÚcheck_adapters_to_mergec@sVeZdZdZdZd&dd„Zed'dd„ƒZd(dd„Z  d)d*d d!„Z  "d+d,d#d$„Z d%S)-Ú DeloraLayer)Údelora_AÚdelora_BÚ delora_lambda)ÚrÚdelora_dropoutÚ delora_w_normÚ base_layerú nn.ModuleÚreturnÚNonecKs˜||_i|_t i¡|_t i¡|_t i¡|_t i¡|_t idd|_ d|_ g|_ ||_ | ¡}t|tjƒrC|j|j|_|_dStdt|ƒ›ƒ‚)NT)Ú persistentFzUnsupported layer type )rr ÚnnÚ ModuleDictr Ú ParameterDictr r r rrÚ_disable_adaptersÚmerged_adaptersÚkwargsÚget_base_layerÚ isinstanceÚLinearÚ in_featuresÚ out_featuresÚ ValueErrorÚtype)ÚselfrrÚbase_layer_mod©r#úL/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/delora/layer.pyÚ__init__)s     zDeloraLayer.__init__ÚAú torch.TensorÚBr r ÚintÚw_normc Cs`tj|jdddd}tj|jdddd}t ||||¡}|||}|| d¡}|S)zpCompute delta = B @ diag(delora_lambda/r / (||A_i||*||B^j||)) @ A, scaled by provided w_norm (per-input channel)é©Údimç-Cëâ6?©Úminr)ÚtorchÚclampÚnormÚ diag_embedÚ unsqueeze) r&r(r r r*ÚAnÚBnÚdiagÚdeltar#r#r$Ú_compute_delta=s  zDeloraLayer._compute_deltaÚadapterÚstrcCsX||jvs ||jvrtd|›dƒ‚| |j||j||j||j||j|¡}|S)NzAdapter z not found.)r r rr:r r r)r!r;r9r#r#r$Úget_delta_weightIsûzDeloraLayer.get_delta_weightTFÚ adapter_nameÚfloatÚmodule_dropoutÚ init_weightsÚboolÚinference_moderrc KsÊ|dkr td|›ƒ‚||j|<t t ||j¡¡|j|<t t |j|¡¡|j |<t t d¡¡|j |<|dkr@tj |d}nt  ¡}|j  t ||i¡¡| |||¡| |¡|j|j|ddS)a¡Internal function to create delora adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. delora_lambda (`float`): Boundary for the adapter's norm. module_dropout (`float`): The dropout probability for disabling adapter during training. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is r+g)Úp)rCN)rr rÚ Parameterr1Úemptyrr rr r ÚDropoutÚIdentityr ÚupdaterÚreset_delora_parametersÚ%_move_adapter_to_device_of_base_layerÚ set_adapterÚactive_adapters) r!r>r r r@rArCrÚmodule_dropout_layerr#r#r$Ú update_layerVs  zDeloraLayer.update_layerç.@cCs||j ¡vr dS|dur%tjj|j|t d¡dtj |j|¡ntjj|j|t d¡dtjj|j|t d¡d|j |j   t |ƒ¡t  ¡/| ¡j}|jjdkrgt j|j dd ¡}n t j|jd|jd}||j|<WdƒdS1s‚wYdS) NTé)ÚaÚmetarr,r+)Údevice)r ÚkeysrÚinitÚkaiming_uniform_ÚmathÚsqrtÚzeros_r r ÚdataÚfill_r?r1Úno_gradrÚweightrTr r3ÚdetachÚonesÚshaper)r!r>rAr Úwr*r#r#r$rJ}s    "ùz#DeloraLayer.reset_delora_parametersN)rrrr) r&r'r(r'r r'r r)r*r'rr')r;r<rr')TF)r>r<r r)r r?r@r?rArBrCrBrrrr)TrP)r>r<rArBr r?rr) Ú__name__Ú __module__Ú __qualname__Úadapter_layer_namesÚother_param_namesr%Ú staticmethodr:r=rOrJr#r#r#r$rs    ù*ürcsPeZdZ d!d"‡fd d„ Zd#d$dd„Zd%dd„Zd&dd„Zd'‡fdd „ Z‡ZS)(Ú DeloraLinearTr>r<r r)r r?r@rArBrrc s:tƒ ¡tj||fi|¤Ž||_| |||||¡dS)N)Úsuperr%rÚ_active_adapterrO)r!rr>r r r@rAr©Ú __class__r#r$r%œs zDeloraLinear.__init__FNÚ safe_mergeÚ adapter_namesúOptional[list[str]]c CsØt||ƒ}|s dS|D]^}||j ¡vri| ¡}| |¡ ¡j|jj|jj d}t   ¡/|rM|jj   ¡}||}t  |¡ ¡sHtd|›dƒ‚||j_ n|jj  |¡Wdƒn1s^wY|j |¡q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. N)ÚdtyperTz1NaNs detected in the merged weights. The adapter z seems to be broken)rr rUrr=r_Útor^rqrTr1r]r[ÚcloneÚisfiniteÚallrÚadd_rÚappend)r!rnroÚactive_adapterrÚ delta_weightÚ orig_weightsr#r#r$Úmerge«s4 þÿ   ÿ €ô €êzDeloraLinear.mergecCsj|js t d¡dSt|jƒdkr3|j ¡}||j ¡vr*| ¡j j |  |¡8_ t|jƒdksdSdS)zJ Unmerge all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) ÚmergedÚwarningsÚwarnÚlenrÚpopr rUrr^r[r=)r!rxr#r#r$ÚunmergeÔs  ýzDeloraLinear.unmergeÚxr'ÚargsrrcOst|j}|jr|jr | ¡|j|g|¢Ri|¤Ž}n™|jr*|j|g|¢Ri|¤Ž}n‰|js<|j|g|¢Ri|¤Ž |¡S|j|g|¢Ri|¤Ž}t |¡}|jD]Z}||j vrXqP|j ||ƒ} t j   | |j||j |¡} tj|j |jdddd} tj|j|jdddd} |j||j|| | } | | } t j   | |j|¡} || 7}qP|| |j¡}| |¡}|S)Nr+r,r.r/r)rqÚdisable_adaptersr|rrrMrrr1Ú zeros_liker r rÚ functionalÚlinearrr2r3r r r )r!r‚rƒrÚprevious_dtypeÚresultÚbase_outÚadd_outr;Úx_dÚhr6r7Úscalingr#r#r$Úforwardàs2     zDeloraLinear.forwardcstƒ ¡}d|S)Nzdelora.)rjÚ__repr__)r!Úreprlr#r$r s zDeloraLinear.__repr__)T) r>r<r r)r r?r@r?rArBrr)FN)rnrBrorprr)rr)r‚r'rƒrrrrr')rr<) rcrdrer%r{rrrÚ __classcell__r#r#rlr$rišs ù  ) +ri)Ú __future__rrXr}Útypingrrr1Útorch.nnrÚpeft.tuners._buffer_dictrÚpeft.tuners.tuners_utilsrrrÚModulerir#r#r#r$Ús