o Nis,@sddlmZddlZddlmZddlmZmZddlZddl m Z ddl m m Z ddlmZddlmZddlmZmZGd d d eZGd d d e jeZdddZdS)) annotationsN)partial)AnyOptional) BufferDict)BaseTunerLayer)decompose_weight_matrixreconstruct_weight_matrixc@steZdZUdZded<dZded<d%d d Zd&ddZd'ddZddZ d(ddZ d)d*dd Z d+d!d"Z d#d$Z dS),OSFLayer)osf_svd_paramsztuple[str, ...]adapter_layer_names) _osf_U_high _osf_S_high _osf_V_highother_param_names base_layer nn.ModulereturnNonecKs:||_i|_ti|_ti|_ti|_ti|_g|_ d|_ g|_ | }t |drAt|jtjrA|jjdkrA|jj\}}nTt|tjrO|j|j}}nFt |drat |dra|j|j}}n4t |drst |drs|j|j}}n"t |drt |d r|j|j}}nd \}}td t|d t||_||_dS) NFweight infeatures outfeatures input_size output_size in_features out_features)NNzUnsupported layer type 'z/' encountered; could not infer in/out features.)reffective_ranknn ModuleDictr rrrr hook_handles_disable_adaptersmerged_adaptersget_base_layerhasattr isinstancertorchTensorndimshapeLinearrrrrrrwarningswarntype UserWarning)selfrkwargsrrr2I/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/osf/layer.py__init__'s>        zOSFLayer.__init__ adapter_namestrrintcKs|dkr td|||j|<|}|jj}t||d}|d|j|<|d|j|<|d|j|<t |d|d|d d }||j |<| || |jd S) z&Update layer to add a new OSF adapter.rzL`effective_rank` should be a positive integer value but the value passed is )top_kU_highS_highV_highU_lowS_lowV_low)r<r=r>N) ValueErrorrr$rdatar rrrr ParameterDictr _attach_hooks set_adapteractive_adapters)r0r5rr1rrsvd_dict svd_paramsr2r2r3 update_layerTs(    zOSFLayer.update_layercCsj||jvrdS|j|}d dd}|d t|d ||d }|d t|d ||d }|j||gdS) z,Attach gradient hooks for the given adapter.Nnamer6adapterlayerr cSs`|dkr|j|}||dd|}||S|dkr.|j|}||dd|}||S|S)Nr<rrr>)r transposer)gradrHrIrJr9projr;r2r2r3hooks  z$OSFLayer._attach_hooks..hookr<)rHrIrJr>)rHr6rIr6rJr )r register_hookrr!extend)r0r5 svd_modulerNhandle_uhandle_vr2r2r3rBxs   zOSFLayer._attach_hookscCs"|jD]}|q|jdS)zRemove all gradient hooks.N)r!removeclear)r0handler2r2r3 _detach_hookss  zOSFLayer._detach_hooks torch.TensorcCsV||jvr |jS|j|}|j||j||j||d|d|dd}t|S)z@Reconstruct weight matrix from SVD components for given adapter.r<r=r>)r9r:r;r<r=r>)r r$rrrrr )r0r5rQrEr2r2r3_reconstruct_weights   zOSFLayer._reconstruct_weightFN safe_mergebool adapter_namesOptional[list[str]]cCs|dur|j}|D]A}||jvrJ|}|r;|jj}||}t | s2t d|d| |j |j_n ||}||j_|j|q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rDr keysr$rr@clonerYr'isfiniteallr?todtyper#append)r0rZr\active_adapterr orig_weight new_weightr2r2r3merges$      zOSFLayer.mergecCs"|js tddStddS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.NzJOSF does not support unmerging. Original weights are permanently modified.)mergedr,r-r0r2r2r3unmerges zOSFLayer.unmergecCs |dS)zCleanup hooks on deletion.N)rWrjr2r2r3__del__s zOSFLayer.__del__)rrrr)r5r6rr7)r5r6)r5r6rrX)FN)rZr[r\r]rr)rr)__name__ __module__ __qualname__r __annotations__rr4rGrBrWrYrhrkrlr2r2r2r3r !s    - $   % r cs:eZdZ ddfdd ZdddZdfdd ZZS)r+Nr5r6rr7rrc sVttj||fi||durt|j|jd}||_|j||fi|dS)Nr)superr4r minrr_active_adapterrG)r0rr5rr1 __class__r2r3r4s zLinear.__init__xrXargsrr1c Os|jr|j|g|Ri|}|S|jr"|j|g|Ri|}|S|}|j}|jr1|jdnd}|rH||jvrH||}t |||}|S|j|g|Ri|}|S)Nr) disable_adaptersrrir$biasrDr rYFlinear) r0rvrwr1resultrryrerr2r2r3forwards zLinear.forwardcst}d|S)Nzosf.)rq__repr__)r0reprtr2r3r~s zLinear.__repr__N)r5r6rr7rr)rvrXrwrr1rrrX)rr6)rmrnror4r}r~ __classcell__r2r2rtr3r+s  r+targettorch.nn.Moduler5r6rOptional[torch.nn.Module]cKs@d}t|tr |}n|}t|tjjrt||fi|}|Sr)r&rr$r'rr+)rr5 osf_configr1 new_moduletarget_base_layerr2r2r3dispatch_default s  r)rrr5r6rr) __future__rr, functoolsrtypingrrr'torch.nnrtorch.nn.functional functionalrzpeft.tuners._buffer_dictrpeft.tuners.tuners_utilsrutilsr r r Moduler+rr2r2r2r3s     ?-