o Ni:@sddlZddlZddlmZmZmZddlZddlmZddl mm Z ddl m Z mZGddde ZGdddejeZdS)N)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_mergec @seZdZdZdZdejddfddZ dd ed e d e d e ddf d dZ d efddZ d efddZ d efddZdeddfddZddddZdS) BoneLayer) bone_block)bone_r base_layerreturnNcKsl||_i|_ti|_d|_g|_d|_||_| }t |tj r-|j |j |_ |_ dStdt|)NFTzUnsupported layer type )r r nn ParameterDictr_disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfr rrJ/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/bone/layer.py__init__s  zBoneLayer.__init__F adapter_namer init_weightsinference_modecKs|dkr td|||j|<|}t|tjr*tjt||j dd|j |<n t dt |j |dkrQ|j|dksF|j |dkrJtd|||n|rZ|||n|||||j|j|dd S) zInternal function to create bone adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is T requires_gradz0Bone is not implemented for base layers of type batz=The weight matrix must be fully divisible into [r, r] blocks.)r N)rr rrr r Parametertorchzerosrr TypeErrorr__name__rreset_bat_parametersreset_bone_parametersreset_bone_parameters_random%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)rrrrr rr rrr update_layer0s      zBoneLayer.update_layercCs"tjt||jdd|j|<dSNTr!r r$r%r&rrrrrrrrr*Ys"zBoneLayer.reset_bone_parameterscCs(tjt|j|||dd|j|<dSr0r1r2rrrr)\s(zBoneLayer.reset_bat_parameterscCs tjj|j|tdddS)N)a)r initkaiming_uniform_rmathsqrt)rrrrrr+_s z&BoneLayer.reset_bone_parameters_randomscalecCs6|dkrdS|jD]}||jvrq tdq dS)NzGScaling operation for Bone not supported! Automatically set scale to 1.r.rkeyswarningswarnrr9active_adapterrrr scale_layerbs  zBoneLayer.scale_layercCs*|jD]}||jvr qtdqdS)Nz?Unscaling operation for Bone not supported! Keeping scale at 1.r;r?rrr unscale_layerls  zBoneLayer.unscale_layerFNr N)r( __module__ __qualname__adapter_layer_namesother_param_namesr Modulerstrintboolr/r*r)r+floatrArBrrrrrs* ) rc seZdZdZ  ddededeeefddffd d Zdd ed e e eddfddZ d ddZ d!dede jfddZd!dede jfddZde jdedede jfddZdeffdd ZZS)" BoneLinearz, Bone implemented in a dense layer. rTrrrr Nc sDttj||fi|||_|j|||fi|||_dSrD)superrr_active_adapterr/bone_fn)rr rrrr __class__rrrys  zBoneLinear.__init__F safe_merge adapter_namescCst||}|s dS|D]{}||jvr|}|jj}|rV|jj}|jdkr4| ||}||7}n | ||j jj}|}t |sNtd|d|||j_n*|jdkrp| ||j jj}|jj||7_n| ||j jj}|||j_|j|q dS)ab Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If `None`, all active adapters will be merged. Defaults to `None`. Nr#z1NaNs detected in the merged weights. The adapter z seems to be broken)rrr<rweightdtypedataclonerRget_delta_weightget_delta_weight_boner r%isfiniteallrtorappend)rrUrVr@r orig_dtype orig_weight delta_weightrrrmerges6        zBoneLinear.mergecCs|js tddSt|jdkrS|j}|}|jj}||j vrJ|jj }|j dkr;|j||dd}n|j||dd}|||j_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nrr#T)re)mergedr=r>lenrpoprrWrXrr<rYrZrRr[r\r_)rr@r rarbrcrrrunmerges   zBoneLinear.unmergerecCsd|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&|}||j}| d}|rw| | d||| d|| dddd} t | d|j} t | |} | |j} | || } | ddddj |j} n(| | d||| d|| dddd||} | ddddj |j} |r| j|d} |||j|_| S) Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpurr:rX)rdevicerXrr%float16bfloat16rNr_sizereshapepermuteeyeinverseshaperY)radapterrbrerprX cast_to_fp32 weight_boneroone inv_I_plus_bw output_tensorrrrr[s4     0  . zBoneLinear.get_delta_weightcCst|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&|}|d}|d} || dkr|| } || } | | } |r|ddd| f d| |  ddd| dddj |ddd| fj |ddd| f<|dd| df| ddddd| f|dd| df<nO|ddd| f d| |  ddd| dddj |ddd| fj |ddd| f<|dd| df| ddddd| f|dd| df<|} n@|r| d|d| |  ddd|}| dddj |j } n| d|d| |  ddd|}| dddj |j } |r8| j |d} | ||j|_| S)rjrkrlrNr:rmro)rrprXrr%rqrrrNrsrtrurx transposer_rY)rryrbrerprXrzr{rr last_sizen_block n_block_sizerrrrrr\sJ      * .* .&& z BoneLinear.get_delta_weight_bonexargsrc Os|j}|jr|jr ||j|g|Ri|}n|jr*|j|g|Ri|}n|jdkrg|jjj}|j D]}||j vrCq9| ||}||}q9| ||j}| |jj|j} tj||| d}nf|j|g|Ri|}|j D]V}||j vrqv|j |} | d} |d| dkr| |d| | } t|d| f}| || j}|tj|jg|jdd|d| | Rdd| }qv||}|S)Nr#)inputrWbiasrrl)dim)rXdisable_adaptersrfrir rRrWrYrZr.rr<r[_cast_input_dtyperFlinearrspadr%sumrtrxr_) rrrrprevious_dtyperesultrbr@rcrboner padding_sizerrrforward9s<       @ zBoneLinear.forwardcst}d|S)Nzbone.)rP__repr__)rreprSrrr^s zBoneLinear.__repr__)rT)FNrErC)r(rFrG__doc__rKrLrrMrrlistrdrir%Tensorr[r\rrr __classcell__rrrSrrOts&   0.?%rO)r7r=typingrrrr%torch.nnr torch.nn.functional functionalrpeft.tuners.tuners_utilsrrrrJrOrrrrs  [