o 8wiM:@sddlZddlZddlmZmZmZddlZddlmZddl mm Z ddl m Z mZGddde ZGdddejeZdS)N)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_mergec@seZdZdZdZdejddfddZded e d e ddfd d Z defd dZ defddZ defddZdeddfddZddddZdS) BoneLayer) bone_block)bone_r base_layerreturnNcKsl||_i|_ti|_d|_g|_d|_||_| }t |tj r-|j |j |_ |_ dStdt|)NFTzUnsupported layer type )r r nn ParameterDictr_disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfr rrS/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/bone/layer.py__init__s  zBoneLayer.__init__ adapter_namer init_weightscKs|dkr td|||j|<|}t|tjr*tjt||j dd|j |<n t dt |j |dkrQ|j|dksF|j |dkrJtd|||n|rZ|||n||||||jdS) zInternal function to create bone adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is T requires_gradz0Bone is not implemented for base layers of type batz=The weight matrix must be fully divisible into [r, r] blocks.N)rr rrr r Parametertorchzerosrr TypeErrorr__name__rreset_bat_parametersreset_bone_parametersreset_bone_parameters_random%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)rrrrrr rrr update_layer0s      zBoneLayer.update_layercCs"tjt||jdd|j|<dSNTr r r#r$r%rrrrrrrrr)Xs"zBoneLayer.reset_bone_parameterscCs(tjt|j|||dd|j|<dSr/r0r1rrrr([s(zBoneLayer.reset_bat_parameterscCs tjj|j|tdddS)N)a)r initkaiming_uniform_rmathsqrt)rrrrrr*^s z&BoneLayer.reset_bone_parameters_randomscalecCs6|dkrdS|jD]}||jvrq tdq dS)NzGScaling operation for Bone not supported! Automatically set scale to 1.r-rkeyswarningswarnrr8active_adapterrrr scale_layeras  zBoneLayer.scale_layercCs*|jD]}||jvr qtdqdS)Nz?Unscaling operation for Bone not supported! Keeping scale at 1.r:r>rrr unscale_layerks  zBoneLayer.unscale_layerNr N)r' __module__ __qualname__adapter_layer_namesother_param_namesr Modulerstrintboolr.r)r(r*floatr@rArrrrrs$ ( rc seZdZdZ  ddededeeefddffd d Zdd ed e e eddfddZ d ddZ d!dede jfddZd!dede jfddZde jdedede jfddZdeffdd ZZS)" BoneLinearz, Bone implemented in a dense layer. rTrrrr Nc sDttj||fi|||_|j|||fi|||_dSrB)superrr_active_adapterr.bone_fn)rr rrrr __class__rrrxs  zBoneLinear.__init__F safe_merge adapter_namescCst||}|s dS|D]{}||jvr|}|jj}|rV|jj}|jdkr4| ||}||7}n | ||j jj}|}t |sNtd|d|||j_n*|jdkrp| ||j jj}|jj||7_n| ||j jj}|||j_|j|q dS)ab Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If `None`, all active adapters will be merged. Defaults to `None`. Nr"z1NaNs detected in the merged weights. The adapter z seems to be broken)rrr;rweightdtypedataclonerPget_delta_weightget_delta_weight_boner r$isfiniteallrtorappend)rrSrTr?r orig_dtype orig_weight delta_weightrrrmerges6        zBoneLinear.mergecCs|js tddSt|jdkrS|j}|}|jj}||j vrJ|jj }|j dkr;|j||dd}n|j||dd}|||j_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nrr"T)re)mergedr<r=lenrpoprrUrVrr;rWrXrPrYrZr])rr?r r_r`rarrrunmerges   zBoneLinear.unmergerccCsd|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&|}||j}| d}|rw| | d||| d|| dddd} t | d|j} t | |} | |j} | || } | ddddj |j} n(| | d||| d|| dddd||} | ddddj |j} |r| j|d} |||j|_| S) Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpurr9rV)rdevicerVrr$float16bfloat16rLr]sizereshapepermuteeyeinverseshaperW)radapterr`rcrnrV cast_to_fp32 weight_boneroone inv_I_plus_bw output_tensorrrrrYs4     0  . zBoneLinear.get_delta_weightcCst|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&|}|d}|d} || dkr|| } || } | | } |r|ddd| f d| |  ddd| dddj |ddd| fj |ddd| f<|dd| df| ddddd| f|dd| df<nO|ddd| f d| |  ddd| dddj |ddd| fj |ddd| f<|dd| df| ddddd| f|dd| df<|} n@|r| d|d| |  ddd|}| dddj |j } n| d|d| |  ddd|}| dddj |j } |r8| j |d} | ||j|_| S)rhrirjrNr9rkrm)rrnrVrr$rorprLrqrrrsrv transposer]rW)rrwr`rcrnrVrxryrr last_sizen_block n_block_sizer~r}rrrrZsJ      * .* .&& z BoneLinear.get_delta_weight_bonexargsrc Os|j}|jr|jr ||j|g|Ri|}n|jr*|j|g|Ri|}n|jdkrg|jjj}|j D]}||j vrCq9| ||}||}q9| ||j}| |jj|j} tj||| d}nf|j|g|Ri|}|j D]V}||j vrqv|j |} | d} |d| dkr| |d| | } t|d| f}| || j}|tj|jg|jdd|d| | Rdd| }qv||}|S)Nr")inputrUbiasrrj)dim)rVdisable_adaptersrdrgr rPrUrWrXr-rr;rY_cast_input_dtyperFlinearrqpadr$sumrrrvr]) rrrrprevious_dtyperesultr`r?rarboner padding_sizerrrforward8s<       @ zBoneLinear.forwardcst}d|S)Nzbone.)rN__repr__)rreprQrrr]s zBoneLinear.__repr__)rT)FNrC)F)r'rDrE__doc__rIrJrrKrrlistrbrgr$TensorrYrZrrr __classcell__rrrQrrMss&   0.?%rM)r6r<typingrrrr$torch.nnr torch.nn.functional functionalrpeft.tuners.tuners_utilsrrrrHrMrrrrs  Z