o NÆÏi Cã@sŽddlmZddlZddlZddlmZmZmZddlZddl m Z ddl m m Z ddlmZmZGdd„deƒZGdd„de jeƒZdS) é)Ú annotationsN)ÚAnyÚOptionalÚUnion)ÚBaseTunerLayerÚcheck_adapters_to_mergec@sjeZdZdZdZd$dd„Z d%d&dd„Zd'dd„Zd'dd„Zd'dd„Z d'dd„Z d(dd „Z d)d*d"d#„Z d!S)+Ú MissLayer)Ú miss_block)Úmiss_rÚ miss_dropoutÚ miss_mini_rÚ base_layerú nn.ModuleÚreturnÚNonecKs~||_i|_t i¡|_i|_t i¡|_d|_g|_ d|_ ||_ |  ¡}t |tjƒr6|j|j|_|_dStdt|ƒ›ƒ‚)NFTzUnsupported layer type )r r ÚnnÚ ModuleDictr r Ú ParameterDictr Ú_disable_adaptersÚmerged_adaptersÚcast_input_dtype_enabledÚkwargsÚget_base_layerÚ isinstanceÚLinearÚ in_featuresÚ out_featuresÚ ValueErrorÚtype)Úselfr r©r úJ/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/miss/layer.pyÚ__init__!s   zMissLayer.__init__FÚ adapter_nameÚstrÚrÚintÚmini_rÚ init_weightsú bool | strÚinference_modeÚboolc Ks<|dkr td|›ƒ‚||j|<||j|<|dkr tj|d}nt ¡}||j|<| ¡} t| tj ƒrCtj t   ||j ¡dd|j|<n tdt| ƒj›ƒ‚|dkrj|j|dks_|j |dkrctd ƒ‚| ||¡n%|d kr|j |dkrytd ƒ‚| |||¡n|rŠ| ||¡n| |¡| |¡|j|j|d d S)zôInternal function to create miss adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is ç)ÚpT©Ú requires_gradz0MiSS is not implemented for base layers of type Úbatz=The weight matrix must be fully divisible into [r, r] blocks.ÚminizÍmini_r is divided along the out_features dimension. For optimal performance and implementation simplicity,it is recommended that out_features be divisible by mini_r.Error: {self.out_features} % mini_r != 0)r*N)rr r rÚDropoutÚIdentityr rrrÚ ParameterÚtorchÚzerosrr Ú TypeErrorrÚ__name__rÚreset_bat_parametersÚreset_mini_parametersÚreset_miss_parametersÚreset_miss_parameters_randomÚ%_move_adapter_to_device_of_base_layerÚ set_adapterÚactive_adapters) rr#r%r'r r(r*rÚmiss_dropout_layerr r r r!Ú update_layer4s6     ÿ  zMissLayer.update_layercCs"tjt ||j¡dd|j|<dS©NTr.©rr4r5r6rr ©rr#r%r r r!r;ms"zMissLayer.reset_miss_parameterscCs(tjt |j|||¡dd|j|<dSrBrCrDr r r!r9ps(zMissLayer.reset_bat_parameterscCs tjt ||¡dd|j|<dSrB)rr4r5r6r )rr#r%r'r r r!r:só zMissLayer.reset_mini_parameterscCs tjj|j|t d¡ddS)Né)Úa)rÚinitÚkaiming_uniform_r ÚmathÚsqrt)rr#r r r!r<vrEz&MissLayer.reset_miss_parameters_randomÚscaleÚfloatcCs6|dkrdS|jD]}||j ¡vrq t d¡q dS)NézGScaling operation for MiSS not supported! Automatically set scale to 1.©r?r ÚkeysÚwarningsÚwarn©rrLÚactive_adapterr r r!Ú scale_layerys  üzMissLayer.scale_layerNcCs*|jD]}||j ¡vr qt d¡qdS)Nz?Unscaling operation for MiSS not supported! Keeping scale at 1.rOrSr r r!Ú unscale_layerƒs  üzMissLayer.unscale_layer)r rrr©F) r#r$r%r&r'r&r(r)r*r+rr)r#r$)rLrMrr©N©rr) r8Ú __module__Ú __qualname__Úadapter_layer_namesÚother_param_namesr"rAr;r9r:r<rUrVr r r r!rs  ù 9     rcsreZdZdZ    d*d+‡fdd„ Zd,d-dd„Zd.dd„Zd/d0dd„Zd/d0d d!„Zd1d&d'„Z d2‡fd(d)„ Z ‡Z S)3Ú MissLinearz, MiSS implemented in a dense layer. rr,Tr#r$r%r&r'r rMr(úUnion[bool, str]rrc sHtƒ ¡tj||fi|¤Ž||_|j|||||fi|¤Ž||_dSrX)Úsuperr"rÚ_active_adapterrAÚmiss_fn)rr r#r%r'r r(r©Ú __class__r r!r"s  zMissLinear.__init__FNÚ safe_merger+Ú adapter_namesúOptional[list[str]]cCs`t||ƒ}|s dS|D]¢}||j ¡vr­| ¡}|jj}|rg|jj ¡}|jdkr4|  ||¡}||7}n|jdkrE|  ||j jj¡}|}n |  ||j jj¡}|}t   |¡ ¡s_td|›dƒ‚| |¡|j_n@|jdkr|  ||j jj¡}|jj| |¡7_n&|jdkr—|  ||j jj¡}| |¡|j_n|  ||j jj¡}| |¡|j_|j |¡q dS)ab Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If `None`, all active adapters will be merged. Defaults to `None`. Nr0r1z1NaNs detected in the merged weights. The adapter z seems to be broken)rr rPrÚweightÚdtypeÚdataÚclonerbÚget_delta_weightÚget_delta_weight_missr r5ÚisfiniteÚallrÚtorÚappend)rrerfrTr Ú orig_dtypeÚ orig_weightÚ delta_weightr r r!Úmerge sB       ÿ   €ÞzMissLinear.mergecCsÆ|js t d¡dSt|jƒdkra|j ¡}| ¡}|jj}||j   ¡vrX| ¡jj   ¡}|j dkr;|j||dd}n|j dkrI|j||dd}n|j||dd}| |¡|j_ t|jƒdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nrr0T)Úrer1)ÚmergedrQrRÚlenrÚpoprrhrir rPrjrkrbrlrmrp)rrTr rrrsrtr r r!ÚunmergeÖs     ózMissLinear.unmergervú torch.TensorcCsd|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&| ¡}| |j¡}|  d¡}|rw|  |  d¡|||  d¡||¡  dddd¡} t  |  d¡¡ |j¡} t  | |¡} |  |j¡} | || } |   dddd¡j |jŽ} n(|  |  d¡|||  d¡||¡  dddd¡||} |   dddd¡j |jŽ} |r°| j|d} | |¡|j|_| S)ú Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. ÚcpuéÿÿÿÿrrNéé©ri)r Údevicerirr5Úfloat16Úbfloat16rMrpÚsizeÚreshapeÚpermuteÚeyeÚinverseÚshaperj)rÚadapterrsrvr‚riÚ cast_to_fp32Ú weight_missr%ÚoÚoneÚ inv_I_plus_bÚwÚ output_tensorr r r!rlís4     0  .ÿþÿ zMissLinear.get_delta_weightcCsž|j|j}|j|j}|jdko|tjkp|tjk}|j|}|r&| ¡}| d¡}| d¡} | d¡} |j dkrE|  d| |j |¡}|| dkrû|| } || } | | } |r©|dd…d| …f  d| | ¡  ddd¡|  ddd¡j |dd…d| …fjŽ|dd…d| …f<|dd…| d…f| dd¡dd…d| …f|dd…| d…f<nO|dd…d| …f  d| | ¡  ddd¡|  ddd¡j |dd…d| …fjŽ|dd…d| …f<|dd…| d…f| dd¡dd…d| …f|dd…| d…f<|}n@|r|  d| d¡| | ¡  ddd¡|}|  ddd¡j |jŽ}n|  d| d¡| | ¡  ddd¡|}|  ddd¡j |jŽ}|rM|j|d}| |¡|j|_|S) r|r}r~rr1rNNrr)r r‚rirr5rƒr„rMr…rbÚrepeatr r†r‡rŠÚ transposerprj)rr‹rsrvr‚rirŒrrrr%Ú last_sizeÚn_blockÚ n_block_sizer’r‘r r r!rmsP        * þÿ.ÿ* þÿ.ÿ&& z MissLinear.get_delta_weight_missÚxÚargsrrcOsÚ|j}|jr|jr | ¡|j|g|¢Ri|¤Ž}nÌ|jr*|j|g|¢Ri|¤Ž}n¼|jdkrg|jjj ¡}|j D]}||j   ¡vrCq9|  ||¡}||}q9|  ||j¡}|  |jj|j¡} tj||| d}n|j|g|¢Ri|¤Ž}|j D]o}||j   ¡vr€qv|j |} |jdkr—|  d|jj|j|¡} |j|} |  d¡} | d¡| dkr½| | d¡| | } t |d| f¡}|  || j¡}|tj| |ƒjg|jdd…¢| d¡| ‘| ‘RŽdd| }qv| |¡}|S) Nr0)ÚinputrhÚbiasr1rNrr~éþÿÿÿ)Údim)riÚdisable_adaptersrwrzr rbrhrjrkr?r rPrlÚ_cast_input_dtyper›ÚFÚlinearr“rr r r…Úpadr5Úsumr†rŠrp)rr˜r™rÚprevious_dtypeÚresultrsrTrtr›ÚmissÚdropoutr%Ú padding_sizer r r!Úforward^sB         D zMissLinear.forwardcstƒ ¡}d|S)Nzmiss.)r`Ú__repr__)rÚreprcr r!rª‡s zMissLinear.__repr__)rrr,T) r#r$r%r&r'r&r rMr(r_rr)FN)rer+rfrgrrrYrW)rvr+rr{)r˜r{r™rrrrr{)rr$) r8rZr[Ú__doc__r"rurzrlrmr©rªÚ __classcell__r r rcr!r^‹sù  6  . C)r^)Ú __future__rrJrQÚtypingrrrr5Útorch.nnrÚtorch.nn.functionalÚ functionalr Úpeft.tuners.tuners_utilsrrrÚModuler^r r r r!Ús  p