o 8wiA!@sddlmZddlZddlZddlmZmZmZddlZddl m Z ddl m Z m Z ddlmZmZGddde ZGd d d e jeZdS) ) annotationsN)AnyLiteralOptional)BaseTunerLayercheck_adapters_to_merge)BlockCircularConvolutionget_circulant_fastc@s@eZdZdZdZdddZdd d Zd d Ze ddZ dS)C3ALayer) c3a_kernel) block_size base_layer nn.ModulereturnNonecKsf||_i|_ti|_d|_g|_||_|}t |tj r*|j |j |_ |_ dSt dt|)NFzUnsupported layer type )rr nn ParameterDictr _disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfrrrR/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/c3a/layer.py__init__"s  zC3ALayer.__init__ torch.TensorcCs\||jvrtd|d|j}|j}|j|}t|tj |}|| dS)NzAdapter z not found.) r keysrrweightdtyper totorchfloat32size)radapterbase_layer_weightbase_layer_weight_dtyper delta_weightrrr get_delta_weight1s  zC3ALayer.get_delta_weightc Cs|dkr td||j|dkrtd|jd||j|dkr/td|jd|||j|<|j}ttj |j||j||tj |j d|j |<| ||||||jdS)NrzH`block_size` should be a positive integer value but the value passed is zPThe block size should be a factor of the input size. However, the input size is z and the block size is zRThe block size should be a factor of the output size. However, the output size is )r&device)rrrr rr%r Parameterr(zerosr)r0r reset_c3a_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r adapter_namer init_weightsr%rrr update_layer;s0   zC3ALayer.update_layercCs|durdS||jvrp|dkrtj|j|dS|dvrI|j|j}}dtdt ||}td|}tj |j|| |dS|dkri|j}dtdt |}tj |j|| |dSt d|dS) NTgaussian)xavier_uniformFg?g@g@kaiming_uniformzUnknown init_weights: ) r r$rinitnormal_rrmathsqrtfloatuniform_r)rr7r8fan_infan_outstdarrr r3Xs zC3ALayer.reset_c3a_parametersN)rrrr)rr") __name__ __module__ __qualname__adapter_layer_namesother_param_namesr!r/r9r(no_gradr3rrrr r s   r csLeZdZdfd d Zdd ddZd!ddZd"ddZd#fdd ZZS)$ C3ALinearr7strr intr8?bool | Literal['gaussian', 'kaiming_uniform', 'xavier_uniform']rrc s6ttj||fi|||_||||dS)N)superr!r _active_adapterr9)rrr7r r8r __class__rr r!os zC3ALinear.__init__FN safe_mergebool adapter_namesOptional[list[str]]cCst||}|s dS|D]E}||jvrP|}|r=|jj}||}||}t | s8t d|d||j_n ||}|jj||j_|j |q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr r$rr%datacloner/r(isfiniteallrrappend)rrUrWactive_adapterr orig_weightsr.rrr merge|s(       zC3ALinear.mergecCsj|js tddSt|jdkr3|j}||jvr*|j j | |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) mergedwarningswarnlenrpopr r$rr%rYr/)rr^rrr unmerges  zC3ALinear.unmergexr"argsrrcOs|j}|jr|jr ||j|g|Ri|}nL|jr*|j|g|Ri|}n<|j|g|Ri|}|tj}|jD]&}||j vrIq?|j |tj}t ||| d}|||j7}q?||}|S)Nr#)r&disable_adaptersrarfrr'r(r)r6r r$r applyr*)rrgrhrprevious_dtyperesultr^r rrr forwards"   zC3ALinear.forwardcst}d|S)Nzc3a.)rQ__repr__)rreprSrr rns zC3ALinear.__repr__)r7rNr rOr8rPrr)FN)rUrVrWrXrr)rr)rgr"rhrrrrr")rrN) rGrHrIr!r`rfrmrn __classcell__rrrSr rMms  ( rM) __future__rr?rbtypingrrrr(torch.nnrpeft.tuners.tuners_utilsrrutilsr r r ModulerMrrrr s  Q