o 8wih/@sddlZddlmZddlZddlmZddlmmZddl m Z ddl m Z m Z ddlmZddlmZGdd d e ZGd d d ejeZdS) N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transpose) BufferDictc@s`eZdZdZdZdejfddZede fddZ dd e d e d e fd dZ dd e fddZdS) VeraLayer) vera_lambda_b vera_lambda_d)vera_Avera_B base_layercKs||_i|_ti|_ti|_ti|_d|_d|_ d|_ g|_ | }t |tjr6|j|j}}nt |trKt|jdrE|jjn|jj\}}||_||_||_dS)NFds_shape)rrnn ModuleDict vera_dropout ParameterDictr r r r _disable_adaptersmerged_adaptersget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshapekwargs)selfrrrrr!S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/vera/layer.py__init__"s$      zVeraLayer.__init__returncCs t|jSN)boolr)r r!r!r"merged@s zVeraLayer.merged皙?r r d_initialc Cs|dkr td|||j|<|dkrtj|d}nt}|jt||itjt |j dd|j |<tjt |dd|j|<||_||_||vrt|jdkrZtdt|jd} t|jd} d } | jd|jkrt| d | jd|j| jd|j krt| d | jd|j d } | jd|j|krt| d | jd|j|| jd|j|krt| d | jd|j|| |j|<| |j|<|r|j||d ||||jdS)Nrz?`r` should be a positive integer value but the value passed is )pT) requires_gradz^The `vera_A` and `vera_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional VeRA adapter was added after the first one with incompatible shapes.r r z{} has a size of {} but {} or greater is required; this probably happened because an additional VeRA adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.r)) ValueErrorrrDropoutIdentityrupdater Parametertorchonesrr randnr r r lenlistvaluesrrformatreset_vera_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) r adapter_namer r rr init_weightsr)vera_dropout_layer vera_A_param vera_B_param error_tmplr!r!r" update_layerDsH     zVeraLayer.update_layercCsj||jvr3ttj|j||tj|j|WddS1s,wYdSdSr%) r keysr4no_gradrinitzeros_fill_r )r r?r)r!r!r"r;s  "zVeraLayer.reset_vera_parametersN)r()__name__ __module__ __qualname__adapter_layer_namesother_param_namesrModuler#propertyr&r'rfloatrEr;r!r!r!r"r s  Br cseZdZ      d dededed ed ed ed ed ededdffdd Zd!dede e eddfddZ d"ddZ de jfddZde jde jfddZdeffdd ZZS)#rrr*FTr(r r r?rrfan_in_fan_outis_target_conv_1d_layerr@r)r$Nc sRttj|tj||fi| ||_||_|j|||||| | d||_dS)Nr.) superrrr#r rS_active_adapterrErT) r rr r r?rrrSrTr@r)r __class__r!r"r#s  zLinear.__init__ safe_merge adapter_namescCst||}|s dS|D]A}||jvrL|}|r;|jj}|||7}t | s6t d|d||j_n |jj||7_|j |q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr rFrrdatacloneget_delta_weightr4isfiniteallr/rappend)r rYrZactive_adapterr orig_weightsr!r!r"merges$     z Linear.mergecCsj|js tddSt|jdkr3|j}||jvr*|j j | |8_ t|jdksdSdS)Nz Already unmerged. Nothing to do.r) r'warningswarnr7rpopr rFrrr[r])r rar!r!r"unmerges  zLinear.unmergec Cs|j|}|j|}|j}|j}|jdko|tjkp|tjk}|j|}|j |}|r;| }| }| }| }|ddd|j f |j} |d|j ddf |j} |d}|d}t|| || |j} |rw| j |d} | S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpuN)dtype)r r devicerjtyper4float16bfloat16r r rRrtor unsqueezerrS) r adapterr r rkrj cast_to_fp32lambda_dlambda_bsliced_Asliced_B output_tensorr!r!r"r]s(       zLinear.get_delta_weightxc Os4|j}|jr|jr ||j|g|Ri|}ny|jr*|j|g|Ri|}ni|j|g|Ri|}|jD]Y}||jvrCq9|j|}|j|}|j |} |j |} | ddd|j f |j } | d|jddf |j } |j|} | |j}||t|t| || | }q9| |}|Sr%)rjdisable_adaptersr'rgrr>r rFr r r rrorkrrFlinear)r rxargsrprevious_dtyperesultrarsrtr r rurvdropoutr!r!r"forwards,       & zLinear.forwardcst}d|S)Nzvera.)rU__repr__)r reprWr!r"rs zLinear.__repr__)rr*FFTr()FN)r$N)rKrLrMrstrintrRr&r#rr8rcrgr4Tensorr]rr __classcell__r!r!rWr"rsB      & '"r)rdtypingrr4torch.nnrtorch.nn.functional functionalrztransformers.pytorch_utilsrpeft.tuners.tuners_utilsrrpeft.utils.otherr _buffer_dictrr rr!r!r!r"s     p