o Ni7@sddlmZddlZddlZddlZddlmZmZmZddl Z ddl m Z ddl m m Z ddlmZmZddlmZddlmZddlmZGd d d e jeZdddZdS)) annotationsN)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_merge) transpose)gather_params_ctx) LoraLayercseZdZdZ         d0d1fdd Zdejdddfd2ddZd3d d!Zd4d5d(d)Z d6d*d+Z d7d,d-Z d8fd.d/ Z Z S)9LoraParallelLineara When the target layer parallel_linear is RowParallelLinear, in order to keep the input and output shapes consistent, we need to split the lora matrix A into rows, and the lora_B at this time should be a complete linear layer; In the same way, when the target layer is ColumnParallelLinear, we perform column segmentation on lora_B, while lora_A is still a complete linear layer. rr FT adapter_namestrrint lora_alpha lora_dropoutfloatfan_in_fan_outboolis_target_conv_1d_layerinit_lora_weightsUnion[bool, str] use_rslorause_dora lora_biasc  s| r t|jjdttj|fd|i| | r&t|jjd||_t||j|_ ||_ ||_ | d}d|i}t j }t|drI|j}d}d}|j rT|j}n|j}|j||f||| | | |||d||rut|jjd d|_dS) Nz0 does not support lora_bias yet, set it to False base_layerz2 does not support DoRA yet, please set it to Falsemegatron_config init_methodTF)rrrrrrinput_is_parallel gather_outputzB does not support target_conv_1d_layer yet, please set it to False) ValueError __class____name__super__init__r backend isinstanceRowParallelLinear is_parallel_ar_active_adapterinitxavier_normal_hasattrrr r! update_layerr)selfrrr'rrrrrrrrrkwargsrparallel_linear_kwargsrr r!r#M/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lora/tp_layer.pyr&(sP     zLoraParallelLinear.__init__inference_modec  Kst} | d=|dkrtd|||j|<||j|<|dkr(tj|d}nt}||j|<| d}t j |_ |j rV|j j|j|d| d||d }tj||jdt j d }ntj|j|dt j d }|j j||jd| ||d }||j|<||j|<|r|t||j|<n|||j|<||j|<t|tr|d rt|j|||Wdn1swYnt|tr|d rt|j| ||Wdn1swYnWt|tr |!dkr t|j |"|Wdn 1swYn,|dkr/t|j |#|Wdn 1s)wYn |r8|$|||%|||j&vrO|j&|j'|fi| |j(|j)| ddS)Nr0rz?`r` should be a positive integer value but the value passed is r )prFT) input_size output_sizebiasr skip_bias_addrconfig) in_features out_featuresr:dtype)r8r9r:r!rr<pissacordaoloraloftq)r6)*localscopyr"rrnnDropoutIdentityrtorchfloat32 params_dtyper*r'r)r=Linearr>ColumnParallelLinearlora_Alora_Bmathsqrtscalingrr(r startswithr get_base_layerweight pissa_init corda_initlower olora_init loftq_initreset_lora_parameters%_move_adapter_to_device_of_base_layer lora_variantr, set_adapteractive_adapters)r0rrrrrrrrr r!r6r2r1lora_dropout_layerrlora_alora_br4r4r5r/es              zLoraParallelLinear.update_layerx torch.Tensorargsrr1c Os@|j|g|Ri||dd}|jr.|jr||j|g|Ri|\}}||fS|dur;t|jjd|jrP|j|g|Ri|\}}||fS|j|g|Ri|\}}|j }|j D]2}||j vrnqd|j |} |j |} |j|} |j|} ||| jj }|| | | || }qd||}||fS)N adapter_namesz* does not support mixed_batch_forward yet.)_check_forward_argspopdisable_adaptersmergedunmergerr"r#r$r?r_rNkeysrOrrR_cast_input_dtyperUto) r0rcrer1rfresultr:torch_result_dtypeactive_adapterrNrOdropoutrRr4r4r5forwards2       zLoraParallelLinear.forwardN safe_mergerfOptional[list[str]]returnNonecCst||}|s dS|D]E}||jvrP|}|r=|jj}||}||}t | s8t d|d||j_n ||}|jj||j_|j |q dS)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rrNrlrTrUdatacloneget_delta_weightrIisfiniteallr"merged_adaptersappend)r0rtrfrqr orig_weights delta_weightr4r4r5merges(       zLoraParallelLinear.mergecCsr|js tddSt|jdkr7|j}||jvr.|j }| |}|j |8_ t|jdksdSdS)zW This method unmerges all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) rjwarningswarnlenr}rhrNrlrTrUrzrx)r0rqrUrr4r4r5rks    zLoraParallelLinear.unmergecCs|j|jj}|j|jj}|jdko|tjkp|tjk}|j|j}|j|j}|r3| }| }t |||j |j |}|r\|j |d}| ||j|j_| ||j|j_|S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpu)r?)rOrUdevicer?typerIfloat16bfloat16rNrrrrRrnrx)r0adapterrr? cast_to_fp32weight_Aweight_B output_tensorr4r4r5rzs   z#LoraParallelLinear.get_delta_weightcst}d|S)Nzlora.)r%__repr__)r0repr3r4r5r2s zLoraParallelLinear.__repr__) rr r FFTFFF)rrrrrrrrrrrrrrrrrrrr)r6r)rcrdrerr1r)FN)rtrrfrurvrw)rvrw)rvrd)rvr)r$ __module__ __qualname____doc__r&r,r-r/rsrrkrzr __classcell__r4r4r3r5r s. E W  ( "r targettorch.nn.Modulerrr1rrvOptional[torch.nn.Module]c Ksd}t|tr |}n|}|jrt|j}nd}|r`t||jj|jj fr`| }|j}t|t r@|j j j} | di|j}||d<|drTtdd|d<|_td|||jd|}|S)Nrrzfan_in_fan_out is set to True but the target module is `ColumnParallelLinear` or `RowParallelLinear`. Setting fan_in_fan_out to False.F)rrr'r4)r(rrTr importlib import_module megatron_coretensor_parallelrMr)rEdict transformertransformer_configTransformerConfigrrrr ) rr lora_configr1 new_moduletarget_base_layerrmegatron_kwargsrtransformer_config_classr4r4r5dispatch_megatron7s8    r)rrrrr1rrvr) __future__rrrPrtypingrrrrItorch.nnrF torch.nn.initr,peft.tuners.tuners_utilsrr peft.utilsrpeft.utils.integrationsr layerr Moduler rr4r4r4r5s