o ۷iII@sddlZddlmmZddlmZddlmZmZddlm Z e r,ddl m Z m Z e eZdejfdd Zdejfd d Zdd efddZGdddejjZGdddejZGdddejZGdddejZGdddejZdS)N)nn) deprecatelogging)is_transformers_available) CLIPTextModelCLIPTextModelWithProjection text_encodercCs`g}t|ttfr't|jjjD]\}}d|d}|j}|||fq|St d|j j )Ntext_model.encoder.layers.z .self_attnz.do not know how to get attention modules for: ) isinstancerr enumerate text_modelencoderlayers self_attnappend ValueError __class____name__)r attn_modulesilayernamemodrK/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/models/lora.pytext_encoder_attn_modules's rcCs`g}t|ttfr't|jjjD]\}}|j}d|d}|||fq|St d|j j )Nr z.mlpz(do not know how to get mlp modules for: ) r rrr r rrmlprrrr)r mlp_modulesrrmlp_modrrrrtext_encoder_mlp_modules5s r ? lora_scalecCspt|D]\}}t|jtr||j_||j_||j_||j_qt|D]\}}t|j tr5||j _||j _q#dSN) rr q_projPatchedLoraProjectionr"k_projv_projout_projr fc1fc2)r r"_ attn_module mlp_modulerrradjust_lora_scale_text_encoderCs  r.csNeZdZdfdd Zddddfd d Zdd d ZddZddZZS)r%Nc spd}tdd|tddlm}||_|jjj}|dur$|jjj}||jj |jj ||||d|_ ||_ dS)NzsUse of `PatchedLoraProjection` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r%1.0.0r)LoRALinearLayer) network_alphadevicedtyperank) rsuper__init__ models.lorar2regular_linear_layerweightr4r5 in_features out_featureslora_linear_layerr") selfr:r"r3r6r5deprecation_messager2r4rrrr8Rs"      zPatchedLoraProjection.__init__F destinationprefix keep_varscs6|jdur|jj||||dStj||||dS)NrC)r>r: state_dictr7)r?rDrErFargsrArrrGms z PatchedLoraProjection.state_dictr!c Cs|jdurdS|jjjj|jjjj}}|jjj}|jjjj}|jjjj}|jj dur;||jj |jj }||t |dddf|dddfd}|rft |rftd|d|j||d|jj_d|_||_||_||_dSNraThis LoRA weight seems to be broken. Encountered NaN values when trying to fuse LoRA weights for .LoRA weights will not be fused.r4r5)r>r:r;datar5r4floatupdownr3r6torchbmmisnananyitemrtocpuw_upw_downr" r?r" safe_fusingr5r4w_origrXrY fused_weightrrr _fuse_loraus(  0   z PatchedLoraProjection._fuse_lorac Cst|dddurt|dddusdS|jjj}|j|j}}|jj|d}|j |}||j t |dddf|dddfd}|j||d|jj_d|_d|_ dSNrXrYr4rrL) getattrr:r;rMr5r4rXrVrNrYr"rQrRr?r]r5r4rXrYunfused_weightrrr _unfuse_loras  6 z"PatchedLoraProjection._unfuse_loracCs>|jdurd|_|jdur||S|||j||S)Nr!)r"r>r:)r?inputrrrforwards   zPatchedLoraProjection.forward)r/Nr0Nr!F) r __module__ __qualname__r8rGr^rdrf __classcell__rrrArr%Qs   r%csneZdZdZ    ddededededBdejeBdBd ej dBf fd d Z d ej d ej fddZ Z S)r2a A linear layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning device (`torch.device`, `optional`, defaults to `None`): The device to use for the layer's weights. dtype (`torch.dtype`, `optional`, defaults to `None`): The dtype to use for the layer's weights. r0Nr<r=r6r3r4r5cstd}tdd|tj||d||d|_tj||d||d|_||_||_||_ ||_ tj j |jj d|dtj |jj dS)NzmUse of `LoRALinearLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r2r1F)biasr4r5r/std)r7r8rrLinearrPrOr3r6r=r<initnormal_r;zeros_)r?r<r=r6r3r4r5r@rArrr8s  zLoRALinearLayer.__init__ hidden_statesreturncCN|j}|jjj}|||}||}|jdur"||j|j9}||Sr#r5rPr;rVrOr3r6r?rr orig_dtyper5down_hidden_statesup_hidden_statesrrrrf    zLoRALinearLayer.forward)r0NNN)rrhri__doc__intrNrQr4strr5r8TensorrfrjrrrArr2s( r2cseZdZdZ     ddededed eeeefBd eeeefBd eeeefBeBd edBffd d Zde j de j fddZ Z S)LoRAConv2dLayera" A convolutional layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. kernel_size (`int` or `tuple` of two `int`, `optional`, defaults to 1): The kernel size of the convolution. stride (`int` or `tuple` of two `int`, `optional`, defaults to 1): The stride of the convolution. padding (`int` or `tuple` of two `int` or `str`, `optional`, defaults to 0): The padding of the convolution. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning r0r/r/rNr<r=r6 kernel_sizestridepaddingr3c std}tdd|tj|||||dd|_tj||dddd|_||_||_tj j |jj d|d tj |jj dS) NzmUse of `LoRAConv2dLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr1F)rrrrkr)rrrkr/rl) r7r8rrConv2drPrOr3r6rorpr;rq) r?r<r=r6rrrr3r@rArrr8s  zLoRAConv2dLayer.__init__rrrscCrtr#rurvrrrrfrzzLoRAConv2dLayer.forward)r0rrrN) rrhrir{r|tupler}rNr8rQr~rfrjrrrArrs.rcszeZdZdZdddedBffddZdedBfddZdd ed efd dZ ddZ dde j dede j fddZ ZS)LoRACompatibleConvz; A convolutional layer that can be used with LoRA. N lora_layerrc,d}tdd|tj|i|||_dS)NzpUse of `LoRACompatibleConv` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr1rr7r8rr?rrHkwargsr@rArrr8.  zLoRACompatibleConv.__init__cCd}tdd|||_dSNznUse of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.set_lora_layerr1rrr?rr@rrrr5s  z!LoRACompatibleConv.set_lora_layerr!Fr"r[c Cs|jdurdS|jjj|jjj}}|jj}|jjjj}|jjjj}|jjdur8||jj|jj }t |j dd|j dd}| |j}|||} |ret | retd|d| j||d|j_d|_||_||_||_dS)Nr/ start_dimrJrKrL)rr;rMr5r4rNrOrPr3r6rQmmflattenreshapeshaperSrTrUrrVrWrXrY _lora_scale) r?r"r[r5r4r\rXrYfusionr]rrrr^;s,        zLoRACompatibleConv._fuse_loracCst|dddurt|dddusdS|jj}|jj|jj}}|jj|d|_|j||_t |jj dd|jj dd}| |j }||j|}|j||d|j_d|_d|_dS)NrXrYr`r/rrL)rar;rMr5r4rXrVrNrYrQrrrrr)r?r]r5r4rrcrrrrd]s    zLoRACompatibleConv._unfuse_lorarrscalersc Csj|jdkrtj||j|jd}d}n|j}t||j|j|j||j |j }|j dur,|S||| |S)Nzeros)mode)rr) padding_modeFpad _reversed_padding_repeated_twicerconv2dr;rkrdilationgroupsr)r?rrrroriginal_outputsrrrrfos  zLoRACompatibleConv.forwardrgr!)rrhrir{rr8rrNboolr^rdrQr~rfrjrrrArr)s"$rcs~eZdZdZdddedBffddZdedBfddZdd ed efd dZ ddZ dde j dede j ffdd Z ZS)LoRACompatibleLinearz4 A Linear layer that can be used with LoRA. Nrrcr)NzrUse of `LoRACompatibleLinear` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr1rrrArrr8rzLoRACompatibleLinear.__init__cCrrrrrrrrs  z#LoRACompatibleLinear.set_lora_layerr!Fr"r[c Cs|jdurdS|jjj|jjj}}|jj}|jjjj}|jjjj}|jjdur8||jj|jj }||t |dddf|dddfd}|rct | rctd|d|j||d|j_d|_||_||_||_dSrI)rr;rMr5r4rNrOrPr3r6rQrRrSrTrUrrVrWrXrYrrZrrrr^s(   0   zLoRACompatibleLinear._fuse_lorac Cst|dddurt|dddusdS|jj}|j|j}}|jj|d}|j|}||j t |dddf|dddfd}|j||d|j_d|_d|_dSr_) rar;rMr5r4rXrVrNrYrrQrRrbrrrrds 6 z!LoRACompatibleLinear._unfuse_lorarrrrscs8|jdur t|}|St||||}|Sr#)rr7rf)r?rrroutrArrrfs  zLoRACompatibleLinear.forwardrgr)rrhrir{r2r8rrNrr^rdrQr~rfrjrrrArrs (rr)rQtorch.nn.functionalr functionalrutilsrrutils.import_utilsr transformersrr get_loggerrloggerModulerr rNr.r%r2rrrrnrrrrrs    \<@W