o
    ۷iGZ                     @   s  d Z ddlZddlZddlmZ ddlmZ e rddlZeeZ	G dd dej
Zdd	d
dddddddddZi ddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7id8d9 edd:D d;d9 edd:D d<d=d>d?d@dAd9 edd:D dBd9 edd:D dCdDdEZdFdGdHdIdJdKdLdMdddNdOdPZdFdGdHdIdJdKdLdMdddQ
Zi dFdRdGdSdHdTdIdUdJdVdKdWdLdXdMdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdiZdRdSdTdUdVdWdXdYdjdkdldmdnZdodpdqZejeejeiZejeejeiZejeiZdrdsiZdtdu ZddvdwZddxdyZdzd{ Zd|d} Zd~d Z dddZ!dddZ"de#fddZ$dS )zI
State dict utilities: utility methods for converting state dicts easily
    N   )is_torch_available)
get_loggerc                   @   s    e Zd ZdZdZdZdZdZdS )StateDictTypez6
    The mode to use when converting state dicts.
    diffusers_oldkohya_sspeft	diffusersN)__name__
__module____qualname____doc__DIFFUSERS_OLDKOHYA_SSPEFT	DIFFUSERS r   r   V/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/utils/state_dict_utils.pyr       s    r   z.to_out.0.lora_Bz.to_out.0.lora_Az.to_q.lora_Az.to_q.lora_Bz.to_k.lora_Az.to_k.lora_Bz.to_v.lora_Az.to_v.lora_Bz.lora_B.lora_Az.to_out.0.lora_magnitude_vector).to_out_lora.up.to_out_lora.down.to_q_lora.down.to_q_lora.up.to_k_lora.down.to_k_lora.up.to_v_lora.down.to_v_lora.upz.lora.upz
.lora.down.to_out.lora_magnitude_vectorz
.to_q.downz.to_q.lora_A.weightz.to_q.upz.to_q.lora_B.weightz
.to_k.downz.to_k.lora_A.weightz.to_k.upz.to_k.lora_B.weightz
.to_v.downz.to_v.lora_A.weightz.to_v.upz.to_v.lora_B.weightz.to_out.0.downz.to_out.0.lora_A.weightz.to_out.0.upz.to_out.0.lora_B.weightz.ff.net.0.proj.downz.ff.net.0.proj.lora_A.weightz.ff.net.0.proj.upz.ff.net.0.proj.lora_B.weightz.ff.net.2.downz.ff.net.2.lora_A.weightz.ff.net.2.upz.ff.net.2.lora_B.weightz.proj_in.downz.proj_in.lora_A.weightz.proj_in.upz.proj_in.lora_B.weightz.proj_out.downz.proj_out.lora_A.weightz.proj_out.upz.proj_out.lora_B.weightz
.conv.downz.conv.lora_A.weightz.conv.upz.conv.lora_B.weightc                 C   "   i | ]}d | dd | dqS ).conv.down.lora_A.weightr   .0ir   r   r   
<dictcomp>N      " r%      c                 C   r   )r   .up.lora_B.weightr   r"   r   r   r   r%   O   r&   zconv_in.lora_A.weightzconv_in.lora_B.weightz.conv_shortcut.lora_A.weightz.conv_shortcut.lora_B.weight)zconv_in.downz
conv_in.upz.conv_shortcut.downz.conv_shortcut.upc                 C   r   ).linear_r    r!   r   r"   r   r   r   r%   T   r&   c                 C   r   )r*   r(   r)   r   r"   r   r   r   r%   U   r&   ztime_emb_proj.lora_A.weightztime_emb_proj.lora_B.weight)ztime_emb_proj.downztime_emb_proj.upz.q_proj.lora_Bz.q_proj.lora_Az.k_proj.lora_Bz.k_proj.lora_Az.v_proj.lora_Bz.v_proj.lora_Az.out_proj.lora_Bz.out_proj.lora_Aztext_projection.lora_A.weightztext_projection.lora_B.weight).q_proj.lora_linear_layer.up.q_proj.lora_linear_layer.down.k_proj.lora_linear_layer.up.k_proj.lora_linear_layer.down.v_proj.lora_linear_layer.up.v_proj.lora_linear_layer.down.out_proj.lora_linear_layer.up .out_proj.lora_linear_layer.down.lora_linear_layer.up.lora_linear_layer.downz text_projection.lora.down.weightztext_projection.lora.up.weight)
r   r   r   r   r   r   r   r   r3   r4   r+   r,   r-   r.   r/   r0   r1   r2   zto_k.lora_Azto_k.lora.downzto_k.lora_Bzto_k.lora.upzto_q.lora_Azto_q.lora.downzto_q.lora_Bzto_q.lora.upzto_v.lora_Azto_v.lora.downzto_v.lora_Bzto_v.lora.upzto_out.0.lora_Azto_out.0.lora.downzto_out.0.lora_Bzto_out.0.lora.upz.k_proj.lora_magnitude_vectorz.v_proj.lora_magnitude_vectorz.q_proj.lora_magnitude_vectorz.out_proj.lora_magnitude_vector)r   r   r   r   r   r   r   r   z.to_k.lora_magnitude_vectorz.to_v.lora_magnitude_vectorz.to_q.lora_magnitude_vectorr   	lora_downlora_up)lora_Alora_Bz.processor..c                 C   sz   i }|   D ]4\}}t D ]}||v rt| }|||}q| D ]}||v r5|| }|||} nq#|||< q|S )a&  
    Simply iterates over the state dict and replaces the patterns in `mapping` with the corresponding values.

    Args:
        state_dict (`dict[str, torch.Tensor]`):
            The state dict to convert.
        mapping (`dict[str, str]`):
            The mapping to use for conversion, the mapping should be a dictionary with the following structure:
                - key: the pattern to replace
                - value: the pattern to replace with

    Returns:
        converted_state_dict (`dict`)
            The converted state dict.
    )itemsKEYS_TO_ALWAYS_REPLACEkeysreplace)
state_dictmappingconverted_state_dictkvpatternnew_patternr   r   r   convert_state_dict   s   
rE   c                 K   sz   |du r&t dd |  D rtj}nt dd |  D r"tj}ntd|t vr4td| dt| }t| |S )a  
    Converts a state dict to the PEFT format The state dict can be from previous diffusers format (`OLD_DIFFUSERS`), or
    new diffusers format (`DIFFUSERS`). The method only supports the conversion from diffusers old/new to PEFT for now.

    Args:
        state_dict (`dict[str, torch.Tensor]`):
            The state dict to convert.
        original_type (`StateDictType`, *optional*):
            The original type of the state dict, if not provided, the method will try to infer it automatically.
    Nc                 s       | ]}d |v V  qdS to_out_loraNr   r#   rA   r   r   r   	<genexpr>       z-convert_state_dict_to_peft.<locals>.<genexpr>c                 s   rF   lora_linear_layerNr   rI   r   r   r   rJ      rK   -Could not automatically infer state dict typeOriginal type  is not supported)anyr<   r   r   r   
ValueErrorPEFT_STATE_DICT_MAPPINGSrE   r>   original_typekwargsr?   r   r   r   convert_state_dict_to_peft   s   
rW   c                    s   | dd  durd   nd |du rFtdd |  D r$tj}n"t fdd|  D r5tj}ntdd |  D rB| S td	|t vrTtd
| dt| }t| |S )a  
    Converts a state dict to new diffusers format. The state dict can be from previous diffusers format
    (`OLD_DIFFUSERS`), or PEFT format (`PEFT`) or new diffusers format (`DIFFUSERS`). In the last case the method will
    return the state dict as is.

    The method only supports the conversion from diffusers old, PEFT to diffusers new for now.

    Args:
        state_dict (`dict[str, torch.Tensor]`):
            The state dict to convert.
        original_type (`StateDictType`, *optional*):
            The original type of the state dict, if not provided, the method will try to infer it automatically.
        kwargs (`dict`, *args*):
            Additional arguments to pass to the method.

            - **adapter_name**: For example, in case of PEFT, some keys will be prepended
                with the adapter name, therefore needs a special handling. By default PEFT also takes care of that in
                `get_peft_model_state_dict` method:
                https://github.com/huggingface/peft/blob/ba0477f2985b1ba311b83459d29895c809404e99/src/peft/utils/save_and_load.py#L92
                but we add it here in case we don't want to rely on that method.
    adapter_nameNr9    c                 s   rF   rG   r   rI   r   r   r   rJ     rK   z2convert_state_dict_to_diffusers.<locals>.<genexpr>c                 3        | ]}d   d|v V  qdS r   .weightNr   rI   peft_adapter_namer   r   rJ         c                 s   rF   rL   r   rI   r   r   r   rJ     rK   rN   rO   rP   )	poprQ   r<   r   r   r   rR   DIFFUSERS_STATE_DICT_MAPPINGSrE   rT   r   r]   r   convert_state_dict_to_diffusers   s    

rb   c                 C   s   t }t| |S )za
    Converts a state dict from UNet format to diffusers format - i.e. by removing some keys
    )UNET_TO_DIFFUSERSrE   )r>   r?   r   r   r   convert_unet_state_dict_to_peft  s   
rd   c                 C   s   dd }|| } t }t| |S )Nc              	      s  dv}t d|rdnd d tdd D }fdd	t|D }d
}dd D }i }|d D ]}|dd}|||< q4dd D }	|	D ]}|dddd}|||< qMdd D }
|
D ]}|dddd}|||< qjtd|D ]  d |d  } d |d  } fdd|  D }|D ]3}|ddddd d!d"d#d$d%d&d'}|d(  d)d*| d+| }|||< qd(  d,v r fd-d|D D ]}|d(  d.d*| d/}|||< q fd0d|  D }|r*|D ]}|d(  d1d*| d2| }|||< qqt|D ]! d3  d4|d5  d6< d3  d7|d5  d8< q/td9d D }fd:d	t|D }| D ]c}t|d d}|d
 dkr|| D ]0}|ddddd d!d"d#d$d%d&d'}|d;| d<| }|||< q}qi|| D ]}|d;| d=| }|||< qqid>|d?< d@|dA< dBd D }t|}td|d D ])}|d }d
| }dC| d6|dD| d6< dC| d8|dD| d8< qdEd D D ]}|dFdG}|||< qdHd D D ]}|dIdJ}|||< q6|S )KNz$input_blocks.11.0.in_layers.0.weightzUsing ControlNet lora (SDXLSD15)c                 S   ,   h | ]}d |v rd |ddd qS )input_blocksr9   N   joinsplitr#   layerr   r   r   	<setcomp>*     , zkconvert_sai_sd_control_lora_state_dict_to_peft.<locals>._convert_controlnet_to_diffusers.<locals>.<setcomp>c                        i | ]   fd dD qS )c                       g | ]}d   |v r|qS )input_blocks.r   r#   keylayer_idr   r   
<listcomp>,      wconvert_sai_sd_control_lora_state_dict_to_peft.<locals>._convert_controlnet_to_diffusers.<locals>.<dictcomp>.<listcomp>r   r#   r>   rw   r   r%   +      zlconvert_sai_sd_control_lora_state_dict_to_peft.<locals>._convert_controlnet_to_diffusers.<locals>.<dictcomp>rj   c                 S      g | ]}d |v r|qS )z0.opr   ru   r   r   r   ry   2      zlconvert_sai_sd_control_lora_state_dict_to_peft.<locals>._convert_controlnet_to_diffusers.<locals>.<listcomp>r   zinput_blocks.0.0conv_inc                 S   r   )
time_embedr   ru   r   r   r   ry   ;  r   ztime_embed.0ztime_embedding.linear_1ztime_embed.2ztime_embedding.linear_2c                 S   r   )	label_embr   ru   r   r   r   ry   C  r   zlabel_emb.0.0zadd_embedding.linear_1zlabel_emb.0.2zadd_embedding.linear_2r   c                    s0   g | ]}d   d|v rd   d|vr|qS )rt   .0.0.opr   ru   r$   r   r   ry   O  s    *zin_layers.0norm1zin_layers.2conv1zout_layers.0norm2zout_layers.3conv2zemb_layers.1time_emb_projskip_connectionconv_shortcutrt   r   zdown_blocks.z	.resnets.z
.0.op.biasc                        g | ]}d   d|v r|qS )rt   r   r   ru   r   r   r   ry   a       r   z.downsamplers.0.convc                    r   )rt   .1r   ru   r   r   r   ry   g  r   r   z.attentions.zzero_convs.z	.0.weightzcontrolnet_down_blocks.r\   z.0.biasz.biasc                 S   rh   )middle_blockr9   Nrj   rk   rn   r   r   r   rp   u  rq   c                    rr   )c                    rs   )middle_block.r   ru   rw   r   r   ry   w  rz   r{   r   r|   r}   rw   r   r%   v  r~   r   zmid_block.resnets.zmid_block.attentions.zmiddle_block_out.0.weightzcontrolnet_mid_block.weightzmiddle_block_out.0.biaszcontrolnet_mid_block.biasc                 S   s<   h | ]}d |v rd|vrd|vrd |ddd qS )input_hint_blockinput_hint_block.0input_hint_block.14r9   Nrj   rk   rn   r   r   r   rp     s    zinput_hint_block.z!controlnet_cond_embedding.blocks.c                 S   r   )r   r   ru   r   r   r   ry     r   r   z!controlnet_cond_embedding.conv_inc                 S   r   )r   r   ru   r   r   r   ry     r   r   z"controlnet_cond_embedding.conv_out)loggerinfolenranger=   getr<   max)r>   is_sdxlnum_input_blocksri   layers_per_block	op_blocksr@   rv   diffusers_keytime_embedding_blockslabel_embedding_blocksblock_idlayer_in_block_idresnets
attentionsnum_middle_blocksmiddle_blocksrA   diffusers_key_hfcond_embedding_blocksnum_cond_embedding_blocksidxdiffusers_idxcond_block_idr   )r$   r>   r    _convert_controlnet_to_diffusers%  s   


"



zXconvert_sai_sd_control_lora_state_dict_to_peft.<locals>._convert_controlnet_to_diffusers)CONTROL_LORA_TO_DIFFUSERSrE   )r>   r   r?   r   r   r   .convert_sai_sd_control_lora_state_dict_to_peft$  s
    
r   c              
   C   sl   zt | }W n ty$ } zt|dkrt| }n W Y d}~nd}~ww tdd | D s4td|S )z
    Attempts to first `convert_state_dict_to_peft`, and if it doesn't detect `lora_linear_layer` for a valid
    `DIFFUSERS` LoRA for example, attempts to exclusively convert the Unet `convert_unet_state_dict_to_peft`
    rN   Nc                 s   s     | ]}d |v pd|v V  qdS )r7   r8   Nr   ru   r   r   r   rJ     r_   z1convert_all_state_dict_to_peft.<locals>.<genexpr>z#Your LoRA was not converted to PEFT)rW   	Exceptionstrrd   rQ   r<   rR   )r>   	peft_dicter   r   r   convert_all_state_dict_to_peft  s   
r   c           	         sh  zddl }W n ty   td  w |dd  dur#d   nd |du r9t fdd|  D r9tj}|t	 vrGt
d	| d
t| t	tj }i }| D ]\\}}d|v rd|dd}n d|v ro|dd}nd|v rz|dd}n
d|v r|dd}|dd|dd }| d}|||< d|v r|dd  d}|t|||< qU|S )a  
    Converts a `PEFT` state dict to `Kohya` format that can be used in AUTOMATIC1111, ComfyUI, SD.Next, InvokeAI, etc.
    The method only supports the conversion from PEFT to Kohya for now.

    Args:
        state_dict (`dict[str, torch.Tensor]`):
            The state dict to convert.
        original_type (`StateDictType`, *optional*):
            The original type of the state dict, if not provided, the method will try to infer it automatically.
        kwargs (`dict`, *args*):
            Additional arguments to pass to the method.

            - **adapter_name**: For example, in case of PEFT, some keys will be prepended
                with the adapter name, therefore needs a special handling. By default PEFT also takes care of that in
                `get_peft_model_state_dict` method:
                https://github.com/huggingface/peft/blob/ba0477f2985b1ba311b83459d29895c809404e99/src/peft/utils/save_and_load.py#L92
                but we add it here in case we don't want to rely on that method.
    r   NzDConverting PEFT state dicts to Kohya requires torch to be installed.rX   r9   rY   c                 3   rZ   r[   r   rI   r]   r   r   rJ     r_   z.convert_state_dict_to_kohya.<locals>.<genexpr>rO   rP   ztext_encoder_2.z	lora_te2.ztext_encoder.z	lora_te1.unet	lora_unetlora_magnitude_vector
dora_scale_rj   r5   z.alpha)torchImportErrorr   errorr`   rQ   r<   r   r   KOHYA_STATE_DICT_MAPPINGSrR   rE   r:   r=   countrm   tensorr   )	r>   rU   rV   r   kohya_ss_partial_state_dictkohya_ss_state_dict	kohya_keyweight	alpha_keyr   r]   r   convert_state_dict_to_kohya  sD   

r   c                    sD    d urt  tr g  fdd|  D } tdd |  D S )Nc                    s,   i | ]\ }t  fd dD r |qS )c                 3   s    | ]}| v V  qd S Nr   )r#   frA   r   r   rJ     rK   z1state_dict_all_zero.<locals>.<dictcomp>.<genexpr>)rQ   )r#   rB   
filter_strr   r   r%     rq   z'state_dict_all_zero.<locals>.<dictcomp>c                 s   s"    | ]}t |d k V  qdS )r   N)r   allitem)r#   paramr   r   r   rJ     s     z&state_dict_all_zero.<locals>.<genexpr>)
isinstancer   r:   r   values)r>   r   r   r   r   state_dict_all_zero  s
   
r   
model_filec                 C   s   dd l }ddlm} |jj| ddd}| pi }W d    n1 s$w   Y  |dd  |r?||}|r=t	|S d S d S )Nr   rj   )LORA_ADAPTER_METADATA_KEYptcpu)	frameworkdeviceformat)
safetensors.torchloaders.lora_baser   r   	safe_openmetadatar`   r   jsonloads)r   safetensorsr   r   r   rawr   r   r   _load_sft_state_dict_metadata  s   
r   r   )%r   enumr   import_utilsr   loggingr   r   r
   r   Enumr   rc   r   r   DIFFUSERS_TO_PEFTDIFFUSERS_OLD_TO_PEFTPEFT_TO_DIFFUSERSDIFFUSERS_OLD_TO_DIFFUSERSPEFT_TO_KOHYA_SSr   r   rS   r   ra   r   r;   rE   rW   rb   rd   r   r   r   r   r   r   r   r   r   r   <module>   sL  	
 	
	

!
/ 

?	