o
    ۷iU                     @   sx   d dl mZ ddlmZmZ ddlmZ ddlmZ ddl	m
Z
mZmZ ddlmZ e
 r.	 eeZG dd	 d	Zd
S )    )nullcontext   )ImageProjectionMultiIPAdapterImageProjection)load_model_dict_into_meta)_LOW_CPU_MEM_USAGE_DEFAULT)is_accelerate_availableis_torch_versionlogging)empty_device_cachec                   @   s4   e Zd ZdZefddZefddZefddZdS )	FluxTransformer2DLoadersMixinz8
    Load layers into a [`FluxTransformer2DModel`].
    c                 C   s,  |rt  rddlm} nd}td |du r tdds tdi }d }|r(|nt}d	|v rxd
}|d	 jd dkr;d}|d	 jd }|d	 jd | }	|  t	|	||d}W d    n1 s`w   Y  |
 D ]\}
}|
dd}|||< qi|s|j|dd |S d| ji}t|||| jd t  |S )Nr   init_empty_weightsF,  Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: 
```
pip install accelerate
```
.T>=1.9.0~Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set `low_cpu_mem_usage=False`.proj.weight         )cross_attention_dimimage_embed_dimnum_image_text_embedsprojimage_embeds)strict 
device_mapdtype)r   
accelerater   loggerwarningr	   NotImplementedErrorr   shaper   itemsreplaceload_state_dictdevicer   r!   r   )self
state_dictlow_cpu_mem_usager   updated_state_dictimage_projectioninit_contextr   clip_embeddings_dimr   keyvaluediffusers_namer     r5   X/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/loaders/transformer_flux.py+_convert_ip_adapter_image_proj_to_diffusers%   sJ   

zIFluxTransformer2DLoadersMixin._convert_ip_adapter_image_proj_to_diffusersc              
   C   s  ddl m} |rt rddlm} nd}td |du r&tdd	s&td
i }d}|r.|nt	}| j
 D ]}|drH| j
| j}	|	 ||< q5| jj}
| j}|}	g }|D ]}d|d v rqd}|d d jd dkrld}||g7 }qU|  |	||
d|| j| jd||< W d    n1 sw   Y  i }t|D ]L\}}|d| d|d | d i |d| d|d | d i |d| d|d | d i |d| d|d | d i q|s|| | nd| ji}| j}t|| |||d |d7 }q5t  |S )Nr   )FluxIPAdapterAttnProcessorr   r   Fr   Tr   r   r   single_transformer_blocksr   
image_projr   r   r   g      ?)hidden_sizer   scale
num_tokensr!   r*   zto_k_ip.z.weight
ip_adapterz.to_k_ip.weightzto_v_ip.z.to_v_ip.weightz.biasz.to_k_ip.biasz.to_v_ip.biasr   r      )$models.transformers.transformer_fluxr8   r   r"   r   r#   r$   r	   r%   r   attn_processorskeys
startswith	__class__configjoint_attention_dim	inner_dimr&   r!   r*   	enumerateupdater)   r   r   )r+   state_dictsr-   r8   r   
attn_procskey_idr0   nameattn_processor_classr   r;   r   r,   num_image_text_embed
value_dictir    r!   r5   r5   r6   %_convert_ip_adapter_attn_to_diffusersY   sl   


$$$&

zCFluxTransformer2DLoadersMixin._convert_ip_adapter_attn_to_diffusersc                 C   sn   t |ts|g}d | _| j||d}| | g }|D ]}| j|d |d}|| qt|| _d| j_	d S )N)r-   r:   ip_image_proj)

isinstancelistencoder_hid_projrR   set_attn_processorr7   appendr   rE   encoder_hid_dim_type)r+   rJ   r-   rK   image_projection_layersr,   image_projection_layerr5   r5   r6   _load_ip_adapter_weights   s   


z6FluxTransformer2DLoadersMixin._load_ip_adapter_weightsN)__name__
__module____qualname____doc__r   r7   rR   r\   r5   r5   r5   r6   r       s
    4Ir   N)
contextlibr   models.embeddingsr   r   models.model_loading_utilsr   models.modeling_utilsr   utilsr   r	   r
   utils.torch_utilsr   
get_loggerr]   r#   r   r5   r5   r5   r6   <module>   s   
