o
    pi                     @   s8  d dl Z d dlmZ d dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlZd dlZd dlm  mZ d dlmZ d dlmZ dd	lmZmZmZmZmZmZ dd
lmZmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddlm,Z, e# rd dl-m.Z.m/Z/m0Z0 e&1e2Z3dZ4dZ5G dd dZ6dS )    N)defaultdict)nullcontext)Path)CallableDictUnion)validate_hf_hub_args)nn   )ImageProjectionIPAdapterFaceIDImageProjection"IPAdapterFaceIDPlusImageProjectionIPAdapterFullImageProjectionIPAdapterPlusImageProjectionMultiIPAdapterImageProjection)load_model_dict_into_metaload_state_dict)	USE_PEFT_BACKEND_get_model_fileconvert_unet_state_dict_to_peftget_adapter_nameget_peft_kwargsis_accelerate_availableis_peft_versionis_torch_versionlogging   )LORA_WEIGHT_NAMELORA_WEIGHT_NAME_SAFETEXT_ENCODER_NAME	UNET_NAME)AttnProcsLayers)AlignDevicesHook
CpuOffloadremove_hook_from_modulez$pytorch_custom_diffusion_weights.binz,pytorch_custom_diffusion_weights.safetensorsc                   @   s   e Zd ZdZeZeZede	e
ee
ejf f fddZdd Zdd Zed	d
 Z				dde	e
ejf dede
dedef
ddZdd Zd ddZd ddZd ddZdd ZdS )!UNet2DConditionLoadersMixinz:
    Load LoRA layers into a [`UNet2DCondtionModel`].
    %pretrained_model_name_or_path_or_dictc                 K   sJ  | dd}| dd}| dd}| dd}| dd}| dd}| d	d}	| d
d}
| dd}| dd}| dd}| dd}d}|du rRd}d}ddd}d}t|ts|rd|
du sm|
dur|
drzt||
pst|||||||	|d
}tjj|dd}W n t	y } z
|s|W Y d}~nd}~ww |du rt||
pt
|||||||	|d
}t|}n|}tdd | D }tdd | D }d}d}|r| j|d}n|r| j|| j|||d\}}nt| d|r|dur| j|d\}}| | | j| j| jd |r|  dS |r#|  dS dS )a  
        Load pretrained attention processor layers into [`UNet2DConditionModel`]. Attention processor layers have to be
        defined in
        [`attention_processor.py`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py)
        and be a `torch.nn.Module` class. Currently supported: LoRA, Custom Diffusion. For LoRA, one must install
        `peft`: `pip install -U peft`.

        Parameters:
            pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`):
                Can be either:

                    - A string, the model id (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on
                      the Hub.
                    - A path to a directory (for example `./my_model_directory`) containing the model weights saved
                      with [`ModelMixin.save_pretrained`].
                    - A [torch state
                      dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).

            cache_dir (`Union[str, os.PathLike]`, *optional*):
                Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
                is not used.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.

            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            local_files_only (`bool`, *optional*, defaults to `False`):
                Whether to only load local model weights and configuration files or not. If set to `True`, the model
                won't be downloaded from the Hub.
            token (`str` or *bool*, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
                `diffusers-cli login` (stored in `~/.huggingface`) is used.
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
                allowed by Git.
            subfolder (`str`, *optional*, defaults to `""`):
                The subfolder location of a model file within a larger model repository on the Hub or locally.
            network_alphas (`Dict[str, float]`):
                The value of the network alpha used for stable learning and preventing underflow. This value has the
                same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this
                link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning).
            adapter_name (`str`, *optional*, defaults to None):
                Adapter name to be used for referencing the loaded adapter model. If not specified, it will use
                `default_{i}` where i is the total number of adapters being loaded.
            weight_name (`str`, *optional*, defaults to None):
                Name of the serialized state dict file.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.unet.load_attn_procs(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic"
        )
        ```
        	cache_dirNforce_downloadFproxieslocal_files_onlytokenrevision	subfolderweight_nameuse_safetensorsadapter_name	_pipelinenetwork_alphasTattn_procs_weightspytorch)	file_type	frameworkz.safetensors)	weights_namer'   r(   r)   r*   r+   r,   r-   
user_agentcpu)devicec                 s       | ]}d |v V  qdS )custom_diffusionN .0kr=   r=   T/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/loaders/unet.py	<genexpr>       z>UNet2DConditionLoadersMixin.load_attn_procs.<locals>.<genexpr>c                 s   s"    | ]}d |v p| dV  qdS )loraz.alphaN)endswithr>   r=   r=   rA   rB      s     )
state_dict)rF   unet_identifier_keyr2   r0   r1   zQ does not seem to be in the correct format expected by Custom Diffusion training.)r1   dtyper:   )pop
isinstancedictrE   r   r   safetensorstorch	load_fileIOErrorr   r   anykeysall_process_custom_diffusion_process_lora	unet_name
ValueError_optionally_disable_offloadingset_attn_processortorI   r:   enable_model_cpu_offloadenable_sequential_cpu_offload)selfr&   kwargsr'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   allow_pickler8   
model_filerF   eis_custom_diffusionis_lorais_model_cpu_offloadis_sequential_cpu_offloadattn_processorsr=   r=   rA   load_attn_procsD   s   A
	
z+UNet2DConditionLoadersMixin.load_attn_procsc                 C   s@  ddl m} i }tt}| D ]L\}}t|dkri ||< qd|v r=d|dd d d|ddd  }}nd|dd d d|ddd  }}||| |< q| D ]<\}}	t|	dkrv|ddd d d	||< qa|	d
 jd }
|	d
 jd }d|	v rdnd}|d|||
d	||< || 	|	 qa|S )Nr
   )CustomDiffusionAttnProcessorr   to_out.F)train_kvtrain_q_outhidden_sizecross_attention_dimzto_k_custom_diffusion.weightr   zto_q_custom_diffusion.weightT)
models.attention_processorrh   r   rL   itemslenjoinsplitshaper   )r]   rF   rh   rf   custom_diffusion_grouped_dictkeyvalueattn_processor_keysub_key
value_dictrp   ro   rn   r=   r=   rA   rT      s4   
42
z5UNet2DConditionLoadersMixin._process_custom_diffusionc                    s  t stdddlm}m}m} t| }	fdd|	D fdd| D }
|d urFfdd| D   fd	d| D }d
}d
}t	|
dkrR|
n|}t	|dkr|t
| di v rjtd| dt|}|d urvt|}i }| D ]\}}d|v r|jd ||< q|t|||dd}d|v r|d rtddrtdn
tddr|d |di |}|d u rt| }| |\}}||| |d || ||}|d urt
|dd }|rtd| d ||fS )Nz)PEFT backend is required for this method.r   )
LoraConfiginject_adapter_in_modelset_peft_model_state_dictc                       g | ]	}|  r|qS r=   
startswithr>   rG   r=   rA   
<listcomp>      z=UNet2DConditionLoadersMixin._process_lora.<locals>.<listcomp>c                    s,   i | ]\}}|v r|   d d|qS rj    replacer?   r@   v)rG   	unet_keysr=   rA   
<dictcomp>      &z=UNet2DConditionLoadersMixin._process_lora.<locals>.<dictcomp>c                    r   r=   r   r>   r   r=   rA   r   $  r   c                    s,   i | ]\}}| v r|  d d|qS r   r   r   )
alpha_keysrG   r=   rA   r   %  r   Fpeft_configzAdapter name z? already in use in the Unet - please select a new adapter name.lora_Br   T)is_unetuse_dora<z0.9.0zeYou need `peft` 0.9.0 at least to use DoRA-enabled LoRAs. Please upgrade your installation of `peft`.)r0   unexpected_keyszXLoading adapter weights from state_dict led to unexpected keys not found in the model:  z. r=   )r   rW   peftr}   r~   r   listrR   rr   rs   getattrr   rv   r   r   rJ   r   rX   loggerwarning)r]   rF   rG   r2   r0   r1   r}   r~   r   rR   unet_state_dictrd   re   state_dict_to_be_usedrankrx   vallora_config_kwargslora_configincompatible_keysr   r=   )r   rG   r   rA   rU     sl   



z)UNet2DConditionLoadersMixin._process_lorac                 C   s   d}d}|durL|j du rL|j D ]9\}}t|tjrKt|drK|s)t|jt}|s@t|jt	p?t|jdo?t|jj
d t	}td t||d q||fS )ar  
        Optionally removes offloading in case the pipeline has been already sequentially offloaded to CPU.

        Args:
            _pipeline (`DiffusionPipeline`):
                The pipeline to disable offloading for.

        Returns:
            tuple:
                A tuple indicating if `is_model_cpu_offload` or `is_sequential_cpu_offload` is True.
        FN_hf_hookhooksr   zAccelerate hooks detected. Since you have called `load_lora_weights()`, the previous hooks will be first removed. Then the LoRA parameters will be loaded and the hooks will be applied again.)recurse)hf_device_map
componentsrr   rK   r	   Modulehasattrr   r#   r"   r   r   infor$   )clsr1   rd   re   _	componentr=   r=   rA   rX   a  s$   z:UNet2DConditionLoadersMixin._optionally_disable_offloadingTNsave_directoryis_main_processr.   save_functionsafe_serializationc                    sP  ddl m mm tj|rtd| d dS t	 fdd| j
 D }|rZ|  }|du rY|rYdd	 | D }	t|	d
krPtd|	   dd	 | D }nts`tdd
dlm}
 |
| }|du rx|rudd }ntj}tj|dd |du r|r|rtnt}n|rtnt}t|| }||| td|  dS )az  
        Save attention processor layers to a directory so that it can be reloaded with the
        [`~loaders.UNet2DConditionLoadersMixin.load_attn_procs`] method.

        Arguments:
            save_directory (`str` or `os.PathLike`):
                Directory to save an attention processor to (will be created if it doesn't exist).
            is_main_process (`bool`, *optional*, defaults to `True`):
                Whether the process calling this is the main process or not. Useful during distributed training and you
                need to call this function on all processes. In this case, set `is_main_process=True` only on the main
                process to avoid race conditions.
            save_function (`Callable`):
                The function to use to save the state dictionary. Useful during distributed training when you need to
                replace `torch.save` with another method. Can be configured with the environment variable
                `DIFFUSERS_SAVE_MODE`.
            safe_serialization (`bool`, *optional*, defaults to `True`):
                Whether to save the model using `safetensors` or with `pickle`.

        Example:

        ```py
        import torch
        from diffusers import DiffusionPipeline

        pipeline = DiffusionPipeline.from_pretrained(
            "CompVis/stable-diffusion-v1-4",
            torch_dtype=torch.float16,
        ).to("cuda")
        pipeline.unet.load_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin")
        pipeline.unet.save_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin")
        ```
        r
   rh   CustomDiffusionAttnProcessor2_0$CustomDiffusionXFormersAttnProcessorzProvided path (z#) should be a directory, not a fileNc                 3   s$    | ]\}}t | fV  qd S )NrK   )r?   r   xr   r=   rA   rB     s    
z>UNet2DConditionLoadersMixin.save_attn_procs.<locals>.<genexpr>c                 S   s"   i | ]\}}t |tjs||qS r=   rK   rN   Tensorr   r=   r=   rA   r        " z?UNet2DConditionLoadersMixin.save_attn_procs.<locals>.<dictcomp>r   zfSafetensors does not support saving dicts with non-tensor values. The following keys will be ignored: c                 S   s"   i | ]\}}t |tjr||qS r=   r   r   r=   r=   rA   r     r   zOPEFT backend is required for saving LoRAs using the `save_attn_procs()` method.)get_peft_model_state_dictc                 S   s   t jj| |ddidS )Nformatpt)metadata)rM   rN   	save_file)weightsfilenamer=   r=   rA   r     s   zBUNet2DConditionLoadersMixin.save_attn_procs.<locals>.save_functionT)exist_okzModel weights saved in )rq   rh   r   r   ospathisfiler   errorrQ   rf   rr    _get_custom_diffusion_state_dictrs   r   rR   r   rW   
peft.utilsr   rN   savemakedirs!CUSTOM_DIFFUSION_WEIGHT_NAME_SAFEr   CUSTOM_DIFFUSION_WEIGHT_NAMEr   r   as_posixr   )r]   r   r   r.   r   r   r^   rb   rF   empty_state_dictr   	save_pathr=   r   rA   save_attn_procs  sF   )

z+UNet2DConditionLoadersMixin.save_attn_procsc                    sl   ddl m mm t fdd| j D }| }| j D ]\}}t| dkr3i ||< q#|S )Nr
   r   c                    s&   i | ]\}}t | fr||qS r=   r   )r?   yr   r   r=   rA   r     s    zPUNet2DConditionLoadersMixin._get_custom_diffusion_state_dict.<locals>.<dictcomp>r   )	rq   rh   r   r   r!   rf   rr   rF   rs   )r]   model_to_saverF   nameattnr=   r   rA   r     s   z<UNet2DConditionLoadersMixin._get_custom_diffusion_state_dictFc                 C   s  |rt  rddlm} nd}td |du r tdds tdi }d }|r(|nt}d	|v rod
}|d	 jd }|d	 jd d
 }	|  t	|	||d}W d    n1 sUw   Y  |
 D ]\}
}|
dd}|||< q^nLd|v r|d jd }|d jd }	|  t|	|d}W d    n1 sw   Y  |
 D ]\}
}|
dd}|dd}|dd}|||< qn d|v r|d jd }|d jd }|d jd }|d jd }|d jd d }|  t|||||d}W d    n1 sw   Y  |
 D ]\}
}|
dd}|d d!}|d"d#}|d$d%}|d&d'}|d(d)}|d*d+}|d,d-}|d.d/}|d0d1}|d2d3}|d4d5}|d6d7}|d8d9}|d:d;}|d<d=}|d>d?}|d@dA}|dBdC}|dDdE}|dFdG}|dHdI}dJ|v r|||dKdL< qdM|v r|||dNdO< qdP|v r|jdQddR}|d ||dPdS< |d ||dPdT< qdU|v r|||dUdV< qd|kr||dW< qdX|kr||dY< qdZ|kr||d[< qd\|kr	||d]< q|||< qnd^|v rr|d jd }|d jd }|| }d^}|| jd }	|dZ jd |	 }|  t|	|||d_}W d    n	1 sQw   Y  |
 D ]\}
}|
dd}|dd}|||< qZnI|d` jd }|da jd }|db jd }|d` jdQ }tdcdd |D }|r|de jd d n|df jd d }|  t|||||dg}W d    n	1 sw   Y  |
 D ]\}
}|
d dh}|didj}|dkdl}|dmdn}|dodp}|dqdr}|dsdt}|dudv}|dwdx}dP|v r7|dy}dz|dQ< dy|}|jdQddR}|d ||dPdS< |d ||dPdT< qd{|v rP|dy}dz|dQ< dy|}|||< qdU|v rm|dy}dz|dQ< dy|}|||dUdV< q|d|d}}|d~d}|dd}|dd}|dd}|dd}|dd}|dd}|dd}|dd}|dd}|dd}|||< q|s|j|dd |S t||| j| jd |S )Nr   init_empty_weightsF,  Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: 
```
pip install accelerate
```
.T>=1.9.0~Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set `low_cpu_mem_usage=False`.proj.weight   )rp   image_embed_dimnum_image_text_embedsprojimage_embedsproj.3.weightzproj.0.weight)rp   r   zproj.0zff.net.0.projzproj.2zff.net.2zproj.3norm"perceiver_resampler.proj_in.weightr   z#perceiver_resampler.proj_out.weightz*perceiver_resampler.layers.0.0.to_q.weight@   )
embed_dimsoutput_dimshidden_dimsheadsid_embeddings_dimzperceiver_resampler.r   z0.tozattn.toz0.1.0.z0.ff.0.z0.1.1.weightz0.ff.1.net.0.proj.weightz0.1.3.weightz0.ff.1.net.2.weightz1.1.0.z1.ff.0.z1.1.1.weightz1.ff.1.net.0.proj.weightz1.1.3.weightz1.ff.1.net.2.weightz2.1.0.z2.ff.0.z2.1.1.weightz2.ff.1.net.0.proj.weightz2.1.3.weightz2.ff.1.net.2.weightz3.1.0.z3.ff.0.z3.1.1.weightz3.ff.1.net.0.proj.weightz3.1.3.weightz3.ff.1.net.2.weightz
layers.0.0zlayers.0.ln0z
layers.0.1zlayers.0.ln1z
layers.1.0zlayers.1.ln0z
layers.1.1zlayers.1.ln1z
layers.2.0zlayers.2.ln0z
layers.2.1zlayers.2.ln1z
layers.3.0zlayers.3.ln0z
layers.3.1zlayers.3.ln1norm1z0.norm10norm2z0.norm21to_kvr
   )dimto_kto_vri   zto_out.0zproj.net.0.proj.weightzproj.0.biaszproj.net.0.proj.biaszproj.2.weightzproj.net.2.weightzproj.2.biaszproj.net.2.biasnorm.weight)rp   r   mult
num_tokenslatentszproj_in.weightzproj_out.weightc                 s   r;   )r   Nr=   r>   r=   r=   rA   rB     rC   zZUNet2DConditionLoadersMixin._convert_ip_adapter_image_proj_to_diffusers.<locals>.<genexpr>zlayers.0.attn.to_q.weightzlayers.0.0.to_q.weight)r   r   r   r   num_queriesz2.toz	0.0.norm1z0.ln0z	0.0.norm2z0.ln1z	1.0.norm1z1.ln0z	1.0.norm2z1.ln1z	2.0.norm1z2.ln0z	2.0.norm2z2.ln1z	3.0.norm1z3.ln0z	3.0.norm2z3.ln1rj   r   to_qz0.1.0z0.ff.0z0.1.1z0.ff.1.net.0.projz0.1.3z0.ff.1.net.2z1.1.0z1.ff.0z1.1.1z1.ff.1.net.0.projz1.1.3z1.ff.1.net.2z2.1.0z2.ff.0z2.1.1z2.ff.1.net.0.projz2.1.3z2.ff.1.net.2z3.1.0z3.ff.0z3.1.1z3.ff.1.net.0.projz3.1.3z3.ff.1.net.2)strictr:   rI   )r   
accelerater   r   r   r   NotImplementedErrorr   rv   r   rr   r   r   r   chunkr   rQ   r   ru   rt   r   r   r:   rI   )r]   rF   low_cpu_mem_usager   updated_state_dictimage_projectioninit_contextr   clip_embeddings_dimrp   rx   ry   diffusers_namer   r   r   r   r   v_chunkid_embeddings_dim_inid_embeddings_dim_out
multiplier
norm_layerr   attn_key_presentpartsr=   r=   rA   +_convert_ip_adapter_image_proj_to_diffusers  sp  


	








-	








zGUNet2DConditionLoadersMixin._convert_ip_adapter_image_proj_to_diffusersc              	   C   s  ddl m}m} |rt rddlm} nd}td |du r(tdd	s(t	d
i }d}|r0|nt
}| j D ]}	|	drAd n| jj}
|	drQ| jjd }n+|	drit|	td }tt| jj| }n|	dr|t|	td }| jj| }|
d u sd|	v r| j|	 j}| ||	< q7ttdr|n|}g }|D ]>}d|d v r|dg7 }qd|d v r|dg7 }qd|d v r|dg7 }qd|d v r|dg7 }q||d d jd g7 }q|  |||
d|d||	< W d    n1 sw   Y  i }t|D ](\}}|d| d |d! | d" i |d#| d |d! | d$ i q|s3||	 | ntt| j }tt| j!}t"||	 |||d% |d7 }q7|S )&Nr
   )IPAdapterAttnProcessorIPAdapterAttnProcessor2_0r   r   Fr   Tr   r   r   r   zattn1.processor	mid_blockr   	up_blocksz
up_blocks.down_blockszdown_blocks.motion_modulesscaled_dot_product_attentionr   
image_projr   r   i  r   r   r   g      ?)ro   rp   scaler   zto_k_ip.z.weight
ip_adapterz.to_k_ip.weightzto_v_ip.z.to_v_ip.weightr   )#rq   r
  r  r   r   r   r   r   r   r   r   rf   rR   rE   configrp   r   block_out_channelsintrs   r   reversed	__class__r   Frv   	enumerateupdater   nextitervaluesr:   rI   r   )r]   state_dictsr   r
  r  r   
attn_procskey_idr   r   rp   ro   block_idattn_processor_classr   rF   r|   ir:   rI   r=   r=   rA   %_convert_ip_adapter_attn_to_diffusers  sx   


$&
zAUNet2DConditionLoadersMixin._convert_ip_adapter_attn_to_diffusersc                 C   s   t |ts|g}| jd ur| jjdkrt| ds| j| _d | _| j||d}| | g }|D ]}| j	|d |d}|
| q/t|| _d| j_| j| j| jd d S )N	text_projtext_encoder_hid_proj)r   r  ip_image_projrH   )rK   r   encoder_hid_projr  encoder_hid_dim_typer   r'  r%  rY   r	  appendr   rZ   rI   r:   )r]   r  r   r   image_projection_layersrF   image_projection_layerr=   r=   rA   _load_ip_adapter_weights7  s&   



z4UNet2DConditionLoadersMixin._load_ip_adapter_weightsc                 C   s  i }t | j D ]\}}t |D ]\}}| d|d v r||vr&i ||< || d| d|d | d i || d| d|d | d i || d| d|d | d i || d| d|d | d i || d| d|d | d i || d| d|d | d i || d| d	|d | d	 i || d| d
|d | d
 i qq	|S )Nz.to_k_lora.down.weightr  zunet.z.to_q_lora.down.weightz.to_v_lora.down.weightz.to_out_lora.down.weightz.to_k_lora.up.weightz.to_q_lora.up.weightz.to_v_lora.up.weightz.to_out_lora.up.weight)r  rf   rR   r  )r]   r  
lora_dictsr!  r   r$  rF   r=   r=   rA   _load_ip_adapter_lorasW  s`   0z2UNet2DConditionLoadersMixin._load_ip_adapter_loras)TNNT)F)__name__
__module____qualname____doc__r   text_encoder_namer    rV   r   r   strr   rN   r   rg   rT   rU   classmethodrX   r   PathLikeboolr   r   r   r	  r%  r.  r0  r=   r=   r=   rA   r%   <   s@    " )"R
%
c
 
Z
Z r%   )7r   collectionsr   
contextlibr   pathlibr   typingr   r   r   rM   rN   torch.nn.functionalr	   
functionalr  huggingface_hub.utilsr   models.embeddingsr   r   r   r   r   r   models.modeling_utilsr   r   utilsr   r   r   r   r   r   r   r   r   lora_pipeliner   r   r   r    r!   accelerate.hooksr"   r#   r$   
get_loggerr1  r   r   r   r%   r=   r=   r=   rA   <module>   s*    ,
