o
    ߥib"                     @   s4  d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZm	Z	 d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ dgZ,e"j-e)j.ej.dG dd de Z/dS )    N)AnyDictOptionalUnion)disable_verbosityenable_sliced_attention)create_modelload_state_dict)DDIMSampler)Image)Models)Tensor)
TorchModel)MODELS)compatible_position_ids)Config)	ModelFileTasks)
get_logger
ControlNet)module_namec                       s\   e Zd Zdef fddZdd Zdd Zdd	 Zd
eee	f deee	f fddZ
  ZS )r   	model_dirc                    s  t  j|g|R i | || _ttj| jtj	| _
| j
jdd}| j
jj| _| j
j| _t  |r8t  |dd}|dkrG| jj}nP|dkrP| jj}nG|dkrY| jj}n>|dkrb| jj}n5|dkrk| jj}n,|d	krt| jj}n#|d
kr}| jj}n|dkr| jj}n|dkr| jj}ntd | jj}|| _|| _tj| j|j}tj| j|j}|dd}	|	dkrd}	t| }
t||	d}t |d |
| |
!|	| _"t#| j"| _$dS )aJ  initialize ControlNet from theq `model_dir` path.
        ControlNet:
            Adding Conditional Control to Text-to-Image Diffusion Models.
            Paper: https://arxiv.org/abs/2302.05543
            Origin codes: https://github.com/lllyasviel/ControlNet
        Args:
            model_dir (str): the model path.
        r   Tcontrol_typehedscribblecannyhoughdepthnormalposesegfake_scribblez&Error input type, use HED for default!devicecudagpu)locationz?cond_stage_model.transformer.text_model.embeddings.position_idsN)%super__init__r   r   	from_fileospathjoinr   CONFIGURATIONconfigmodelsettinggetimage_resolutionr   r   r   r   r   r   r   r   r   r    printinit_control_typeinput_setting	yaml_path	ckpt_pathr   cpur	   r   tomodelr
   ddim_sampler)selfr   argskwargs	enable_sar2   r3   r4   r5   r"   r8   
state_dict	__class__ q/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/cv/controllable_image_generation/controlnet.pyr'   '   sd   	











zControlNet.__init__c                 C      | j S N)r0   r:   rA   rA   rB   get_resolutione      zControlNet.get_resolutionc                 C   rC   rD   )r.   rE   rA   rA   rB   
get_configh   rG   zControlNet.get_configc                 C   rC   rD   )r   rE   rA   rA   rB   get_model_dirk   rG   zControlNet.get_model_dirinputsreturnc                    s(  |d }|d }|d }| dd}| d| jj}| d| jj}| d| jj}| d	| jj}	| d
| jj}
| d| jj}| d| jj}| d| jj	t
d| d d t
d| d| d| d|	  t
d|
 d| d t  |j\}}}t|   d  tj fddt|D dd t d  |r| jjdd  g| j|d |
 g| gd }|rt n g| j|g| gd }d!|d" |d" f}|r| jjd#d |rfd$dtd%D ngd% | j_| jj||||d|	||d&\}}|r| jjdd | j|td'd( d(    !dd)"t#j$fd*dt|D }W d+   n	1 sJw   Y  | j%d,krdt&j'|t#j(d-t#j$d.d/d0}n(| j%d1kry|d+d+d+d+d+d+d2f }n| j%d3ks| j%d4krd)| }n|}|||d5 d6S )7a  return the result by the model
        Args:
            inputs (Dict[str, Any]) should contains the keys:
            - "image", a numpy array
            - "prompt", string of prompt
            - "detected_map", a numpy array of detected map
            - "save_memory", boolean indicating whether to save memory
            - "is_cat_img", boolean indicating whether to concatenate results

            inputs (Dict[str, Any]) can also contains the keys, but not required:
            - "image_resolution", int
            - "strength", float
            - "guess_mode", bool
            - "ddim_steps", int
            - "scale", float
            - "num_samples", int
            - "eta", float
            - "a_prompt", string of added prompt
            - "n_prompt", string of negative prompt

        Returns:
            Dict[str, Any]: A dict contains result, detected_map and boolean 'is_cat_img'
            indicating whether to concatenate the result and the detected_map.

        imagepromptdetected_mapsave_memoryFnum_samplesscale
ddim_stepsetaa_promptn_prompt
guess_modestrengthzProcess with guess_mode:z
,strength:,znum_samples:z,scale:z,ddim_steps:z,eta:z
a_prompt:'z',n_prompt:'z',g     o@c                    s   g | ]} qS rA   rA   ).0_)controlrA   rB   
<listcomp>   s    z&ControlNet.forward.<locals>.<listcomp>r   )dimzb h w c -> b c h w)is_diffusingz, )c_concatc_crossattn      Tc                    s    g | ]} d t d|   qS )gffffff?   )floatrY   i)rW   rA   rB   r\      s       )verboserS   unconditional_guidance_scaleunconditional_conditioningzb c h w -> b h w cg     _@   c                    s   g | ]} | qS rA   rA   re   )	x_samplesrA   rB   r\      s    Nr   )   rm   )shapedtype   )
iterationsr   r!   r   
is_cat_img)resultrN   rs   ))r/   r3   rP   rQ   rR   rS   rT   rU   rV   rW   r1   torchno_gradrn   
from_numpycopyrd   r#   stackrangeeinops	rearrangecloner8   low_vram_shiftget_learned_conditioning
zeros_likecontrol_scalesr9   sampledecode_first_stager6   numpyclipastypenpuint8r2   cv2dilateones)r:   rJ   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   HWCcondun_condrn   samplesintermediatesresultsshow_det_maprA   )r[   rW   rl   rB   forwardn   s   

 	


6
zControlNet.forward)__name__
__module____qualname__strr'   rF   rH   rI   r   r   r   __classcell__rA   rA   r?   rB   r   "   s    >*)0mathr)   randomsystempfiletypingr   r   r   r   r   r{   r   r   ru   torch.nnnncontrol_ldm.cldm.hackr   r   control_ldm.cldm.modelr   r	   %control_ldm.ldm.models.diffusion.ddimr
   PILr   modelscope.metainfor   modelscope.models.baser   'modelscope.models.base.base_torch_modelr   modelscope.models.builderr   -modelscope.utils.compatible_with_transformersr   modelscope.utils.configr   modelscope.utils.constantr   r   modelscope.utils.loggerr   __all__register_modulecontrollable_image_generationr   rA   rA   rA   rB   <module>   s:   