o
    GiTy                     @   s   d dl mZ d dlZd dlZd dlZd dlm  mZ	 d dlm
Z
 ddlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ eeZG d
d deZdS )    )AnyN)Image   )ConfigMixin)register_to_config)PipelineImageInput)CONFIG_NAMElogging)is_matplotlib_availablec                       s  e Zd ZeZe			dLdededef fddZe	de
jejB d	e
jejB fd
dZe	de
jd	ejfddZe	dejd	e
jfddZe		dMde
jdeeef dededB d	e
jf
ddZe	de
jdeded	e
jfddZe	de
jded	ee
jeeef f fddZe	de
jdeeef d	e
jfddZe	e
d e
jfde
jejB ejB d!e
jd"e
jd	ee
jef fd#d$Ze	de
jd	dfd%d&Zdd'e
d e
jfded(edB d)ed!e
jd"e
jf
d*d+Ze		,	-	dNdeje
jB d.ed/ed0edB d	eje
jB f
d1d2Ze		3	4	,dOd5e jjejB e
jB e!e jj B e!ej B e!e
j B d6e"d7e"d8ed	e!e jj f
d9d:Z#e		3	4dPd5eje
jB e!ej B e!e
j B d6e"d7e"d	e!e jj fd;d<Z$e		-	-	-dQd=eje
jB e!ej B e!e
j B d>ed?ed@ed	e!e jj f
dAdBZ%e		CdRdDeje
jB e!ej B e!e
j B dEe&ee'f d8ee&eef B d	e!e&ee jjf  fdFdGZ(e		HdSdIeje
jB e!ej B e!e
j B d	e!e jj fdJdKZ)  Z*S )TMarigoldImageProcessor   Tvae_scale_factordo_normalizedo_range_checkc                    s   t    d S N)super__init__)selfr   r   r   	__class__ j/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/marigold/marigold_image_processing.pyr   (   s   zMarigoldImageProcessor.__init__imagesreturnc                 C   s~   t | tjr| jdkr| d } | jdkr| d } | S t | tjr6| jdkr+| d } | S | jdkr4| d } | S tdt|  )zK
        Expand a tensor or array to a specified number of images.
           )N.Nr   N)NNUnexpected input type: )
isinstancenpndarrayndimtorchTensor
ValueErrortyper   r   r   r   expand_tensor_or_array1   s   



z-MarigoldImageProcessor.expand_tensor_or_arrayc                 C   s    |   dddd  } | S )z<
        Convert a PyTorch tensor to a NumPy image.
        r   r   r      )cpupermutefloatnumpyr$   r   r   r   pt_to_numpyD   s   z"MarigoldImageProcessor.pt_to_numpyc                 C   s   t | jt jrt | jt jstd| j dt | jt jr*td| j dt | jtr:td| j dt	| 
dddd} | S )	z<
        Convert a NumPy image to a PyTorch tensor.
        Input image dtype= cannot be a signed integer. cannot be complex. cannot be boolean.r   r   r&   r   )r   
issubdtypedtypeintegerunsignedintegerr"   complexfloatingboolr    
from_numpy	transposer$   r   r   r   numpy_to_ptL   s    z"MarigoldImageProcessor.numpy_to_ptNimagesizemodeis_aac                 C   sz   t | stdt|  dt | std| j d|  dkr,td| j d|o1|dv }tj	| |||d} | S )NInvalid input type=.Invalid input dtype=    Invalid input dimensions; shape=)bilinearbicubic)r;   	antialias)
r    	is_tensorr"   r#   is_floating_pointr1   dimshapeFinterpolate)r9   r:   r;   r<   rD   r   r   r   resize_antialias[   s   

z'MarigoldImageProcessor.resize_antialiasmax_edge_szc                 C   s   t | stdt|  dt | std| j d|  dkr,td| j d| jdd  \}}t||}|| | }|| | }|dksN|dkrYtd| d	| d
t	j
| ||f|dd} | S )Nr=   r>   r?   r@   rA   r   z*Extreme aspect ratio of the input image: [z x ]T)r<   )r    rE   r"   r#   rF   r1   rG   rH   maxr   rK   )r9   rL   r;   hwmax_orignew_hnew_wr   r   r   resize_to_max_edgek   s   


z)MarigoldImageProcessor.resize_to_max_edgealignc                 C   s   t | stdt|  dt | std| j d|  dkr,td| j d| jdd  \}}| | | | }}tj	| d|d|fdd	} | ||ffS )
Nr=   r>   r?   r@   rA   rM   r   	replicater;   )
r    rE   r"   r#   rF   r1   rG   rH   rI   pad)r9   rV   rP   rQ   phpwr   r   r   	pad_image   s   

z MarigoldImageProcessor.pad_imagepaddingc                 C   s   t | stdt|  dt | std| j d|  dkr,td| j d|\}}|dkr6d n| }|dkr?d n| }| d d d d d |d |f } | S )Nr=   r>   r?   r@   rA   r   )r    rE   r"   r#   rF   r1   rG   rH   )r9   r]   rZ   r[   uhuwr   r   r   unpad_image   s   

 z"MarigoldImageProcessor.unpad_imager'   devicer1   c                 C   s  t | tjrt| } d }t | tjtjfr$t| } | j	dkr$t
dt | tjr~t| jtjrCt| jtjsCt
d| j dt| jtjrTt
d| j dt| jtrdt
d| j dt| jtjryt| jj}| tj} t| } t| rt| s|d u r| jtjkrt
d| j dd	}t| st
d
t|  d| jd dkr| dddd} | jd dkrt
d| j d| j||d} |d ur| | } | S )Nr@   z,Input image is not 2-, 3-, or 4-dimensional.r,   r-   r.   r/   zImage dtype=z is not supported.   zInput type unsupported: r>   r&   r   z$Input image is not 1- or 3-channel: )ra   r1   )r   r   r   arrayr   r    r!   r   r%   r   r"   r0   r1   r2   r3   r4   r5   iinforO   astypefloat32r8   rE   rF   uint8r#   rH   repeatto)r9   ra   r1   image_dtype_maxr   r   r   load_image_canonical   s@   


 

z+MarigoldImageProcessor.load_image_canonicalc                 C   sf   t | stdt|  dt | std| j d|   dk s-|   dkr1tdd S )Nr=   r>   r?                 ?z9Input image data is partially outside of the [0,1] range.)	r    rE   r"   r#   rF   r1   minitemrO   )r9   r   r   r   check_image_values_range   s   

 z/MarigoldImageProcessor.check_image_values_rangerB   processing_resolutionresample_method_inputc              
   C   s  t |trPd }t|D ]@\}}| |||}|d u r|}q|jdd  |jdd  krBtd| d|jdd   d|jdd   tj||fdd}q|}~n| |||}|jdd  }	| jj	rg| 
| | jjrq|d d }|d ur|dkr| |||}| || jj\}}
||
|	fS )	Nr   zInput image[z] has incompatible dimensions z with the previous images r   )rG   g       @rm   )r   list	enumeraterk   rH   r"   r    catconfigr   rp   r   rU   r\   r   )r   r9   rq   rr   ra   r1   r   iimgoriginal_resolutionr]   r   r   r   
preprocess   s2   


z!MarigoldImageProcessor.preprocessSpectralFcmapbytes_force_methodc                    s   t | st| tjstd|dvrtdddgg dd dd	d
}d fdd	}|du r?t | r?|dkr?|| ||S d}|dkrK|| ||}|dkrW|du rWtd|du ra|| ||}|S )a  
        Converts a monochrome image into an RGB image by applying the specified colormap. This function mimics the
        behavior of matplotlib.colormaps, but allows the user to use the most discriminative color maps ("Spectral",
        "binary") without having to install or import matplotlib. For all other cases, the function will attempt to use
        the native implementation.

        Args:
            image: 2D tensor of values between 0 and 1, either as np.ndarray or torch.Tensor.
            cmap: Colormap name.
            bytes: Whether to return the output as uint8 or floating point image.
            _force_method:
                Can be used to specify whether to use the native implementation (`"matplotlib"`), the efficient custom
                implementation of the select color maps (`"custom"`), or rely on autodetection (`None`, default).

        Returns:
            An RGB-colorized tensor corresponding to the input image.
        z/Argument must be a numpy array or torch tensor.)N
matplotlibcustomzB_force_method must be either `None`, `'matplotlib'` or `'custom'`.)rm   rm   rm   )rl   rl   rl   ))g?gp?g?)g?g??)g?g[[[[[[?g?)g?g?gXXXXXX?)g?g?gqqqqqq?)rm   rm   g?)g?g?g?)guuuuuu?g?g?)g?gXXXXXX?g?)g?g?g?)g?r   gTTTTTT?)binaryr{   Fc              	   S   s   t  rdd l}nd S t| d }}|r|   | j} }||jvr5td| dd	t
|j  |j| }|| |d}|dd df }|rQtj||d}|S )	Nr   zUnexpected color map z; available options are: , )r}   .r   )ra   )r
   r   r    rE   r'   r*   ra   	colormapsr"   joinrs   keystensor)r9   r|   r}   r   	arg_is_ptra   outr   r   r   method_matplotlib2  s    


z:MarigoldImageProcessor.colormap.<locals>.method_matplotlibc                    s:  t | tj}|rt| } | jtjkr|  d } n|  } |d}|r+|d d }| vr;t	dt
   d | }|rH|d d d }tj|tj| jd}|jd }| jdd	d
|d	  }| }|d	 j|d	 d}||  d}	|| }
|| }d	|	 |
 |	|  }|r|d tj}|r| }|S )Nrb   _rrM   zOnly z8 color maps are available without installing matplotlib.r1   ra   r   r&   )rn   rO   )rO   )r   r   r   r    r   r1   rg   r)   endswithr"   rs   r   ra   rH   clamplong	unsqueezeri   r*   )r9   r|   r}   	arg_is_npis_cmap_reversedKposleftrightdleft_colorsright_colorsr   supported_cmapsr   r   method_customJ  s<   


z6MarigoldImageProcessor.colormap.<locals>.method_customNr{   r   r   zUMake sure to install matplotlib if you want to use a color map other than 'Spectral'.)F)r    rE   r   r   r   r"   ImportError)r9   r|   r}   r~   r   r   r   r   r   r   colormap  s*   
*zMarigoldImageProcessor.colormaprl   rm   depthval_minval_max	color_mapc                    s   krt d d dd fdd	| du s*t| tr.tdd | D r.t d	t| tjtjfrgt	| } t| tjrGt
| } | jd
krS| jd dks\t d| j dfddt| D S t| trwfddt| D S t dt|  )a  
        Visualizes depth maps, such as predictions of the `MarigoldDepthPipeline`.

        Args:
            depth (`PIL.Image.Image | np.ndarray | torch.Tensor | list[PIL.Image.Image, list[np.ndarray],
                list[torch.Tensor]]`): Depth maps.
            val_min (`float`, *optional*, defaults to `0.0`): Minimum value of the visualized depth range.
            val_max (`float`, *optional*, defaults to `1.0`): Maximum value of the visualized depth range.
            color_map (`str`, *optional*, defaults to `"Spectral"`): Color map used to convert a single-channel
                      depth prediction into colored representation.

        Returns: `list[PIL.Image.Image]` with depth maps visualization.
        zInvalid values range: [r   z].Nc                    s*  d|r	d| dnd }t | tjjr.| jdkr#t| d| j dt| tjd } t | tj	s9t
| re| jd	krIt| d
| j dt | tj	rTt
| } t
| sdt| d| j dnt| dt|  ddksydkr|    } tj|  dd} tj|   } | S )NDepth[rN    I;16z: invalid PIL mode=r>     r   : unexpected shape=: unexpected dtype=: unexpected type=rl   rm   Tr|   r}   )r   PILr   r;   r"   r   rc   re   rf   r   r    rE   r   rH   r6   rF   r1   r#   r   r   	fromarrayr'   r*   rx   idxprefix)r   r   r   r   r   visualize_depth_one  s&   



zCMarigoldImageProcessor.visualize_depth.<locals>.visualize_depth_onec                 s       | ]}|d u V  qd S r   r   .0or   r   r   	<genexpr>      z9MarigoldImageProcessor.visualize_depth.<locals>.<genexpr>Input depth is `None`r@   r&   Unexpected input shape=, expecting [N,1,H,W].c                       g | ]\}} |d  |qS r   r   r   r   rx   r   r   r   
<listcomp>      z:MarigoldImageProcessor.visualize_depth.<locals>.<listcomp>c                       g | ]	\}} ||qS r   r   r   r   r   r   r         r   r   )r"   r   rs   anyr   r   r    r!   r   r%   r8   r   rH   rt   r#   )r   r   r   r   r   )r   r   r   r   r   visualize_depth  s   $


z&MarigoldImageProcessor.visualize_depthc                    s   dfdd	 | d u st | trtdd | D rtdt | tjtjfrWt	| } t | tjr7t
| } | jdkrC| jd dksLtd| j d	 fd
dt| D S t | trg fddt| D S tdt|  )Nc                    s   d|r	d| dnd }t | tjs#t| s#t| dt|  d| jdkr3t| d| j dt| r>| 	 
 } t| jtjsQt| d	| j dd
ksY dkra|     } | d tj} tjj| dd} | S )Nr   r   rN   r   r   r>   r   r   r   rl   rm   r   r   rX   )r   r   r   r    rE   r"   r#   r   rH   r'   r*   r0   r1   floatingre   uint16r   r   r   r   )r   r   r   r   export_depth_to_16bit_png_one  s   

zWMarigoldImageProcessor.export_depth_to_16bit_png.<locals>.export_depth_to_16bit_png_onec                 s   r   r   r   r   r   r   r   r     r   zCMarigoldImageProcessor.export_depth_to_16bit_png.<locals>.<genexpr>r   r@   r&   r   r   c                    r   r   r   r   r   r   r   r     r   zDMarigoldImageProcessor.export_depth_to_16bit_png.<locals>.<listcomp>c                    r   r   r   r   r   r   r   r     r   r   r   r   rs   r   r"   r   r   r    r!   r   r%   r8   r   rH   rt   r#   )r   r   r   r   )r   r   r   r   export_depth_to_16bit_png  s   $


z0MarigoldImageProcessor.export_depth_to_16bit_pngnormalsflip_xflip_yflip_zc                    s  d t |||frtjd| d| d| gtjd d fdd	| du s3t| tr7t dd | D r7tdt| tjtj	frpt
| } t| tjrPt
| } | jd	kr\| jd
 dksetd| j dfddt| D S t| trfddt| D S tdt|  )a4  
        Visualizes surface normals, such as predictions of the `MarigoldNormalsPipeline`.

        Args:
            normals (`np.ndarray | torch.Tensor | list[np.ndarray, list[torch.Tensor]]`):
                Surface normals.
            flip_x (`bool`, *optional*, defaults to `False`): Flips the X axis of the normals frame of reference.
                      Default direction is right.
            flip_y (`bool`, *optional*, defaults to `False`):  Flips the Y axis of the normals frame of reference.
                      Default direction is top.
            flip_z (`bool`, *optional*, defaults to `False`): Flips the Z axis of the normals frame of reference.
                      Default direction is facing the observer.

        Returns: `list[PIL.Image.Image]` with surface normals visualization.
        Nr   )r1   c                    sZ   |  ddd}  d ur|  | j9 } | d d } | d jtjdd } tj| } | S )	Nr&   r   r   rm   g      ?rb   r'   r   )	r(   ri   ra   r    rg   r*   r   r   r   )rx   r   )flip_vecr   r   visualize_normals_one  s   zGMarigoldImageProcessor.visualize_normals.<locals>.visualize_normals_onec                 s   r   r   r   r   r   r   r   r     r   z;MarigoldImageProcessor.visualize_normals.<locals>.<genexpr>zInput normals is `None`r@   r&   r   r   z, expecting [N,3,H,W].c                    r   r   r   r   r   r   r   r     r   z<MarigoldImageProcessor.visualize_normals.<locals>.<listcomp>c                    r   r   r   r   r   r   r   r     r   r   r   )r   r    r   rf   r   rs   r"   r   r   r!   r   r%   r8   r   rH   rt   r#   )r   r   r   r   r   )r   r   r   visualize_normals  s,   	$	


z(MarigoldImageProcessor.visualize_normalsr   
predictiontarget_propertiesc           	         sh  dvrt dt ts!t trtdd   D s!t dtd }d fdd	| du sAt| trEtd	d | D rEt d
t| t	j
tjfrt| } t| t	j
r^t| } | jdkrs| jd dkrs| jd | dks|t d| j d| j\}}}}|| }| ||d||} fddt| D S t| trfddt| D S t dt|  )a  
        Visualizes intrinsic image decomposition, such as predictions of the `MarigoldIntrinsicsPipeline`.

        Args:
            prediction (`np.ndarray | torch.Tensor | list[np.ndarray, list[torch.Tensor]]`):
                Intrinsic image decomposition.
            target_properties (`dict[str, Any]`):
                Decomposition properties. Expected entries: `target_names: list[str]` and a dictionary with keys
                `prediction_space: str`, `sub_target_names: list[str | Null]` (must have 3 entries, null for missing
                modalities), `up_to_scale: bool`, one for each target and sub-target.
            color_map (`str | dict[str, str]`, *optional*, defaults to `"Spectral"`):
                Color map used to convert a single-channel predictions into colored representations. When a dictionary
                is passed, each modality can be colored with its own color map.

        Returns: `list[dict[str, PIL.Image.Image]]` with intrinsic image decomposition visualization.
        target_namesz+Missing `target_names` in target_propertiesc                 s   s(    | ]\}}t |tot |tV  qd S r   r   str)r   kvr   r   r   r   8  s   & z>MarigoldImageProcessor.visualize_intrinsics.<locals>.<genexpr>z7`color_map` must be a string or a dictionary of stringsNc                    s  i }t d | D ]\}}|ddd}| dd}|dkr| d }t|d	ks5td
d |D r?td| d| t|D ]\\}}|d u rLqC|d d d d |f }	| dd}
|
dkr|| dd}|rx|	t|	  d }	|	d }	t	 t
r n |d}tj|	|dd}	tj|	  }	|	||< qCn#|dkr| dd}|r|t|  d }|d }n|dkr	 |d jtjdd }tj|}|||< q	|S )Nr   r&   r   r   prediction_spacesrgbstacksub_target_namesr   c                 s   s$    | ]}t |tp|d u  V  qd S r   r   )r   sr   r   r   r   E  s    
z]MarigoldImageProcessor.visualize_intrinsics.<locals>.visualize_targets_one.<locals>.<genexpr>zUnexpected target sub-names z in linearup_to_scaleFgư>g]tE?r   Tr   rb   r'   r   )zipr(   getlenr   r"   rt   rO   ro   r   r   r   r   r   r   r   r'   r*   ri   r    rg   )r   r   r   target_namerx   r   r   rw   sub_target_namesub_imgsub_prediction_spacesub_up_to_scale	cmap_namer   )r   r   r   r   visualize_targets_one=  sJ   


zJMarigoldImageProcessor.visualize_intrinsics.<locals>.visualize_targets_onec                 s   r   r   r   r   r   r   r   r   e  r   zInput prediction is `None`r@   r&   r   r   r   z, expecting [N*T,3,H,W].c                    r   r   r   r   r   r   r   r   p  r   z?MarigoldImageProcessor.visualize_intrinsics.<locals>.<listcomp>c                    r   r   r   r   r   r   r   r   r  r   r   r   )r"   r   r   dictallitemsr   rs   r   r   r   r    r!   r   r%   r8   r   rH   reshapert   r#   )	r   r   r   	n_targetsN_T_HWNr   )r   r   r   r   visualize_intrinsics  s2   
$(

*
z+MarigoldImageProcessor.visualize_intrinsics_   uncertaintyc                    s   d fdd	| du st | trtdd | D rtdt | tjtjfrVt	| } t | tjr6t
| } | jdkrB| jd d	v sKtd
| j dfddt| D S t | trffddt| D S tdt|  )a"  
        Visualizes dense uncertainties, such as produced by `MarigoldDepthPipeline`, `MarigoldNormalsPipeline`, or
        `MarigoldIntrinsicsPipeline`.

        Args:
            uncertainty (`np.ndarray | torch.Tensor | list[np.ndarray, list[torch.Tensor]]`):
                Uncertainty maps.
            saturation_percentile (`int`, *optional*, defaults to `95`):
                Specifies the percentile uncertainty value visualized with maximum intensity.

        Returns: `list[PIL.Image.Image]` with uncertainty visualization.
        Nc                    s   d|r	d| dnd }|   dk rt| d|    d| dd	d} | d	  } t|  }t| d
 | dd
} | 	tj
} tj| } | S )NUncertaintyr   rN   r   r   z: unexpected data range, min=r>   r&   r   rb   )rn   r"   r(   squeezer'   r*   r   
percentileclipre   rg   r   r   r   )rx   r   r   saturation_value)saturation_percentiler   r   visualize_uncertainty_one  s   zOMarigoldImageProcessor.visualize_uncertainty.<locals>.visualize_uncertainty_onec                 s   r   r   r   r   r   r   r   r     r   z?MarigoldImageProcessor.visualize_uncertainty.<locals>.<genexpr>zInput uncertainty is `None`r@   r&   )r&   r   r   z&, expecting [N,C,H,W] with C in (1,3).c                    r   r   r   r   r   r   r   r     r   z@MarigoldImageProcessor.visualize_uncertainty.<locals>.<listcomp>c                    r   r   r   r   r  r   r   r     r   r   r   r   )r   r   r   )r   r   r   visualize_uncertaintyv  s   $


z,MarigoldImageProcessor.visualize_uncertainty)r   TTr   )r{   FN)rl   rm   r{   )rl   rm   )FFF)r   )r   )+__name__
__module____qualname__r   config_namer   intr5   r   staticmethodr    r!   r   r   r%   r+   r8   tupler   rK   rU   r\   r`   ra   rf   r   r1   rk   rp   r   rz   r   r   rs   r)   r   r   r   r   r   r   r  __classcell__r   r   r   r   r   %   s2   $
 ,$-
)

 
	

>
#
7
W
r   )typingr   r*   r   r   r    torch.nn.functionalnn
functionalrI   r   r   r   configuration_utilsr   image_processorr   utilsr   r	   utils.import_utilsr
   
get_loggerr  loggerr   r   r   r   r   <module>   s   
