o
    ۷i                     @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZ d dlmZ d dlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  d dl!m"Z" e"e#Z$e  a%da&dej'e(B dej'e(B de)fddZ*de(fddZ+dd Z,dej'dej'fddZ-dej'e(B dej'e(B de)fd d!Z.dej'e(B dej'e(B de)fd"d#Z/dej'e(B dej'e(B de)fd$d%Z0e.fd&ej'e(B d'ej'e(B d(e1deee)e2f ed) f fd*d+Z3d,e(d-e(fd.d/Z4e.fd,e(d-e(d0e1deee)e2f ed) f fd1d2Z5e	e3e*d3Z6ed4eee)e2f ed) f f e7d5< e	e3e.d3Z8ed4eee)e2f ed) f f e7d6< e	e3e0d3Z9ed4eee)e2f ed) f f e7d7< e	e3e/d3Z:ed4eee)e2f ed) f f e7d8< e	e5e*d9Z;ed4eee)e2f ed) f f e7d:< e	e5e.d9Z<ed4eee)e2f ed) f f e7d;< e	e5e0d9Z=ed4eee)e2f ed) f f e7d<< e	e5e/d9Z>ed4eee)e2f ed) f f e7d=< g d>a?d?d@ Z@dAdB ZAe#dCkreA  dS dS )D    N)tqdm)partial)CallableUnionTupleList)mean_squared_error)peak_signal_noise_ratio)structural_similarity   )set_metrics_verbose)get_metrics_verbose)_IMAGE_EXTENSIONS)_VIDEO_EXTENSIONS)compute_fid)compute_video_fid)compute_lpips_img)compute_clip_score)compute_reward_score)init_loggercustom
image_true
image_testreturnc                    sj   dd l ddlm  ddlmmm  fdd}t| tr'|| } t|tr0||}t	| |S )Nr   )Image)convert_image_dtype	normalizepil_to_tensorc                    s:     | }|}|jd}|g dg dd}|S )N)dtype)      ?r   r   )meanstd)openfloat32)pathpilimgr   r   r   r   torch O/home/ubuntu/vllm_env/lib/python3.10/site-packages/cache_dit/metrics/metrics.pyload_img_as_tensor+   s
   
z.compute_lpips_file.<locals>.load_img_as_tensor)
r(   PILr   $torchvision.transforms.v2.functionalr   r   r   
isinstancestrr   )r   r   r+   r)   r'   r*   compute_lpips_file   s   

r0   	psnr_typec                 C   s   | a t dv sJ d S )N)skimager   	PSNR_TYPE)r1   r)   r)   r*   set_psnr_type<   s   r5   c                   C   s   t S Nr3   r)   r)   r)   r*   get_psnr_typeB   s   r7   c                 C   s:   t | | d }|dkrtdS dt dt |  S )zCalculate PSNR (Peak Signal-to-Noise Ratio).

    Ref: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio

    Args:
        image_true (ndarray): Images with range [0, 255].
        image_test (ndarray): Images with range [0, 255].
       r   inf   g     o@)npr    floatlog10sqrt)r   r   mser)   r)   r*   calculate_psnrG   s   r@   c                 C   sF   t | tr
t| } t |trt|}t dkrt| |S t| |S )z
    img_true = cv2.imread(img_true_file)
    img_test = cv2.imread(img_test_file)
    PSNR = compute_psnr(img_true, img_test)
    r2   )r.   r/   cv2imreadr7   r	   r@   r   r   r)   r)   r*   compute_psnr_fileY   s   
	




rD   c                 C   s2   t | tr
t| } t |trt|}t| |S )z
    img_true = cv2.imread(img_true_file)
    img_test = cv2.imread(img_test_file)
    MSE = compute_mse(img_true, img_test)
    )r.   r/   rA   rB   r   rC   r)   r)   r*   compute_mse_fileo   s   
	


rE   c                 C   s8   t | tr
t| } t |trt|}t| |dddS )z
    img_true = cv2.imread(img_true_file)
    img_test = cv2.imread(img_test_file)
    SSIM = compute_ssim(img_true, img_test)
    Tr8   )multichannelchannel_axis)r.   r/   rA   rB   r
   rC   r)   r)   r*   compute_ssim_file   s   
	


rH   image_true_dirimage_test_dircompute_file_funcNNc                    s  t tjst  tjr| dfS tjrtj s&| dfS dd }tfddtD }dd |D }t	||d}t   fddtD }d	d |D }t	||d}g }g }t
tt|t|D ] }|| }	|| }
tj|
tj|	kr||	 ||
 qo| }| }t|d
krtd dS td|  td|  d}d
}d
}tt||t|tdD ]\}}|||}|tdkr||7 }|d7 }|d7 }q|d
kr|| }td|d ||fS td dS )Nr   c                 S   s"   t d| }|rt|dS | S )Nz	(\d+)\D*$r   )researchintgroup)filenamematchr)   r)   r*   natural_sort_key   s   z,compute_dir_metric.<locals>.natural_sort_keyc                    &   g | ]}  d |D ]}|qqS z*.{}rglobformat.0extfile)rI   r)   r*   
<listcomp>   
    z&compute_dir_metric.<locals>.<listcomp>c                 S      g | ]}|  qS r)   as_posixrZ   r\   r)   r)   r*   r]          keyc                    rT   rU   rV   rY   )rJ   r)   r*   r]      r^   c                 S   r_   r)   r`   rb   r)   r)   r*   r]      rc   r   zONo valid Image pairs, please note that Image pairs must have the same basename.rL   zimage_true_files: zimage_test_files:         totaldisabler9   	Average: .2fzNo valid files to compare)r.   r;   ndarrayosr$   isdirpathlibPathr   sortedrangeminlenbasenameappendcopyloggererrordebugr   zipDISABLE_VERBOSEr<   )rI   rJ   rK   rS   image_true_filesimage_test_filesimage_true_files_selectedimage_test_files_selectediselected_image_trueselected_image_testtotal_metricvalid_filestotal_filesr   r   metricaverage_metricr)   )rJ   rI   r*   compute_dir_metric   sl   








r   
video_true
video_testc                 C   s   t | }t |}| r| std g g dfS tt|t jt|t j}d}g }g }t	d|  	 |
 \}}	|
 \}
}|rL|
sMn||	 || |d7 }q<|  |  |dkrmg g dfS |||fS )NzCould not open video filesr   zTotal frames: Tr   )rA   VideoCaptureisOpenedrx   ry   rs   rO   getCAP_PROP_FRAME_COUNTrz   readrv   release)r   r   cap1cap2frame_countvalid_framesvideo_true_framesvideo_test_framesret1frame1ret2frame2r)   r)   r*   _fetch_video_frames   s6   







r   compute_frame_funcc                    sD  t j| rt j|rt| |d\}}}nt j| rt j|rt| tfddtD }t| t fddtD }dd |D }dd |D }g }g }	t	t
t|t|D ] }
||
 }||
 }t j|t j|kr|| |	| q_| }|	 }t|dkrtd d	S td
|  td|  g }g }d}t||D ]\}}t||d\}}}|| || ||7 }qntd|dkrtd d	S d}d}tt||t|tdD ]\}}|||}|tdkr||7 }|d7 }q|dkr|| }td|d ||fS td d	S )z
    video_true = "video_true.mp4"
    video_test = "video_test.mp4"
    PSNR = compute_video_psnr(video_true, video_test)
    )r   r   c                    rT   rU   rV   rY   )video_true_dirr)   r*   r]   "     & z(compute_video_metric.<locals>.<listcomp>c                    rT   rU   rV   rY   )video_test_dirr)   r*   r]   &  r   c                 S   r_   r)   r`   rb   r)   r)   r*   r]   (  rc   c                 S   r_   r)   r`   rb   r)   r)   r*   r]   )  rc   r   zONo valid Video pairs, please note that Video pairs must have the same basename.rL   zvideo_true_files: zvideo_test_files: z0video_true and video_test must be files or dirs.zNo valid frames to comparerf   rg   r9   r   rj   rk   )rm   r$   isfiler   rn   ro   rp   rq   r   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   extend
ValueErrorr   r|   r<   )r   r   r   r   r   r   video_true_filesvideo_test_filesvideo_true_files_selectedvideo_test_files_selectedr   selected_video_trueselected_video_testvideo_true_video_test_video_true_frames_video_test_frames_valid_frames_r   r   r   r   r   r)   )r   r   r*   compute_video_metric  s   











r   )rK   .compute_lpipscompute_psnrcompute_ssimcompute_mse)r   compute_video_lpipscompute_video_psnrcompute_video_ssimcompute_video_mse)lpipspsnrssimr?   fidall
clip_scoreimage_rewardc                  C   s  t jdt jd} | jdtddtdt d | jdd	td d
d | jddtd dd | jddtd dd | jddtd dd | jddtd dd | jddtd dd | jddtd dd | jddtd dd | jd d!td d"d | jd#d$td d%d | jd&d'td(d)d | jd*d+d,d-d.d/ | jd0d1d,d-d2d/ | jd3d4td d5d | jd6d7dtg d8d9 | jd:d;dg d<d= | jd>td?d?d@gdAdB | jdCd,d-dDd/ | jdEdFd,d-dGd/ |  S )HNzCacheDiT's Metrics CLI)descriptionformatter_classmetrics+r   zMetric choices: )typenargsdefaultchoiceshelpz
--img-truez-i1z8Path to ground truth image or Dir to ground truth images)r   r   r   z--prompt-truez-pzGPath to ground truth prompt file for CLIP Score and Image Reward Score.z
--img-testz-i2z2Path to predicted image or Dir to predicted imagesz--video-truez-v1z8Path to ground truth video or Dir to ground truth videosz--video-testz-v2z2Path to predicted video or Dir to predicted videosz--img-source-dirz-dz(Path to dir that contains dirs of imagesz--ref-img-dirz-rz1Path to ref dir that contains ground truth imagesz--ref-prompt-truez-rpz--video-source-dirz-vdz%Path to dir that contains many videosz--ref-videoz-rvzPath to ground truth videoz--fid-batch-sizez-br   zBatch size for FID computez--enable-verbosez-verbose
store_trueFzShow metrics progress verbose)actionr   r   z	--summaryz-sz"Summary the outupt metrics resultsz
--perf-logz-plogzPath to addtional perf logz--perf-tagsz-ptagsz$Tag to parse perf time from perf log)r   r   r   r   z--extra-perf-tagsz-extra-ptagsz+Extra tags to parse perf time from perf log)r   r   r   z--psnr-typer   r2   z+The compute type of PSNR, [custom, skimage])r   r   r   r   z--cal-speedupzCalculate performance speedup.z--gen-markdown-tablez-tablez#Generate performance markdown table)argparseArgumentParserArgumentDefaultsHelpFormatteradd_argumentr/   METRICS_CHOICESrO   
parse_args)parserr)   r)   r*   get_args  s$  						r   c            )         s  t  t jdv r%jd usjd usJ jd us%jd us%J jr0t	d t
  atj i  	 	 	 	 	 dKdtdtdtdtdtdtd	d f fd
d} ts]tdj  d	tffdd}d	tffdd}d	tffdd}t| | | frJ | rtjjstj d d S tjjstj d d S g }tjD ]#}tjj|}tj|rtj|tjjkrq|| qt|dkrd S t|}tstdj d| dt|  jD ]}|D ]
}| |j|d qqn| rtjjs#tj d d S tjjs6tj d d S t !jtfddt"D }	dd |	D }	g }
|	D ]}tj|tjjkrdqR|
| qRt|
dkrtd S t|
}
tstdj d|
 dt|
  jD ]}|
D ]}| |j|d qqn| rtjjstj d d S g }tjD ]}tjj|}tj|r|| qt|dkrd S t|}tstdj d| dt|  jD ]}|D ]}| |j|d qqnjD ]}| |j#jjj$j%d qj&rfdd }| 	!dLd"td#td	t'd B ffd$d%d&td'td	t'd B ffd(d)d*tffd+d,}d*tdtd-t'd.t(ffd/d0}d1t)t dtfd2d3}j}d4|v rt*+ }|,d4 |D ]u}i } - D ]}|. |v s|/ |v r | ||< q|/ d5v rdnd6}t|0 d7d8 |d9}d:d |- D }t1d;d< |D }||d d=}g }|rd |vr|+ }|D ]C\}}|||||\}}}d>|v rq|rd |v s|rj2sq|s|}qt3t|D ]}t1|| || ||< q!qg }|D ]U\}}|||||\}}}d>|v rKq6|r]d |v s]|r]|r]j2sd|| q6t4|||D ]\} }!}"|"|  }#||! d?|#d@dA7 }qj|5dA}|| q6t1dBd< |D }$|$tdC|.   }%|%dD }&|%|& }'t6dE|$  t6dF|& dC|.   dF|'   t6dE|$  |D ]}t6| qt6dE|$  j7r|||}(|(8dGdH}(|(8dIdH}(|(8dJdH}(t6dE|$  t6|(  t6dE|$  qd S d S )MN)r   r   Tr   img_trueprompt_trueimg_testr   r   r   c                    s  |   } |d ur|d urttj| tj| fr*td| d| d d S tj|tj| dtdt	f fdd}| dksK| d	krXt
||\}}||d| | d
ks`| d	krmt||\}	}||	d
| | dksu| d	krt||\}
}||
d| | dks| d	krt||\}}||d| | dks| d	krt||\}}||d| |d ur|d urttj| tj| frtd| d| d d S tj|tj| dtdt	f fdd}| dkrt||\}}||d| | dkrt||\}}||d| |d ur|d urttj| tj| fr9td| d| d d S tj|tj|dtdt	ffdd}| dks\| d	krit||\}}||d| | d
kss| d	krt||\}}||d
| | dks| d	krt||\}}||d| | dks| d	krt||\}}||d| | dks| d	krt||\}}||d| d S d S d S d S )NzNot exist: z or z, skip.valuenc              
      P   | d u s|d u r
d S  d d| d|   d| d	}|  |< t| d S N vs , Num: , : .5fupperrx   infor   namer   msg)METRICS_METAimg_test_infoimg_true_infor)   r*   _logging_msgg  s   $z5entrypoint.<locals>._run_metric.<locals>._logging_msgr   r   r   r   r?   r   c              
      r   r   r   r   )r   r   prompt_true_infor)   r*   r        r   r   c              
      r   )Nr   
, Frames: r   r   r   r   r   )r   video_test_infovideo_true_infor)   r*   r     r   )loweranyrm   r$   existsrx   ry   ru   r<   rO   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   	img_lpipsr   img_psnrimg_ssimimg_mseimg_fidr   r   video_lpips
video_psnr
video_ssim	video_mse	video_fid)r   )r   r   r   r   r   r*   _run_metricP  s   		


$zentrypoint.<locals>._run_metriczSelected metrics: c                          j d uo	 jd uS r6   )img_source_dirref_img_dirr)   argsr)   r*   _is_image_1vsN_pattern     z*entrypoint.<locals>._is_image_1vsN_patternc                      r   r6   )video_source_dir	ref_videor)   r   r)   r*   _is_video_1vsN_pattern  r  z*entrypoint.<locals>._is_video_1vsN_patternc                      r   r6   )r   ref_prompt_truer)   r   r)   r*   _is_prompt_1vsN_pattern  r  z+entrypoint.<locals>._is_prompt_1vsN_patternz not exist!r   zCompare r   z, Num compares: )r   r   r   c                    rT   rU   rV   rY   )r  r)   r*   r]   	  s    zentrypoint.<locals>.<listcomp>c                 S   r_   r)   r`   rb   r)   r)   r*   r]     rc   )r   r   r   )r   r   r   )r   r   r   r   r   r   c                     s    j d u st jdkrg S tj j sg S g } t j dO}| }|D ]?}| } jD ]5}|	 |	 v rdt j
dkrG| |  nd} j
D ]}|	 |	 vrZd} nqL|rd| |  nq/q&W d    | S 1 sqw   Y  | S )Nr   rTF)perf_logrt   	perf_tagsrm   r$   r   r"   	readlinesstripr   extra_perf_tagsrv   )
perf_textsr\   
perf_lineslineperf_taghas_all_extra_tagext_tagr   r)   r*   _fetch_perfT  s:   




zentrypoint.<locals>._fetch_perfNumtexttagc           	         s   dd l }||}|dd}|| d|j}|| }|s"d S |d}z*| tv r3t	|W S t
 jdkrMdd  jD }| |v rMt	|W S t|W S  ty[   Y d S w )Nr   z\ z\s+z:\s*(\d+\.?\d*)\D*r   c                 S   r_   r)   )r   )rZ   r  r)   r)   r*   r]     rc   z4entrypoint.<locals>._parse_value.<locals>.<listcomp>)rM   escapereplacecompile
IGNORECASErN   rP   r   r   r<   rt   r	  rO   r   )	r  r  rM   escaped_tagprocessed_tagpatternrR   	value_strr	  r   r)   r*   _parse_valuep  s&   





z entrypoint.<locals>._parse_valuecompare_tagr  c                    sR   g } D ]}| |v r||}|d ur| | qt|dkr!d S t|t| S )Nr   )rv   rt   sum)r   r  perf_valuesr  
perf_value)
PERF_TEXTSr  r)   r*   _parse_perf  s   

zentrypoint.<locals>._parse_perfre   c                    sf   |  dd  }d }jd u r| dd  }|d u rg S g }jD ]} ||}|| q$|S )N,r   vs)splitr  r   r	  rv   )re   headerreference_tagref_perf_valuesr  r#  )r%  r   r)   r*   	_ref_perf  s   


zentrypoint.<locals>._ref_perfr   max_key_lenc                    s  |  dd  }| dd  }t dk}dtfdd}d}g }	g }
| d	 }r|sC|d
| d| d|  d|d}n|d
| d| d|  d|dd}jD ]'}||}|	| |rm|dnd }||}|| d| d7 }|
| q[js|d}nk| d }r|s|d
| d| d|  d|d}nM|d
| d| d|  d|dd}jD ]'}||}|	| |r|dnd }||}|| d| d7 }|
| qjs|d}nt	d||	|
fS )Nr&  r   r'  r   r  c                 S   sH   d|   v r
d}|S d|   v rd}|S d|   v rd}|S |  }|S )Ntimez
Latency(s)tflopsTFLOPsflopsFLOPs)r   r   )r  perf_msgr)   r)   r*   	_perf_msg  s   z3entrypoint.<locals>._format_item.<locals>._perf_msg r  <r   r   r   z<7.4f<.2fFramesr   z"Num or Frames can not be NoneType.)
r(  r  rt   r/   r   r	  rv   cal_speedupremovesuffixr   )re   r   r   r-  r)  r   has_perf_textsr4  
format_strr"  	perf_msgsr   r  r#  r3  )r$  r%  r  r   r)   r*   _format_item  sZ   &(



&




z entrypoint.<locals>._format_itemformat_strsc                    s  | sdS |   d h}g }| D ]t}dd |dD }|d  }d|v r?|ddd  }d	|v r>|d	dd  }n|}t fd
d|D }i }	|D ](}
|
dd fr[qPd|
v rx|
dd\}}| }| }||	|< || qP|d| |i|	 qd gt fdd|D  }dd| d }|dddgt	|  d 7 }|D ]fdd|D }|dd| d 7 }q| S )Nr5  Configc                 S   r_   r)   )r  rZ   pr)   r)   r*   r]     rc   z5entrypoint.<locals>._format_table.<locals>.<listcomp>r&  r   r'  r   	_DBCACHE_c                 3   s,    | ]}|  r|d d  V  qdS ):r   N)
startswithr(  r  rA  metric_upperr)   r*   	<genexpr>  s    

z4entrypoint.<locals>._format_table.<locals>.<genexpr>zNum:zFrames:rD  c                    s   g | ]
}|d  fvr|qS )r@  r)   )rZ   hrF  r)   r*   r]     s    z| z | z |
z---c                    s   g | ]}  |d qS )r5  )r   )rZ   r)  )rowr)   r*   r]   %  s    )
r   r(  r  nextrE  addrv   rq   joinrt   )r?  r   all_headersrow_datar  partsconfig_partconfigmetric_value	perf_datapartre   r   sorted_headerstable
row_valuesr)   )rG  rJ  r*   _format_table  sL   
 z!entrypoint.<locals>._format_tabler   )r   r   r   r   Fc                 S   s   | d S )Nr   r)   )xr)   r)   r*   <lambda>@  s    zentrypoint.<locals>.<lambda>)re   reversec                 S   s   g | ]}| d d  qS )r&  r   )r(  r  rZ   re   r)   r)   r*   r]   A  s    c                 s       | ]}t |V  qd S r6   rt   r]  r)   r)   r*   rH  B      zentrypoint.<locals>.<genexpr>rd   Noneu   (↑): r7  r   c                 s   r^  r6   r_  )rZ   r<  r)   r)   r*   rH  t  r`  z	Summary: r8   - u   Latency(s)(↑)u   SpeedUp(↑)u   TFLOPs(↑)u
   FLOPs(↑))NNNNN)r  )9r   rx   rz   r   r   r  r   r   enable_verboser   r   r|   r5   r1   r/   r   boolr   rm   r$   r   ry   r   listdirrM  rn   ru   rv   rt   rq   r  r  ro   rp   r   r   r   r   summaryr<   rO   r   r   rw   removekeysr   r   itemsmaxr9  rr   r{   r:  printgen_markdown_tabler  ))r   r   r  r  directoriesitem	item_pathr   img_test_dirvideo_source_filesvideo_source_selectedvideo_source_filer   r  r,  r>  rY  selected_metricsselected_itemsre   r\  sorted_itemsselected_keysr-  r+  max_perf_valuesr   r<  r"  r=  r   r?  r#  r3  max_perf_valueperf_speedup
format_lenres_lenleft_len	right_lenrW  r)   )r   r$  r%  r  r   r  r*   
entrypoint=  s  


x








	
	K1







"

   ^r  __main__)Brm   rM   rA   ro   r   numpyr;   r   	functoolsr   typingr   r   r   r   skimage.metricsr   r	   r
   rR  r   r   r   r   r   r   r   r   r   r   r   r   r   cache_dit.loggerr   __name__rx   r|   r4   rl   r/   r<   r0   r5   r7   r@   rD   rE   rH   callablerO   r   r   r   r   __annotations__r   r   r   r   r   r   r   r   r   r  r)   r)   r)   r*   <module>   s   
 





M
.
Y******** #    
Q
