o
    ߥiy                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlZ d dl!Z!d dl"m#  m$Z% d dl&m'Z( d dl)m*Z* d dl+m,Z, g dZ-da.de/d/e0e12ddd d	fd
dZ3dd Z4dd Z5dd Z6i e6_7dd Z8dkddZ9dlddZ:dlddZ;dlddZ<dd Z=dmd!d"Z>dld#d$Z?dld%d&Z@dld'd(ZAdld)d*ZBdld+d,ZCdnd/d0ZDe!E 	-	1	2	dod3d4ZFe!E dpd6d7ZGe!E g d8g d8d-dfd9d:ZHe!E g d8g d8d-dd1dfd;d<ZIe!E g d8g d8d-dfd=d>ZJe!E g d8g d8d-dfd?d@ZKe!E g d8g d8dddfdAdBZLe!E dqdCdDZMe!E dldEdFZNe!E dmdGdHZOdrdJdKZPdLdM ZQdsdNdOZRdtdPdQZSdudRdSZTdTdU ZUdvdWdXZVG dYdZ dZeWZXdld[d\ZYd]d^ ZZd_d` Z[dtdadbZ\dcdd Z]dwdfdgZ^dxdidjZ_dS )y    N)BytesIO)
ThreadPool	rearrange)Image)$parse_oss_urlparse_bucketread
read_image	read_gzipceil_divide	to_device
put_objectput_torch_objectput_object_from_file
get_objectget_object_to_file	rand_name
save_image
save_videosave_video_vs_conditions(save_video_multiple_conditions_with_datasave_video_multiple_conditionsdownload_video_to_filesave_video_grid_mp4	save_capsemaparallelexistsdownloadunzipload_state_dictinverse_indicesdetect_duplicatesread_tfsmd5ropeformat_statebreakup_gridhuggingface_tokenizerhuggingface_model/model_weightsc                 C   s   t ||p	t | }|S )zDownload OSS [file or folder] to the cache folder.
        Only the 0th process on each node will run the downloading.
        Barrier all processes until the downloading is completed.
    )ospjoinbasename)oss_keyfile_or_dirname	cache_dir	base_path r5   k/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/multi_modal/videocomposer/utils/utils.pyDOWNLOAD_TO_CACHE2   s   
r7   c                  C   sx   ddl } ddlm} ||  | j| j}|d || j| jd t	|
 d W  d   S 1 s5w   Y  dS )zYhttps://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-numberr   N)closing) r      )socket
contextlibr8   AF_INETSOCK_STREAMbind
setsockopt
SOL_SOCKETSO_REUSEADDRstrgetsockname)r;   r8   sr5   r5   r6   find_free_portA   s   
$rF   c                 C   s:   t |  t j|  tj|  t|  dt jj_	d S )NT)
torchmanual_seedcudamanual_seed_allnprandomseedbackendscudnndeterministic)rM   r5   r5   r6   
setup_seedK   s
   

rQ   c                 C   s   |  dr| tdd  } tdd tdd tdd tdd d}| jddd	\}} d
|v rO|jd
dd	\}}|dD ]}|jddd	\}}|||< q?tj| dt  t	
 }t	jt	|d |d |d ||d}|| fS )Noss://OSS_ENDPOINTOSS_ACCESS_KEY_IDOSS_ACCESS_KEY_SECRETOSS_SECURITY_TOKEN)endpointaccessKeyIDaccessKeySecretsecurityTokenr+   r:   )maxsplit?&=@rX   rY   rW   )authrW   bucket_namesession)
startswithlenosgetenvsplitr   	_sessions
setdefaultgetpidossSessionBucketAuth)pathconfigsbucketconfigpairkvrb   r5   r5   r6   r   S   s.   





r   c                 C   s   t t| dd S )N_placeholderr   )r   r.   r/   )urlr5   r5   r6   r   u   s   r   r   c           
      C   s   |dv sJ d }t |D ]f}zP| dr+t| \}}|| }|dkr*|d}n/| dr@t| j}|dkr?|d}nt	| |d}| }W d    n1 sUw   Y  |W   S  t
yr }	 z|	}W Y d }	~	qd }	~	ww |)N)rx   rbrR   rx   utf-8http)mode)rangerc   r   r   r	   decoderequestsgetcontentopen	Exception)
filenamer}   retry	exception_rq   ro   r   fer5   r5   r6   r	   y   s2   





r	   c                 C   s\   d }t |D ]%}zttt| d|dW   S  ty+ } z|}W Y d }~qd }~ww |)Nrz   )r}   r   )r~   r   r   r   r	   r   )r   r   r   r   r   r5   r5   r6   r
      s   r
   c                 C   s^   d }t |D ]&}zt| \}}||| W  d S  ty, } z|}W Y d }~qd }~ww |N)r~   r   r   r   )r   
local_filer   r   r   rq   ro   r   r5   r5   r6   r      s   r   c           
      C   s   d }t |D ]Z}zDd}| dr(t| \}}tt| d d} |||  d}t| }|	 }W d    n1 s<w   Y  |rHt
|  |W   S  ty` }	 z|	}W Y d }	~	qd }	~	ww |)NFrR   r:   suffixT)r~   rc   r   r   r.   splitextr   gzipr   r	   re   remover   )
r   r   r   r   r   rq   ro   r   r   r   r5   r5   r6   r      s*   



r   c                 C   s   t t| | S r   )intmathceil)abr5   r5   r6   r      s   r   Fc                    s|   t | ttfrt|  fdd| D S t | tr*t|  fdd|  D S t | tjr<| j kr<| j	 d} | S )Nc                    s   g | ]}t | qS r5   r   .0udevicenon_blockingr5   r6   
<listcomp>       zto_device.<locals>.<listcomp>c                    s    g | ]\}}|t | fqS r5   r   r   rt   ru   r   r5   r6   r      s    )r   )

isinstancelisttupletypedictitemsrG   Tensorr   to)batchr   r   r5   r   r6   r      s   
r   c                 C   h   d }t |D ]}z	| ||W   S  ty% } z|}W Y d }~qd }~ww td| d| dd d S )Nzput_object to  failed with error: Tflush)r~   r   r   print)rq   r1   datar   r   r   r   r5   r5   r6   r         
r   c                 C   s~   d }t |D ]*}zt }t|| | || W   S  ty0 } z|}W Y d }~qd }~ww td| d| dd d S )Nzput_torch_object to r   Tr   )r~   r   rG   saver   getvaluer   r   )rq   r1   r   r   r   r   bufferr   r5   r5   r6   r      s   
r   c                 C   r   )Nzput_object_from_file to r   Tr   )r~   r   r   r   rq   r1   r   r   r   r   r   r5   r5   r6   r      r   r   c                 C   sj   d }t |D ] }z
| | W   S  ty& } z|}W Y d }~qd }~ww td| d| dd d S )Nzget_object from r   Tr   )r~   r   r	   r   r   )rq   r1   r   r   r   r   r5   r5   r6   r      s   
r   c                 C   r   )Nzget_object_to_file from r   Tr   )r~   r   r   r   r   r5   r5   r6   r     r   r      r9   c                 C   s8   t t| d}|r|dsd| }||7 }|S )Nr{   .)binasciib2a_hexre   urandomr   rc   )lengthr   namer5   r5   r6   r     s   
r   Tr:   c                 C   s   t dd}d g| D ]+}ztj|||||d | || d }	W  n ty5 }
 z|
}	W Y d }
~
q
d }
~
ww t|r@t| |	d urPt	d
||	dd d S d S )Nz.jpgr   )nrow	normalizer~   z"save image to {} failed, error: {}Tr   )r   tvutilsr   r   r   r.   r   re   r   r   format)rq   r1   tensorr   r   r~   r   r   r   r   r   r5   r5   r6   r   "  s,   





r   x   c                 C   s>   |  dddd} | jdd}dd |D }tj||dd	 |S )
Nr:         r   dimc                 S   s   g | ]}|  d  dqS )   uint8)numpyastype)r   imager5   r5   r6   r   C  s    z'video_tensor_to_gif.<locals>.<listcomp>r   )fps)permuteunbindimageiomimwrite)r   ro   durationloopoptimizeimagesr5   r5   r6   video_tensor_to_gif?  s
   r   )      ?r   r   c                 C   s   t j||jdddddd}t j||jdddddd}|||}|dd tdd}d g| D ]-}zt|d|d}	t	|	| | 
|| d }
W  n tyc } z|}
W Y d }~q6d }~ww t|rnt| |
d ur~td	||
d
d d S d S )Nr   r:   r   r   .gifr    (i j) c f h w -> c f (i h) (j w)i"save video to {} failed, error: {}Tr   )rG   r   r   viewmul_add_clamp_r   r   r   r   r   r.   r   re   r   r   r   )rq   r1   r   meanstdr   r   r   r   one_gifr   r   r5   r5   r6   r   H  s6   





r   c                    sj  t j||jdddddd}t j||jdddddd}|||}z|dd W n tyL } z| dd}t	| W Y d }~nd }~ww |
 }|j\}}}}}t||||f}|
 }i }|d  D ]\}}|jd dkrzqnt|jdkr|
  }g }|D ]}g }|D ]}||j|dddd	 q|t| qt t|}t|d
}n|ddkrt j|||gdd}t||||f}|ddkrt j||d d d df gdd}t||||f}nE|ddkrt||||f}n3|ddkr8|d d ddf d d }|d d ddf }|| dd|   }t||||f}|jr@|
 n|||< qn| }d g| D ]V}z>t|d d} fdd| D }t|d d}|	ryt j|g| |g dd}n
t j||g dd}t|| d }W  n ty } z	|}W Y d }~qLd }~ww |d urtd| | d S d S )Nr   r:   r   r   i   r   Z      
percentilewidthheightb n h w c -> b c n h wr   r            ?       @r   r   c                       g | ]\}}t |d  dqS )r   r   r   r   r   conr   r5   r6   r         z2save_video_multiple_conditions.<locals>.<listcomp>r   )rG   r   r   r   r   r   r   r   floatr   cpushapeFadaptive_avg_pool3dr   rd   r   appendget_palette_imagerK   stack
from_numpyr   sizecatis_cudar   logginginfor   )r1   video_tensormodel_kwargssource_imgspaletter   r   r   r   save_origin_videorq   r   r   cnhwmodel_kwargs_channel3key
conditionsconditions_npr   vis_ijcoloralphar   r   vid_gif	cons_listr   r5   r   r6   r   k  s   



r   c           &         s`  t j||jdddddd}t j|	|jdddddd}	||	|}|dd |j\}}}}}t	||||f}|
 }i }|d  D ]\}}t|jdkr|
  }g }|D ]}g }|D ]}||j|dddd qd|t| q^t t|}t|d	}n|ddkrt j|||gdd
}t	||||f}|ddkrt j||d d d df gdd
}t	||||f}nD|ddkrt	||||f}n3|ddkr
|d d ddf d d }|d d ddf }|| dd|   }t	||||f}|jr|
 n|||< qI| }| }tdd}d g| D ]N}z6t|d d} fdd| D }t|d d}t j|g| |g dd
}t|| | || d } W  n tyw }! z	|!} W Y d }!~!q)d }!~!ww t|rt| tdd}"d g| D ]7}z| d  d}#t!"|#t#|"d | ||" W  n ty }! zt$d||! W Y d }!~!qd }!~!ww t|"rt|" tdd}$d g| D ]7}z| d  d}%t!"|%t#|$d | ||$ W  n ty }! zt$d||! W Y d }!~!qd }!~!ww t|$rt|$ | d ur.t$d%|| dd d S d S )Nr   r:   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   c                    r   )r   )r  r   r   r   r5   r6   r     r   z<save_video_multiple_conditions_with_data.<locals>.<listcomp>z.pklr   r   wbzerror! r   Tr   )&rG   r   r   r   r   r   r   r   r   r   r   r   rd   r   r   r   rK   r   r   r   r   r   r   cloner   r   r   r   r.   r   re   r   r   pickledumpr   r   r   )&rq   video_save_keygt_video_save_keyvis_oss_keyr  r  r  r  r   r   r   r   r   r  r  r	  r
  r  r  r  r  r   r  r  r  r  copy_video_tensorcopy_source_imgsr   r   r  r  r   r   filename_predcopy_video_npfilename_gtcopy_source_npr5   r   r6   r     s   









r   c	                 C   s  t j||jdddddd}t j||jdddddd}|||}|dd |j\}	}
}}}t	||||f}|
 }|ddkr[t j|||gdd}t	||||f}tdd}d g| D ]E}z/t|d|d	}t|d|d	}t|d|d	}t j|||gd
d}t|| | || d }W  n ty } z|}W Y d }~qed }~ww t|rt| |d urtd||dd d S d S )Nr   r:   r   r   r   r   r   r   r   r   r   Tr   )rG   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r.   r   re   r   r   r   )rq   r1   r  r  r  r   r   r   r   r   r  r  r	  r
  r   r   r  con_gifr   r   r5   r5   r6   r   J  sP   






r   c                 C   s  t j||jdddddd}t j||jdddddd}|||}|dd |j\}}	}
}}|ddddd}|	 
 d d	}td
d}d g| D ]}zr|d u ratt|}t|| }d}tj|
|| | | || | | |	fd	d}t|D ])}|| }|| }|| | }|| | }|| |d d ||| ||| f< qtjj||dt|id | || d }W  n ty } z|}W Y d }~qRd }~ww t|rt| |d urtd||dd d S d S )Nr   r:   r   r   r   r   r   r   r   z.mp4r   )dtypez-r)	inputdictr   Tr   )rG   r   r   r   r   r   r   r   r   r   r   r   r   r   r   sqrtrK   zerosr~   skvideoiovwriterC   r   r   r.   r   re   r   r   r   )rq   r1   r   r   r   r   r   r   r   r  tr	  r
  r   r   ncolpadding
video_gridr   rx   c_start_rstart_cr   r   r5   r5   r6   r   }  sZ   	





r   c                 C   s   |j d }t|| }||| ksJ d}t|D ]}t|D ]}	t||| |	  }
||
d 7 }q|d7 }qd g| D ]!}z| || d }W  n ty] } z|}W Y d }~q<d }~ww |d urntd||dd d S d S )Nr   r9   
r   Tr   )r   r   r~   dec_bytes2objr   r   r   r   )rq   r1   r   r   r   rd   num_per_rowtextsr   r  textr   r   r   r5   r5   r6   	save_text  s2   



r6  c           	      C   s   d}|D ]
}||7 }|d7 }qd g| D ]!}z|  || d }W  n ty5 } z|}W Y d }~qd }~ww |d urFtd||dd d S d S )Nr9   r1  r   Tr   )r   r   r   r   )	rq   r1   capsr   r4  capr   r   r   r5   r5   r6   r     s(   


r   c                 C   s|   d|  krdksJ  J t |  | D ]\}}|||| q|r:t |  | D ]\}}|| q0d S d S )Ng        r   )zip
parameterscopy_lerpbuffers)net_emanetbetacopy_bufferp_emapb_emar   r5   r5   r6   r     s   r       c                    s   t |tsJ t |d tsdd |D }|dkr" fdd|D S t|d fdd|D }fdd|D }W d    |S 1 sFw   Y  |S )Nr   c                 S   s   g | ]}|fqS r5   r5   r   argsr5   r5   r6   r     s    zparallel.<locals>.<listcomp>c                    s   g | ]} | qS r5   r5   rF  )funcr5   r6   r     s    )	processesc                    s   g | ]}  |qS r5   )apply_asyncrF  )rH  poolr5   r6   r     r   c                    s   g | ]}|j  d qS )timeout)r   )r   resrL  r5   r6   r     r   )r   r   r   Pool)rH  	args_listnum_workersrM  resultsr5   )rH  rK  rM  r6   r     s   
r   c                 C   s*   |  drt| \}}||S t| S )NrR   )rc   r   object_existsr.   r   )r   rq   ro   r5   r5   r6   r     s   


r   c              
   C   s   |d u r	t | }t |r|rPz)| dr#t| \}}||| ntj| | |s8t	d|  d| dd W n t
yO } z
td| d| d }~ww t |S )NrR   zDownloaded z to Tr   zDownloading z failed with error )r.   r0   r   rc   r   r   urllibrequesturlretriever   r   
ValueErrorabspath)rw   r   replacequietrq   r1   r   r5   r5   r6   r     s    


r   c                 C   sN   |d u r	t | }t| d}|| W d    d S 1 s w   Y  d S )Nrx   )r.   dirnamezipfileZipFile
extractall)r   dst_dirzip_refr5   r5   r6   r      s
   
"r    c                    s  ||    rt fdd D fddD }fddD }fdd  @ D }t|| tfdd D | jdd t|d	krgtd
d| dd t|d	krxtdd| dd t|d	krtdd| dd d S d S )Nc                    s2   g | ]\}}|  r|t d  n||fqS r   )rc   rd   r   )drop_prefixr5   r6   r     s     z#load_state_dict.<locals>.<listcomp>c                       g | ]}| vr|qS r5   r5   r   rt   )srcr5   r6   r     r   c                    rb  r5   r5   rc  )dstr5   r6   r      r   c                    s$   g | ]}| j  | j kr|qS r5   )r   rc  )re  rd  r5   r6   r   !  s    c                    s    g | ]\}}| vr||fqS r5   r5   r   )incompatibler5   r6   r   '  s     F)strictr   z  Missing: z, Tr   z  Unexpected: z  Shape unmatched: )	
state_dictr   r   keyssetr!   rd   r   r/   )modulerh  ra  missing
unexpected	unmatchedr5   )ra  re  rf  rd  r6   r!     s(   r!   c                 C   s&   t | }t t| | || < |S )zXInverse map of indices.
        E.g., if A[indices] == B, then B[inv_indices] == A.
    )rG   
empty_likearangerd   r   )indicesinv_indicesr5   r5   r6   r"   3  s   
r"   ?c                 C   sb   | j dksJ tj| ddd} t| | j}|d tj  |	|j
dd }t|d S )Nr   r:   )rC  r   r   r   )ndimr   r   rG   mmTtriu_rI   synchronizegtanywhere)featsthrsimmatmaskr5   r5   r6   r#   <  s   

r#   c                   @   s.   e Zd Z		d
ddZedd Zdd Zd	S )	TFSClient!restful-store.vip.tbsite.net:38005354c9fae75f5c                 C   sV   || _ || _dd td| d ddd  D | _t| jdks&J d| _d S )Nc                 S   s   g | ]}d |v r|qS ):r5   r   r5   r5   r6   r   S  s
    z&TFSClient.__init__.<locals>.<listcomp>http://z	/url.listr1  r:   r   )hostapp_keyr	   striprg   serversrd   _TFSClient__server_id)selfr  r  r5   r5   r6   __init__L  s    
zTFSClient.__init__c                 C   s"   | j d t| j | _ | j| j  S )Nr:   )r  rd   r  )r  r5   r5   r6   serverZ  s   zTFSClient.serverc                 C   sh   t |}ttd| j d| j d| d}tt	td| j d| j d| d|d  d}|S )	Nr  z/v1/z
/metadata/z?force=0r+   z?offset=0&size=SIZErz   )
r.   r0   jsonloadsr	   r  r  r   r   r   )r  tfsmetaimgr5   r5   r6   r	   _  s   
"zTFSClient.readN)r  r  )__name__
__module____qualname__r  propertyr  r	   r5   r5   r5   r6   r  J  s    

r  c                 C   s\   d }t |D ]%}ztd u rt at| W   S  ty+ } z|}W Y d }~qd }~ww |r   )r~   
TFS_CLIENTr  r	   r   )r  r   r   r   r   r5   r5   r6   r$   m  s   r$   c                 C   s@   t | d}t|  W  d    S 1 sw   Y  d S )Nrz   )r   hashlibr%   r	   	hexdigest)r   r   r5   r5   r6   r%   |  s   $r%   c           
   	   C   s   | j }| | dd| d} | j dd \}}|d dks!J |d }tt|| tdt|| | }t	|t
|}}| jddd\}}	tj|| |	|  |	| ||  gdd} | |S )zQApply rotary position embedding on x of shape [B, *(spatial dimensions), C].
    r   r   r,   Nr   i'  r   )r   r   r   rG   outerrp  r   powdivsincoschunkr   )
xr   lr  halfsinusoidr  r  x1x2r5   r5   r6   r&     s   *
r&   c                 C   sZ   d dd |  D }|r+t|d}|| W d   dS 1 s$w   Y  dS dS )z'For comparing/aligning state_dict.
    r1  c                 S   s$   g | ]\}}| d t |j qS )	)r   r   r   r5   r5   r6   r     s   $ z format_state.<locals>.<listcomp>r
  N)r/   r   r   write)stater   r   r   r5   r5   r6   r'     s   "r'   c                 C   s   | j | }| j| }d }}g }t|D ]-}t|D ]&}|| |d |  }	|| |d |  }
|| |	|
|	| |
| f qq|S )z=The inverse operator of ``torchvision.utils.make_grid``.
    r   r:   )r   r   r~   r   crop)r  	grid_sizer   r+  wrowwcolgridsr   r  r  y1r5   r5   r6   r(     s   

"r(   google/mt5-xxlc                 K   s*   ddl m} |jtd|  | fi |S )Nr   )AutoTokenizerzhuggingface/tokenizers/)transformersr  from_pretrainedr7   )r   kwargsr  r5   r5   r6   r)     s   r)   	AutoModelc                 K   s,   dd l }t||jtd|  | fi |S )Nr   zhuggingface/models/)r  getattrr  r7   )r   
model_typer  r  r5   r5   r6   r*     s   
r*   )rx   ry   )ry   )F)r   r9   )r   Tr   ry   )r   r   T)r   ry   )rE  N)NFFr   )r9   )rs  )r  )r  r  )`base64r   copyglobr   r  r   r   re   os.pathro   r.   r  rL   systimeurllib.requestrT  r\  r(  r   multiprocessing.poolr   rO  r   r  r   rK   oss2rk   r   
skvideo.ior'  rG   torch.nn.functionalnn
functionalr   torchvision.utilsutilsr   einopsr   PILr   __all__r  r/   rX  __file__rg   r7   rF   rQ   r   rh   r   r	   r
   r   r   r   r   r   r   r   r   r   r   no_gradr   r   r   r   r   r   r   r6  r   r   r   r   r   r    r!   r"   r#   objectr  r$   r%   r&   r'   r(   r)   r*   r5   r5   r5   r6   <module>   s   












	"a	|24
	



	
#
	
