o
    ϯi~                     @   s  d dl mZ d dlmZ d dlmZ d dlZd dlZd dlmZ d dl	m
Z
mZ d dlmZ dd	 Zed
kre Zejdde_edZde_e \e_e_e_ejre
erejD ]2Zee D ]+Zejde de s{ede de  ede de de de d	 qbq\eejej ej!ej"eej#ej$ej%ej&dej'ej(ej)ej*d\Z+Z,eee,Z-e-d j.e-d j/Z.Z/e0de-d j.j1 e0dej2 e0de-d j.j1ej2  e  dS dS )    )create_model)get_data)
parse_argsN)tqdm)	is_masterworld_info_from_env)dataset_splitc                  C   s,   t tttd jjtj dD ]\} }qd S )Ntrain)total)	enumerater   
dataloaderdatanum_samplesargs
batch_size)ibatch r   J/home/ubuntu/.local/lib/python3.10/site-packages/tests/data_loader_test.pyrun_dataloader   s   &r   __main__/-cpuFz./json_files/z,aws s3 cp s3://s-laion-audio/webdataset_tar/z/sizes.json ./json_files/z/sizes.jsonT)
	precisiondevicejitforce_quick_geluopenai_model_cache_dirskip_paramspretrained_audiopretrained_textenable_fusionfusion_typer	   zdataset size:zbatch size:znum batches:)3
laion_clapr   laion_clap.training.datar   laion_clap.trainingr   torchosr   laion_clap.training.distributedr   r   laion_clap.utilsr   r   __name__r   amodelreplacer   distributed
local_rankrank
world_size
remotedatadatasetnamesdataset_namesplitpathexistsmakedirssystemtmodel
pretrainedr   torchscriptr   
expanduserr   r    r!   r"   r#   model	model_cfgr   r   samplerprintr   r   r   r   r   r   <module>   s\    




