o
    Xi6                     @   s   d Z ddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ ddlZG dd de
Zdd Zd	d
 Zdd Zdd ZedkrMe  dS dS )a   Example Usage
cpu:

s3tokenizer --wav_scp xxx.scp             --device "cpu"             --output_dir "./"             --batch_size 32

gpu:

torchrun --nproc_per_node=8 --nnodes=1      --rdzv_id=2024 --rdzv_backend="c10d" --rdzv_endpoint="localhost:0"     `which s3tokenizer` --wav_scp xxx.scp                 --device "cuda"                 --output_dir "./"                 --batch_size 32

    N)
DataLoaderDatasetDistributedSampler)tqdmc                   @   s$   e Zd Zdd Zdd Zdd ZdS )AudioDatasetc                 C   sr   g | _ g | _t|ddd"}|D ]}|  \}}| j | | j| qW d    d S 1 s2w   Y  d S )Nrzutf-8)encoding)datakeysopenstripsplitappend)selfwav_scpflinekey	file_path r   C/home/ubuntu/.local/lib/python3.10/site-packages/s3tokenizer/cli.py__init__/   s   "zAudioDataset.__init__c                 C   s
   t | jS N)lenr	   )r   r   r   r   __len__9   s   
zAudioDataset.__len__c                 C   s0   | j | }| j| }t|}t|}||fS r   )r	   r
   s3tokenizer
load_audiolog_mel_spectrogram)r   idxr   r   audiomelr   r   r   __getitem__<   s
   



zAudioDataset.__getitem__N)__name__
__module____qualname__r   r   r!   r   r   r   r   r   -   s    
r   c                 C   s4   dd | D }dd | D }t |\}}|||fS )Nc                 S      g | ]}|d  qS )r   r   .0itemr   r   r   
<listcomp>E       zcollate_fn.<locals>.<listcomp>c                 S   r%   )   r   r&   r   r   r   r)   F   r*   )r   padding)batchr
   mels	mels_lensr   r   r   
collate_fnD   s   
r0   c                  C   sp   t tjdd} t tjdd}t tjdd}td|d||   tj| t	
d | ||fS )	N
WORLD_SIZEr+   
LOCAL_RANKr   RANKz'Inference on multiple gpus, this gpu {}z, rank {}, world_size {}nccl)intosenvirongetprintformattorchcuda
set_devicedistinit_process_group)
world_size
local_rankrankr   r   r   init_distributedK   s   



rC   c                  C   s   t jdd} | jddtg ddd | jddtd	d
 | jddtddgdd | jddtdd
 | jddtdd
 | jdtddd | jdtddd |  }|S )Nzextract speech code)descriptionz--modelT)speech_tokenizer_v1speech_tokenizer_v1_25hzspeech_tokenizer_v2_25hzzmodel version)requiredtypechoiceshelpz	--wav_scpz&each line contains `wav_name wav_path`)rH   rI   rK   z--devicer<   cpuzdevice for inferencez--output_dirzdir to save resultz--batch_sizez%batch size (per-device) for inferencez--num_workers   zworkers for dataloader)rI   defaultrK   z
--prefetch   zprefetch for dataloader)argparseArgumentParseradd_argumentstrr5   
parse_args)parserargsr   r   r   get_argsV   sP   rW   c               	   C   s  t  } tj| jdd | jdkrtj sJ t \}}}nd\}}}t| j}t	
| j|}t| j}| jdkrNtjjj||gd}t|||d}nd }t|| j|d| j| jtd}t|}	|d	krmt|	d
dd}
t| j d|d  d| d}|D ]E\}}}|||||\}}t|D ]!\}}||d ||  f  }|tj ||dddd  q|d	kr|
!|t|  q|d	kr|
"  |"  | jdkrt#$  t#%  d S d S )NT)exist_okr<   )r+   r   r   )
device_ids)num_replicasrB   F)
batch_sizesamplershufflenum_workersprefetch_factorr0   r   
Processingwavs)totaldescunitz/part_r+   _of_w)r   code)ensure_ascii
)&rW   r6   makedirs
output_dirdevicer;   r<   is_availablerC   r   
load_modelmodeltor   r   nnparallelDistributedDataParallelr   r   r[   r^   prefetchr0   r   r   r   	enumerater(   tolistwritejsondumpsupdatecloser>   barrierdestroy_process_group)rV   r@   rA   rB   rl   ro   datasetr\   
dataloadertotal_stepsprogress_barwriterr
   r.   r/   codes
codes_lensikrg   r   r   r   main}   sl   



 
r   __main__)__doc__rP   rx   r6   r;   torch.distributeddistributedr>   torch.utils.datar   r   r   r   r   r   r0   rC   rW   r   r"   r   r   r   r   <module>   s"   '9
