o
    [ei                     @   sJ   d Z ddlZddlZddlmZmZ G dd deZdd Zdd
dZdS )u1   Dataset for v7 — 8-level EnCodec tokens + text.    N)Dataset
DataLoaderc                   @   s&   e Zd Zd
ddZdd Zdd Zd	S )TokenDataset     c                    sb   t |ddd} fddt|D | _W d    n1 sw   Y  tdt| j d d S )Nrutf-8)encodingc                    s   g | ]
}|d   kr|qS )n_tokens ).0e
max_tokensr   #/home/ubuntu/lewm-tts/dataset_v7.py
<listcomp>	   s    z)TokenDataset.__init__.<locals>.<listcomp>z	Dataset: z samples)openjsonloaddataprintlen)selfmanifest_pathr   max_textfr   r   r   __init__   s   zTokenDataset.__init__c                 C   s
   t | jS )N)r   r   )r   r   r   r   __len__   s   
zTokenDataset.__len__c                 C   s   | j | }tj|d dd }tjt|d dd d tjd}|||jd |jd	 d
}d|v r<|d |d< |S d|v rMt|d t	rM|d |d< |S )N
token_pathT)weights_onlytextr   r   dtype   r   )tokenstext_tokensr
   text_lenspeaker_id_num
speaker_id)
r   torchr   longtensorlistencodeshape
isinstanceint)r   idxr   r$   r    resultr   r   r   __getitem__   s   
&zTokenDataset.__getitem__N)r   r   )__name__
__module____qualname__r   r   r3   r   r   r   r   r      s    
r   c                 C   s6  t dd | D }t dd | D }t| }| d d jd }tj|||tjd}tj||tjd}tj||tjd}tj||tjd}t| D ]4\}	}
|
d |
d }}|
d ||	d d d |f< |
d	 ||	d |f< d
||	d |f< d
||	d |f< qH||||d}d| d v rtj	dd | D tjd|d< |S )Nc                 s       | ]}|d  V  qdS )r
   Nr   r   sr   r   r   	<genexpr>       zcollate_fn.<locals>.<genexpr>c                 s   r7   )r&   Nr   r8   r   r   r   r:      r;   r   r$   r!   r
   r&   r%   F)r$   r%   
token_mask	text_maskr(   c                 S   s   g | ]}|d  qS )r(   r   r8   r   r   r   r   1   s    zcollate_fn.<locals>.<listcomp>)
maxr   r.   r)   zerosr*   onesbool	enumerater+   )batchmax_tmax_txBn_rvqr$   r    tok_maskr=   ir9   ttxr2   r   r   r   
collate_fn   s(   rL      Tc              	   C   s&   t | }t||||tddd}||fS )NT)
batch_sizeshufflenum_workersrL   
pin_memory	drop_last)r   r   rL   )r   rN   rP   rO   dsdlr   r   r   build_dataloader5   s
   
rU   )rM   r   T)	__doc__r   r)   torch.utils.datar   r   r   rL   rU   r   r   r   r   <module>   s    