o
    }oi3*                     @   s   d dl Zd dlZd dlZd dlmZmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZ dd	ed
edefddZejdd Zdd Zejg dg ddd Zejg dg ddd Zdd ZG dd dejjjZ dd Z!dS )     N)CutSetSecondsSupervisionSegment)DynamicBucketingSampler)	dummy_cut)deterministic_rng)BucketingFilter FixedBucketBatchSizeConstraint2D!get_lhotse_dataloader_from_config      ?
   id_duration
num_tokensc                 C   sJ   t d|  d|  d|d| d}tj|ftjd |_t| ||gdS )Nzblah-g        a)text)dtype)r   supervisions)r   npzerosint32tolisttokensr   )r   r   r   supervision r   i/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/common/test_2d_bucketing_constraint.pymake_cut   s   "r   c                   C   s   t dd tdD dd tdD  dd tdD  dd tdD  dd tdD  dd tdD  d	d tdD  d
d tdD  dd tdD  S )Nc                 S      g | ]	}t |d ddqS )       @   r   r   r   .0ir   r   r   
<listcomp>'       zcuts.<locals>.<listcomp>   c                 S   r   )r      r    r!   r"   r   r   r   r%   (   r&   c                 S   r   )r      r    r!   r"   r   r   r   r%   )   r&   c                 S   r   )       @r(   r    r!   r"   r   r   r   r%   *   r&   c                 S   r   )r*   r)   r    r!   r"   r   r   r   r%   +   r&   c                 S   r   )r*      r    r!   r"   r   r   r   r%   ,   r&   c                 S   r   )      ,@r)   r    r!   r"   r   r   r   r%   -   r&   c                 S   r   )r,   r+   r    r!   r"   r   r   r   r%   .   r&   c                 S   r   )r,   r'   r    r!   r"   r   r   r   r%   /   r&   )r   ranger   r   r   r   cuts$   s&   r.   c              	      s  g d}g d}t |  d|t||ddddd}t|D ]m\}}|d	kr( d S t|d
 }|| \ |D ]S}|j ks?J t|jd jksKJ |dkr fdd|d | D }|rit|}	|j|	ksiJ  fdd|d | D }|rt|}
t|jd j|
ksJ q6qd S )N))      @   )r/      )r/      )      @r   )r3      )r3   r'   )r*   r2   )r*      )r*      )      .@r'   )r7      )r7      )         r   r0         r(   	   r   r1   r)   TF)max_seq_len_bucketsbatch_sizes	strict_2d  r   )shuffleduration_bins
constraintbuffer_sizeseedd   r:   c                    s   g | ]
\}}| k r|qS r   r   r#   durtok)max_durationr   r   r%   e   s    z@test_2d_bucketing_expected_bucket_allocation.<locals>.<listcomp>c                    s$   g | ]\}}| kr|k r|qS r   r   rJ   rM   max_num_tokensr   r   r%   j   s    )	r   repeatr	   	enumeratelenr   r   r   max)r.   rE   rA   sampler	batch_idxbatch	bin_indexcut
candidatesprev_max_durationprev_max_num_tokensr   rN   r   ,test_2d_bucketing_expected_bucket_allocation3   sH   
r\   )r   r   should_keep
bucket_idx)r   r   Tr   r   r0   Tr   r   r   Tr   r   r1   Tr:   r   r'   Tr:   )r      Tr<   )r   r9   Tr<   r      FNr/   r   Tr   r/   r0   Tr   r/   r   Tr   r/   r1   Tr:   r/   r'   Tr:   )r/   rd   Tr<   )r/   r9   Tr<   r/   rf   FNr*   r   Tr;   r*   r2   Tr;   r*   r+   Tr<   r*   r9   Tr<   r*   rf   FN      $@r   Tr;   rs   r2   Tr;   rs   r+   Tr<   rs   r9   Tr<   rs   rf   FN      4@r   FNry   rC   FNc           	      C   \   g d}g d}t ||dd}t|}t| |d}|||ks!J ||j||ks,J d S )Nr/   r   r/   r'   )rs   r2   )rs   r9   r   r<   r;   r:   FrB   r    r	   r   r   select_bucketr@   	r   r   r]   r^   bucketsrA   rF   	filter_2drX   r   r   r    test_2d_bucketing_filter_lenientr      )r   )r_   r`   ra   rb   rc   )r   rd   FN)r   r9   FNre   rg   rh   ri   rj   rk   )r/   rd   FN)r/   r9   FNrl   rm   rn   ro   rp   rq   rr   rt   ru   rv   rw   rx   rz   c           	      C   r{   )Nr|   r   Tr   r    r   r   r   r   r   test_2d_bucketing_filter_strict   r   r   c                  C   s   g d} g d}g d}t | |dd}t|}tddd}||dks%J ||j|d	ks0J t | |d|d
}t|}tddd}||dksJJ ||j|d ksUJ d S )Nr|   )      @r         @r   r   Tr   r   r'   r    r:   )rB   	max_ratioFr   )r   r   rA   rF   r   rX   r   r   r   )test_2d_bucketing_filter_strict_max_ratio   s   r   c                   @   s   e Zd Zdd ZdS )	_Identityc                 C   s   |S )Nr   )selfitemr   r   r   __getitem__   s   z_Identity.__getitem__N)__name__
__module____qualname__r   r   r   r   r   r      s    r   c                 C   sB  |d }t tddddtddddg| t|dg dg d	dd
ddt d}dd |D }t|dks9J t|d dksCJ t|d d jd jdksSJ t|dg dg d	dd
ddt d}dd |D }t|dksuJ t|d dksJ t|d d jd jdksJ t|d d jd jdksJ d S )Nz
cuts.jsonlr   r   r   r    rI   T)r}   r~   )rs      )rs   i,  )r:   r:   r:   r:   )	cuts_pathuse_bucketingbucket_duration_binsbucket_batch_sizebucketing_2d_strict_moder:   )global_rank
world_sizedatasetc                 S      g | ]}|qS r   r   r#   br   r   r   r%         z<test_2d_bucketing_strict_mode_flag_works.<locals>.<listcomp>Fc                 S   r   r   r   r   r   r   r   r%     r   r;   )r   r   to_filer
   r   rR   r   r   )r   tmp_pathr   dloaderbatchesr   r   r   (test_2d_bucketing_strict_mode_flag_works   sB   &  $r   )r   r   r   )"numpyr   pytesttorch.utils.datatorchlhotser   r   r   lhotse.datasetr   lhotse.testing.dummiesr   lhotse.testing.randomr   .nemo.collections.common.data.lhotse.dataloaderr   r	   r
   intr   fixturer.   r\   markparametrizer   r   r   utilsdataDatasetr   r   r   r   r   r   <module>   s2   
?
(
(