o
    }oi                     @   sj   d dl mZ d dlZd dlm  mZ d dlmZ 		ddeej	 deej	 de
de
d	e
dB f
d
dZdS )    )ListN)PackedSeqParams@   tokenslabelsignore_indexpad_to_multiple_offinal_padding_toc              	   C   s  g }g }g }g }dg}	dg}
t t| |D ]\}\}}t|}|| d | | }|t| d krO|rO||
d | ksIJ d|d|
d |  ||
d  }|| }|dkrkt|d|fdd}t|d|fd|}|| || |tjt|tj|j	d || |	|	d |  |
|
d |  qtj
|ddd}tj
|ddd}tj
|ddd}tj|tj|j	d}d	||dk < t|	}	t|
}
t|	|	|
|
tt|tt|d
d}|||||fS )a  
    Convert tokens, labels, and associated inputs into a packed version with padded sequence parameters.

    Args:
        tokens (list[torch.Tensor]): List of token tensors for each instance.
        labels (list[torch.Tensor]): List of label tensors for each instance.
        num_image_embeddings_per_tile (int): Number of image embeddings per tile.
        media_token_index (int): Token ID representing media.
        ignore_index (int): Value to use for padding labels.
        pad_to_multiple_of (int): Sequence length will be padded to a multiple of this value. Default is 8.
        final_padding_to(int): Pad the final seq to make everything to same size
    r      zfinal_padding_to = z, constant)dtypedevice)dimg        thd)cu_seqlens_qcu_seqlens_kvcu_seqlens_q_paddedcu_seqlens_kv_paddedmax_seqlen_qmax_seqlen_kv
qkv_format)	enumerateziplenFpadappendtorcharangeintr   cat	unsqueeze	ones_likefloat	IntTensorr   max)r   r   r   r   r	   packed_tokenspacked_labelspacked_position_idsseqlens_padded
cu_seqlenscu_seqlens_paddediinstance_tokensinstance_labelsseqlenseqlen_paddedpad_lenpacked_loss_maskpacked_seq_params r5   ^/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/llava_next/data/utils.pyconvert_to_packed_llava_next   sR   







r7   )r   N)typingr   r   torch.nn.functionalnn
functionalr   megatron.core.packed_seq_paramsr   Tensorr    r7   r5   r5   r5   r6   <module>   s"   