o
    wi                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ ejdejjdd Zejdejjejjddd	d
 Zdd Zdd ZdS )    N)HuggingFaceLLMDeploy)broadcast_listGPUc                  C   s   t dddddd} | jddgddd	}t|d
ksJ dt|d dks)J dt|d dks5J d| jddgdddddd}d|v sJJ dd|v sRJ dd|v sZJ dt|d d
ksfJ ddS )z.Tests HF deployable class's generate function.)/home/TestData/llm/models/llama3.2-1B-hf/text-generationTNhf_model_id_pathtasktrust_remote_code
device_maptp_planWhat is the color of a banana? Tell me a joke.    )text_inputs
max_length	do_sample   z Output should have to be a list.r   :First list in the output should have more than 0 elements.   ;Second list in the output should have more than 0 elements.)r   r   r   output_logitsoutput_scoresreturn_dict_in_generatelogitszOutput should have logits.scoreszOutput should have scores.	sentenceszOutput should have sentences.zOutput should have 2 sentences.)r   generatelen)hf_deployableoutput r!   X/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/deploy/test_hf_import.pytest_hf_generate   s6   r#   zwill be enabled later.)reasonc                   C   s   t jtdd dS )zATests HF deployable class's generate function with multiple GPUs.r   )nprocsN)mpspawn_run_generater!   r!   r!   r"   test_hf_multigpu_generateA   s   r)   c                 C   s   dt jd< dt jd< dt jd< | dkr+t| t jd< tjd	| d
d t  t  dS t| t jd< tjd	| d
d t  t  dS )z"Code to run generate in each rank.2
WORLD_SIZE	localhostMASTER_ADDR12355MASTER_PORTr   RANKncclr   )rank
world_sizeN)osenvironstrdistinit_process_group_hf_generate_ranksdestroy_process_group)r2   r!   r!   r"   r(   J   s   


r(   c            	   
   C   s2  t jt  tdddddd} t dkred}d}d	}d
}d}d}ddg}tjt jdgt jdddd t	|dd t	||||||gdd | j
||d|||||d}tjt jdgt jdddd n|   t  t dkrt|dks}J dt|d dksJ dt|d dksJ ddS dS )zGenerate by Ranksr   r   TNr   r   g      ?r   g        r   Fr   r   cuda)dtypedevice)src)datar>   )r   r   r   temperaturetop_ktop_pr   r   r   z!Output should have to be a lists.r   r   )torchr;   
set_devicer7   get_rankr   	broadcasttensorlongr   r   generate_other_ranksbarrierr   )	r   r@   rA   rB   num_tokens_to_generater   r   promptsr    r!   r!   r"   r9   ]   s^    
r9   )r4   pytestrC   torch.distributeddistributedr7   torch.multiprocessingmultiprocessingr&   nemo.deploy.nlp.hf_deployabler   nemo.deploy.utilsr   markrun_only_onunitr#   skipr)   r(   r9   r!   r!   r!   r"   <module>   s    

$