o
    5tiK                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZmZ d dlZd dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ eeZd
d ZdefddZdedededej j!fddZ"dd Z#edG dd deZ$dS )    N)deepcopy)ListLiteral)tqdm)Instance)LM)register_model)Collator)get_rolling_token_windowsmake_disjoint_windowsimple_parse_args_stringc           	   
   C   s   zdd l }W n ty } zt|dd }~ww |j| d || ' dd d |jd ||dd}| D ]\}}t| |rC|| |< q6W d    | S 1 sOw   Y  | S )Nr   Attempted to use 'nemo_lm' model type, but package `nemo` is not installedPlease install nemo following the instructions in the README: either with a NVIDIA PyTorch or NeMo container, or installing nemo following https://github.com/NVIDIA/NeMo.TF)sequence_parallel"activations_checkpoint_granularityactivations_checkpoint_method	precisionglobal_batch_sizetensor_model_parallel_sizepipeline_model_parallel_sizeapply_rope_fusion)		omegaconfModuleNotFoundErrortype	OmegaConf
set_struct	open_dictr   itemshasattr)	pretrained_cfgtrainerr   r   r   	exceptionattributes_to_updatenamevalue r$   J/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/models/nemo_lm.py_patch_pretrained_cfg(   s8   


r&   returnc                 C   s   | j  d| j S )N.)
__module____name__)target_classr$   r$   r%   _get_target_from_classF   s   r,   
model_pathr   r   c              
      sN  zddl m} ddlm} W n ty  } zt|dd }~ww t| } | }|  r2| 	 |_
|jd | 	 d|d}t|dsHt||d< t||||d}| }	|}
|
jd	d
\}}tt||}td| j d  fdd}|j|_||_|j|		 ||
|d|j d}|  d|_z	d |jjj_W |S  t y   Y |S w )Nr   )MegatronGPTModel)NLPSaveRestoreConnectorr   T)return_configr   targetr   r   r(      z/tmp/z.tokenizer.lockc                    s2     |    W d    d S 1 sw   Y  d S N)_original_build_tokenizerselftokenizer_lockr$   r%   _synced_build_tokenizerv   s   
"z+load_model.<locals>._synced_build_tokenizercuda:)restore_pathr   override_config_pathsave_restore_connectormap_locationF)!@nemo.collections.nlp.models.language_modeling.megatron_gpt_modelr.   (nemo.collections.nlp.parts.nlp_overridesr/   r   r   pathlibPathis_diras_posixmodel_extracted_dirrestore_fromr   r,   r&   r1   rsplitgetattr	importlibimport_modulefilelockFileLockr"   _build_tokenizerr5   
local_rankfreezetrainingmodellanguage_modelencoderr   AttributeError)r-   r   r   r   r.   r/   r    r>   r   model_to_load_pathoverride_configmodule_name
class_namemodel_classr:   rR   r$   r8   r%   
load_modelJ   s`   



r[   c              
   C   sp   zddl m} W n ty } zt|dd }~ww dd }| jjd ur.| jjj|| d | j  | }|S )Nr   )AppStater   c                   S   s   d S r4   r$   r$   r$   r$   r%   dummy   s   z,setup_distributed_environment.<locals>.dummy)r   )nemo.utils.app_stater\   r   r   strategylauncherlaunchsetup_environment)r   r\   r    r]   	app_stater$   r$   r%   setup_distributed_environment   s   
rd   nemo_lmc                       s.  e Zd Z								d9dedededed	ed
edededed f fddZed:ddZe	dd Z
e	dd Ze	dd Ze	dd Ze	dd Ze	dd Ze	d d! Ze	d"d# ZG d$d% d%Zd&efd'd(Zd)d* Zd+d, Zd-d. Z	/d;d0ee d1ed2ee fd3d4Zd;d5d6Zd7d8 Z  ZS )<NeMoLM   r3      bf16path
max_length
batch_sizemax_gen_toksdevices	num_nodesr   r   r   )z16-mixedz
bf16-mixedz32-truez64-true@          643216ri   c
              
      s  zddl m} ddlm} ddlm} || _W n ty) } zt|dd }~ww t 	  |dkrS|dkrS|dkrSt
d| d t
d	| d t
d
 n%|| |krtt
d| d| d t
d	| d t
d ntd|dkrtd|| |d||	dddd}|dkr|dkr|dkrtd|j | _|j| _|j| _t||||d | _| jj| _t|| _|| _t|| _|| _d S )Nr   )Trainer)generate)NLPDDPStrategyr   r3   z.The number of data replicas for evaluation is r(   zThe total number of devices is z9No tensor parallelism or pipeline parallelism is applied.zSetting tensor parallelism to z and pipeline parallelism to zNo data parallelism is applied.z~Please set the product of tensor_model_parallel_size and pipeline_model_parallel_sizeequal to the specified number of devices.zQA number of nodes greater than 1 is not supported yet. Please set num_nodes as 1.gpuF)r_   rn   acceleratorro   r   loggerenable_checkpointinguse_distributed_samplerr;   r2   )!lightning.pytorch.trainer.trainerrv   9nemo.collections.nlp.modules.common.text_generation_utilsrw   rA   rx   r   r   super__init__eval_loggerinfo
ValueErrortorchdeviceglobal_rank_device_rank
world_size_world_sizer[   cudarR   	tokenizerrd   rc   _max_lengthint_batch_size_max_gen_toks)r7   rj   rk   rl   rm   rn   ro   r   r   r   kwargsrv   rw   rx   r    r   	__class__r$   r%   r      s   






zNeMoLM.__init__Nc                 C   s*   t |}|r|dd|d< | di |S )Nrl   r3   r$   )r   get)cls
arg_stringadditional_configargsr$   r$   r%   create_from_arg_string  s   zNeMoLM.create_from_arg_stringc                 C   s    z| j jW S  ty   Y d S w r4   )r   eos_idrU   r6   r$   r$   r%   eot_token_id  s
   
zNeMoLM.eot_token_idc                 C      | j S r4   )r   r6   r$   r$   r%   rk        zNeMoLM.max_lengthc                 C   r   r4   )r   r6   r$   r$   r%   rm   #  r   zNeMoLM.max_gen_toksc                 C   r   r4   )r   r6   r$   r$   r%   rl   '  r   zNeMoLM.batch_sizec                 C   r   r4   )r   r6   r$   r$   r%   r   +  r   zNeMoLM.devicec                 C   r   r4   )r   r6   r$   r$   r%   rank/  r   zNeMoLM.rankc                 C   r   r4   )r   r6   r$   r$   r%   r   3  r   zNeMoLM.world_sizec                 C   s   |  | jS r4   )_Acceleratorr   r6   r$   r$   r%   rz   7  s   zNeMoLM.acceleratorc                   @   s$   e Zd Zdd Zdd Zdd ZdS )zNeMoLM._Acceleratorc                 C   s
   || _ d S r4   )r   )r7   r   r$   r$   r%   r   <  s   
zNeMoLM._Accelerator.__init__c                 C   s   t j  d S r4   )r   distributedbarrierr6   r$   r$   r%   wait_for_everyone?  s   z%NeMoLM._Accelerator.wait_for_everyonec                    s0    fddt | jD }tj|  t|S )Nc                    s    g | ]}t jd  jd qS )r3   )dtype)r   zerosr   r   ).0_local_tensorr$   r%   
<listcomp>C  s    z.NeMoLM._Accelerator.gather.<locals>.<listcomp>)ranger   r   r   
all_gathercat)r7   r   gathered_tensorsr$   r   r%   gatherB  s
   

zNeMoLM._Accelerator.gatherN)r*   r)   __qualname__r   r   r   r$   r$   r$   r%   r   ;  s    r   stringc                 C      | j |S r4   )r   text_to_ids)r7   r   r$   r$   r%   
tok_encodeJ     zNeMoLM.tok_encodec                 C   r   r4   )r   ids_to_text)r7   tokensr$   r$   r%   
tok_decodeM  r   zNeMoLM.tok_decodec                 C   sp   t |t |  }|dkr|| d  | }|d |  }| || }| |}t |}||d  }||fS Nr   )lenrstripr   )r7   contextcontinuationn_spaces	whole_enccontext_enccontext_enc_lencontinuation_encr$   r$   r%   _encode_pairP  s   
zNeMoLM._encode_pairc                 C   sh   g }dd |D D ]%\}}|dkr| j g| |}}n| ||\}}|||f||f q	| |S )Nc                 S      g | ]}|j qS r$   r   r   reqr$   r$   r%   r   ]      z(NeMoLM.loglikelihood.<locals>.<listcomp> )r   r   r   append_loglikelihood_tokens)r7   requestsnew_reqsr   r   r   r   r$   r$   r%   loglikelihood[  s   
zNeMoLM.loglikelihoodFr   disable_tqdmr'   c              
   C   s   g }t dd |D |dD ]<\}tttt| || j| jd dd}dd |D }| |}dd |D }t	|}|
| | jd|f| q|S )	Nc                 S   r   r$   r   r   r$   r$   r%   r   p  r   z0NeMoLM.loglikelihood_rolling.<locals>.<listcomp>)disabler3   )
token_listprefix_tokenmax_seq_lencontext_lenc                 S   s   g | ]}d | qS )r4   r$   r   xr$   r$   r%   r   }      c                 S   s   g | ]}|d  qS r   r$   r   r$   r$   r%   r     r   loglikelihood_rolling)r   listmapr   r
   r   r   rk   r   sumr   
cache_hookadd_partial)r7   r   r   loglikelihoodsr   rolling_token_windows
string_nllr$   r$   r%   r   k  s,   
zNeMoLM.loglikelihood_rollingc           !   	   C   sR  g }dd }t ||d}|j| jd d}tt||p| jdkdd}|D ]}g }	g }
g }|D ]9\}}}|| | jd  d  }t|tdt|t| | jd   }|
| |t| |	| 	| q.| j
| j|	ddd	d	d
}t|d d d d df }|d d d d df }|d d d d dd d f }t|
}t|d d |d d d d f d  }t||||
||D ]\\}}}}}\}}}||d d  d | }t| }||d  d | }|| }|||d  d | k}t|ts| }||f} |d ur| jd||  ||  |d qq$|  ||S )Nc                 S   s"   | d | d  }t | t|fS )Nr3      )r   tupler   toksr$   r$   r%   _collate  s   z.NeMoLM._loglikelihood_tokens.<locals>._collate)sort_fnnbatch_fnr   zRunning loglikelihood requests)totalr   descr3   T)inputstokens_to_generatemin_tokens_to_generatecompute_logprob	all_probs	token_idslogprobfull_logprobr   )r	   get_batchedrl   r   r   r   rk   maxr   r   rw   rR   npasarrayminr   argmaxcpunumpyzipr   tolist
isinstanceboolallr   r   updatecloseget_original)!r7   r   r   resr   re_ordchunkspbarchunkinpsctxlenscontlensr   r   r   inpctxlenoutputbatch_token_idsbatch_logprobsbatch_full_logprob
min_ctxlenbatch_greedy_tokensr   greedy_tokenslogprobscontlen	cache_keyr   continuation_tokenslen_diff	is_greedyanswerr$   r$   r%   r     s   
	$


!
zNeMoLM._loglikelihood_tokensc                    sd  |sg S g } fdd} fdd}t dd |D |dd}|j jd d	}|D ]}t| \}}	|	d
 }
||
}|
d j} j| }g }|D ]\}} |}|| d  }| 	| qH j
 j|||dd}|d }g }t||D ]\}}||t|d   qw|D ]fdd|D }qt||D ]\}} jd|| || qq(||S )Nc                    sJ   |  dg }t|} j jgd |vr#| j jgd  |S )Nuntilr   )r   r   r   ids_to_tokensr   r   )req_argsr  r6   r$   r%   	get_until  s
   z(NeMoLM.generate_until.<locals>.get_untilc                    s     | d }t|| d fS r   )r   r   r   r6   r$   r%   r     s   z'NeMoLM.generate_until.<locals>._collatec                 S   r   r$   r   )r   regr$   r$   r%   r     r   z)NeMoLM.generate_until.<locals>.<listcomp>
gen_kwargs)r   group_byr   r   rm   T)r   r   end_stringsgreedy	sentencesc                    s   g | ]	}|  d  qS r   )split)r   r  )termr$   r%   r     s    greedy_until)r	   r   rl   r   r   rm   rk   r   r   r   rw   rR   r   r   r   r  )r7   r   r  r!  r   re_ordsr  r
  contextsall_gen_kwargsr   r  rm   remaining_lengthr   r   encoded_contextr  answerscontinuationsr  requestr$   )r7   r)  r%   generate_until  sL   


zNeMoLM.generate_until)rg   r3   rh   r3   r3   r3   r3   ri   r4   )F)r*   r)   r   strr   r   r   classmethodr   propertyr   rk   rm   rl   r   r   r   rz   r   r   r   r   r   r   r   r  floatr   r   r3  __classcell__r$   r$   r   r%   rf      s~    	
i









"\rf   )%rJ   loggingrB   copyr   typingr   r   rL   r   r   r   r   lm_eval.api.instancer   lm_eval.api.modelr   lm_eval.api.registryr   lm_eval.models.utilsr	   lm_eval.utilsr
   r   r   	getLoggerr*   r   r&   r4  r,   r   nnModuler[   rd   rf   r$   r$   r$   r%   <module>   s:   

E