o
    }oiM                     @   s  d dl Zd dlZd dlmZ d dlmZ d dlm	Z
 d dlmZ d dlmZ dZefdd	Zed
kr~ee edkrd dlZe Zejddd ejdded ejdded ejdded ejdded ejdddd ejdddgd ejdddgd ejded d! ejd"ddd ejd#edd! ejd$edd! e ZdZejdurd d%lmZ d&ej d'd(d Zeeje d)ej! d*ej" d+ZdZ#dZ$dZ%ej%dkrd d,l&m'Z' d d-lm(Z( e'e(ej)d.Z%d d-lm(Z( ej*eje%d/Zej+Z+ej,j-eee+e
j.ej!ej/ej0e
j1ej2ej3ej4ej5d0dd1d de#e$g ed2e6ej7j8d3d4dd5 ej9j:dd6d7 d7 d8k sUJ d9ej9j:dd6d7 d7  d:ej%rpej%dkrpeejZ;e;skJ d;e<d<e; ej=dure>ej= dS dS dS dS )=    N)Version)get_torch_version_str)	lightning)llm)is_te_acceleratedz#/home/TestData/lite/hf_cache/squad/c                    s8    fdd}t j|d jd}|j|ddg dd |S )	Nc                    s   d| d  d| d  dd| d d d	    g}tt j|\}}t|d	kr8|d	  jkr8|d	 j t|d	krK|d
  jkrK| j t	|| dd  || d d
 d	gt|d  dgt|  dS )Nz	Context: contextz Question: questionz Answer: answerstextr      )labels	input_ids	loss_mask)
striplistmaptext_to_idslenbos_idinserteos_idappenddict)exampleformatted_textcontext_ids
answer_ids	tokenizer V/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/hf/sft_fsdp2.pyformatting_prompts_func   s   z6make_squad_hf_dataset.<locals>.formatting_prompts_funcz
train[:10])splitpad_token_idF   )idtitler   r   r
   )batched
batch_sizeremove_columns)r   HFDatasetDataModuler   r   )r    	data_pathr#   
datamoduler!   r   r"   make_squad_hf_dataset   s   r/   __main__z2.4z--modelzmeta-llama/Llama-3.2-1B)defaultz	--devicesr&   )r1   typez	--dp-sizez	--tp-sizer   z	--cp-sizez--sequence-parallelF
store_true)r1   actionz--acceleratorgpu)r1   choicesz--model-acceleratortez--max-steps   )r2   r1   z--fp8-autocastz--wandb-projectz--model-save-path)WandbLogger_/_dev_strat_)projectname)partial)te_accelerate)fp8_autocast)
model_namemodel_accelerator)data_parallel_sizetensor_parallel_sizecontext_parallel_sizesequence_parallelg        )devices	max_stepsacceleratorstrategylog_every_n_stepslimit_val_batchesnum_sanity_val_stepsaccumulate_grad_batchesgradient_clip_valuse_distributed_sampler	callbacksloggergh㈵>)lr)modeldatatraineroptimlog)devicei   ir  zusing z9 MB, larger than 29326 MB when not using parallelization.z0Transformer Engine acceleration was unsuccessfulzTE Accelerated: )?fiddlefdltorchpackaging.versionr   
PkgVersionutilsr   nemor   nlnemo.collectionsr   4nemo.lightning.pytorch.accelerate.transformer_enginer   	DATA_PATHr/   __name__argparseArgumentParserparseradd_argumentintstr
parse_argsargswandbwandb_projectlightning.pytorch.loggersr9   joinrW   r$   rJ   rM   	grad_clipuse_dist_samprE   	functoolsrA   rB   rC   HFAutoModelForCausalLMr    apifinetuneTrainerrK   rL   FSDP2Strategydp_sizetp_sizecp_sizerI   buildadampytorch_adam_with_flat_lrcudamax_memory_allocatedte_accprintmodel_save_pathsave_pretrainedr!   r!   r!   r"   <module>   s   




P