o
    }oi                     @   sR  d dl Zd dlmZ d dlmZ d dlmZ dZ	dd Z
edkr%d dlZe Zejd	d
d ejdedg dd ejddd ejdddgd ejdddgd ejdedd ejdddd ejdedd ejdedd e ZdZejdurd d lmZ d!ejd"d#d Zeeje d$ej d%ej d&Zd'Zejd(krdZdZdZejdkrd d)lm Z  e ej!d*Zd d+lm"Z" ej#ejedd,Zej$Z$ej%j&ee
e	e$ej'ejej(ej)ejdd-d deeg ed.e*ej+j,d/d0dd1 ejrejdkreejZ-e-sJ d2e.d3e- ej/dur'e0ej/ dS dS dS )4    N)	lightning)llm)is_te_acceleratedz#/home/TestData/lite/hf_cache/squad/c                    sL   j   fdd}tdtj| djd}|j|ddg dd	 |S )
Nc                    s\   d}| d }| d }| d d }t |tr|d }||||  }|}|d |d< |S )	NzBelow is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

    ### Instruction:
    {}

    ### Input:
    {}

    ### Response:
    {}contextquestionanswerstextr   	input_idslabels)
isinstancelistformat)examplesalpaca_promptinstructioninputoutputr   ans	EOS_TOKEN	tokenizer U/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/hf/pretrain.pyformatting_prompts_func   s   

z6make_squad_hf_dataset.<locals>.formatting_prompts_funcr   ztrain[:100])splitpad_token_idF   )idtitler   r   r   )batched
batch_sizeremove_columns)	eos_tokengetattrr   HFDatasetDataModuleeos_token_idmap)	data_pathr   r   
datamoduler   r   r   make_squad_hf_dataset   s   r)   __main__z--modelzmeta-llama/Llama-3.2-1B)defaultz
--strategyauto)r,   ddpfsdp)typer+   choicesz	--devices   z--acceleratorgpu)r+   r0   z--model-acceleratortez--max-stepsd   )r/   r+   z--fp8-autocastF
store_true)r+   actionz--wandb-projectz--model-save-path)WandbLogger_/_dev_strat_)projectnameg      ?r.   )TEConfig)fp8_autocast)te_accelerate)
model_namemodel_acceleratorload_pretrained_weightsg        )devices	max_stepsacceleratorstrategylog_every_n_stepslimit_val_batchesnum_sanity_val_stepsaccumulate_grad_batchesgradient_clip_valuse_distributed_sampler	callbacksloggergh㈵>)lr)modeldatatraineroptimlogz0Transformer Engine acceleration was unsuccessfulzTE Accelerated: )1fiddlefdlnemor   nlnemo.collectionsr   4nemo.lightning.pytorch.accelerate.transformer_enginer   	DATA_PATHr)   __name__argparseArgumentParserparseradd_argumentstrint
parse_argsargswandbwandb_projectlightning.pytorch.loggersr7   joinrR   r   rE   rH   	grad_clipuse_dist_samprC   r?   r@   rA   HFAutoModelForCausalLMr   apifinetuneTrainerrF   rG   buildadampytorch_adam_with_flat_lrte_accprintmodel_save_pathsave_pretrainedr   r   r   r   <module>   s   
%




F