o
    wi:                  
   @   s  d dl Z d dlmZ d dlmZmZ d dlmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZmZmZmZ d dlmZmZ d dlm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) ede%fddZ*de+de+fddZ,	d&de+de+de-ddfddZ.dedeej/ef fddZ0dededej/d eddf
d!d"Z1de+d#e2dej/defd$d%Z3ddgZ4dS )'    N)Path)TupleUnion)	TrainerFn)dist_checkpointing)Console)AutoTokenizer)LoRA	LoRAMerge)factory)MegatronStrategyTrainer_strategy_libio)ADAPTER_META_FILENAMEckpt_to_context_subdir)api)TrainerContextckpt_to_weights_subdir)PEFT)RestoreConfig)loggingreturnc                   C   s   t  S ) )r	    r   r   Z/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/llm/peft/api.pygpt_lora%   s   r   lora_checkpoint_pathoutput_pathc                 C   s4   t jt| dt|d}t }|d| d |S )a  
    Export the LoRA adapter weights to HF format. Requires an implementation of HF PEFT exporter class.
    See HFLlamaPEFTExporter for an example.

    Python Usage:
    ```python
    if __name__ == '__main__':
        llm.peft.export_lora(
            lora_checkpoint_path=your_lora_checkpoint_path,
            output_path=your_output_path,
        )
    ```

    Args:
        lora_checkpoint_path: The path to the LoRA checkpoint.
        output_path: The path to save the HF checkpoint.

    zhf-peft)pathtargetr   u'   [green]✓ LoRA checkpoint exported to [/green])r   export_ckptr   r   print)r   r   outputconsoler   r   r   export_lora+   s   r&   Flegacy_ckptc                 C   s   ddl m} tddtdd| dd}|rd|j_t| \}}tt| ||| t	 }||jj
}d	d
 |  D }	t||	|| t }
|
d| d dS )a  
    Merges the LoRA adapter weights into the base model's weights.

    Python Usage:
    ```python
    if __name__ == '__main__':
        llm.peft.merge_lora(
            lora_checkpoint_path=your_lora_checkpoint_path,
            output_path=your_output_path,
        )
    ```

    Args:
        lora_checkpoint_path: The path to the LoRA checkpoint.
        output_path: The path to save the merged checkpoint.

    r   )
bf16_mixed   cpupytorchF)ddpsetup_optimizersplugins)devicesacceleratorstrategyc                 S   s   i | ]\}}d |vr||qS z	.adapter.r   .0kvr   r   r   
<dictcomp>s   s    zmerge_lora.<locals>.<dictcomp>u/   [green]✓ LoRA checkpoint merged and saved to r!   N)6nemo.collections.llm.recipes.precision.mixed_precisionr(   r   r   r1   ckpt_load_strictness_load_base_model_and_lora,_setup_trainer_and_restore_model_and_adapterr   r
   megatron_parallelsharded_state_dictitems_save_merged_weightr   r#   )r   r   r'   r(   trainermodellora
lora_mergemerged_modelmerged_weightsr%   r   r   r   
merge_loraL   s    rF   c                 C   sZ   t t| d}d\|_|j_d|j_tj|j_	t t| d}t
|ts)J d||fS )NrA   )NNTzmodel.model_transformz#LoRA config not found in checkpoint)r   load_contextr   model_transform__io__configbf16torchbfloat16params_dtype
isinstancer	   )r   rA   rB   r   r   r   r:   z   s   
r:   r@   rA   rB   c                 C   sn  t | ddt  } r0t|d}t|}W d    n1 s!w   Y  t|d ddd}ntd|  ||j_	d|j_
d |_|j| |j  | skt|j |  W d    n1 sfw   Y  |j| tj|j_|jj|d ||j_||_|| t | dd}|jj|}	d	d
 |jjj|	d D }
|jjj ||
d}|jj!|dd d S )NF	is_savingrmodel_ckpt_pathT)r   load_model_stateload_optim_statez!Cannot find adapter meta file in )r@   c                 S   s   i | ]\}}d |v r||qS r2   r   r3   r   r   r   r7      s
    z@_setup_trainer_and_restore_model_and_adapter.<locals>.<dictcomp>)metadata)r=   )strict)"r   r   existsopenjsonloadr   
ValueErrorr1   restore_config_setup_optimizers	ckpt_pathconnectsetup_environment
state_dictr   megatron_cpu_init_contextrJ   configure_modelsetupr   TESTINGstatefnsetup_megatron_parallelr@   unwrapped_checkpoint_ioload_content_metadatar<   r=   r>   checkpoint_ioload_checkpointload_model_state_dict)r   r@   rA   rB   adapter_meta_pathfrV   r]   weights_dirsharded_sd_metadataadapter_sharded_state_dictadapter_stater   r   r   r;      sH   


r;   rE   c                 C   s   t | dd}t|jddd tj|tt | dd|jjd t|j	dr2|j	
d tddd|_	t|jdrGt|jj	drG|jj	j|jj_	t|jt| d	gd
 td|   d S )NTrP   )parentsexist_ok)content_metadatasave_pretrainedz/tmp/nemo_tokenizer)trust_remote_coderI   rA   )
yaml_attrszMerged checkpoint saved to )r   r   mkdirr   savestrr1   sharded_state_dict_metadatahasattr	tokenizerrx   r   rA   rI   r   from_trainerio_dumpr   r   info)r   rE   rA   r@   weight_pathr   r   r   r?      s   r?   )F)5rZ   pathlibr   typingr   r   lightning.pytorchr+   plrL    lightning.pytorch.trainer.statesr   megatron.corer   rich.consoler   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr   nemo.collections.llm.peft.lorar	   r
   nemo.collections.llm.utilsr   nemo.lightningr   r   r   r   nemo.lightning.ckpt_utilsr   r   nemo.lightning.ior   nemo.lightning.io.plr   r    nemo.lightning.pytorch.callbacksr   'nemo.lightning.pytorch.strategies.utilsr   
nemo.utilsr   r   r}   r&   boolrF   LightningModuler:   r;   dictr?   __all__r   r   r   r   <module>   sb   
$
.
.