o
    }oi                  
   @   s  d dl Z d dlmZ d dlmZmZ d dlmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZmZmZmZ d dlmZmZ d dlm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) ede%fddZ*de+de+fddZ,	d&de+de+de-ddfddZ.dedeej/ef fddZ0dededej/d eddf
d!d"Z1de+d#e2dej/defd$d%Z3ddgZ4dS )'    N)Path)TupleUnion)	TrainerFn)dist_checkpointing)Console)AutoTokenizer)LoRA	LoRAMerge)factory)MegatronStrategyTrainer_strategy_libio)ADAPTER_META_FILENAMEckpt_to_context_subdir)api)TrainerContextckpt_to_weights_subdir)PEFT)RestoreConfig)loggingreturnc                   C   s   t  S ) )r	    r   r   Q/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/llm/peft/api.pygpt_lora%   s   r   lora_checkpoint_pathoutput_pathc                 C   s4   t jt| dt|d}t }|d| d |S )a  
    Export the LoRA adapter weights to HF format. Requires an implementation of HF PEFT exporter class.
    See HFLlamaPEFTExporter for an example.

    Python Usage:
    ```python
    if __name__ == '__main__':
        llm.peft.export_lora(
            lora_checkpoint_path=your_lora_checkpoint_path,
            output_path=your_output_path,
        )
    ```

    Args:
        lora_checkpoint_path: The path to the LoRA checkpoint.
        output_path: The path to save the HF checkpoint.

    zhf-peft)pathtargetr   u'   [green]✓ LoRA checkpoint exported to [/green])r   export_ckptr   r   print)r   r   outputconsoler   r   r   export_lora+   s   r&   Flegacy_ckptc                 C   s   ddl m} tddtdd| dd}|rd|j_t| \}}tt| ||| t	 }||jj
}d	d
 |  D }	t||	|| t }
|
d| d dS )a  
    Merges the LoRA adapter weights into the base model's weights.

    Python Usage:
    ```python
    if __name__ == '__main__':
        llm.peft.merge_lora(
            lora_checkpoint_path=your_lora_checkpoint_path,
            output_path=your_output_path,
        )
    ```

    Args:
        lora_checkpoint_path: The path to the LoRA checkpoint.
        output_path: The path to save the merged checkpoint.

    r   )
bf16_mixed   cpupytorchF)ddpsetup_optimizersplugins)devicesacceleratorstrategyc                 S   s   i | ]\}}d |vr||qS z	.adapter.r   .0kvr   r   r   
<dictcomp>s   s    zmerge_lora.<locals>.<dictcomp>u/   [green]✓ LoRA checkpoint merged and saved to r!   N)6nemo.collections.llm.recipes.precision.mixed_precisionr(   r   r   r1   ckpt_load_strictness_load_base_model_and_lora,_setup_trainer_and_restore_model_and_adapterr   r
   megatron_parallelsharded_state_dictitems_save_merged_weightr   r#   )r   r   r'   r(   trainermodellora
lora_mergemerged_modelmerged_weightsr%   r   r   r   
merge_loraL   s    rF   c                 C   sZ   t t| d}d\|_|j_d|j_tj|j_	t t| d}t
|ts)J d||fS )NrA   )NNTzmodel.model_transformz#LoRA config not found in checkpoint)r   load_contextr   model_transform__io__configbf16torchbfloat16params_dtype
isinstancer	   )r   rA   rB   r   r   r   r:   z   s   
r:   r@   rA   rB   c           
      C   sX  t | ddt  } r0t|d}t|}W d    n1 s!w   Y  t|d ddd}ntd|  ||j_	d|j_
d |_|j| |j  | skt|j |  W d    n1 sfw   Y  |j| tj|j_|jj|d ||j_||_|| d	d
 |jj  D }|jjjt | dd|d}	|jj|	dd d S )NF	is_savingrmodel_ckpt_pathT)r   load_model_stateload_optim_statez!Cannot find adapter meta file in )r@   c                 S   s   i | ]\}}d |v r||qS r2   r   r3   r   r   r   r7      s    z@_setup_trainer_and_restore_model_and_adapter.<locals>.<dictcomp>)r=   )strict) r   r   existsopenjsonloadr   
ValueErrorr1   restore_config_setup_optimizers	ckpt_pathconnectsetup_environment
state_dictr   megatron_cpu_init_contextrJ   configure_modelsetupr   TESTINGstatefnsetup_megatron_parallelr@   r<   r=   r>   checkpoint_ioload_checkpointload_model_state_dict)
r   r@   rA   rB   adapter_meta_pathfmetadatar\   adapter_sharded_state_dictadapter_stater   r   r   r;      sD   


r;   rE   c                 C   s   t | dd}t|jddd t|tt | dd t|jdr,|jd t	d|_t|j
drAt|j
jdrA|j
jj|j
j_t|jt| dgd td	|   d S )
NTrP   )parentsexist_oksave_pretrainedz/tmp/nemo_tokenizerrI   rA   )
yaml_attrszMerged checkpoint saved to )r   r   mkdirr   savestrhasattr	tokenizerrs   r   rA   rI   r   from_trainerio_dumpr   r   info)r   rE   rA   r@   weight_pathr   r   r   r?      s   
r?   )F)5rY   pathlibr   typingr   r   lightning.pytorchr+   plrL    lightning.pytorch.trainer.statesr   megatron.corer   rich.consoler   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr   nemo.collections.llm.peft.lorar	   r
   nemo.collections.llm.utilsr   nemo.lightningr   r   r   r   nemo.lightning.ckpt_utilsr   r   nemo.lightning.ior   nemo.lightning.io.plr   r    nemo.lightning.pytorch.callbacksr   'nemo.lightning.pytorch.strategies.utilsr   
nemo.utilsr   r   rw   r&   boolrF   LightningModuler:   r;   dictr?   __all__r   r   r   r   <module>   sb   
$
.
*