o
    ©Ì³i°  ã                   @   st  d dl mZmZmZ d dlmZmZ d dlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6 d d	l7m8Z8m9Z9 d d
l:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA d dlBmCZC d dlDmEZEmFZFmGZG d dlHmIZI d dlJmKZK g d¢ZLdS )é    )Úget_act_offloading_ctx_managerÚNoOpManagerÚOffloadActivations)Úcompile_lossÚcompile_model)Úgather_cpu_state_dictÚget_distributed_backendÚget_full_optimizer_state_dictÚget_shard_conditionsÚget_world_size_and_rankÚinit_distributedÚis_distributedÚload_from_full_model_state_dictÚ#load_from_full_optimizer_state_dictÚprepare_mha_for_tpÚset_torch_num_threadsÚshard_modelÚ!validate_no_params_on_meta_device)Úscale_grads)Údisable_dropout)ÚDEFAULT_PROFILE_DIRÚDEFAULT_PROFILER_ACTIVITIESÚDEFAULT_SCHEDULEÚDEFAULT_TRACE_OPTSÚDummyProfilerÚPROFILER_KEYÚsetup_torch_profiler)Ú(apply_selective_activation_checkpointing)ÚADAPTER_CONFIGÚADAPTER_KEYÚCheckpointerÚDATALOADER_KEYÚDistributedCheckpointerÚ
EPOCHS_KEYÚFormattedCheckpointFilesÚFullModelHFCheckpointerÚFullModelMetaCheckpointerÚFullModelTorchTuneCheckpointerÚMAX_STEPS_KEYÚ	MODEL_KEYÚ	ModelTypeÚOPT_KEYÚRNG_KEYÚSEED_KEYÚ	STEPS_KEYÚTOTAL_EPOCHS_KEYÚ update_state_dict_for_classifier)Úget_cosine_schedule_with_warmupÚget_lr)Úcleanup_before_trainingÚcreate_optim_in_bwd_wrapperÚget_memory_statsÚlog_memory_statsÚOptimizerInBackwardWrapperÚregister_optim_in_bwd_hooksÚset_activation_checkpointing)Úget_unmasked_sequence_lengths)Ú	get_dtypeÚset_default_dtypeÚvalidate_expected_param_dtype)Úget_quantizer_mode)Úset_seed)>r   r   r   r;   r<   r=   r%   r&   r"   r'   r*   r    r0   r   r   r#   r(   r)   r+   r,   r-   r.   r/   r>   r1   r   r2   r3   r4   r5   r6   r7   r8   r9   r   r   r   r   r   r
   r   r   r	   r   r   r?   r:   r   r   r   r   r   r   r   r   r   r   r   r$   r   r   r!   N)MÚ)torchtune.training._activation_offloadingr   r   r   Útorchtune.training._compiler   r   Útorchtune.training._distributedr   r   r	   r
   r   r   r   r   r   r   r   r   r   Útorchtune.training._grad_scalerr   Útorchtune.training._model_utilr   Útorchtune.training._profilerr   r   r   r   r   r   r   Útorchtune.training.activationsr   Ú torchtune.training.checkpointingr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   Ú torchtune.training.lr_schedulersr1   r2   Útorchtune.training.memoryr3   r4   r5   r6   r7   r8   r9   Útorchtune.training.poolingr:   Útorchtune.training.precisionr;   r<   r=   Útorchtune.training.quantizationr>   Útorchtune.training.seedr?   Ú__all__© rO   rO   úO/home/ubuntu/.local/lib/python3.10/site-packages/torchtune/training/__init__.pyÚ<module>   s   <$	T$	