o
    Ii^                  	   @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZ ejejd e ZG dd dej j!Z"dd Z#dd Z$dd Z%dd Z&dd Z'dd Z(dd Z)dd Z*dd  Z+e,d!kre# \Z-Z.Z/e/j0d"krzYz=ed#Z1e12 se1j3d$d$d% e$e/e-Z-e/4  e 5  ee.e/e-e1 e/4  e(e-e1 e)e-e1 e*e-e1 e+e-e1 W n e6y    w W e/j7re8e1 e/4  e/9  dS e/j7re8e1 e/4  e/9  w dS dS )&    N)Path)	load_file)ShardingStrategyStateDictType)
DataLoader)AcceleratorFullyShardedDataParallelPlugin)merge_commandmerge_command_parser)AcceleratorState)torch_device)RegressionDataset)merge_fsdp_weightspatch_environmentsave_fsdp_model)levelc                       s$   e Zd Z fddZdd Z  ZS )	TinyModelc                    sF   t    tjdd| _tj | _tjdd| _tj	 | _
d S )N   )super__init__torchnnLinearlinear1ReLU
activationlinear2Softmaxsoftmax)self	__class__ d/home/ubuntu/.local/lib/python3.10/site-packages/accelerate/test_utils/scripts/test_merge_weights.pyr   '   s
   
zTinyModel.__init__c                 C   s   |  | | |S )N)r   r   r   )r   xr"   r"   r#   forward.   s   zTinyModel.forward)__name__
__module____qualname__r   r%   __classcell__r"   r"   r    r#   r   &   s    r   c                  C   s|   t ji kr
t    ttjtjd} t }t	dd | 
| W d    n1 s*w   Y  t| d}||}|| |fS )N)sharding_strategystate_dict_typeSIZE_BASED_WRAP)fsdp_auto_wrap_policy)fsdp_plugin)r   _shared_state_reset_stater   r   
FULL_SHARDr   SHARDED_STATE_DICTr   r   set_auto_wrap_policyr   prepare)pluginmodelacceleratorr"   r"   r#   setup2   s   




r8   c           	      C   s   t ddd}t|ddd}tjj| dd}| |||\}}}td	D ]$}|D ]}|  ||d
 }tj	j
||d }| | |  q)q%|S )N   *   )lengthseedr   F)
batch_sizeshuffleg?)lr   r$   y)r   r   r   optimSGD
parametersr4   range	zero_gradr   
functionalmse_lossbackwardstep)	r7   r6   	train_settrain_dl	optimizer_batchoutputlossr"   r"   r#   mock_training@   s   

rR   c                 C   sJ   t | | D ]\}}| dkrt||sJ q	t||r"J q	d S )Nsame)zipvaluesr   allclose)	operationstate_1state_2weight_1weight_2r"   r"   r#   check_weightsP   s
   r\   c                 C   sN   t | d }t t}td| |  || td| |  d S )Nzmodel.safetensorsdiffrS   )r   r   tor   r\   
state_dictload_state_dict)pathr6   safe_state_dictsafe_loaded_modelr"   r"   r#   check_safetensors_weightsX   s
   
rd   c                 C   sT   t j| d dd}t t}td| |  || td| |  d S )Nzpytorch_model.binT)weights_onlyr]   rS   )r   loadr   r^   r   r\   r_   r`   )ra   r6   nonsafe_state_dictnonsafe_loaded_modelr"   r"   r#   check_pytorch_weights`   s
   
ri   c                 C       t |d |dd t||  d S )Npytorch_model_fsdp_0Tsafe_serialization)r   rd   r6   ra   r"   r"   r#   test_merge_weights_safetensorsh      ro   c                 C   s0   t t|d t|g}t| t||  d S )Nrk   )parser
parse_argsstrr	   rd   r6   ra   argsr"   r"   r#   &test_merge_weights_command_safetensorsn   s   rv   c                 C   rj   )Nrk   Frl   )r   ri   rn   r"   r"   r#   test_merge_weights_pytorcht   rp   rw   c                 C   s2   t t|d t|dg}t| t||  d S )Nrk   z--unsafe_serialization)rq   rr   rs   r	   ri   rt   r"   r"   r#   "test_merge_weights_command_pytorchz   s   rx   __main__   test_merge_weights_fsdp_weightsT)parentsexist_ok):gcloggingshutilpathlibr   r   safetensors.torchr   2torch.distributed.fsdp.fully_sharded_data_parallelr   r   torch.utils.datar   
accelerater   r   accelerate.commands.merger	   r
   accelerate.stater   accelerate.test_utilsr   accelerate.test_utils.trainingr   accelerate.utilsr   r   r   basicConfigINFOrq   r   Moduler   r8   rR   r\   rd   ri   ro   rv   rw   rx   r&   r6   r5   r7   num_processesout_pathexistsmkdirwait_for_everyonecollect	Exceptionis_main_processrmtreeend_trainingr"   r"   r"   r#   <module>   sp   







