o
    Ii                     @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZmZ d d	lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& e" rpd
Z'd
Z(ndZ'dZ(d>ddZ)dd Z*dd Z+dd Z,dd Z-dd Z.dd Z/dd Z0dd Z1dd  Z2d!d" Z3d#d$ Z4d%d& Z5d'd( Z6d>d)d*Z7d>d+d,Z8d-d. Z9d/d0 Z:d1d2 Z;d3d4 Z<d5d6 Z=d7d8 Z>d9d: Z?d;d< Z@eAd=kre@  dS dS )?    N)deepcopy)Path)
DataLoaderDataset)Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetRegressionModelare_the_same_tensors)DataLoaderConfigurationDistributedTypegathergather_objectis_bf16_availableis_cuda_availableis_datasets_availableis_fp16_availableis_hpu_availableis_mps_availableis_pytest_availableset_seedsynchronize_rng_statesgMbP?gư>Fc                 C   s4   |rt || t| d}t| ||dS t| |d|dS )zBCreates a dataloader that can also use the `SeedableRandomSampler`)	generatordata_sourcenum_samples)
batch_sizesamplerT)r   shuffler   )r   lenr   )	train_setr   r   use_seedable_samplerr    r#   ]/home/ubuntu/.local/lib/python3.10/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloader9   s   r%   c                 C      t d| j  d S )NzPrinting from the main process printprocess_indexstater#   r#   r$   
print_mainH      r,   c                 C   r&   )Nz%Printing from the local main process )r(   local_process_indexr*   r#   r#   r$   print_local_mainL   r-   r/   c                 C   r&   )NPrinting from the last process r'   r*   r#   r#   r$   
print_lastP   r-   r1   c                 C   s   t d| d| j  d S )NPrinting from process : r'   )r+   process_idxr#   r#   r$   print_onT   s   r5   c               	   C   s  t  } | j}td}|  E | jr2td t|d}|d W d    n1 s,w   Y  nt|d}|d W d    n1 sGw   Y  W d    n1 sVw   Y  | 	  | jrt|}d
| }W d    n1 sxw   Y  z2|dsJ d|dkr|dsJ d|d| jd ksJ d	|d
d  d| j W n ty   |   w | jr| r|  | 	  t }t| | t| j W d    n1 sw   Y  |  }| jr|dksJ | dn|  dksJ | d|d |d t| | t| j W d    n	1 s6w   Y  | jrK|  dksJJ n|  dksVJ |d |d t| |  t!| j W d    n	1 syw   Y  | j"r|  d| jjd  ksJ n|  dksJ |d |d t#|D ]R}t| | j$t%|d| j| W d    n	1 sw   Y  | j&|kr|  d| d| j& ksJ n|  dksJ |d |d qd S )Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
 zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r0   )r)   r2   r3   )'r   num_processesr   main_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithendswithcountAssertionErrorunlinkexistsioStringIO
contextlibredirect_stdouton_main_processr,   r+   getvaluerstriptruncateseekon_local_main_processr/   is_local_main_processon_last_processr1   is_last_processrange
on_processr5   r)   )acceleratorr9   pathftextresultr4   r#   r#   r$   process_execution_checkX   s   


 



&

&
r]   c                  C   s$   t  } | jdkrtd t|  d S )Nr   zTesting, testing. 1, 2, 3.)r	   r.   r(   r*   r#   r#   r$   init_state_check   s   
r^   c                  C   s   t  } tdg tt sJ d| jtjkr)tdg ttj s(J dn| jtj	kr?tdg ttj
 s?J dt }tdg|d t| sTJ d	| jd
kr_td d S d S )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.r   )r   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r	   r   r   r_   get_rng_statedistributed_typer   	MULTI_GPUr`   	MULTI_XPUra   	Generator	get_stater.   r(   )r+   r   r#   r#   r$   rng_sync_check   s   



rh   c                  C   s  t  } d| j }tt|dd}t|| j| j| jdd}g }|D ]	}|t| q t	
|}t	| t	d| sAJ dtt|dd}t|| j| j| jddd}g }|D ]	}|t| qZt	
|}t	| t	d| s{J d| jdkrtd	 tt|ddd
}t|| j| j| jdd}g }|D ]	}|t| qt	
| }|  |tt|ksJ dtt|ddd
}t|| j| j| jddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ d| jdkrtd d S d S )N       r   T)put_on_devicer   %Wrong non-shuffled dataloader result.)rl   split_batchesz Non-shuffled dataloader passing.r   r   !Wrong shuffled dataloader result.zShuffled dataloader passing.r	   r9   r   rV   r   devicer)   appendr   r_   catequalcpuarangelongr(   tolistsortlistr.   r+   lengthdlr\   batchr#   r#   r$   dl_preparation_check   sf   

$
$
r   c               	   C   s  t  } d| j }tt|dd}t|| j| j| jddd}g }|D ]	}|t| q!t	
|}t	| t	d| sBJ dtt|dd}t|| j| j| jdddd}g }|D ]	}|t| q\t	
|}t	| t	d| s}J d| jdkrtd	 tt|ddd
}t|| j| j| jddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ dtt|ddd
}t|| j| j| jdddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ d| jdkr	td d S d S )Nri   rj   rk   T)rl   dispatch_batchesr   rm   )rl   rn   r   z(Non-shuffled central dataloader passing.ro   rp   z$Shuffled central dataloader passing.rq   r|   r#   r#   r$   central_dl_preparation_check   sr   

$	
$
	r   c                  C   s   t  } G dd dt}G dd d}|td| j }|t|dd}t||d}t|| j| j| j}t	|j
d	rFt|j
j
|sDJ d
d S t|j
|sPJ d
d S )Nc                   @   s$   e Zd Zdd Zdd Zdd ZdS )z+custom_sampler_check.<locals>.CustomDatasetc                 S   s
   || _ d S Ndata)selfr   r#   r#   r$   __init__<     
z4custom_sampler_check.<locals>.CustomDataset.__init__c                 S   s
   t | jS r   )r    r   r   r#   r#   r$   __len__?  r   z3custom_sampler_check.<locals>.CustomDataset.__len__c                 S   s
   | j | S r   r   )r   indexr#   r#   r$   __getitem__B  r   z7custom_sampler_check.<locals>.CustomDataset.__getitem__N)__name__
__module____qualname__r   r   r   r#   r#   r#   r$   CustomDataset;  s    r   c                   @   s4   e Zd ZddededefddZdd Zd	d
 ZdS )z0custom_sampler_check.<locals>.CustomBatchSamplerTdataset_lengthr   r   c                 S   s   || _ t|| _|| _d S r   )r   nprw   
data_indexr   )r   r   r   r   r#   r#   r$   r   F  s   
z9custom_sampler_check.<locals>.CustomBatchSampler.__init__c                 s   s@    t | }| jrtj| j}n| j}t||}|E d H  d S r   )r    r   r   randompermutationr   array_split)r   num_batchesr   outputr#   r#   r$   __iter__K  s   z9custom_sampler_check.<locals>.CustomBatchSampler.__iter__c                 S   s   t t| j| j S r   )mathceilr    r   r   r   r#   r#   r$   r   T  s   z8custom_sampler_check.<locals>.CustomBatchSampler.__len__N)T)r   r   r   intboolr   r   r   r#   r#   r#   r$   CustomBatchSamplerE  s    	r   ri   rj   rk   )batch_samplerr   z>Custom sampler was changed after calling `prepare_data_loader`)r	   r   rV   r9   r    r   r   rr   r)   hasattrr   
isinstance)r+   r   r   datasetr   r~   r#   r#   r$   custom_sampler_check8  s   
r   c                  C   s   t d tddd} t| ddd}tdd}t|d}||}g }td	D ]}|D ]	}||d
  q*q&t	|}t d |
d g }td	D ]}|D ]	}||d
  qMqIt	|}t||sgJ dd S )N*   
   r}   seed   Tro   r"   dataloader_config   xr   z;Did not obtain the same items with the same seed and epoch.)r   r
   r   r   r   preparerV   rs   r_   rt   	set_epochallclose)r!   train_dlconfigrX   original_items_r   	new_itemsr#   r#   r$   check_seedable_samplerf  s*   





r   c                  C   sp   t d tdd} t| d}|jdksJ dtttdddd}t|dd	}|jjj	}t
|ts6J d
d S )Nr   Tr   r   r8   z)This test requires more than one process.r   ro   )
dataloaderr"   z:Sampler in BatchSamplerShard is not SeedableRandomSampler.)r   r   r   r9   r   r{   rV   r   r   r   r   r   )r   rX   r   prepared_data_loadertarget_samplerr#   r#   r$   -check_seedable_sampler_in_batch_sampler_shard  s   


r   c            
      C   s   t d d} tddd}t|ddd}td| d}t|d}||}g }td	D ]}|D ]	}||d
  q-q)t	|}d|_
t|d}||}g }	td	D ]}|D ]	}|	|d
  qTqPt	|	}	t||	rnJ dd S )Nr   r   r   r   Tro   )r"   	data_seedr   r   r   +   z1Obtained the same items with different data seed.)r   r
   r   r   r   r   rV   rs   r_   rt   r   r   )
r   r!   r   r   rX   prepared_dlr   r   r   r   r#   r#   r$   %check_seedable_sampler_with_data_seed  s.   





r   c                 C   s   t d |d t| dd}t||||}t }tjj| dd}t	dD ]#}|D ]}	|
  ||	d }
tjj|
|	d }|  |  q+q'||fS )Nr   r   r6   lrr   r   y)r   manual_seedr
   r%   r   r_   optimSGD
parametersrV   	zero_gradnn
functionalmse_lossbackwardstep)r}   r   r   r"   r!   r   model	optimizerepochr   r   lossr#   r#   r$   mock_training  s   

r   c                 C   s&  t  }t }d}|d |j }t|||j || \}}t|js%J dt|js.J dt }t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]$}|D ]}|	  |	|d }tjj||d	 }|| |
  q`q\||	 }	tjj|j|	jttd
d d tjj|j|	jttdd d |d td| d}t|d}t	||||j | }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]$}|D ]}|	  |	|d }tjj||d	 }|| |
  qq||	 }	tjj|j|	jttdd d tjj|j|	jttdd d |d t  s;t! rmtd t "  tdd}tj#dd}	||	}	|j|	dd}t$ddgj%tj&|j'd}||}t( rtd t "  t| d}td|d}t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]&}|D ] }|	  |	|d }tjj||d	 }|| |
  qq||	 }	tjj|j|	jttd d d tjj|j|	jttd!d d t) rt* std" t "  t| d}td|d}t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]&}|D ] }|	  |	|d }tjj||d	 }|| |
  qEqA||	 }	tjj|j|	jttd#d d tjj|j|	jttd$d d d S d S d S )%Nrj      z0Did not obtain the same model on both processes.r6   r   r   r   r   r   c                 S   
   d|  S Nz>Did not obtain the same model on CPU or distributed training.
r#   msgr#   r#   r$   <lambda>     
 z training_check.<locals>.<lambda>)atolrtolr   c                 S   r   r   r#   r   r#   r#   r$   r     r   zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)rn   r"   r   c                 S   r   r   r#   r   r#   r#   r$   r     r   c                 S   r   r   r#   r   r#   r#   r$   r     r   zSTraining yielded the same results on one CPU or distributed setup with batch split.zKeep fp32 wrapper check.fp16)mixed_precisionr   )keep_fp32_wrapperr8   )dtyperr   zBF16 training check.r   bf16)r   r   c                 S   r   r   r#   r   r#   r#   r$   r   >  r   c                 S   r   r   r#   r   r#   r#   r$   r   E  r   zFP16 training check.c                 S   r   r   r#   r   r#   r#   r$   r   d  r   c                 S   r   r   r#   r   r#   r#   r$   r   k  r   )+r	   r_   rf   r9   r   r   abr   r%   r   r   r   r   r   r   r   rV   r   r   r   r   r   r   unwrap_modelrv   testingassert_closeATOLRTOLr(   r   r   r   _reset_stateLinearTensortofloat16rr   r   r   r   )r"   r+   r   r   r}   r!   	old_modelrX   r   r   r   r   r   r   r   r   model_with_fp32_wrapperinput_tensorr#   r#   r$   training_check  s,  













	



r   c              	   C   sj  t  }| dd td|j D }|j|dd}t|dks-J d|j dt| W d    n1 s7w   Y  | dd td|j d	 D }|j|dd2}|jrlt|d	kskJ d
|j dt| nt|dksJ d|j dt| W d    n1 sw   Y  |  | dd td|j d	 D }| dd td|j D }||fD ]|}|d }|j|ddB}|jd	krt|t|ksJ d|j dt| nt|dksJ d|j dt| g }|D ]}|	| qW d    n	1 s	w   Y  |  t
|}	dd |	d t| D }
||
ks2J d| d|
 qd S )Nc                 S      g | ]}t |d qS kdict.0vr#   r#   r$   
<listcomp>q      z8test_split_between_processes_dataset.<locals>.<listcomp>r   Fapply_padding4Each process did not have two items. Process index: 
; Length: c                 S   r   r   r   r   r#   r#   r$   r   w  r   r8   z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: c                 S   r   r   r   r   r#   r#   r$   r     r   c                 S   r   r   r   r   r#   r#   r$   r     r   r   Tz9Single process did not receive all items. Process index: c                 S   s   g | ]}|d  qS r   r#   )r   rr#   r#   r$   r     s    )Gathered results is incorrect. Expected: ; Got: )r	   	from_listrV   r9   split_between_processesr    r)   rU   r@   rs   r   )datasets_Datasetr+   r   resultsodd_data	even_dataexpected_outputresults_per_processr\   gathered_resultsr   r#   r#   r$   $test_split_between_processes_dataseto  sR   "	"
 r
  c               
   C   sx  t  } ttdd| j }| |}t|dks&J d| j dt| W d    n1 s0w   Y  |   ttdd| j }ttdd| j d }||fD ]f}|}| j|dd5}t	t|| j }t||ksJ d| j
 d	| j dt| g }|D ]}|| qW d    n1 sw   Y  |   t|}	|	d t| }
||
ksJ d
| d|
 qSd S )Nr   r   r   r   r8   Tr   Process ; did not get the correct number of item(s). Process index: r   r   )r	   r{   rV   r9   r  r    r)   r@   r   r   rr   rs   r   )r+   r   r  r  r  r  num_samples_per_devicer  r\   r	  r   r#   r#   r$   !test_split_between_processes_list  s6   r  c                  C   s  t  } g d}g d}tg d}| jdv r>|||d}t|}| |}| jdkr@|d |d d d| j  ks?J n5| jdkrT|d |d d	d  ksSJ n!| jd
kru|d |d dd  ksuJ d|d d  d|d  | jdkr|d |d d d| j  ksJ n'| jdkr|d |d d	d  ksJ n| jd
kr|d |d dd  ksJ | jdkrt|d |d d d| j  sJ d|d d d| j   d|d  nO| jdkrt|d |d d	d  sJ d|d d	d   d|d  n'| jd
kr.t|d |d dd  s.J d|d dd   d|d  W d    n	1 s9w   Y  |   d S )N)r8   r   r   r            rj   )r   r   cderZ   gh)r8   r   r   )r   r   r  r   r   rj   r   r   r   z
Expected: 
, Actual: r   r  z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)	r	   r_   tensorr9   r   r  r)   r   r@   )r+   r   r   r  r   	data_copyr  r#   r#   r$   (test_split_between_processes_nested_dict  sF   
$

8
$


$$  r  c               
   C   s  t  } | jdkrUtg dg dg| j}| |-}| jdkr/tg dg| j}ntg dg| j}tj	|| W d    n1 sLw   Y  | 
  tdd td| j D | j}tdd td| j d D | j}||fD ]p}d	d | D }| j|d
d8}tt|| j }t||ksJ d| j d| j dt| g }|D ]
}	||	d qW d    n1 sw   Y  | 
  t|}
|
d t| }||ksJ d| d| qd S )Nr8   )r   r8   r   r   )r   r  r  r  r   c                 S      g | ]}|gqS r#   r#   r   ir#   r#   r$   r         z7test_split_between_processes_tensor.<locals>.<listcomp>r   c                 S   r  r#   r#   r  r#   r#   r$   r     r  c                 S   s   g | ]}t |qS r#   )r_   r  r  r#   r#   r$   r     r   Tr   r  r  r   rv   r   r   )r	   r9   r_   r  r   rr   r  r)   r   r   r@   rV   ry   r   r   r    rs   r   )r+   r   r  expectedr  r  r  r  r  r\   r	  r   r#   r#   r$   #test_split_between_processes_tensor  s:   

&*	r!  c                  C   s   t  } | jdv r_ttd}t|| j }t|| j }| |6}| j|k r>t||d ks=J d|d  dt| nt||ksPJ d| dt| W d    n1 sZw   Y  |   d S )N)r8   r   r   rj      r8   z2Each Process should have even elements. Expected: r  )r	   r9   r{   rV   r    r  r)   r@   )r+   r   num_samples_per_process
num_extrasr  r#   r#   r$   #test_split_between_processes_evenly  s    

	r%  c                  C   sH   t  } |  du sJ | jr|   |  du sJ |  du s"J d S )NFT)r   check_triggerr;   set_trigger)rX   r#   r#   r$   test_trigger  s   r(  c                  C   s   dd l } t  tjdd}t }t  | t}|	| W d    n1 s+w   Y  dt
|jjd v s<J dt
|jjd v sHJ d S )Nr   r8   z*`AcceleratorState` object has no attributez1This happens if `AcceleratorState._reset_state()`)pytestr	   r   r_   r   r   r   raisesAttributeErrorr   strvalueargs)r)  simple_modelrX   cmr#   r#   r$   test_reinstantiated_state+  s   r1  c                  C   s  t  } | j}|jdkrtd t  |  |jtjkr"t	j
 }n|j}||jkr|jdkr3td t  |jdkr?td t  |jdkrKtd t  |jdkrWtd t  |jdkrctd t  |jdkrotd t r}dd	lm} t| ntd
 |jdkrtd t  |jdkrtd t  |jtjkrt  t  t  t  |jdkrt  |jtjkrd S |jdkrtd t dd t dd |jdkrtd t!  t" r|jdkrtd t#  |$  d S )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z(
**Test split between processes evenly**z7
**Test split between processes as a datasets.Dataset**)r   z6Skipped because Hugging Face datasets is not availablez1
**Test random number generator synchronization**z 
**DataLoader integration test**r8   z
**Training integration test**Fr   Tz
**Breakpoint trigger test**z
**Test reinstantiated state**)%r   r+   r.   r(   r^   r@   rc   r   rd   r_   r`   device_countr9   r)   r]   r  r  r!  r%  r   datasetsr   r
  rh   r   XLAr   r   r   r   r   	DEEPSPEEDr   r(  r   r1  destroy_process_group)rX   r+   num_processes_per_noder  r#   r#   r$   main;  st   
















r8  __main__)F)BrK   rI   r   r<   copyr   pathlibr   numpyr   r_   torch.utils.datar   r   
accelerater   accelerate.data_loaderr   r   accelerate.stater	   accelerate.test_utilsr
   r   r   accelerate.utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r%   r,   r/   r1   r5   r]   r^   rh   r   r   r   r   r   r   r   r   r
  r  r  r!  r%  r(  r1  r8  r   r#   r#   r#   r$   <module>   s^   <
I<A.

 /0!&"Q
