o
    3wi^                     @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZ d d	lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% e# rpd d
lm&Z' nd dlm'Z' e  r~dZ(dZ)ndZ(dZ)d@ddZ*dd Z+dd Z,dd Z-dd Z.dd Z/dd Z0dd Z1dd  Z2d!d" Z3d#d$ Z4d%d& Z5d'd( Z6d)d* Z7d@d+d,Z8d@d-d.Z9d/d0 Z:d1d2 Z;d3d4 Z<d5d6 Z=d7d8 Z>d9d: Z?d;d< Z@d=d> ZAeBd?kreA  dS dS )A    N)deepcopy)Path)
DataLoaderDataset)Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetare_the_same_tensors)DataLoaderConfigurationDistributedTypegathergather_objectis_bf16_availableis_datasets_availableis_fp16_availableis_hpu_availableis_ipex_availableis_pytest_availableis_xpu_availableset_seedsynchronize_rng_states)RegressionModel4XPU)RegressionModelgMbP?gư>Fc                 C   s4   |rt || t| d}t| ||dS t| |d|dS )zBCreates a dataloader that can also use the `SeedableRandomSampler`)	generatordata_sourcenum_samples)
batch_sizesamplerT)r   shuffler   )r   lenr   )	train_setr   r   use_seedable_samplerr    r$   f/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloader?   s   r&   c                 C      t d| j  d S )NzPrinting from the main process printprocess_indexstater$   r$   r%   
print_mainN      r-   c                 C   r'   )Nz%Printing from the local main process )r)   local_process_indexr+   r$   r$   r%   print_local_mainR   r.   r0   c                 C   r'   )NPrinting from the last process r(   r+   r$   r$   r%   
print_lastV   r.   r2   c                 C   s   t d| d| j  d S )NPrinting from process : r(   )r,   process_idxr$   r$   r%   print_onZ   s   r6   c               	   C   s  t  } | j}td}|  E | jr2td t|d}|d W d    n1 s,w   Y  nt|d}|d W d    n1 sGw   Y  W d    n1 sVw   Y  | 	  | jrt|}d
| }W d    n1 sxw   Y  z2|dsJ d|dkr|dsJ d|d| jd ksJ d	|d
d  d| j W n ty   |   w | jr| r|  | 	  t }t| | t| j W d    n1 sw   Y  |  }| jr|dksJ | dn|  dksJ | d|d |d t| | t| j W d    n	1 s6w   Y  | jrK|  dksJJ n|  dksVJ |d |d t| |  t!| j W d    n	1 syw   Y  | j"r|  d| jjd  ksJ n|  dksJ |d |d t#|D ]R}t| | j$t%|d| j| W d    n	1 sw   Y  | j&|kr|  d| d| j& ksJ n|  dksJ |d |d qd S )Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
 zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r1   )r*   r3   r4   )'r   num_processesr   main_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithendswithcountAssertionErrorunlinkexistsioStringIO
contextlibredirect_stdouton_main_processr-   r,   getvaluerstriptruncateseekon_local_main_processr0   is_local_main_processon_last_processr2   is_last_processrange
on_processr6   r*   )acceleratorr:   pathftextresultr5   r$   r$   r%   process_execution_check^   s   


 



&

&
r^   c                  C   s$   t  } | jdkrtd t|  d S )Nr   zTesting, testing. 1, 2, 3.)r	   r/   r)   r+   r$   r$   r%   init_state_check   s   
r_   c                  C   s   t  } tdg tt sJ d| jtjkr)tdg ttj s(J dn| jtj	kr?tdg ttj
 s?J dt }tdg|d t| sTJ d	| jd
kr_td d S d S )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.r   )r   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r	   r   r   r`   get_rng_statedistributed_typer   	MULTI_GPUra   	MULTI_XPUrb   	Generator	get_stater/   r)   )r,   r   r$   r$   r%   rng_sync_check   s   



ri   c                  C   s  t  } d| j }tt|dd}t|| j| j| jdd}g }|D ]	}|t| q t	
|}t	| t	d| sAJ dtt|dd}t|| j| j| jddd}g }|D ]	}|t| qZt	
|}t	| t	d| s{J d| jdkrtd	 tt|ddd
}t|| j| j| jdd}g }|D ]	}|t| qt	
| }|  |tt|ksJ dtt|ddd
}t|| j| j| jddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ d| jdkrtd d S d S )N       r   T)put_on_devicer   %Wrong non-shuffled dataloader result.)rm   split_batchesz Non-shuffled dataloader passing.r   r    !Wrong shuffled dataloader result.zShuffled dataloader passing.r	   r:   r   rW   r   devicer*   appendr   r`   catequalcpuarangelongr)   tolistsortlistr/   r,   lengthdlr]   batchr$   r$   r%   dl_preparation_check   sf   

$
$
r   c               	   C   s  t  } d| j }tt|dd}t|| j| j| jddd}g }|D ]	}|t| q!t	
|}t	| t	d| sBJ dtt|dd}t|| j| j| jdddd}g }|D ]	}|t| q\t	
|}t	| t	d| s}J d| jdkrtd	 tt|ddd
}t|| j| j| jddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ dtt|ddd
}t|| j| j| jdddd}g }|D ]	}|t| qt	
| }|  |tt|ksJ d| jdkr	td d S d S )Nrj   rk   rl   T)rm   dispatch_batchesr   rn   )rm   ro   r   z(Non-shuffled central dataloader passing.rp   rq   z$Shuffled central dataloader passing.rr   r}   r$   r$   r%   central_dl_preparation_check   sr   

$	
$
	r   c                  C   s   t  } G dd dt}G dd d}|td| j }|t|dd}t||d}t|| j| j| j}t	|j
d	rFt|j
j
|sDJ d
d S t|j
|sPJ d
d S )Nc                   @   s$   e Zd Zdd Zdd Zdd ZdS )z+custom_sampler_check.<locals>.CustomDatasetc                 S   s
   || _ d S Ndata)selfr   r$   r$   r%   __init__B     
z4custom_sampler_check.<locals>.CustomDataset.__init__c                 S   s
   t | jS r   )r!   r   r   r$   r$   r%   __len__E  r   z3custom_sampler_check.<locals>.CustomDataset.__len__c                 S   s
   | j | S r   r   )r   indexr$   r$   r%   __getitem__H  r   z7custom_sampler_check.<locals>.CustomDataset.__getitem__N)__name__
__module____qualname__r   r   r   r$   r$   r$   r%   CustomDatasetA  s    r   c                   @   s4   e Zd ZddededefddZdd Zd	d
 ZdS )z0custom_sampler_check.<locals>.CustomBatchSamplerTdataset_lengthr   r    c                 S   s   || _ t|| _|| _d S r   )r   nprx   
data_indexr    )r   r   r   r    r$   r$   r%   r   L  s   
z9custom_sampler_check.<locals>.CustomBatchSampler.__init__c                 s   s@    t | }| jrtj| j}n| j}t||}|E d H  d S r   )r!   r    r   randompermutationr   array_split)r   num_batchesr   outputr$   r$   r%   __iter__Q  s   z9custom_sampler_check.<locals>.CustomBatchSampler.__iter__c                 S   s   t t| j| j S r   )mathceilr!   r   r   r   r$   r$   r%   r   Z  s   z8custom_sampler_check.<locals>.CustomBatchSampler.__len__N)T)r   r   r   intboolr   r   r   r$   r$   r$   r%   CustomBatchSamplerK  s    	r   rj   rk   rl   )batch_samplerr   z>Custom sampler was changed after calling `prepare_data_loader`)r	   r   rW   r:   r!   r   r   rs   r*   hasattrr   
isinstance)r,   r   r   datasetr   r   r$   r$   r%   custom_sampler_check>  s   
r   c                  C   s   t d tddd} t| ddd}tdd}t|d}||}g }td	D ]}|D ]	}||d
  q*q&t	|}t d |
d g }td	D ]}|D ]	}||d
  qMqIt	|}t||sgJ dd S )N*   
   r~   seed   Trp   r#   dataloader_config   xr   z;Did not obtain the same items with the same seed and epoch.)r   r
   r   r   r   preparerW   rt   r`   ru   	set_epochallclose)r"   train_dlconfigrY   original_items_r   	new_itemsr$   r$   r%   check_seedable_samplerl  s*   





r   c                  C   sp   t d tdd} t| d}|jdksJ dtttdddd}t|dd	}|jjj	}t
|ts6J d
d S )Nr   Tr   r   r9   z)This test requires more than one process.r   rp   )
dataloaderr#   z:Sampler in BatchSamplerShard is not SeedableRandomSampler.)r   r   r   r:   r   r|   rW   r   r   r   r   r   )r   rY   r   prepared_data_loadertarget_samplerr$   r$   r%   -check_seedable_sampler_in_batch_sampler_shard  s   


r   c            
      C   s   t d d} tddd}t|ddd}td| d}t|d}||}g }td	D ]}|D ]	}||d
  q-q)t	|}d|_
t|d}||}g }	td	D ]}|D ]	}|	|d
  qTqPt	|	}	t||	rnJ dd S )Nr   r   r   r   Trp   )r#   	data_seedr   r   r   +   z1Obtained the same items with different data seed.)r   r
   r   r   r   r   rW   rt   r`   ru   r   r   )
r   r"   r   r   rY   prepared_dlr   r   r   r   r$   r$   r%   %check_seedable_sampler_with_data_seed  s.   





r   c                 C   s   t d |d t| dd}t||||}t }tjj| dd}t	dD ]#}|D ]}	|
  ||	d }
tjj|
|	d }|  |  q+q'||fS )Nr   r   r7   lrr   r   y)r   manual_seedr
   r&   r   r`   optimSGD
parametersrW   	zero_gradnn
functionalmse_lossbackwardstep)r~   r   r   r#   r"   r   model	optimizerepochr   r   lossr$   r$   r%   mock_training  s   

r   c                 C   s<  t  }t }d}|d |j }t|||j || \}}t|js%J dt|js.J dt }t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]$}|D ]}|	  |	|d }tjj||d	 }|| |
  q`q\||	 }	tjj|j|	jttd
d d tjj|j|	jttdd d |d td| d}t|d}t	||||j | }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]$}|D ]}|	  |	|d }tjj||d	 }|| |
  qq||	 }	tjj|j|	jttdd d tjj|j|	jttdd d |d tj ! rktd t "  tdd}tj#dd}	||	}	|j|	dd}t$ddgj%tj&|j'd}||}t( rtd t "  t| d}td|d}t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]&}|D ] }|	  |	|d }tjj||d	 }|| |
  qq||	 }	tjj|j|	jttd d d tjj|j|	jttd!d d t) rt* std" t "  t| d}td|d}t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]&}|D ] }|	  |	|d }tjj||d	 }|| |
  qCq?||	 }	tjj|j|	jttd#d d tjj|j|	jttd$d d t+ rtd% t "  t| d}tdd|d&}t	|||| }t
 }	tjj|	 dd}
|||	|
\}}	}
td |d tdD ]&}|D ] }|	  |	|d }tjj||d	 }|| |
  qҐq||	 }	tjj|j|	jttd'd d tjj|j|	jttd(d d d S d S ))Nrk      z0Did not obtain the same model on both processes.r7   r   r   r   r   r   c                 S   
   d|  S Nz>Did not obtain the same model on CPU or distributed training.
r$   msgr$   r$   r%   <lambda>     
 z training_check.<locals>.<lambda>)atolrtolr   c                 S   r   r   r$   r   r$   r$   r%   r     r   zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)ro   r#   r   c                 S   r   r   r$   r   r$   r$   r%   r     r   c                 S   r   r   r$   r   r$   r$   r%   r     r   zSTraining yielded the same results on one CPU or distributed setup with batch split.zKeep fp32 wrapper check.fp16)mixed_precisionr   )keep_fp32_wrapperr9   )dtypers   zBF16 training check.r   bf16)r   r   c                 S   r   r   r$   r   r$   r$   r%   r   D  r   c                 S   r   r   r$   r   r$   r$   r%   r   K  r   zFP16 training check.c                 S   r   r   r$   r   r$   r$   r%   r   j  r   c                 S   r   r   r$   r   r$   r$   r%   r   q  r   zipex BF16 training check.)r   rw   r   c                 S   r   r   r$   r   r$   r$   r%   r     r   c                 S   r   r   r$   r   r$   r$   r%   r     r   ),r	   r`   rg   r:   r   r   abr   r&   r   r   r   r   r   r   r   rW   r   r   r   r   r   r   unwrap_modelrw   testingassert_closeATOLRTOLr)   r   ra   is_available_reset_stateLinearTensortofloat16rs   r   r   r   r   )r#   r,   r   r   r~   r"   	old_modelrY   r   r   r   r   r   r   r   r   model_with_fp32_wrapperinput_tensorr$   r$   r%   training_check  sp  













	


	



r   c              	   C   sj  t  }| dd td|j D }|j|dd}t|dks-J d|j dt| W d    n1 s7w   Y  | dd td|j d	 D }|j|dd2}|jrlt|d	kskJ d
|j dt| nt|dksJ d|j dt| W d    n1 sw   Y  |  | dd td|j d	 D }| dd td|j D }||fD ]|}|d }|j|ddB}|jd	krt|t|ksJ d|j dt| nt|dksJ d|j dt| g }|D ]}|	| qW d    n	1 s	w   Y  |  t
|}	dd |	d t| D }
||
ks2J d| d|
 qd S )Nc                 S      g | ]}t |d qS kdict.0vr$   r$   r%   
<listcomp>      z8test_split_between_processes_dataset.<locals>.<listcomp>r   Fapply_padding4Each process did not have two items. Process index: 
; Length: c                 S   r   r   r   r   r$   r$   r%   r     r   r9   z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: c                 S   r   r   r   r   r$   r$   r%   r     r   c                 S   r   r   r   r   r$   r$   r%   r     r   r   Tz9Single process did not receive all items. Process index: c                 S   s   g | ]}|d  qS r   r$   )r   rr$   r$   r%   r     s    )Gathered results is incorrect. Expected: ; Got: )r	   	from_listrW   r:   split_between_processesr!   r*   rV   rA   rt   r   )datasets_Datasetr,   r   resultsodd_data	even_dataexpected_outputresults_per_processr]   gathered_resultsr   r$   r$   r%   $test_split_between_processes_dataset  sR   "	"
 r  c               
   C   sx  t  } ttdd| j }| |}t|dks&J d| j dt| W d    n1 s0w   Y  |   ttdd| j }ttdd| j d }||fD ]f}|}| j|dd5}t	t|| j }t||ksJ d| j
 d	| j dt| g }|D ]}|| qW d    n1 sw   Y  |   t|}	|	d t| }
||
ksJ d
| d|
 qSd S )Nr   r   r   r   r9   Tr   Process ; did not get the correct number of item(s). Process index: r  r  )r	   r|   rW   r:   r  r!   r*   rA   r   r   rs   rt   r   )r,   r   r  r  r  r	  num_samples_per_devicer
  r]   r  r   r$   r$   r%   !test_split_between_processes_list  s6   r  c                  C   s  t  } g d}g d}tg d}| jdv r>|||d}t|}| |}| jdkr@|d |d d d| j  ks?J n5| jdkrT|d |d d	d  ksSJ n!| jd
kru|d |d dd  ksuJ d|d d  d|d  | jdkr|d |d d d| j  ksJ n'| jdkr|d |d d	d  ksJ n| jd
kr|d |d dd  ksJ | jdkrt|d |d d d| j  sJ d|d d d| j   d|d  nO| jdkrt|d |d d	d  sJ d|d d	d   d|d  n'| jd
kr.t|d |d dd  s.J d|d dd   d|d  W d    n	1 s9w   Y  |   d S )N)r9   r   r   r            rk   )r   r   cder[   gh)r9   r   r   )r   r   r  r   r   rk   r   r   r   z
Expected: 
, Actual: r   r  z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)	r	   r`   tensorr:   r   r  r*   r   rA   )r,   r   r   r  r   	data_copyr  r$   r$   r%   (test_split_between_processes_nested_dict  sF   
$

8
$


$$  r  c               
   C   s  t  } | jdkrUtg dg dg| j}| |-}| jdkr/tg dg| j}ntg dg| j}tj	|| W d    n1 sLw   Y  | 
  tdd td| j D | j}tdd td| j d D | j}||fD ]p}d	d | D }| j|d
d8}tt|| j }t||ksJ d| j d| j dt| g }|D ]
}	||	d qW d    n1 sw   Y  | 
  t|}
|
d t| }||ksJ d| d| qd S )Nr9   )r   r9   r   r   )r   r  r  r  r   c                 S      g | ]}|gqS r$   r$   r   ir$   r$   r%   r         z7test_split_between_processes_tensor.<locals>.<listcomp>r   c                 S   r  r$   r$   r  r$   r$   r%   r     r!  c                 S   s   g | ]}t |qS r$   )r`   r  r  r$   r$   r%   r      r   Tr   r  r  r   rw   r  r  )r	   r:   r`   r  r   rs   r  r*   r   r   rA   rW   rz   r   r   r!   rt   r   )r,   r   r  expectedr  r  r	  r  r
  r]   r  r   r$   r$   r%   #test_split_between_processes_tensor  s:   

&*	r#  c                  C   s   t  } | jdv r_ttd}t|| j }t|| j }| |6}| j|k r>t||d ks=J d|d  dt| nt||ksPJ d| dt| W d    n1 sZw   Y  |   d S )N)r9   r   r   rk      r9   z2Each Process should have even elements. Expected: r  )r	   r:   r|   rW   r!   r  r*   rA   )r,   r   num_samples_per_process
num_extrasr  r$   r$   r%   #test_split_between_processes_evenly3  s    

	r'  c                  C   sH   t  } |  du sJ | jr|   |  du sJ |  du s"J d S )NFT)r   check_triggerr<   set_trigger)rY   r$   r$   r%   test_triggerE  s   r*  c                  C   s   dd l } t  tjdd}t }t  | t}|	| W d    n1 s+w   Y  dt
|jjd v s<J dt
|jjd v sHJ d S )Nr   r9   z*`AcceleratorState` object has no attributez1This happens if `AcceleratorState._reset_state()`)pytestr	   r   r`   r   r   r   raisesAttributeErrorr   strvalueargs)r+  simple_modelrY   cmr$   r$   r%   test_reinstantiated_stateV  s   r3  c                  C   s  t  } | j}|jdkrtd t  |  |jtjkr"t	j
 }n|j}||jkr|jdkr3td t  |jdkr?td t  |jdkrKtd t  |jdkrWtd t  |jdkrctd t  |jdkrotd t r}dd	lm} t| ntd
 |jdkrtd t  |jdkrtd t  |jtjkrt  t  t  t  |jdkrt  |jtjkrd S |jdkrtd t dd t dd |jdkrtd t!  t" r|jdkrtd t#  |$  d S )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z(
**Test split between processes evenly**z7
**Test split between processes as a datasets.Dataset**)r   z6Skipped because Hugging Face datasets is not availablez1
**Test random number generator synchronization**z 
**DataLoader integration test**r9   z
**Training integration test**Fr   Tz
**Breakpoint trigger test**z
**Test reinstantiated state**)%r   r,   r/   r)   r_   rA   rd   r   re   r`   ra   device_countr:   r*   r^   r  r  r#  r'  r   datasetsr   r  ri   r   XLAr   r   r   r   r   	DEEPSPEEDr   r*  r   r3  destroy_process_group)rY   r,   num_processes_per_noder  r$   r$   r%   mainf  st   
















r:  __main__)F)CrL   rJ   r   r=   copyr   pathlibr   numpyr   r`   torch.utils.datar   r   
accelerater   accelerate.data_loaderr   r   accelerate.stater	   accelerate.test_utilsr
   r   accelerate.utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   r-   r0   r2   r6   r^   r_   ri   r   r   r   r   r   r   r   r   r  r  r  r#  r'  r*  r3  r:  r   r$   r$   r$   r%   <module>   sd   <
I<A.

 T0!&"Q
