o
    $it$                     @   s  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dl	m  m
Z d dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ e Zdd	 Zd
d ZG dd dejZ dd Z!d'deddfddZ"d(ddZ#e$dkrd dl%Z%e%& Z'e'j(dddd e'j(dd dd! e') \Z*Z+e*j,rej-dd" e#d#d#d d$d% dS e-e*j. e#ddd d& dS dS ))    N)Dict)FileLock)random_split)tune)
Checkpoint)ASHASchedulerc                 C   s   t t  t ddg}ttjd tj	j
| dd|d}tj	j
| dd|d}W d    ||fS 1 s7w   Y  ||fS )N)gHPs?gec]?g~jt?)gۊe?ggDio?g|?5^?z~/.data.lockT)roottraindownload	transformF)
transformsComposeToTensor	Normalizer   ospath
expandusertorchvisiondatasetsCIFAR10)data_dirr   trainsettestset r   ^/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/tune/examples/cifar10_pytorch.py	load_data   s   

r   c                  C   s8   t jjdddt d} t jjdddt d}| |fS )N   )       r   
   )num_classesr      )r   r   FakeDatar   r   )r   r   r   r   r   load_test_data1   s   r#   c                       s&   e Zd Zd fdd	Zdd Z  ZS )Netx   T   c                    sj   t t|   tddd| _tdd| _tddd| _t	d|| _
t	||| _t	|d| _d S )Nr            r!     r   )superr$   __init__nnConv2dconv1	MaxPool2dpoolconv2Linearfc1fc2fc3)selfl1l2	__class__r   r   r,   >   s   zNet.__init__c                 C   sf   |  t| |}|  t| |}|dd}t| |}t| |}| |}|S )Nr*   )	r1   Frelur/   r2   viewr4   r5   r6   )r7   xr   r   r   forwardG   s   
zNet.forward)r%   r&   )__name__
__module____qualname__r,   rA   __classcell__r   r   r:   r   r$   =   s    	r$   c           "   
   C   sj  t | d | d }d}tj rd}tj dkrt|}|| t }t	j
| | d dd}t ret }| }ttj|d	\}}|| || W d    n1 s`w   Y  | d
 rot \}	}
ntt\}	}
tt|	d }t|	|t|	| g\}}tjjj|t| d d| d
 rdndd}tjjj|t| d d| d
 rdndd}tdD ]}d}d}t|D ]J\}}|\}}||||}}|  ||}|||}|   |!  ||" 7 }|d7 }|d dkrt#d|d |d || f  d}qd}d}d}d}t|dD ]X\}}t$ F |\}}||||}}||}t%|jd\}}||&d7 }|||k' " 7 }|||}||( ) 7 }|d7 }W d    n	1 slw   Y  qt*+ .}tj|d	} t,|- |- f|  t./|}!tj0|| || d|!d W d    n	1 sw   Y  qt#d d S )Nr8   r9   cpucuda:0   lrg?)rI   momentumcheckpoint.pt
smoke_testg?
batch_sizeTr      rM   shufflenum_workersr   g        i  i  z[%d, %5d] loss: %.3f)lossaccuracy)
checkpointzFinished Training)1r$   torchcudais_availabledevice_countr-   DataParalleltoCrossEntropyLossoptimSGD
parametersr   get_checkpointas_directoryloadr   r   joinload_state_dictr#   r   DATA_DIRintlenr   utilsdata
DataLoaderrange	enumerate	zero_gradbackwardstepitemprintno_gradmaxsizesumrF   numpytempfileTemporaryDirectorysave
state_dictr   from_directoryreport)"confignetdevice	criterion	optimizerloaded_checkpointloaded_checkpoint_dirmodel_stateoptimizer_stater   r   test_abstrain_subset
val_subsettrainloader	valloaderepochrunning_lossepoch_stepsirh   inputslabelsoutputsrR   val_loss	val_stepstotalcorrect_	predictedtemp_checkpoint_dirr   rT   r   r   r   train_cifarS   s   













r   Fr|   rT   r   c                 C   sX  t | d | d }tj rdnd}|| | }tj|d}t	|\}}|
| W d    n1 s9w   Y  |rFt \}	}
ntt\}	}
tjjj|
dddd	}d
}d
}t : |D ]/}|\}}||||}}||}t|jd\}	}||d
7 }|||k  7 }qbW d    n1 sw   Y  td||  d S )Nr8   r9   rG   rF   rK      Fr)   rO   r   rH   z Best trial test set accuracy: {})r$   rU   rV   rW   rZ   r`   r   r   rb   ra   rc   r#   r   rd   rg   rh   ri   rq   rr   rs   rt   ro   rp   format)r|   rT   rL   best_trained_modelr~   checkpoint_dircheckpoint_pathr   r   r   r   
testloaderr   r   rh   imagesr   r   r   r   r   r   test_best_model   s6   



r   r   r)   c           	      C   s   t dd t dd t ddt g d|d}t|dd	d
}t jt jt td	|ddt j	dd| |d|d}|
 }|dd}td|j td|jd  td|jd  t|j|j|d d S )Nc                 S      dt jdd S Nr)   	   nprandomrandintr   r   r   r   <lambda>       zmain.<locals>.<lambda>c                 S   r   r   r   r   r   r   r   r      r   g-C6?g?)r)   r   rN   r!   )r8   r9   rI   rM   rL   rH   r)   )max_tgrace_periodreduction_factor)rF   gpu)	resourcesrR   min)metricmodenum_samples	scheduler)tune_configparam_spacezBest trial config: {}z$Best trial final validation loss: {}z(Best trial final validation accuracy: {}rS   )rL   )r   sample_from
loguniformchoicer   Tunerwith_resourceswith_parametersr   
TuneConfigfitget_best_resultrp   r   r|   metricsr   rT   )	r   max_num_epochsgpus_per_trialrL   r|   r   tunerresultsbest_resultr   r   r   main   sD   
r   __main__z--smoke-test
store_truezFinish quickly for testing)actionhelpz--ray-addressz:Address of Ray cluster for seamless distributed execution.)r   required)num_cpusrH   T)r   r   r   rL   )r   r   r   )F)r   r   r)   F)/r   rv   typingr   ru   r   rU   torch.nnr-   torch.nn.functional
functionalr=   torch.optimr\   r   torchvision.transformsr   filelockr   torch.utils.datar   rayr   ray.tuner   ray.tune.schedulersr   mkdtemprd   r   r#   Moduler$   r   r   r   rB   argparseArgumentParserparseradd_argumentparse_known_argsargsr   rL   initray_addressr   r   r   r   <module>   sR   m
#(