o
    8wih                     @   s  d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlZd dlmZ d d	lmZ d d
lmZmZ d dlmZ d dl m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z' d dl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z= d dl>m?Z? eG dd dZ@G dd deZAdS )    N)ChainMapOrderedDictdefaultdict)IterableIterator)	dataclass)AnyOptionalUnion)apply_to_collection)Tensor)_set_sampler_epoch)_RICH_AVAILABLE)_DataFetcher_DataLoaderIterDataFetcher)_Loop)_BatchProgress)_no_grad_context_select_data_fetcher"_verify_dataloader_idx_requirement)call)_check_dataloader_iterable_DataLoaderSource_parse_num_batches_process_dataloader_request_dataloader_resolve_overfit_batches)	_OUT_DICT_ResultCollection)RunningStage	TrainerFn)CombinedLoader)has_len_all_ranks)SIGTERMException)_ModuleModeis_overridden)is_param_in_hook_signaturec                   @   s   e Zd ZdZdZdS )RestartStagenonerestarted_mid_evaluationN)__name__
__module____qualname__NONERESTARTED_MID_EVALUATION r/   r/   d/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.pyr'   3   s    r'   c                       sf  e Zd ZdZ		dWdddedededed	d
f fddZed	e	fddZ
ed	eee	ef  fddZed	efddZed	efddZed	efddZed	ee fddZdXddZed	efddZdXddZdXdd ZdXd!d"ZdXd#d$ZdXd%d&Zd	ee fd'd(ZdXd)d*Zd+ed,ed	d
fd-d.ZdXd/d0Z dXd1d2Z!d+ed,ed	d
fd3d4Z"d+ed,ed	d
fd5d6Z#dXd7d8Z$dXd9d:Z%dXd;d<Z&dXd=d>Z'd?ed@e	dAe	dBe(e) d	d
f
dCdDZ*d?ed@e	dAe(e	 d	e+fdEdFZ,dGe+dHe-d	e.fdIdJZ/dXdKdLZ0e1dMe2d	e3e.e-dNf  fdOdPZ4e1dMe2dQe3e- d	e(e fdRdSZ5e1dTee de-d	d
fdUdVZ6  Z7S )Y_EvaluationLoopz/Top-level loop where validation/testing starts.Ttrainerz
pl.Trainer
trainer_fnstageverboseinference_modereturnNc                    s   t  | || _|| _t | _g | _tdd| _g | _	d| _
|| _|| _td |j d| _d | _d | _tt| _td| _t | _tj| _d S )NF)training_dataloaderz-inf)super__init__r5   r6   r   batch_progress_max_batchesr   _results_logged_outputs_has_run_trainer_fn_stager   dataloader_prefix_data_source_combined_loader_data_fetcherr   int_seen_batches_per_dataloaderfloat_last_val_dl_reload_epochr$   _module_moder'   r-   _restart_stage)selfr2   r3   r4   r5   r6   	__class__r/   r0   r;   <   s"   

z_EvaluationLoop.__init__c                 C   s   | j }|dus	J t|jS )z-Returns the number of prediction dataloaders.N)rE   len	flattened)rM   combined_loaderr/   r/   r0   num_dataloadersW   s   
z_EvaluationLoop.num_dataloadersc                    s$    j } jjs	|S  fdd|D S )z0The max number of batches to run per dataloader.c                    s   g | ]	}t  jj|qS r/   )minr2   num_sanity_val_steps).0batchesrM   r/   r0   
<listcomp>d       z/_EvaluationLoop.max_batches.<locals>.<listcomp>)r=   r2   sanity_checking)rM   max_batchesr/   rX   r0   r\   ^   s   z_EvaluationLoop.max_batchesc                 C   s   t | jdkS )z1Returns whether the evaluation should be skipped.r   )sumr\   rX   r/   r/   r0   skipf   s   z_EvaluationLoop.skipc                 C   s"   | j j}t|o| j j| j |kS )z2Check if validation dataloader should be reloaded.)r2   !reload_dataloaders_every_n_epochsboolcurrent_epochrJ   )rM   n_epochsr/   r/   r0   _should_reload_val_dlk   s   z%_EvaluationLoop._should_reload_val_dlc                 C   s   | j d usJ | j jdkS )N
sequential)rE   _moderX   r/   r/   r0   _is_sequentialq   s   z_EvaluationLoop._is_sequentialc                 C   s   |    | jr	g S |   |   | j}|d usJ d}	 zJz5t|tr2t|}|j}|j	}|j
}n	d }t|\}}}||krC|   |}|j| j_| |||| W n tya   Y W |   nw W |   n|   w q|   |  S )Nr   )
setup_datar^   reseton_run_startrF   
isinstancer   next_batch
_batch_idx_dataloader_idx_store_dataloader_outputsdoner<   is_last_batch_evaluation_stepStopIterationon_iteration_done
on_run_end)rM   data_fetcherprevious_dataloader_idxdataloader_iterbatch	batch_idxdataloader_idxr/   r/   r0   runv   s>   


z_EvaluationLoop.runc                 C   s  | j }| j}| jd ur|tjkr| jsd S |j}|jr|jn|j	}|jr&dnd}|dks1t
||s3d S |tjkrH|jrA|jj sD|jsH|j| _| j}| j}t|}|j|j d t|tsgt|d}	n|}	|tjkrx|jdkrxt|	| g }|	jD ]}
t|
|| t||||
}
||
 q}||	_|	| _|j}|j d ur||j jO }g | _!|	jD ]'}
t"|
|j|rt#|
nt$d}t%|d|j d}t&|||}| j!| qt't(| _)d S )	N	test_stepvalidation_stepr   z_dataloader()rd   inflimit__batches)*r2   rA   rE   r    FITTINGrc   lightning_moduletestinglimit_test_batcheslimit_val_batchesr%   r[   fit_loop
epoch_loop_should_check_val_epochra   rJ   rB   rD   r   strategybarrierrC   rj   r!   overfit_batchesr   rQ   r   r   append2allow_zero_length_dataloader_with_multiple_devices
datamoduler=   r"   rP   rI   getattrr   r   rG   rH   )rM   r2   r3   	pl_modulelimit_batches	hook_namer4   sourcedataloadersrR   dlallow_zero_lengthlengthnum_batchesr/   r/   r0   rg      sV   






z_EvaluationLoop.setup_datac                 C   s   | j tjkS N)rL   r'   r.   rX   r/   r/   r0   r)      s   z(_EvaluationLoop.restarted_mid_evaluationc                 C   s^   | j r)| jjj| jjjkr)| jjj| jjjd kr)| jjj| jjjkr)tj| _	d S tj
| _	d S )N   )
restartingr<   totalstartedready	processed	completedr'   r.   rL   r-   rX   r/   r/   r0   update_restart_stage   s   z$_EvaluationLoop.update_restart_stagec                 C   s   t j| _d S r   )r'   r-   rL   rX   r/   r/   r0   reset_restart_stage   s   z#_EvaluationLoop.reset_restart_stagec                 C   s   | j }d| _g | _| js| j  n| j  |jj}|dus!J |t	j
kr+| j  |jjdus3J t||jj}| j}|dusCJ |t	j
kr[t|jD ]\}}t||jjjj qM| j|_|| t| | j| jjj7  _| j|_| j|_|| _dS )z&Resets the internal state of the loop.FN) r2   r@   r?   r   r<   reset_on_runreset_on_restartstatefnr    r   r4   r   rE   	enumeraterQ   r   r   epoch_progresscurrentr   r\   limitssetupiterfetchedr   _on_before_fetch_start_profiler_on_after_fetch_stop_profilerrF   )rM   r2   r   rv   rR   ir   r/   r/   r0   rh      s0   





z_EvaluationLoop.resetc                 C   sb   |    | jr	d S |   t| j}t|trt|rd S t	|}|dkr(d S | j
|d d S )NT)rg   r^   rh   maxr\   rj   rI   mathisinfrG   r<   increment_by)rM   	max_batchr/   r/   r0   $increment_progress_to_evaluation_end  s   
z4_EvaluationLoop.increment_progress_to_evaluation_endc                 C   s$   |    |   |   |   dS )zrRuns the ``_on_evaluation_model_eval``, ``_on_evaluation_start`` and ``_on_evaluation_epoch_start``
        hooks.N)r   _on_evaluation_model_eval_on_evaluation_start_on_evaluation_epoch_startrX   r/   r/   r0   ri     s   z_EvaluationLoop.on_run_startc                 C   s   | j j  | j j  |   | jg }| _| j j }tt| }|	| |D ]}|	| q*| j j
| |   |   | jrP| j jrP| || jj |S )z+Runs the ``_on_evaluation_epoch_end`` hook.)r2   _logger_connectorepoch_end_reached_evaluation_epoch_end_on_evaluation_epoch_endr?   update_eval_epoch_metricsdictr   updatelog_eval_end_metrics_on_evaluation_end_on_evaluation_model_trainr5   is_global_zero_print_resultsrB   value)rM   logged_outputsepoch_end_logged_outputsall_logged_outputs
dl_outputsr/   r/   r0   ru      s   
z_EvaluationLoop.on_run_endc                 C   s(   | j d ur| j   d | _ | j  d S r   )rF   teardownr>   cpurX   r/   r/   r0   r   ?  s   

z_EvaluationLoop.teardownargskwargsc                 O   sf   | j }|jrdnd}tj||g|R i | tj||g|R i | tj||g|R i | dS )z*Runs ``on_{validation/test}_start`` hooks.on_test_starton_validation_startN)r2   r   r   _call_callback_hooks_call_lightning_module_hook_call_strategy_hookrM   r   r   r2   r   r/   r/   r0   r   E  s
   z$_EvaluationLoop._on_evaluation_startc                 C   s2   | j }|jrdnd}| j|j t|| dS )zSets model to eval mode.on_test_model_evalon_validation_model_evalN)r2   r   rK   capturer   r   r   rM   r2   r   r/   r/   r0   r   N  s   z)_EvaluationLoop._on_evaluation_model_evalc                 C   sB   | j }|jrdnd}t||jrt|| dS | j|j dS )zUndoes the eval mode.on_test_model_trainon_validation_model_trainN)r2   r   r%   r   r   r   rK   restorer   r/   r/   r0   r   U  s
   z*_EvaluationLoop._on_evaluation_model_trainc                 O   sp   | j }|jrdnd}tj||g|R i | tj||g|R i | tj||g|R i | |j  dS )z'Runs ``on_{validation/test}_end`` hook.on_test_endon_validation_endN)r2   r   r   r   r   r   r   reset_resultsr   r/   r/   r0   r   ^  s   z"_EvaluationLoop._on_evaluation_endc                 O   sL   | j }|jrdnd}tj||g|R i | tj||g|R i | dS )z4Runs the ``on_{validation/test}_epoch_start`` hooks.on_test_epoch_starton_validation_epoch_startN)r2   r   r   r   r   r   r/   r/   r0   r   i  s   z*_EvaluationLoop._on_evaluation_epoch_startc                 C   s:   | j }|jrdnd}t|| t|| |j  dS )z-Runs ``on_{validation/test}_epoch_end`` hook.on_test_epoch_endon_validation_epoch_endN)r2   r   r   r   r   r   on_epoch_endr   r/   r/   r0   r   q  s
   z(_EvaluationLoop._on_evaluation_epoch_endc                 C   s&   | j }|j  | j|j  d S r   )r2   r   r   r?   r   r   )rM   r2   r/   r/   r0   ro   {  s   
z)_EvaluationLoop._store_dataloader_outputsc                 C   *   | j jdt| j d| jj d d S N[z]._next)r2   profilerstarttyper*   rB   rC   rX   r/   r/   r0   r     s   *z _EvaluationLoop._on_before_fetchc                 C   r   r   )r2   r   stopr   r*   rB   rC   rX   r/   r/   r0   r     s   *z_EvaluationLoop._on_after_fetchry   rz   r{   rx   c                 C   s  | j }| j}|dusJ t|t }s*|j|}|jj||d}tj	|d||d}| 
||| jr8| jdkr8|nd}| j  |j|| jrN| jdkrN|nd |jrVdnd}	tj||	g| R   tj||	g| R   | j  |jrzdnd}	|s| ||	n|f}
tj	||	g|
R  }| j  |r|j}|j}|j}| 
||| jr| jdkr|nd}|jrd	nd
}	tj||	|g| R   tj||	|g| R   |j  | j  |jsd| _|j| j |  | j |  d7  < | jj!s|j"rt#dS dS )a  Runs the actual evaluation step together with all the necessary bookkeeping and the hooks tied to it.

        Args:
            batch: The current batch to run through the step.
            batch_idx: The index of the current batch.
            dataloader_idx: the index of the dataloader producing the current batch.
            dataloader_iter: The iterator if using this step flavor.

        N)r{   batch_to_devicer   on_test_batch_starton_validation_batch_startr}   r~   on_test_batch_endon_validation_batch_endT)$r2   rF   rj   r   precision_pluginconvert_inputr   _on_before_batch_transferr   r   _build_kwargsrf   rS   r<   increment_readyr   on_batch_startr   r   valuesr   increment_started!_build_step_args_from_hook_kwargsincrement_processedrl   rm   rn   on_batch_endincrement_completedr[   r@   update_eval_step_metricsrH   rq   received_sigtermr#   )rM   ry   rz   r{   rx   r2   rv   using_dataloader_iterhook_kwargsr   	step_argsoutputr/   r/   r0   rr     sX   




z _EvaluationLoop._evaluation_stepc                 C   s(   t d|fd|fg}|dur||d< |S )a  Helper method to build the arguments for the current step.

        Args:
            batch: the current batch to run through the step.
            batch_idx: the index of the current batch.
            dataloader_idx: the index of the dataloader producing the current batch. None if not multiple dataloaders
                in sequential mode.

        Returns:
            the dictionary containing all the keyboard arguments for the step

        ry   rz   Nr{   )r   )rM   ry   rz   r{   step_kwargsr/   r/   r0   r     s   z_EvaluationLoop._build_kwargsr  step_hook_namec                 C   s<   |  }t| jj|}t|ddds|dd t| S )zAHelper method to build args for `test_step` or `validation_step`.rz      )min_argsN)copyr   r2   r   r&   poptupler   )rM   r  r  r   step_hook_fxr/   r/   r0   r     s
   z1_EvaluationLoop._build_step_args_from_hook_kwargsc                 C   s   | j }|jrdnd}|jrdnd}|jrdnd}t|f| jo)| jdko)t| jt | j|j	 t||f| jo:| jdk| j|j	 d S )Nr}   r~   r   r   r   r   r   )
r2   r   r   rf   rS   rj   rF   r   rB   r   )rM   r2   	step_hookbatch_start_hookbatch_end_hookr/   r/   r0   r     s&   z2_EvaluationLoop._verify_dataloader_idx_requirementdata.c                 c   sN    |   D ]\}}t|tr t|ttjD ]	}|g|R V  qq|fV  qd S r   )itemsrj   r   r   r1   	_get_keys)r  kvnew_keyr/   r/   r0   r    s   

z_EvaluationLoop._get_keystargetc                 C   s0   |^}}|| vr
d S | | }|s|S t ||S r   )r1   _find_value)r  r  target_startrestresultr/   r/   r0   r    s   z_EvaluationLoop._find_valueresultsc              	      sR  dd | D } dd t | ttjD }|sd S dd |D }ttt|| \}}dd tt| D }tj	ddj
p<d	}ttttt|td
tt|td
d|d dd |D }| D ]5}t||D ]-\}}	t||}
|
d urt|
tr|
 dkr|
 n|
 }
|	|
  qf|	d qfq_t|  tdt|D ] |  }fdd|D }|d| d  trddlm} ddlm m}  fdd|D }d|d _|| }t||D ]\}}	|	d| |j|	  q| }|| qd dt| }t|d }zt j!j"d ur&d#t j!j" W n t$y3   d}Y nw d}|| }||j%| & |g}t||D ]O\}}	t||krt||kr||d | }||d  }||j%|g|	R  &  t||ks[||%|d&  qI||j%|g|	R  &  qI|| tt'j()| qd S )Nc                 S   s   g | ]}d d |  D qS )c                 S   s    i | ]\}}| d d |qS )z/dataloader_idx_r   )split)rV   r  r  r/   r/   r0   
<dictcomp>  s     z=_EvaluationLoop._print_results.<locals>.<listcomp>.<dictcomp>)r  )rV   r  r/   r/   r0   rY     s    z2_EvaluationLoop._print_results.<locals>.<listcomp>c                 S   s   h | ]	}|D ]}|qqS r/   r/   )rV   keysr  r/   r/   r0   	<setcomp>  rZ   z1_EvaluationLoop._print_results.<locals>.<setcomp>c                 S   s   g | ]}d  |qS ):)join)rV   metricr/   r/   r0   rY         c                 S   s   g | ]}d | qS )zDataLoader r/   )rV   r   r/   r/   r0   rY   "  r%  )x      )fallbackr&  )key   r	  c                 S   s   g | ]}g qS r/   r/   )rV   _r/   r/   r0   rY   (  s    r    r   c                    s   g | ]
}|    qS r/   r/   )rV   row)r   num_colsr/   r0   rY   9      z Metric)get_console)ColumnTablec                    s   g | ]
} |d ddqS )centermagenta)justifystylewidthr/   )rV   h)r1  
max_lengthr/   r0   rY   A  r/  cyanz{:^}u   ─-)*r   r   r1   r  zipsortedrangerP   shutilget_terminal_sizecolumnsrG   rT   r   r  rj   r   numelitemtolistr   insert
capitalizer   richr0  
rich.tabler1  r2  r6  add_rowprintsysstdoutencodingencodeUnicodeEncodeErrorformatrstriposlinesepr#  )r  r4   metrics_pathsmetrics_strsheaders	term_sizerowsr  r$  r-  valtable_headers
table_rowsr0  r2  rB  tableconsole
row_formathalf_term_sizebar_characterbarlines
row_metricr/   )r1  r   r9  r.  r0   r     sv   2


 
z_EvaluationLoop._print_results)TT)r7   N)8r*   r+   r,   __doc__r    r   r`   r;   propertyrG   rS   listr
   rI   r\   r^   rc   rf   r   r   r|   rg   r)   r   r   rh   r   ri   ru   r   r   r   r   r   r   r   r   ro   r   r   r	   r   rr   r   r   strr  r   r   staticmethodr   r   r  r  r   __classcell__r/   r/   rN   r0   r1   9   s    
#8



'


	
	





M
  	$r1   )Br   rS  r@  rL  collectionsr   r   r   collections.abcr   r   dataclassesr   typingr   r	   r
   #lightning_utilities.core.apply_funcr   torchr   pytorch_lightningpllightning_fabric.utilities.datar   2pytorch_lightning.callbacks.progress.rich_progressr    pytorch_lightning.loops.fetchersr   r   pytorch_lightning.loops.loopr    pytorch_lightning.loops.progressr   !pytorch_lightning.loops.utilitiesr   r   r   pytorch_lightning.trainerr   3pytorch_lightning.trainer.connectors.data_connectorr   r   r   r   r   r   <pytorch_lightning.trainer.connectors.logger_connector.resultr   r    pytorch_lightning.trainer.statesr   r    +pytorch_lightning.utilities.combined_loaderr!    pytorch_lightning.utilities.datar"   &pytorch_lightning.utilities.exceptionsr#   )pytorch_lightning.utilities.model_helpersr$   r%   +pytorch_lightning.utilities.signature_utilsr&   r'   r1   r/   r/   r/   r0   <module>   s:    