o
    zir;                     @   s   d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZ ddlZddlmZ ddlmZ dd	lmZ dd
lmZ G dd deZdS )zr

Learning Rate Monitor
=====================

Monitor and logs learning rate for lr schedulers during training.

    N)defaultdict)	AnyDefaultDictDictListLiteralOptionalSetTupleType)	Optimizer)override)Callback)MisconfigurationException)rank_zero_warn)LRSchedulerConfigc                   @   s  e Zd ZdZ			d?deed  dededdfd	d
Zeddde	de	ddfddZ
eddde	de	ddfddZeddde	de	ddfddZdddedeeef fddZdedee deeef fddZdeee	f dedeee	f fddZd@deee  d!eddfd"d#Zdeeef ded$edeeef fd%d&Zdeee	f dedeee	f fd'd(Zded)ee d*eee ef defd+d,ZdAded.ee d/ed0edef
d1d2Zd.ee dee fd3d4Zd5ee de eee  ee eee ef f fd6d7Z!d8ee	 d9ee d*eee ef de eee  ee f fd:d;Z"deded9ee d*eee ef d<ee dee fd=d>Z#dS )BLearningRateMonitora  Automatically monitor and logs learning rate for learning rate schedulers during training.

    Args:
        logging_interval: set to ``'epoch'`` or ``'step'`` to log ``lr`` of all optimizers
            at the same interval, set to ``None`` to log at individual interval
            according to the ``interval`` key of each scheduler. Defaults to ``None``.
        log_momentum: option to also log the momentum values of the optimizer, if the optimizer
            has the ``momentum`` or ``betas`` attribute. Defaults to ``False``.
        log_weight_decay: option to also log the weight decay values of the optimizer. Defaults to
            ``False``.

    Raises:
        MisconfigurationException:
            If ``logging_interval`` is none of ``"step"``, ``"epoch"``, or ``None``.

    Example::

        >>> from pytorch_lightning import Trainer
        >>> from pytorch_lightning.callbacks import LearningRateMonitor
        >>> lr_monitor = LearningRateMonitor(logging_interval='step')
        >>> trainer = Trainer(callbacks=[lr_monitor])

    Logging names are automatically determined based on optimizer class name.
    In case of multiple optimizers of same type, they will be named ``Adam``,
    ``Adam-1`` etc. If an optimizer has multiple parameter groups they will
    be named ``Adam/pg1``, ``Adam/pg2`` etc. To control naming, pass in a
    ``name`` keyword in the construction of the learning rate schedulers.
    A ``name`` keyword can also be used for parameter groups in the
    construction of the optimizer.

    Example::

        def configure_optimizer(self):
            optimizer = torch.optim.Adam(...)
            lr_scheduler = {
                'scheduler': torch.optim.lr_scheduler.LambdaLR(optimizer, ...)
                'name': 'my_logging_name'
            }
            return [optimizer], [lr_scheduler]

    Example::

        def configure_optimizer(self):
            optimizer = torch.optim.SGD(
                [{
                    'params': [p for p in self.parameters()],
                    'name': 'my_parameter_group_name'
                }],
                lr=0.1
            )
            lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, ...)
            return [optimizer], [lr_scheduler]

    NFlogging_interval)stepepochlog_momentumlog_weight_decayreturnc                 C   s8   |dvrt d|| _|| _|| _i | _i | _i | _d S )N)Nr   r   z7logging_interval should be `step` or `epoch` or `None`.)r   r   r   r   lrslast_momentum_valueslast_weight_decay_values)selfr   r   r    r   Z/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/callbacks/lr_monitor.py__init__^   s   
zLearningRateMonitor.__init__trainerz
pl.Trainerargskwargsc                    s    j std| jr#dtdtf fdd}|dr#|dr#tdtd	 g }|  j\}}}|	| | j
 j||d
\}	}
|	|	 ttj|}dd |D | _dd |D | _dd |D | _dS )a  Called before training, determines unique names for all lr schedulers in the case of multiple of the same
        type or in the case of multiple parameter groups.

        Raises:
            MisconfigurationException:
                If ``Trainer`` has no ``logger``.

        zLCannot use `LearningRateMonitor` callback with `Trainer` that has no logger.keyr   c                    s6   j rt fddj D S t fddjD S )Nc                 3   s    | ]
} |j jjvV  qd S N)	scheduler	optimizerdefaults).0configr#   r   r   	<genexpr>   s    
zLLearningRateMonitor.on_train_start.<locals>._check_no_key.<locals>.<genexpr>c                 3   s    | ]} |j vV  qd S r$   )r'   )r(   r&   r*   r   r   r+      s    )lr_scheduler_configsany
optimizersr*   r    r*   r   _check_no_key   s
   z9LearningRateMonitor.on_train_start.<locals>._check_no_keymomentumbetaszsYou have set log_momentum=True, but some optimizers do not have momentum. This will log a value 0 for the momentum.)categoryseen_optimizersseen_optimizer_typesc                 S   s   i | ]}|g qS r   r   r(   namer   r   r   
<dictcomp>   s    z6LearningRateMonitor.on_train_start.<locals>.<dictcomp>c                 S      i | ]}|d  dqS )	-momentumNr   r7   r   r   r   r9          c                 S   r:   )-weight_decayNr   r7   r   r   r   r9      r<   N)loggersr   r   strboolr   RuntimeWarning_find_names_from_schedulersr,   extend_find_names_from_optimizersr.   list	itertoolschainfrom_iterabler   r   r   )r   r    r!   r"   r0   namessched_hparam_keysoptimizers_with_scheduleroptimizers_with_scheduler_typesoptimizer_hparam_keys_names_flattenr   r/   r   on_train_starto   s8   




z"LearningRateMonitor.on_train_startc                 O   sd   |j jsd S | jdkr,| jd u rdnd}| ||}|r.|jD ]}|j||jjjd qd S d S d S )Nr   r   r-   r   )	_logger_connectorshould_update_logsr   _extract_statsr>   log_metricsfit_loop
epoch_loop_batches_that_steppedr   r    r!   r"   intervallatest_statloggerr   r   r   on_train_batch_start   s   

z(LearningRateMonitor.on_train_batch_startc                 O   sX   | j dkr&| j d u rdnd}| ||}|r(|jD ]}|j||jjjd qd S d S d S )Nr   r   r-   rQ   )r   rT   r>   rU   rV   rW   rX   rY   r   r   r   on_train_epoch_start   s   

z(LearningRateMonitor.on_train_epoch_startrZ   c                    s   i }|   j\}}}| | t| jD ]\}}||jdfv r0|jj}	| |	|}
||
 q| j	 j
||d\}}| | t||D ]\}	}| |	|}
||
 qF j fdd| D  |S )Nr-   r4   c                    s$   i | ]\}}|t j| jjd qS ))device)torchtensorstrategyroot_device)r(   r8   valuer/   r   r   r9      s    z6LearningRateMonitor._extract_stats.<locals>.<dictcomp>)rB   r,   _remap_keysziprZ   r%   r&   _get_optimizer_statsupdaterD   r.   callback_metricsitems)r   r    rZ   r[   scheduler_hparam_keysrK   rL   r8   r)   optcurrent_statrM   optimizers_without_schedulerrI   r   r/   r   rT      s4   




z"LearningRateMonitor._extract_statsr&   rI   c                 C   s   i }|j }d|jv }t||D ]1\}}| ||}|| | j|||| d|d}	||	 | || d}
||
 q|S )Nr2   r;   )param_groupr8   	use_betasr=   )param_groupsr'   rf   _extract_lrrh   _extract_momentumreplace_extract_weight_decay)r   r&   rI   statsrq   rp   pgr8   lrr1   weight_decayr   r   r   rg      s   


z(LearningRateMonitor._get_optimizer_statsro   r8   c                 C   s    |d }| j | | ||iS )Nrx   )r   append)r   ro   r8   rx   r   r   r   rr      s   zLearningRateMonitor._extract_lr/pg1tokenc                 C   s^   |D ]*}|D ]%}| |d}||v r!|| jv r!| j|| j|< q|| jvr+g | j|< qqdS )zYThis function is used the remap the keys if param groups for a given optimizer increased. N)rt   r   pop)r   rI   r|   group_new_namesnew_nameold_namer   r   r   re      s   

zLearningRateMonitor._remap_keysrp   c                 C   s8   | j si S |r|d d n|dd}|| j|< ||iS )Nr2   r   r1   )r   getr   )r   ro   r8   rp   r1   r   r   r   rs      s
   
z%LearningRateMonitor._extract_momentumc                 C   s$   | j si S |d }|| j|< ||iS )z<Extracts the weight decay statistics from a parameter group.ry   )r   r   )r   ro   r8   ry   r   r   r   ru     s
   
z)LearningRateMonitor._extract_weight_decayoptimizer_clsr6   c                 C   s2   ||vr|S || }|dkr|d|d   S |S )N   -r   )r   r8   r   r6   countr   r   r   _add_prefix  s   zLearningRateMonitor._add_prefixTrq   param_group_index	use_namesc                 C   sv   t |dkr%|s| d|d  S || dd|d  }| d| S |r9|| d}|r7| d| S |S |S )Nr   z/pgr8   rw   /)lenr   )r   r8   rq   r   r   pg_namer   r   r   _add_suffix  s   zLearningRateMonitor._add_suffixc                    sF   dd t |ddD  t }t t|krt S  fdd D S )Nc                 S   s"   g | ]\}}| d d| qS )r8   rw   )r   )r(   irw   r   r   r   
<listcomp>#  s   " zDLearningRateMonitor._duplicate_param_group_names.<locals>.<listcomp>r   )startc                    s   h | ]}  |d kr|qS )r   )r   )r(   nrI   r   r   	<setcomp>'  s    zCLearningRateMonitor._duplicate_param_group_names.<locals>.<setcomp>)	enumeratesetr   )r   rq   uniquer   r   r   _duplicate_param_group_names"  s
   z0LearningRateMonitor._duplicate_param_group_namesr,   c           	      C   sf   g }g }t t}|D ]#}|j}|jd ur|jnd|jjj }| |j||||}|| q
|||fS )Nlr-)	r   intr%   r8   r&   	__class____name__!_check_duplicates_and_update_namerz   )	r   r,   rI   r5   r6   r)   schr8   updated_namesr   r   r   rB   )  s   
z/LearningRateMonitor._find_names_from_schedulersr.   r5   c           	      C   sj   g }g }|D ]*}t |dr|jn|}||v rqd|jj }| ||||d }|| || q||fS )Nr&   r   )hasattrr&   r   r   r   rz   )	r   r.   r5   r6   rI   rn   r&   r8   r   r   r   r   rD   =  s   

z/LearningRateMonitor._find_names_from_optimizerslr_scheduler_configc                    s   | | t|}|d u s|jd u r||  d7  < |j}|r.td  d|  ||  fddttD S )Nr   zYA single `Optimizer` cannot have multiple parameter groups with identical `name` values. z& has duplicated parameter group names c                    s   g | ]	}  |qS r   )r   )r(   r   r8   rq   r   r   r   r   l  s    zILearningRateMonitor._check_duplicates_and_update_name.<locals>.<listcomp>)	rz   typer8   rq   r   r   r   ranger   )r   r&   r8   r5   r6   r   r   
duplicatesr   r   r   r   U  s    

z5LearningRateMonitor._check_duplicates_and_update_name)NFF)r{   )T)$r   
__module____qualname____doc__r   r   r@   r   r   r   rP   r]   r^   r?   r   floatrT   r   r   rg   rr   re   rs   ru   r   r   r   r   r   r	   r   r   r
   rB   rD   r   r   r   r   r   r   &   s    9

6	"!& *
&	
$$

r   )r   rF   collectionsr   typingr   r   r   r   r   r   r	   r
   r   r`   torch.optim.optimizerr   typing_extensionsr   pytorch_lightningpl$pytorch_lightning.callbacks.callbackr   &pytorch_lightning.utilities.exceptionsr   %pytorch_lightning.utilities.rank_zeror   !pytorch_lightning.utilities.typesr   r   r   r   r   r   <module>   s   	,