o
    ziY                     @   sV  d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
mZmZmZmZmZ ddlm  m  mZ ddlmZ ddlmZmZ ddlmZmZ ddlZdd	lmZ dd
l m!Z! ddl"m#Z# ddl$m%Z%m&Z& e'e(Z)ej*ej+ej,ej-dZ.ej/ej0ej1ej2dZ3eej4e5f Z6ee6 Z7e!ej8ej9ej:fZ;G dd deZ<G dd deZ=dS )z
ModelPruning
^^^^^^^^^^^^
    N)deepcopypartial)AnyCallableDictListOptionalSequenceTupleUnion)apply_to_collection)Tensornn)	TypedDictoverride)Callback)LightningModule)MisconfigurationException)rank_zero_debugrank_zero_only)ln_structuredl1_unstructuredrandom_structuredrandom_unstructuredc                   @   s,   e Zd ZU ejed< eeee	f  ed< dS )	_LayerRefdatanamesN)
__name__
__module____qualname__r   Module__annotations__r   r   intstr r%   r%   W/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/callbacks/pruning.pyr   9   s   
 
r   c                   @   s
  e Zd ZdZ												dPdeeef d	ed
ee	e  de
deeeeegeeef f f dee
eege
f f de
dee
eege
f f de
dee dee dede
ddfddZdQd	edefddZdededeeejf fddZedededefddZdejddfd d!Zed"ejd#ejd$eddfd%d&ZdRd'd(Zdeddfd)d*Zdedeeef fd+d,Zdeddfd-d.Zedejd$edeeef fd/d0Zdeeef ddfd1d2Z e!	dSd3e	eeef  d4e	eeef  deeef ddfd5d6Z"e#d7d8d9e$d:eddfd;d<Z%d=eddfd>d?Z&e#d7d8d9e$ddfd@dAZ'e#d7d8d9e$ddfdBdCZ(e#d7d8d9e$ddfdDdEZ)d9e$deeef fdFdGZ*e#d7d8d9e$dHeeef ddfdIdJZ+e	dTd9e$d	ed
e,e defdKdLZ-edMede
fdNdOZ.dS )UModelPruning)weightbiasr%   NT      ?Fr   
pruning_fnparameters_to_pruneparameter_namesuse_global_unstructuredamountapply_pruningmake_pruning_permanentuse_lottery_ticket_hypothesisresample_parameterspruning_dimpruning_normverboseprune_on_train_epoch_endreturnc                 C   s  || _ || _|| _|	| _|| _|p| j| _i | _d| _d| _	| jD ]}|| jvr3t
d| d| j q!t|tr}i }| }|tvrRt
d| dtt  d|drs|
du r_t
d|d	kro|du rkt
d
||d< |
|d< | j|fi |}n| |r|st
dnt
dtt  d| d|r|jdkrt
d| d|j d|| _|| _|| _t|ttfst|st
d|| _|dvrt
d|| _dS )a  Model pruning Callback, using PyTorch's prune utilities. This callback is responsible of pruning networks
        parameters during training.

        To learn more about pruning with PyTorch, please take a look at
        `this tutorial <https://pytorch.org/tutorials/intermediate/pruning_tutorial.html>`_.

        .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

        .. code-block:: python

            parameters_to_prune = [(model.mlp_1, "weight"), (model.mlp_2, "weight")]

            trainer = Trainer(
                callbacks=[
                    ModelPruning(
                        pruning_fn="l1_unstructured",
                        parameters_to_prune=parameters_to_prune,
                        amount=0.01,
                        use_global_unstructured=True,
                    )
                ]
            )

        When ``parameters_to_prune`` is ``None``, ``parameters_to_prune`` will contain all parameters from the model.
        The user can override ``filter_parameters_to_prune`` to filter any ``nn.Module`` to be pruned.

        Args:

            pruning_fn: Function from torch.nn.utils.prune module or your own PyTorch ``BasePruningMethod`` subclass.
                Can also be string e.g. `"l1_unstructured"`. See pytorch docs for more details.

            parameters_to_prune: List of tuples ``(nn.Module, "parameter_name_string")``.

            parameter_names: List of parameter names to be pruned from the nn.Module.
                Can either be ``"weight"`` or ``"bias"``.

            use_global_unstructured: Whether to apply pruning globally on the model.
                If ``parameters_to_prune`` is provided, global unstructured will be restricted on them.

            amount: Quantity of parameters to prune:

                - ``float``. Between 0.0 and 1.0. Represents the fraction of parameters to prune.
                - ``int``. Represents the absolute number of parameters to prune.
                - ``Callable``. For dynamic values. Will be called every epoch. Should return a value.

            apply_pruning: Whether to apply pruning.

                - ``bool``. Always apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            make_pruning_permanent: Whether to remove all reparametrization pre-hooks and apply masks
                when training ends or the model is saved.

            use_lottery_ticket_hypothesis: See `The lottery ticket hypothesis <https://arxiv.org/abs/1803.03635>`_:

                - ``bool``. Whether to apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            resample_parameters: Used with ``use_lottery_ticket_hypothesis``. If True, the model parameters will
                be resampled, otherwise, the exact original parameters will be used.

            pruning_dim: If you are using a structured pruning method you need to specify the dimension.

            pruning_norm: If you are using ``ln_structured`` you need to specify the norm.

            verbose: Verbosity level. 0 to disable, 1 to log overall sparsity, 2 to log per-layer sparsity

            prune_on_train_epoch_end: whether to apply pruning at the end of the training epoch.
                If this is ``False``, then the check runs at the end of the validation epoch.

        Raises:
            MisconfigurationException:
                If ``parameter_names`` is neither ``"weight"`` nor ``"bias"``,
                if the provided ``pruning_fn`` is not supported,
                if ``pruning_dim`` is not provided when ``"unstructured"``,
                if ``pruning_norm`` is not provided when ``"ln_structured"``,
                if ``pruning_fn`` is neither ``str`` nor :class:`torch.nn.utils.prune.BasePruningMethod`, or
                if ``amount`` is none of ``int``, ``float`` and ``Callable``.

        Nz%The provided `parameter_names` name: z
 isn't in zThe provided `pruning_fn` z2 isn't available in PyTorch's built-in functions:  _structuredzKWhen requesting `structured` pruning, the `pruning_dim` should be provided.r   zOWhen requesting `ln_structured` pruning, the `pruning_norm` should be provided.ndimz\PyTorch `BasePruningMethod` is currently only supported with `use_global_unstructured=True`.z(`pruning_fn` is expected to be a str in z* or a PyTorch `BasePruningMethod`. Found: zI. HINT: if passing a `BasePruningMethod`, pass the class, not an instanceunstructuredzdOnly the "unstructured" PRUNING_TYPE is supported with `use_global_unstructured=True`. Found method z	 of type z. zO`amount` should be provided and be either an int, a float or Callable function.)r         z"`verbose` must be any of (0, 1, 2))_use_global_unstructured_parameters_to_prune_use_lottery_ticket_hypothesis_resample_parameters_prune_on_train_epoch_endPARAMETER_NAMES_parameter_names_global_kwargs_original_layers_pruning_method_namer   
isinstancer$   lower_PYTORCH_PRUNING_FUNCTIONSlistkeysendswith_create_pruning_fn_is_pruning_methodPRUNING_TYPEr+   _apply_pruning_make_pruning_permanentr#   floatcallabler/   _verbose)selfr+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   namepruning_kwargsr%   r%   r&   __init__A   s   a






zModelPruning.__init__c                 C   s   |S )zAThis function can be overridden to control which module to prune.r%   )rX   r,   r%   r%   r&   filter_parameters_to_prune   s   z'ModelPruning.filter_parameters_to_prunekwargsc                 K   sV   | j rt| nt| }t|sJ d| j r|| _|j| _| j r"|S tj|fi |S )a  This function takes `pruning_fn`, a function name.

        IF use_global_unstructured, pruning_fn will be resolved into its associated ``PyTorch BasePruningMethod`` ELSE,
        pruning_fn will be resolved into its function counterpart from `torch.nn.utils.prune`.

        z'Selected pruning method is not callable)	r@   _PYTORCH_PRUNING_METHODrL   rV   rG   r   rI   r'   _wrap_pruning_fn)rX   r+   r]   pruning_methr%   r%   r&   rP      s   	
zModelPruning._create_pruning_fnc                 K   s   t | fi |S Nr   )r+   r]   r%   r%   r&   r_     s   zModelPruning._wrap_pruning_fnmodulec                 C   sN   |  D ] \}}t|jD ]}|j| }t|tjr#|| |j|= qqdS )zRemoves pruning buffers from any pruned modules.

        Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/nn/utils/prune.py#L1118-L1122

        N)named_modulesrM   _forward_pre_hooksrJ   pytorch_pruneBasePruningMethodremove)rX   rb   _khookr%   r%   r&   r1     s   

z#ModelPruning.make_pruning_permanentnewoldrY   c                 C   sP   t | |}t ||}|d u s|d u st|trt|tsd S |j|j|_d S ra   )getattrrJ   r   r   todevice)rk   rl   rY   dstsrcr%   r%   r&   _copy_param  s
   

$zModelPruning._copy_paramc                 C   s   | j dusJ | j  D ]4}|d }|d }| jr+t|dr+t|jr+t|}|  |D ]\}}| j| \}}| ||| q-qdS )a  Lottery ticket hypothesis algorithm (see page 2 of the paper):

            1. Randomly initialize a neural network :math:`f(x; \theta_0)` (where :math:`\theta_0 \sim \mathcal{D}_\theta`).
            2. Train the network for :math:`j` iterations, arriving at parameters :math:`\theta_j`.
            3. Prune :math:`p\%` of the parameters in :math:`\theta_j`, creating a mask :math:`m`.
            4. Reset the remaining parameters to their values in :math:`\theta_0`, creating the winning ticket :math:`f(x; m \odot \theta_0)`.

        This function implements the step 4.

        The ``resample_parameters`` argument can be used to reset the parameters with a new :math:`\theta_z \sim \mathcal{D}_\theta`

        Nr   r   reset_parameters)	rH   valuesrC   hasattrrV   rs   r   rA   rr   )rX   dcopyr   irY   rk   rh   r%   r%   r&   apply_lottery_ticket_hypothesis  s   z,ModelPruning.apply_lottery_ticket_hypothesisc                 C   s$   | j D ]\}}| j|||d qd S )N)rY   r/   )rA   r+   )rX   r/   rb   rY   r%   r%   r&   _apply_local_pruning5  s   z!ModelPruning._apply_local_pruningc                    s>   || j d< tt| jj  d  fdd| j  D S )Nr/   rX   c                    s   i | ]\}}| v r||qS r%   r%   ).0ri   vparamsr%   r&   
<dictcomp>=  s    z7ModelPruning._resolve_global_kwargs.<locals>.<dictcomp>)rG   setinspect	signaturer+   
parametersdiscarditemsrX   r/   r%   r}   r&   _resolve_global_kwargs9  s   

z#ModelPruning._resolve_global_kwargsc                 C   s$   t j| jfd| ji| | d S )Npruning_method)re   global_unstructuredrA   r+   r   r   r%   r%   r&   _apply_global_pruning?  s   
z"ModelPruning._apply_global_pruningc                 C   s:   | d}t | |sdS t| |}|dk  | fS )N_mask)r   r>   r   )ru   rm   sumitemnumel)rb   rY   attrmaskr%   r%   r&   _get_pruned_statsD  s
   


zModelPruning._get_pruned_statsc                    sh    j r fdd jD } jr | n |  j r2 fdd jD } j|||d dS dS )z+Applies pruning to ``parameters_to_prune``.c                       g | ]
\}}  ||qS r%   r   r{   mr;   rX   r%   r&   
<listcomp>O      z.ModelPruning.apply_pruning.<locals>.<listcomp>c                    r   r%   r   r   r   r%   r&   r   W  r   )r/   N)rW   rA   r@   r   rz   _log_sparsity_stats)rX   r/   
prev_stats
curr_statsr%   r   r&   r0   L  s   
zModelPruning.apply_pruningprevcurrc                 C   s  t dd | jD }t dd |D }t dd |D }td| j d| d| d|| d	d
| d| d|| d	d | jdkrt| jD ]9\}\}}	|| \}
}|| \}}td| j d|d|	 d| d|
 d|
| d	d
| d|| d	d qHd S d S )Nc                 s   s*    | ]\}}|  D ]}| V  q
qd S ra   )r   r   )r{   layerrh   pr%   r%   r&   	<genexpr>^     ( z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>c                 s       | ]\}}|V  qd S ra   r%   r{   zerosrh   r%   r%   r&   r   _      c                 s   r   ra   r%   r   r%   r%   r&   r   `  r   z	Applied `z`. Pruned: /z (z.2%z) -> )r?   z` to `.z` with amount=z
. Pruned: )r   rA   loginforI   rW   	enumerate)rX   r   r   r/   total_paramsprev_total_zeroscurr_total_zerosrx   rb   rY   prev_mask_zerosprev_mask_sizecurr_mask_zeroscurr_mask_sizer%   r%   r&   r   Z  sF   

z ModelPruning._log_sparsity_statstrainerz
pl.Trainer	pl_modulestagec           	      C   s   | j || j| jd}| || _| jr?i | _t| jD ]%\}\}}t|}| j|t	t
|g d | j| d ||f qd S d S )N)r-   )r   r   r   )sanitize_parameters_to_prunerA   rF   r\   rB   rH   r   id
setdefaultr   r   append)	rX   r   r   r   r,   rx   rb   rY   id_r%   r%   r&   setupp  s   
zModelPruning.setupcurrent_epochc                 C   sz   t | jr
| |n| j}t | jr| |n| j}|r|s d S | | t | jr0| |r;n| jr9|   d S d S d S ra   )rV   rS   r/   r0   rB   ry   )rX   r   pruner/   r%   r%   r&   _run_pruning  s   
zModelPruning._run_pruningc                 C   s"   | j rtd | |j d S d S )Nz3`ModelPruning.on_train_epoch_end`. Applying pruning)rD   r   r   r   rX   r   r   r%   r%   r&   on_train_epoch_end  s   zModelPruning.on_train_epoch_endc                 C   s,   |j s| jstd | |j d S d S d S )Nz8`ModelPruning.on_validation_epoch_end`. Applying pruning)sanity_checkingrD   r   r   r   r   r%   r%   r&   on_validation_epoch_end  s   z$ModelPruning.on_validation_epoch_endc                 C   s    | j rtd | | d S d S )NzJ`ModelPruning.on_train_end`. Pruning is made permanent for this checkpoint)rT   r   r1   r   r%   r%   r&   on_train_end  s   zModelPruning.on_train_endc                 C   sp   |  }dd |D }|D ]}||d }||d }|j|jd| ||< qdtdtfdd	}t|t|S )
Nc                 S   s"   h | ]}| d r|d dqS )r    )rO   replace)r{   ri   r%   r%   r&   	<setcomp>  s   " zEModelPruning._make_pruning_permanent_on_state_dict.<locals>.<setcomp>_origr   )dtypetensorr8   c                 S   s   |   S ra   )cpu)r   r%   r%   r&   move_to_cpu  s   zGModelPruning._make_pruning_permanent_on_state_dict.<locals>.move_to_cpu)
state_dictpoprn   r   r   r   )rX   r   r   map_pruned_paramstensor_nameorigr   r   r%   r%   r&   %_make_pruning_permanent_on_state_dict  s   z2ModelPruning._make_pruning_permanent_on_state_dict
checkpointc                 C   s$   | j rtd | ||d< d S d S )NzP`ModelPruning.on_save_checkpoint`. Pruning is made permanent for this checkpointr   )rT   r   r   )rX   r   r   r   r%   r%   r&   on_save_checkpoint  s   zModelPruning.on_save_checkpointc                    s   |pt j}dd |  D  |s fdd|D }|S t|ttfrjt|dkrjtdd |D rjtdd |D rjg g }}|D ]\}}| vrO|| qAt	||sY|| qA|s^|rht
d| d	| |S t
d
)a  This function is responsible of sanitizing ``parameters_to_prune`` and ``parameter_names``. If
        ``parameters_to_prune is None``, it will be generated with all parameters of the model.

        Raises:
            MisconfigurationException:
                If ``parameters_to_prune`` doesn't exist in the model, or
                if ``parameters_to_prune`` is neither a list nor a tuple.

        c                 S   s   g | ]	}t |ts|qS r%   )rJ   _MODULE_CONTAINERS)r{   r   r%   r%   r&   r     s    z=ModelPruning.sanitize_parameters_to_prune.<locals>.<listcomp>c                    s.   g | ]} D ]}t ||d d ur||fqqS ra   )rm   )r{   r   r   current_modulesr%   r&   r     s     r   c                 s   s    | ]	}t |d kV  qdS )r?   N)len)r{   r   r%   r%   r&   r     s    z<ModelPruning.sanitize_parameters_to_prune.<locals>.<genexpr>c                 s   s*    | ]\}}t |tjot |tV  qd S ra   )rJ   r   r!   r$   )r{   abr%   r%   r&   r     r   zUSome provided `parameters_to_prune` don't exist in the model. Found missing modules: z and missing parameters: zThe provided `parameters_to_prune` should either be list of tuple with 2 elements: (nn.Module, parameter_name_to_prune) or None)r'   rE   modulesrJ   rM   tupler   allr   ru   r   )r   r,   r-   r   missing_modulesmissing_parametersrb   rY   r%   r   r&   r     sD   






z)ModelPruning.sanitize_parameters_to_prunemethodc                 C   s   t | sdS t| tjS )NF)r   isclass
issubclassre   rf   )r   r%   r%   r&   rQ     s   
zModelPruning._is_pruning_method)r%   NTr*   TTTFNNr   T)r%   )r8   N)r   )r%   r%   )/r   r   r    rE   r   r   r$   _PARAM_LISTr	   r   boolr#   rU   r[   r\   r   re   rf   rP   staticmethodr_   r   r!   r1   rr   ry   rz   r   r   r   r   r   r0   r   r   r   r   r   r   r   r   r   r   r   r
   r   rQ   r%   r%   r%   r&   r'   >   s    

	

 *  
"
$/r'   )>__doc__r   loggingrw   r   	functoolsr   typingr   r   r   r   r	   r
   r   r   torch.nn.utils.pruner   utilsr   re   #lightning_utilities.core.apply_funcr   torchr   typing_extensionsr   r   pytorch_lightningpl$pytorch_lightning.callbacks.callbackr   pytorch_lightning.core.moduler   &pytorch_lightning.utilities.exceptionsr   %pytorch_lightning.utilities.rank_zeror   r   	getLoggerr   r   r   r   r   r   rL   LnStructuredL1UnstructuredRandomStructuredRandomUnstructuredr^   r!   r$   _PARAM_TUPLEr   
Sequential
ModuleList
ModuleDictr   r   r'   r%   r%   r%   r&   <module>   s>   (
