o
    پiR                     @   s   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 e 
eZ		ddejd	ed
ee fddZdd ZdddZeZ						ddejd	ed
ee dededee defddZdS )    N)islice)
CollectionOptional)nn)group_parametersh㈵> modelweight_decayno_weight_decay_listc                 C   sp   t |}g }g }|  D ]!\}}|jsq|jdks"|ds"||v r(|| q|| q|dd||dgS )N   z.bias        )paramsr
   )setnamed_parametersrequires_gradndimendswithappend)r	   r
   r   decayno_decaynameparamr   r   L/home/ubuntu/.local/lib/python3.10/site-packages/timm/optim/_param_groups.pyparam_groups_weight_decay   s   r   c                    s   t   t  fdddS )Nc                      s   t t S N)tupler   r   itsizer   r   <lambda>%   s    z_group.<locals>.<lambda>r   )iterr   r   r   r   _group#   s   r"      c                    s   dd }t | di dd }g }g }|  D ]\}}|||r$||n|| qt|}	|d ur8|	|   }tt||}t| dd t|D }
|
 fdd|D  |
S )Nc                    s6   |sdS t |ttfrt fdd|D S  |S )NTc                    s   g | ]}  |qS r   )
startswith).0hpinr   r   
<listcomp>-   s    z7auto_group_layers.<locals>._in_head.<locals>.<listcomp>)
isinstancer   listanyr$   )r(   hpr   r'   r   _in_head)   s
   
z#auto_group_layers.<locals>._in_headpretrained_cfg
classifierc                 S   s    i | ]\}}|D ]}||qqS r   r   )r%   ilr(   r   r   r   
<dictcomp>>   s     z%auto_group_layers.<locals>.<dictcomp>c                    s   i | ]}| qS r   r   )r%   r(   num_trunk_groupsr   r   r3   ?   s    )	getattrgetr   r   lenr+   r"   	enumerateupdate)r	   layers_per_group
num_groupsr.   head_prefixnames_trunk
names_headr(   _num_trunk_layers	layer_mapr   r4   r   auto_group_layers(   s   rC   皙?T      ?Fweight_decay_exclude_1dlayer_decayend_layer_decayverbosec                    sL  t |}i }i }t| drt| | jdddd}	nt| }	t|	 d }
|
d t fddt|
D }| 	 D ]R\}}|j
sCq;|rJ|jdksN||v rSd	}d
}nd}|}|	|}d||f }||vr{|| }||g d||< ||g d||< || d | || d | q;|rddl}td|j|dd  t| S )z
    Parameter groups for layer-wise lr decay & weight decay
    Based on BEiT: https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L58
    group_matcherF)coarseT)reverser   c                 3   s    | ]	} |  V  qd S r   r   )r%   r1   rG   	layer_maxr   r   	<genexpr>^   s    z+param_groups_layer_decay.<locals>.<genexpr>r   r   r   zlayer_%d_%s)lr_scaler
   param_names)rP   r
   r   rQ   r   r   Nzparameter groups: 
%s   )indent)r   hasattrr   rJ   rC   maxvaluesr+   ranger   r   r   r7   r   json_loggerinfodumps)r	   r
   r   rF   rG   rH   rI   param_group_namesparam_groupsrB   
num_layerslayer_scalesr   r   g_decay
this_decaylayer_id
group_name
this_scalerX   r   rM   r   param_groups_layer_decayE   sF   


re   )r   r   )r#   N)rD   r   TrE   NF)logging	itertoolsr   typingr   r   torchr   timm.modelsr   	getLogger__name__rY   Modulefloatstrr   r"   rC   
_layer_mapboolre   r   r   r   r   <module>   sN    


