o
    NÆÏi«J  ã                   @   s–   d dl Z d dlZd dlmZmZmZ d dlZd dlmZ d dl	m  m
Z d dlmZmZ G dd„ deƒZG dd„ dejeƒZG dd	„ d	ejeƒZdS )
é    N)ÚAnyÚOptionalÚUnion)ÚBaseTunerLayerÚcheck_adapters_to_mergec                   @   sˆ   e Zd ZdZdZdejddfdd„Z	dd	ed
e	de
de
de
ddfdd„Zd	efdd„Zd	efdd„Zdeddfdd„Zdddd„ZdS )ÚHRALayer)Úhra_u)Úhra_rÚhra_apply_GSÚ
base_layerÚreturnNc                 K   s”   || _ i | _i | _t i ¡| _d| _g | _d| _|| _	|  
¡ }t|tjƒr0|j|j| _| _d S t|tjƒrA|j|j| _| _d S tdt|ƒ› ƒ‚)NFTzUnsupported layer type )r   r	   r
   ÚnnÚParameterDictr   Ú_disable_adaptersÚmerged_adaptersÚcast_input_dtype_enabledÚkwargsÚget_base_layerÚ
isinstanceÚLinearÚin_featuresÚout_featuresÚConv2dÚin_channelsÚout_channelsÚ
ValueErrorÚtype)Úselfr   r   © r   úI/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/hra/layer.pyÚ__init__    s   zHRALayer.__init__FÚadapter_nameÚrÚapply_GSÚinit_weightsÚinference_modec                 K   sê   |dkrt d|› ƒ‚|| j|< || j|< |  ¡ }t|tjƒr/tjt 	| j
|¡dd| j|< n*t|tjƒrOtjt 	| j
|jd  |jd  |¡dd| j|< n
tdt|ƒj› ƒ‚|ra|  |¡ n|  |¡ |  |¡ | j| j|d dS )aJ  Internal function to create hra adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            init_weights (`bool`): Whether to initialize weights.
            apply_GS (`bool`): Whether to apply Gram-Schmidt orthogonalization or not.
        r   z?`r` should be a positive integer value but the value passed is T)Úrequires_gradz/HRA is not implemented for base layers of type )r%   N)r   r	   r
   r   r   r   r   Ú	ParameterÚtorchÚemptyr   r   r   Úkernel_sizeÚ	TypeErrorr   Ú__name__Úreset_hra_parametersÚreset_hra_parameters_randomÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r!   r"   r#   r$   r%   r   r   r   r   r   Úupdate_layer4   s$   

  þ

zHRALayer.update_layerc                 C   s–   | j | d dkrt d¡ tjj| j| t d¡d d S | j| j	}t
 |d |d d ¡}tjj|t d¡d t t
j|ddd¡| j|< d S )Né   r   z@The symmetric initialization can NOT be performed when r is odd!é   ©Úaé   ©Údim)r	   ÚwarningsÚwarnr   ÚinitÚkaiming_uniform_r   ÚmathÚsqrtÚshaper(   Úzerosr'   Úrepeat_interleave)r   r!   r@   Úhalf_ur   r   r   r-   a   s   
  zHRALayer.reset_hra_parametersc                 C   s    t jj| j| t d¡d d S )Nr4   r5   )r   r<   r=   r   r>   r?   )r   r!   r   r   r   r.   k   s    z$HRALayer.reset_hra_parameters_randomÚscalec                 C   s6   |dkrd S | j D ]}|| j ¡ vrq	t d¡ q	d S )Nr7   zFScaling operation for HRA not supported! Automatically set scale to 1.©r1   r   Úkeysr:   r;   ©r   rD   Úactive_adapterr   r   r   Úscale_layern   s   
üzHRALayer.scale_layerc                 C   s*   | j D ]}|| j ¡ vrqt d¡ qd S )Nz>Unscaling operation for HRA not supported! Keeping scale at 1.rE   rG   r   r   r   Úunscale_layerx   s
   
üzHRALayer.unscale_layer©F©N©r   N)r,   Ú
__module__Ú__qualname__Úadapter_layer_namesÚother_param_namesr   ÚModuler    ÚstrÚintÚboolr2   r-   r.   ÚfloatrI   rJ   r   r   r   r   r      s,    úþýüûú
ø-

r   c                       sº   e Zd ZdZ			ddedededeeef d	d
f
‡ fdd„Zddede	e
e  d	d
fdd„Zddd„Zd deded	ejfdd„Zdejdeded	ejfdd„Zd	ef‡ fdd„Z‡  ZS )!Ú	HRALinearz+
    HRA implemented in a dense layer.
    r   FTr!   r"   r#   r$   r   Nc                    s@   t ƒ  ¡  tj| |fi |¤Ž || _| j||||fi |¤Ž d S rL   ©Úsuperr    r   Ú_active_adapterr2   ©r   r   r!   r"   r#   r$   r   ©Ú	__class__r   r   r    …   s   
	zHRALinear.__init__Ú
safe_mergeÚadapter_namesc           	      C   sÖ   t | |ƒ}|s	dS |D ]]}|| j ¡ v rh|  ¡ }|jj}|rJ|jj ¡ }|  |¡}t	 
| |j¡|¡}t	 |¡ ¡ sBtd|› dƒ‚| |¡|j_n|  |¡}t	 
|jj |j¡|¡}| |¡|j_| j |¡ qdS )áb  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
        Nú1NaNs detected in the merged weights. The adapter ú seems to be broken)r   r   rF   r   ÚweightÚdtypeÚdataÚcloneÚget_delta_weightr(   ÚmmÚtoÚisfiniteÚallr   r   Úappend)	r   r^   r_   rH   r   Ú
orig_dtypeÚorig_weightÚdelta_weightÚ
new_weightr   r   r   Úmerge“   s,   


ÿ
€ëzHRALinear.mergec                 C   sœ   | j s
t d¡ dS t| jƒdkrL| j ¡ }|  ¡ }|jj}|| j	 
¡ v rC|jj ¡ }| j|dd}t | |j¡|¡}| |¡|j_t| jƒdksdS dS ©zW
        This method unmerges all merged adapter layers from the base weights.
        z Already unmerged. Nothing to do.Nr   T)Úreverse)Úmergedr:   r;   Úlenr   Úpopr   rc   rd   r   rF   re   rf   rg   r(   rh   ri   )r   rH   r   rm   rn   ro   rp   r   r   r   Úunmerge¼   s   

øzHRALinear.unmergers   c                 C   ó€  | j | }| j| }| j| }|j}|r~|d d …df |d d …df  ¡   dd¡g}td|ƒD ]0}|d d …|f  dd¡}	t|ƒD ]}
|	||
  ¡ |	 ||
   }	qB| |	|	 ¡   dd¡¡ q0t	j
|dd}t	j|d |j|jdd| | ¡   }|S ||jdd }t	j|d |j|jd}|rt|d ddƒ}nt|ƒ}|D ]}|d d …|f  dd¡}	|d| |	 |	 ¡   }q£|S ©Nr   éÿÿÿÿr7   r8   )Údevicerd   r3   ©r	   r
   r   r@   ÚnormÚviewÚrangeÚtrl   r(   ÚcatÚeyer{   rd   ©r   r!   rs   Úrankr#   Úopt_ur@   rc   ÚiÚuiÚjÚindicesr   r   r   rg   Î   ó.   


.(õzHRALinear.get_delta_weightÚxÚargsr   c                 O   s   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}no| jr*| j|g|¢R i |¤Ž}n_tj| j|jd}| j	D ]}|| j
 ¡ vr@q6|  |¡}t | |j ¡|¡}q6|  ¡ jj}	|  |	|j ¡}	t |	|¡}|  | jj|j ¡}
| jrw|  ||j ¡}n
| |  ¡ jjj ¡}tj|||
d}| |¡}|S )N©r{   )Úinputrc   Úbias)rd   Údisable_adaptersrt   rw   r   r(   r‚   r   r{   r1   r   rF   rg   rh   ri   r   rc   re   Ú_cast_input_dtyper   r   ÚFÚlinear©r   r‹   rŒ   r   Úprevious_dtypeÚresultrp   rH   ro   rn   r   r   r   r   Úforwardì   s.   


zHRALinear.forwardc                    ó   t ƒ  ¡ }d| S ©Nzhra.©rY   Ú__repr__©r   Úrepr\   r   r   r›     ó   
zHRALinear.__repr__©r   FT©FNrM   rK   ©r,   rN   rO   Ú__doc__rS   rT   rU   r   r    r   Úlistrq   rw   r(   ÚTensorrg   r   r—   r›   Ú__classcell__r   r   r\   r   rW   €   s*    úýüû
úø 
) rW   c                       s¶   e Zd ZdZ			ddedededeeef f‡ fd	d
„Zddede	e
e  ddfdd„Zddd„Zd dededejfdd„Zdejdededejfdd„Zdef‡ fdd„Z‡  ZS )!Ú	HRAConv2dzHRA implemented in Conv2d layerr   FTr!   r"   r#   r$   c                    s8   t ƒ  ¡  t | |¡ || _| j||||fi |¤Ž d S rL   rX   r[   r\   r   r   r      s   
	zHRAConv2d.__init__Nr^   r_   r   c                 C   sh  t | |ƒ}|s	dS |D ]¦}|| j ¡ v r±|  ¡ }|jj}|rm|jj ¡ }| | j	| j
|jd  | jjd  ¡}|  |¡}t | |j¡|¡}| | j	| j
|jd |jd ¡}t |¡ ¡ setd|› dƒ‚| |¡|j_n>|jj}| | j	| j
| jjd  | jjd  ¡}|  |¡}t | |j¡|¡}| | j	| j
|jd |jd ¡}| |¡|j_| j |¡ qdS )r`   Nr   ra   rb   )r   r   rF   r   rc   rd   re   rf   r~   r   r   r*   r   rg   r(   rh   ri   rj   rk   r   r   rl   )r   r^   r_   rH   r   rm   rn   ro   r   r   r   rq   "  sV   
þ
ü
ÿþ
ü€ÕzHRAConv2d.mergec                 C   sà   | j s
t d¡ dS t| jƒdkrn| j ¡ }|  ¡ }|jj}|| j	 
¡ v re|jj ¡ }| | j| j|jd  |jd  ¡}| j|dd}t | |j¡|¡}| | j| j|jd |jd ¡}| |¡|j_t| jƒdksdS dS rr   )rt   r:   r;   ru   r   rv   r   rc   rd   r   rF   re   rf   r~   r   r   r*   rg   r(   rh   ri   )r   rH   r   rm   rn   ro   r   r   r   rw   a  s(   

þÿðzHRAConv2d.unmergers   c                 C   rx   ry   r|   rƒ   r   r   r   rg   z  rŠ   zHRAConv2d.get_delta_weightr‹   rŒ   r   c                 O   s–  |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nª| jr*| j|g|¢R i |¤Ž}nštj| j| jjd  | jjd  |j	d}| j
D ]}|| j ¡ vrLqB|  |¡}t | |j ¡|¡}qB| jjj}	|	 | j| j| jjd  | jjd  ¡}	|  |	|j ¡}	|  | jj|j ¡}
t |	|¡}| | j| j| jjd | jjd ¡}| jr¨|  ||j ¡}n
| |  ¡ jjj ¡}tj|||
| jjd | jjd d}| |¡}|S )Nr   r   )rŽ   rc   r   ÚpaddingÚstride)rd   r   rt   rw   r   r(   r‚   r   r*   r{   r1   r   rF   rg   rh   ri   rc   re   r~   r   r‘   r   r   r   r’   Úconv2dr§   r¨   r”   r   r   r   r—   ˜  sT   þ


þ

ü

û
zHRAConv2d.forwardc                    r˜   r™   rš   rœ   r\   r   r   r›   Ë  rž   zHRAConv2d.__repr__rŸ   r    rM   rK   r¡   r   r   r\   r   r¦     s&    úýüû
ú 
?3r¦   )r>   r:   Útypingr   r   r   r(   Útorch.nnr   Útorch.nn.functionalÚ
functionalr’   Úpeft.tuners.tuners_utilsr   r   r   rR   rW   r¦   r   r   r   r   Ú<module>   s   f 