o
    8wÖi?:  ã                   @   s°   d dl Z d dlmZmZ d dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ G dd„ de
ƒZG dd	„ d	ejeƒZG d
d„ dejeƒZG dd„ deƒZG dd„ deƒZdS )é    N)ÚAnyÚOptional)ÚConv1D)ÚBaseTunerLayerÚcheck_adapters_to_merge)Ú	transposec                   @   s8   e Zd ZdZdejdeddfdd„Zdd	„ Zd
d„ Z	dS )ÚIA3Layer)Úia3_lÚ
base_layerÚis_feedforwardÚreturnNc                 K   sÔ   || _ t i ¡| _d| _g | _|| _|  ¡ }t|tj	ƒr$|j
|j}}n>t|tjtjfƒr5|j|j}}n-t|tjƒrC|j|j}}nt|tƒrYt|jdƒrR|jjn|jj\}}n	tdt|ƒ› ƒ‚|| _
|| _d S )NFÚds_shapezUnsupported layer type )r
   ÚnnÚParameterDictr	   Ú_disable_adaptersÚmerged_adaptersr   Úget_base_layerÚ
isinstanceÚLinearÚin_featuresÚout_featuresÚConv2dÚConv3dÚin_channelsÚout_channelsÚ	EmbeddingÚnum_embeddingsÚembedding_dimr   ÚhasattrÚweightr   ÚshapeÚ
ValueErrorÚtype)Úselfr
   r   Úkwargsr   r   © r%   úR/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/ia3/layer.pyÚ__init__   s$   
ÿ
zIA3Layer.__init__c                 C   s`   | j rt d| jf¡}nt | jdf¡}t |¡| j|< |r#|  |¡ |  	|¡ |  
| j¡ d S ©Né   )r   ÚtorchÚrandnr   r   r   Ú	Parameterr	   Úreset_ia3_parametersÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r#   Úadapter_nameÚinit_ia3_weightsr   r%   r%   r&   Úupdate_layer6   s   

zIA3Layer.update_layerc                 C   s*   || j  ¡ v rtj | j | d¡ d S d S )Ng      ð?)r	   Úkeysr   ÚinitÚ	constant_)r#   r1   r%   r%   r&   r-   C   s   þzIA3Layer.reset_ia3_parameters)
Ú__name__Ú
__module__Ú__qualname__Úadapter_layer_namesr   ÚModuleÚboolr'   r3   r-   r%   r%   r%   r&   r      s
    r   c                       sŽ   e Zd Z				ddejdededededed	d
f‡ fdd„Zddedee	e  d	d
fdd„Z
ddd„Zdejdeded	ejfdd„Z‡  ZS )r   FTr
   r1   Úfan_in_fan_outr   Úis_target_conv_1d_layerr2   r   Nc                    s<   t ƒ  ¡  tj| ||d || _|| _|| _|  ||¡ d S ©N)r   )Úsuperr'   r   r=   r>   Ú_active_adapterr3   )r#   r
   r1   r=   r   r>   r2   r$   ©Ú	__class__r%   r&   r'   K   s   

zLinear.__init__Ú
safe_mergeÚadapter_namesc           	      C   s
  t | |ƒ}|s	dS |D ]w}|| j ¡ v r‚|  ¡ }t| j| j| jƒ}|jjj}|rJ|jj}t	 
||¡}t	 |¡ ¡ sBtd|› dƒ‚| |¡|j_nt	 
|jj|¡ |¡|j_| js||jdur|| j|  |jj¡}|jjj}t	 
|jj|j¡ |¡|j_| j |¡ qdS )á^  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        Nú1NaNs detected in the merged weights. The adapter ú seems to be broken)r   r	   r4   r   r   Údatar=   r   Údtyper*   ÚmulÚisfiniteÚallr!   Útor   ÚbiasÚreshaper    r   Úappend)	r#   rD   rE   Úactive_adapterr
   r	   Ú
orig_dtypeÚorig_weightsÚscalingr%   r%   r&   Úmerge\   s0   


ÿ
€êzLinear.mergec                 C   sì   | j s
t d¡ dS t d¡ t| jƒdkrt| j ¡ }|| j ¡ v rk|  ¡ }t	| j| j
| jƒd }|jj
j}t |jj
|¡ |¡|j_
| jsk|jdurk| j|  |jj¡}|jj
j}t |jj
|j
d ¡ |¡|j_
t| jƒdksdS dS )úW
        This method unmerges all merged adapter layers from the base weights.
        ú Already unmerged. Nothing to do.Nú,Unmerge result can be inaccurate for (IA)^3.r   ç:Œ0âŽyE>)ÚmergedÚwarningsÚwarnÚlenr   Úpopr	   r4   r   r   rI   r=   r   rJ   r*   ÚdivrN   r   rO   rP   r    )r#   rR   r
   r	   rS   rU   r%   r%   r&   Úunmerge†   s    




 ôzLinear.unmergeÚxÚargsr$   c                 O   s  |j  }}| jr| jr|  ¡  | j|g|¢R i |¤Ž}|S | jr.| j|g|¢R i |¤Ž}|S d}| jD ]}|| j ¡ vr=q3| j| j }|| j|  ¡ 9 }q3| j	rj| 
|¡}||  
|¡}	| j|	g|¢R i |¤Ž}|S | j|g|¢R i |¤Ž}|j }
||  
|
¡}|S r(   )rJ   Údisable_adaptersr[   ra   r
   r0   r	   r4   Úflattenr   rN   )r#   rb   rc   r$   rJ   Úprevious_dtypeÚresultÚia3_scalingrR   ÚintermÚresult_dtyper%   r%   r&   Úforward   s0   
ëî

üzLinear.forward)FFFT©FN©r   N)r7   r8   r9   r   r;   Ústrr<   r'   r   ÚlistrV   ra   r*   ÚTensorr   rk   Ú__classcell__r%   r%   rB   r&   r   I   s.    ùþýüûúù	÷ 
*&r   c                       s   e Zd Z			ddejdededededd	f‡ fd
d„Zdd„ Zddede	e
e  dd	fdd„Zddd„Zdejdededejfdd„Z‡  ZS )Ú_ConvNdFTr
   r1   r=   r   r2   r   Nc                    sB   t ƒ  ¡  tj| ||d || _|| _|j ¡ | _|  ||¡ d S r?   )	r@   r'   r   r=   rA   r   ÚdimÚ_kernel_dimr3   )r#   r
   r1   r=   r   r2   r$   rB   r%   r&   r'   ¼   s   
	z_ConvNd.__init__c                 C   sj   | j r| jn| j}d|fd| jd   }t |¡}t |¡| j|< |r(|  	|¡ |  
|¡ |  | j¡ d S )Nr)   )r)   é   )r   r   r   rt   r*   r+   r   r,   r	   r-   r.   r/   r0   )r#   r1   r2   Únum_featuresÚweights_sizer   r%   r%   r&   r3   Í   s   


z_ConvNd.update_layerrD   rE   c           	      C   s
  t | |ƒ}|s	dS |D ]w}|| j ¡ v r‚|  ¡ }|jjj}| j| j}| js,| dd¡}|rOt	 
|jj|¡ ¡ }t	 |¡ ¡ sGtd|› dƒ‚| |¡|j_nt	 
|jj|¡ |¡|j_| js||jdur|| j|  |jj¡}t	 
|jj|j¡ |¡|j_| j |¡ qdS )rF   Nr   r)   rG   rH   )r   r	   r4   r   r   rI   rJ   r   r   r*   rK   ÚclonerL   rM   r!   rN   rO   rP   r    r   rQ   )	r#   rD   rE   rR   r
   rS   rh   Úoutput_weightrU   r%   r%   r&   rV   Ø   s0   


ÿ€èz_ConvNd.mergec                 C   sò   | j s
t d¡ dS t d¡ t| jƒdkrw| j ¡ }|| j ¡ v rn|  ¡ }|j	j
j}| j| j
}| js:| dd¡}t |j	j
|d ¡ |¡|j	_
| jsn|jdurn| j|  |jj¡}|jj
j}t |jj
|j
¡ |¡|j_
t| jƒdksdS dS )rW   rX   NrY   r   r)   rZ   )r[   r\   r]   r^   r   r_   r	   r4   r   r   rI   rJ   r   r   r*   r`   rN   rO   rP   r    rK   )r#   rR   r
   rS   rh   rU   r%   r%   r&   ra     s$   




òz_ConvNd.unmergerb   rc   r$   c           
      O   s  |j  }}| jr| jr|  ¡  | j|g|¢R i |¤Ž}n`| jr,| j|g|¢R i |¤Ž}nPd}| jD ]}|| j ¡ vr;q1| j| j }|| j| 9 }q1| jri| 	|¡}||  	|  
¡ jj ¡}	| j|	g|¢R i |¤Ž}n| j|g|¢R i |¤Ž}| 	|¡| }| 	|¡}|S r(   )rJ   rd   r[   ra   r
   r0   r	   r4   r   rN   r   r   )
r#   rb   rc   r$   rJ   rf   rg   rh   rR   ri   r%   r%   r&   rk     s*   



z_ConvNd.forward)FFTrl   rm   )r7   r8   r9   r   r;   rn   r<   r'   r3   r   ro   rV   ra   r*   rp   r   rk   rq   r%   r%   rB   r&   rr   »   s*    úþýüûúø 
,&rr   c                       ó   e Zd Z‡ fdd„Z‡  ZS )r   c                    ó0   t ƒ j|i |¤Ž | jdkstd| j› ƒ‚d S )Né   z0Conv2d layer kernel must have 4 dimensions, not ©r@   r'   rt   r!   ©r#   rc   r$   rB   r%   r&   r'   ?  ó   
ÿzConv2d.__init__©r7   r8   r9   r'   rq   r%   r%   rB   r&   r   <  s    r   c                       rz   )r   c                    r{   )Né   z0Conv2d layer kernel must have 5 dimensions, not r}   r~   rB   r%   r&   r'   G  r   zConv3d.__init__r€   r%   r%   rB   r&   r   E  s    r   )r\   Útypingr   r   r*   Útorch.nnr   Útransformers.pytorch_utilsr   Úpeft.tuners.tuners_utilsr   r   Ú
peft.utilsr   r   r;   r   rr   r   r   r%   r%   r%   r&   Ú<module>   s   /r 	