o
    8wi                     @   st   d dl Z d dlmZ d dlZd dlmZ d dlmZ ddlm	Z	 ddl
mZ G dd deZG d	d
 d
ejeZdS )    N)Any)BaseTunerLayer   )
PolyConfig)
get_routerc                   @   s4   e Zd ZdZdZdejfddZdd Zdd	 Z	d
S )	PolyLayer)poly_lora_Apoly_lora_Bpoly_router)rn_tasksn_skillsn_splits
base_layerc                 K   s   || _ i | _i | _i | _i | _i | _t | _t	 | _
t	 | _|| _|  }t|tjr6|j|j}}n	tdt| || _|| _d S )NzUnsupported layer type )r   r   r   r   r   	poly_typenn
ModuleDictr
   ParameterDictr   r	   kwargsget_base_layer
isinstanceLinearin_featuresout_features
ValueErrortype)selfr   r   r   r    r   S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/poly/layer.py__init__!   s    



zPolyLayer.__init__c              	   C   s   |j dkrtd|j  |j | j |< |j| j|< |j| j|< |j| j|< |j| j|< tt	|j|j| j
|j |j | j|< tt	|j|j|j | j|j | j|< t|| j|< | j||jd | | | | j d S )Nr   z?`r` should be a positive integer value but the value passed is )init_weights)r   r   r   r   r   r   r   	Parametertorchemptyr   r   r   r	   r   r
   reset_poly_parametersr    %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   adapter_namepoly_configr   r   r   update_layer6   s6   





zPolyLayer.update_layerc           
   	   C   s.  || j  v r| j | j\}}}}t|D ]-}t|D ]&}t||f}	tjjj|	t	
dd |	j| j | j||d d d d f< qq|rPtjj| j|  n<| j| j\}}}}t|D ]-}t|D ]&}t||f}	tjjj|	t	
dd |	j| j| j||d d d d f< qdq^| j|   d S d S )N   )a)r   keysshaperanger"   r#   r   initkaiming_uniform_mathsqrtTdatazeros_r	   r
   reset)
r   r(   r    r   r   dr   skillsplitparamr   r   r   r$   W   s&   $$zPolyLayer.reset_poly_parametersN)
__name__
__module____qualname__adapter_layer_namesother_param_namesr   Moduler   r*   r$   r   r   r   r   r      s    !r   c                       sf   e Zd Zdededdf fddZdddejd	ed
ejdedejf
ddZ	def fddZ
  ZS )r   r(   r)   returnNc                    s4   t    tj| |fi | || _| || d S )N)superr   r   _active_adapterr*   )r   r   r(   r)   r   	__class__r   r   r   t   s   
zLinear.__init__)task_idsxargsrG   r   c                O   s  |j }| jr| j|g|R i |}nq| j|g|R i |}| jD ]a}|| j vr,q"| j| }| j| }	| j| }
| j| }|	||d}|	 \}}}t
d||
f}t
d||f}||| j|}|dd||| j}||j }||||| 7 }q"||}|S )N)rG   	input_idszbqs,qsdr->bqdrzbqs,qsrd->bqrdr      )dtypedisable_adaptersr   r'   r   r-   r   r
   r	   sizer"   einsumreshaper   	transposer   tobmm)r   rH   rG   rI   r   previous_dtyperesultactive_adapterr   r
   r   r	   mixing_weightsbsr   r   ABr   r   r   forward   s*   





zLinear.forwardc                    s   t   }d| S )Nzpoly.)rC   __repr__)r   reprE   r   r   r\      s   
zLinear.__repr__)r<   r=   r>   strr   r   r"   Tensorr   r[   r\   __classcell__r   r   rE   r   r   r   s    *"r   )r2   typingr   r"   torch.nnr   peft.tuners.tuners_utilsr   configr   routerr   r   rA   r   r   r   r   r   <module>   s   W