o
     iG                  
   @   s   d dl mZmZ d dlZd dlZddlmZmZ ddlm	Z	m
Z
 dejdeejgejf dejjd	ed
df
ddZG dd dejjZG dd dejjZdS )    )CallableListN   )copy_to_model_parallel_region!reduce_from_model_parallel_region) sequence_parallel_leading_matmul!sequence_parallel_trailing_matmulweightinit_methodprocess_grouppartition_dimreturnc           
      C   s   |  }| }| j\}}|dkr%| || |}||d |d d f }	n| ||| }|d d |d |f }	|| t  | |	 W d    d S 1 sQw   Y  d S )Nr   )ranksizeshape	new_emptytorchno_gradcopy_)
r	   r
   r   r   r   
world_sizenrowsncolsfull_weightmy_weight_slice r   N/home/ubuntu/.local/lib/python3.10/site-packages/xformers/ops/modpar_layers.py_init_2d_weight   s   


"r   c                       s   e Zd Zddejjjddddedee dej	j
deded	eejgejf d
ededdf fddZdejdeej fddZ  ZS )ColumnParallelLinearTF)biasgather_outputr
   sequence_parallelfuse_sequence_parallelin_featuresout_featuresr   r   r   r
   r    r!   r   Nc          
         s   t t|   t|tstd|rtd|rtd | _|| _|| _	|| _
|| _| tfdd|D s<J fdd|D | _tj fdd| jD | _| jD ]
}	t|	||d	d
 qXd S )NzTxFormers's implementation of ColumnParallelLinear requires out_features to be a listzExFormers's implementation of ColumnParallelLinear requires bias=FalsezNxFormers's implementation of ColumnParallelLinear requires gather_output=Falsec                 3   s    | ]	}|  d kV  qdS )r   Nr   .0dimmp_sizer   r   	<genexpr>Q   s    z0ColumnParallelLinear.__init__.<locals>.<genexpr>c                    s   g | ]}|  qS r   r   r$   r'   r   r   
<listcomp>R       z1ColumnParallelLinear.__init__.<locals>.<listcomp>c                    s"   g | ]}t jt | fqS r   )r   nn	Parameteremptyr$   )r"   r   r   r*   U   s    r   r   )superr   __init__
isinstancelist	TypeError
ValueErrorr"   global_out_featuresr    r!   r   r   allmy_out_featuresr   r,   ParameterListweightsr   )
selfr"   r#   r   r   r   r
   r    r!   w	__class__)r"   r(   r   r1   .   s:   


zColumnParallelLinear.__init__input_c                    sN   | j rt dd | jD | j| jd}|S t | j  fdd| jD }|S )Nc                 S   s   g | ]}|  qS r   )tr%   r<   r   r   r   r*   b   r+   z0ColumnParallelLinear.forward.<locals>.<listcomp>fuser   c                    s   g | ]
}t  | qS r   )r   matmulr@   rA   r?   r   r   r*   h   s    )r    r   r:   r!   r   r   )r;   r?   outputsr   rE   r   forward^   s   	zColumnParallelLinear.forward)__name__
__module____qualname__r   r,   initxavier_normal_intr   distributedProcessGroupboolr   Tensorr1   rG   __classcell__r   r   r=   r   r   -   s8    "0r   c                       sz   e Zd Zddejjjddddededejj	de
de
d	eejgejf d
e
de
f fddZdejdejfddZ  ZS )RowParallelLinearTF)r   input_is_parallelr
   r    r!   r"   r#   r   r   rT   r
   r    r!   c          
         s   t t|   |rtd|std|| _|| _|| _|| _|| _|	 }	||	 dks.J ||	 | _
tjt|| j
f| _t| j||dd d S )NzBxFormers's implementation of RowParallelLinear requires bias=FalsezNxFormers's implementation of RowParallelLinear requires input_is_parallel=Truer   r   r/   )r0   rS   r1   r5   global_in_featuresr#   r    r!   r   r   my_in_featuresr   r,   r-   r.   r	   r   )
r;   r"   r#   r   r   rT   r
   r    r!   r(   r=   r   r   r1   m   s*   
zRowParallelLinear.__init__r?   r   c                 C   sF   | j rt|| j | j| jd}|S t|| j }t|| j}|S )NrB   )	r    r   r	   r@   r!   r   r   rD   r   )r;   r?   outputr   r   r   rG      s   	zRowParallelLinear.forward)rH   rI   rJ   r   r,   rK   rL   rM   rN   rO   rP   r   rQ   r1   rG   rR   r   r   r=   r   rS   l   s4    (rS   )typingr   r   r   torch.distributeddifferentiable_collectivesr   r   seqparr   r   rQ   rN   rO   rM   r   r,   Moduler   rS   r   r   r   r   <module>   s$   
?