o
    iG                     @  s  d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	m
Z
 d dlm	  mZ d dlmZmZ d dlmZmZ d d	lmZmZmZmZ d d
lmZmZ d dlmZ 	 dd Zdd Zdd Z dd Z!dd Z"dd Z#d-ddZ$d-ddZ%				d.ddZ&	 						d/d!d"Z'G d#d$ d$eZ(G d%d& d&eZ)G d'd( d(eZ*e*Z+e,e&e*_&e,e'e*_'G d)d* d*eZ-G d+d, d,eZ.dS )0    )annotations)Callable)partial)	randrangeN)nncat)Module
Sequential)tree_flattentree_unflatten)	rearrangerepeatreduceeinsum)	RearrangeReduce)pack_with_inversec                 C  s   | d uS N vr   r   K/home/ubuntu/.local/lib/python3.10/site-packages/hyper_connections/mHCv2.pyexists      r   c                 C  s   | | dkS )Nr   r   )numdenr   r   r   divisible_by"   s   r   c                 C  s   t | r| S |S r   )r   )r   dr   r   r   default%      r   c                 C  s   | S r   r   )tr   r   r   identity(   s   r!   c                 C  s   | | S r   r   )xyr   r   r   add+   r   r$   c                 C  s   t j| d|dS )N   )pdim)F	normalize)r    r'   r   r   r   l1norm0   r   r*      c                 C  s   |dkr| S | j d | j d ksJ | j}|  } | | jddd  } |  }t|D ]}t|dd}t|dd}q,||S )Nr   T)r'   keepdimr'   )	shapedtypefloatamaxdetachexpranger*   to)	log_alphaitersr1   alpha_r   r   r   sinkhorn_knopps3   s   
r<   c                 C  sj   |dkr| S | j d | j d ksJ | j}|  } t|D ]}tj| dd} tj| dd} q|  |S )Nr   r,   r-   r/   )r0   r1   r2   r6   r(   log_softmaxr5   r7   )r8   r9   r1   r;   r   r   r   log_domain_sinkhorn_knoppsG   s   r>   Fc                 C  st   |r
t  t  fS |s|rt|sJ d|r t| |dd}ntdd| d}|r1t|}||fS tdd}||fS )	Nz`dim` must be passed into get_init_and_expand_reduce_stream_functions for returning an expansion function with stream embeddings addedT)expand_to_streams... d -> ... s dr   sz... s d -> ... dsum)r   Identityr   StreamEmbedr   AttentionPoolReduceStream)num_streamsadd_stream_embedadd_attn_pool_reduce_streamr'   disable	expand_fn	reduce_fnr   r   r   "get_expand_reduce_stream_functionsY   s   
rM   r%   c                 K  sn   t || dko	|dk}|stnt}	t|	| f|||d|}
t| ||||d}t|r1t|
|d}
|
g|R S )Nr%   )	num_fracssinkhorn_itersuse_triton_sinkhorn)rH   rI   r'   rJ   r/   )r   #ManifoldConstrainedHyperConnectionsResidualr   rM   r   )rG   rN   r'   rH   rI   rJ   rO   rP   kwargshyper_conn_klassinit_hyper_conn_fnexpand_reduce_fnsr   r   r   +get_init_and_expand_reduce_stream_functionsr   s   rW   c                      $   e Zd Z fddZdd Z  ZS )RMSNormc                   s*   t    |d | _tt|| _d S )Ng      ?)super__init__scaler   	Parametertorchzerosgammaselfr'   	__class__r   r   r[      s   

zRMSNorm.__init__c                 C  s   t j|dd| j | jd  S )Nr-   r/   r%   )r(   r)   r\   r`   )rb   r"   r   r   r   forward   s   zRMSNorm.forward__name__
__module____qualname__r[   re   __classcell__r   r   rc   r   rY          rY   c                      sH   e Zd Zdddd fddZdd	 Zd
d ZdddZdd Z  ZS )rR   N)branchresidual_transformrl   Module | Nonerm   c                  s$   t    || _t|t | _d S r   )rZ   r[   rl   r   r   rD   rm   )rb   rl   rm   argsrS   rc   r   r   r[      s   
zResidual.__init__c                 C  s   ||t  fS r   )dictrb   	residualsr   r   r   width_connection   s   zResidual.width_connectionc                 C  s   ||  | S r   )rm   )rb   branch_outputrr   r   r   r   depth_connection   s   zResidual.depth_connectionr   c                   $   t jr	J d fdd}|S )N"branch was already wrapped on initc                   0    | \}} |g|R i |}||} | S r   re   residualro   rS   branch_inputadd_residualrt   rl   rb   r   r   forward_and_add_residual      z:Residual.decorate_branch.<locals>.forward_and_add_residualr   rl   rb   rl   r   r   r~   r   decorate_branch      	zResidual.decorate_branchc                   R    \}  fdd}tjs||fS j|g|R i |}||S )Nc                   s6   t | \^} }}j| fi  } t| g|R |S r   )r
   ru   r   
branch_outrest	tree_specresidual_kwargsrr   rb   r   r   add_residual_fn   s   z)Residual.forward.<locals>.add_residual_fnrs   r   rl   rb   rr   branch_argsbranch_kwargsr|   r   rt   r   r   r   re      s   
zResidual.forward)rl   rn   rm   rn   rl   r   )	rg   rh   ri   r[   rs   ru   r   re   rj   r   r   rc   r   rR      s    
rR   c                      sb   e Zd Zddddddedddddddddd fddZdd Zdd ZdddZdd Z  Z	S )rQ   Ng        Tr%   r+   Fr   )rl   layer_indexdropoutrm   add_branch_out_to_residualnum_input_viewsdepth_residual_fnrN   rO   log_domain_sinkhornresidual_mix_constraint_fnforward_method_namesnum_dynamic_alpha_proposalsrP   mix_streams_before_normrl   rn   rm   r   Callable | Noner   tuple[str, ...]c                  s  t    || _|
dksJ |
| _|
dk| _td|
d| _td| _t||
s2J d| d|
 d||
 }d| _	|rMt
j||dd	d
| _	t
j| j	j t|| _|dksZJ d|| _t|t|| }||
 }||
 }|
| _|dkswJ || _|dk| _|| _t||f}d||ddf< t
t|t|fdd| _t
t|||| | _t
tdd | _t
tdd | _ || _!|rt
t||
d| _"t
t||
| _#t
tdd | _$d	}|rzddl%m&}m'} | }W n t(y   d	}Y nw |rt)||d| _*nt|t)|st+nt,|d| _*t
-|| _.t|t
/ | _0|	| _1|| _2| j2D ]}t3| |rBJ t4| j|}t5| || q8dS )zN
        Appendix J, Algorithm2 in - https://arxiv.org/abs/2409.19606
        r%   zb ... (f d) -> b ... f dfzb ... f d -> b ... (f d)zfeature dimension (z() must be divisible by the `num_fracs` ()NFbiasr   z-`num_residual_streams` must be greater than 0g      ?r/   g{Gz?r   )triton_sinkhornis_triton_available)r9   )6rZ   r[   rl   rN   	has_fracsr   split_fracsmerge_fracsr   maybe_mix_streamsr   Conv2dinitdirac_weightrY   normnum_residual_streamsr   r   r   has_dynamic_alpha_proposalsr   r^   r_   r]   r   eyestatic_alphadynamic_alpha_fnonespre_branch_scaleresidual_scaler   static_betadynamic_beta_fnh_post_scale!hyper_connections.triton_sinkhornr   r   ImportErrorr   r   r<   r>   Dropoutr   rD   rm   r   r   hasattrgetattrsetattr)rb   r   r'   rl   r   r   rm   r   r   r   rN   rO   r   r   r   r   rP   r   init_residual_indexnum_residual_streams_fracsnum_input_views_fracsinit_alpha0!use_triton_sinkhorn_and_availabler   r   forward_method_namefnrc   r   r   r[      st   


 



z,ManifoldConstrainedHyperConnections.__init__c                   s  | j | j}}| |}| |}|}t| jr(t|d\}}| |}||}| |}|j |	 }t
|| j	 d}t|d}t| j	 d| jd}t| j	 d| j| d}	t||	f}
t|
d| jd}
||
 }|| j	  }|dd | j| j f |d| j| j d f }}| }| |}t||fd	d
}| jrt|dd}nt|d}t|d|d}d }| jr|| j	  }|| j	  }|| j	  }| d }t
||	 d}t|d|d}| jdkr|ddd d d d f |ddd d d d d f }}n$|dd | jd d d d f |d| jd d d d d f }}t|d}| |}t|d}t fdd||fD \}}t|rK| }||t|dfS )Nz* c h wz... d, p d mix -> p ... mixz ... s1 f2 mix -> ... (s1 f2) mixz1 -> srA   z
n -> (v n)r   .r-   r/   zp ... -> ...meanz1 ... -> ...z... (s f) t -> ... s f t   z!... s f tf, ... s f d -> ... tf dz... (t f) d -> ... t f dr   r%   r   zb ... v f d -> v b ... f dzb ... s f d -> b ... s (f d)c                 3  s    | ]}|  V  qd S r   )r7   ).0r    r1   r   r   	<genexpr>  s    zGManifoldConstrainedHyperConnections.width_connection.<locals>.<genexpr>)beta)r   rN   rm   r   r   r   r   r   r1   r2   r   r   r   r   r   r   r   r   r   sigmoidr   r   r   r   r   r   r   r   tupler7   rp   )rb   rr   streamsfracs
norm_inputinverse_pack_lead_dimsnormed	wc_weightr   r   alpha_scaledynamic_alphar:   	alpha_prealpha_residualr   	dc_weightdynamic_betamix_hr|   r   r   r   rs   t  sZ   





2

8>




z4ManifoldConstrainedHyperConnections.width_connectionc                C  sT   | j sJ | |}|j}t| | d}| |}| |||}| |S )Nz)b ... f1 d, b ... s f1 f2 -> b ... s f2 d)	r   r   r1   r   r2   r   r   r7   r   )rb   rt   rr   r   r1   outputr   r   r   ru     s   



z4ManifoldConstrainedHyperConnections.depth_connectionr   c                   rv   )Nrw   c                   rx   r   ry   rz   r~   r   r   r     r   zUManifoldConstrainedHyperConnections.decorate_branch.<locals>.forward_and_add_residualr   r   r   r~   r   r     r   z3ManifoldConstrainedHyperConnections.decorate_branchc                   r   )Nc                   s@   j s| S t| \^} }}j| fi  } t| g|R |S r   )r   r
   ru   r   r   r   r   r   r     s
   zDManifoldConstrainedHyperConnections.forward.<locals>.add_residual_fnr   r   r   r   r   re     s   
z+ManifoldConstrainedHyperConnections.forward)rl   rn   rm   rn   r   r   r   r   r   )
rg   rh   ri   r$   r[   rs   ru   r   re   rj   r   r   rc   r   rQ      s,     d
rQ   c                      s(   e Zd Z	d fdd	Zdd Z  ZS )rE   Fc                   s.   t    || _|| _tt||| _d S r   )	rZ   r[   rG   r?   r   r]   r^   r_   stream_embed)rb   rG   r'   r?   rc   r   r   r[   )  s   
zStreamEmbed.__init__c                 C  s    | j rt|d| jd}|| j S )Nr@   rA   )r?   r   rG   r   rq   r   r   r   re   5  s   
zStreamEmbed.forward)Frf   r   r   rc   r   rE   (  s    rE   c                      rX   )rF   c                   s6   t    tj||dd| _| jjjt	| d S )NFr   )
rZ   r[   r   Linearto_attn_logitsr   datacopy_r^   r   ra   rc   r   r   r[   ?  s   
z"AttentionPoolReduceStream.__init__c                 C  s"   |  |}|jdd}t||dS )Nr,   r/   z... s d, ... s d -> ... d)r   softmaxr   )rb   rr   attn_logitsattnr   r   r   re   D  s   
z!AttentionPoolReduceStream.forwardrf   r   r   rc   r   rF   >  rk   rF   )r+   )FFNF)r%   NFFNr+   F)/
__future__r   typingr   	functoolsr   randomr   r^   r   r   torch.nn.functional
functionalr(   torch.nnr   r	   torch.utils._pytreer
   r   einopsr   r   r   r   einops.layers.torchr   r   torch_einops_utilsr   r   r   r   r!   r$   r*   r<   r>   rM   rW   rY   rR   rQ   mHCstaticmethodrE   rF   r   r   r   r   <module>   sX    



 C  A

