o
    Ni$$                     @  s   d dl mZ d dlZd dlmZmZ d dlmZmZ d dlm  m	Z
 d dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZ dd	 Zd
d ZG dd deZdS )    )annotationsN)nnTensor)Module
ModuleDict)Dict)packrepeatunpack)AttentionLayersScaledSinusoidalEmbeddingAbsolutePositionalEmbeddingLayerIntermediates	LayerNormalways
pad_at_dimis_emptyc                 C  s   | d uS N )valr   r   N/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/multi_input.pyexists   s   r   c                 C  s   t | r| S t|r| S |S r   )r   callable)r   dr   r   r   default   s   r   c                      sj   e Zd Ze dddddddddddddd fddZ																ddddZ  ZS )MultiInputTransformerWrapperNr           FTg      ?-C6?)
num_tokensemb_dimmax_mem_lenshift_mem_downemb_dropoutpost_emb_normnum_memory_tokens memory_tokens_interspersed_everyreturn_only_embeduse_abs_pos_embscaled_sinu_pos_embemb_frac_gradientattn_z_loss_weightr   Dict[str, int]attn_layersr   c                  sL  t    |j t | _|| _|| _|| _|dkp#|o"|j  }|r,t	d| _
n|r4t| _
nt|| _
tfdd| D | _|| _|rQtnt | _t|| _ krft nt | _|| _|rtd | _nt fdd| D | _t|	d}	|	| _|	dkrtt|	 | _|
| _ | jdk| _!|| _"d S )Nr   c                   s$   i | ]\}}| d t | qS )_embed)r   	Embedding).0nameone_num_tokens)r   r   r   
<dictcomp>K   s   $ z9MultiInputTransformerWrapper.__init__.<locals>.<dictcomp>c                   s"   i | ]\}}|t j |d dqS )F)bias)r   Linear)r/   r0   
logits_dimdimr   r   r2   \   s   " )#super__init__r7   r   r   max_seq_lenr    r!   disable_abs_pos_embr   pos_embr   r   r   itemsembedsr)   r   r   Identityr#   Dropoutr"   r4   project_embr,   	to_logitsr$   	Parametertorchrandnmemory_tokensr%   can_cache_kv can_cache_kv_outside_max_seq_len)selfr   r:   r,   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   no_abs_pos_emb	__class__)r7   r   r   r9   #   s:   



z%MultiInputTransformerWrapper.__init__xDict[str, Tensor]cacheLayerIntermediates | Nonec           -   	     st  t rJ t d }g |j|jjjdkjR \ }}}||B |B |B }|tj B }t	t	j
ksCJ d} D ]\}}| d}|j
v sYJ j
| |}|| }qIt|
on|
jtjk}|syj||
|dn|
}|| }t|r|| }|t|r|jdd  \}|jd ksJ dtj|fdd	t|st|rt| fd
d}t| fdd}tj||fdd	}|dk r|dksJ |  d|   |rcj} t| r/| dksJ tjtsJ dt|  |  }!td|! fdddtd| dtjdjd d}"t |"fd\}#t| sWt|rWt||dfddd}t| rctd dj!rt|r|d j! |j!d  }$}%g |%|$}jf|||	|d|d|\}&|rt| rtd| | dt"|#d\}"|"|&_t| rtd dd d d f |sԇfddj D }'|r|'f}(n|r}(n|'}(|rdd |&j#D })t$|)|d|&_%d}|r"|&j&}*t|rdd t'||*D n|*}+fdd|+D }+|s|(|+fS |+|&_(|r)|(|&fS |r8dd |&j#D },|(|,fS |(S )Nr   r   r-   )posseq_start_pos   zJprepended embeddings need to have same dimensions as text model dimensionsr6   c                        t j ft jdS N)devicedtyperD   onesboolr   )brX   nr   r   <lambda>       z6MultiInputTransformerWrapper.forward.<locals>.<lambda>c                     rV   rW   rZ   r   )r]   rX   prepend_seqr   r   r_      r`   zonly for decoder)r7   valuezb (n m) d -> (b n) m d)mzn d -> b n d)r]   zb * dTz(b n) m d -> b (n m) d)maskmems	mem_masksrO   return_hiddensrR   c                   s   i | ]	\}}|| qS r   r   )r/   r0   fn)rM   r   r   r2          z8MultiInputTransformerWrapper.forward.<locals>.<dictcomp>c                 S     g | ]}|j qS r   )pre_softmax_attnr/   tr   r   r   
<listcomp>       z8MultiInputTransformerWrapper.forward.<locals>.<listcomp>)weightc                 S  s   g | ]	}t j|d dqS )rU   r6   )rD   cat)r/   pairr   r   r   rn     ri   c                   s*   g | ]}|d  j  dddf  qS ).N)r    detachrl   )rI   r   r   rn     s   * c                 S  rj   r   )post_softmax_attnrl   r   r   r   rn     ro   ))r   listvaluesshaperX   r$   r)   r   rB   lenr>   r=   rY   rD   longr<   r#   rq   r   rs   r"   rA   r%   
isinstancer,   Decodermathceilr   	rearranger	   rF   r   r!   r
   attn_intermediatescalc_z_lossattn_z_losshiddenszipre   )-rI   rM   return_embeddingsreturn_logits_and_embeddingsreturn_intermediatesrd   return_memsreturn_attnre   rf   rQ   prepend_embedsprepend_mask
sum_embedsreturn_attn_z_lossr*   rR   rO   kwargsfirst_inputnum_memshas_memory_tokensr)   rg   	token_embr0   embed_id	embed_keyembedexternal_pos_embr<   prepend_dim	mem_everynext_seq_lenmemmem_packed_shapemems_lmems_rintermediateslogitsoutpre_softmax_attnsr   new_mems	attn_mapsr   )r]   rX   r^   ra   rI   rM   r   forwardl   s   4






$


"z$MultiInputTransformerWrapper.forward)r   r+   r,   r   )FFFNFFNNNNNNFr   NN)rM   rN   rO   rP   )__name__
__module____qualname__dictr9   r   __classcell__r   r   rK   r   r   "   s@    Lr   )
__future__r   rD   r   r   torch.nnr   r   torch.nn.functional
functionalFtypingr   einopsr   r	   r
   x_transformers.x_transformersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s    (