o
    Ni                     @  s   d dl mZ d dlmZ d dlmZmZ d dlZd dlmZm	Z	m
Z
mZ d dlmZmZmZ d dlmZmZ dd	 Zd
d Zdd Z		dddZG dd deZG dd deZdS )    )annotations)partial)	randrangeuniformN)nncattensorrandperm)LSTMGRUModule)TransformerWrapperAutoregressiveWrapperc                 C  s   | d uS N )vr   r   M/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/up_wrapper.pyexists   s   r   c                 C  s   t | r| S |S r   )r   )r   dr   r   r   default   s   r   c                 C  s   | | dkS )Nr   r   )numdenr   r   r   divisible_by   s   r   Tc           
      C  s|   |dks
|dks
J t d| ||f}t ||ft| }t||f}t|r,||}|s0|S t|jd |j	d}	||	 S )Nr   device)
torchrandintfullr   r   r   tor	   shaper   )

num_tokensseq_lennum_samples_randomnum_samples_constantshuffler   rand_seq	const_seqall_seqrand_indicesr   r   r   random_sequences   s   
r)   c                      sh   e Zd Z				d fdd	Zdd Ze dd	 Ze ej				
dddZ
	dddZ  ZS )SyntheticDataGenerator   NFc                   st   t    || _t||| _t||}t|stnt	dd}t||}|||| _
tj||dd| _| | j d S )NT)batch_firstF)bias)super__init__max_seq_lenr   	Embeddingembedr   r   r
   r   netLinear	to_logitsapplyinit_)selfdimr    r0   hidden_sizeuse_grunetwork_klassdefault_network_klass	__class__r   r   r/   ;   s   
	

zSyntheticDataGenerator.__init__c                 C  s0   |   D ]}t|dr|  q| | j d S )Nreset_parameters)moduleshasattrr@   r6   r7   r8   mr   r   r   reset_U   s
   
zSyntheticDataGenerator.reset_c                 C  s(   t |tjr| jtdd9  _d S d S )Ng        g?)
isinstancer   r4   weightr   rC   r   r   r   r7   \   s   zSyntheticDataGenerator.init_-C6?c                 C  s   t |s
t |s
J g tt ||f}| j}tj|dd}|}d }	t|D ]+}
| ||	\}}	|d d df }|| jdd}t|d}|}tj||fdd}q%|d d | d f S )N)r9      )	r   filterr0   r   r   rangeforwardsoftmaxmultinomial)r8   lengthseed	conditiontemperatureprefixr!   seq	net_inputhiddens_logits
last_logitprobsampledr   r   r   generatea   s   	zSyntheticDataGenerator.generatec                 C  s,   |  |}| ||\}}| |}||fS r   )r2   r3   r5   )r8   inputrW   tokensr2   hiddenrY   r   r   r   rM      s   

zSyntheticDataGenerator.forward)r+   NFN)NNrH   r   )__name__
__module____qualname__r/   rE   r   no_gradr7   inference_modecompiler]   rM   __classcell__r   r   r>   r   r*   :   s"    
!r*   c                      sN   e Zd Z								dd fddZedd ZdddZdd Z  ZS )UniversalPretrainWrapperN       r+      r   Fmodelr   data_generator&SyntheticDataGenerator | Module | Nonec
                   s   t    || _t|| _|jjsJ |j}
|jj}t	|s%t
|
||d}|| _|| _|| _|| _|| _t||d }||ksCJ dt|dsJJ || _|| _tt|
|| _| |d |d }|	rg|| _n| d| | dtd d S )N)r    r9   r0   ri   z0data buffer size must be greater than batch size   synth_data_bufferstepr   )r.   r/   rl   r   
ar_wrappedattn_layerscausalr    r9   r   r*   reset_turing_machine_everyr!   rm   seed_length
batch_sizer   r   	num_resetbuffer_sizer   r)   random_sequences_fnrp   register_bufferr   )r8   rl   rm   ry   rx   rw   r!   rv   ru   keep_buffer_on_cpur    r9   init_data_bufferr>   r   r   r/      s8   

z!UniversalPretrainWrapper.__init__c                 C  s   | j jS r   )rq   r   )r8   r   r   r   r      s   zUniversalPretrainWrapper.devicec                 C  s.   t || j}t| j| jdd | }| j| S )Nr   )r   rw   r	   ry   r   rp   )r8   sizer(   r   r   r   get_rand_sequences_from_buffer   s   
z7UniversalPretrainWrapper.get_rand_sequences_from_bufferc                 C  s|  |   }|   }t| j}td| j| j | jf}|d d d f | }tj| j| jdd d d f }|||f }| jj	| j|
| j|
| jd}| jd | jdkret| j | jre| j  | jdkr|  | j}t  | j| jd | jd | jd}	||	 W d    n1 sw   Y  t  || W d    n1 sw   Y  |   
| j}
| |
S )Nr   r   )rR   rQ   rJ   ro   )r   r   arangerv   r   r!   rw   r   rm   r]   r   rq   add_ru   r   itemrE   rx   rd   rz   copy_rr   )r8   
conditionsseeds
seq_arangerand_offsetseq_start_posbatch_arange	generatedbuffer_to_resetreset_sequencesdatar   r   r   rM      s4   






z UniversalPretrainWrapper.forward)NNri   rj   r+   rk   r   F)rl   r   rm   rn   r   )	ra   rb   rc   r/   propertyr   r   rM   rg   r   r   r>   r   rh      s    8

rh   )TN)
__future__r   	functoolsr   randomr   r   r   r   r   r   r	   torch.nnr
   r   r   x_transformers.x_transformersr   r   r   r   r   r)   r*   rh   r   r   r   r   <module>   s   

W