o
    Nic                     @   s   d dl mZ d dlZd dlmZ d dlm  mZ d dlmZm	Z	m
Z
 d dlmZmZmZ dd Zdd	 ZG d
d dejZdS )    )ceilN)nn)	rearrangepackunpack)top_ptop_keval_decoratorc                 C   s   | d uS N )valr   r   \/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/xl_autoregressive_wrapper.pyexists   s   r   c                 C   s   | | dkS )Nr   r   )numerdenomr   r   r   divisible_by   s   r   c                       sV   e Zd Z		d fdd	Ze eddee dfdefdd	Z		dd
dZ
  ZS )XLAutoregressiveWrapperr   c                    s(   t    || _|| _|| _|j| _d S r
   )super__init__	pad_valueignore_indexnetmax_seq_len)selfr   r   r   	__class__r   r   r      s
   
z XLAutoregressiveWrapper.__init__Ng      ?filter_kwargsc                 K   s  |j | j}	}
t|gd\}}|j\}}|j|
dd^ }}|D ]}| j|f|dd|\}}q t||
 }|}d }|}t|D ]}|jd }t||
}|d d |d f }| j|f||ddd|\}}|j	}|d d df }||fi |}t
j|| dd}t|d}|r|}|}tj||fdd}t|r||k}|jdd rt
|d}| jdddk}||| j} nqA|d d |d f }t||d\}|S )	Nz* ndimTmemsreturn_mems)r"   cacher#   return_intermediates   )r&   r   )devicer   r   shapesplitr   lenranger   r"   Fsoftmaxtorchmultinomialcatr   anyallpadfloatcumsummasked_fillr   r   )r   start_tokensseq_len	eos_tokentemperaturefilter_logits_fnr   r"   kwargsr'   r   psbtall_leading_tokens_leading_tokenscurr_pos	curr_memsr$   outcurr_segment_lenis_last_segment_tokensxlogitsfiltered_logitsprobssampleis_eos_tokensshifted_is_eos_tokensmaskr   r   r   generate"   sf   
	


	z XLAutoregressiveWrapper.generatec                    s   | j | j}}|d d d df |d d dd f }}|jd  |j|dd}|j|dd}t fdd|D }	| jjsAtjntj	}
d}t
|||	D ]#\}}}| j|f|dd|\}}|
t|d	||d
}|||  }qL|S )Nr   r&   r   c                 3   s    | ]
}|j d    V  qdS )r   N)r(   ).0r?   r8   r   r   	<genexpr>   s    z2XLAutoregressiveWrapper.forward.<locals>.<genexpr>g        Tr!   zb n c -> b c n)r   )r   r   r(   r)   tupler   output_is_log_probr,   cross_entropynll_losszipr   )r   rH   r"   r<   r   r   labelssplit_xsplit_labelsloss_weightsloss_fn
total_losschunkchunk_labelsloss_weightrI   lossr   rR   r   forwardv   s0   *

zXLAutoregressiveWrapper.forward)r   r   r
   )__name__
__module____qualname__r   r.   no_gradr	   r   dictrP   rc   __classcell__r   r   r   r   r      s     Ur   )mathr   r.   r   torch.nn.functional
functionalr,   einopsr   r   r   %x_transformers.autoregressive_wrapperr   r   r	   r   r   Moduler   r   r   r   r   <module>   s    