o
    zi                     @   s2  d Z ddlZddlZddlmZ ddlmZmZmZm	Z	 ddl
Z
ddlmZ ddlm  mZ ddlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ ed
Zeedraeedsaeje_G dd dejZG dd dejZ G dd deZ!G dd dZ"dede	ee"f fddZ#G dd deZ$dS )zDemo of a simple transformer language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

    N)Path)DictListOptionalTuple)RequirementCache)Tensor)MultiheadAttention)
DataLoaderDataset)LightningModulerequests_reset_parametersreset_parametersc                       sh   e Zd Z						ddedededed	ed
eddf fddZddededee defddZ  Z	S )Transformer        皙?
vocab_sizeninpnheadnhidnlayersdropoutreturnNc              	      sb   t    t||| _t||| _tj||||||dd| _t	||| _
|| _|| _d | _d S )NT)d_modelr   num_encoder_layersnum_decoder_layersdim_feedforwardr   batch_first)super__init__PositionalEncodingpos_encodernn	Embedding	embeddingr   transformerLineardecoderr   r   src_mask)selfr   r   r   r   r   r   	__class__ W/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/demos/transformer.pyr"       s    
		
zTransformer.__init__inputstargetmaskc                 C   s   |j \}}|d u r)ttj|||jddk}| |dktd|dkd}| | |t	
| j }| | |t	
| j }| j|||d}| |}tj|dd}|d| j}|S )	Ndevice   r   z-infg        )tgt_mask)dim)shapetorchtrilonesr5   floatmasked_fillr$   r'   mathsqrtr   r(   r*   Flog_softmaxviewr   )r,   r1   r2   r3   _tsrcoutputr/   r/   r0   forward;   s   
$
zTransformer.forward)r   r   r   r   r   r   N)
__name__
__module____qualname__intr>   r"   r   r   rI   __classcell__r/   r/   r-   r0   r      s.    (r   c                	       sV   e Zd Zddedededdf fdd	Zd
edefddZdej	defddZ
  ZS )r#   皙?  r9   r   max_lenr   Nc                    s.   t    tj|d| _|| _|| _d | _d S )N)p)r!   r"   r%   Dropoutr   r9   rR   pe)r,   r9   r   rR   r-   r/   r0   r"   M   s
   

zPositionalEncoding.__init__xc                 C   sD   | j d u r| j|jd| _ || j d |dd d f  }| |S )Nr4   r   )rU   _init_pos_encodingr5   sizer   )r,   rV   r/   r/   r0   rI   T   s   
 
zPositionalEncoding.forwardr5   c                 C   s   t j| j| j|d}t jd| jt j|dd}t t jd| jd|d t	d | j  }t 
|| |d d dd df< t || |d d dd df< |ddd}|S )Nr4   r   )dtyper5   r6   r   g     @)r;   zerosrR   r9   aranger>   	unsqueezeexpr@   logsincos	transpose)r,   r5   rU   positiondiv_termr/   r/   r0   rW   ^   s   0  z%PositionalEncoding._init_pos_encoding)rP   rQ   )rK   rL   rM   rN   r>   r"   r   rI   r;   r5   rW   rO   r/   r/   r-   r0   r#   L   s     
r#   c                	       s   e Zd ZdZedddfdedededd	f fd
dZedefddZ	defddZ
dedeeef fddZededd	fddZ  ZS )	WikiText2zMini version of WikiText2.z./data#   Tdata_dir
block_sizedownloadr   Nc                    s@   t    |d | _|r| | j t| j\| _| _|| _d S )Nzwikitext-2.txt)r!   r"   pathrh   tokenizedata
dictionaryrg   )r,   rf   rg   rh   r-   r/   r0   r"   k   s   


zWikiText2.__init__c                 C   
   t | jS rJ   )lenrl   r,   r/   r/   r0   r   s   s   
zWikiText2.vocab_sizec                 C   s   t | j| j d S Nr6   )rn   rk   rg   ro   r/   r/   r0   __len__w   s   zWikiText2.__len__indexc                 C   s@   || j  }|| j  }| j|| }| j|d |d  }||fS rp   )rg   rk   )r,   rr   startendr1   r2   r/   r/   r0   __getitem__z   s
   

zWikiText2.__getitem__destinationc                 C   s~   t sttt dd l}tj| jdd d}tj| rd S t	| d}|
||j W d    d S 1 s8w   Y  d S )Nr   T)exist_okzehttps://raw.githubusercontent.com/pytorch/examples/main/word_language_model/data/wikitext-2/train.txtw)_REQUESTS_AVAILABLEModuleNotFoundErrorstrr   osmakedirsparentri   existsopenwritegettext)rv   r   urlfr/   r/   r0   rh      s   "zWikiText2.download)rK   rL   rM   __doc__r   rN   boolr"   propertyr   rq   r   r   ru   staticmethodrh   rO   r/   r/   r-   r0   rd   h   s    *rd   c                   @   s6   e Zd Zd
ddZdedefddZdefdd	ZdS )
Dictionaryr   Nc                 C   s   i | _ g | _d S rJ   )word2idxidx2wordro   r/   r/   r0   r"      s   
zDictionary.__init__wordc                 C   s4   || j vr| j| t| jd | j |< | j | S rp   )r   r   appendrn   )r,   r   r/   r/   r0   add_word   s   

zDictionary.add_wordc                 C   rm   rJ   )rn   r   ro   r/   r/   r0   rq      s   
zDictionary.__len__r   N)rK   rL   rM   r"   r{   rN   r   rq   r/   r/   r/   r0   r      s    
r   ri   r   c                 C   s   t  }tj| sJ t| dd}|D ]}| dg }|D ]}|| qqW d    n1 s2w   Y  t| dd1}g }|D ]$}| dg }g }|D ]
}||j|  qO|t	
|t	j qBW d    n1 sqw   Y  t	||fS )Nutf8)encodingz<eos>)r   r|   ri   r   r   splitr   r   r   r;   tensortypeint64cat)ri   rl   r   linewordsr   idssidsr/   r/   r0   rj      s*   	rj   c                       s   e Zd Zddeddf fddZdededefd	d
Zdeeef dedefddZde	j
jfddZdddZdefddZ  ZS )LightningTransformerr   r   r   Nc                    s   t    t|d| _d S )N)r   )r!   r"   r   model)r,   r   r-   r/   r0   r"      s   
zLightningTransformer.__init__r1   r2   c                 C   s   |  ||S rJ   )r   )r,   r1   r2   r/   r/   r0   rI      s   zLightningTransformer.forwardbatch	batch_idxc                 C   s,   |\}}| ||}t jj||d}|S )Nr8   )r;   r%   
functionalnll_lossrD   )r,   r   r   r1   r2   rH   lossr/   r/   r0   training_step   s   
z"LightningTransformer.training_stepc                 C   s   t jj| j ddS )NrP   )lr)r;   optimSGDr   
parametersro   r/   r/   r0   configure_optimizers   s   z)LightningTransformer.configure_optimizersc                 C   s   t dd d S )NT)rh   )rd   ro   r/   r/   r0   prepare_data   s   z!LightningTransformer.prepare_datac                 C   s   t  }t|S rJ   )rd   r
   )r,   datasetr/   r/   r0   train_dataloader   s   z%LightningTransformer.train_dataloader)r   r   )rK   rL   rM   rN   r"   r   rI   r   r   r;   r   	Optimizerr   r   r
   r   rO   r/   r/   r-   r0   r      s    
r   )%r   r@   r|   pathlibr   typingr   r   r   r   r;   torch.nnr%   torch.nn.functionalr   rB    lightning_utilities.core.importsr   r   torch.nn.modulesr	   torch.utils.datar
   r   pytorch_lightningr   ry   hasattrr   r   Moduler   r#   rd   r   rj   r   r/   r/   r/   r0   <module>   s,    -(