o
    8wi|                     @   s&  d Z ddlZddlZddlmZ ddlmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ ed
Zeedr[eeds[eje_G dd de	jZG dd de	jZG dd deZG dd dZdede eef fddZ!G dd deZ"dS )zDemo of a simple transformer language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

    N)Path)Optional)RequirementCache)Tensor)MultiheadAttention)
DataLoaderDataset)LightningModulerequests_reset_parametersreset_parametersc                       sh   e Zd Z						ddedededed	ed
eddf fddZddededee defddZ  Z	S )Transformer        皙?
vocab_sizeninpnheadnhidnlayersdropoutreturnNc              	      sb   t    t||| _t||| _tj||||||dd| _t	||| _
|| _|| _d | _d S )NT)d_modelr   num_encoder_layersnum_decoder_layersdim_feedforwardr   batch_first)super__init__PositionalEncodingpos_encodernn	Embedding	embeddingr   transformerLineardecoderr   r   src_mask)selfr   r   r   r   r   r   	__class__ `/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/demos/transformer.pyr       s    
		
zTransformer.__init__inputstargetmaskc                 C   s   |j \}}|d u r)ttj|||jddk}| |dktd|dkd}| | |t	
| j }| | |t	
| j }| j|||d}| |}tj|dd}|d| j}|S )	Ndevice   r   z-infg        )tgt_mask)dim)shapetorchtrilonesr2   floatmasked_fillr!   r$   mathsqrtr   r%   r'   Flog_softmaxviewr   )r)   r.   r/   r0   _tsrcoutputr,   r,   r-   forward;   s   
$
zTransformer.forward)r   r   r   r   r   r   N)
__name__
__module____qualname__intr;   r   r   r   rF   __classcell__r,   r,   r*   r-   r      s.    (r   c                	       sV   e Zd Zddedededdf fdd	Zd
edefddZdej	defddZ
  ZS )r    皙?  r6   r   max_lenr   Nc                    s.   t    tj|d| _|| _|| _d | _d S )N)p)r   r   r"   Dropoutr   r6   rO   pe)r)   r6   r   rO   r*   r,   r-   r   M   s
   

zPositionalEncoding.__init__xc                 C   s@   | j d u r| j|jd| _ || j d d |df  }| |S )Nr1   r3   )rR   _init_pos_encodingr2   sizer   )r)   rS   r,   r,   r-   rF   T   s   

zPositionalEncoding.forwardr2   c                 C   s   t j| j| j|d}t jd| jt j|dd}t t jd| jd|d t	d | j  }t 
|| |d d dd df< t || |d d dd df< |d}|S )Nr1   r   )dtyper2   r3   r   g     @)r8   zerosrO   r6   aranger;   	unsqueezeexpr=   logsincos)r)   r2   rR   positiondiv_termr,   r,   r-   rT   ^   s   0  
z%PositionalEncoding._init_pos_encoding)rM   rN   )rH   rI   rJ   rK   r;   r   r   rF   r8   r2   rT   rL   r,   r,   r*   r-   r    L   s     
r    c                	       s   e Zd ZdZedddfdedededd	f fd
dZedefddZ	defddZ
dedeeef fddZededd	fddZ  ZS )	WikiText2zMini version of WikiText2.z./data#   Tdata_dir
block_sizedownloadr   Nc                    s@   t    |d | _|r| | j t| j\| _| _|| _d S )Nzwikitext-2.txt)r   r   pathrd   tokenizedata
dictionaryrc   )r)   rb   rc   rd   r*   r,   r-   r   k   s   


zWikiText2.__init__c                 C   
   t | jS rG   )lenrh   r)   r,   r,   r-   r   s   s   
zWikiText2.vocab_sizec                 C   s   t | j| j d S Nr3   )rj   rg   rc   rk   r,   r,   r-   __len__w   s   zWikiText2.__len__indexc                 C   s@   || j  }|| j  }| j|| }| j|d |d  }||fS rl   )rc   rg   )r)   rn   startendr.   r/   r,   r,   r-   __getitem__z   s
   

zWikiText2.__getitem__destinationc                 C   s~   t sttt dd l}tj| jdd d}tj| rd S t	| d}|
||j W d    d S 1 s8w   Y  d S )Nr   T)exist_okzehttps://raw.githubusercontent.com/pytorch/examples/main/word_language_model/data/wikitext-2/train.txtw)_REQUESTS_AVAILABLEModuleNotFoundErrorstrr
   osmakedirsparentre   existsopenwritegettext)rr   r
   urlfr,   r,   r-   rd      s   "zWikiText2.download)rH   rI   rJ   __doc__r   rK   boolr   propertyr   rm   tupler   rq   staticmethodrd   rL   r,   r,   r*   r-   r`   h   s    *r`   c                   @   s6   e Zd Zd
ddZdedefddZdefdd	ZdS )
Dictionaryr   Nc                 C   s   i | _ g | _d S rG   )word2idxidx2wordrk   r,   r,   r-   r      s   
zDictionary.__init__wordc                 C   s4   || j vr| j| t| jd | j |< | j | S rl   )r   r   appendrj   )r)   r   r,   r,   r-   add_word   s   

zDictionary.add_wordc                 C   ri   rG   )rj   r   rk   r,   r,   r-   rm      s   
zDictionary.__len__r   N)rH   rI   rJ   r   rw   rK   r   rm   r,   r,   r,   r-   r      s    
r   re   r   c                 C   s   t  }tj| sJ t| dd}|D ]}| dg }|D ]}|| qqW d    n1 s2w   Y  t| dd1}g }|D ]$}| dg }g }|D ]
}||j|  qO|t	
|t	j qBW d    n1 sqw   Y  t	||fS )Nutf8)encodingz<eos>)r   rx   re   r{   r|   splitr   r   r   r8   tensortypeint64cat)re   rh   r   linewordsr   idssidsr,   r,   r-   rf      s*   	rf   c                       s   e Zd Zddeddf fddZdededefd	d
Zdeeef dedefddZde	j
jfddZdddZdefddZ  ZS )LightningTransformerr   r   r   Nc                    s   t    t|d| _d S )N)r   )r   r   r   model)r)   r   r*   r,   r-   r      s   
zLightningTransformer.__init__r.   r/   c                 C   s   |  ||S rG   )r   )r)   r.   r/   r,   r,   r-   rF      s   zLightningTransformer.forwardbatch	batch_idxc                 C   s,   |\}}| ||}t jj||d}|S )Nr5   )r8   r"   
functionalnll_lossrA   )r)   r   r   r.   r/   rE   lossr,   r,   r-   training_step   s   
z"LightningTransformer.training_stepc                 C   s   t jj| j ddS )NrM   )lr)r8   optimSGDr   
parametersrk   r,   r,   r-   configure_optimizers   s   z)LightningTransformer.configure_optimizersc                 C   s   t dd d S )NT)rd   )r`   rk   r,   r,   r-   prepare_data   s   z!LightningTransformer.prepare_datac                 C   s   t  }t|S rG   )r`   r   )r)   datasetr,   r,   r-   train_dataloader   s   z%LightningTransformer.train_dataloader)r   r   )rH   rI   rJ   rK   r   r   rF   r   r   r8   r   	Optimizerr   r   r   r   rL   r,   r,   r*   r-   r      s    
r   )#r   r=   rx   pathlibr   typingr   r8   torch.nnr"   torch.nn.functionalr   r?    lightning_utilities.core.importsr   r   torch.nn.modulesr   torch.utils.datar   r   pytorch_lightningr	   ru   hasattrr   r   Moduler   r    r`   r   r   rf   r   r,   r,   r,   r-   <module>   s,    -(