o
    „o™i¦  ã                   @   sÄ   d Z ddlmZmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZ G d	d
„ d
e	jƒZG dd„ deee  ƒZG dd„ deƒZdS )z—Demo of a simple LSTM language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

é    )ÚIteratorÚListÚOptionalÚSizedÚTupleN)ÚTensor)Ú	Optimizer)Ú
DataLoaderÚSampler)ÚLightningModule©Ú	WikiText2c                       s€   e Zd Z	ddedededed	ef
‡ fd
d„Zddd„Zdedeeef deeef fdd„Z	dedeeef fdd„Z
‡  ZS )Ú
SimpleLSTMéþ  é   é   çš™™™™™É?Ú
vocab_sizeÚninpÚnhidÚnlayersÚdropoutc                    sf   t ƒ  ¡  || _t |¡| _t ||¡| _tj||||dd| _	t 
||¡| _|| _|| _|  ¡  d S )NT)r   Úbatch_first)ÚsuperÚ__init__r   ÚnnÚDropoutÚdropÚ	EmbeddingÚencoderÚLSTMÚrnnÚLinearÚdecoderr   r   Úinit_weights)Úselfr   r   r   r   r   ©Ú	__class__© úP/home/ubuntu/.local/lib/python3.10/site-packages/lightning/pytorch/demos/lstm.pyr      s   
zSimpleLSTM.__init__ÚreturnNc                 C   s<   t j | jjdd¡ t j | jj¡ t j | jjdd¡ d S )Ngš™™™™™¹¿gš™™™™™¹?)r   ÚinitÚuniform_r   ÚweightÚzeros_r#   Úbias©r%   r(   r(   r)   r$   #   s   zSimpleLSTM.init_weightsÚinputÚhiddenc                 C   sP   |   |  |¡¡}|  ||¡\}}|   |¡}|  |¡ d| j¡}tj|dd|fS )Néÿÿÿÿé   )Údim)r   r   r!   r#   Úviewr   ÚFÚlog_softmax)r%   r1   r2   ÚembÚoutputÚdecodedr(   r(   r)   Úforward(   s
   
zSimpleLSTM.forwardÚ
batch_sizec                 C   s0   t |  ¡ ƒ}| | j|| j¡| | j|| j¡fS ©N)ÚnextÚ
parametersÚ	new_zerosr   r   )r%   r=   r-   r(   r(   r)   Úinit_hidden/   s   þzSimpleLSTM.init_hidden)r   r   r   r   r   ©r*   N)Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr   r$   r   r   r<   rB   Ú__classcell__r(   r(   r&   r)   r      s     ÿÿÿÿÿÿ
&"r   c                       sN   e Zd Zdededdf‡ fdd„Zdeee  fdd„Zdefd	d
„Z	‡  Z
S )ÚSequenceSamplerÚdatasetr=   r*   Nc                    s,   t ƒ  ¡  || _|| _t| jƒ| j | _d S r>   )r   r   rK   r=   ÚlenÚ
chunk_size)r%   rK   r=   r&   r(   r)   r   8   s   
zSequenceSampler.__init__c                 c   s>    t | jƒ}t| jƒD ]}tt|||| j  | jƒƒV  qd S r>   )rL   rK   ÚrangerM   Úlistr=   )r%   ÚnÚir(   r(   r)   Ú__iter__>   s
   €
 ÿzSequenceSampler.__iter__c                 C   s   | j S r>   )rM   r0   r(   r(   r)   Ú__len__C   s   zSequenceSampler.__len__)rD   rE   rF   r   rG   r   r   r   rR   rS   rI   r(   r(   r&   r)   rJ   7   s    rJ   c                       sr   e Zd Zddef‡ fdd„Zddd„Zd	eeef d
edefdd„Zddd„Z	de
fdd„Zdefdd„Z‡  ZS )ÚLightningLSTMr   r   c                    s    t ƒ  ¡  t|d| _d | _d S )N)r   )r   r   r   Úmodelr2   )r%   r   r&   r(   r)   r   H   s   

zLightningLSTM.__init__r*   Nc                 C   s
   d | _ d S r>   )r2   r0   r(   r(   r)   Úon_train_epoch_endM   s   
z LightningLSTM.on_train_epoch_endÚbatchÚ	batch_idxc                 C   s~   |\}}| j d u r| j | d¡¡| _ | j d  ¡ | j d  ¡ f| _ |  || j ¡\}| _ t || d¡¡}| jd|dd |S )Nr   r4   r3   Ú
train_lossT)Úprog_bar)	r2   rU   rB   ÚsizeÚdetachr7   Únll_lossr6   Úlog)r%   rW   rX   r1   Útargetr:   Úlossr(   r(   r)   Útraining_stepP   s   
zLightningLSTM.training_stepc                 C   s   t dd d S )NT)Údownloadr   r0   r(   r(   r)   Úprepare_dataZ   s   zLightningLSTM.prepare_datac                 C   s   t ƒ }t|t|dddS )Né   )r=   )Úbatch_sampler)r   r	   rJ   )r%   rK   r(   r(   r)   Útrain_dataloader]   s   zLightningLSTM.train_dataloaderc                 C   s   t jj|  ¡ ddS )Ng      4@)Úlr)ÚtorchÚoptimÚSGDr@   r0   r(   r(   r)   Úconfigure_optimizersa   s   z"LightningLSTM.configure_optimizers)r   rC   )rD   rE   rF   rG   r   rV   r   r   ra   rc   r	   rf   r   rk   rI   r(   r(   r&   r)   rT   G   s    


rT   )Ú__doc__Útypingr   r   r   r   r   rh   Útorch.nnr   Útorch.nn.functionalÚ
functionalr7   r   Útorch.optimr   Útorch.utils.datar	   r
   Úlightning.pytorch.corer   Ú#lightning.pytorch.demos.transformerr   ÚModuler   rG   rJ   rT   r(   r(   r(   r)   Ú<module>   s    "