o
    8wi                     @   s   d Z ddlmZmZ ddlmZ ddlZddlmZ ddl	m  m
Z ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ G d
d dejZG dd deee  ZG dd deZdS )zDemo of a simple LSTM language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

    )IteratorSized)OptionalN)Tensor)	Optimizer)
DataLoaderSampler)LightningModule	WikiText2c                       s   e Zd Z	ddedededed	ef
 fd
dZdddZdedeeef deeef fddZ	dedeeef fddZ
  ZS )
SimpleLSTM        皙?
vocab_sizeninpnhidnlayersdropoutc                    sf   t    || _t|| _t||| _tj||||dd| _	t
||| _|| _|| _|   d S )NT)r   batch_first)super__init__r   nnDropoutdrop	EmbeddingencoderLSTMrnnLineardecoderr   r   init_weights)selfr   r   r   r   r   	__class__ Y/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/demos/lstm.pyr      s   
zSimpleLSTM.__init__returnNc                 C   s<   t j| jjdd t j| jj t j| jjdd d S )Ngg?)r   inituniform_r   weightzeros_r!   biasr#   r&   r&   r'   r"   $   s   zSimpleLSTM.init_weightsinputhiddenc                 C   sP   |  | |}| ||\}}|  |}| |d| j}tj|dd|fS )N   )dim)r   r   r   r!   viewr   Flog_softmax)r#   r/   r0   emboutputdecodedr&   r&   r'   forward)   s
   
zSimpleLSTM.forward
batch_sizec                 C   s0   t |  }|| j|| j|| j|| jfS N)next
parameters	new_zerosr   r   )r#   r;   r+   r&   r&   r'   init_hidden0   s   zSimpleLSTM.init_hidden)r   r   r   r   r   r(   N)__name__
__module____qualname__intfloatr   r"   r   tupler:   r@   __classcell__r&   r&   r$   r'   r      s     
&"r   c                       sN   e Zd Zdededdf fddZdeee  fddZdefd	d
Z	  Z
S )SequenceSamplerdatasetr;   r(   Nc                    s,   t    || _|| _t| j| j | _d S r<   )r   r   rJ   r;   len
chunk_size)r#   rJ   r;   r$   r&   r'   r   9   s   
zSequenceSampler.__init__c                 c   s>    t | j}t| jD ]}tt|||| j  | jV  qd S r<   )rK   rJ   rangerL   listr;   )r#   nir&   r&   r'   __iter__?   s
   
 zSequenceSampler.__iter__c                 C   s   | j S r<   )rL   r.   r&   r&   r'   __len__D   s   zSequenceSampler.__len__)rB   rC   rD   r   rE   r   r   rN   rQ   rR   rH   r&   r&   r$   r'   rI   8   s    rI   c                       sr   e Zd Zddef fddZdddZd	eeef d
edefddZdddZ	de
fddZdefddZ  ZS )LightningLSTMr   r   c                    s    t    t|d| _d | _d S )N)r   )r   r   r   modelr0   )r#   r   r$   r&   r'   r   I   s   

zLightningLSTM.__init__r(   Nc                 C   s
   d | _ d S r<   )r0   r.   r&   r&   r'   on_train_epoch_endN   s   
z LightningLSTM.on_train_epoch_endbatch	batch_idxc                 C   s~   |\}}| j d u r| j|d| _ | j d  | j d  f| _ | || j \}| _ t||d}| jd|dd |S )Nr   r2   r1   
train_lossT)prog_bar)	r0   rT   r@   sizedetachr5   nll_lossr4   log)r#   rV   rW   r/   targetr8   lossr&   r&   r'   training_stepQ   s   
zLightningLSTM.training_stepc                 C   s   t dd d S )NT)downloadr
   r.   r&   r&   r'   prepare_data[   s   zLightningLSTM.prepare_datac                 C   s   t  }t|t|dddS )N   )r;   )batch_sampler)r   r   rI   )r#   rJ   r&   r&   r'   train_dataloader^   s   zLightningLSTM.train_dataloaderc                 C   s   t jj|  ddS )Ng      4@)lr)torchoptimSGDr>   r.   r&   r&   r'   configure_optimizersb   s   z"LightningLSTM.configure_optimizers)r   rA   )rB   rC   rD   rE   r   rU   rG   r   r`   rb   r   re   r   rj   rH   r&   r&   r$   r'   rS   H   s    


rS   )__doc__collections.abcr   r   typingr   rg   torch.nnr   torch.nn.functional
functionalr5   r   torch.optimr   torch.utils.datar   r   pytorch_lightning.corer	   #pytorch_lightning.demos.transformerr   Moduler   rN   rE   rI   rS   r&   r&   r&   r'   <module>   s    "