o
    9wi                     @   s   d Z ddlmZmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZ G d	d
 d
e	jZG dd deee  ZG dd deZdS )zDemo of a simple LSTM language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

    )IteratorListOptionalSizedTupleN)Tensor)	Optimizer)
DataLoaderSampler)LightningModule	WikiText2c                       s   e Zd Z	ddedededed	ef
 fd
dZdddZdedeeef deeef fddZ	dedeeef fddZ
  ZS )
SimpleLSTM        皙?
vocab_sizeninpnhidnlayersdropoutc                    sf   t    || _t|| _t||| _tj||||dd| _	t
||| _|| _|| _|   d S )NT)r   batch_first)super__init__r   nnDropoutdrop	EmbeddingencoderLSTMrnnLineardecoderr   r   init_weights)selfr   r   r   r   r   	__class__ Y/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/lightning/pytorch/demos/lstm.pyr      s   
zSimpleLSTM.__init__returnNc                 C   s<   t j| jjdd t j| jj t j| jjdd d S )Ngg?)r   inituniform_r   weightzeros_r#   biasr%   r(   r(   r)   r$   #   s   zSimpleLSTM.init_weightsinputhiddenc                 C   sP   |  | |}| ||\}}|  |}| |d| j}tj|dd|fS )N   )dim)r   r   r!   r#   viewr   Flog_softmax)r%   r1   r2   emboutputdecodedr(   r(   r)   forward(   s
   
zSimpleLSTM.forward
batch_sizec                 C   s0   t |  }|| j|| j|| j|| jfS N)next
parameters	new_zerosr   r   )r%   r=   r-   r(   r(   r)   init_hidden/   s   zSimpleLSTM.init_hidden)r   r   r   r   r   r*   N)__name__
__module____qualname__intfloatr   r$   r   r   r<   rB   __classcell__r(   r(   r&   r)   r      s     
&"r   c                       sN   e Zd Zdededdf fddZdeee  fddZdefd	d
Z	  Z
S )SequenceSamplerdatasetr=   r*   Nc                    s,   t    || _|| _t| j| j | _d S r>   )r   r   rK   r=   len
chunk_size)r%   rK   r=   r&   r(   r)   r   8   s   
zSequenceSampler.__init__c                 c   s>    t | j}t| jD ]}tt|||| j  | jV  qd S r>   )rL   rK   rangerM   listr=   )r%   nir(   r(   r)   __iter__>   s
   
 zSequenceSampler.__iter__c                 C   s   | j S r>   )rM   r0   r(   r(   r)   __len__C   s   zSequenceSampler.__len__)rD   rE   rF   r   rG   r   r   r   rR   rS   rI   r(   r(   r&   r)   rJ   7   s    rJ   c                       sr   e Zd Zddef fddZdddZd	eeef d
edefddZdddZ	de
fddZdefddZ  ZS )LightningLSTMr   r   c                    s    t    t|d| _d | _d S )N)r   )r   r   r   modelr2   )r%   r   r&   r(   r)   r   H   s   

zLightningLSTM.__init__r*   Nc                 C   s
   d | _ d S r>   )r2   r0   r(   r(   r)   on_train_epoch_endM   s   
z LightningLSTM.on_train_epoch_endbatch	batch_idxc                 C   s~   |\}}| j d u r| j|d| _ | j d  | j d  f| _ | || j \}| _ t||d}| jd|dd |S )Nr   r4   r3   
train_lossT)prog_bar)	r2   rU   rB   sizedetachr7   nll_lossr6   log)r%   rW   rX   r1   targetr:   lossr(   r(   r)   training_stepP   s   
zLightningLSTM.training_stepc                 C   s   t dd d S )NT)downloadr   r0   r(   r(   r)   prepare_dataZ   s   zLightningLSTM.prepare_datac                 C   s   t  }t|t|dddS )N   )r=   )batch_sampler)r   r	   rJ   )r%   rK   r(   r(   r)   train_dataloader]   s   zLightningLSTM.train_dataloaderc                 C   s   t jj|  ddS )Ng      4@)lr)torchoptimSGDr@   r0   r(   r(   r)   configure_optimizersa   s   z"LightningLSTM.configure_optimizers)r   rC   )rD   rE   rF   rG   r   rV   r   r   ra   rc   r	   rf   r   rk   rI   r(   r(   r&   r)   rT   G   s    


rT   )__doc__typingr   r   r   r   r   rh   torch.nnr   torch.nn.functional
functionalr7   r   torch.optimr   torch.utils.datar	   r
   lightning.pytorch.corer   #lightning.pytorch.demos.transformerr   Moduler   rG   rJ   rT   r(   r(   r(   r)   <module>   s    "