o
    ﰓiw
                     @   s   d dl mZmZmZmZ zd dlZd dlmZ W n ey#   edw ddl	m
Z
 ddlmZmZ ded	eeeeef  fd
dZddeded	efddZG dd dZddgZdS )    )ListSetTupleUnionN)ExLlamaV2TokenizerzJexllamav2 is not installed. Please install it with "pip install exllamav2"   )CharacterLevelParser)TokenEnforcerTokenEnforcerTokenizerData	tokenizerreturnc                 C   s   | j  }t| j }|| j| j| j| j	h | 
 }g }t|D ]}||v r*q#|| }t|dko9|d dk}||||f q#|S )Nr    )r   
vocab_sizesetextended_id_to_piecekeysupdatebos_token_ideos_token_idpad_token_idunk_token_idget_id_to_piece_listrangelenappend)r   r   all_special_idsid_to_pieceregular_tokens	token_idxdecodedis_word_start_token r!   [/home/ubuntu/.local/lib/python3.10/site-packages/lmformatenforcer/integrations/exllamav2.py_build_regular_tokens_list   s   
r#   Fuse_bitmaskc                    s6   t  }dtt dtf fdd}t|| j| jS )Ntokensr   c                    s   t j| t jd} |S )N)dtype)torchtensorlongdecode)r%   r(   r   r!   r"   _decode   s   
z4build_token_enforcer_tokenizer_data.<locals>._decode)r#   r   intstrr
   r   actual_vocab_size)r   r$   r   r,   r!   r+   r"   #build_token_enforcer_tokenizer_data   s   r0   c                   @   s   e Zd ZU dZee ed< dedee	e
f fddZdedd	fd
dZdejdd	fddZdd Zdeee ee f fddZd	S )ExLlamaV2TokenEnforcerFilterzkExLlamaV2Sampler.Settings.filters filter that uses the token enforcer to only allow format-complying tokenstoken_sequencecharacter_level_parsertokenizer_datac                 C   s(   t |tr	t|}t||| _g | _d S N)
isinstancer   r0   r	   token_enforcerr2   )selfr3   r4   r!   r!   r"   __init__*   s   

z%ExLlamaV2TokenEnforcerFilter.__init__
prefix_strr   Nc                 C   s
   g | _ d S r5   )r2   )r8   r:   r!   r!   r"   begin2   s   
z"ExLlamaV2TokenEnforcerFilter.begintokenc                 C   s   | j t|d d  d S )Nr   )r2   r   r-   )r8   r<   r!   r!   r"   feed5   s   z!ExLlamaV2TokenEnforcerFilter.feedc                 C   s   | S r5   r!   )r8   r!   r!   r"   clone8   s   z"ExLlamaV2TokenEnforcerFilter.clonec                 C   s   | j | jj}t|t fS r5   )r7   get_allowed_tokensr2   allowed_tokensr   )r8   r@   r!   r!   r"   next;   s   z!ExLlamaV2TokenEnforcerFilter.next)__name__
__module____qualname____doc__r   r-   __annotations__r   r   r   r
   r9   r.   r;   r'   Tensorr=   r>   r   r   rA   r!   r!   r!   r"   r1   &   s   
 

"r1   )F)typingr   r   r   r   r'   	exllamav2r   ImportErrorcharacterlevelparserr   tokenenforcerr	   r
   r-   r.   boolr#   r0   r1   __all__r!   r!   r!   r"   <module>   s     
