o ]۷iP@s:zddlZdZWn eydZYnwGdddZdS)NTFc@s<eZdZdedefddZddZddZd efd d Zd S) TokenList use_bitmask vocab_sizecCsH||_|r ts td|jr|dd}tj|tjd|_dSg|_dS)Nz'LMFormatEnforcer bitmasks require torch )dtype)r _HAS_TORCH ValueErrortorchzerosint32allowed_tokens)selfrr tensor_sizerP/home/ubuntu/vllm_env/lib/python3.10/site-packages/lmformatenforcer/tokenlist.py__init__s  zTokenList.__init__cCs@|jr|d?}|d@}|j|d|>O<dS|j|dSNr)rr appendrtoken_id element_index bit_indexrrrrs zTokenList.appendcCsj|jr-t|tjrtj|j|dS|D]}|d?}|d@}|j|d|>O<qdS|j|dSr)r isinstancer Tensor bitwise_or_r extend)r token_idsrrrrrrrs zTokenList.extendreturncCs6|jr|d}|d}|j|d|>@dkS||jvS)Nrrr)rr rrrris_token_allowed)s  zTokenList.is_token_allowedN) __name__ __module__ __qualname__boolintrrrr!rrrrrs   r)r r ImportErrorrrrrrs