o
    }oiq                     @   s  d dl mZmZ d dlmZ zBd dlmZ d dlm	Z	 d dl
mZ ee	efD ]Zeedd dD d	 q&d dlmZ eeed
ddedddedddgd	 W n	 eyY   Y nw z#d dlmZmZmZ eeedgd	 eeed
gd	 ee W dS  ey   Y dS w )    )DirOrStringArtifactFileArtifact)track_io)AutoTokenizer)LlamaTokenizer)LlamaTokenizerFastc                 C   s   g | ]}t |d dqS )Frequired)r   ).0	attr_name r   N/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/io/registry.py
<listcomp>   s    
r   )
vocab_filemerges_filetokenizer_filename_or_path)	artifactsr   Fr   r   pretrained_model_name)ByteLevelTokenizerSentencePieceTokenizerTiktokenTokenizer
model_pathN)nemo.lightning.io.artifactr   r   nemo.lightning.io.mixinr   transformersr   HfAutoTokenizer,transformers.models.llama.tokenization_llamar   1transformers.models.llama.tokenization_llama_fastr   cls"nemo.collections.common.tokenizersImportErrorr   r   r   r   r   r   r   <module>   s@   



