o
    i                     @   s0   d dl Z dg dfgZe jdedd ZdS )    NuI   Lääʹddjânnmest lie nuʹtt 10 000 säʹmmliʹžžed. Seeʹst pâʹjjel)	u   Lääʹddjânnmestlieu   nuʹtt10000u   säʹmmliʹžžed.u   Seeʹstu	   pâʹjjelztext,expected_tokensc                 C   s&   | |}dd |D }||ksJ d S )Nc                 S   s   g | ]}|j s|jqS  )is_spacetext).0tokenr   r   V/home/ubuntu/.local/lib/python3.10/site-packages/spacy/tests/lang/xx/test_tokenizer.py
<listcomp>   s    z+test_xx_tokenizer_basic.<locals>.<listcomp>r   )xx_tokenizerr   expected_tokenstokens
token_listr   r   r   test_xx_tokenizer_basic   s   r   )pytestXX_BASIC_TOKENIZATION_TESTSmarkparametrizer   r   r   r   r   <module>   s    