o
     ¢i1  ã                   @   s0   d dl Z dg d¢fgZe j de¡dd„ ƒZdS )é    Nuh   Askush nuk mund tâ€™i nÃ«nshtrohet torturÃ«s ose dÃ«nimeve ose trajtimeve Ã§njerÃ«zore ose poshtÃ«ruese.)ÚAskushÚnukÚmundu   tâ€™iu   nÃ«nshtrohetu	   torturÃ«sÚoseu	   dÃ«nimever   Ú
trajtimeveu   Ã§njerÃ«zorer   u   poshtÃ«rueseÚ.ztext,expected_tokensc                 C   s&   | |ƒ}dd„ |D ƒ}||ksJ ‚d S )Nc                 S   s   g | ]}|j s|j‘qS © )Úis_spaceÚtext)Ú.0Útokenr   r   úV/home/ubuntu/.local/lib/python3.10/site-packages/spacy/tests/lang/sq/test_tokenizer.pyÚ
<listcomp>   s    z+test_sq_tokenizer_basic.<locals>.<listcomp>r   )Úsq_tokenizerr
   Úexpected_tokensÚtokensÚ
token_listr   r   r   Útest_sq_tokenizer_basic   s   r   )ÚpytestÚSQ_BASIC_TOKENIZATION_TESTSÚmarkÚparametrizer   r   r   r   r   Ú<module>   s    ýÿ