o
    i                     @   sX   d dl Z d dlZdd Zejdddgdd Zejdg d	d
d Zdd ZdS )    Nc                 C   s  d}| |}|d j dksJ |d j dksJ |d j dks!J |d j d	ks*J |d
 j dks3J |d j dks<J |d j dksEJ |d j dksNJ |d j dksWJ |d j dks`J |d j dksiJ |d j dksrJ |d j dks{J |d j dksJ |d j dksJ |d  j d!ksJ |d" j d#ksJ |d$ j d%ksJ |d& j d'ksJ |d( j d)ksJ |d* j d+ksJ d S ),NzI:o :/ :'( >:o (: :) >.< XD -__- o.O ;D :-) @_@ :P 8D :1 >:( :D =| :> ....r   z:o   z:/   z:'(   z>:o   z(:   z:)   z>.<   XD   z-__-	   zo.O
   z;D   z:-)   z@_@   z:P   8D   z:1   z>:(   z:D   z=|   z:>   z....text)	tokenizerr   tokens r   Y/home/ubuntu/.local/lib/python3.10/site-packages/spacy/tests/tokenizer/test_exceptions.py test_tokenizer_handles_emoticons   s0   r   ztext,length)z108)r   )XDNr   c                 C   s   | |}t ||ksJ d S )N)lenr   r   lengthr   r   r   r   +test_tokenizer_excludes_false_pos_emoticons#   s   r#   ))u"   can you still dunk?🍕🍔😵LOLr
   )u   i💙your   )u   🤘🤘yay!r   c                 C   s*   t jdkr| |}t||ksJ d S d S )Ni )sys
maxunicoder    r!   r   r   r   test_tokenizer_handles_emoji)   s
   
r&   c                 C   sd   dD ]-}dd | d| dD d| dgksJ dd |  d| dD d| dgks/J qd S )NcfkCFKc                 S   s   g | ]}|j qS r   r   .0tr   r   r   
<listcomp>5   s    z)test_tokenizer_degree.<locals>.<listcomp>   °.c                 S   s   g | ]}|d  qS )r   r   r(   r   r   r   r+   6   s    )explain)r   ur   r   r   test_tokenizer_degree3   s   *.r0   )r$   pytestr   markparametrizer#   r&   r0   r   r   r   r   <module>   s    

