o
    c²“i2  ã                   @   sX   d dl Z d dlmZ d dlmZ ededee fdd„ƒZedededefd	d
„ƒZ	dS )é    N)ÚList)ÚDeveloperAPIÚvalueÚreturnc                 C   s
   |   d¡S )z*Tokenize a string using a split on spaces.ú )Úsplit)r   © r   úP/home/ubuntu/.local/lib/python3.10/site-packages/ray/data/preprocessors/utils.pyÚsimple_split_tokenizer   s   
r
   Únum_featuresc                 C   s,   t | ƒ ¡ }t |¡}t| ¡ dƒ}|| S )z6Deterministically hash a value into the integer space.é   )ÚstrÚencodeÚhashlibÚsha1ÚintÚ	hexdigest)r   r   Úencoded_valueÚhashed_valueÚhashed_value_intr   r   r	   Úsimple_hash   s   
r   )
r   Útypingr   Úray.util.annotationsr   r   r
   Úobjectr   r   r   r   r   r	   Ú<module>   s    