o
    .iB                     @   s|  d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ G dd	 d	ejZd
d Zdd Zdd Zeje
dk rJdd Znejjdd e
dje
dje
djfv rgdd Zeedd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zeje
dk reed%d& ZdS ejjdd e
dje
dje
djfv reed'd( ZdS dS ))zPExtends `dill` to support pickling more types and produce more consistent dumps.    N)BytesIO)CodeTypeFunctionType)version   )configc                       sF   e Zd Zejejj ZdZ	d	ddZ
 fddZdd Z  ZS )
PicklerFTc           	      C   s:  t |}|| jvrdtjv rdd l}||ju rt|t dtjv r2dd l}t	||j
r2t|t dtjv rFdd l}||ju rFt|t dtjv rsdd l}t	||jr[t|t ||ju rft|t t	||jjrst|d|}dtjv rdd l}t	||jrt|t |tu rt|d|}tjj| ||d	 d S )
Nregexr   spacytiktokentorch	_orig_modtransformers_torchdynamo_orig_callable)save_persistent_id)typedispatchsysmodulesr	   Patternpklregister_save_regexPatternr
   
issubclassLanguage_save_spacyLanguager   Encoding_save_tiktokenEncodingr   Tensor_save_torchTensor	Generator_save_torchGeneratornnModulegetattrr   PreTrainedTokenizerBase)_save_transformersPreTrainedTokenizerBaser   dillr   save)	selfobjr   obj_typer	   r
   r   r   r    r+   H/home/ubuntu/.local/lib/python3.10/site-packages/datasets/utils/_dill.pyr'      s:   








zPickler.savec                    sd   | j r	t |S zt|}W n ty(   ddlm  t| fddd}Y nw tj| | d S )Nr   Hasherc                    s     | d S )Nr   )hash)xr-   r+   r,   <lambda>R   s    z)Pickler._batch_setitems.<locals>.<lambda>key)	_legacy_no_dict_keys_sortingsuper_batch_setitemssorted	Exceptiondatasets.fingerprintr.   r&   r   )r(   items	__class__r-   r,   r6   H   s   zPickler._batch_setitemsc                 C   s"   t |turtj| | d S d S N)r   strr&   r   memoize)r(   r)   r+   r+   r,   r?   U   s   zPickler.memoize)T)__name__
__module____qualname__r&   _dillMetaCatchingDictr   r   copyr4   r'   r6   r?   __classcell__r+   r+   r;   r,   r      s    
)r   c                    s    fdd}|S )z'Register a custom reducer for the type.c                    s   | t j < | S r=   )r   r   )functr+   r,   proxy^   s   
zpklregister.<locals>.proxyr+   )rI   rJ   r+   rH   r,   r   [   s   r   c                 C   s   t |dd|  dS )zPickle an object to a file.T)recurseN)r   dumpr)   filer+   r+   r,   rL   e   s   rL   c                 C   s   t  }t| | | S )zPickle an object to a string.)r   rL   getvaluerM   r+   r+   r,   dumpsj   s   
rP   z0.3.6c                 C   s   t jj| d S r=   )r&   rC   loginfopicklermsgr+   r+   r,   rQ   s   s   rQ      z0.3.7z0.3.8c                 C   s   t jj| | d S r=   )r&   rC   loggertracerS   r+   r+   r,   rQ   |   s   c                 C   sl   t | d|  zt|f}W n ty&   ddlm} t||jdf}Y nw | jt||d t | d d S )NzSe: r   r-   r2   r)   z# Se)rQ   r7   r8   r9   r.   r/   save_reduceset)rT   r)   argsr.   r+   r+   r,   	_save_set   s   r]   c                 C   sD   dd l }t| d|  |j|jf}| j|j||d t| d d S )Nr   zRe: rY   z# Re)r	   rQ   patternflagsrZ   compile)rT   r)   r	   r\   r+   r+   r,   r      s
   r   c                 C   sL   dd l }t| d|  |j|j|j|jf}| j|j||d t| d d S )Nr   zEnc: rY   z# Enc)r   rQ   name_pat_str_mergeable_ranks_special_tokensrZ   r   )rT   r)   r   r\   r+   r+   r,   r      s
   r   c                    s   dd l  d fdd	}t| d|  |j jkr)|  j   jf}n	|   f}| j	|||d t| d d S )Nr   c                    s     | }|r||}|S r=   )
from_numpyr   )np_arraydtypetensorr   r+   r,   create_torchTensor   s   

z-_save_torchTensor.<locals>.create_torchTensorzTo: rY   z# Tor=   )
r   rQ   rg   bfloat16detachtofloatcpunumpyrZ   )rT   r)   rj   r\   r+   ri   r,   r      s    r   c                    sL   dd l   fdd}t| d|  | f}| j|||d t| d d S )Nr   c                    s      }||  |S r=   )r   	set_state)state	generatorri   r+   r,   create_torchGenerator   s   
z3_save_torchGenerator.<locals>.create_torchGeneratorzGe: rY   z# Ge)r   rQ   	get_staterZ   )rT   r)   rt   r\   r+   ri   r,   r       s   
r    c                    sP   dd l   fdd}t| d|  |j| f}| j|||d t| d d S )Nr   c                    s(    j | d d }|| }||S )Nnlplang)utilget_lang_classfrom_config
from_bytes)r   byteslang_cls	lang_instr
   r+   r,   create_spacyLanguage   s   

z1_save_spacyLanguage.<locals>.create_spacyLanguagezSp: rY   z# Sp)r
   rQ   r   to_bytesrZ   )rT   r)   r   r\   r+   r   r,   r      s   r   c                 C   sX   t | d|  |j}d|v rt|d tri |d< | jt|d||d t | d d S )NzTok: cacher+   )rr   r)   z# Tok)rQ   __dict__
isinstancedictrZ   r   )rT   r)   rr   r+   r+   r,   r%      s   r%   c                 C   s^  t jjd|  |jds.t|jtj	j
dkr)|jtj	j
d ds.|jdkr0dntj	|j}d}t jjrt|drc|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}n;|j|j|j|j|j|j|j|j|j||j||j|j|jf}n|j|j|j|j|j|j|j|j||j||j|j|jf}| jt||d	 t jjd
 dS )z
        From dill._dill.save_code
        This is a modified version that removes the origin (filename + line no.)
        of functions created in notebooks or shells for example.
        zCo: <   
ipykernel_<lambda> co_posonlyargcountrY   # CoN)r&   rC   rQ   rR   co_filename
startswithlensplitospathsepco_namebasenamePY3hasattrco_argcountr   co_kwonlyargcount
co_nlocalsco_stacksizeco_flagsco_code	co_constsco_namesco_varnames	co_lnotabco_freevarsco_cellvarsrZ   r   rT   r)   r   co_firstlinenor\   r+   r+   r,   
_save_code   s   



r   c                 C   s(  t jj| d| |jds-t|jtj	j
dkr(|jtj	j
d ds-|jdkr/dntj	|j}d}t|drh|j|j|j|j|j|j|j|j|j|j|j||j|j||j|j|j|j|j|jf}nt|d	r|j|j|j|j|j|j|j|j|j|j|j||j|j||j|j|j|jf}nlt|d
r|j|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}nDt|dr|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}n|j|j|j|j|j|j|j|j|j||j||j|j|jf}| j t jj!||d t jj| d d S )NzCo: %sr   r   r   r   r   r   co_endlinetableco_exceptiontableco_linetabler   rY   r   )"r&   rC   rW   rX   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   co_qualnamer   r   co_columntabler   r   r   rZ   _create_coder   r+   r+   r,   	save_codeA  s   






r   )__doc__r   r   ior   typesr   r   r&   	packagingr   r   r   r   r   rL   rP   DILL_VERSIONparserQ   releaser[   r]   r   r   r   r    r   r%   r   r   r+   r+   r+   r,   <module>   sN   @





		
^


