o .i>l@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZmZddlmZddlmZddlmZdd lmZdd lmZmZmZmZmZdd lmZddl Z ddl!Z ddl"Z#dd l$m%Z%d dl&m'Z'd dl(m)Z)ddl&m*Z*dd l&m%Z+ddl,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2zddlm3Z3ddl4Z5ddl4m6Z6Wne7ydZ5Z6Z3Ynwe*8e9Z:ej;Ze?fde>fddZ@ddZAde?de?deeBe?e?ffd d!ZCd"d#ZDed$d%ZEedkd'e>fd(d)ZFd*d+ZGd,d-ZHd.d/ZId0d1ZJGd2d3d3eBZKGd4d5d5eLZMd6d7ZN & 8 & &  & 9  8 dld:eegefd;edeOd?eOd@ee>dAe>dBeOdCee>dDeePdEeOdFee?defdGdHZQGdIdJdJZRdmdLdMZSdNe?dePe?ee?ffdOdPZTdQe?de2fdRdSZUdTe?dePe?e?e?e?ffdUdVZVdWdXZWedYZXdZejYd[ed\e eXfd]eBde>fd^d_ZZd`eej[j\e j[j\fde]e>fdadbZ^d`eej[j\e j[j\fd[ed\e eXfdce eBde eXfdddeZ_edfZ`dge e`dhe>de eae`fdidjZbdS)nz'Some python utils function and classes.N)Iterable)contextmanager)fields is_dataclass)Manager)Path)Empty) disk_usage)AnyCallableOptionalTypeVarUnion)urlparse)tqdm)config) parallel_map)logging)Picklerdumpdumps pklregister)FileLock)Final)LiteralcCsX|sdSgd}t|}|D]\}}||}|dkr$|dd|Sqt|dS)a6Returns a human readable size string. If size_in_bytes is None, then returns "Unknown size". For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`. Args: size_in_bytes: `int` or `None`, the size, in bytes, that we want to format as a human-readable size string. z Unknown size))PiB )TiB)GiB@)MiB)KiBg?z.2f z bytes)floatint) size_in_bytes _NAME_LISTname size_bytesvaluer/K/home/ubuntu/.local/lib/python3.10/site-packages/datasets/utils/py_utils.pysize_strKs  r1sizereturncCst|tr|S|drt|dddS|dr)t|dddS|dr:t|dddS|d rKt|ddd S|d r\t|ddd S|d rxt|ddd}|drv|dS|S|drt|ddd}|dr|dS|S|drt|ddd}|dr|dS|S|drt|ddd}|dr|dS|S|drt|ddd}|dr|dS|Std|d)a) Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes). Args: size (`int` or `str`): The size to convert. Will be directly returned if an `int`. Example: ```py >>> convert_file_size_to_int("1MiB") 1048576 ``` PIBNrTIBr GIBr"MIBr$KIBr&PBlI5bTBlJ)GBiʚ;MBi@BKBz`size=zM` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.) isinstancer)upperendswith ValueError)r2int_sizer/r/r0convert_file_size_to_intcs8 rHcCsb|ddddddddd d d d d dddddddd ddS)N\z\\.z\.*z.*+z\+z///(z\()z\)|z\|^z\^$z\$?)replacerstrip)patternr/r/r0glob_pattern_to_regexs rWstringrVcCs^tdd|}tdd|}t||}|durdSt|}td|}tt||}|S)aUn-format a string using a python f-string pattern. From https://stackoverflow.com/a/36838374 Example:: >>> p = 'hello, my name is {name} and I am a {age} year old {what}' >>> s = p.format(name='cody', age=18, what='quarterback') >>> s 'hello, my name is cody and I am a 18 year old quarterback' >>> string_to_dict(s, p) {'age': '18', 'name': 'cody', 'what': 'quarterback'} Args: string (str): input string pattern (str): pattern formatted like a python f-string Returns: Optional[dict[str, str]]: dictionary of variable -> value, retrieved from the input using the pattern, or `None` if the string does not match the pattern. z{([^:}]+)(?::[^}]+)?}z{\1}z{(.+?)}z (?P<_\1>.+)N)resubsearchlistgroupsfindalldictzip)rXrVregexresultvalueskeys_dictr/r/r0string_to_dicts   rfcs>ddfddt|ts|st|d|S)zbConvert an object to its dictionary representation recursively. cSst|o t|t SN)rrCtypeobjr/r/r0_is_dataclass_instancesz&asdict.._is_dataclass_instancecs|r+i}t|D]}t||j}|jr#||jks#|jddr(|||j<q |St|trBt |drBt |fdd|DSt|t tfrVt |fdd|DSt|t rffdd | DSt|S) N$include_in_asdict_even_if_is_defaultF_fieldscsg|]}|qSr/r/.0v _asdict_innerr/r0 sz1asdict.._asdict_inner..c3s|]}|VqdSrgr/rnrqr/r0 z0asdict.._asdict_inner..csi|] \}}||qSr/r/rokrprqr/r0 z1asdict.._asdict_inner..)rgetattrr,initdefaultmetadatagetrCtuplehasattrrhr\r_itemscopydeepcopy)rjrbfr.rrrkr/r0rrs    zasdict.._asdict_innerz is not a dict or a dataclass)rCr_ TypeErrorrir/rr0asdicts rc csBt||d}t|||z dVWt|||dSt|||w)z%Temporarily assign obj.attr to value.N)rzsetattr)rjattrr.originalr/r/r0temporary_assignments   rFseedc cstj}tj||r0tjr0ddl}|j}|j||j r0|j }|j ||rotj roddl}ddlm}|j} |jj|} |j| |sYtd|} | j} t| d} | rj| j}| |z@dVWtj||rtjr|j||j r|j ||rtj r|j| | | _| r|| _dSt| ddSdSdStj||rtjr|j||j r|j ||rtj r|j| | | _| r|| _wt| dwww)zUTemporarily set the random seed. This works for python numpy, pytorch and tensorflow.rN)contextzBSetting random seed for TensorFlow is only available in eager mode_rng)nprandom get_staterrTORCH_AVAILABLEtorch get_rng_state manual_seedcuda is_availableget_rng_state_allmanual_seed_all TF_AVAILABLE tensorflowtensorflow.python.eagerrget_global_generator Generator from_seedset_global_generatorexecuting_eagerlyrF_seedrr_set_global_seed set_state set_rng_stateset_rng_state_alldelattr)r set_pytorchset_tensorflownp_stater torch_statetorch_cuda_statestf tfpycontexttf_statetemp_gen tf_contexttf_seedtf_rng_initializedtf_rngr/r/r0 temp_seedsd                             rccs.t}|D]}||vr|||VqdS)z=Iterate over iterable and return only unique values in order.N)setadd)rcseenr.r/r/r0 unique_values*s rcsfdd}|S)z4If the value is None, return None, else call `func`.cs|dur|SdSrgr/)r.funcr/r0wrapper6sz'no_op_if_value_is_null..wrapperr/)rrr/rr0no_op_if_value_is_null3s rcCs*t|D]\}}|dur||fSqdS)zwReturn the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index.N)N) enumerate)iterableir.r/r/r0first_non_null_value<s  rc's6ttj|D]tfdd|DfVqdS)z9Iterate over items of dictionaries grouped by their keys.c3s|]}|VqdSrgr/)rodkeyr/r0rtHruzzip_dict..N)r itertoolschainr)dictsr/rr0zip_dictDsrcs8eZdZdZfddZfddZfddZZS)NonMutableDictzDict where keys can only be added but not modified. Will raise an error if the user try to overwrite one key. The error message can be customized during construction. It will be formatted using {key} for the overwritten key. cs0|dd|_|r tdtj|i|dS)N error_msgz$Try to overwrite existing key: {key}z1NonMutableDict cannot be initialized with kwargs.)pop _error_msgrFsuper__init__)selfargskwargs __class__r/r0rSszNonMutableDict.__init__cs(||vr t|jj|dt||S)Nr)rFrformatr __setitem__)rrr.rr/r0r\szNonMutableDict.__setitem__cs@tfdd|Drtjjtt|@dt|S)Nc3s|]}|vVqdSrgr/)rorwrr/r0rtbruz(NonMutableDict.update..r)anyrFrrrrupdate)rotherrrr0ras zNonMutableDict.update)__name__ __module__ __qualname____doc__rrr __classcell__r/r/rr0rKs   rc@seZdZdZdddZdS) classpropertyz5Descriptor to be used as decorator for @classmethods.NcCs|jd|Srg)fget__get__)rrjobjtyper/r/r0rjzclassproperty.__get__rg)rrrrrr/r/r/r0rgsrc s|\}}}}t|ts!t|s!r|gdS|SrDt|tsDt|rDtfdd|DrDfddt|DS|durSttjkrSt|durj|sjtddt j Drjt dd d d t|trs| n|}|dur|dur|dnd d t |n|}t|||d |dO}t|trfdd|DWdSfdd|D}t|tr|WdSt|trt|WdSt|WdS1swYdS)zEApply a function recursively to each element of a nested data struct.rc3s |] }t|tf VqdSrg)rCr_rn)typesr/r0rt|sz%_single_map_nested..csg|] }|D]}|qqSr/r/)robatch mapped_item)functionr/r0rs~ryz&_single_map_nested..Ncss|]}d|jvVqdS)notebookN)r)rotqdm_clsr/r/r0rtsr'T)endflush#rj)disablepositionunitdescc s*i|]\}}|t|dddfqSNT_single_map_nestedrv batch_sizebatchedrrr/r0rxsz&_single_map_nested..c $g|]}t|dddfqSrrrnrr/r0rss$)rCr_all iter_batchedr get_verbosityWARNINGset_verbosity_warningrr__mro__printrstrhf_tqdmr\rrarray) r data_structrank disable_tqdmr pbar_iterable pbar_descpbarmappedr/rr0rnsF ,    $rTrBrr dict_onlymap_list map_tuple map_numpynum_procparallel_min_lengthrrrrrc  sbdur"g|s|rt|rt|rtjtt|ts=t|s=r1|g}|} r;| d} | St|trHt|n|durPdtfddDrkfddD} ndkrsdksyt krrdusdkrt t t t dkdtt fd dt | | d D} rd d| D} nStFtjd d tdrdusψdkrt t t dktt t| | t } rdd| D} Wdn 1swYt|trtt|| St|tr"| St|tr,t| St| S)a4 Apply a function recursively to each element of a nested data struct. Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to `parallel_min_length`. Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``. Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and multiprocessing is used. Args: function (`Callable`): Function to be applied to `data_struct`. data_struct (`Any`): Data structure to apply `function` to. dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in `data_struct`. map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict` values). map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides `dict` values). map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides `dict` values). num_proc (`int`, *optional*): Number of processes. The level in the data struct used for multiprocessing is the first level that has smaller sub-structs, starting from the root. parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel processing. batched (`bool`, defaults to `False`): Provide batch of items to `function`. batch_size (`int`, *optional*, defaults to `1000`): Number of items per batch provided to `function` if `batched=True`. If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`. types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their elements. disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar. desc (`str`, *optional*): Prefix for the tqdm progressbar. Returns: `Any` Nrrc3s*|]}t|ot|tkVqdSrg)rClenrn)rrr/r0rts(zmap_nested..c s"g|] }t|dqS))rrrrrrr) map_nestedrorj)rrrrrrr/r0rss zmap_nested..rc rrrrrr/r0rss)rrcSg|] }|D]}|qqSr/r/ro mapped_batchrr/r/r0rsignorezL.* is experimental and might be subject to breaking changes in the future\.$)messagecategorycSrr/r/rr/r/r0rsr)appendr\rrndarrayrCr_rcrrmaxr)rrwarningscatch_warningsfilterwarnings UserWarningrrr`rdr)rrr r r rrrrrrrrr r/)rrrrrrrr0rsv=     &       rc@s eZdZdddZdddZdS)NestedDataStructureNcCs|dur ||_dSg|_dSrg)datarr#r/r/r0r$szNestedDataStructure.__init__csT|dur|nj}t|trt|St|ttfr'fdd|DS|gS)Ncs g|] }|D]}|q qSr/)flatten)roitem flattenedrr/r0rs,s z/NestedDataStructure.flatten..)r#rCr_r%r\rcrr$r/rr0r%'s  zNestedDataStructure.flattenrg)rrrrr%r/r/r/r0r"#s r"rJcCs2zttj|j}W||kStyYdSwr)r ospathabspathfreeOSError) needed_bytes directory free_bytesr/r/r0has_sufficient_disk_space1s  r0url_pathcCst|}d}|jdvr[|jdkr[d|vr+|ds!td|d|dd}||fS|jd d}d |vr;|d n|d f\}}|d \}}d |d |d|d}|d|}||fS)zMConvert a link to a file on a github repo in a link to the raw github object.N)httphttpss3z github.comblobz.pyzExternal import from github at z) should point to a file ending with '.py'rawrz/tree/masterrMzhttps://github.com/z /archive/z.zip-)rschemenetlocrErFrTr)split)r1parsed sub_directory github_path repo_infobranch repo_owner repo_namer/r/r0_convert_github_url9s  rCimportable_local_filecCs$tt|jj}|d}t|S)Nz.lock)rrresolveparentr)rDimportable_directory_path lock_pathr/r/r0lock_importable_fileLsrI file_pathc sg}t|dd}||Wdn1swYtd|dg}d}|D]}td|}t|dkr?| }|rBq.tjd |tj d dur]tjd |tj d dur]q. drt fd d |Drnq. dr d}t |\}}| d d||fq. dr| d d ddfq. dr d}| d d|dfq.| d d ddfq.|S)aFind whether we should import or clone additional files for a given processing script. And list the import. We allow: - library dependencies, - local dependencies and - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. external dependencies will be downloaded (and extracted if needed in the dataset folder). We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. Note that only direct import in the dataset processing script will be handled We don't recursively explore the additional import to download further files. Example:: import tensorflow import .c4_utils import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset zutf-8)encodingNz Checking z for additional imports.Fz[\s\S]*?"""[\s\S]*?rz=^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*))flagszQ^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)c3s"|] }|ddkVqdS)rrN)group)roimpmatchr/r0rts zget_imports..externalrinternallibrary)openextend readlinesloggerdebugrYr^rrP MULTILINErMrrCr) rJlinesrimportsis_in_docstringlinedocstr_start_matchr1r=r/rOr0 get_importsUsJ          r`cCs(t|j|j|j|j|j}|j|_|Srg)r FunctionType__code__ __globals__r __defaults__ __closure____kwdefaults__)rrbr/r/r0copyfuncsrgYqueuer.rcCs*t|di|D] \}}||q |S)Nr/)rput)rirrrrbr/r/r0_write_generator_to_queues rkpoolcCsdd|jDS)NcSsh|]}|jqSr/)pid)rorr/r/r0 sz _get_pool_pid..)_pool)rlr/r/r0 _get_pool_pidrrpkwargs_iterablec #st}d}ttjjrtntj}|_}|fdd|D}z; z jddVWnt yIt dd|DrG rGYnYnwt|krVd}t d q(W|sad d|Dn |sld d|DwwWddS1sxwYdS) NFcsg|] }t|fqSr/) apply_asyncrk)rorrrlrir/r0rssz&iflatmap_unordered..T皙?timeoutcss|]}|VqdSrg)readyro async_resultr/r/r0rtruz%iflatmap_unordered..zkOne of the subprocesses has abruptly died during map operation.To debug the error, disable multiprocessing.cSsg|]}|jddqS)rtru)r~rxr/r/r0rss) rprCmultiprocessingrlPoolr multiprocessQueuer~rrempty RuntimeError)rlrrqinitial_pool_pid pool_changed manager_clsmanager async_resultsr/rsr0iflatmap_unorderedsB   "rTrnccsX|dkr td|g}|D]}||t||kr"|Vg}q|r*|VdSdS)NrzInvalid batch size )rFrr)rrrr&r/r/r0rs   r)FF) FTFFNrFrBNTN)rJ)crr functoolsrmultiprocessing.poolrzr(rirYrrcollections.abcr contextlibr dataclassesrrrpathlibrrshutilr typingr r r r r urllib.parserr|multiprocess.poolnumpyr tqdm.autorrrparallelrrr_dillrrrr _filelockrrtyping_extensions_typing_extensionsr ImportError get_loggerrrX lru_cachememoizer1r)rrHrWr_rfrrrrrrrrpropertyrrboolrrr"r0rCrIr`rgrhr}rkrlr{rrprrr\rr/r/r/r0s                ," '  5  /         I(& "&