o .i@sUddlZddlmZddlmZmZddlZddlZ ddl m Z ddl m Z ddlmZdd lmZer=ddlZddlZe Zdaeeed <Gd d d eed efZdS)N)Mapping) TYPE_CHECKINGOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGcseZdZdfdd ZedeedffddZdd Zd d Z d d Z defddZ de j defddZde j ddfddZde j defddZZS) JaxFormatterNc stj||dddl}ddlm}t||r$td|dt|dt|tr+|nt| d|_ t dur<| a |j t t vrhtd|j dt t d t| dd t| d|_ ||_dS) N)featurestoken_per_repo_idr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfr rrr rr __class__U/home/ubuntu/.local/lib/python3.10/site-packages/datasets/formatting/jax_formatter.pyr's(      zJaxFormatter.__init__returnzjaxlib.xla_extension.DevicecCsddl}dd|DS)NrcSsi|]}t||qSr$)r).0rr$r$r% Fz4JaxFormatter._map_devices_to_str..)rr)rr$r$r%rBsz JaxFormatter._map_devices_to_strcsLddlddlm}ttr$r$tfddDr$|jddSS)Nrc3s<|]}t|jo|jdjko|jdjkVqdS)rN)rArrayshapedtype)r'xcolumnrr$r% Ms, z,JaxFormatter._consolidate..)axis)r jax.numpynumpyrrallstack)r!r/jnpr$r.r% _consolidateHs zJaxFormatter._consolidatecCsddl}ddlm}t|tttdfr|St|tjtj fr+t |j tjr+| Si}t|tj tj frNt |j tjrN|jjrHd|ji}nd|ji}nt|tj tj frdt |j tjrdd|ji}tjr|dtjvr|ddl}t||jjr|t|}tjrdtjvrddlm}t||r|Stdur|a| t|j!|j"|fii||j#WdS1swYdS)Nrr,PIL torchvision) VideoReader)$rr2r3rrbytesrnp characterndarray issubdtyper,tolistnumberintegerrjax_enable_x64int64int32floatingfloat32 PIL_AVAILABLEsysmodules PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior:r rdefault_devicerarrayr )r!valuerr6 default_dtyper8r:r$r$r% _tensorizeSs4 ""  "    $zJaxFormatter._tensorizecsddl}tjr#dtjvr#ddl}t||jr#| dSt |dr2t||j s2|}t|tjrJ|jtkrIfdd|DSnt|ttfr]fdd|DS|S)Nrtorchr$ __array__cg|]}|qSr$recursive_tensorizer' substructr!r$r% r)z5JaxFormatter._recursive_tensorize..crWr$rXrZr\r$r%r]r))rrTORCH_AVAILABLErIrJrUrTensorrTdetachcpur3hasattrr*rVr<r>r,objectr7rtuple)r! data_structrrUr$r\r%_recursive_tensorize~s    z!JaxFormatter._recursive_tensorizerecCst|j|ddS)NF)map_list)rrf)r!rer$r$r%rYsz JaxFormatter.recursive_tensorizepa_tablecCs$||}|j|}||SN)numpy_arrow_extractor extract_rowpython_features_decoder decode_rowrY)r!rhrowr$r$r% format_rows  zJaxFormatter.format_row jax.ArraycCs:||}|j||jd}||}||}|S)Nr)rjextract_columnrl decode_column column_namesrYr7)r!rhr/r$r$r% format_columns   zJaxFormatter.format_columncCsD||}|j|}||}|D] }|||||<q|Sri)rj extract_batchrl decode_batchrYr7)r!rhbatch column_namer$r$r% format_batchs   zJaxFormatter.format_batch)NNN)__name__ __module__ __qualname__r staticmethoddictrrr7rTrfrYpaTablerrortry __classcell__r$r$r"r%r &s +r rp)rIcollections.abcrtypingrrr3r<pyarrowrr utils.loggingrutils.py_utilsr formattingr rjaxlibrr r~__annotations__r r$r$r$r%s