o 8wi@sUddlZddlmZddlmZmZddlZddlZ ddl m Z ddl m Z ddlmZdd lmZer=ddlZddlZe Zdaeeed <Gd d d eed efZdS)N)Mapping) TYPE_CHECKINGOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGcseZdZdfdd ZedeedffddZdd Zd d Z d d Z defddZ de j defddZde j ddfddZde j defddZZS) JaxFormatterNc stj||dddl}ddlm}t||r$td|dt|dt|tr+|nt| d|_ t dur<| a |j t t vrhtd|j dt t d t| dd t| d|_ ||_dS) N)featurestoken_per_repo_idr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfr rrr rr __class__^/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/datasets/formatting/jax_formatter.pyr's(      zJaxFormatter.__init__returnzjaxlib.xla_extension.DevicecCsddl}dd|DS)NrcSsi|]}t||qSr$)r).0rr$r$r% Fz4JaxFormatter._map_devices_to_str..)rr)rr$r$r%rBsz JaxFormatter._map_devices_to_strcsLddlddlm}ttr$r$tfddDr$|jddSS)Nrc3s<|]}t|jo|jdjko|jdjkVqdS)rN)rArrayshapedtype)r'xcolumnrr$r% Ms, z,JaxFormatter._consolidate..)axis)r jax.numpynumpyrrallstack)r!r/jnpr$r.r% _consolidateHs zJaxFormatter._consolidatec Csddl}ddlm}t|tttdfr|St|tjtj fr+t |j tjr+| Si}t|tj tj frNt |j tjrN|jjrHd|ji}nd|ji}nt|tj tj frdt |j tjrdd|ji}tjr|dtjvr|ddl}t||jjr|t|}tjrdtjvrddlm}t||r|Stjrdtjvrddlm }m!}t|||fr|St"dur|#a"|$t"|j%|j&|fii||j'WdS1swYdS)Nrr,PIL torchvision) VideoReader torchcodec) AudioDecoder VideoDecoder)(rr2r3rrbytesrnp characterndarray issubdtyper,tolistnumberintegerrjax_enable_x64int64int32floatingfloat32 PIL_AVAILABLEsysmodules PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior:TORCHCODEC_AVAILABLEtorchcodec.decodersr<r=r rdefault_devicerarrayr ) r!valuerr6 default_dtyper8r:r<r=r$r$r% _tensorizeSs< ""  "    $zJaxFormatter._tensorizecsddl}tjr#dtjvr#ddl}t||jr#| dSt |dr2t||j s2|}t|tjrJ|jtkrIfdd|DSnt|ttfr]fdd|DS|S)Nrtorchr$ __array__cg|]}|qSr$recursive_tensorizer' substructr!r$r% r)z5JaxFormatter._recursive_tensorize..cr\r$r]r_rar$r%rbr))rrTORCH_AVAILABLErLrMrZrTensorrYdetachcpur3hasattrr*r[r?rAr,objectr7rtuple)r! data_structrrZr$rar%_recursive_tensorizes    z!JaxFormatter._recursive_tensorizerjcCst|j|ddS)NF)map_list)rrk)r!rjr$r$r%r^sz JaxFormatter.recursive_tensorizepa_tablecCs$||}|j|}||SN)numpy_arrow_extractor extract_rowpython_features_decoder decode_rowr^)r!rmrowr$r$r% format_rows  zJaxFormatter.format_row jax.ArraycCs:||}|j||jd}||}||}|S)Nr)roextract_columnrq decode_column column_namesr^r7)r!rmr/r$r$r% format_columns   zJaxFormatter.format_columncCsD||}|j|}||}|D] }|||||<q|Srn)ro extract_batchrq decode_batchr^r7)r!rmbatch column_namer$r$r% format_batchs   zJaxFormatter.format_batch)NNN)__name__ __module__ __qualname__r staticmethoddictrrr7rYrkr^paTablerrtryr~ __classcell__r$r$r"r%r &s 0r ru)rLcollections.abcrtypingrrr3r?pyarrowrr utils.loggingrutils.py_utilsr formattingr rjaxlibrr r__annotations__r r$r$r$r%s