o bi@sddlZddlmZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z m Z mZddlZddlZddlmZddlmZddlmZmZmZmZddlmZmZddl m!Z!dd l"m#Z#m$Z$dd l%m&Z&eZ'ed Z(ed Z)d Z*e+e,Z-e#Gddde.Z/ddZ0e#dee1ee eej2ffdej3fddZ4e#dddee eej5efde1dej2fddZ6dee eej5fde1dej2fddZ7de eej2ej8ffddZ9dej5d ej:de1fd!d"Z;dee eej5fde ej<fd#d$Z=d%d%d%d%d%d%d%d&d'd(d(d(d(d) Z>dee eej5fde ej<fd*d+Z?e#d,d-Z@e#d.d/ZAe#d0d1ZBGd2d3d3ejCejDZEe$d4dGd5d6d6eEZFe$ddGd7d8d8eEZGe0rhe$d4dGd9d:d:ejHZIGd;d<d<ZJe$d4dGd=d>d>eJejKZLe$ddGd?d@d@ejCZMe$ddGdAdBdBeJejKZNdCej5dDe ej5deOfdEdFZPdGej5deQfdHdIZRdJdKZSdLdMZTz!eUeFdNeVeUeGdNeVeUeMeVdWdSejWyYdSw)ON)datetime)AnyDictIterableListOptionalSequenceTupleUnion)parse)get_pyarrow_version)"_is_ndarray_variable_shaped_tensorcreate_ragged_ndarray_should_convert_to_tensor ArrayLike)convert_to_numpy _convert_datetime_to_np_datetime)log_once) DeveloperAPI PublicAPI) INT32_MAXz9.0.0z13.0.0cs*eZdZdZdZdeffdd ZZS)ArrowConversionErrorz=Error raised when there is an issue converting data to Arrow.data_strcs:t||jkr|d|jd}d|}t|dS)Nz...z Error converting data to Arrow: )lenMAX_DATA_STR_LENsuper__init__)selfrmessage __class__X/home/ubuntu/.local/lib/python3.10/site-packages/ray/air/util/tensor_extensions/arrow.pyr0s zArrowConversionError.__init__)__name__ __module__ __qualname____doc__rstrr __classcell__r#r#r!r$r*srcCstdupttkS)z Whether Arrow ExtensionScalars support subclassing in the current pyarrow version. This returns True if the pyarrow version is 9.0.0+, or if the pyarrow version is unknown. N)PYARROW_VERSION#MIN_PYARROW_VERSION_SCALAR_SUBCLASSr#r#r#r$)_arrow_extension_scalars_are_subclassable7s r-pydictreturnc Cs6ztj|WSty}ztt||d}~ww)z} Convert a Python dictionary to a pyarrow Table. Raises: ArrowConversionError: if the conversion fails. N)paTable from_pydict Exceptionrr))r.er#r#r$pyarrow_table_from_pydictEs  r5alpha) stability column_values column_namec Csz t|dkrt||rddlm}|t||WSt||WSty}zWddlm }ddl m }m }| j}|sJd}dtdtd} n |d upO|}|rUd } nd } |d us]|sqtd rqtjd |d|d| |d|st||WYd }~Sd }~ww)aConverts provided NumPy `ndarray` into PyArrow's `array` while utilizing both Arrow's natively supported types as well as custom extension types: - ArrowTensorArray (for tensors) - ArrowPythonObjectArray (for user-defined python class objects, as well as any python object that aren't represented by a corresponding Arrow's native scalar type) rArrowTensorArray DataContext)ArrowPythonObjectArray_object_extension_type_allowedFz[skipping fallback to serialize as pickled python objects (due to unsupported Arrow version z, min required version is )Nz3falling back to serialize as pickled python objectsz~skipping fallback to serialize as pickled python objects (due to DataContext.enable_fallback_to_arrow_object_ext_type = False)0_fallback_to_arrow_object_extension_type_warningzFailed to convert column 'z' into pyarrow array due to: z; exc_info)rr$ray.data.extensions.tensor_extensionr; from_numpyr _convert_to_pyarrow_native_arrayrray.datar=$ray.data.extensions.object_extensionr>r? get_current(enable_fallback_to_arrow_object_ext_typer+r,rloggerwarning from_objects) r8r9r;acer=r>r?enable_fallback_config object_ext_type_fallback_allowedobject_ext_type_detailr#r#r$convert_to_pyarrow_arrayUsZ      rRc Cszlt|dkrt|dtrt|}t|dkr(t|dtjtjfr(t|WSt|}|r@tj |r@t|t j r@t |||}ttdd|d|dt|dkret|dtjredd|D}tj||dWSty}ztt||d }~ww) zConverts provided NumPy `ndarray` into PyArrow's `array` while only utilizing Arrow's natively supported types (ie no custom extension types)rTRACEzInferred dtype of 'z' for column ''cSsg|] }|jr |ndqSN)is_valid.0vr#r#r$ z4_convert_to_pyarrow_native_array..typeN)r isinstancerrr0Array ChunkedArray_combine_as_list_array_infer_pyarrow_typetypes is_timestampnpndarray-_coerce_np_datetime_to_pa_timestamp_precisionrKloglogging getLevelName ListScalararrayr3rr))r8r9pa_typer4r#r#r$rFs0   rFcCsddd|D}tjtdgt|gtd}ttjdd|D}tj ||t |j S)z7Combines list of Arrow arrays into a single `ListArray`cSsg|]}t|qSr#)rrWr#r#r$rZz*_combine_as_list_array..rr\cSs$g|]}t|tjr |jn|gqSr#)r^r0r`chunksrWr#r#r$rZs) r0rlre concatenatecumsumint32 concat_arrays itertoolschain ListArray from_arrayslist_r])r8lensoffsetscombinedr#r#r$ras$ radtypecCspt|jtjs Jt|j\}}|j}||kr6|d|d}td|dr6t d|d|d|S)Nz datetime64[]column__timestamp_warningz Converting a z$ precision datetime NumPy array to 'z' precision Arrow timestamp. This conversion occurs because Arrow supports fewer precisions than Arrow and might result in a loss of precision or unrepresentable values.) re issubdtyper| datetime64 datetime_dataunitastyperrKrL)r8r|r9numpy_precision_arrow_precisionr#r#r$rgsrgcst|dkrdSt|}|dur|St|}dtdtfddtj|r5tfdd|Dr5t Stj |rJtfd d|DrJt S|S) aInfers target Pyarrow `DataType` based on the provided columnar values. NOTE: This is a wrapper on top of `pa.infer_type(...)` utility performing up-casting of `binary` and `string` types to corresponding `large_binary` and `large_string` types in case any of the array elements exceeds 2Gb in size therefore making it impossible for original types to accommodate such values. Unfortunately, for unknown reasons PA doesn't perform that upcasting itself henceforth we have to do perform it manually Args: column_values: List of columnar values Returns: Instance of PyArrow's `DataType` based on the provided column values rNobjr/cSst|ttfr t|tkSdSNF)r^r)bytesrr)rr#r#r$_len_gt_overflow_threshold;s z7_infer_pyarrow_type.._len_gt_overflow_thresholdcg|]}|qSr#r#rWrr#r$rZIrnz'_infer_pyarrow_type..crr#r#rWrr#r$rZMrn) r_try_infer_pa_timestamp_typer0 infer_typerboolrc is_binaryany large_binary is_string large_string)r8dtype_with_timestamp_typeinferred_pa_dtyper#rr$rbs   rbsmsusns) YDMWhmrrrrpsfsascCsrt|trt|dkrt|d}|rt|SdSt|tjr7t|j tj r7t |j \}}t t |SdS)Nr)r^listrrr0rxrerfrr|rr timestamp_NUMPY_TO_ARROW_PRECISION_MAP)r8 element_type np_precisionrr#r#r$rgs rcCsgttRS)zXReturns list of extension types of Arrow Array holding multidimensional tensors ),get_arrow_extension_fixed_shape_tensor_types/get_arrow_extension_variable_shape_tensor_typesr#r#r#r$ get_arrow_extension_tensor_types{s rcCsttfSz`Returns list of Arrow extension types holding multidimensional tensors of *fixed* shape )ArrowTensorTypeArrowTensorTypeV2r#r#r#r$rsrcCstfSr)ArrowVariableShapedTensorTyper#r#r#r$rrcseZdZdZdeedfdejdeffdd Z e dd Z e d d Z d d Z ddZddZddZer;ddZdddejfddZdefddZdefddZedeed defd!d"ZZS)#_BaseFixedShapeArrowTensorTypea Arrow ExtensionType for an array of fixed-shaped, homogeneous-typed tensors. This is the Arrow side of TensorDtype. See Arrow extension type docs: https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types shape. tensor_dtype ext_type_idcs||_t||dSrU)_shaperr)rrrrr!r#r$rsz'_BaseFixedShapeArrowTensorType.__init__cC|jS)z- Shape of contained tensors. )rrr#r#r$rrz$_BaseFixedShapeArrowTensorType.shapecCs|jjS)3Returns the type of the underlying tensor elements.) storage_type value_typerr#r#r$ scalar_typez*_BaseFixedShapeArrowTensorType.scalar_typecCsddlm}||j|jS) Convert Arrow extension type to corresponding Pandas dtype. Returns: An instance of pd.api.extensions.ExtensionDtype. r TensorDtype)%ray.air.util.tensor_extensions.pandasrrrto_pandas_dtyperrr#r#r$rs z._BaseFixedShapeArrowTensorType.to_pandas_dtypecC|j|j|ffSrU__arrow_ext_deserialize__r__arrow_ext_serialize__rr#r#r$ __reduce__z)_BaseFixedShapeArrowTensorType.__reduce__cCt|jSrU)jsondumpsrencoderr#r#r$rz6_BaseFixedShapeArrowTensorType.__arrow_ext_serialize__cCtSz ExtensionArray subclass with custom logic for this array of tensors type. Returns: A subclass of pd.api.extensions.ExtensionArray. r:rr#r#r$__arrow_ext_class__z2_BaseFixedShapeArrowTensorType.__arrow_ext_class__cCrzd ExtensionScalar subclass with custom logic for this array of tensors type. ArrowTensorScalarrr#r#r$__arrow_ext_scalar_class__z9_BaseFixedShapeArrowTensorType.__arrow_ext_scalar_class__scalarpa.ExtensionScalarr/cCs6|jj}|jj}|j}|j}|d}t||||S)A Convert an ExtensionScalar to a tensor element. )valuevaluesr]roffsetbuffers_to_ndarray_helper)rr raw_valuesrrr data_bufferr#r#r$_extension_scalar_to_ndarrays  z;_BaseFixedShapeArrowTensorType._extension_scalar_to_ndarraycCsd|jd|jjdS)Nznumpy.ndarray(shape=, dtype=r@)rrrrr#r#r$__str__sz&_BaseFixedShapeArrowTensorType.__str__cCt|SrUr)rr#r#r$__repr__z'_BaseFixedShapeArrowTensorType.__repr__ array_types)rrrcCsZd}|D]&}t|trdSt|tstd||dur'|j|kr'dS|j}qdS)a Whether the provided list of tensor types needs a variable-shaped representation (i.e. `ArrowVariableShapedTensorType`) when concatenating or chunking. If one or more of the tensor types in `array_types` are variable-shaped and/or any of the tensor arrays have a different shape than the others, a variable-shaped tensor array representation will be required and this method will return True. Args: array_types: List of tensor types to check if a variable-shaped representation is required for concatenation Returns: True if concatenating arrays with types `array_types` requires a variable-shaped representation NTzqAll provided array types must be an instance of either ArrowTensorType or ArrowVariableShapedTensorType, but got F)r^rr ValueErrorr)clsrrarr_typer#r#r$"_need_variable_shaped_tensor_arrays  zA_BaseFixedShapeArrowTensorType._need_variable_shaped_tensor_array)r%r&r'r(r intr0DataTyper)rpropertyrrrrrrr-rrerfrrr classmethodrr rrr*r#r#r!r$rs@      rbetacNeZdZdZejZdeedfde j ffdd Z e ddZ d d ZZS) rzArrow ExtensionType (v1) for tensors. NOTE: This type does *NOT* support tensors larger than 4Gb (due to overflow of int32 offsets utilized inside Pyarrow `ListType`) r.r|ct|t|ddS) Construct the Arrow extension type for array of fixed-shaped tensors. Args: shape: Shape of contained tensors. dtype: pyarrow dtype of tensor elements. zray.data.arrow_tensorN)rrr0rxrrr|r!r#r$r$ zArrowTensorType.__init__cCtt|}|||jSrUtuplerloadsrrr serializedrr#r#r$r/ z)ArrowTensorType.__arrow_ext_deserialize__cC"t|to|j|jko|j|jkSrU)r^rrrrotherr#r#r$__eq__4   zArrowTensorType.__eq__)r%r&r'r(rerr OFFSET_DTYPEr rr0rrrrrr*r#r#r!r$rs  rcr) rz@Arrow ExtensionType (v2) for tensors (supporting tensors > 4Gb).r.r|cr)rzray.data.arrow_tensor_v2N)rrr0 large_listrr!r#r$rBrzArrowTensorTypeV2.__init__cCrrUrrr#r#r$rMrz+ArrowTensorTypeV2.__arrow_ext_deserialize__cCrrU)r^rrrrr#r#r$rRrzArrowTensorTypeV2.__eq__)r%r&r'r(reint64rr rr0rrrrrr*r#r#r!r$r<s  rc@s,eZdZdejfddZdejfddZdS)rr/cKs |j|SrU)r]r)rkwargsr#r#r$as_py^s zArrowTensorScalar.as_pycCs|SrU)r rr#r#r$ __array__arzArrowTensorScalar.__array__N)r%r&r'rerfr r r#r#r#r$r\srcs>eZdZdZesddZddZfddZZSZS)_ArrowTensorScalarIndexingMixinz A mixin providing support for scalar indexing in tensor extension arrays for Arrow < 9.0.0, before full ExtensionScalar support was added. This mixin overrides __getitem__, __iter__, and to_pylist. ccs$tt|D]}||VqdSrU)ranger __getitem__)rir#r#r$__iter__qsz(_ArrowTensorScalarIndexingMixin.__iter__cCrrU)rrr#r#r$ to_pylistyrz)_ArrowTensorScalarIndexingMixin.to_pylistcs&t|}t|ts|j|}|SrU)rr r^slicer]r)rkeyitemr!r#r$r ~s  z+_ArrowTensorScalarIndexingMixin.__getitem__) r%r&r'r(r-rrr r*r#r#r!r$r fsr c @seZdZdZe ddeejeejfde e dedfddZ edejdedfd d Z dd e e d efddZdd efddZe d deeddededfddZedeeddejfddZd!ddZdS)"r;z An array of fixed-shape, homogeneous-typed tensors. This is the Arrow side of TensorArray. See Arrow docs for customizing extension arrays: https://arrow.apache.org/docs/python/extending_types.html#custom-extension-array-class Narrr9r/)r;ArrowVariableShapedTensorArrayc Cs>t|tjst|trt|}t|ttfrN|rNt|dtjrNz tj|dd}Wn$tyM}ztj d|d|dtj |t d}WYd}~nd}~wwt|tjs`tdt |d |zt |}|rmt|||}||WSty}z d }|r|d |d 7}|d |jd|jd|7}t||d}~ww)aN Convert an ndarray or an iterable of ndarrays to an array of homogeneous-typed tensors. If given fixed-shape tensor elements, this will return an ``ArrowTensorArray``; if given variable-shape tensor elements, this will return an ``ArrowVariableShapedTensorArray``. Args: arr: An ndarray or an iterable of ndarrays. column_name: Optional. Used only in logging outputs to provide additional details. Returns: - If fixed-shape tensor elements, an ``ArrowTensorArray`` containing ``len(arr)`` tensors of fixed shape. - If variable-shaped tensor elements, an ``ArrowVariableShapedTensorArray`` containing ``len(arr)`` tensors of variable shape. - If scalar elements, a ``pyarrow.Array``. r)axiszFailed to stack lists due to: z3; falling back to using np.array(..., dtype=object)rB)r|Nz/Must give ndarray or iterable of ndarrays, got  z column: 'z', zshape: z , dtype: z, data: )r^rerfrrrstackrrKrLrlobjectr]rrg _from_numpyr3rr|r)rrr9vetimestamp_dtyper4rr#r#r$rEs@"    zArrowTensorArray.from_numpyc st|dkrt|drt|St|rt|S|jj s$t |}t |j }tj |rP|j jdksA|j jdkrItjdkrItd|j t|j j}|jd}|jdd}|j}|rft|ndtj |rutj|dd}t|}tj||d|g}dd lm}|jrt ||} nt!||} t| "fd d t#|dD} tjj| j$|d| g|gd } tj%&| | S) Nr>=big:Only little-endian string tensors are supported, but got: rlittlebitorderr<csg|]}|qSr#r#rXrnum_items_per_elementr#r$rZrnz0ArrowTensorArray._from_numpy..)children)'rreisscalarr0rlr rrEflags c_contiguousascontiguousarrayfrom_numpy_dtyper|rcr byteordersysrbinaryitemsizersizeprod is_booleanpackbits py_bufferr_ from_buffersrGr=rIuse_arrow_tensor_v2rrrr rExtensionArray from_storage) rr scalar_dtype outer_len element_shapetotal_num_itemsr data_arrayr=pa_type_ offset_bufferstorager#r&r$rsT              zArrowTensorArray._from_numpyFindexzero_copy_onlycCs`|}|d}|jj}|j}|}|jj}tj|r!|j } n|j d} |r-t |nd} |j | } | | } |durX|d} t j t|f| |jjd}||}| | |7} nt|f|}tj|r| d}| d}d|t |dd}t j |ft j||d}t j|dd}t j |t j||dStj|rt d |jt}t j |||| dS) at Helper for getting either an element of the array of tensors as an ndarray, or the entire array of tensors as a single ndarray. Args: index: The index of the tensor element that we wish to return as an ndarray. If not given, the entire array of tensors is returned as an ndarray. zero_copy_only: If True, an exception will be raised if the conversion to a NumPy array would require copying the underlying data (e.g. in presence of nulls, or for non-primitive types). This argument is currently ignored, so zero-copy isn't enforced even if this argument is true. Returns: The corresponding tensor element as an ndarray if an index was given, or the entire array of tensors as an ndarray otherwise. rN)bufferr|r|rGrr"r#.cSsg|] }|D]}|qqSr#r#)rXar4r#r#r$rZr[rrcSrbr#)rB)rXcr#r#r$rZrd) rrrrErr0rsr;r:r])rr`rato_concat_typesrBr#r#r$_concat_same_types  z"ArrowTensorArray._concat_same_typearrscCsddd|D}t|r-g}|D]}t|jtr|}t|jts%J||q|}t |S)zD Create a ChunkedArray from multiple tensor arrays. cSrbr#r\rcr#r#r$rZrdz9ArrowTensorArray._chunk_tensor_arrays..) rrr^r]rto_variable_shaped_tensor_arrayrappendr0 chunked_array)rri arrs_typesnew_arrsrer#r#r$_chunk_tensor_arrayss   z%ArrowTensorArray._chunk_tensor_arraysrcCst|S)a Convert this tensor array to a variable-shaped tensor array. This is primarily used when concatenating multiple chunked tensor arrays where at least one chunked array is already variable-shaped and/or the shapes of the chunked arrays differ, in which case the resulting concatenated tensor array will need to be in the variable-shaped representation. )rrEr_rr#r#r$rjs z0ArrowTensorArray.to_variable_shaped_tensor_arrayrUrT)F)r/r)r%r&r'r(rr rerfrrr)rErrrrZr_rrhr0r`rorjr#r#r#r$r;sJ  A @V %  r;cseZdZdZdejdeffdd ZddZe defd d Z e d d Z d dZ ddZ eddZddZer>ddZdefddZdefddZdddejfddZZS)ra Arrow ExtensionType for an array of heterogeneous-shaped, homogeneous-typed tensors. This is the Arrow side of TensorDtype for tensor elements with different shapes. Note that this extension only supports non-ragged tensor elements; i.e., when considering each tensor element in isolation, they must have a well-defined, non-ragged shape. See Arrow extension type docs: https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types r|ndimc s:||_ttdt|fdttfgddS)z Construct the Arrow extension type for array of heterogeneous-shaped tensors. Args: dtype: pyarrow dtype of tensor elements. ndim: The number of dimensions in the tensor elements. datarz%ray.data.arrow_variable_shaped_tensorN)_ndimrrr0structrrxr)rr|rqr!r#r$rsz&ArrowVariableShapedTensorType.__init__cCs*ddlm}|d|j|jdjjS)rrrrUrr)rrrqrr]rrrr#r#r$rs z-ArrowVariableShapedTensorType.to_pandas_dtyper/cCr)z7Return the number of dimensions in the tensor elements.)rsrr#r#r$rqsz"ArrowVariableShapedTensorType.ndimcCs|jd}|j|jjS)rrr)rget_field_indexr]r)rdata_field_indexr#r#r$rs z)ArrowVariableShapedTensorType.scalar_typecCrrUrrr#r#r$rrz(ArrowVariableShapedTensorType.__reduce__cCrrU)rrrsrrr#r#r$r rz5ArrowVariableShapedTensorType.__arrow_ext_serialize__cCs t|}|djj}|||S)Nrr)rrr]r)rrrrqr|r#r#r$r s   z7ArrowVariableShapedTensorType.__arrow_ext_deserialize__cCrr)rrr#r#r$rrz1ArrowVariableShapedTensorType.__arrow_ext_class__cCrrrrr#r#r$rrz8ArrowVariableShapedTensorType.__arrow_ext_scalar_class__cCs"|jdjj}d|jd|dS)Nrrznumpy.ndarray(ndim=rr@)rr]rrq)rr|r#r#r$r%sz%ArrowVariableShapedTensorType.__str__cCrrUrrr#r#r$r)rz&ArrowVariableShapedTensorType.__repr__rrcCsL|jd}|j}t|jd}|j}|j}|d}t||||S)rrrrr) rgetrrr r]rrr)rrrrrrrrrr#r#r$r,s  z:ArrowVariableShapedTensorType._extension_scalar_to_ndarray)r%r&r'r(r0rrrrrrqrrrrrrr-rr)rrrerfrr*r#r#r!r$rs$    rc@sfeZdZdZedeejeeje ejfddfddZ dde e d e fd d Zdd e fd dZdS)raB An array of heterogeneous-shaped, homogeneous-typed tensors. This is the Arrow side of TensorArray for tensor elements that have differing shapes. Note that this extension only supports non-ragged tensor elements; i.e., when considering each tensor element in isolation, they must have a well-defined shape. This extension also only supports tensor elements that all have the same number of dimensions. See Arrow docs for customizing extension arrays: https://arrow.apache.org/docs/python/extending_types.html#custom-extension-array-class rr/cCst|tttjfstdt|t|dkrtdggg}}}d}|D]6}t|}|durC|j |krCtd|d|j |j }| |j | |j tj |dd}| |q(t|}t|}|d }td d t|Dr~|d j} nt|} | j} t| } tj| r| jd ks| jd krtjdkrtd| t| j} | jtjurtj| dd} t| } tj !| |d| g} t"|dd}t|}tj#$|| }t|}tj%$||gddg}t&| |}tj'(||S)ax Convert an ndarray or an iterable of heterogeneous-shaped ndarrays to an array of heterogeneous-shaped, homogeneous-typed tensors. Args: arr: An ndarray or an iterable of heterogeneous-shaped ndarrays. Returns: An ArrowVariableShapedTensorArray containing len(arr) tensors of heterogeneous shape. zmArrowVariableShapedTensorArray can only be constructed from an ndarray or a list/tuple of ndarrays, but got: rz5Creating empty ragged tensor arrays is not supported.NzArrowVariableShapedTensorArray only supports tensor elements that all have the same number of dimensions, but got tensor elements with dimensions: z, C)ordercss|] \}}t||VqdSrU)_is_contiguous_view)rXprevcurrr#r#r$ sz.rrr r!r"r#rrr))r^rrrerfrr]rasarrayrqrkrr2ravelrlrqall _pairwisebaserpr|r0r-rcrr.r/r0r1rMr5r6r_r7insertLargeListArrayrw StructArrayrr9r:)rrshapessizesraveledrqre size_offsets total_sizenp_data_bufferr|pa_dtyper value_arrayrUr? shape_arrayrBtype_r#r#r$rENsp                  z)ArrowVariableShapedTensorArray.from_numpyNFrCrDc s|durfddttD}t|Sjd}jd}||}|jj}|j|}| d} t |||| S)ad Helper for getting either an element of the array of tensors as an ndarray, or the entire array of tensors as a single ndarray. Args: index: The index of the tensor element that we wish to return as an ndarray. If not given, the entire array of tensors is returned as an ndarray. zero_copy_only: If True, an exception will be raised if the conversion to a NumPy array would require copying the underlying data (e.g. in presence of nulls, or for non-primitive types). This argument is currently ignored, so zero-copy isn't enforced even if this argument is true. Returns: The corresponding tensor element as an ndarray if an index was given, or the entire array of tensors as an ndarray otherwise. Ncsg|]}|qSr#r]r%r^r#r$rZsz.rrrrE) r rrrBfieldr r]rrzrr) rrCrDrirrrrrrrr#r^r$rZs    z(ArrowVariableShapedTensorArray._to_numpyTcCr[)a  Convert the entire array of tensors into a single ndarray. Args: zero_copy_only: If True, an exception will be raised if the conversion to a NumPy array would require copying the underlying data (e.g. in presence of nulls, or for non-primitive types). This argument is currently ignored, so zero-copy isn't enforced even if this argument is true. Returns: A single ndarray representing the entire array of tensors. r\r]r^r#r#r$r_s z'ArrowVariableShapedTensorArray.to_numpyrrp)r%r&r'r(rr rerfrr rErrrrZr_r#r#r#r$r=s  c#rr}r|cCsZ|jdus|jjr|dur|j|jurdS|dur+t|t||jjj|jkr+dSdS)abCheck if the provided tensor element is contiguous with the previous tensor element. Args: curr: The tensor element whose contiguity that we wish to check. prev: The previous tensor element in the tensor array. Returns: Whether the provided tensor element is contiguous with the previous tensor element. NFT)rrrr+_get_buffer_addressr|r1r2)r}r|r#r#r$r{s r{rcCs|jddS)zDGet the address of the buffer underlying the provided NumPy ndarray.rrr)__array_interface__)rr#r#r$r srcCs"t|\}}t|dt||SrU)rtteenextzip)iterablerebr#r#r$rs  rc Cstj|r |j}n|jd}||}tj|rJ|d}|d}d|t|dd}tj|ftj||d} tj| dd} tj|tj | |dS| } tj |r_t d|j t} tj|| ||dS)NrFrrHr"r#rI)r0rcr4rJrer3rfrKrLrMrrNr|rOrP) rrrrrS data_offsetrWrXrYrrRr#r#r$rs(    r)r)Xabcrrtrrir/typingrrrrrrr r numpyrepyarrowr0packaging.versionr parse_versionray._private.arrow_utilsr $ray.air.util.tensor_extensions.utilsr rrr ray.data._internal.numpy_supportrrray.utilrray.util.annotationsrrray.util.commonrr+r,9MIN_PYARROW_VERSION_CHUNKED_ARRAY_TO_NUMPY_ZERO_COPY_ONLYrP getLoggerr%rKr3rr-r)r_r1r5rfrRrFr`ra TimestampTypergrrbrrrrr ExtensionTypeABCrrrExtensionScalarrr r9r;rrrr{rrrrregister_extension_typer ArrowKeyErrorr#r#r#r$s (        Y 7  A    ! +<p ($+