o bi7@sjddlZddlZddlmZmZmZmZmZmZm Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlmZmZddlmZmZddlmZmZdd lm Z zdd lm!Z!Wne"ymdd lm#Z!Ynwd ee$fd dZ%d ee$fddZ&dZ'e(e'ddkree j)edkrddl*m+Z+e+Z,nddl*m-Z-e-Z,e,j.e,_/edee j)kredkrnne%e,_.ne&e,_.de,_0e dde j1j2j3Gddde j1j2j4Z5Gddde j1j2j6Z7Gdd d Z8e ddGd!d"d"e7e8Z9e ddGd#d$d$e j1j2j:e7e8Z;e9<e9=e9>e;<e;=e;>e ddd%e j?d e@fd&d'ZAdS)(N)AnyCallableListOptionalSequenceTupleUnion)Version)Dtype)set_function_name) ABCDataFrame ABCSeries)check_array_indexervalidate_indices)_create_possibly_ragged_ndarray"_is_ndarray_variable_shaped_tensor) PublicAPI)ABCIndex) ABCIndexClassreturnc sddlm}ddlmtjtsS|jdd}t |}|j dkr+Sfdd}j }|dur?|j dd }| d }t |||}|jjrRd nd }|j|j|d } || dS)Nr extract_array format_arrayT extract_numpyc s0||jjjjjjjjd }|S)N float_formatna_repdigitsspacejustifydecimal leading_spacequotingrarray_ formatter_ fmt_valuesrselfY/home/ubuntu/.local/lib/python3.10/site-packages/ray/air/util/tensor_extensions/pandas.pyformat_array_wrapMs z2_format_strings_patched..format_array_wrapboxedKFCorder)pandas.core.constructionrpandas.io.formats.formatr isinstancevalues TensorArray_format_strings_orignpasarrayndim formatter _formatterravelflags f_contiguousreshapeshape) r+rr9arrayr.flat_formatter flat_arrayfmt_flat_arrayr5 fmt_arrayr,r*r-_format_strings_patched@s"        rKc sddlmddlm}ddlmddlmtj t s" S|j dd}t |}|jdkr7 Sfd d }j}|durK|jdd }|d }t |||}|jjr^d nd}|j|j|d} fdd} | | jS)Nr)partialrr) pprint_thingTrrc s,||jjjjjjjd }|S)Nrrr r!r"r#r$rNr&r*r,r-r.zs z9_format_strings_patched_v1_0_0..format_array_wrapr/r1r2r3r4csPddfdd}g}|D]}|durdnd}||j||dq|S) N)   ) escape_charscs t|SNstr)xr?r,r-_formats zL_format_strings_patched_v1_0_0..format_strings_slim.._formatFz{v}z {v})v)appendformat)r'r$rXr)rYtpl)rLrMrWr-format_strings_slims z;_format_strings_patched_v1_0_0..format_strings_slim) functoolsrLr6rr7rpandas.io.formats.printingrMr8r9r:r;r<r=r>r?r@rArBrCrDrEr$) r+rr9rFr.rGrHrIr5rJr]r,)rrLrMr+r-_format_strings_patched_v1_0_0js(          r`)TENSOR_COLUMN_EXTENSION_FORMATTER_ENABLED1z2.2.0)ExtensionArrayFormatter)_ExtensionArrayFormatterz1.1.0z1.3.0Tbeta) stabilityc@seZdZdZdZdeeedfdej fddZ e dd Z e d d Z e d d Ze ddZe defddZedefddZeddZdeejejffddZdefddZdefddZe dd ZdS)! TensorDtypea Pandas extension type for a column of homogeneous-typed tensors. This extension supports tensors in which the elements have different shapes. However, each tensor element must be non-ragged, i.e. each tensor element must have a well-defined, non-ragged shape. See: https://github.com/pandas-dev/pandas/blob/master/pandas/core/dtypes/base.py for up-to-date interface documentation and the subclassing contract. The docstrings of the below properties and methods were copied from the base ExtensionDtype. Examples: >>> # Create a DataFrame with a list of ndarrays as a column. >>> import pandas as pd >>> import numpy as np >>> import ray >>> df = pd.DataFrame({ ... "one": [1, 2, 3], ... "two": list(np.arange(24).reshape((3, 2, 2, 2)))}) >>> # Note the opaque np.object dtype for this column. >>> df.dtypes # doctest: +SKIP one int64 two object dtype: object >>> # Cast column to our TensorDtype extension type. >>> from ray.data.extensions import TensorDtype >>> df["two"] = df["two"].astype(TensorDtype(np.int64, (3, 2, 2, 2))) >>> # Note that the column dtype is now TensorDtype instead of >>> # np.object. >>> df.dtypes # doctest: +SKIP one int64 two TensorDtype(shape=(3, 2, 2, 2), dtype=int64) dtype: object >>> # Pandas is now aware of this tensor column, and we can do the >>> # typical DataFrame operations on this column. >>> col = 2 * (df["two"] + 10) >>> # The ndarrays underlying the tensor column will be manipulated, >>> # but the column itself will continue to be a Pandas type. >>> type(col) # doctest: +SKIP pandas.core.series.Series >>> col # doctest: +SKIP 0 [[[ 2 4] [ 6 8]] [[10 12] [14 16]]] 1 [[[18 20] [22 24]] [[26 28] [30 32]]] 2 [[[34 36] [38 40]] [[42 44] [46 48]]] Name: two, dtype: TensorDtype(shape=(3, 2, 2, 2), dtype=int64) >>> # Once you do an aggregation on that column that returns a single >>> # row's value, you get back our TensorArrayElement type. >>> tensor = col.mean() >>> type(tensor) # doctest: +SKIP ray.data.extensions.tensor_extension.TensorArrayElement >>> tensor # doctest: +SKIP array([[[18., 20.], [22., 24.]], [[26., 28.], [30., 32.]]]) >>> # This is a light wrapper around a NumPy ndarray, and can easily >>> # be converted to an ndarray. >>> type(tensor.to_numpy()) # doctest: +SKIP numpy.ndarray >>> # In addition to doing Pandas operations on the tensor column, >>> # you can now put the DataFrame into a Dataset. >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> # Internally, this column is represented the corresponding >>> # Arrow tensor extension type. >>> ds.schema() # doctest: +SKIP one: int64 two: extension> >>> # You can write the dataset to Parquet. >>> ds.write_parquet("/some/path") # doctest: +SKIP >>> # And you can read it back. >>> read_ds = ray.data.read_parquet("/some/path") # doctest: +SKIP >>> read_ds.schema() # doctest: +SKIP one: int64 two: extension> >>> read_df = ray.get(read_ds.to_pandas_refs())[0] # doctest: +SKIP >>> read_df.dtypes # doctest: +SKIP one int64 two TensorDtype(shape=(3, 2, 2, 2), dtype=int64) dtype: object >>> # The tensor extension type is preserved along the >>> # Pandas --> Arrow --> Parquet --> Arrow --> Pandas >>> # conversion chain. >>> read_df.equals(df) # doctest: +SKIP True NrE.dtypecCs||_||_dSrS_shape_dtype)r+rErhr,r,r-__init__&s zTensorDtype.__init__cCtS)a  The scalar type for the array, e.g. ``int`` It's expected ``ExtensionArray[item]`` returns an instance of ``ExtensionDtype.type`` for scalar ``item``, assuming that value is valid (not NA). NA values do not need to be instances of `type`. )TensorArrayElementr+r,r,r-type* zTensorDtype.typecC|jS)z> The dtype of the underlying tensor elements. )rkror,r,r- element_dtype5szTensorDtype.element_dtypecCrr)z The shape of the underlying tensor elements. This will be a tuple of Nones if the corresponding TensorArray for this TensorDtype holds variable-shaped tensor elements. )rjror,r,r- element_shape<szTensorDtype.element_shapecCstdd|jDS)z{ Whether the corresponding TensorArray for this TensorDtype holds variable-shaped tensor elements. css|]}|duVqdSrSr,).0dim_sizer,r,r- Ksz1TensorDtype.is_variable_shaped..)allrEror,r,r-is_variable_shapedEszTensorDtype.is_variable_shapedrcCsd|jd|jdS)zp A string identifying the data type. Will be used for display in, e.g. ``Series.dtype`` znumpy.ndarray(shape=z, dtype=)riror,r,r-nameMszTensorDtype.namestringc Csddl}ddl}t|tstdt|d}|||}d|jd|d}|dur0t||}t |dkr>t||\}} } | | } t | } || | S) a= Construct this type from a string. This is useful mainly for data types that accept parameters. For example, a period dtype accepts a frequency parameter that can be set as ``period[H]`` (where H means hourly frequency). By default, in the abstract class, just the name of the type is expected. But subclasses can overwrite this method to accept parameters. Parameters ---------- string : str The name of the type, for example ``category``. Returns ------- ExtensionDtype Instance of the dtype. Raises ------ TypeError If a class cannot be constructed from this 'string'. Examples -------- For extension dtypes with arguments the following may be an adequate implementation. >>> import re >>> @classmethod ... def construct_from_string(cls, string): ... pattern = re.compile(r"^my_type\[(?P.+)\]$") ... match = pattern.match(string) ... if match: ... return cls(**match.groupdict()) ... else: ... raise TypeError( ... f"Cannot construct a '{cls.__name__}' from '{string}'" ... ) rNz.'construct_from_string' expects a string, got zP^(TensorDtype|numpy.ndarray)\(shape=(\((?:(?:\d+|None),?\s?)*\)), dtype=(\w+)\)$zCannot construct a 'z' from 'zF'; expected a string like 'TensorDtype(shape=(1, 2, 3), dtype=int64)'.) astrer8rU TypeErrorrpsearch__name__groupslen literal_evalr<rh) clsr|r~rregexmerr_msgr_rErhr,r,r-construct_from_stringUs(-        z!TensorDtype.construct_from_stringcCrm)zq Return the array type associated with this dtype. Returns ------- type )r:)rr,r,r-construct_array_typerqz TensorDtype.construct_array_typerFcCs\t|tjr&|jdkrtdd|D}t |S|d}t |S|}t |S)a| Convert a pyarrow (chunked) array to a TensorArray. This and TensorArray.__arrow_array__ make up the Pandas extension type + array <--> Arrow extension type + array interoperability protocol. See https://pandas.pydata.org/pandas-docs/stable/development/extending.html#compatibility-with-apache-arrow for more information. rcSsg|]}|qSr,)to_numpy)ruchunkr,r,r- z.TensorDtype.__from_arrow__..r) r8pa ChunkedArray num_chunksr< concatenate iterchunksrrr:)r+rFr9r,r,r-__from_arrow__s zTensorDtype.__from_arrow__cCrrrS)r{ror,r,r-__str__szTensorDtype.__str__cCst|SrSrTror,r,r-__repr__szTensorDtype.__repr__cCsddlm}||jS)B Whether this extension array should be considered boolean. By default, ExtensionArrays are assumed to be non-numeric. Setting this to True will affect the behavior of several places, e.g. * is_bool * boolean indexing Returns ------- bool r) is_bool_dtype)pandas.core.dtypes.commonrrk)r+rr,r,r- _is_booleans  zTensorDtype._is_boolean)r __module__ __qualname____doc__baserrintr<rhrlpropertyrprsrtryrUr{ classmethodrrrrArrayrrrrrr,r,r,r-rgs.d     H  rgc@s*eZdZdZedddZeddZdS) _TensorOpsMixinze Mixin for TensorArray operator support, applying operations on the underlying ndarrays. TNcs*fdd}djdt|S)zc Add support for binary operators by unwrapping, applying, and rewrapping. cs|j}t|tttfr tSdvrtt|ttfr|j}n|}||}t|tr:t|tr4t |s:t|}|S|}|S)N) __divmod__ __rdivmod__) _tensorr8r r rNotImplementedNotImplementedErrorr:rnr<isscalar)r+otherlvaluesrvaluesresultresult_wrappedropop_namer,r-_binops$  z._TensorOpsMixin._create_method.._binop__)rr )rrcoerce_to_dtype result_dtyperr,rr-_create_methods  z_TensorOpsMixin._create_methodcCs ||SrS)r)rrr,r,r-_create_logical_methods z&_TensorOpsMixin._create_logical_method)TN)rrrrrrrr,r,r,r-rs  *rc@sPeZdZdZdeegeffddZddZddZd d Z d d Z d dZ dS)_TensorScalarCastMixinzH Mixin for casting scalar tensors to a particular numeric type. funccCs ||jSrSr)r+rr,r,r- _scalarfunc z"_TensorScalarCastMixin._scalarfunccC |tSrS)rcomplexror,r,r- __complex__rz"_TensorScalarCastMixin.__complex__cCrrS)rfloatror,r,r- __float__rz _TensorScalarCastMixin.__float__cCrrS)rrror,r,r-__int__!rz_TensorScalarCastMixin.__int__cCrrS)rhexror,r,r-__hex__$rz_TensorScalarCastMixin.__hex__cCrrS)roctror,r,r-__oct__'rz_TensorScalarCastMixin.__oct__N) rrrrrrrrrrrrr,r,r,r-rs rc@seZdZdZdejfddZddZddZe d d Z e d d Z e d dZ e ddZ ddZddejdejfddZdS)rnzJ Single element of a TensorArray, wrapping an underlying ndarray. r9cCs ||_dS)z Construct a TensorArrayElement from a NumPy ndarray. Args: values: ndarray that underlies this TensorArray element. Nrr+r9r,r,r-rl1s zTensorArrayElement.__init__cC |jSrSrrror,r,r-r:rzTensorArrayElement.__repr__cCrrSrrror,r,r-r=rzTensorArrayElement.__str__cC|jjSzf Get the dtype of the tensor. :return: The numpy dtype of the backing ndarray rrhror,r,r- numpy_dtype@zTensorArrayElement.numpy_dtypecCrzl Get the number of tensor dimensions. :return: integer for the number of dimensions rr>ror,r,r- numpy_ndimHrzTensorArrayElement.numpy_ndimcCrz~ Get the shape of the tensor. :return: A tuple of integers for the numpy shape of the backing ndarray rrEror,r,r- numpy_shapePrzTensorArrayElement.numpy_shapecCrzo Get the size of the tensor. :return: integer for the number of elements in the tensor rsizeror,r,r- numpy_sizeXrzTensorArrayElement.numpy_sizecCs t|jS)zG Return the values of this element as a NumPy ndarray. r<r=rror,r,r-r`s zTensorArrayElement.to_numpyNrhrcKtj|jfd|i|SNrhrr+rhkwargsr,r,r- __array__fzTensorArrayElement.__array__rS)rrrrr<ndarrayrlrrrrrrrrrhrr,r,r,r-rn+s     rnc seZdZdZejejejejej ej ej ej ej ejd ZdeejeeeejefeeffddZedddd eed efd d Zedejd ejjjfddZ dee!e"ejfdedfddZ#de!fddZ$e%dejjj&fddZ'e%ddZ(e%de!fddZ)dUddZ* dVdee!d ed!eddfd"d#Z+dUd$d%Z,ed&edddfd'd(Z-d)ee!ejfd*eddfd+d,Z.deffd-d. Z/d/d0Z0d1d2Z1de2ejeffd3d4Z3dWd6e4d7efd8d9Z5dXd ej'dejfd:d;Z6dd?Z8ddejjj9fd ej'd ed@efdAdBZ:e%dCdDZ;e%dEdFZdWdKdLZ?dYdMdNZdYdOdPZdXdQdRZ@e%dSdTZAZBS)Zr:a Pandas `ExtensionArray` representing a tensor column, i.e. a column consisting of ndarrays as elements. This extension supports tensors in which the elements have different shapes. However, each tensor element must be non-ragged, i.e. each tensor element must have a well-defined, non-ragged shape. Examples: >>> # Create a DataFrame with a list of ndarrays as a column. >>> import pandas as pd >>> import numpy as np >>> import ray >>> from ray.data.extensions import TensorArray >>> df = pd.DataFrame({ ... "one": [1, 2, 3], ... "two": TensorArray(np.arange(24).reshape((3, 2, 2, 2)))}) >>> # Note that the column dtype is TensorDtype. >>> df.dtypes # doctest: +SKIP one int64 two TensorDtype(shape=(3, 2, 2, 2), dtype=int64) dtype: object >>> # Pandas is aware of this tensor column, and we can do the >>> # typical DataFrame operations on this column. >>> col = 2 * (df["two"] + 10) >>> # The ndarrays underlying the tensor column will be manipulated, >>> # but the column itself will continue to be a Pandas type. >>> type(col) # doctest: +SKIP pandas.core.series.Series >>> col # doctest: +SKIP 0 [[[ 2 4] [ 6 8]] [[10 12] [14 16]]] 1 [[[18 20] [22 24]] [[26 28] [30 32]]] 2 [[[34 36] [38 40]] [[42 44] [46 48]]] Name: two, dtype: TensorDtype(shape=(3, 2, 2, 2), dtype=int64) >>> # Once you do an aggregation on that column that returns a single >>> # row's value, you get back our TensorArrayElement type. >>> tensor = col.mean() # doctest: +SKIP >>> type(tensor) # doctest: +SKIP ray.data.extensions.tensor_extension.TensorArrayElement >>> tensor # doctest: +SKIP array([[[18., 20.], [22., 24.]], [[26., 28.], [30., 32.]]]) >>> # This is a light wrapper around a NumPy ndarray, and can easily >>> # be converted to an ndarray. >>> type(tensor.to_numpy()) # doctest: +SKIP numpy.ndarray >>> # In addition to doing Pandas operations on the tensor column, >>> # you can now put the DataFrame into a Dataset. >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> # Internally, this column is represented the corresponding >>> # Arrow tensor extension type. >>> ds.schema() # doctest: +SKIP one: int64 two: extension> >>> # You can write the dataset to Parquet. >>> ds.write_parquet("/some/path") # doctest: +SKIP >>> # And you can read it back. >>> read_ds = ray.data.read_parquet("/some/path") # doctest: +SKIP >>> read_ds.schema() # doctest: +SKIP one: int64 two: extension> >>> read_df = ray.get(read_ds.to_pandas_refs())[0] # doctest: +SKIP >>> read_df.dtypes # doctest: +SKIP one int64 two TensorDtype(shape=(3, 2, 2, 2), dtype=int64) dtype: object >>> # The tensor extension type is preserved along the >>> # Pandas --> Arrow --> Parquet --> Arrow --> Pandas >>> # conversion chain. >>> read_df.equals(df) # doctest: +SKIP True ) sumrxanyminmaxmeanmedianprodstdvarr9cCst|tr t|}n!t|trdd|D}t|}nt|tr+tjt|gdd}t|tjra|j j tj ur`t |dkr?n9t dd|DrTdd|D}t|}n$td t |dd nt|trjtd td |d t |jdt|tjsJ||_d|_dS)zy Args: values: A NumPy ndarray or sequence of NumPy ndarrays of equal shape. cS$g|]}t|trt|n|qSr,r8rnr<r=rurYr,r,r-rsz(TensorArray.__init__..Fcopyrcss.|]}t|tjttfot|t VqdSrS)r8r<rrnrrUrr,r,r-rws   z'TensorArray.__init__..cSsg|]}t|qSr,)r<r=rr,r,r-rszExpected a well-typed ndarray or an object-typed ndarray of ndarray pointers, but got an object-typed ndarray whose subndarrays are of type .z8Use the copy() method to create a copy of a TensorArray.zDExpected a numpy.ndarray or sequence of numpy.ndarray, but received z of type z instead.N)r8r rrrnr<rFr=rrhrpobject_rrxrr:rr_is_variable_shapedrr,r,r-rlsH           zTensorArray.__init__NFrhrrhrcCsF|rt|tjr|}t|St|tr|r|jn|j}t|S)aZ Construct a new ExtensionArray from a sequence of scalars. Parameters ---------- scalars : Sequence Each element will be an instance of the scalar type for this array, ``cls.dtype.type`` or be converted into this type in this method. dtype : dtype, optional Construct for this particular dtype. This should be a Dtype compatible with the ExtensionArray. copy : bool, default False If True, copy the underlying data. Returns ------- ExtensionArray )r8r<rrr:r)rscalarsrhrr,r,r-_from_sequences  zTensorArray._from_sequenceoriginalcCt)a Reconstruct an ExtensionArray after factorization. Parameters ---------- values : ndarray An integer ndarray with the factorized values. original : ExtensionArray The original ExtensionArray that factorize was called on. See Also -------- factorize : Top-level factorize method that dispatches here. ExtensionArray.factorize : Encode the extension array as an enumerated type. r)rr9rr,r,r-_from_factorized,szTensorArray._from_factorizeditemr)r:rncCst|tr|j|}t|r|St|St|tr4t|dkr4|dtkr4t|dkr0t d|d}t|t r>t |}t ||}t |j|S)aA Select a subset of self. Parameters ---------- item : int, slice, or ndarray * int: The position in 'self' to get. * slice: A slice object, where 'start', 'stop', and 'step' are integers or None * ndarray: A 1-d boolean NumPy ndarray the same length as 'self' Returns ------- item : scalar or ExtensionArray Notes ----- For scalar ``item``, return a scalar value suitable for the array's type. This should be an instance of ``self.dtype.type``. For slice ``key``, return an instance of ``ExtensionArray``, even if the slice is length 0 or 1. For a boolean mask, return an instance of ``ExtensionArray``, filtered to the values where ``item`` is True. rrzCWorkaround Pandas issue #42430 not implemented for tuple length > 2) r8rrr<rrntuplerEllipsis ValueErrorr:r=r)r+rvaluer,r,r- __getitem__Bs   "    zTensorArray.__getitem__cCs t|jS)z] Length of this array. Returns ------- length : int )rrror,r,r-__len__us zTensorArray.__len__cCsB|jr|jdj}d|jdj}n |j}|jdd}t||S)z2 An instance of 'ExtensionDtype'. rrSrN)ryrrhr>rrrg)r+rhrEr,r,r-rhs   zTensorArray.dtypecCs|jdur t|j|_|jS)zQ Whether this TensorArray holds variable-shaped tensor elements. N)rrrror,r,r-rys  zTensorArray.is_variable_shapedcCr)zL The number of bytes needed to store this object in memory. )rnbytesror,r,r-rszTensorArray.nbytescsjjjtjurtjfddttDtdSjjjtj ur3tj jdkt tdjj dStj t jt tdjj dS)a A 1-D array indicating if each value is missing. Returns ------- na_values : Union[np.ndarray, ExtensionArray] In most cases, this should return a NumPy ndarray. For exceptional cases like ``SparseArray``, where returning an ndarray would be expensive, an ExtensionArray may be returned. Notes ----- If returning an ExtensionArray, then * ``na_values._is_boolean`` should be True * `na_values` should implement :func:`ExtensionArray._reduce` * ``na_values.any`` and ``na_values.all`` should be implemented csg|] }j|duqSrSr)ruiror,r-rsz$TensorArray.isna..rhraxis)rrhrpr<rrFrangerboolstr_rxrr>isnanror,ror-isnas"zTensorArray.isnaindices allow_fill fill_valuecCs|rLtj|tjd}t|t|jt|dk}|rL|dur"tj}tt|f|jj dd|}|dk}t |t |d|j||t |S|jj |dd}t |S)a Take elements from an array. Parameters ---------- indices : sequence of int Indices to be taken. allow_fill : bool, default False How to handle negative values in `indices`. * False: negative values in `indices` indicate positional indices from the right (the default). This is similar to :func:`numpy.take`. * True: negative values in `indices` indicate missing values. These values are set to `fill_value`. Any other other negative values raise a ``ValueError``. fill_value : any, optional Fill value to use for NA-indices when `allow_fill` is True. This may be ``None``, in which case the default NA value for the type, ``self.dtype.na_value``, is used. For many ExtensionArrays, there will be two representations of `fill_value`: a user-facing "boxed" scalar, and a low-level physical NA value. `fill_value` should be the user-facing version, and the implementation should handle translating that to the physical version for processing the take if necessary. Returns ------- ExtensionArray Raises ------ IndexError When the indices are out of bounds for the array. ValueError When `indices` contains negative values other than ``-1`` and `allow_fill` is True. See Also -------- numpy.take : Take elements from an array along an axis. api.extensions.take : Take elements from an array. Notes ----- ExtensionArray.take is called by ``Series.__getitem__``, ``.loc``, ``iloc``, when `indices` is a sequence of values. Additionally, it's called by :meth:`Series.reindex`, or any other method that causes realignment, with a `fill_value`. Examples -------- Here's an example implementation, which relies on casting the extension array to object dtype. This uses the helper method :func:`pandas.api.extensions.take`. .. code-block:: python def take(self, indices, allow_fill=False, fill_value=None): from pandas.core.algorithms import take # If the ExtensionArray is backed by an ndarray, then # just pass that here instead of coercing to object. data = self.astype(object) if allow_fill and fill_value is None: fill_value = self.dtype.na_value # fill value should always be translated from the scalar # type for the array, to the physical storage type for # the data, before passing to take. result = take(data, indices, fill_value=fill_value, allow_fill=allow_fill) return self._from_sequence(result, dtype=self.dtype) rrNrr)r<r=intprrrrnanfullrEputwherer:take)r+rrr has_missingr9 is_nonnegr,r,r-rsR""zTensorArray.takecCst|jS)ze Return a copy of the array. Returns ------- ExtensionArray )r:rrror,r,r-r,s zTensorArray.copy to_concatcCs|d}d}|D]}|dur|jj}|js|jj|krd}nq|r0ttjdd|Dtd}|Sttdd|D}|S)z Concatenate multiple array of this dtype. Parameters ---------- to_concat : sequence of this type Returns ------- ExtensionArray FNTcSsg|] }|jD]}|qqSr,r)ruaer,r,r-rNsz1TensorArray._concat_same_type..rcSsg|]}|jqSr,r)rurr,r,r-rQs)rhrtryr:r<rFobjectr)rrshould_flattenrErconcatedr,r,r-_concat_same_type7s  zTensorArray._concat_same_typekeyrcCst||}t|tst|rt|}t|tr dd|D}t|tr.t|jt r.|j }|dus=t|t rLt |dkrLt |j|tj|j|<dSt|tttjfr\||j|<dStdt|d)a^ Set one or more values inplace. This method is not required to satisfy the pandas extension array interface. Parameters ---------- key : int, ndarray, or slice When called from, e.g. ``Series.__setitem__``, ``key`` will be one of * scalar int * ndarray of integers. * boolean ndarray * slice object value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object value or values to be set of ``key``. Returns ------- None cSrr,rrr,r,r-rqsz+TensorArray.__setitem__..Nrz__setitem__ with key type 'z' not implemented)rr8rnr<rr=listr rhrgr9rr full_likerrrslicerrrp)r+r!rr,r,r- __setitem__Ts   zTensorArray.__setitem__csDt|trt|}|jdkrt|r|St |S)z, Return for `item in self`. r) r8rnr<r=rr rxrrsuper __contains__)r+rnp_item __class__r,r-r's    zTensorArray.__contains__cCrrSrror,r,r-rrzTensorArray.__repr__cCrrSrror,r,r-rrzTensorArray.__str__cCrrSrror,r,r-_values_for_factorizesz!TensorArray._values_for_factorizeTr{skipnac Kszdg}i}|D]}z||||<WqtyYqwzt|j||jfddi|WSty<td|ddw)a Return a scalar result of performing the reduction operation. Parameters ---------- name : str Name of the function, supported values are: { any, all, min, max, sum, mean, median, prod, std, var, sem, kurt, skew }. skipna : bool, default True If True, skip NaN values. **kwargs Additional keyword arguments passed to the reduction function. Currently, `ddof` is the only supported kwarg. Returns ------- scalar Raises ------ TypeError : subclass does not define reductions ddofr r'z' aggregate not implemented.N)KeyErrorrnSUPPORTED_REDUCERSrr)r+r{r,rsupported_kwargsreducer_kwargskwr,r,r-_reduces  zTensorArray._reducecKrrrrr,r,r-rrzTensorArray.__array__ufuncmethodcs|dd}||D]}t|ttjtjfstSq tdd|D}|r1tdd|D|d<t |||i|}t |turLtfdd|DS|dkrRdSt |S) z` Supports NumPy ufuncs without requiring sloppy coercion to an ndarray. outr,cs$|] }t|tr |jn|VqdSrSr8r:rrurVr,r,r-rws"z.TensorArray.__array_ufunc__..csr8rSr9r:r,r,r-rws c3s|] }t|VqdSrS)rpr:ror,r-rwsatN) getr8r:r<rnumbersNumberrrgetattrrp)r+r5r6inputsrr7rVrr,ror-__array_ufunc__s     zTensorArray.__array_ufunc__na_valuecCs\|dur tjj|}|rtj|j|dd}|S|j|}|S|r)|j}|S|j}|S)aj Convert to a NumPy ndarray. .. versionadded:: 1.0.0 This is similar to :meth:`numpy.asarray`, but may provide additional control over how the conversion is done. Parameters ---------- dtype : str or numpy.dtype, optional The dtype to pass to :meth:`numpy.asarray`. copy : bool, default False Whether to ensure that the returned value is a not a view on another array. Note that ``copy=False`` does not *ensure* that ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensure that a copy is made, even if not strictly necessary. na_value : Any, optional The value to use for missing values. The default value depends on `dtype` and the type of the array. Returns ------- numpy.ndarray NTr) pdapitypes pandas_dtyper<rFrastyper)r+rhrrBr9r,r,r-rs  zTensorArray.to_numpycCrrrror,r,r-rrzTensorArray.numpy_dtypecCrrrror,r,r-r rzTensorArray.numpy_ndimcCrrrror,r,r-rrzTensorArray.numpy_shapecCrrrror,r,r-rrzTensorArray.numpy_sizecCstjj|}t|tr|rt|j}|S|}|Stjj |r)tjj |sEt dd|jD}t|tj rC|j|ddS|Stjj |rgt jt|td}tt|D] }|j|||<q[|S|jj||d}|S)a Cast to a NumPy array with 'dtype'. Parameters ---------- dtype : str or dtype Typecode or data-type to which the array is cast. copy : bool, default True Whether to copy the data, even if not necessary. If False, a copy is made only if the old dtype does not match the new dtype. Returns ------- array : ndarray NumPy ndarray with 'dtype' for its dtype. cSsg|]}t|qSr,rT)rutr,r,r-r<rz&TensorArray.astype..Frr)rCrDrErFr8rgr:rris_object_dtypeis_string_dtyper<rF StringDtyperremptyrrr rG)r+rhrr9rr,r,r-rG#s*    zTensorArray.astypecC&|jj|||d}|dur|St|S)ab Test whether any array element along a given axis evaluates to True. See numpy.any() documentation for more information https://numpy.org/doc/stable/reference/generated/numpy.any.html#numpy.any :param axis: Axis or axes along which a logical OR reduction is performed. :param out: Alternate output array in which to place the result. :param keepdims: If this is set to True, the axes which are reduced are left in the result as dimensions with size one. :return: single boolean unless axis is not None else TensorArray r r7keepdimsN)rrr:r+r r7rOrr,r,r-rJszTensorArray.anycCrM)a Test whether all array elements along a given axis evaluate to True. :param axis: Axis or axes along which a logical AND reduction is performed. :param out: Alternate output array in which to place the result. :param keepdims: If this is set to True, the axes which are reduced are left in the result as dimensions with size one. :return: single boolean unless axis is not None else TensorArray rNN)rrxr:rPr,r,r-rx[s zTensorArray.allcCs.ddlm}m}|jr||jS||jS)a Convert this TensorArray to an ArrowTensorArray extension array. This and TensorDtype.__from_arrow__ make up the Pandas extension type + array <--> Arrow extension type + array interoperability protocol. See https://pandas.pydata.org/pandas-docs/stable/development/extending.html#compatibility-with-apache-arrow for more information. r)ArrowTensorArrayArrowVariableShapedTensorArray)$ray.air.util.tensor_extensions.arrowrQrRry from_numpyr)r+rprQrRr,r,r-__arrow_array__is   zTensorArray.__arrow_array__cCr)r)rhrror,r,r-r}s zTensorArray._is_boolean)rr:)FN)TrS)NNF)Crrrrr<rrxrrrrrrrrr0rrr rrnrrlrrr r rrCrD extensionsExtensionArrayrrr$rrrExtensionDtyperhryrrrrr r%r'rrrr+rUr4rrrA no_defaultrrrrrrGrUr __classcell__r,r,r)r-r:jsV ;   3   " m  + & +      '  r:scCs(|jjtjuo|j ot|jdtjS)aReturn whether the provided pandas Series column needs a tensor extension representation. This tensor extension representation provides more efficient slicing and interop with ML frameworks. Args: s: The pandas Series column that may need to be represented using the tensor extension. Returns: Whether the provided Series needs a tensor extension representation. r)rhrpr<rrLr8ilocr)r[r,r,r-column_needs_tensor_extensions&r])Br=ostypingrrrrrrrnumpyr<pandasrCpyarrowrpackaging.versionr pandas._typingr pandas.compatr pandas.core.dtypes.genericr r pandas.core.indexersrr$ray.air.util.tensor_extensions.utilsrrray.util.annotationsrr ImportErrorrrUrKr`_FORMATTER_ENABLED_ENV_VARgetenv __version__r7rc formatter_clsrd_format_stringsr;_patched_by_ray_datasetsrDrVregister_extension_dtyperXrgExtensionScalarOpsMixinrrrnrWr:_add_arithmetic_ops_add_comparison_ops_add_logical_opsSeriesr r]r,r,r,r-sz $      *<  & 6> .