o
    biC                     @   st  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z zd dlZW n ey5   dZY nw ejfZerDeejejf ZG dd dZG dd	 d	eZG d
d deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZe dg dZdd Zd d! Z d"d# Z!d(d$d%Z"d&d' Z#dS ))    N)backend)tree)data_adapter_utils
tensorflowc                   @   s\   e Zd ZdZdd Zdd Zedd Zedd	 Zed
d Z	edd Z
edd ZdS )	Sliceableaa  `Sliceable` wrapping a tensor.

    A `Sliceable` implements the subscript operator to slice or index against
    the first dimension of the array. It also has conversion methods for each
    one of the backends.

    Args:
        array: the native array or tensor to wrap.

    Attributes:
        shape: the shape of the full dense native array.
    c                 C   s
   || _ d S Narrayselfr
    r   b/home/ubuntu/.local/lib/python3.10/site-packages/keras/src/trainers/data_adapters/array_slicing.py__init__'   s   
zSliceable.__init__c                 C   s
   | j | S )a/  Select elements in the 0th dimension.

        Args:
            indices: the indices to select. Only needs to support one dimension,
                the 0th dimension. Should support a `slice` or a list, tuple,
                `np.array` or 1D tensor.
        Returns: A slice of `self.array`.
        r	   r   indicesr   r   r   __getitem__*   s   
	zSliceable.__getitem__c                 C   s
   | |S )zCast a tensor to a different dtype.

        Only called on a full array as provided by the user.

        Args:
            x: the tensor to cast.
        Returns: the cast tensor.
        )astype)clsxdtyper   r   r   cast5   s   

zSliceable.castc                 C      |S )zConvert a tensor to a NumPy array.

        Only called after slicing using `__getitem__`.

        Args:
            x: the tensor to convert.
        Returns: the converted tensor.
        r   r   r   r   r   r   convert_to_numpyA   s   
zSliceable.convert_to_numpyc                 C   r   )ak  Convert a tensor to something compatible with `tf.data.Dataset`.

        This can be a NumPy array, `tf.Tensor` or any other type of tensor that
        `tf.data.Dataset.from_tensors` can consume.
        Only called on a full array as provided by the user.

        Args:
            x: the tensor to convert.
        Returns: converted version tensor.
        r   r   r   r   r    convert_to_tf_dataset_compatibleM      z*Sliceable.convert_to_tf_dataset_compatiblec                 C   r   )a`  Convert a tensor to something that the JAX backend can consume.

        This can be a `JAX` array, `JAXSparse` or a NumPy array.
        Only called after slicing using `__getitem__`.
        Used to convert sparse tensors and densify ragged tensors.

        Args:
            x: the tensor to convert.
        Returns: the converted tensor.
        r   r   r   r   r   convert_to_jax_compatible[   r   z#Sliceable.convert_to_jax_compatiblec                 C   r   )a  Convert a tensor to something that the Torch backend can consume.

        This can be a Torch tensor, NumPy array or any other type of tensor that
        `keras.backend.torch.core.convert_to_tensor()` can consume.
        Only called after slicing using `__getitem__`.
        Used to densify sparse tensors and ragged tensors.

        Args:
            x: the tensor to convert.
        Returns: the converted tensor.
        r   r   r   r   r   convert_to_torch_compatiblei   s   z%Sliceable.convert_to_torch_compatibleN)__name__
__module____qualname____doc__r   r   classmethodr   r   r   r   r   r   r   r   r   r      s    



r   c                   @      e Zd ZdS )NumpySliceableNr   r    r!   r   r   r   r   r%   y       r%   c                   @   s,   e Zd Zdd Zedd Zedd ZdS )TensorflowSliceablec                 C   s2   ddl m} t|tr| j| S |j| j|ddS )Nr   r   axis)keras.src.utils.module_utilsr   
isinstanceslicer
   gather)r   r   tfr   r   r   r   ~   s   

zTensorflowSliceable.__getitem__c                 C      ddl m} |||S Nr   )r   )!keras.src.backend.tensorflow.corer   r   r   r   r   r   r   r   r         
zTensorflowSliceable.castc                 C      ddl m} ||S Nr   r   )r2   r   r   r   r   r   r   r   r         z$TensorflowSliceable.convert_to_numpyN)r   r    r!   r   r#   r   r   r   r   r   r   r(   }   s    
r(   c                   @   $   e Zd Zedd Zedd ZdS )TensorflowRaggedSliceablec                 C   
   |  |S r   r7   r   r   r   r   r         
z3TensorflowRaggedSliceable.convert_to_jax_compatiblec                 C      |  S r   )	to_tensorr   r   r   r   r         z5TensorflowRaggedSliceable.convert_to_torch_compatibleN)r   r    r!   r#   r   r   r   r   r   r   r;      s
    
r;   c                       sT   e Zd Z fddZedd Zdd Zedd Zed	d
 Z	edd Z
  ZS )TensorflowSparseSliceablec                    s   t  t| d S r   )superr   to_tensorflow_sparse_wrapperr   	__class__r   r   r      s   z"TensorflowSparseSliceable.__init__c                 C   s
   | j jjS r   )r
   sparseshape)r   r   r   r   rG      r=   zTensorflowSparseSliceable.shapec                 C   s   t | j|S r   )slice_tensorflow_sparse_wrapperr
   r   r   r   r   r         z%TensorflowSparseSliceable.__getitem__c                 C   s   t |S r   )rC   r   r   r   r   r      r@   z:TensorflowSparseSliceable.convert_to_tf_dataset_compatiblec                 C   
   t |S r   )r   tf_sparse_to_jax_sparser   r   r   r   r      r=   z3TensorflowSparseSliceable.convert_to_jax_compatiblec                 C   s   ddl m} ||S )Nr   )rF   )keras.src.backend.tensorflowrF   sparse_to_dense)r   r   	tf_sparser   r   r   r      r4   z5TensorflowSparseSliceable.convert_to_torch_compatible)r   r    r!   r   propertyrG   r   r#   r   r   r   __classcell__r   r   rD   r   rA      s    


rA   c                   @   s8   e Zd Zdd Zedd Zedd Zedd Zd	S )
JaxSparseSliceablec                 C   s   | j |df S )N.r	   r   r   r   r   r      s   zJaxSparseSliceable.__getitem__c                 C   r5   r6   )keras.src.backend.jax.corer   r8   r   r   r   r      r9   z#JaxSparseSliceable.convert_to_numpyc                 C      t t|S r   )rC   r   jax_sparse_to_tf_sparse)r   r
   r   r   r   r         z3JaxSparseSliceable.convert_to_tf_dataset_compatiblec                 C   r>   r   todenser   r   r   r   r      r@   z.JaxSparseSliceable.convert_to_torch_compatibleN)r   r    r!   r   r#   r   r   r   r   r   r   r   rQ      s    

rQ   c                   @   r:   )TorchSliceablec                 C   r0   r1   )keras.src.backend.torch.corer   r3   r   r   r   r      r4   zTorchSliceable.castc                 C   r5   r6   )rY   r   r8   r   r   r   r      r9   zTorchSliceable.convert_to_numpyN)r   r    r!   r#   r   r   r   r   r   r   rX      s
    
rX   c                   @   sD   e Zd Zdd Zedd Zedd Zedd Zed	d
 ZdS )PandasSliceablec                 C   s   | j j| S r   )r
   ilocr   r   r   r   r      rI   zPandasSliceable.__getitem__c                 C   r>   r   )to_numpyr   r   r   r   r      r@   z PandasSliceable.convert_to_numpyc                 C   r<   r   r7   r   r   r   r   r      r=   z0PandasSliceable.convert_to_tf_dataset_compatiblec                 C   r<   r   r7   r   r   r   r   r      r=   z)PandasSliceable.convert_to_jax_compatiblec                 C   r<   r   r7   r   r   r   r   r      r=   z+PandasSliceable.convert_to_torch_compatibleN)	r   r    r!   r   r#   r   r   r   r   r   r   r   r   rZ      s    


rZ   c                   @   r$   )PandasDataFrameSliceableNr&   r   r   r   r   r]      r'   r]   c                   @   s   e Zd Zedd ZdS )PandasSeriesSliceablec                 C   s   t j| ddS )Nr)   )npexpand_dimsr\   r   r   r   r   r      s   z&PandasSeriesSliceable.convert_to_numpyN)r   r    r!   r#   r   r   r   r   r   r^      s    r^   c                       sL   e Zd Z fddZedd Zedd Zedd Zed	d
 Z  Z	S )ScipySparseSliceablec                    s   t  |  d S r   )rB   r   tocsrr   rD   r   r   r      s   zScipySparseSliceable.__init__c                 C   r>   r   rV   r   r   r   r   r      r@   z%ScipySparseSliceable.convert_to_numpyc                 C   rS   r   )rC   r   scipy_sparse_to_tf_sparser   r   r   r   r     rU   z5ScipySparseSliceable.convert_to_tf_dataset_compatiblec                 C   rJ   r   )r   scipy_sparse_to_jax_sparser   r   r   r   r   	  r=   z.ScipySparseSliceable.convert_to_jax_compatiblec                 C   r>   r   rV   r   r   r   r   r     r@   z0ScipySparseSliceable.convert_to_torch_compatible)
r   r    r!   r   r#   r   r   r   r   rP   r   r   rD   r   rb      s    


rb   TensorflowSparseWrapper)rF   ragged_indicesragged_valuesc                 C   sf   ddl m} | jd d df }|jj| }||j	| j||j
}|j	| j|}t| ||S )Nr   r   )r+   r   r   experimentalRowPartitionfrom_value_rowids
row_splitsr   RaggedTensorfrom_row_splitsint64valuesrf   )rF   r/   row_idsrl   rg   rh   r   r   r   rC   %  s   rC   c                 C   s   ddl m} t|tr| j| }| j| }|j|j }n(|| j|}|| j|}t|t	r4t
|}n|jd }|d u rD||d }| }|jd d dd f }|j||d|gdd}|j}|ft| jj dd   }||||S )Nr   r      r_   r)   )r+   r   r,   r-   rg   rh   stopstartr.   listlenrG   value_rowidsflat_valuesconcatra   tuplerF   as_listSparseTensor)sparse_wrapperr   r/   sparse_indicessparse_values	batch_dimrq   sparse_shaper   r   r   rH   4  s,   





rH   c                 C   sD   | d u p!t | tp!t| p!t| p!t| p!t| p!t| dS )N	__array__)r,   ARRAY_TYPESr   is_tensorflow_tensoris_jax_arrayis_torch_tensoris_scipy_sparsehasattrr   r   r   r   can_slice_arrayR  s   r   c                    s    fdd}t || S )a  Convert a structure of arrays into `Sliceable` instances

    Args:
        arrays: the arrays to convert.
        target_backend: the target backend for the output:
            - `None` indicates that `arrays` will be wrapped into `Sliceable`s
              as-is without using a different representation. This is used by
              `train_validation_split()`.
            - `tensorflow` indicates that
              `Sliceable.convert_to_tf_dataset_compatible` will be called. The
              returned structure therefore contains arrays, not `Sliceable`s.
            - `numpy`, `jax` or `torch` indices that the arrays will eventually
              be converted to this backend type after slicing. In this case,
              the intermediary `Sliceable`s may use a different representation
              from the input `arrays` for better performance.
    Returns: the same structure with `Sliceable` instances or arrays.
    c                    s  | d u r| S t | tjr)t| jdkr)t dkr)tdd | D r)tj| dd} t | tjr2t	}nqt
| rJt
| r?t}ndt
| rGt}n\t}nYt
| r_t
| rWt}nLt| } t	}nDt
| rgt}n<td urtt | tjrtt}n/td urt | tjrt}n"t
| rt}nt| drt| } t	}ntd|  d	t |  d
dd  d }td urt | tjrt! fdd| j"j#D rt$ }n	 | jrt$ }|d ur|%| |} d u r|| S dkr|&| S dkr|ttfv rt| } t	}|| S )Nobjectr   c                 s   s    | ]}t |tV  qd S r   )r,   str).0er   r   r   	<genexpr>z  s    zEconvert_to_sliceable.<locals>.convert_single_array.<locals>.<genexpr>stringr   r   zExpected a NumPy array, tf.Tensor, tf.RaggedTensor, tf.SparseTensor, jax.np.ndarray, jax.experimental.sparse.JAXSparse, torch.Tensor, Pandas Dataframe, or Pandas Series. Received invalid input: z
 (of type )c                 S   s&   | t uot| ot| t k S r   )r   r   is_float_dtypestandardize_dtypefloatxr   r   r   r   is_non_floatx_float  s
   zOconvert_to_sliceable.<locals>.convert_single_array.<locals>.is_non_floatx_floatc                 3   s    | ]} |V  qd S r   r   )r   dr   r   r   r         jax)'r,   r`   ndarrayr   r   r   allr/   convert_to_tensorr%   r   r   is_tensorflow_raggedr;   is_tensorflow_sparserA   r(   r   is_jax_sparserQ   asarrayr   rX   pandas	DataFramer]   Seriesr^   r   rb   r   
ValueErrortypeanydtypesrp   r   r   r   )r   sliceable_class
cast_dtypetarget_backendr   r   convert_single_arrayq  sz   










	



z2convert_to_sliceable.<locals>.convert_single_array)r   map_structure)arraysr   r   r   r   r   convert_to_sliceable^  s   Yr   c           	         s   t | }dd |D }|rtd| tdd |D r"| | fS d}|D ]
}|dur0|} nq&t|jd ttd|  dksKkrVtd	 d
| ddd  t| }t 	 fdd|}t 	 fdd|}||fS )a  Split arrays into train and validation subsets in deterministic order.

    The last part of data will become validation data.

    Args:
        arrays: Tensors to split. Allowed inputs are arbitrarily nested
            structures of Tensors and NumPy arrays.
        validation_split: Float between 0 and 1. The proportion of the dataset
            to include in the validation split. The rest of the dataset will be
            included in the training split.

    Returns:
        `(train_arrays, validation_arrays)`
    c                 S   s   g | ]
}t |st|qS r   )r   r   r   tr   r   r   
<listcomp>  s    z*train_validation_split.<locals>.<listcomp>zpArgument `validation_split` is only supported for tensors or NumPy arrays.Found incompatible type in the input: c                 s   s    | ]}|d u V  qd S r   r   r   r   r   r   r     r   z)train_validation_split.<locals>.<genexpr>Nr   g      ?zTraining data contains zs samples, which is not sufficient to split it into a validation and training set as specified by `validation_split=zV`. Either provide more data, or a different value for the `validation_split` argument.c                 S   s   | d u r| S | || S r   r   )r   rt   endr   r   r   _split  s   z&train_validation_split.<locals>._splitc                    s    | ddS )Nr   rt   r   r   r   )r   split_atr   r   <lambda>      z(train_validation_split.<locals>.<lambda>c                    s    | dS )Nr   r   r   r   r   r   r   r   r     r   )
r   flattenr   r   intrG   mathfloorr   r   )	r   validation_splitflat_arraysunsplitablefirst_non_noner   
sliceablestrain_arrays
val_arraysr   r   r   train_validation_split  sB   
r   r   )$collectionsr   numpyr`   	keras.srcr   r    keras.src.trainers.data_adaptersr   r+   r   r/   r   ImportErrorr   r   r   r   r   r%   r(   r;   rA   rQ   rX   rZ   r]   r^   rb   
namedtuplerf   rC   rH   r   r   r   r   r   r   r   <module>   sD    `
'
o