o `۷iQ@sXddlmZmZmZmZmZmZddlZddl Z ddl Z ddl mZddlmZmZmZmZmZmZddlmZddlmZmZddlmZddlmZddl m!Z!m"Z"er\dd l#m$Z$d Z%e"d d ed ddGdddeZ&e"d d ed ddGdddeZ'e"d d ed ddGdddeZ(e"d d ed ddGdddeZ)dS)) TYPE_CHECKINGAnyDictListOptionalTupleN)AbsMaxApproximateQuantileMaxMeanMinStd) BlockAccessor) PreprocessorSerializablePreprocessorBase)SerializablePreprocessor) BatchFormat) DeveloperAPI PublicAPI)Datasetg:0yE>alpha) stabilityz$io.ray.preprocessors.standard_scaler)version identifierc seZdZdZd!deedeeeffdd Zddd efd d Z d e j fd dZ e dejdeded ejfddZdejd ejfddZeed efddZd eeeffddZdeeefdefddZdd ZZS)"StandardScalerav Translate and scale each column by its mean and standard deviation, respectively. The general formula is given by .. math:: x' = \frac{x - \bar{x}}{s} where :math:`x` is the column, :math:`x'` is the transformed column, :math:`\bar{x}` is the column average, and :math:`s` is the column's sample standard deviation. If :math:`s = 0` (i.e., the column is constant-valued), then the transformed column will contain zeros. .. warning:: :class:`StandardScaler` works best when your data is normal. If your data isn't approximately normal, then the transformed features won't be meaningful. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import StandardScaler >>> >>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]}) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 1 1 0 -3 1 2 2 3 1 Columns are scaled separately. >>> preprocessor = StandardScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.224745 -0.707107 1 1 0.000000 -0.707107 1 2 1.224745 1.414214 1 Constant-valued columns get filled with zeros. >>> preprocessor = StandardScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 0.0 1 0 -3 0.0 2 2 3 0.0 >>> preprocessor = StandardScaler( ... columns=["X1", "X2"], ... output_columns=["X1_scaled", "X2_scaled"] ... ) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 -1.224745 -0.707107 1 0 -3 1 0.000000 -0.707107 2 2 3 1 1.224745 1.414214 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. Ncolumnsoutput_columnsc"t||_t|||_dSNsuper__init__rr#_derive_and_validate_output_columnsrselfrr __class__S/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/data/preprocessors/scaler.pyr"^  zStandardScaler.__init__datasetrreturncCs,|jjt|jd|jjdd|jd|S)N) aggregator_fnrcSs t|ddS)Nr)ddof)r )colr(r(r)ks z%StandardScaler._fit..)stat_computation_planadd_aggregatorr r)r%r+r(r(r)_fiteszStandardScaler._fitdfc.dtjffdd }|j||j<|S)Nscsbjd|jd}jd|jd}|dus|dur%tj|dd<|S|tkr+d}|||S)Nmean()std(r)stats_namenpnan_EPSILON)r6s_means_stdr%r(r)column_standard_scalerqs z@StandardScaler._transform_pandas..column_standard_scalerpdSeriesr transformr)r%r4rBr(rAr)_transform_pandaspsz StandardScaler._transform_pandascolumnmeanstdc Cs4|tkrd}tt|tt|tt|S)Nr)r>pcdividesubtractpascalarfloat)rHrIrJr(r(r) _scale_columns  zStandardScaler._scale_columntablec sfdd|jD}t|j|j|D]C\}}}|jd|d}|jd|d}|dus1|durEtjt||jd}t  ||q| |||} t  || qS)z/Transform using fast native PyArrow operations.csg|]}|qSr()rH).0 input_colrRr(r) sz3StandardScaler._transform_arrow..r7r8r9N)type) rziprr:rNnullslenrWr for_block upsert_columnrQ) r%rR input_columnsrT output_colrHr?r@ null_array scaled_columnr(rUr)_transform_arrows"   zStandardScaler._transform_arrowcCstjSr)rARROW)clsr(r(r)preferred_batch_formatsz%StandardScaler.preferred_batch_formatcC|j|jt|dddSN_fitted)rrrgrrgetattrrAr(r(r)_get_serializable_fields z'StandardScaler._get_serializable_fieldsfieldsrcC$|d|_|d|_|d|_dSNrrrgrrgetrgr%rlrr(r(r)_set_serializable_fields  z'StandardScaler._set_serializable_fieldscC|jjd|jd|jdSN (columns=z, output_columns=r8r'__name__rrrAr(r(r)__repr__zStandardScaler.__repr__r)rx __module__ __qualname____doc__rstrrr"rr3rD DataFramerG staticmethodrNArrayrPrQTablera classmethodrrrdrrrjintrrry __classcell__r(r(r&r)rs$C   rz#io.ray.preprocessors.min_max_scalerceZdZdZddeedeeeffdd Zddd efd d Z d e j fd dZ d e eeffddZde eefdefddZddZZS) MinMaxScaleraScale each column by its range. The general formula is given by .. math:: x' = \frac{x - \min(x)}{\max{x} - \min{x}} where :math:`x` is the column and :math:`x'` is the transformed column. If :math:`\max{x} - \min{x} = 0` (i.e., the column is constant-valued), then the transformed column will get filled with zeros. Transformed values are always in the range :math:`[0, 1]`. .. tip:: This can be used as an alternative to :py:class:`StandardScaler`. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import MinMaxScaler >>> >>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]}) # noqa: E501 >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 1 1 0 -3 1 2 2 3 1 Columns are scaled separately. >>> preprocessor = MinMaxScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 0.0 0.0 1 1 0.5 0.0 1 2 1.0 1.0 1 Constant-valued columns get filled with zeros. >>> preprocessor = MinMaxScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 0.0 1 0 -3 0.0 2 2 3 0.0 >>> preprocessor = MinMaxScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 0.0 0.0 1 0 -3 1 0.5 0.0 2 2 3 1 1.0 1.0 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. Nrrcrrr r$r&r(r)r"r*zMinMaxScaler.__init__r+rr,cs&fddttfD}|j|_S)Ncs g|] }jD]}||qqSr()r)rSAggr/rAr(r)rV s z%MinMaxScaler._fit..)r r aggregater:r%r+ aggregatesr(rAr)r3 s zMinMaxScaler._fitr4cr5)Nr6csHjd|jd}jd|jd}||}|tkrd}|||S)Nzmin(r8zmax(r)r:r;r>)r6s_mins_maxdiffrAr(r)column_min_max_scalers  z=MinMaxScaler._transform_pandas..column_min_max_scalerrC)r%r4rr(rAr)rG zMinMaxScaler._transform_pandascCrerfrhrAr(r(r)rj rkz%MinMaxScaler._get_serializable_fieldsrlrcCrmrnrorqr(r(r)rr'rsz%MinMaxScaler._set_serializable_fieldscCrtrurwrAr(r(r)ry.rzzMinMaxScaler.__repr__rrxr{r|r}rr~rr"rr3rDrrGrrrjrrrryrr(r(r&r)rs$?rz#io.ray.preprocessors.max_abs_scalercr) MaxAbsScalera@Scale each column by its absolute max value. The general formula is given by .. math:: x' = \frac{x}{\max{\vert x \vert}} where :math:`x` is the column and :math:`x'` is the transformed column. If :math:`\max{\vert x \vert} = 0` (i.e., the column contains all zeros), then the column is unmodified. .. tip:: This is the recommended way to scale sparse data. If you data isn't sparse, you can use :class:`MinMaxScaler` or :class:`StandardScaler` instead. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import MaxAbsScaler >>> >>> df = pd.DataFrame({"X1": [-6, 3], "X2": [2, -4], "X3": [0, 0]}) # noqa: E501 >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -6 2 0 1 3 -4 0 Columns are scaled separately. >>> preprocessor = MaxAbsScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.0 0.5 0 1 0.5 -1.0 0 Zero-valued columns aren't scaled. >>> preprocessor = MaxAbsScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -6 2 0.0 1 3 -4 0.0 >>> preprocessor = MaxAbsScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 -1.0 -1.0 1 0 -3 1 0.0 -1.0 2 2 3 1 1.0 1.0 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. Nrrcrrr r$r&r(r)r"pr*zMaxAbsScaler.__init__r+rr,cCs dd|jD}|j||_|S)NcSsg|]}t|qSr()rrSr/r(r(r)rVxsz%MaxAbsScaler._fit..)rrr:rr(r(r)r3ws zMaxAbsScaler._fitr4cr5)Nr6cs(jd|jd}|dkrd}||S)Nzabs_max(r8rr)r:r;)r6 s_abs_maxrAr(r)column_abs_max_scaler}sz=MaxAbsScaler._transform_pandas..column_abs_max_scalerrC)r%r4rr(rAr)rG|s zMaxAbsScaler._transform_pandascCrerfrhrAr(r(r)rjrkz%MaxAbsScaler._get_serializable_fieldsrlrcCrmrnrorqr(r(r)rrrsz%MaxAbsScaler._set_serializable_fieldscCrtrurwrAr(r(r)ryrzzMaxAbsScaler.__repr__rrr(r(r&r)r2s$;rz"io.ray.preprocessors.robust_scalerc seZdZdZdZddefdeedeeefde eede ffd d Z d d d e fddZ dejfddZd eeeffddZdeeefde fddZddZZS) RobustScaleraI Scale and translate each column using approximate quantiles. The general formula is given by .. math:: x' = \frac{x - \mu_{1/2}}{\mu_h - \mu_l} where :math:`x` is the column, :math:`x'` is the transformed column, :math:`\mu_{1/2}` is the column median. :math:`\mu_{h}` and :math:`\mu_{l}` are the high and low quantiles, respectively. By default, :math:`\mu_{h}` is the third quartile and :math:`\mu_{l}` is the first quartile. Internally, the `ApproximateQuantile` aggregator is used to calculate the approximate quantiles. .. tip:: This scaler works well when your data contains many outliers. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import RobustScaler >>> >>> df = pd.DataFrame({ ... "X1": [1, 2, 3, 4, 5], ... "X2": [13, 5, 14, 2, 8], ... "X3": [1, 2, 2, 2, 3], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 1 13 1 1 2 5 2 2 3 14 2 3 4 2 2 4 5 8 3 :class:`RobustScaler` separately scales each column. >>> preprocessor = RobustScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.0 0.625 1 1 -0.5 -0.375 2 2 0.0 0.750 2 3 0.5 -0.750 2 4 1.0 0.000 3 >>> preprocessor = RobustScaler( ... columns=["X1", "X2"], ... output_columns=["X1_scaled", "X2_scaled"] ... ) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 1 13 1 -1.0 0.625 1 2 5 2 -0.5 -0.375 2 3 14 2 0.0 0.750 3 4 2 2 0.5 -0.750 4 5 8 3 1.0 0.000 Args: columns: The columns to separately scale. quantile_range: A tuple that defines the lower and upper quantiles. Values must be between 0 and 1. Defaults to the 1st and 3rd quartiles: ``(0.25, 0.75)``. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. quantile_precision: Controls the accuracy and memory footprint of the sketch (K in KLL); higher values yield lower error but use more memory. Defaults to 800. See https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html for details on accuracy and size. i )g?g?Nrquantile_rangerquantile_precisioncs.t||_||_||_t|||_dSr)r!r"rrrrr#r)r%rrrrr&r(r)r"s  zRobustScaler.__init__r+rr,csjddjdgfddjD}|j|}i_jD](}|d|d\}}}|jd|d<|jd |d<|jd |d<q!S) Nrg?rcsg|] }t|jdqS))on quantilesr)r rrrr%r(r)rVsz%RobustScaler._fit..zapprox_quantile(r8 low_quantile(median(high_quantile()rrrr:)r%r+r aggregatedr/low_qmed_qhigh_qr(rr)r3s   zRobustScaler._fitr4cr5)Nr6csbjd|jd}jd|jd}jd|jd}||}|dkr+t|S|||S)Nrr8rrr)r:r;r< zeros_like)r6s_low_qs_medians_high_qrrAr(r)column_robust_scalers  z.column_robust_scalerrC)r%r4rr(rAr)rGrzRobustScaler._transform_pandascCs |j|j|j|jt|dddS)Nrg)rrrrrg)rrrrrirAr(r(r)rj&s  z%RobustScaler._get_serializable_fieldsrlrcCs8|d|_|d|_|d|_|d|_|d|_dS)Nrrrrrg)rrrrrprgrqr(r(r)rr/s    z%RobustScaler._set_serializable_fieldscCs&|jjd|jd|jd|jdS)Nrvz, quantile_range=z), output_columns=r8)r'rxrrrrAr(r(r)ry8s zRobustScaler.__repr__)rxr{r|r}DEFAULT_QUANTILE_PRECISIONrr~rrPrrr"rr3rDrrGrrrjrrryrr(r(r&r)rs(K    r)*typingrrrrrrnumpyr<pandasrDpyarrowrNpyarrow.computecomputerKray.data.aggregaterr r r r r ray.data.blockrray.data.preprocessorrr&ray.data.preprocessors.version_supportr#ray.data.util.data_batch_conversionrray.util.annotationsrrray.data.datasetrr>rrrrr(r(r(r)s6        ( o h