o cit@s~ddlmZmZddlmZddlmZmZmZddl Z ddl Z ddl Z ddlmZddlmZddlmZmZddlmZed d Gd d d eZed d Gd ddeZed d GdddeZed d GdddeZed d GdddeZ    d'dedeedededeeeefdedeeeeefffdd Zd!e j deddfd"d#Z!d$e j"defd%d&Z#dS)()Counter OrderedDict)partial)DictListOptionalN) BatchFormat)Dataset) PreprocessorPreprocessorNotFittedException) PublicAPIalpha) stabilityc @sdeZdZdZddddeededeeefdd Zd e d e fd d Z de j fddZddZdS)OrdinalEncodera Encode values within columns as ordered integer values. :class:`OrdinalEncoder` encodes categorical features as integers that range from :math:`0` to :math:`n - 1`, where :math:`n` is the number of categories. If you transform a value that isn't in the fitted datset, then the value is encoded as ``float("nan")``. Columns must contain either hashable values or lists of hashable values. Also, you can't have both scalars and lists in the same column. Examples: Use :class:`OrdinalEncoder` to encode categorical features as integers. >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import OrdinalEncoder >>> df = pd.DataFrame({ ... "sex": ["male", "female", "male", "female"], ... "level": ["L4", "L5", "L3", "L4"], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder = OrdinalEncoder(columns=["sex", "level"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP sex level 0 1 1 1 0 2 2 1 0 3 0 1 :class:`OrdinalEncoder` can also be used in append mode by providing the name of the output_columns that should hold the encoded values. >>> encoder = OrdinalEncoder(columns=["sex", "level"], output_columns=["sex_encoded", "level_encoded"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP sex level sex_encoded level_encoded 0 male L4 1 1 1 female L5 0 2 2 male L3 1 0 3 female L4 0 1 If you transform a value not present in the original dataset, then the value is encoded as ``float("nan")``. >>> df = pd.DataFrame({"sex": ["female"], "level": ["L6"]}) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder.transform(ds).to_pandas() # doctest: +SKIP sex level 0 0 NaN :class:`OrdinalEncoder` can also encode categories in a list. >>> df = pd.DataFrame({ ... "name": ["Shaolin Soccer", "Moana", "The Smartest Guys in the Room"], ... "genre": [ ... ["comedy", "action", "sports"], ... ["animation", "comedy", "action"], ... ["documentary"], ... ], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder = OrdinalEncoder(columns=["genre"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP name genre 0 Shaolin Soccer [2, 0, 4] 1 Moana [1, 2, 0] 2 The Smartest Guys in the Room [3] Args: columns: The columns to separately encode. encode_lists: If ``True``, encode list elements. If ``False``, encode whole lists (i.e., replace each list with an integer). ``True`` by default. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. .. seealso:: :class:`OneHotEncoder` Another preprocessor that encodes categorical data. TN) encode_listsoutput_columnscolumnsrrcC||_||_t|||_dSN)rrr #_derive_and_validate_output_columnsr)selfrrrrR/home/ubuntu/.local/lib/python3.10/site-packages/ray/data/preprocessors/encoder.py__init__f  zOrdinalEncoder.__init__datasetreturncCst||j|jd|_|S)Nr)_get_unique_value_indicesrrstats_rrrrr_fitts zOrdinalEncoder._fitdfcsXt|gjRdtdtffdd dtjffdd }|j||j<|S)Nelementnamecsfdd|DS)Ncs$g|]}jdd|qS)unique_values())rget.0x)r$rrr ~s$zIOrdinalEncoder._transform_pandas..encode_list..r)r#r$rr$r encode_list}sz5OrdinalEncoder._transform_pandas..encode_listscsXtrjrtjdSfdd}|Sjdjd}|S)Nr-cs"t|}jdjd|SNr%r&)tuplerr$r'r#)r/rrrlist_as_categoryszZOrdinalEncoder._transform_pandas..column_ordinal_encoder..list_as_categoryr%r&)_is_series_composed_of_listsrmaprr$applyr)r/r3s_valuesr.r)r/rcolumn_ordinal_encoders  z@OrdinalEncoder._transform_pandas..column_ordinal_encoder) _validate_dfrliststrpdSeriesr6r)rr"r9rr8r_transform_pandaszs z OrdinalEncoder._transform_pandascC&|jjd|jd|jd|jdS)N (columns=z, encode_lists=, output_columns=r&) __class____name__rrrr,rrr__repr__ zOrdinalEncoder.__repr__)rD __module__ __qualname____doc__rr<boolrrr r r!r= DataFramer?rErrrrrsY   rc @peZdZdZddddeedeeeefdeeefddZ d e d e fd d Z d e jfddZddZdS) OneHotEncodera- `One-hot encode `_ categorical data. This preprocessor transforms each specified column into a one-hot encoded vector. Each element in the vector corresponds to a unique category in the column, with a value of 1 if the category matches and 0 otherwise. If a category is infrequent (based on ``max_categories``) or not present in the fitted dataset, it is encoded as all 0s. Columns must contain hashable objects or lists of hashable objects. .. note:: Lists are treated as categories. If you want to encode individual list elements, use :class:`MultiHotEncoder`. Example: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import OneHotEncoder >>> >>> df = pd.DataFrame({"color": ["red", "green", "red", "red", "blue", "green"]}) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder = OneHotEncoder(columns=["color"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP color 0 [0, 0, 1] 1 [0, 1, 0] 2 [0, 0, 1] 3 [0, 0, 1] 4 [1, 0, 0] 5 [0, 1, 0] OneHotEncoder can also be used in append mode by providing the name of the output_columns that should hold the encoded values. >>> encoder = OneHotEncoder(columns=["color"], output_columns=["color_encoded"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP color color_encoded 0 red [0, 0, 1] 1 green [0, 1, 0] 2 red [0, 0, 1] 3 red [0, 0, 1] 4 blue [1, 0, 0] 5 green [0, 1, 0] If you one-hot encode a value that isn't in the fitted dataset, then the value is encoded with zeros. >>> df = pd.DataFrame({"color": ["yellow"]}) >>> batch = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder.transform(batch).to_pandas() # doctest: +SKIP color color_encoded 0 yellow [0, 0, 0] Likewise, if you one-hot encode an infrequent value, then the value is encoded with zeros. >>> encoder = OneHotEncoder(columns=["color"], max_categories={"color": 2}) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP color 0 [1, 0] 1 [0, 1] 2 [1, 0] 3 [1, 0] 4 [0, 0] 5 [0, 1] Args: columns: The columns to separately encode. max_categories: The maximum number of features to create for each column. If a value isn't specified for a column, then a feature is created for every category in that column. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. .. seealso:: :class:`MultiHotEncoder` If you want to encode individual list elements, use :class:`MultiHotEncoder`. :class:`OrdinalEncoder` If your categories are ordered, you may want to use :class:`OrdinalEncoder`. Nmax_categoriesrrrOrcCrrrrOr rrrrrOrrrrrrzOneHotEncoder.__init__rrcCt||j|jdd|_|S)NFrOrrrrOrr rrrr!zOneHotEncoder._fitr"c Cst|g|jRt|j|jD]R\}}t||r#||t||<|jd|d}t|}t j t||ft d}|| |fdd }|dk}d|t |d||t f<|||<q|S) Nr%r&)dtypecSs ||dS)Nr')vmrrrs z1OneHotEncoder._transform_pandas..rWr)r:rziprr4r5r1rlennpzerosintr6to_numpynonzeroastypetolist) rr"column output_columnstatsnum_categoriesone_hotcodes valid_rowsrrrr? s  zOneHotEncoder._transform_pandascCr@NrAz, max_categories=rBr&rCrDrrOrr,rrrrE rFzOneHotEncoder.__repr__rDrGrHrIrr<rrrarr r r!r=rKr?rErrrrrMs]   rMc @rL)MultiHotEncoderaMulti-hot encode categorical data. This preprocessor replaces each list of categories with an :math:`m`-length binary list, where :math:`m` is the number of unique categories in the column or the value specified in ``max_categories``. The :math:`i\\text{-th}` element of the binary list is :math:`1` if category :math:`i` is in the input list and :math:`0` otherwise. Columns must contain hashable objects or lists of hashable objects. Also, you can't have both types in the same column. .. note:: The logic is similar to scikit-learn's [MultiLabelBinarizer][1] Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import MultiHotEncoder >>> >>> df = pd.DataFrame({ ... "name": ["Shaolin Soccer", "Moana", "The Smartest Guys in the Room"], ... "genre": [ ... ["comedy", "action", "sports"], ... ["animation", "comedy", "action"], ... ["documentary"], ... ], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> >>> encoder = MultiHotEncoder(columns=["genre"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP name genre 0 Shaolin Soccer [1, 0, 1, 0, 1] 1 Moana [1, 1, 1, 0, 0] 2 The Smartest Guys in the Room [0, 0, 0, 1, 0] :class:`MultiHotEncoder` can also be used in append mode by providing the name of the output_columns that should hold the encoded values. >>> encoder = MultiHotEncoder(columns=["genre"], output_columns=["genre_encoded"]) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP name genre genre_encoded 0 Shaolin Soccer [comedy, action, sports] [1, 0, 1, 0, 1] 1 Moana [animation, comedy, action] [1, 1, 1, 0, 0] 2 The Smartest Guys in the Room [documentary] [0, 0, 0, 1, 0] If you specify ``max_categories``, then :class:`MultiHotEncoder` creates features for only the most frequent categories. >>> encoder = MultiHotEncoder(columns=["genre"], max_categories={"genre": 3}) >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP name genre 0 Shaolin Soccer [1, 1, 1] 1 Moana [1, 1, 0] 2 The Smartest Guys in the Room [0, 0, 0] >>> encoder.stats_ # doctest: +SKIP OrderedDict([('unique_values(genre)', {'comedy': 0, 'action': 1, 'sports': 2})]) Args: columns: The columns to separately encode. max_categories: The maximum number of features to create for each column. If a value isn't specified for a column, then a feature is created for every unique category in that column. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. .. seealso:: :class:`OneHotEncoder` If you're encoding individual categories instead of lists of categories, use :class:`OneHotEncoder`. :class:`OrdinalEncoder` If your categories are ordered, you may want to use :class:`OrdinalEncoder`. [1]: https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MultiLabelBinarizer.html NrNrrOrcCrrrPrQrrrrzrzMultiHotEncoder.__init__rrcCrR)NTrSrTr rrrr!rUzMultiHotEncoder._fitr"cs^t|gjRdtdtffdd }tjjD]\}}||t||d||<q|S)Nr#r$csRt|tjr |}nt|ts|g}jd|d}t|fdd|DS)Nr%r&csg|]}|dqSrrXr(counterrrr+szJMultiHotEncoder._transform_pandas..encode_list..) isinstancer_ndarrayrer;rr)r#r$rhr,rrrr.s   z6MultiHotEncoder._transform_pandas..encode_listr-)r:rr;r<r]rr5r)rr"r.rfrgrr,rr?s  z!MultiHotEncoder._transform_pandascCs&|jjd|jd|jd|jdSrmrnr,rrrrErFzMultiHotEncoder.__repr__rorrrrrp(sT   rpc@speZdZdZdddedeefddZded efd d Z d e j fd dZ dddZ d e j fddZddZdS) LabelEncodera Encode labels as integer targets. :class:`LabelEncoder` encodes labels as integer targets that range from :math:`0` to :math:`n - 1`, where :math:`n` is the number of unique labels. If you transform a label that isn't in the fitted datset, then the label is encoded as ``float("nan")``. Examples: >>> import pandas as pd >>> import ray >>> df = pd.DataFrame({ ... "sepal_width": [5.1, 7, 4.9, 6.2], ... "sepal_height": [3.5, 3.2, 3, 3.4], ... "species": ["setosa", "versicolor", "setosa", "virginica"] ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> >>> from ray.data.preprocessors import LabelEncoder >>> encoder = LabelEncoder(label_column="species") >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP sepal_width sepal_height species 0 5.1 3.5 0 1 7.0 3.2 1 2 4.9 3.0 0 3 6.2 3.4 2 You can also provide the name of the output column that should hold the encoded labels if you want to use :class:`LabelEncoder` in append mode. >>> encoder = LabelEncoder(label_column="species", output_column="species_encoded") >>> encoder.fit_transform(ds).to_pandas() # doctest: +SKIP sepal_width sepal_height species species_encoded 0 5.1 3.5 setosa 0 1 7.0 3.2 versicolor 1 2 4.9 3.0 setosa 0 3 6.2 3.4 virginica 2 If you transform a label not present in the original dataset, then the new label is encoded as ``float("nan")``. >>> df = pd.DataFrame({ ... "sepal_width": [4.2], ... "sepal_height": [2.7], ... "species": ["bracteata"] ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> encoder.transform(ds).to_pandas() # doctest: +SKIP sepal_width sepal_height species 0 4.2 2.7 NaN Args: label_column: A column containing labels that you want to encode. output_column: The name of the column that will contain the encoded labels. If None, the output column will have the same name as the input column. .. seealso:: :class:`OrdinalEncoder` If you're encoding ordered features, use :class:`OrdinalEncoder` instead of :class:`LabelEncoder`. N)rg label_columnrgcCs||_|p||_dSr)rwrg)rrwrgrrrrszLabelEncoder.__init__rrcCst||jg|_|Sr)rrwrr rrrr!szLabelEncoder._fitr"cs:t|jdtjffdd }|j||j<|S)Nr/csjd|jd}||Sr0)rr$r5)r/r7r,rrcolumn_label_encoders z.column_label_encoder)r:rwr=r> transformrg)rr"rxrr,rr?s zLabelEncoder._transform_pandasdsr cCsF|}|tjjtjjfvrtd|}|j|jfdt j i|S)a/Inverse transform the given dataset. Args: ds: Input Dataset that has been fitted and/or transformed. Returns: ray.data.Dataset: The inverse transformed Dataset. Raises: PreprocessorNotFittedException: if ``fit`` is not called yet. z1`fit` must be called before `inverse_transform`, batch_format) fit_statusr FitStatusPARTIALLY_FITTED NOT_FITTEDr _get_transform_config map_batches_inverse_transform_pandasrPANDAS)rrzr|kwargsrrrinverse_transforms  zLabelEncoder.inverse_transformcs.dtjffdd }|j||j<|S)Nr/cs,ddjdjdD}||S)NcSi|]\}}||qSrr)r)keyvaluerrr szXLabelEncoder._inverse_transform_pandas..column_label_decoder..r%r&)rrwitemsr5)r/inverse_valuesr,rrcolumn_label_decoders  zDLabelEncoder._inverse_transform_pandas..column_label_decoder)r=r>rgryrw)rr"rrr,rrs z&LabelEncoder._inverse_transform_pandascCs|jjd|jd|jdS)Nz(label_column=z, output_column=r&)rCrDrwrgr,rrrrE(szLabelEncoder.__repr__)rzr rr )rDrGrHrIr<rrr r r!r=rKr?rrrErrrrrvs@  rvc @speZdZdZ  ddeedeeeej fdeeefddZ de d e fd d Z d ejfd dZddZdS) Categorizera^ Convert columns to ``pd.CategoricalDtype``. Use this preprocessor with frameworks that have built-in support for ``pd.CategoricalDtype`` like LightGBM. .. warning:: If you don't specify ``dtypes``, fit this preprocessor before splitting your dataset into train and test splits. This ensures categories are consistent across splits. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import Categorizer >>> >>> df = pd.DataFrame( ... { ... "sex": ["male", "female", "male", "female"], ... "level": ["L4", "L5", "L3", "L4"], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> categorizer = Categorizer(columns=["sex", "level"]) >>> categorizer.fit_transform(ds).schema().types # doctest: +SKIP [CategoricalDtype(categories=['female', 'male'], ordered=False), CategoricalDtype(categories=['L3', 'L4', 'L5'], ordered=False)] :class:`Categorizer` can also be used in append mode by providing the name of the output_columns that should hold the categorized values. >>> categorizer = Categorizer(columns=["sex", "level"], output_columns=["sex_cat", "level_cat"]) >>> categorizer.fit_transform(ds).to_pandas() # doctest: +SKIP sex level sex_cat level_cat 0 male L4 male L4 1 female L5 female L5 2 male L3 male L3 3 female L4 female L4 If you know the categories in advance, you can specify the categories with the ``dtypes`` parameter. >>> categorizer = Categorizer( ... columns=["sex", "level"], ... dtypes={"level": pd.CategoricalDtype(["L3", "L4", "L5", "L6"], ordered=True)}, ... ) >>> categorizer.fit_transform(ds).schema().types # doctest: +SKIP [CategoricalDtype(categories=['female', 'male'], ordered=False), CategoricalDtype(categories=['L3', 'L4', 'L5', 'L6'], ordered=True)] Args: columns: The columns to convert to ``pd.CategoricalDtype``. dtypes: An optional dictionary that maps columns to ``pd.CategoricalDtype`` objects. If you don't include a column in ``dtypes``, the categories are inferred. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. NrdtypesrcCs&|si}||_||_t|||_dSr)rrr rr)rrrrrrrris zCategorizer.__init__rrcsXfddjD}|rt||ddd}dd|D}ni}ij|}|_S)Ncsg|] }|tjvr|qSr)setrr)rfr,rrr+ysz$Categorizer._fit..Tz{0})drop_na_values key_formatcSs i|] \}}|t|qSr)r=CategoricalDtypekeys)r)rfvalues_indicesrrrrsz$Categorizer._fit..)rrrrr)rrcolumns_to_getunique_indicesrr,rr!xs zCategorizer._fitr"cCs||j|j||j<|Sr)rrdrr)rr"rrrr?szCategorizer._transform_pandascCr@)NrAz , dtypes=rBr&)rCrDrrrr,rrrrEs zCategorizer.__repr__)NN)rDrGrHrIrr<rrr=rrr r r!rKr?rErrrrr,s>   rFunique_values({0})TrrrrrOrrcs|duri}t}|D]}||vrtd|ddq dtjffdd dtjd ttttfffd d }|j |d d } ddD} | j ddD]#} | D]\} } | D]}dd| D}| | t|7<q[qUqOD]/} |r| | }t |}dd| D}t|| | <qut dd| | Drtd| dqut}D]2}||vrddt| |||D|||<qddttt | |D|||<q|S)z8If drop_na_values is True, will silently drop NA values.NzYou set `max_categories` for z, which is not present in .colcsNt|rrtfdd}||S|dd}t|jddS)Ncs||Sr)updater2rrrrupdate_counters zY_get_unique_value_indices..get_pd_value_counts_per_column..update_countercSst|Sr)r1)r*rrrr[szS_get_unique_value_indices..get_pd_value_counts_per_column..F)dropna)r4rr5 value_countsto_dict)rrrrrrget_pd_value_counts_per_columns  zA_get_unique_value_indices..get_pd_value_counts_per_columnr"rcsJ|j}i}D]}||vr||g||<q td|d||S)NzColumn 'z2' does not exist in DataFrame, which has columns: )rre ValueError)r" df_columnsresultr)rrrrget_pd_value_countss z6_get_unique_value_indices..get_pd_value_countspandas)r{cSsi|]}|tqSr)r)r)rrrrrsz-_get_unique_value_indices..) batch_sizecSsi|] \}}|dur||qSrrr)krYrrrrscSs i|] \}}t|s||qSrr=isnullrrrrrs css|]}t|VqdSrr)r)rrrr sz,_get_unique_value_indices..zUnable to fit column 'zJ' because it contains null values. Consider imputing missing values first.cSsi|] \}}|d|qSrqrr)jrrrrrscSrrrrrrrrs )rrr=r>rKrrr<rr iter_batchesrdictanyr enumerate most_commonformatsortedr)rrrrrOr columns_setrfrrfinal_countersbatchrcountersrs counter_dictsanitized_dictunique_values_with_indicesr)rrrrrsZ &  rr"cs*fdd|D}|rtd|ddS)Ncs"g|] }|jr|qSr)rvaluesrrr"rrr+s"z _validate_df..zUnable to transform columns zJ because they contain null values. Consider imputing missing values first.)r)r"r null_columnsrrrr:s  r:seriescCs4tdd|Dd}tjj|jot|ttj fS)Ncss|] }|dur|VqdSrr)r)r#rrrrsz/_is_series_composed_of_lists..) nextrapitypesis_object_dtyperVrtr;r_ru)rfirst_not_none_elementrrrr4s  r4)FrNT)$ collectionsrr functoolsrtypingrrrnumpyr_rr=pandas.api.types"ray.air.util.data_batch_conversionrray.datar ray.data.preprocessorr r ray.util.annotationsr rrMrprvrr<rJrarrKr:r>r4rrrrsZ      k Y