o R i\@sddlZddlmZddlmZddlZddlmZddlZddl m Z ddl Z ddl Z ddl Z ddlZzddlZWn eyEdZYnwddlZddlmZmZmZiaiaiaddZd d Zd d Zd dZddZddZ dQddZ!ddZ"ddZ#ddZ$ddZ%ddZ&dd Z'd!d"Z(d#d$Z)d%d&Z*d'd(Z+dQd)d*Z,  ,dRd-d.Z-d/d0Z.dSd1d2Z/d3d4Z0 dTd6d7Z1hd8Z2d9d:Z3d;d<Z4d=d>Z5dQd?d@Z6 dQdAdBZ7dCdDZ8dEdFZ9dGdHZ:dIdJZ;dKdLZdS)UN)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesis_threading_enabledc Cstsjtitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGr2r2I/home/ubuntu/.local/lib/python3.10/site-packages/pyarrow/pandas_compat.pyget_logical_type_map.sR     r4cCst}z||jWStyLt|tjjrYdSt|tjjr,dt|j dYSt|tjj r@|j durz&construct_metadata..cSs"g|] \}}t|ts||fqSr2)r=dict)r{level descriptorr2r2r3r|s )rnrFroFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesattrsz(Could not serialize pd.DataFrame.attrs: z!, defaulting to empty attributes.spandaspyarrow)libraryversion) index_columnscolumn_indexescolumns attributescreatorpandas_versionutf8)zipraryappendrnr=rT_column_name_to_stringswarningswarn UserWarningr`r_get_simple_index_descriptorrSrjsondumps Exceptionr __version__rrencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indexrDcolumn_field_namesserialized_index_levelsnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolrnrorFrhindex_column_metadatanon_str_index_namesrrrrrrer2r2r3construct_metadatas          rcCsTt|\}}t|}d|vrtjdtdd|dkr"|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rrrencodingUTF-8rm)rjrZrrr)rrnrwrxrpr2r2r3r2s rcCsxt|tr|St|tr|dSt|trtttt|St|tr(td|dus6t|t r8t |r8|St|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex levelN) r=rTrdecodetuplemaprrrtrrrKrsrnr2r2r3rFs     rcCs,|jdur|j|vrt|jSd|ddS)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str N__index_level_d__)rnr)indexirr2r2r3_index_level_namems rcCsPt|||}|jjstdt|j|durt|||Sg}g}|dur+t|jng}g}g}|D]-} || } t| } t | rJt d| d| | | d| | | t | q3g} g} t|D]0\} }t|| |} t|t jjr|durt|}n| || d| }| | | |qi|| }|||| | |||fS)NzDuplicate column names found: FSparse pandas data (column ) not supported.)_resolve_columns_of_interestr is_unique ValueErrorlist$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparsertrrT enumeraterr=pd RangeIndex_get_range_index_descriptor)rschemarrrrrrconvert_fieldsrnrrindex_column_namesr index_leveldescr all_namesr2r2r3_get_columns_to_convertsR              rc Cs<g}g}g}g}g}g}|jD]} z|| } d} WnCty\zt|| } Wnttfy8td| dw|durEtd| d|durXt| tjjrXtd| dd} Ynwt | rjt d| d | | } | | | | | | | r| | | | | | q||} | |||||||fS) z Specialized version of _get_columns_to_convert in case a Schema is specified. In that case, the Schema is used as the single point of truth for the table structure (types, which columns are included, order of columns, ...). Fzname 'zF' present in the specified schema is not found in the columns or indexzd' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNz' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTrr) rr<_get_index_level IndexErrorrr=rrrrrtfieldr)rrrrrrrrrrnris_indexrrr2r2r3rs^               rcCs8|}||jjvrt|rt|tdd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). r)rr_is_generated_index_nameintraget_level_values)rrnkeyr2r2r3rs rcCs,zt||WStyt|YSwN)rrrtrTrr2r2r3 _level_name s    rcCs.dt|jt|dt|dt|ddS)Nrangestartstopstep)kindrnrrr)rrnrget_rangeindex_attribute)rr2r2r3rs    rcs(ttdg}fddt|DS)Nrcsg|]}|qSr2)rr{rrr2r3r|#z+_get_index_level_values..)rar`r)rnr2rr3r!srcsR|dur |dur td|dur|j}|S|dur$fdd|D}|Sj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themcsg|] }|jvr|qSr2)rr{crr2r3r|-sz0_resolve_columns_of_interest..)rrr)rrrr2rr3r&src Cst|d||\}}}}}}} }g} | D]T} | j} t| r&tj| ddj} n       rTc" sDt||||\}}}} } } } } |dur.t|t|j}}||dkr,|dkr,t}nd}ts3d}fdddd}|dkrNfddt| | D}nNg}t|*}t| | D]\}}||j rm| ||q[| | ||q[Wdn1swYt |D]\}}t |tjr|||<qd d|D}|durg}t||D] \}}| t||qt|}t| ||| | |||d }|jrt|jnt}||||}d}t|d krz'| d d }|d kr| d d}| d d} | d d}!tt|| |!}Wn tyYnw|||fS)Ndrc s|dur d}d}n|j}|j}z tj||dd}Wn$tjtjtjfy>}z|jd|jd|j f7_|d}~ww|sR|j dkrRt d|d|j d|S) NT)rRrsafezConversion failed for column z with type rzField z( was non-nullable but pandas column had z null values) nullablerRrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrnrQ null_countr)rrfield_nullablerrYr)rr2r3convert_columnps,   z+dataframe_to_arrays..convert_columncSs$t|tjo|jjot|jjtjSr) r=rKndarrayflags contiguous issubclassrQrRinteger)arrr2r2r3_can_definitely_zero_copys z6dataframe_to_arrays.._can_definitely_zero_copycsg|] \}}||qSr2r2)r{rf)rr2r3r|sz'dataframe_to_arrays..cSsg|]}|jqSr2rR)r{xr2r2r3r|srrrrrrr)rrarr cpu_countrrrThreadPoolExecutorrrsubmitrr=FuturerYrrrrhrr~r with_metadatarr)"rrrnthreadsrrrrrrrrrrnrowsncolsrarraysexecutorrrr maybe_futrDfieldsrnrpandas_metadatarhn_rowsrrrrr2)rrr3dataframe_to_arraysWs~                  rcCsd|jjtjkr ||fSt|r$|dur$|j}|j}t ||}||fS|dur.t |j}||fSr) rQrRrKrPr is_datetimetzrCrer timestampfrom_numpy_dtype)rrQrrCrer2r2r3rs  rcCs$ddlmm}|dd}|d}d|vr%tjj||d|dd}n`d|vrZt|j \}} t ||d} t rItj j |d | d d }n<|}|rY|j|||j| d } | Sn+d |vr|d }t|dksjJ||d} || } t| ds}td| |}n|}|r|j||dS||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryr^)rbr^r_rF)rQcopy)rklassrQpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r)pandas.core.internalscore internalsgetrcategorical_type from_codesrK datetime_datarQmake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrarSrr")itemrextension_columns return_block_int block_arrrrrerrQrrn pandas_dtyper2r2r3_reconstruct_blocksF     r5cCs&trd}tj|}tj||dS)NnsrC)ris_v1rrstring_to_tzinfodatetimetz_type)rerCr2r2r3r*s r*Fcspg}g}i}|jj}|s;|dur;|d}|dg}|di}|d} t||}t|| ||\}} t|||||ntj|j } t|g|||t |t |||} |j t j|||t} trddlm} fdd| D}| || | d }||_|Sdd lm}dd lm}fd d| D}| | g}|||}tr|||j}n||}||_|S) Nrrrrr)create_dataframe_from_blockscsg|] }t|ddqS)F)r1r5r{r/rext_columns_dtypesr2r3r|;s z&table_to_dataframe..)rr) BlockManager) DataFramecsg|]}t|qSr2r<r=r>r2r3r|Hs )rrr&_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsr;rr#r@pandasrAr+ _from_mgraxes)optionstablerbignore_metadata types_mapper all_columnsrrrrrrrYr;blocksrr@rArNmgrr2r>r3table_to_dataframes\               rV> r r r r rrr:rrrrrrc Cs|d}|pg}i}tjdur|S|r)|jD]}|j}||} | dur(| ||j<q|jD]%}|j}|j|vrQt|tjrQz|} Wn t yKYq,w| ||j<q,|D]]} z| d} Wn t yi| d} Ynw| d} | |vr| t vrt | } t| tjrt| tj jr|s| |vrqTztj|j| jrWqTWn t yYnwt| dr| || <qTtr|s|jD],}|j|vrtj|jstj|jstj|jr|j|vrtj jtjd||j<q|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. strings_to_categoricalNrornrqr")na_value)rextension_dtyperrRrnr=rBaseExtensionTypeto_pandas_dtypeNotImplementedErrorr<_pandas_supported_numpy_typesr4r StringDtyperD is_dictionaryrrSuses_string_dtype is_stringis_large_stringis_string_viewrKnan) rPcolumns_metadatarRrOrbrW ext_columnsrtypr4col_metarnrQr2r2r3rDbst                       rDcCstdd|Ds JdS)Ncss0|]}|ddurd|vp|dduVqdS)rnNror2rr2r2r3 s  z:_check_data_column_metadata_consistency..)all)rSr2r2r3rFsrFcs|rdd|Dfdd|jD}n|j}t|dkr2tjjjtttj |dd|Dd}ntjj ||r>|dd ndd }t|dkrMt ||}|S) NcSs&i|]}|dt|d|dqSrorn)r&rrr2r2r3 sz-_deserialize_column_index..csg|]}||qSr2r&rzcolumns_name_dictr2r3r|s z-_deserialize_column_index..rcSsg|]}|dqSrr2)r{ col_indexr2r2r3r|r}rrrnr) rrarr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tablerSrcolumns_valuesrr2rnr3rGs&     rGc Cs6dd|D}g}g}|}|D]O}t|tr't|||||\}} } | dur&qn-|ddkrK|d} tjj|d|d|d| d } t| t|krJqn td |d|| || qtj} t|d krt| j j ||d } || fSt|d kr|d } t| | j s| j | |d d} || fS| |j } || fS)NcSsi|] }|d|d|qSrkrmrr2r2r3rlsz&_reconstruct_index..rrrnrrr)rrnzUnrecognized index kind: rrqrr) r=rT_extract_index_levelrrrrarrrr from_arraysrvrE) rPrrSrRfield_name_to_metadata index_arrays index_names result_tablerr index_namerrr2r2r3rCsL           rCc Csl||d}t||}|j|}|dkr|ddfS||}|j|d} d| _||j|}|| |fS)Nrn)rR) _backwards_compatible_index_namerget_field_indexrf to_pandasrn remove_column) rPrror|rR logical_namerrrrr2r2r3rzs        rzcCs||kr t|r dS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namerr2r2r3r)srcCsd}t||duS)Nz^__index_level_\d+__$)rematch)rnpatternr2r2r3rAsrc Cs2tstddddtjdtjtjtjtjd tS)NrHzdatetime64[ns]rT) rr8r7rrrIrfloatingr9r )_pandas_logical_type_maprrKrNrrobject_r2r2r2r3get_pandas_logical_type_mapFs rcCs@t}z||WStyd|vrtjYSt|YSw)a Get the numpy dtype that corresponds to a pandas type. Parameters ---------- pandas_type : str The result of a call to pandas.lib.infer_dtype. Returns ------- dtype : np.dtype The dtype that corresponds to `pandas_type`. r)rr<rKrrQ)rppandas_logical_type_mapr2r2r3_pandas_type_to_numpy_typeYs    rc Cstj}t|ddp |g}t|ddpdg}ddt||idD}g}tdd}|D]\}} } t| } | tjkr?| |}nZ| d krht j |d d d } |j |d d| }trg|t| d }n1| dkrxtjdd|D}n!|jdkr| dkrd| vs| dvr||q+|j| kr|| }|j| kr| d kr|| }||q+t|dkr|j|||jdS|j|d |d j|jdS)a_Construct a pandas MultiIndex from `columns` and column index metadata in `column_indexes`. Parameters ---------- columns : List[pd.Index] The columns coming from a pyarrow.Table column_indexes : List[Dict[str, str]] The column index metadata deserialized from the JSON schema metadata in a :class:`~pyarrow.Table`. Returns ------- result : MultiIndex The index reconstructed using `column_indexes` metadata with levels of the correct type. Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` rNrccSs0g|]\}}||dt|j|ddfqS)rprqN)r&rTrQ)r{rrpr2r2r3r|s  z6_reconstruct_columns_from_metadata..) fillvaluerrr7rrhr_T)utcr9cSsg|]}t|qSr2)r9Decimalrr2r2r3r|rrTr:r)rrIrrq)rQrn)rrr`roperator methodcallerrrKrNrrrr9 to_datetime tz_convertrJas_unitr)rvrQrastyperarrrrn) rrrrlabels levels_dtypes new_levelsencoderrr4 numpy_dtyperQrCr2r2r3rwpsH         rwcCsi}i}|j}|d}dd|D}t|}t|d|}t|dD]q\}} | d} | sC| d} ||kr=|||} | durCd} || } | dkr| d d kr|| } t| jtjj s_q$| d } | sfq$| d }|r|| jj kr| }tj d |d}tj j||d}t|| j||| <||| <q$t|dkrg}g}tt|jD]$}||vr||||||q|||||j|qtjj|t|dS|S)NrcSsg|] }t|tr|qSr2)r=rT)r{idx_colr2r2r3r|s z%_add_any_metadata..rrornNonerrpr7rhr_r6r7rr)r)rrarr&rr=rRrrrBrCrrArrayrrrnrrTabler{)rPrmodified_columnsmodified_fieldsrrn_index_levels n_columnsrrhridxrrh metadata_tz converted tz_aware_typerrrr2r2r3rBs^       rBcCs$tj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrr9dt tz_localizer)seriesrCr2r2r3 make_tz_awares  rr)rNT)NNT)NFN)?rtcollections.abcr concurrentrconcurrent.futures.threadrrr9 itertoolsrrrrrnumpyrK ImportErrorrr pyarrow.librrrrrJrr4r@rOrZrjryrrrrrrrrrrrrrrr5r*rVr]rDrFrGrCrzrrrrrwrBrr2r2r2r3s|      1 m'B>    # d E @S !6 W A