o uyi@sDddlmZddlZddlmZmZddlmZddlm Z m Z m Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZddlmZmZmZddlmZdd lm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:e rddl;ZZ>ed Z?ed Z@dZAdZBdZCGddde!ZDGdddee?eZEGdddee?eZFGdddee@e?feZGGdddeGe@e?fZHGdddee@eZIe dd$d%ZJeJKeDdd(d)ZLeJKe3dd,d)ZLeJKe-dd/d)ZLeJKe/dd2d)ZLeJKe1dd5d)ZLGd6d7d7eEe?eZMed8d8d8d9Gd:d;d;ZNe dd>d?ZOeOKeDdd@d)ZLeOKe3ddAd)ZLeOKe-ddBd)ZLeOKe/ddCd)ZLeOKe1ddDd)ZLe ddFdGZPePKeDddHd)ZLePKe3ddId)ZLePKe-ddJd)ZLePKe/ddKd)ZLePKe1ddLd)ZLGdMdNdNeEeeQe0fZRddPdQZSGdRdSdSeEeeQeQfZTddUdVZUGdWdXdXeEeeVeQfZWddZd[ZXdd]d^ZYeQZZGd_d`d`eEeeZeNfZ[ddbdcZ\dddfdgZ]GdhdidieFe+Z^ddldmZ_ddodpZ`ddqdrZaddtduZbddvdwZcGdxdydyeEee+Zdddd}d~ZeGdddeEee+Zfe dddZgegKe,ddd)ZLegKe*ddd)ZLegKe2ddd)ZLegKe$ddd)ZLegKe'ddd)ZLegKe)ddd)ZLegKe9ddd)ZLdddZhGdddeFeiZjdS)) annotationsN)ABCabstractmethod) dataclass)cached_propertypartialsingledispatch) TYPE_CHECKINGAnyCallableDictGenericListLiteralOptionalSetTupleTypeVarUnion)Field PrivateAttrmodel_validator ResolveError) EMPTY_DICTIcebergBaseModelStructProtocol) BinaryType BooleanTypeDateType DecimalType DoubleType FixedType FloatType IcebergType IntegerTypeListTypeLongTypeMapType NestedField PrimitiveType StringType StructTypeTimestampNanoType TimestampTypeTimestamptzNanoTypeTimestamptzTypeTimeType UnknownTypeUUIDType) NameMappingTPzfield-idziceberg-field-namecseZdZUdZdZded<eedZded<ede d Z d ed <ed e d Z ded<e Zded<dffdd ZdgddZdgddZdhddZdid!d"Zed#d$djd%d&Zedkd'd(Zedld*d+Zedmd-d.Zednd/d0Zedod2d3Zedpd5d6Zdqd8d9Zdrd;d<ZdsdtdAdBZ dsdudDdEZ!edhdFdGZ"edvdIdJZ#dwdMdNZ$edxdPdQZ%dydTdUZ&dzdWdXZ'd=dYd{d[d\Z(ed|d^d_Z)d}dadbZ*d~dddeZ+Z,S)SchemazsA table Schema. Example: >>> from pyiceberg import schema >>> from pyiceberg import types structzLiteral['struct']type)default_factoryTuple[NestedField, ...]fieldsz schema-id)aliasdefaultint schema_idzidentifier-field-ids)r=r:z List[int]identifier_field_idsDict[str, int] _name_to_idr)datar cs,|r||d<tjdi|t||_dS)Nr<)super__init__ index_by_namerC)selfr<rD __class__rES/home/ubuntu/maya3_transcribe/venv/lib/python3.10/site-packages/pyiceberg/schema.pyrGdszSchema.__init__returnstrcCsdddd|jDdS)5Return the string representation of the Schema class.ztable {  cSsg|]}dt|qS)z )rN.0fieldrErErL lz"Schema.__str__..z })joincolumnsrIrErErL__str__jszSchema.__str__cCs.dddd|jDd|jd|jdS)rOzSchema(z, css|]}t|VqdSN)repr)rRcolumnrErErL psz"Schema.__repr__..z , schema_id=z, identifier_field_ids=))rVrWr@rArXrErErL__repr__ns.zSchema.__repr__cCs t|jS)z6Return the length of an instance of the Literal class.)lenr<rXrErErL__len__r zSchema.__len__otherboolcCs^|sdSt|ts dSt|jt|jkrdS|j|jk}tddt|j|jD}|o.|S)z9Return the equality of two instances of the Schema class.Fcss|] \}}||kVqdSrZrE)rRlhsrhsrErErLr]sz Schema.__eq__..) isinstancer7r`rWrAallzip)rIrcidentifier_field_ids_is_equalschema_is_equalrErErL__eq__vs  z Schema.__eq__after)modecCs |jr|jD]}||q|SrZ)rA_validate_identifier_fieldrIfield_idrErErL check_schemas  zSchema.check_schemacC|jS)z A tuple of the top-level fields.)r<rXrErErLrWszSchema.columnsDict[int, NestedField]cCt|S)zReturn an index of field ID to NestedField instance. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. ) index_by_idrXrErErL_lazy_id_to_fieldzSchema._lazy_id_to_fieldDict[int, int]cCru)zReturns an index of field ID to parent field IDs. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )_index_parentsrXrErErL_lazy_id_to_parentrxzSchema._lazy_id_to_parentcCsdd|jDS)zReturn an index of lower-case field names to field IDs. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. cSsi|] \}}||qSrE)lower)rRnamerqrErErL z1Schema._lazy_name_to_id_lower..)rCitemsrXrErErL_lazy_name_to_id_lowerszSchema._lazy_name_to_id_lowerDict[int, str]cCru)zReturn an index of field ID to full name. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )index_name_by_idrXrErErL_lazy_id_to_namerxzSchema._lazy_id_to_nameDict[int, Accessor]cCru)zReturn an index of field ID to accessor. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )build_position_accessorsrXrErErL_lazy_id_to_accessorrxzSchema._lazy_id_to_accessorr,cCs t|jS)zReturn the schema as a struct.)r,r<rXrErErL as_structrbzSchema.as_struct 'pa.Schema'cCddlm}||S)z%Return the schema as an Arrow schema.r)schema_to_pyarrow)pyiceberg.io.pyarrowr)rIrrErErLas_arrow zSchema.as_arrowT name_or_idUnion[str, int]case_sensitivecCstt|tr||jvrtd||j|S|r|j|}n|j|}|dur5td|d||j|S)aFind a field using a field name or field ID. Args: name_or_id (Union[str, int]): Either a field name or a field ID. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Raises: ValueError: When the value cannot be found. Returns: NestedField: The matched NestedField. zCould not find field with id: NzCould not find field with name , case_sensitive=)rgr?rw ValueErrorrCgetrr|)rIrrrqrErErL find_fields   zSchema.find_fieldr$cCs,|j||d}|std|d||jS)adFind a field type using a field name or field ID. Args: name_or_id (Union[str, int]): Either a field name or a field ID. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Returns: NestedField: The type of the matched NestedField. )rrz%Could not find field with name or id r)rr field_type)rIrrrSrErErL find_types zSchema.find_typecCst|jddS)Nr)r>)maxrkeysrXrErErLhighest_field_idszSchema.highest_field_idr4cCr)Nr)create_mapping_from_schema)pyiceberg.table.name_mappingr)rIrrErErL name_mappingrzSchema.name_mapping column_id Optional[str]cCs |j|S)zFind a column name given a column ID. Args: column_id (int): The ID of the column. Returns: str: The column name (or None if the column ID cannot be found). )rr)rIrrErErLfind_column_name zSchema.find_column_name List[str]cCt|jS)z Return a list of all the column names, including nested fields. Excludes short names. Returns: List[str]: The column names. )listrvaluesrXrErErL column_namess zSchema.column_namesrqAccessorcCs"||jvr td||j|S)aFind a schema position accessor given a field ID. Args: field_id (int): The ID of the field. Raises: ValueError: When the value cannot be found. Returns: Accessor: An accessor for the given field ID. z+Could not find accessor for field with id: )rrrprErErLaccessor_for_fields  zSchema.accessor_for_fieldSet[str]cCs@t}|jD]}||}|durtd|||q|S)zwReturn the names of the identifier fields. Returns: Set of names of the identifier fields Nz%Could not find identifier column id: )setrArradd)rIidsrq column_namerErErLidentifier_field_names"s   zSchema.identifier_field_names)rnamesc sbz|r fdd|D}n fdd|D}Wnty+}ztd||d}~wwt|S)aReturn a new schema instance pruned to a subset of columns. Args: names (List[str]): A list of column names. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup for each column name. Defaults to True. Returns: Schema: A new schema with pruned columns. Raises: ValueError: If a column is selected that doesn't exist. csh|]}j|qSrE)rCrRr}rXrErL @z Schema.select..csh|] }j|qSrE)rr|rrXrErLrBrzCould not find column: N)KeyErrorr prune_columns)rIrrrerErXrLselect1s  z Schema.selectSet[int]cCr)z%Return the IDs of the current schema.)rrCrrXrErErL field_idsHszSchema.field_idsNonecCs||}|jjstd|d|jstd|dt|jttfr,td|d|j |j }g}|durH| ||j |}|dus9|rr|| }|jj s`td|jd||jsntd|jd||sJdSdS) zValidate that the field with the given ID is a valid identifier field. Args: field_id: The ID of the field to validate. Raises: ValueError: If the field is not valid. zIdentifier field z$ invalid: not a primitive type fieldz invalid: not a required fieldz+ invalid: must not be float or double fieldNzCannot add field z/ as an identifier field: must not be nested in zA as an identifier field: must not be nested in an optional field )rr is_primitiverrequiredrgr!r#r{rrqappendpop is_structr})rIrqrS parent_idr<parentrErErLroMs,    z!Schema._validate_identifier_fieldformat_versioncCsB|jD]}||jkrt|jd|jd|qdS)zCheck that the schema is compatible for the given table format version. Args: format_version: The Iceberg table format version. Raises: ValueError: If the schema is not compatible for the format version. z is only supported in z' or higher. Current format version is: N)rwrrminimum_format_versionr)rIrrSrErErL"check_format_version_compatibilityrs z)Schema.check_format_version_compatibility)r<r)rDr rMrN)rMr?)rcr rMrd)rMr7)rMr;)rMrt)rMryrMrBrMr)rMr)rMr,)rMrT)rrrrdrMr))rrrrdrMr$)rMr4)rr?rMr)rMr)rqr?rMr)rMr)rrNrrdrMr7)rMr)rqr?rMr)rr?rMr)-__name__ __module__ __qualname____doc__r9__annotations__rtupler<INITIAL_SCHEMA_IDr@rrArrCrGrYr_rarlrrrpropertyrWrrwr{rrrrrrrrrrrrrrrror __classcell__rErErJrLr7UsX                        %r7c@seZdZd7ddZd7ddZd8d d Zd8d d Zd9ddZd9ddZd:ddZ d:ddZ e d;ddZ e dd*d+Ze d?d0d1Ze d@d4d5Zd6S)A SchemaVisitorrSr)rMrcCdSzNOverride this method to perform an action immediately before visiting a field.NrErIrSrErErL before_fieldzSchemaVisitor.before_fieldcCrzMOverride this method to perform an action immediately after visiting a field.NrErrErErL after_fieldrzSchemaVisitor.after_fieldelementcC||dSzcOverride this method to perform an action immediately before visiting an element within a ListType.NrrIrrErErLbefore_list_elementz!SchemaVisitor.before_list_elementcCrzbOverride this method to perform an action immediately after visiting an element within a ListType.NrrrErErLafter_list_elementrz SchemaVisitor.after_list_elementkeycCrz]Override this method to perform an action immediately before visiting a key within a MapType.NrrIrrErErLbefore_map_keyrzSchemaVisitor.before_map_keycCrz\Override this method to perform an action immediately after visiting a key within a MapType.NrrrErErL after_map_keyrzSchemaVisitor.after_map_keyvaluecCrz_Override this method to perform an action immediately before visiting a value within a MapType.NrrIrrErErLbefore_map_valuerzSchemaVisitor.before_map_valuecCrz^Override this method to perform an action immediately after visiting a value within a MapType.NrrrErErLafter_map_valuerzSchemaVisitor.after_map_valueschemar7 struct_resultr5cCrzVisit a Schema.NrErIrrrErErLrrzSchemaVisitor.schemar8r, field_resultsList[T]cCrzVisit a StructType.NrErIr8rrErErLr8rzSchemaVisitor.struct field_resultcCrzVisit a NestedField.NrErIrSrrErErLrSrzSchemaVisitor.field list_typer&element_resultcCrzVisit a ListType.NrErIrrrErErLrrzSchemaVisitor.listmap_typer( key_result value_resultcCrzVisit a MapType.NrErIrrrrErErLmaprzSchemaVisitor.map primitiver*cCrVisit a PrimitiveType.NrErIrrErErLrrzSchemaVisitor.primitiveNrSr)rMrrr)rMr)rr)rMrrr)rMr)rr7rr5rMr5)r8r,rrrMr5)rSr)rr5rMr5)rr&rr5rMr5)rr(rr5rr5rMr5rr*rMr5rrrrrrrrrrrrrr8rSrrrrErErErLr*             rc@s`eZdZed$ddZed%d dZed&ddZed'ddZed(ddZed)d!d"Z d#S)*PreOrderSchemaVisitorrr7rCallable[[], T]rMr5cCrrrErrErErLrrzPreOrderSchemaVisitor.schemar8r,rList[Callable[[], T]]cCrrrErrErErLr8rzPreOrderSchemaVisitor.structrSr)rcCrrrErrErErLrSrzPreOrderSchemaVisitor.fieldrr&rcCrrrErrErErLrrzPreOrderSchemaVisitor.listrr(rrcCrrrErrErErLrrzPreOrderSchemaVisitor.maprr*cCrrrErrErErLrrzPreOrderSchemaVisitor.primitiveN)rr7rrrMr5)r8r,rrrMr5)rSr)rrrMr5)rr&rrrMr5)rr(rrrrrMr5r ) rrrrrr8rSrrrrErErErLrs     rc@seZdZdAddZdAd d ZdBd dZdBddZdCddZdCddZdDddZ dDddZ e dEd"d#Z e dFd)d*Z e dGd,d-Ze dHd2d3Ze dId9d:Ze dJd>d?Zd@S)KSchemaWithPartnerVisitorrSr) field_partner Optional[P]rMrcCrrrErIrSrrErErLrrz%SchemaWithPartnerVisitor.before_fieldcCrrrErrErErLrrz$SchemaWithPartnerVisitor.after_fieldrelement_partnercC|||dSrrrIrrrErErLrz,SchemaWithPartnerVisitor.before_list_elementcCrrrrrErErLrrz+SchemaWithPartnerVisitor.after_list_elementr key_partnercCrrrrIrrrErErLrrz'SchemaWithPartnerVisitor.before_map_keycCrrrrrErErLrrz&SchemaWithPartnerVisitor.after_map_keyr value_partnercCrrrrIrrrErErLrrz)SchemaWithPartnerVisitor.before_map_valuecCrrrrrErErLrrz(SchemaWithPartnerVisitor.after_map_valuerr7schema_partnerrr5cCr)zVisit a schema with a partner.NrE)rIrrrrErErLrrzSchemaWithPartnerVisitor.schemar8r,struct_partnerrrcCr)z#Visit a struct type with a partner.NrE)rIr8rrrErErLr8rzSchemaWithPartnerVisitor.structrcCr)z$Visit a nested field with a partner.NrE)rIrSrrrErErLrSrzSchemaWithPartnerVisitor.fieldrr& list_partnerrcCr)z!Visit a list type with a partner.NrE)rIrrrrErErLrrzSchemaWithPartnerVisitor.listrr( map_partnerrrcCr)z Visit a map type with a partner.NrE)rIrr rrrErErLrrzSchemaWithPartnerVisitor.maprr*primitive_partnercCr)z&Visit a primitive type with a partner.NrErIrr!rErErLrrz"SchemaWithPartnerVisitor.primitiveN)rSr)rrrMr)rr)rrrMr)rr)rrrMr)rr)rrrMr)rr7rrrr5rMr5)r8r,rrrrrMr5)rSr)rrrr5rMr5)rr&rrrr5rMr5) rr(r rrr5rr5rMr5rr*r!rrMr5r rErErErLrr rc@seZdZdOddZedPd d ZedQddZedRddZedSddZedTddZ edUd d!Z edVd$d%Z edWd(d)Z edXd,d-Z edYd0d1ZedZd4d5Zed[d8d9Zed\dr8rErErLr)@rz'PrimitiveWithPartnerVisitor.visit_float double_typer!cCrzVisit a DoubleType.NrE)rIr@r8rErErLr*Drz(PrimitiveWithPartnerVisitor.visit_double decimal_typer cCrzVisit a DecimalType.NrE)rIrBr8rErErLr+Hrz)PrimitiveWithPartnerVisitor.visit_decimal date_typercCrrCrE)rIrDr8rErErLr,Lrz&PrimitiveWithPartnerVisitor.visit_date time_typer1cCrrCrE)rIrEr8rErErLr-Prz&PrimitiveWithPartnerVisitor.visit_timetimestamp_typer.cCrzVisit a TimestampType.NrE)rIrFr8rErErLr.Trz+PrimitiveWithPartnerVisitor.visit_timestamptimestamp_ns_typer-cCrzVisit a TimestampNanoType.NrE)rIrHr8rErErLr/Xrz.PrimitiveWithPartnerVisitor.visit_timestamp_nstimestamptz_typer0cCrzVisit a TimestamptzType.NrE)rIrJr8rErErLr0\rz-PrimitiveWithPartnerVisitor.visit_timestamptztimestamptz_ns_typer/cCrzVisit a TimestamptzNanoType.NrE)rIrLr8rErErLr1`rz0PrimitiveWithPartnerVisitor.visit_timestamptz_ns string_typer+cCrzVisit a StringType.NrE)rIrNr8rErErLr2drz(PrimitiveWithPartnerVisitor.visit_string uuid_typer3cCrzVisit a UUIDType.NrE)rIrPr8rErErLr3hrz&PrimitiveWithPartnerVisitor.visit_uuid fixed_typer"cCrzVisit a FixedType.NrE)rIrRr8rErErLr4lrz'PrimitiveWithPartnerVisitor.visit_fixed binary_typercCrzVisit a BinaryType.NrE)rIrTr8rErErLr5prz(PrimitiveWithPartnerVisitor.visit_binary unknown_typer2cCrzVisit a UnknownType.NrE)rIrVr8rErErLr6trz)PrimitiveWithPartnerVisitor.visit_unknownNr#)r7rr8rrMr5)r:r%r8rrMr5)r<r'r8rrMr5)r>r#r8rrMr5)r@r!r8rrMr5)rBr r8rrMr5)rDrr8rrMr5)rEr1r8rrMr5)rFr.r8rrMr5)rHr-r8rrMr5)rJr0r8rrMr5)rLr/r8rrMr5)rNr+r8rrMr5)rPr3r8rrMr5)rRr"r8rrMr5)rTrr8rrMr5)rVr2r8rrMr5)rrrrrr&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6rErErErLr$ H '                r$c@sReZdZedddZedd d ZedddZedddZedddZdS)PartnerAccessorr8rrMcCr)z0Return the equivalent of the schema as a struct.NrE)rIr8rErErLrzrzPartnerAccessor.schema_partnerpartner_structrqr? field_namerNcCr)zGReturn the equivalent struct field by name or id in the partner struct.NrE)rIrZrqr[rErErLr~rzPartnerAccessor.field_partner partner_listcCr)z7Return the equivalent list element in the partner list.NrE)rIr\rErErLlist_element_partnerrz$PartnerAccessor.list_element_partner partner_mapcCr)z1Return the equivalent map key in the partner map.NrErIr^rErErLmap_key_partnerrzPartnerAccessor.map_key_partnercCr)z3Return the equivalent map value in the partner map.NrEr_rErErLmap_value_partnerrz!PartnerAccessor.map_value_partnerN)r8rrMr)rZrrqr?r[rNrMr)r\rrMr)r^rrMr) rrrrrrr]r`rarErErErLrYys    rYschema_or_typeUnion[Schema, IcebergType]r8visitorSchemaWithPartnerVisitor[T, P]accessorPartnerAccessor[P]rMcCtd|)NzUnsupported type: )r)rbr8rdrfrErErLvisit_with_partnersrirSchemaWithPartnerVisitor[P, T]c Cs&||}|||t||||SrZ)rrrir)rr8rdrfrrErErL_s rkr8r,c Cs~g}|jD]2}|||j|j}|||zt|j|||}|||||W| ||q| ||w| |||SrZ) r<rrqr}rrirrrSrr8)r8r8rdrfrrSrrrErErLrks  rr&c CsZ||}||j|zt|j|||}W||j|n||j|w||||SrZ)r]r element_fieldri element_typerr)rr8rdrfrrrErErLrks  rr(c Cs||}||j|zt|j|||}W||j|n||j|w||}||j|zt|j |||}W| |j|n| |j|w| ||||SrZ) r`r key_fieldrikey_typerrar value_field value_typerr)rr8rdrfrrrrrErErLrks    rr*cCs |||SrZr)rr8rdrkrErErLrks c@seZdZdKddZedLd d ZedMd dZedNddZedOddZedPddZ edQddZ edRd!d"Z edSd%d&Z edTd)d*Z edUd-d.ZedVd0d1ZedWd4d5ZedXd8d9ZedYdr#cCrr?rE)rIr>rErErLr) rz)SchemaVisitorPerPrimitiveType.visit_floatr@r!cCrrArE)rIr@rErErLr*rz*SchemaVisitorPerPrimitiveType.visit_doublerDrcCrrCrE)rIrDrErErLr,rz(SchemaVisitorPerPrimitiveType.visit_daterEr1cCrrCrE)rIrErErErLr-rz(SchemaVisitorPerPrimitiveType.visit_timerFr.cCrrGrErIrFrErErLr.rz-SchemaVisitorPerPrimitiveType.visit_timestampr-cCrrIrErtrErErLr/rz0SchemaVisitorPerPrimitiveType.visit_timestamp_nsrJr0cCrrKrE)rIrJrErErLr0"rz/SchemaVisitorPerPrimitiveType.visit_timestamptzrLr/cCrrMrE)rIrLrErErLr1&rz2SchemaVisitorPerPrimitiveType.visit_timestamptz_nsrNr+cCrrOrE)rIrNrErErLr2*rz*SchemaVisitorPerPrimitiveType.visit_stringrPr3cCrrQrE)rIrPrErErLr3.rz(SchemaVisitorPerPrimitiveType.visit_uuidrTrcCrrUrE)rIrTrErErLr52rz*SchemaVisitorPerPrimitiveType.visit_binaryrVr2cCrrWrE)rIrVrErErLr66rz+SchemaVisitorPerPrimitiveType.visit_unknownNr )rRr"rMr5)rBr rMr5)r7rrMr5)r:r%rMr5)r<r'rMr5)r>r#rMr5)r@r!rMr5)rDrrMr5)rEr1rMr5)rFr.rMr5)rFr-rMr5)rJr0rMr5)rLr/rMr5)rNr+rMr5)rPr3rMr5)rTrrMr5)rVr2rMr5)rrrrrr4r+r&r'r(r)r*r,r-r.r/r0r1r2r3r5r6rErErErLrsrXrsT)initeqfrozenc@sDeZdZUdZded<dZded<dd d Zdd d ZdddZdS)rzVAn accessor for a specific position in a container that implements the StructProtocol.r?positionNzOptional[Accessor]innerrMrNcCsd|jd|jdS)7Return the string representation of the Accessor class.zAccessor(position=z,inner=r^rxryrXrErErLrYBszAccessor.__str__cCs|S)rz)rYrXrErErLr_FszAccessor.__repr__ containerrr cCs2|j}||}|}|jr|j}||j}|js |S)zReturn the value at self.position in `container`. Args: container (StructProtocol): A container to access at position `self.position`. Returns: Any: The value at position `self.position` in the container. r{)rIr|posvalryrErErLrJs  z Accessor.getr)r|rrMr ) rrrrrryrYr_rrErErErLr;s    robjSchemaVisitor[T]cCrh)aApply a schema visitor to any point within a schema. The function traverses the schema in post-order fashion. Args: obj (Union[Schema, IcebergType]): An instance of a Schema or an IcebergType. visitor (SchemaVisitor[T]): An instance of an implementation of the generic SchemaVisitor base class. Raises: NotImplementedError: If attempting to visit an unrecognized object type. Cannot visit non-type: NotImplementedErrorrrdrErErLvisit]s rcCs||t||S)z-Visit a Schema with a concrete SchemaVisitor.)rrrrrErErLrkmcCsNg}|jD]}||t|j|}||||||q|||S)z1Visit a StructType with a concrete SchemaVisitor.)r<rrrrrrSr8)rrdresultsrSresultrErErLrkss     cCs0||jt|j|}||j|||S)z/Visit a ListType with a concrete SchemaVisitor.)rrlrrmrr)rrdrrErErLrks    cCsV||jt|j|}||j||jt|j|}||j| |||S)z.Visit a MapType with a concrete SchemaVisitor.) rrnrrorrrprqrr)rrdrrrErErLrks      cC ||S)z4Visit a PrimitiveType with a concrete SchemaVisitor.rrrrErErLrk PreOrderSchemaVisitor[T]cCrh)a\Apply a schema visitor to any point within a schema. The function traverses the schema in pre-order fashion. This is a slimmed down version compared to the post-order traversal (missing before and after methods), mostly because we don't use the pre-order traversal much. Args: obj (Union[Schema, IcebergType]): An instance of a Schema or an IcebergType. visitor (PreOrderSchemaVisitor[T]): An instance of an implementation of the generic PreOrderSchemaVisitor base class. Raises: NotImplementedError: If attempting to visit an unrecognized object type. rrrrErErLpre_order_visitsrcfddS)z5Visit a Schema with a concrete PreOrderSchemaVisitor.cstSrZ)rrrErrErLs_..)rrrErrLrkrcs|fdd|jDS)z9Visit a StructType with a concrete PreOrderSchemaVisitor.csg|] }tfdd|qS)cs|tfdd|S)Ncs t|jSrZ)rrrSrdrErLr z0_.....)rSrrrrErLrrz_...)rrQrrErLrTs z_..)r8r<rrErrLrks  cr)z7Visit a ListType with a concrete PreOrderSchemaVisitor.c tjSrZ)rrmrErrErLrrr)rrrErrLrkrcs"fddfddS)z6Visit a MapType with a concrete PreOrderSchemaVisitor.crrZ)rrorErrErLrrrcrrZ)rrqrErrErLrr)rrrErrLrks"cCr)zZd?S)M _IndexByNamez?A schema visitor for generating a field name to field ID index.rMrcCs"i|_i|_i|_g|_g|_dSrZ)r_short_name_to_id_combined_index _field_names_short_field_namesrXrErErLrG<s  z_IndexByName.__init__rr)cCs,t|jts |j|j|j|jdSrZrgrr,rrr}rrrErErLrCs z_IndexByName.before_map_valuecC$t|jts |j|jdSrZrgrr,rrrrrErErLrH  z_IndexByName.after_map_valuercCs,t|jts |j|j|j|jdS)z@Short field names omit element when the element is a StructType.NrrrErErLrMs z _IndexByName.before_list_elementcCrrZrrrErErLrSrz_IndexByName.after_list_elementrScCs |j|j|j|jdS)zStore the field name.N)rrr}rrrErErLrXsz_IndexByName.before_fieldcCs|j|jdS)z"Remove the last field name stored.N)rrrrrErErLr]s z_IndexByName.after_fieldrr7rrBcCrsrZrrrErErLrbrz_IndexByName.schemar8r,rList[Dict[str, int]]cCrsrZrrrErErLr8erz_IndexByName.structrcCs||j|j|jS)z Add the field name to the index.) _add_fieldr}rqrrrErErLrShrz_IndexByName.fieldrr&rcCs||jj|jj|jS)z'Add the list element name to the index.)rrlr}rqrrrErErLrmsz_IndexByName.listrr(rrcCs.||jj|jj||jj|jj|jS)zAAdd the key name and value name as individual items in the index.)rrnr}rqrprrrErErLrrsz_IndexByName.mapr}rNrqr?cCs|}|jrdd|j|g}||jvr%td|d|j|d|||j|<|jr?dd|j|g}||j|<dSdS)a Add a field name to the index, mapping its full name to its field ID. Args: name (str): The field name. field_id (int): The field ID. Raises: ValueError: If the field name is already contained in the index. .z)Invalid schema, multiple fields for name z: z and N)rrVrrrr)rIr}rq full_name short_namerErErLrxs    z_IndexByName._add_fieldrr*cCrsrZrrrErErLrrz_IndexByName.primitivecCs|j}||j|S)zReturn an index of combined full and short names. Note: Only short names that do not conflict with full names are included. )rcopyupdater)rIcombined_indexrErErLby_names  z_IndexByName.by_namercCsdd|jD}|S)z$Return an index of ID to full names.cSsi|]\}}||qSrErE)rRrrrErErLr~rz&_IndexByName.by_id..)rr)rIid_to_full_namerErErLby_idsz_IndexByName.by_idNrr r r)rr7rrBrMrB)r8r,rrrMrB)rSr)rrBrMrB)rr&rrBrMrB)rr(rrBrrBrMrB)r}rNrqr?rMr)rr*rMrBrr)rrrrrGrrrrrrrr8rSrrrrrrrErErErLr9s$                rrBcCs*t|jdkrt}t|||StS)zGenerate an index of field names to field IDs. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[str, int]: An index of field names to field IDs. r)r`r<rrrrrbindexerrErErLrHs   rHrcCst}t|||S)zGenerate an index of field IDs full field names. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[str, int]: An index of field IDs to full names. )rrrrrErErLrs rc@sLeZdZdZd$ddZd%d dZd&ddZd'ddZd(ddZd)d!d"Z d#S)*_BuildPositionAccessorsaPA schema visitor for generating a field ID to accessor index. Example: >>> from pyiceberg.schema import Schema >>> from pyiceberg.types import * >>> schema = Schema( ... NestedField(field_id=2, name="id", field_type=IntegerType(), required=False), ... NestedField(field_id=1, name="data", field_type=StringType(), required=True), ... NestedField( ... field_id=3, ... name="location", ... field_type=StructType( ... NestedField(field_id=5, name="latitude", field_type=FloatType(), required=False), ... NestedField(field_id=6, name="longitude", field_type=FloatType(), required=False), ... ), ... required=True, ... ), ... schema_id=1, ... identifier_field_ids=[1], ... ) >>> result = build_position_accessors(schema) >>> expected = { ... 2: Accessor(position=0, inner=None), ... 1: Accessor(position=1, inner=None), ... 5: Accessor(position=2, inner=Accessor(position=0, inner=None)), ... 6: Accessor(position=2, inner=Accessor(position=1, inner=None)) ... 3: Accessor(position=2, inner=None), ... } >>> result == expected True rr7rDict[Position, Accessor]rMcC|SrZrErrErErLrz_BuildPositionAccessors.schemar8r,rList[Dict[Position, Accessor]]cCsXi}t|jD]"\}}||r"||D] \}}t||d||<qt|||j<q|S)N)ry) enumerater<rrrq)rIr8rrrxrSinner_field_idaccrErErLr8sz_BuildPositionAccessors.structrSr)rcCrrZrErrErErLrSrz_BuildPositionAccessors.fieldrr&rcCiSrZrErrErErLrrz_BuildPositionAccessors.listrr(rrcCrrZrErrErErLrsz_BuildPositionAccessors.maprr*cCrrZrErrErErLrrz!_BuildPositionAccessors.primitiveN)rr7rrrMr)r8r,rrrMr)rSr)rrrMr)rr&rrrMr)rr(rrrrrMr)rr*rMr) rrrrrr8rSrrrrErErErLrs   rrcCr)zGenerate an index of field IDs to schema position accessors. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[int, Accessor]: An index of field IDs to accessors. )rrrrErErLrrrnext_idOptional[Callable[[], int]]cCst|t|dS)z'Traverses the schema, and sets new IDs.) next_id_func)r _SetFreshIDs)rbrrErErLassign_fresh_schema_ids rrc@sleZdZUdZded<d1d2d d Zd3d dZd4ddZd5ddZd6d d!Z d7d%d&Z d8d+d,Z d9d/d0Z dS):rz>Traverses the schema and assigns monotonically increasing ids.ryold_id_to_new_idNrrrMrcs4i|_td|dur||_dSfdd|_dS)NcstSrZ)nextrEcounterrErLrsz'_SetFreshIDs.__init__..)r itertoolscountr)rIrrErrLrGs $z_SetFreshIDs.__init__ current_idr?cCs|}||j|<|SrZ)rr)rIrnew_idrErErL_get_and_increments z_SetFreshIDs._get_and_incrementrr7rCallable[[], StructType]cs"t|jdfdd|jDiS)NrAcsg|]}j|qSrE)r)rRrqrXrErLrT$rz'_SetFreshIDs.schema..)r7r<rArrErXrLr!s z_SetFreshIDs.schemar8r,rList[Callable[[], IcebergType]]c sZfdd|jD}g}t||j|D]\}}}|t||j||j|jdqt|S)Ncsg|]}|jqSrE)rrqrQrXrErLrT(rUz'_SetFreshIDs.struct..)rqr}rrdoc)r<rirr)r}rrr,)rIr8rnew_ids new_fieldsrqrSrrErXrLr8's z_SetFreshIDs.structrSr)rCallable[[], IcebergType]r$cCs|SrZrErrErErLrS6rz_SetFreshIDs.fieldrr&rcCs||j}t|||jdS)N)rrelement_required)rrr&r)rIrrrrErErLr9s z_SetFreshIDs.listrr(rrcCs0||j}||j}t|||||jdS)N)rrorrqvalue_required)rrrr(r)rIrrrrrrErErLrAs  z_SetFreshIDs.maprr*cC|SrZrErrErErLrLrz_SetFreshIDs.primitiverZ)rrrMr)rr?rMr?)rr7rrrMr7)r8r,rrrMr,)rSr)rrrMr$)rr&rrrMr&)rr(rrrrrMr()rr*rMr*) rrrrrrGrrr8rSrrrrErErErLrs         rr}rNcCst|st|S|S)aMake a field name compatible with Avro specification. This function sanitizes field names to comply with Avro naming rules: - Names must start with [A-Za-z_] - Subsequent characters must be [A-Za-z0-9_] Invalid characters are replaced with _xHHHH where HHHH is the hex code. Names starting with digits get a leading underscore. Args: name: The original field name Returns: A sanitized name that complies with Avro specification )_valid_avro_name_sanitize_name)r}rErErLmake_compatible_nameQsrrdcCsXt|std|d}|s|dksdS|ddD] }|s)|dks)dSqdS)Nz Can not validate empty avro namerrkFrT)r`risalphaisalnum)r}first characterrErErLrfsrcCs|g}|d}|s|dks|t|n|||ddD]}|s3|dks3|t|q!||q!d|S)Nrrkr)rr_sanitize_charrrV)r}sbrrrErErLrss   rrcCs,|rd|Sdtt|ddS)Nrk_x)isdigithexordupper)rrErErLrsrcCs,t|t}t|p tj|j|jdS)a_Sanitize column names to make them compatible with Avro. The column name should be starting with '_' or digit followed by a string only contains '_', digit or alphabet, otherwise it will be sanitized to conform the avro naming convention. Args: schema: The schema to be sanitized. Returns: The sanitized schema. r@rA)rr_SanitizeColumnsVisitorr7r,r<r@rA)rrrErErLsanitize_column_namess   rc@sHeZdZd#ddZd$d d Zd%ddZd&ddZd'ddZd(d d!Zd"S))rrr7rOptional[IcebergType]rMcCrrZrErrErErLrrz_SanitizeColumnsVisitor.schemarSr)rcCst|jt|j||j|jdS)Nrqr}rrr)r)rqrr}rrrrErErLrSsz_SanitizeColumnsVisitor.fieldr8r,rList[Optional[IcebergType]]cCstdd|DS)NcSsg|]}|dur|qSrZrErQrErErLrTrUz2_SanitizeColumnsVisitor.struct..)r,rrErErLr8rz_SanitizeColumnsVisitor.structrr&rcCst|j||jdSN)rrmr)r&rrrrErErLrrz_SanitizeColumnsVisitor.listrr(rrcCst|j|j|||jdSN)rrrorqr)r(rrrrrErErLrsz_SanitizeColumnsVisitor.maprr*cCrrZrErrErErLrrz!_SanitizeColumnsVisitor.primitiveNrr7rrrMrrSr)rrrMrr8r,rrrMrrr&rrrMrrr(rrrrrMrrr*rMr) rrrrrSr8rrrrErErErLrs     rselectedrselect_full_typescCs:t|t||}t|ptj|jt||j dS)aPrunes a column by only selecting a set of field-ids. Args: schema: The schema to be pruned. selected: The field-ids to be included. select_full_types: Return the full struct when a subset is recorded Returns: The pruned schema. r) rr_PruneColumnsVisitorr7r,r<r@r intersectionrA)rrrrrErErLrs   rc@seZdZUded<ded<d1ddZd2d d Zd3ddZd4ddZd5ddZd6d"d#Z d7d&d'Z e d8d)d*Z e d9d,d-Z e d:d.d/Zd0S);rrrrdrcCs||_||_dSrZ)rr)rIrrrErErLrGrz_PruneColumnsVisitor.__init__rr7rrrMcCrrZrErrErErLrrz_PruneColumnsVisitor.schemar8r,rrc Cs|j}g}d}t|D])\}}||}|j|kr||q |dur4d}|t|j|j||j|jdq |rIt |t |krE|durE|St |SdS)NTFr) r<rrrr)rqr}rrr`r,) rIr8rr<selected_fields same_typeidxprojected_typerSrErErLr8s0   z_PruneColumnsVisitor.structrSr)rcCsj|j|jvr-|jr |jS|jjr||S|jjs*td|jd|jd|jd|jS|dur3|SdS)N-Cannot explicitly project List or Map types, : of type was selected) rqrrrr_project_selected_structrrr}rrErErLrSs  z_PruneColumnsVisitor.fieldrr&rcCsx|j|jvr0|jr |S|jr|jjr||}|||S|jjs.td|jd|jd|S|dur:|||SdS)Nrrr ) rrrrmrr  _project_listrr)rIrrprojected_structrErErLrs    z_PruneColumnsVisitor.listrr(rrcCs|j|jvr0|jr |S|jr|jjr||}|||S|jjs.td|jd|jd|S|dur:|||S|j |jvrB|SdS)Nz7Cannot explicitly project List or Map types, Map value rr ) rrrrqrr  _project_maprrr)rIrrrr rErErLr s     z_PruneColumnsVisitor.maprr*cCsdSrZrErrErErLr4rz_PruneColumnsVisitor.primitiveprojected_fieldcCs(|r t|ts td|durtS|S)NzExpected a struct)rgr,r)rrErErLr 7s z-_PruneColumnsVisitor._project_selected_structr$cCs |j|kr|St|j||jdSr)rmr&rr)rrrErErLr As  z"_PruneColumnsVisitor._project_listcCs(|j|kr|St|j|j|j||jdSr)rqr(rrror)rrrErErLr Js z!_PruneColumnsVisitor._project_mapN)rrrrdrrrrrr)rrrMr,)rr&rr$rMr&)rr(rr$rMr()rrrrrGrr8rSrrr staticmethodr r r rErErErLrs          r file_typer$ read_typecCs ||kr|Std|d|)a Promotes reading a file type to a read type. Args: file_type (IcebergType): The type of the Avro file. read_type (IcebergType): The requested read type. Raises: ResolveError: If attempting to resolve an unrecognized object type. Cannot promote  to rrrrErErLpromoteXs rr%cCt|tr|Std|)NzCannot promote an int to )rgr'rrrErErLrki r#cCr)NzCannot promote an float to )rgr!rrrErErLrkrrr+cCr)NzCannot promote an string to )rgrrrrErErLrk{ rcCr)NzCannot promote an binary to )rgr+rrrErErLrkrr cCsHt|tr|j|jkr|j|jkr|Std|d|td|)NzCannot reduce precision from rzCannot promote an decimal to )rgr precisionscalerrrErErLrks r"cCs.t|tr t|dkr |Std|d|)Nrr)rgr3r`rrrErErLrksr2cCs"t|tr|Std|d|)Nrr)rgr*rrrErErLrks requested_schemaprovided_schemarcCst|t|dS)aZ Check if the `provided_schema` is compatible with `requested_schema`. Both Schemas must have valid IDs and share the same ID for the same field names. Two schemas are considered compatible when: 1. All `required` fields in `requested_schema` are present and are also `required` in the `provided_schema` 2. Field Types are consistent for fields that are present in both schemas. I.e. the field type in the `provided_schema` can be promoted to the field type of the same field ID in `requested_schema` Raises: ValueError: If the schemas are not compatible. N)r_SchemaCompatibilityVisitor)rrrErErL_check_schema_compatiblesrc@sfeZdZUded<d*ddZd+d d Zd,ddZd-ddZd.ddZd/ddZ d0d#d$Z d1d'd(Z d)S)2rr7rcCs`ddlm}ddlm}||_|ddd|_|jd|jd|jd |dd |_dS) Nr)Console)TableTbold) show_header header_stylerz Table fieldzDataframe field)record) rich.consoler  rich.tabler!r rich_table add_columnconsole)rIrr  RichTablerErErLrGs     z$_SchemaCompatibilityVisitor.__init__rer)rMrdc sz |jjWn$ty-jr |jdtdYdS|jdtdYdSwjrBjsB|jdttdSjjkrV|jdttdSt fddt t t hDrs|jdttdSzt jj|jdttWdStytjtrtjtsdjd }nd jd }|jdttd |YdS|jdttYdSw) Nu❌MissingFu✅Tc3s(|]}tj|otj|VqdSrZ)rgr)rRcontainer_typererfrErLr]s  zC_SchemaCompatibilityVisitor._is_field_compatible..zANull type (UnknownType) cannot be promoted to non-primitive type za. UnknownType can only be promoted to primitive types (string, int, boolean, etc.) in V3+ tables.z.Null type (UnknownType) cannot be promoted to zi. This may be due to table format version limitations (V1/V2 tables don't support UnknownType promotion).z - )rrrqrrr(add_rowrNranyr,r(r&rrrgr2r*)rIre error_msgrEr.rL_is_field_compatiblesD      "z0_SchemaCompatibilityVisitor._is_field_compatiblerrCallable[[], bool]cCs0|}s|j|jtd|j|S)NzMismatch in fields: )r*printr(r export_text)rIrrrrErErLrs z"_SchemaCompatibilityVisitor.schemar8r,rList[Callable[[], bool]]cCsdd|D}t|S)NcSsg|]}|qSrErE)rRrrErErLrTsz6_SchemaCompatibilityVisitor.struct..)rh)rIr8rrrErErLr8sz"_SchemaCompatibilityVisitor.structrSrcCs||o|SrZ)r2rrErErLrSsz!_SchemaCompatibilityVisitor.fieldrr&rcCs||jo|SrZ)r2rlrrErErLrrz _SchemaCompatibilityVisitor.listrr(rrcCs$t||j||j||gSrZ)rhr2rnrprrErErLrs  z_SchemaCompatibilityVisitor.maprr*cCr)NTrErrErErLrrz%_SchemaCompatibilityVisitor.primitiveN)rr7)rer)rMrd)rr7rr3rMrd)r8r,rr6rMrd)rSr)rr3rMrd)rr&rr3rMrd)rr(rr3rr3rMrd)rr*rMrd) rrrrrGr2rr8rSrrrrErErErLrs   1     r) rbrcr8r6rdrerfrgrMr5) rr7r8r6rdrjrfrgrMr5) r8r,r8r6rdrjrfrgrMr5) rr&r8r6rdrjrfrgrMr5) rr(r8r6rdrjrfrgrMr5) rr*r8r6rdrjrkrgrMr5)rrcrdrrMr5)rr7rdrrMr5)rr,rdrrMr5)rr&rdrrMr5)rr(rdrrMr5)rr*rdrrMr5)rrcrdrrMr5)rr7rdrrMr5)rr,rdrrMr5)rr&rdrrMr5)rr(rdrrMr5)rr*rdrrMr5)rbrcrMrt)rbrcrMry)rbrcrMrB)rbrcrMr)rbrcrMrrZ)rbrcrrrMr7)r}rNrMrN)r}rNrMrd)rrNrMrN)rr7rMr7r)rr7rrrrdrMr7)rr$rr$rMr$)rr%rr$rMr$)rr#rr$rMr$)rr+rr$rMr$)rrrr$rMr$)rr rr$rMr$)rr"rr$rMr$)rr2rr$rMr$)rr7rr7rMr)k __future__rrabcrr dataclassesr functoolsrrrtypingr r r r r rrrrrrrpydanticrrrpyiceberg.exceptionsrpyiceberg.typedefrrrpyiceberg.typesrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3pyarrowparr4r5r6r FIELD_ID_PROPICEBERG_FIELD_NAME_PROPr7rrrr$rYriregisterrkrsrrrr?rrvrrzrNrrHrPositionrrrrrrrrrrrrrrrdrrErErErLs  8 d /88m       m!             " ' h  >  ?    "