o dZ?GdddeZ@Gdddee;eZAGdddee;eZBGdddee>> from pyiceberg import schema >>> from pyiceberg import types structzLiteral['struct']type)default_factorytuple[NestedField, ...]fieldsz schema-id)aliasdefaultint schema_idzidentifier-field-ids)r7r4z list[int]identifier_field_idsdict[str, int] _name_to_idr#datar cs,|r||d<tjdi|t||_dS)Nr6)super__init__ index_by_namer=)selfr6r> __class__r?M/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/pyiceberg/schema.pyrA_szSchema.__init__returnstrcCsdddd|jDdS)5Return the string representation of the Schema class.ztable {  cSsg|]}dt|qS)z )rH.0fieldr?r?rF gz"Schema.__str__..z })joincolumnsrCr?r?rF__str__eszSchema.__str__cCs2ddd|jD}d|d|jd|jdS)rIz, css|]}t|VqdSN)repr)rLcolumnr?r?rF ksz"Schema.__repr__..zSchema(z , schema_id=z, identifier_field_ids=))rPrQr:r;)rC columns_reprr?r?rF__repr__iszSchema.__repr__cCs t|jS)z6Return the length of an instance of the Literal class.)lenr6rRr?r?rF__len__n zSchema.__len__otherboolcCsb|sdSt|ts dSt|jt|jkrdS|j|jk}tddt|j|jddD}|o0|S)z9Return the equality of two instances of the Schema class.Fcss|] \}}||kVqdSrTr?)rLlhsrhsr?r?rFrW~sz Schema.__eq__..Tstrict) isinstancer1r[rQr;allzip)rCr^identifier_field_ids_is_equalschema_is_equalr?r?rF__eq__rs   z Schema.__eq__after)modecCs |jr|jD]}||q|SrT)r;_validate_identifier_fieldrCfield_idr?r?rF check_schemas  zSchema.check_schemacC|jS)z A tuple of the top-level fields.)r6rRr?r?rFrQszSchema.columnsdict[int, NestedField]cCt|S)zReturn an index of field ID to NestedField instance. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. ) index_by_idrRr?r?rF_lazy_id_to_fieldzSchema._lazy_id_to_fielddict[int, int]cCrr)zReturns an index of field ID to parent field IDs. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )_index_parentsrRr?r?rF_lazy_id_to_parentruzSchema._lazy_id_to_parentcCsdd|jDS)zReturn an index of lower-case field names to field IDs. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. cSsi|] \}}||qSr?)lower)rLnamernr?r?rF z1Schema._lazy_name_to_id_lower..)r=itemsrRr?r?rF_lazy_name_to_id_lowerszSchema._lazy_name_to_id_lowerdict[int, str]cCrr)zReturn an index of field ID to full name. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )index_name_by_idrRr?r?rF_lazy_id_to_nameruzSchema._lazy_id_to_namedict[int, Accessor]cCrr)zReturn an index of field ID to accessor. This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. )build_position_accessorsrRr?r?rF_lazy_id_to_accessorruzSchema._lazy_id_to_accessorr&cCs t|jS)zReturn the schema as a struct.)r&r6rRr?r?rF as_structr]zSchema.as_struct pa.SchemacCddlm}||S)z%Return the schema as an Arrow schema.r)schema_to_pyarrow)pyiceberg.io.pyarrowr)rCrr?r?rFas_arrow zSchema.as_arrowT name_or_id str | intcase_sensitivecCstt|tr||jvrtd||j|S|r|j|}n|j|}|dur5td|d||j|S)aFind a field using a field name or field ID. Args: name_or_id (Union[str, int]): Either a field name or a field ID. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Raises: ValueError: When the value cannot be found. Returns: NestedField: The matched NestedField. zCould not find field with id: NzCould not find field with name , case_sensitive=)rdr9rt ValueErrorr=getr~ry)rCrrrnr?r?rF find_fields   zSchema.find_fieldrcCs,|j||d}|std|d||jS)adFind a field type using a field name or field ID. Args: name_or_id (Union[str, int]): Either a field name or a field ID. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Returns: NestedField: The type of the matched NestedField. )rrz%Could not find field with name or id r)rr field_type)rCrrrMr?r?rF find_types zSchema.find_typecCst|jddS)Nr)r8)maxrkeysrRr?r?rFhighest_field_idszSchema.highest_field_idr.cCr)Nr)create_mapping_from_schema)pyiceberg.table.name_mappingr)rCrr?r?rF name_mappingrzSchema.name_mapping column_id str | NonecCs |j|S)zFind a column name given a column ID. Args: column_id (int): The ID of the column. Returns: str: The column name (or None if the column ID cannot be found). )rr)rCrr?r?rFfind_column_name zSchema.find_column_name list[str]cCt|jS)z Return a list of all the column names, including nested fields. Excludes short names. Returns: List[str]: The column names. )listrvaluesrRr?r?rF column_namess zSchema.column_namesrnAccessorcCs"||jvr td||j|S)aFind a schema position accessor given a field ID. Args: field_id (int): The ID of the field. Raises: ValueError: When the value cannot be found. Returns: Accessor: An accessor for the given field ID. z+Could not find accessor for field with id: )rrrmr?r?rFaccessor_for_field s  zSchema.accessor_for_fieldset[str]cCs@t}|jD]}||}|durtd|||q|S)zwReturn the names of the identifier fields. Returns: Set of names of the identifier fields Nz%Could not find identifier column id: )setr;rradd)rCidsrn column_namer?r?rFidentifier_field_namess   zSchema.identifier_field_names)rnamesc sbz|r fdd|D}n fdd|D}Wnty+}ztd||d}~wwt|S)aReturn a new schema instance pruned to a subset of columns. Args: names (List[str]): A list of column names. case_sensitive (bool, optional): Whether to perform a case-sensitive lookup for each column name. Defaults to True. Returns: Schema: A new schema with pruned columns. Raises: ValueError: If a column is selected that doesn't exist. csh|]}j|qSr?)r=rLrzrRr?rF <z Schema.select..csh|] }j|qSr?)r~ryrrRr?rFr>r|zCould not find column: N)KeyErrorr prune_columns)rCrrrer?rRrFselect-s  z Schema.selectset[int]cCr)z%Return the IDs of the current schema.)rr=rrRr?r?rF field_idsDszSchema.field_idsNonecCs||}|jjstd|d|jstd|dt|jttfr,td|d|j |j }g}|durH| ||j |}|dus9|rr|| }|jj s`td|jd||jsntd|jd||sJdSdS) zValidate that the field with the given ID is a valid identifier field. Args: field_id: The ID of the field to validate. Raises: ValueError: If the field is not valid. zIdentifier field z$ invalid: not a primitive type fieldz invalid: not a required fieldz+ invalid: must not be float or double fieldNzCannot add field z/ as an identifier field: must not be nested in zA as an identifier field: must not be nested in an optional field )rr is_primitiverrequiredrdrrrxrrnappendpop is_structrz)rCrnrM parent_idr6parentr?r?rFrlIs,    z!Schema._validate_identifier_fieldformat_versioncCsB|jD]}||jkrt|jd|jd|qdS)zCheck that the schema is compatible for the given table format version. Args: format_version: The Iceberg table format version. Raises: ValueError: If the schema is not compatible for the format version. z is only supported in z' or higher. Current format version is: N)rtrrminimum_format_versionr)rCrrMr?r?rF"check_format_version_compatibilityns z)Schema.check_format_version_compatibility)r6r#r>r rGrH)rGr9)r^r rGr_)rGr1)rGr5)rGrq)rGrvrGr<rGr)rGr)rGr&)rGrT)rrrr_rGr#)rrrr_rGr)rGr.)rr9rGr)rGr)rnr9rGr)rGr)rrHrr_rGr1)rGr)rnr9rGr)rr9rGr)-__name__ __module__ __qualname____doc__r3__annotations__rtupler6INITIAL_SCHEMA_IDr:rr;rr=rArSrZr\rirropropertyrQrrtrxr~rrrrrrrrrrrrrrrlr __classcell__r?r?rDrFr1PsX                        %r1c@seZdZd7ddZd7ddZd8d d Zd8d d Zd9ddZd9ddZd:ddZ d:ddZ e d;ddZ e dd*d+Ze d?d0d1Ze d@d4d5Zd6S)A SchemaVisitorrMr#rGrcCdSzNOverride this method to perform an action immediately before visiting a field.Nr?rCrMr?r?rF before_fieldzSchemaVisitor.before_fieldcCrzMOverride this method to perform an action immediately after visiting a field.Nr?rr?r?rF after_fieldrzSchemaVisitor.after_fieldelementcC||dSzcOverride this method to perform an action immediately before visiting an element within a ListType.NrrCrr?r?rFbefore_list_elementz!SchemaVisitor.before_list_elementcCrzbOverride this method to perform an action immediately after visiting an element within a ListType.Nrrr?r?rFafter_list_elementrz SchemaVisitor.after_list_elementkeycCrz]Override this method to perform an action immediately before visiting a key within a MapType.NrrCrr?r?rFbefore_map_keyrzSchemaVisitor.before_map_keycCrz\Override this method to perform an action immediately after visiting a key within a MapType.Nrrr?r?rF after_map_keyrzSchemaVisitor.after_map_keyvaluecCrz_Override this method to perform an action immediately before visiting a value within a MapType.NrrCrr?r?rFbefore_map_valuerzSchemaVisitor.before_map_valuecCrz^Override this method to perform an action immediately after visiting a value within a MapType.Nrrr?r?rFafter_map_valuerzSchemaVisitor.after_map_valueschemar1 struct_resultr/cCrzVisit a Schema.Nr?rCrrr?r?rFrrzSchemaVisitor.schemar2r& field_resultsbuiltins.list[T]cCrzVisit a StructType.Nr?rCr2rr?r?rFr2rzSchemaVisitor.struct field_resultcCrzVisit a NestedField.Nr?rCrMrr?r?rFrMrzSchemaVisitor.field list_typer element_resultcCrzVisit a ListType.Nr?rCrrr?r?rFrrzSchemaVisitor.listmap_typer" key_result value_resultcCrzVisit a MapType.Nr?rCrrrr?r?rFmaprzSchemaVisitor.map primitiver$cCrVisit a PrimitiveType.Nr?rCrr?r?rFrrzSchemaVisitor.primitiveNrMr#rGrrr#rGr)rr#rGrrr#rGr)rr1rr/rGr/)r2r&rrrGr/)rMr#rr/rGr/)rr rr/rGr/)rr"rr/rr/rGr/rr$rGr/rrrrrrrrrrrrrr2rMrrrr?r?r?rFr*             rc@s`eZdZed$ddZed%d dZed&ddZed'ddZed(ddZed)d!d"Z d#S)*PreOrderSchemaVisitorrr1rCallable[[], T]rGr/cCrrr?rr?r?rFrrzPreOrderSchemaVisitor.schemar2r&rbuiltins.list[Callable[[], T]]cCrrr?rr?r?rFr2rzPreOrderSchemaVisitor.structrMr#rcCrrr?rr?r?rFrMrzPreOrderSchemaVisitor.fieldrr rcCrrr?rr?r?rFrrzPreOrderSchemaVisitor.listrr"rrcCrrr?rr?r?rFrrzPreOrderSchemaVisitor.maprr$cCrrr?rr?r?rFrrzPreOrderSchemaVisitor.primitiveN)rr1rr rGr/)r2r&rr rGr/)rMr#rr rGr/)rr rr rGr/)rr"rr rr rGr/r) rrrrrr2rMrrrr?r?r?rFr s     r c@seZdZdAddZdAd d ZdBd dZdBddZdCddZdCddZdDddZ dDddZ e dEd"d#Z e dFd)d*Z e dGd,d-Ze dHd2d3Ze dId9d:Ze dJd>d?Zd@S)KSchemaWithPartnerVisitorrMr# field_partnerP | NonerGrcCrrr?rCrMrr?r?rFrrz%SchemaWithPartnerVisitor.before_fieldcCrrr?rr?r?rFrrz$SchemaWithPartnerVisitor.after_fieldrelement_partnercC|||dSrrrCrrr?r?rFrz,SchemaWithPartnerVisitor.before_list_elementcCrrrrr?r?rFrrz+SchemaWithPartnerVisitor.after_list_elementr key_partnercCrrrrCrrr?r?rFrrz'SchemaWithPartnerVisitor.before_map_keycCrrrrr?r?rFrrz&SchemaWithPartnerVisitor.after_map_keyr value_partnercCrrrrCrrr?r?rFrrz)SchemaWithPartnerVisitor.before_map_valuecCrrrrr?r?rFrrz(SchemaWithPartnerVisitor.after_map_valuerr1schema_partnerrr/cCr)zVisit a schema with a partner.Nr?)rCrrrr?r?rFrrzSchemaWithPartnerVisitor.schemar2r&struct_partnerrrcCr)z#Visit a struct type with a partner.Nr?)rCr2rrr?r?rFr2rzSchemaWithPartnerVisitor.structrcCr)z$Visit a nested field with a partner.Nr?)rCrMrrr?r?rFrMrzSchemaWithPartnerVisitor.fieldrr list_partnerrcCr)z!Visit a list type with a partner.Nr?)rCrrrr?r?rFrrzSchemaWithPartnerVisitor.listrr" map_partnerrrcCr)z Visit a map type with a partner.Nr?)rCrrrrr?r?rFrrzSchemaWithPartnerVisitor.maprr$primitive_partnercCr)z&Visit a primitive type with a partner.Nr?rCrrr?r?rFrrz"SchemaWithPartnerVisitor.primitiveN)rMr#rrrGr)rr#rrrGr)rr#rrrGr)rr#rrrGr)rr1rrrr/rGr/)r2r&rrrrrGr/)rMr#rrrr/rGr/)rr rrrr/rGr/) rr"rrrr/rr/rGr/rr$rrrGr/r r?r?r?rFrr rc@seZdZdOddZedPd d ZedQddZedRddZedSddZedTddZ edUd d!Z edVd$d%Z edWd(d)Z edXd,d-Z edYd0d1ZedZd4d5Zed[d8d9Zed\dr?)rCr=r?r?rFr' rz*SchemaVisitorPerPrimitiveType.visit_doublerArcCrr@r?)rCrAr?r?rFr)rz(SchemaVisitorPerPrimitiveType.visit_daterBr+cCrr@r?)rCrBr?r?rFr*rz(SchemaVisitorPerPrimitiveType.visit_timerCr(cCrrDr?rCrCr?r?rFr+rz-SchemaVisitorPerPrimitiveType.visit_timestampr'cCrrFr?rqr?r?rFr,rz0SchemaVisitorPerPrimitiveType.visit_timestamp_nsrGr*cCrrHr?)rCrGr?r?rFr-rz/SchemaVisitorPerPrimitiveType.visit_timestamptzrIr)cCrrJr?)rCrIr?r?rFr.#rz2SchemaVisitorPerPrimitiveType.visit_timestamptz_nsrKr%cCrrLr?)rCrKr?r?rFr/'rz*SchemaVisitorPerPrimitiveType.visit_stringrMr-cCrrNr?)rCrMr?r?rFr0+rz(SchemaVisitorPerPrimitiveType.visit_uuidrQrcCrrRr?)rCrQr?r?rFr2/rz*SchemaVisitorPerPrimitiveType.visit_binaryrSr,cCrrTr?)rCrSr?r?rFr33rz+SchemaVisitorPerPrimitiveType.visit_unknownNr)rOrrGr/)r?rrGr/)r4rrGr/)r7rrGr/)r9r!rGr/)r;rrGr/)r=rrGr/)rArrGr/)rBr+rGr/)rCr(rGr/)rCr'rGr/)rGr*rGr/)rIr)rGr/)rKr%rGr/)rMr-rGr/)rQrrGr/)rSr,rGr/)rrrrrr1r(r#r$r%r&r'r)r*r+r,r-r.r/r0r2r3r?r?r?rFrprUrpT)initeqfrozenc@sDeZdZUdZded<dZded<dd d Zdd d ZdddZdS)rzVAn accessor for a specific position in a container that implements the StructProtocol.r9positionNzAccessor | NoneinnerrGrHcCsd|jd|jdS)7Return the string representation of the Accessor class.zAccessor(position=z,inner=rXrurvrRr?r?rFrS?szAccessor.__str__cCs|S)rw)rSrRr?r?rFrZCszAccessor.__repr__ containerrr cCs2|j}||}|}|jr|j}||j}|js |S)zReturn the value at self.position in `container`. Args: container (StructProtocol): A container to access at position `self.position`. Returns: Any: The value at position `self.position` in the container. rx)rCryposvalrvr?r?rFrGs  z Accessor.getr)ryrrGr ) rrrrrrvrSrZrr?r?r?rFr8s    robjSchemaVisitor[T]cCre)aApply a schema visitor to any point within a schema. The function traverses the schema in post-order fashion. Args: obj (Union[Schema, IcebergType]): An instance of a Schema or an IcebergType. visitor (SchemaVisitor[T]): An instance of an implementation of the generic SchemaVisitor base class. Raises: NotImplementedError: If attempting to visit an unrecognized object type. Cannot visit non-type: NotImplementedErrorr|rar?r?rFvisitZs rcCs||t||S)z-Visit a Schema with a concrete SchemaVisitor.)rrrrr?r?rFrhjcCsNg}|jD]}||t|j|}||||||q|||S)z1Visit a StructType with a concrete SchemaVisitor.)r6rrrrrrMr2)r|raresultsrMresultr?r?rFrhps     cCs0||jt|j|}||j|||S)z/Visit a ListType with a concrete SchemaVisitor.)rrirrjrr)r|rarr?r?rFrh~s    cCsV||jt|j|}||j||jt|j|}||j| |||S)z.Visit a MapType with a concrete SchemaVisitor.) rrkrrlrrrmrnrr)r|rarrr?r?rFrhs      cC ||S)z4Visit a PrimitiveType with a concrete SchemaVisitor.rorr?r?rFrh PreOrderSchemaVisitor[T]cCre)a\Apply a schema visitor to any point within a schema. The function traverses the schema in pre-order fashion. This is a slimmed down version compared to the post-order traversal (missing before and after methods), mostly because we don't use the pre-order traversal much. Args: obj (Union[Schema, IcebergType]): An instance of a Schema or an IcebergType. visitor (PreOrderSchemaVisitor[T]): An instance of an implementation of the generic PreOrderSchemaVisitor base class. Raises: NotImplementedError: If attempting to visit an unrecognized object type. r~rrr?r?rFpre_order_visitsrcfddS)z5Visit a Schema with a concrete PreOrderSchemaVisitor.cstSrT)rrr?rr?rFs_..)rrr?rrFrhrcs|fdd|jDS)z9Visit a StructType with a concrete PreOrderSchemaVisitor.csg|] }tfdd|qS)cs|tfdd|S)Ncs t|jSrT)rrrMrar?rFr z0_.....)rMrrrr?rFrr|z_...)rrKrr?rFrNs z_..)r2r6rr?rrFrhs  cr)z7Visit a ListType with a concrete PreOrderSchemaVisitor.c tjSrT)rrjr?rr?rFrrr)rrr?rrFrhrcs"fddfddS)z6Visit a MapType with a concrete PreOrderSchemaVisitor.crrT)rrlr?rr?rFrrrcrrT)rrnr?rr?rFrr)rrr?rrFrhs"cCr)zZd?S)M _IndexByNamez?A schema visitor for generating a field name to field ID index.rGrcCs"i|_i|_i|_g|_g|_dSrT)r_short_name_to_id_combined_index _field_names_short_field_namesrRr?r?rFrA9s  z_IndexByName.__init__rr#cCs,t|jts |j|j|j|jdSrTrdrr&rrrzrrr?r?rFr@s z_IndexByName.before_map_valuecC$t|jts |j|jdSrTrdrr&rrrrr?r?rFrE  z_IndexByName.after_map_valuercCs,t|jts |j|j|j|jdS)z@Short field names omit element when the element is a StructType.Nrrr?r?rFrJs z _IndexByName.before_list_elementcCrrTrrr?r?rFrPrz_IndexByName.after_list_elementrMcCs |j|j|j|jdS)zStore the field name.N)rrrzrrr?r?rFrUsz_IndexByName.before_fieldcCs|j|jdS)z"Remove the last field name stored.N)rrrrr?r?rFrZs z_IndexByName.after_fieldrr1rr<cCrprTrrr?r?rFr_rz_IndexByName.schemar2r&rbuiltins.list[dict[str, int]]cCrprTrrr?r?rFr2brz_IndexByName.structrcCs||j|j|jS)z Add the field name to the index.) _add_fieldrzrnrrr?r?rFrMerz_IndexByName.fieldrr rcCs||jj|jj|jS)z'Add the list element name to the index.)rrirzrnrrr?r?rFrjsz_IndexByName.listrr"rrcCs.||jj|jj||jj|jj|jS)zAAdd the key name and value name as individual items in the index.)rrkrzrnrmrrr?r?rFrosz_IndexByName.maprzrHrnr9cCs|}|jrdd|j|g}||jvr%td|d|j|d|||j|<|jr?dd|j|g}||j|<dSdS)a Add a field name to the index, mapping its full name to its field ID. Args: name (str): The field name. field_id (int): The field ID. Raises: ValueError: If the field name is already contained in the index. .z)Invalid schema, multiple fields for name z: z and N)rrPrrrr)rCrzrn full_name short_namer?r?rFrus    z_IndexByName._add_fieldrr$cCrprTrrr?r?rFrrz_IndexByName.primitivecCs|j}||j|S)zReturn an index of combined full and short names. Note: Only short names that do not conflict with full names are included. )rcopyupdater)rCcombined_indexr?r?rFby_names  z_IndexByName.by_namercCsdd|jD}|S)z$Return an index of ID to full names.cSsi|]\}}||qSr?r?)rLrrr?r?rFr{rz&_IndexByName.by_id..)rr})rCid_to_full_namer?r?rFby_idsz_IndexByName.by_idNrrrr)rr1rr<rGr<)r2r&rrrGr<)rMr#rr<rGr<)rr rr<rGr<)rr"rr<rr<rGr<)rzrHrnr9rGr)rr$rGr<rr)rrrrrArrrrrrrr2rMrrrrrrr?r?r?rFr6s$                rr<cCs*t|jdkrt}t|||StS)zGenerate an index of field names to field IDs. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[str, int]: An index of field names to field IDs. r)r[r6rrrrr_indexerr?r?rFrBs   rBrcCst}t|||S)zGenerate an index of field IDs full field names. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[str, int]: An index of field IDs to full names. )rrrrr?r?rFrs rc@sLeZdZdZd$ddZd%d dZd&ddZd'ddZd(ddZd)d!d"Z d#S)*_BuildPositionAccessorsaPA schema visitor for generating a field ID to accessor index. Example: >>> from pyiceberg.schema import Schema >>> from pyiceberg.types import * >>> schema = Schema( ... NestedField(field_id=2, name="id", field_type=IntegerType(), required=False), ... NestedField(field_id=1, name="data", field_type=StringType(), required=True), ... NestedField( ... field_id=3, ... name="location", ... field_type=StructType( ... NestedField(field_id=5, name="latitude", field_type=FloatType(), required=False), ... NestedField(field_id=6, name="longitude", field_type=FloatType(), required=False), ... ), ... required=True, ... ), ... schema_id=1, ... identifier_field_ids=[1], ... ) >>> result = build_position_accessors(schema) >>> expected = { ... 2: Accessor(position=0, inner=None), ... 1: Accessor(position=1, inner=None), ... 5: Accessor(position=2, inner=Accessor(position=0, inner=None)), ... 6: Accessor(position=2, inner=Accessor(position=1, inner=None)) ... 3: Accessor(position=2, inner=None), ... } >>> result == expected True rr1rdict[Position, Accessor]rGcC|SrTr?rr?r?rFrz_BuildPositionAccessors.schemar2r&r'builtins.list[dict[Position, Accessor]]cCsXi}t|jD]"\}}||r"||D] \}}t||d||<qt|||j<q|S)N)rv) enumerater6r}rrn)rCr2rrrurMinner_field_idaccr?r?rFr2sz_BuildPositionAccessors.structrMr#rcCrrTr?rr?r?rFrMrz_BuildPositionAccessors.fieldrr rcCiSrTr?rr?r?rFrrz_BuildPositionAccessors.listrr"rrcCrrTr?rr?r?rFrsz_BuildPositionAccessors.maprr$cCrrTr?rr?r?rFrrz!_BuildPositionAccessors.primitiveN)rr1rrrGr)r2r&rrrGr)rMr#rrrGr)rr rrrGr)rr"rrrrrGr)rr$rGr) rrrrrr2rMrrrr?r?r?rFrs   rrcCr)zGenerate an index of field IDs to schema position accessors. Args: schema_or_type (Union[Schema, IcebergType]): A schema or type to index. Returns: Dict[int, Accessor]: An index of field IDs to accessors. )rrrr?r?rFrrrnext_idCallable[[], int] | NonecCst|t|dS)z'Traverses the schema, and sets new IDs.) next_id_func)r _SetFreshIDs)r_rr?r?rFassign_fresh_schema_ids rrc@sleZdZUdZded<d1d2d d Zd3d dZd4ddZd5ddZd6d d!Z d7d%d&Z d8d+d,Z d9d/d0Z dS):rz>Traverses the schema and assigns monotonically increasing ids.rvold_id_to_new_idNrrrGrcs4i|_td|dur||_dSfdd|_dS)NcstSrT)nextr?counterr?rFrsz'_SetFreshIDs.__init__..)r itertoolscountr)rCrr?rrFrAs $z_SetFreshIDs.__init__ current_idr9cCs|}||j|<|SrT)rr)rCrnew_idr?r?rF_get_and_increments z_SetFreshIDs._get_and_incrementrr1rCallable[[], StructType]cs"t|jdfdd|jDiS)Nr;csg|]}j|qSr?)r)rLrnrRr?rFrN!rz'_SetFreshIDs.schema..)r1r6r;rr?rRrFrs z_SetFreshIDs.schemar2r&r(builtins.list[Callable[[], IcebergType]]c s^fdd|jD}g}t||j|ddD]\}}}|t||j||j|jdqt|S)Ncsg|]}|jqSr?)rrnrKrRr?rFrN%rOz'_SetFreshIDs.struct..Trb)rnrzrrdoc)r6rfrr#rzrrr&)rCr2rnew_ids new_fieldsrnrMrr?rRrFr2$s z_SetFreshIDs.structrMr#rCallable[[], IcebergType]rcCs|SrTr?rr?r?rFrM3rz_SetFreshIDs.fieldrr rcCs||j}t|||jdS)N)rrelement_required)rrr r)rCrrrr?r?rFr6s z_SetFreshIDs.listrr"rrcCs0||j}||j}t|||||jdS)N)rrlrrnvalue_required)rrrr"r)rCrrrrrr?r?rFr>s  z_SetFreshIDs.maprr$cC|SrTr?rr?r?rFrIrz_SetFreshIDs.primitiverT)rrrGr)rr9rGr9)rr1rrrGr1)r2r&rrrGr&)rMr#rrrGr)rr rrrGr )rr"rrrrrGr")rr$rGr$) rrrrrrArrr2rMrrrr?r?r?rFrs         rrzrHcCst|st|S|S)aMake a field name compatible with Avro specification. This function sanitizes field names to comply with Avro naming rules: - Names must start with [A-Za-z_] - Subsequent characters must be [A-Za-z0-9_] Invalid characters are replaced with _xHHHH where HHHH is the hex code. Names starting with digits get a leading underscore. Args: name: The original field name Returns: A sanitized name that complies with Avro specification )_valid_avro_name_sanitize_name)rzr?r?rFmake_compatible_nameNsrr_cCsXt|std|d}|s|dksdS|ddD] }|s)|dks)dSqdS)Nz Can not validate empty avro namerrhFrT)r[risalphaisalnum)rzfirst characterr?r?rFrcsrcCs|g}|d}|s|dks|t|n|||ddD]}|s3|dks3|t|q!||q!d|S)Nrrhr)rr_sanitize_charrrP)rzsbrrr?r?rFrps   rrcCs,|rd|Sdtt|ddS)Nrh_x)isdigithexordupper)rr?r?rFrsrcCs,t|t}t|p tj|j|jdS)a_Sanitize column names to make them compatible with Avro. The column name should be starting with '_' or digit followed by a string only contains '_', digit or alphabet, otherwise it will be sanitized to conform the avro naming convention. Args: schema: The schema to be sanitized. Returns: The sanitized schema. r:r;)rr_SanitizeColumnsVisitorr1r&r6r:r;)rrr?r?rFsanitize_column_namess   rc@sHeZdZd#ddZd$d d Zd%ddZd&ddZd'ddZd(d d!Zd"S))rrr1rIcebergType | NonerGcCrrTr?rr?r?rFrrz_SanitizeColumnsVisitor.schemarMr#rcCst|jt|j||j|jdS)Nrnrzrrr)r#rnrrzrrrr?r?rFrMsz_SanitizeColumnsVisitor.fieldr2r&r!builtins.list[IcebergType | None]cCstdd|DS)NcSsg|]}|dur|qSrTr?rKr?r?rFrNrOz2_SanitizeColumnsVisitor.struct..)r&rr?r?rFr2rz_SanitizeColumnsVisitor.structrr rcCst|j||jdSN)rrjr)r rrrr?r?rFrrz_SanitizeColumnsVisitor.listrr"rrcCst|j|j|||jdSN)rrrlrnr)r"rrrrr?r?rFrsz_SanitizeColumnsVisitor.maprr$cCrrTr?rr?r?rFrrz!_SanitizeColumnsVisitor.primitiveNrr1rrrGrrMr#rrrGrr2r&rrrGrrr rrrGrrr"rrrrrGrrr$rGr) rrrrrMr2rrrr?r?r?rFrs     rselectedrselect_full_typescCs:t|t||}t|ptj|jt||j dS)aPrunes a column by only selecting a set of field-ids. Args: schema: The schema to be pruned. selected: The field-ids to be included. select_full_types: Return the full struct when a subset is recorded Returns: The pruned schema. r) rr_PruneColumnsVisitorr1r&r6r:r intersectionr;)rrrrr?r?rFrs   rc@seZdZUded<ded<d1ddZd2d d Zd3ddZd4ddZd5ddZd6d"d#Z d7d&d'Z e d8d)d*Z e d9d,d-Z e d:d.d/Zd0S);rrrr_rcCs||_||_dSrT)rr)rCrrr?r?rFrArz_PruneColumnsVisitor.__init__rr1rrrGcCrrTr?rr?r?rFrrz_PruneColumnsVisitor.schemar2r&rrc Cs|j}g}d}t|D])\}}||}|j|kr||q |dur4d}|t|j|j||j|jdq |rIt |t |krE|durE|St |SdS)NTFr) r6rrrr#rnrzrrr[r&) rCr2rr6selected_fields same_typeidxprojected_typerMr?r?rFr2s0   z_PruneColumnsVisitor.structrMr#rcCsj|j|jvr-|jr |jS|jjr||S|jjs*td|jd|jd|jd|jS|dur3|SdS)N-Cannot explicitly project List or Map types, : of type was selected) rnrrrr_project_selected_structrrrzrr?r?rFrMs&  z_PruneColumnsVisitor.fieldrr rcCsx|j|jvr0|jr |S|jr|jjr||}|||S|jjs.td|jd|jd|S|dur:|||SdS)Nrrr) rrrrjrr _project_listrr)rCrrprojected_structr?r?rFr s$    z_PruneColumnsVisitor.listrr"rrcCs|j|jvr0|jr |S|jr|jjr||}|||S|jjs.td|jd|jd|S|dur:|||S|j |jvrB|SdS)Nz7Cannot explicitly project List or Map types, Map value rr) rrrrnrr _project_maprrr)rCrrrr r?r?rFrs(     z_PruneColumnsVisitor.maprr$cCsdSrTr?rr?r?rFr0rz_PruneColumnsVisitor.primitiveprojected_fieldcCs(|r t|ts td|durtS|S)NzExpected a struct)rdr&r)r r?r?rFr3s z-_PruneColumnsVisitor._project_selected_structrcCs |j|kr|St|j||jdSr)rjr rr)rrr?r?rFr=s  z"_PruneColumnsVisitor._project_listcCs(|j|kr|St|j|j|j||jdSr)rnr"rrrlr)rrr?r?rFr Fs z!_PruneColumnsVisitor._project_mapN)rrrr_rrrrrr)r rrGr&)rr rrrGr )rr"rrrGr")rrrrrArr2rMrrr staticmethodrrr r?r?r?rFrs          r file_typer read_typecCs ||kr|Std|d|)a Promotes reading a file type to a read type. Args: file_type (IcebergType): The type of the Avro file. read_type (IcebergType): The requested read type. Raises: ResolveError: If attempting to resolve an unrecognized object type. Cannot promote  to rr rr?r?rFpromoteTs rrcCt|tr|Std|)NzCannot promote an int to )rdr!rrr?r?rFrhe rcCr)NzCannot promote an float to )rdrrrr?r?rFrhnrr%cCr)NzCannot promote an string to )rdrrrr?r?rFrhw rcCr)NzCannot promote an binary to )rdr%rrr?r?rFrhrrcCsHt|tr|j|jkr|j|jkr|Std|d|td|)NzCannot reduce precision from rzCannot promote an decimal to )rdr precisionscalerrr?r?rFrhs rcCs.t|tr t|dkr |Std|d|)Nrr)rdr-r[rrr?r?rFrhsr,cCs"t|tr|Std|d|)Nrr)rdr$rrr?r?rFrhs requested_schemaprovided_schemarcCst|t|dS)aZ Check if the `provided_schema` is compatible with `requested_schema`. Both Schemas must have valid IDs and share the same ID for the same field names. Two schemas are considered compatible when: 1. All `required` fields in `requested_schema` are present and are also `required` in the `provided_schema` 2. Field Types are consistent for fields that are present in both schemas. I.e. the field type in the `provided_schema` can be promoted to the field type of the same field ID in `requested_schema` Raises: ValueError: If the schemas are not compatible. N)r_SchemaCompatibilityVisitor)rrr?r?rF_check_schema_compatiblesrc@sfeZdZUded<d*ddZd+d d Zd,ddZd-ddZd.ddZd/ddZ d0d#d$Z d1d'd(Z d)S)2rr1rcCs`ddlm}ddlm}||_|ddd|_|jd|jd|jd |dd |_dS) Nr)Console)TableTbold) show_header header_stylerz Table fieldzDataframe field)record) rich.consoler rich.tablerr rich_table add_columnconsole)rCrr RichTabler?r?rFrAs     z$_SchemaCompatibilityVisitor.__init__r`r#rGr_c sz |jjWn$ty-jr |jdtdYdS|jdtdYdSwjrBjsB|jdttdSjjkrV|jdttdSt fddt t t hDrs|jdttdSzt jj|jdttWdStytjtrtjtsdjd }nd jd }|jdttd |YdS|jdttYdSw) Nu❌MissingFu✅Tc3s(|]}tj|otj|VqdSrT)rdr)rLcontainer_typer`rar?rFrWs  zC_SchemaCompatibilityVisitor._is_field_compatible..zANull type (UnknownType) cannot be promoted to non-primitive type za. UnknownType can only be promoted to primitive types (string, int, boolean, etc.) in V3+ tables.z.Null type (UnknownType) cannot be promoted to zi. This may be due to table format version limitations (V1/V2 tables don't support UnknownType promotion).z - )rrrnrrr%add_rowrHranyr&r"r rrrdr,r$)rCr` error_msgr?r+rF_is_field_compatiblesH        "z0_SchemaCompatibilityVisitor._is_field_compatiblerrCallable[[], bool]cCs0|}s|j|jtd|j|S)NzMismatch in fields: )r'printr%r export_text)rCrrrr?r?rFrs z"_SchemaCompatibilityVisitor.schemar2r&r!builtins.list[Callable[[], bool]]cCsdd|D}t|S)NcSsg|]}|qSr?r?)rLrr?r?rFrNsz6_SchemaCompatibilityVisitor.struct..)re)rCr2rrr?r?rFr2sz"_SchemaCompatibilityVisitor.structrMrcCs&||}|j|jjvr|S|o|SrT)r/rnrrt)rCrMr is_compatibler?r?rFrMs  z!_SchemaCompatibilityVisitor.fieldrr rcCs||jo|SrT)r/rirr?r?rFr rz _SchemaCompatibilityVisitor.listrr"rrcCs$t||j||j||gSrT)rer/rkrmrr?r?rFrs  z_SchemaCompatibilityVisitor.maprr$cCr)NTr?rr?r?rFrrz%_SchemaCompatibilityVisitor.primitiveN)rr1)r`r#rGr_)rr1rr0rGr_)r2r&rr3rGr_)rMr#rr0rGr_)rr rr0rGr_)rr"rr0rr0rGr_)rr$rGr_) rrrrrAr/rr2rMrrrr?r?r?rFrs   9     r) r_r`r5r0rarbrcrdrGr/) rr1r5r0rargrcrdrGr/) r2r&r5r0rargrcrdrGr/) rr r5r0rargrcrdrGr/) rr"r5r0rargrcrdrGr/) rr$r5r0rargrhrdrGr/)r|r`rar}rGr/)r|r1rar}rGr/)r|r&rar}rGr/)r|r rar}rGr/)r|r"rar}rGr/)r|r$rar}rGr/)r|r`rarrGr/)r|r1rarrGr/)r|r&rarrGr/)r|r rarrGr/)r|r"rarrGr/)r|r$rarrGr/)r_r`rGrq)r_r`rGrv)r_r`rGr<)r_r`rGr)r_r`rGrrT)r_r`rrrGr1)rzrHrGrH)rzrHrGr_)rrHrGrH)rr1rGr1r)rr1rrrr_rGr1)r rrrrGr)r rrrrGr)r rrrrGr)r r%rrrGr)r rrrrGr)r rrrrGr)r rrrrGr)r r,rrrGr)rr1rr1rGr)h __future__rbuiltinsrabcrrcollections.abcr dataclassesr functoolsrrr typingr r r r rpydanticrrrpyiceberg.exceptionsrpyiceberg.typedefrrrpyiceberg.typesrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-pyarrowparr.r/r0r FIELD_ID_PROPICEBERG_FIELD_NAME_PROPr1rr rr!rVrfregisterrhrprrrdictr9rrsrrwrHrrBrPositionrrrrrrrrrrrrrrr_rr?r?r?rFs    d 188m       m!             " ' h  >  ?