o Z>ddl?m@Z@mAZAmBZBdd lCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZd Z[e@fd e:eKBd e\e]ed eBffdefddZ^d e:eKBde4fddZ_Gddde;e4Z`e`Zade:eKBd e:eKBde4fddZbe@e@fd e:eKBde:eKBd e\e]ed eBffde\e]ed effdef ddZcGdddeZdGddde9eKe4fZeGd d!d!e9eKefZfGd"d#d#e8eKZgd$S)%)Callable)Enum) BinaryDecoder) BinaryReader BooleanReader DateReader DecimalReader DefaultReader DoubleReader FixedReader FloatReader IntegerReader ListReader MapReader NoneReader OptionReaderReader StringReader StructReader TimeReaderTimestampNanoReaderTimestampReaderTimestamptzNanoReaderTimestamptzReader UnknownReader UUIDReader) BinaryWriter BooleanWriter DateWriter DecimalWriter DefaultWriter DoubleWriter FixedWriter FloatWriter IntegerWriter ListWriter MapWriter OptionWriter StringWriter StructWriterTimestampNanoWriterTimestamptzNanoWriterTimestamptzWriterTimestampWriter TimeWriter UnknownWriter UUIDWriterWriter) ResolveError)PartnerAccessorPrimitiveWithPartnerVisitorSchemaSchemaVisitorPerPrimitiveTypepromotevisitvisit_with_partner) EMPTY_DICTRecordStructProtocol) BinaryType BooleanTypeDateType DecimalType DoubleType FixedType FloatType IcebergType IntegerTypeListTypeLongTypeMapType NestedField PrimitiveType StringType StructTypeTimestampNanoType TimestampTypeTimestamptzNanoTypeTimestamptzTypeTimeType UnknownTypeUUIDType file_schema read_types.returncCs t|||S)aMConstruct a reader from a file schema. Args: file_schema (Schema | IcebergType): The schema of the Avro file. read_types (Dict[int, Callable[..., StructProtocol]]): Constructors for structs for certain field-ids Raises: NotImplementedError: If attempting to resolve an unrecognized object type. )resolve_reader)rUrVrYT/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/pyiceberg/avro/resolver.pyconstruct_readeros r[cCs t|tS)zConstruct a writer from a file schema. Args: file_schema (Schema | IcebergType): The schema of the Avro file. Raises: NotImplementedError: If attempting to resolve an unrecognized object type. )r8CONSTRUCT_WRITER_VISITOR)rUrYrYrZconstruct_writer~s r]c@seZdZdZdededefddZdedeedefd d Z d e d edefd dZ de dedefddZde dededefddZdedefddZdedefddZdedefdd Zd!edefd"d#Zd$edefd%d&Zd'edefd(d)Zd*edefd+d,Zd-edefd.d/Zd0edefd1d2Z d3e!defd4d5Z"d6e#defd7d8Z$d9e%defd:d;Z&dZ(d?e)defd@dAZ*dBe+defdCdDZ,dEe-defdFdGZ.dHe/defdIdJZ0dKS)LConstructWriterz/Construct a writer tree from an Iceberg schema.schema struct_resultrWcCs|SNrY)selfr_r`rYrYrZr_zConstructWriter.schemastruct field_resultscCsttddt|DS)Ncss|] \}}||fVqdSrarY).0posresultrYrYrZ sz)ConstructWriter.struct..)r)tuple enumerate)rbrdrerYrYrZrdzConstructWriter.structfield field_resultcCs|jr|St|Srarequiredr')rbrmrnrYrYrZrmzConstructWriter.field list_typeelement_resultcCst|Sra)r%)rbrrrsrYrYrZlistszConstructWriter.listmap_type key_result value_resultcCs t||Sra)r&)rbrurvrwrYrYrZmaps zConstructWriter.map fixed_typecC tt|Srar"len)rbryrYrYrZ visit_fixed zConstructWriter.visit_fixed decimal_typecCt|j|jSrar precisionscale)rbrrYrYrZ visit_decimalzConstructWriter.visit_decimal boolean_typecCtSrar)rbrrYrYrZ visit_booleanzConstructWriter.visit_boolean integer_typecCrrar$)rbrrYrYrZ visit_integerrzConstructWriter.visit_integer long_typecCrrar)rbrrYrYrZ visit_longrzConstructWriter.visit_long float_typecCrrar#)rbrrYrYrZ visit_floatrzConstructWriter.visit_float double_typecCrrar!)rbrrYrYrZ visit_doublerzConstructWriter.visit_double date_typecCrrar)rbrrYrYrZ visit_daterzConstructWriter.visit_date time_typecCrrar.)rbrrYrYrZ visit_timerzConstructWriter.visit_timetimestamp_typecCrrar-)rbrrYrYrZvisit_timestamprzConstructWriter.visit_timestamptimestamp_ns_typecCrrar*)rbrrYrYrZvisit_timestamp_nsrz"ConstructWriter.visit_timestamp_nstimestamptz_typecCrrar,)rbrrYrYrZvisit_timestamptzrz!ConstructWriter.visit_timestamptztimestamptz_ns_typecCrrar+)rbrrYrYrZvisit_timestamptz_nsrz$ConstructWriter.visit_timestamptz_ns string_typecCrrar()rbrrYrYrZ visit_stringrzConstructWriter.visit_string uuid_typecCrrar0)rbrrYrYrZ visit_uuidrzConstructWriter.visit_uuid binary_typecCrrar)rbrrYrYrZ visit_binaryrzConstructWriter.visit_binary unknown_typecCrrar/)rbrrYrYrZ visit_unknownrzConstructWriter.visit_unknownN)1__name__ __module__ __qualname____doc__r5r1r_rLrtrdrIrmrFrHrxrBr}r@rr>rrErrGrrCrrArr?rrQrrNrrMrrPrrOrrKrrSrr=rrRrrYrYrYrZr^s0r^ record_schemacCs"||krt|St||ttS)aOResolve the file and read schema to produce a reader. Args: record_schema (Schema | IcebergType): The schema of the record in memory. file_schema (Schema | IcebergType): The schema of the file that will be written Raises: NotImplementedError: If attempting to resolve an unrecognized object type. )r]r9WriteSchemaResolverSchemaPartnerAccessor)rrUrYrYrZresolve_writers r read_schema read_enumscCst||t||tS)a>Resolve the file and read schema to produce a reader. Args: file_schema (Schema | IcebergType): The schema of the Avro file. read_schema (Schema | IcebergType): The requested read schema which is equal, subset or superset of the file schema. read_types (Dict[int, Callable[..., StructProtocol]]): A dict of types to use for struct data. read_enums (Dict[int, Callable[..., Enum]]): A dict of fields that have to be converted to an enum. Raises: NotImplementedError: If attempting to resolve an unrecognized object type. )r9ReadSchemaResolverr)rUrrVrrYrYrZrXsrXc@speZdZUdZdZedefed<eed<dedefdeddfdd Z d e defd d Z d e ddfd dZ dS) EnumReaderz5An Enum reader to wrap primitive values into an Enum.enumreader.rrrWNcCs||_||_dSrar)rbrrrYrYrZ__init__s zEnumReader.__init__decodercCs||j|Sra)rrreadrbrrYrYrZrrqzEnumReader.readcCsdSrarYrrYrYrZskiprczEnumReader.skip) rrrr __slots__rr__annotations__rrrrrrYrYrYrZrs rc seZdZdededBdedefddZdededBd eedefd d Z d e d edBdedefddZ de dedBdedefddZde dedBdededef ddZdededBdeffdd Zded edBdefd!d"Zd#ed edBdefd$d%Zd&ed edBdefd'd(Zd)ed edBdefd*d+Zd,ed edBdefd-d.Zd/ed edBdefd0d1Zd2ed edBdefd3d4Zd5ed edBdefd6d7Z d8e!d edBdefd9d:Z"d;e#d edBdefde%d edBdefd?d@Z&dAe'd edBdefdBdCZ(dDe)d edBdefdEdFZ*dGe+d edBdefdHdIZ,dJe-d edBdefdKdLZ.dMe/d edBdefdNdOZ0dPe1d edBdefdQdRZ2Z3S)SrrUrNrhrWcC|SrarY)rbrUrrhrYrYrZr_ rczWriteSchemaResolver.schema record_struct file_writerscCst|ts td|ddt|jD}g}t||jddD];\}}|j|vr4|||j|fq |jrT|j durI|dt ||j dfq |jrSt d|q |d|fq t t |dS) Nz2File/write schema are not aligned for struct, got cSi|]\}}|j|qSrYfield_idrfrgrmrYrYrZ z.WriteSchemaResolver.struct..Tstrict)writervaluez2Field is required, and there is no write default: ) field_writers) isinstancerLr2rkfieldsziprappendrp write_defaultr ValueErrorr)rj)rbrUrrrecord_struct_positionsresultsr file_fieldrYrYrZrds   zWriteSchemaResolver.structr record_type field_writercC|jr|St|Sraro)rbrrrrYrYrZrm&rqzWriteSchemaResolver.fieldfile_list_type file_listelement_writercCst|jr|St|Sra)r%element_requiredr')rbrrrrYrYrZrt)szWriteSchemaResolver.list file_map_typefile_primitive key_writer value_writercCst||jr|St|Sra)r&value_requiredr')rbrrrrrYrYrZrx,rlzWriteSchemaResolver.maprecord_primitivecs(|dur ||kr t||t||Sra)r7super primitive)rbrr __class__rYrZr/s zWriteSchemaResolver.primitiverpartnercCrrarrbrrrYrYrZr7rz!WriteSchemaResolver.visit_booleanrcCrrarrbrrrYrYrZr:rz!WriteSchemaResolver.visit_integerrcCrrarrbrrrYrYrZr=rzWriteSchemaResolver.visit_longrcCrrarrbrrrYrYrZr@rzWriteSchemaResolver.visit_floatrcCrrarrbrrrYrYrZrCrz WriteSchemaResolver.visit_doublercCrrarrbrrrYrYrZrFrz!WriteSchemaResolver.visit_decimalrcCrrarrbrrrYrYrZrIrzWriteSchemaResolver.visit_datercCrrarrbrrrYrYrZrLrzWriteSchemaResolver.visit_timercCrrarrbrrrYrYrZrOrz#WriteSchemaResolver.visit_timestamprcCrrarrbrrrYrYrZrRrz&WriteSchemaResolver.visit_timestamp_nsrcCrrarrbrrrYrYrZrUrz%WriteSchemaResolver.visit_timestamptzrcCrrarrbrrrYrYrZrXrz(WriteSchemaResolver.visit_timestamptz_nsrcCrrarrbrrrYrYrZr[rz WriteSchemaResolver.visit_stringrcCrrarrbrrrYrYrZr^rzWriteSchemaResolver.visit_uuidrycCrzrar{rbryrrYrYrZr}ar~zWriteSchemaResolver.visit_fixedrcCrrarrbrrrYrYrZrdrz WriteSchemaResolver.visit_binaryrcCrrarrbrrrYrYrZrgrz!WriteSchemaResolver.visit_unknown)4rrrr5rDr1r_rLrtrdrIrmrFrHrxrJrr>rrErrGrrCrrArr@rr?rrQrrNrrMrrPrrOrrKrrSrrBr}r=rrRr __classcell__rYrYrrZr s0"""rc s8eZdZUdZeeedeffed<eeede ffed<e eed<e e fdeeedeffdeeede ffddfdd Z d e d edBd edefd dZdededBddfddZdededBddfddZdededBde edefddZdededBdedefddZdededBd edefd!d"Z d#ed$edBd%ed&edef d'd(Zd)ed*edBdeffd+d, Zd-ed.edBdefd/d0Zd1ed.edBdefd2d3Zd4e d.edBdefd5d6Z!d7e"d.edBdefd8d9Z#d:e$d.edBdefd;d<Z%d=e&d.edBdefd>d?Z'd@e(d.edBdefdAdBZ)dCe*d.edBdefdDdEZ+dFe,d.edBdefdGdHZ-dIe.d.edBdefdJdKZ/dLe0d.edBdefdMdNZ1dOe2d.edBdefdPdQZ3dRe4d.edBdefdSdTZ5dUe6d.edBdefdVdWZ7dXe8d.edBdefdYdZZ9d[e:d.edBdefd\d]Z;d^eS)arrVrcontext.rVrrrWNcCs||_||_g|_dSrar)rbrVrrYrYrZrqs zReadSchemaResolver.__init__r_expected_schemarhcCrrarY)rbr_rrhrYrYrZr_zrczReadSchemaResolver.schemarm field_partnercCs|j|jdSra)rrrrbrmrrYrYrZ before_field}rqzReadSchemaResolver.before_fieldcCs|jdSra)rpopr rYrYrZ after_fieldrzReadSchemaResolver.after_fieldrdexpected_struct field_readersc stjdkr jtnt}j|t}|s!ttt|||St |t s-t d|ddt|j Dfddt |j |ddD}d d |j D}t|j D]0\}} | j|vrt | trs| jdurs||t| jfqU| jr}t | d ||tfqUtt|||S) Nr1File/read schema are not aligned for struct, got cSrrYrrrYrYrZrrz-ReadSchemaResolver.struct..cs<g|]\}}|jj|js|nt|fqSrY)getrrr)rfrm result_reader enum_typeexpected_positionsrbrYrZ s  z-ReadSchemaResolver.struct..TrcSsh|]}|jqSrYr)rfrmrYrYrZ sz,ReadSchemaResolver.struct..z1 is non-optional, and not part of the file schema)r|r STRUCT_ROOTrVrr;rrjrkrrLr2rrrrIinitial_defaultrr rpr) rbrdr rread_struct_idstruct_callabler file_fieldsrg read_fieldrYrrZrds(   zReadSchemaResolver.structexpected_field field_readercCrra)rpr)rbrmrrrYrYrZrmrqzReadSchemaResolver.fieldrr expected_listelement_readercCs4|rt|tstd|t|jr|St|SNz/File/read schema are not aligned for list, got )rrFr2rrr)rbrrrr rYrYrZrtszReadSchemaResolver.listru expected_map key_reader value_readercCs6|rt|tstd|t||jr|St|SNz.File/read schema are not aligned for map, got )rrHr2rrr)rbrur"r#r$rYrYrZrxszReadSchemaResolver.maprexpected_primitivecsF|durt|tstd|d|||krt||t||S)Nz%File/read schema are not aligned for z, got )rrJr2r7rr)rbrr&rrYrZrs   zReadSchemaResolver.primitiverrcCrra)rrrYrYrZrrz ReadSchemaResolver.visit_booleanrcCrrar rrYrYrZrrz ReadSchemaResolver.visit_integerrcCrrar'rrYrYrZrrzReadSchemaResolver.visit_longrcCrra)r rrYrYrZrrzReadSchemaResolver.visit_floatrcCrra)r rrYrYrZrrzReadSchemaResolver.visit_doublercCrra)rrrrrYrYrZrrz ReadSchemaResolver.visit_decimalrcCrra)rrrYrYrZrrzReadSchemaResolver.visit_datercCrra)rrrYrYrZrrzReadSchemaResolver.visit_timercCrra)rrrYrYrZrrz"ReadSchemaResolver.visit_timestamprcCrra)rrrYrYrZrrz%ReadSchemaResolver.visit_timestamp_nsrcCrra)rrrYrYrZrrz$ReadSchemaResolver.visit_timestamptzrcCrra)rrrYrYrZrrz'ReadSchemaResolver.visit_timestamptz_nsrcCrra)rrrYrYrZrrzReadSchemaResolver.visit_stringrcCrra)rrrYrYrZrrzReadSchemaResolver.visit_uuidrycCrzra)r r|rrYrYrZr}r~zReadSchemaResolver.visit_fixedrcCrra)rrrYrYrZrrzReadSchemaResolver.visit_binaryrcCrra)rrrYrYrZrrz ReadSchemaResolver.visit_unknown)?rrrrdictintrr<rrrtr:rr5rDrr_rIr r rLrdrmrFrHrxrJrr>rrErrGrrCrrArr@rr?rrQrrNrrMrrPrrOrrKrrSrrBr}r=rrRrrrYrYrrZrksN    "%" "rc @seZdZdedBdedBfddZdedBdedededBfdd Zd edBdedBfd d Zd edBdedBfddZ d edBdedBfddZ dS)rrNrWcCs t|tr |Std|)Nz1File/read schema are not aligned for schema, got )rr5 as_structr2)rbrrYrYrZschema_partners z$SchemaPartnerAccessor.schema_partnerr field_namecCs2t|tr ||}ntd||r|jSdS)Nr)rrLrmr2 field_type)rbrrr,rmrYrYrZrs  z#SchemaPartnerAccessor.field_partner partner_listcCt|tr|jStd|r!)rrF element_typer2)rbr.rYrYrZlist_element_partner z*SchemaPartnerAccessor.list_element_partner partner_mapcCr/r%)rrHkey_typer2rbr3rYrYrZmap_key_partner r2z%SchemaPartnerAccessor.map_key_partnercCr/r%)rrH value_typer2r5rYrYrZmap_value_partnerr2z'SchemaPartnerAccessor.map_value_partner) rrrrDr+r)strrr1r6r8rYrYrYrZrs "rN)hcollections.abcrrrpyiceberg.avro.decoderrpyiceberg.avro.readerrrrrr r r r r rrrrrrrrrrrrrrpyiceberg.avro.writerrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1pyiceberg.exceptionsr2pyiceberg.schemar3r4r5r6r7r8r9pyiceberg.typedefr:r;r<pyiceberg.typesr=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrr(r)r[r]r^r\rrXrrrrrYrYrYrZs\   d` $ d  F  _