o ˳i0@s,ddlmZmZmZmZmZddlZddlmZddl m Z m Z m Z m Z mZmZmZmZmZddlmZmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!ddl"m#Z#m$Z$ddl%m&Z&m'Z'm(Z(dd l)m*Z*dd l+m,Z,m-Z-dd l.m/Z/dd l0m1Z1m2Z2dd l3m4Z4m5Z5ddl6m7Z7m8Z8m9Z9ddl:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMddlNmOZOmPZPerddlQZRddlSmTZTddlUmVZVmWZWmXZXmYZYmZZZm[Z[dZ\dZ]dZ^dZ_dZ`dZadZbdZcdZddZedZfdZgdZhd Zid!Zjd"Zkd#Zld$ZmekelemgZn   dOd%eod&ed'd(eodBd)e9dBd*e9f d+d,Zpefd?d@ZudAeod&d'd*d>fdBdCZvdDeod=e9d*dEfdFdGZwdHdIdJeod*dfdKdLZxGdMdNdNeZydS)Q) TYPE_CHECKINGAnyOptionalUnioncastN)Config) BOTOCORE_SESSIONEXTERNAL_TABLEICEBERGLOCATIONMETADATA_LOCATIONPREVIOUS_METADATA_LOCATION TABLE_TYPEMetastoreCatalogPropertiesUpdateSummary)CommitFailedExceptionNamespaceAlreadyExistsErrorNamespaceNotEmptyErrorNoSuchIcebergTableErrorNoSuchNamespaceErrorNoSuchPropertyExceptionNoSuchTableErrorTableAlreadyExistsError)AWS_ACCESS_KEY_IDAWS_PROFILE_NAME AWS_REGIONAWS_SECRET_ACCESS_KEYAWS_SESSION_TOKEN)UNPARTITIONED_PARTITION_SPEC PartitionSpec)Schema SchemaVisitorvisit) FromInputFile)CommitTableResponseTable) TableMetadata)UNSORTED_SORT_ORDER SortOrder)TableRequirement TableUpdate) EMPTY_DICT Identifier Properties) BinaryType BooleanTypeDateType DecimalType DoubleType FixedType FloatType IntegerTypeListTypeLongTypeMapType NestedField PrimitiveType StringType StructType TimestampTypeTimestamptzTypeTimeTypeUUIDType)get_first_property_valueproperty_as_bool) GlueClient) ColumnTypeDefDatabaseInputTypeDefDatabaseTypeDefStorageDescriptorTypeDefTableInputTypeDef TableTypeDefzglue.idzglue.skip-archiveTz glue.endpointziceberg.field.idziceberg.field.optionalziceberg.field.currentzglue.profile-namez glue.regionzglue.access-key-idzglue.secret-access-keyzglue.session-tokenzglue.max-retrieszglue.retry-mode standardadaptivelegacymetadata_location glue_tablerIprev_metadata_locationmetadata_propertiesreturncCs\|r|dini}|ttt|i|r||t<|r,|D] \}}t|||<q!|S)N Parameters) getupdaterr upperr r itemsstr)rNrOrPrQnew_parameterskeyvaluer\J/home/ubuntu/.local/lib/python3.10/site-packages/pyiceberg/catalog/glue.py_construct_parameterssr^booleanintbigintfloatdoubledatestring timestampbinaryc@seZdZdededefddZdedeedefdd Zd e d edefd d Z de dedefddZde dededefddZ dedefddZdS)_IcebergSchemaToGlueTypeschema struct_resultrRcCs|SNr\)selfrirjr\r\r]riz_IcebergSchemaToGlueType.schemastruct field_resultscCsdd|dS)Nzstruct<,>)join)rlrnror\r\r]rnz_IcebergSchemaToGlueType.structfield field_resultcCs|jd|S)N:)name)rlrtrur\r\r]rtsz_IcebergSchemaToGlueType.field list_typeelement_resultcCs d|dS)Nzarray._append_to_resultsTF) r9bool schema_by_idcurrent_schema_idcolumnsschemas schema_idrzvalues)rrcurrent_schemartrir\rr] _to_columnss       r table_name propertiesrHcCs<|tt||||t||jdd}d|vr|d|d<|S)N)ColumnsLocation)r TableTyperSStorageDescriptor Description)r r^rlocation)rrNrrrOrP table_inputr\r\r]_construct_table_inputs   r to_table_namecCs~d|i}|ds td|d|d<d|vr|d|d<d|vr&|d|d<d|vr3td|d|d<d|vr=|d|d<|S) Nrrz/Glue table type is missing, cannot rename tableOwnerrSrrGr) ValueErrorr)rrOrename_table_inputr\r\r]_construct_rename_table_inputs    r database_namerEcCsVd|i}i}|D]\}}|dkr||d<q |tkr ||d<q |||<q ||d<|S)Nrr LocationUrirS)rWr )rrdatabase_input parameterskvr\r\r]_construct_database_inputs   rgluerCglue_catalog_idcs:|jj}dtttfdtddffdd }|d|dS)a+ Register the Glue Catalog ID (AWS Account ID) as a parameter on all Glue client methods. It's more ergonomic to do this than to pass the CatalogId as a parameter to every client call since it's an optional parameter and boto3 does not support 'None' values for missing parameters. paramskwargsrRNcsd|vr |d<dSdS)N CatalogIdr\)rrrr\r]add_glue_catalog_id0s zG_register_glue_catalog_id_with_glue_client..add_glue_catalog_idzprovide-client-params.glue)metaeventsdictrXrregister)rr event_systemrr\rr]*_register_glue_catalog_id_with_glue_client's"rcseZdZUded<dKdededdeffdd Zd d d efd d Z dededdd dfddZ dededdded df ddZ deded d fddZ de eefdeeBdeedfdedBdededed efdd ZdeeBd!ed efd"d#Zd$ed%eed&fd'eed&fd efd(d)ZdeeBd efd*d+ZdeeBd dfd,d-Zd.eeBd/eeBd efd0d1Zefd2eeBded dfd3d4Z d2eeBd dfd5d6Z!d2eeBd e"efd7d8Z#dLd2eeBd e"efd:d;Z$d2eeBd efde&edBd'ed e'fd?d@Z(d2eeBd e"efdAdBZ)deeBd dfdCdDZ*deeBd e+fdEdFZ,e-d$d d e+fdGdHZ.deded efdIdJZ/Z0S)M GlueCatalogrCrNrwclientrc stj|fi||dur||_dSt|t}tjt|ttt|t t | t t|t tt|ttt|ttd}|jd| tt| tt|tvrM|ntddd|_| t}ret|j|dSdS)aDGlue Catalog. You either need to provide a boto3 glue client, or one will be constructed from the properties. Args: name: Name to identify the catalog. client: An optional boto3 glue client. properties: Properties for glue client construction and configuration. N) profile_name region_namebotocore_sessionaws_access_key_idaws_secret_access_keyaws_session_tokenr) max_attemptsmode)retries) endpoint_urlconfig)super__init__rrAGLUE_RETRY_MODEboto3SessionGLUE_PROFILE_NAMEr GLUE_REGIONrrTrGLUE_ACCESS_KEY_IDrGLUE_SECRET_ACCESS_KEYrGLUE_SESSION_TOKENrrGLUE_CATALOG_ENDPOINTrGLUE_MAX_RETRIES MAX_RETRIESEXISTING_RETRY_MODESSTANDARD_RETRY_MODEGLUE_IDr)rlrwrrretry_mode_prop_valuesessionr __class__r\r]r:s0          zGlueCatalog.__init__rOrIrRc Cs|d}dur td|d}durtd|d}dur'td|t}dur=tdtd|d ||tkrStd |d td |d ||t}duritd td|d ||j|d}| |}t |} t ||f| ||| j ||dS)N DatabaseNamez+Glue table is missing DatabaseName propertyrz#Glue table is missing Name propertyrSz)Glue table is missing Parameters propertyz Property z$ missing, could not determine type: .zProperty table_type is z , expected z: zTable property z' is missing, cannot find metadata for: r) identifierrrNiocatalog)rTrrrrr rr _load_file_io new_inputr#table_metadatar%r) rlrOrrrglue_table_typerNrfilerr\r\r]_convert_glue_to_icebergas8     z$GlueCatalog._convert_glue_to_icebergrrrrHc Cs|z |jj||dWdS|jjjy&}z td|d|d|d}~w|jjjy=}z td|d|d}~ww)N)r TableInputzTable r already exists Database z does not exist)r create_table exceptionsAlreadyExistsExceptionrEntityNotFoundExceptionr)rlrrrer\r\r]_create_glue_tableszGlueCatalog._create_glue_table version_idc Csz|jj||t|jtt|dWdS|jjjy0}ztd|d|d|d|d}~w|jjj yL}zt d|d|d||d}~ww)N)rr SkipArchive VersionIdTable does not exist: rz (Glue table version rCannot commit z: because Glue detected concurrent update to table version ) r update_tablerBrGLUE_SKIP_ARCHIVEGLUE_SKIP_ARCHIVE_DEFAULTrrrConcurrentModificationExceptionr)rlrrrrrr\r\r]_update_glue_tables(  zGlueCatalog._update_glue_tablec CsPz |jj||d}|dWS|jjjy'}z td|d||d}~ww)Nrrr%rr)r get_tablerrr)rlrrload_table_responserr\r\r]_get_glue_tables zGlueCatalog._get_glue_tablerriz pa.Schemarpartition_spec sort_orderc Csf|j||||||d}||\}} ||j|j|jt| |j||j} |j|| | d|j|dS)a{ Create an Iceberg table. Args: identifier: Table identifier. schema: Table's schema. location: Location for the table. Optional Argument. partition_spec: PartitionSpec for the table. sort_order: SortOrder for the table. properties: Table properties that can be a string based dictionary. Returns: Table: the created table instance. Raises: AlreadyExistsError: If a table with the name already exists. ValueError: If the identifier is invalid, or no path is given to store metadata. )rrirr r rrrrr) _create_staged_table identifier_to_database_and_table_write_metadatarrrNrr load_table) rlrrirr r r staged_tablerrrr\r\r]rs zGlueCatalog.create_tablerNc Cs\||\}}t}|j|d}||}t|}t||||} |j||| d|j|dS)aiRegister a new table using existing metadata. Args: identifier (Union[str, Identifier]): Table identifier for the table metadata_location (str): The location to the metadata Returns: Table: The newly registered table Raises: TableAlreadyExistsError: If the table already exists rr r ) rr+rrr#rrrr) rlrrNrrrrrrrr\r\r]register_tables     zGlueCatalog.register_tabletable requirements.updatesc Cs*|}||t\}}z|j||d}|d}|j|d} Wnty/d}d}d} Ynw|| |||} | rH| j| jkrHt| j| j dS|j | j| j | j d| rz|sbt d|d|d t || j | j| j|| j d } |j||| |d nt || j | j| jd } |j||| d t| j| j dS)aCommit updates to a table. Args: table (Table): The table to be updated. requirements: (Tuple[TableRequirement, ...]): Table requirements. updates: (Tuple[TableUpdate, ...]): Table updates. Returns: CommitTableResponse: The updated metadata. Raises: NoSuchTableError: If a table with the given identifier does not exist. CommitFailedException: Requirement not met, or a conflict with a concurrent commit. rrrrON)rrN)rr metadata_pathrrz) because Glue table version id is missing)rrNrrrOrP)rrrr)rrNrrr )rwrrr rTr_update_and_stage_tablerr$rNrrrrrrr) rlrrrtable_identifierrrcurrent_glue_tableglue_table_version_id current_tableupdated_staged_tableupdate_table_inputcreate_table_inputr\r\r] commit_tables`  zGlueCatalog.commit_tablecCs$||t\}}||j||dS)aLoad the table's metadata and returns the table instance. You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError'. Note: This method doesn't scan data stored in the table. Args: identifier: Table identifier. Returns: Table: the table instance with its metadata. Raises: NoSuchTableError: If a table with the name does not exist, or the identifier is invalid. r)rrrr )rlrrrr\r\r]r5szGlueCatalog.load_tablec Cs\||t\}}z |jj||dWdS|jjjy-}z td|d||d}~ww)zDrop a table. Args: identifier: Table identifier. Raises: NoSuchTableError: If a table with the name does not exist, or the identifier is invalid. rrrN)rrr delete_tablerr)rlrrrrr\r\r] drop_tableHs zGlueCatalog.drop_tablefrom_identifier to_identifierc Cs||t\}}||\}}z |jj||d}Wn|jjjy3}z td|d||d}~ww|d} z|j| dWn/tyX}z td|d|d|d}~wtyo}z td|d|d |d}~wwt || d } |j ||| d z| |WnAt y}z5d |d|d } z| || d|d|d7} Wnty| d|d|d7} Ynwt | |d}~ww||S)aRename a fully classified table name. This method can only rename Iceberg tables in AWS Glue. Args: from_identifier: Existing table identifier. to_identifier: New table identifier. Returns: Table: the updated table instance with its metadata. Raises: ValueError: When from table identifier is invalid. NoSuchTableError: When a table with the name does not exist. NoSuchIcebergTableError: When from table is not a valid iceberg table. NoSuchPropertyException: When from table miss some required properties. NoSuchNamespaceError: When the destination namespace doesn't exist. rrrNr%rzFailed to rename table z( since it is missing required propertiesz& since it is not a valid iceberg table)rrOr zFailed to drop old table z. zRolled back table creation for z'Failed to roll back table creation for z. Please clean up manually)rrrrrrrrrrrr$ Exceptionrr) rlr%r&from_database_namefrom_table_nameto_database_namerget_table_responserrOr log_messager\r\r] rename_tableWsX     zGlueCatalog.rename_table namespacec CsV||}z |jjt||dWdS|jjjy*}z td|d|d}~ww)aTCreate a namespace in the catalog. Args: namespace: Namespace identifier. properties: A string dictionary of properties for the given namespace. Raises: ValueError: If the identifier is invalid. AlreadyExistsError: If a namespace with the given name already exists. ) DatabaseInputrrN)identifier_to_databasercreate_databaserrrr)rlr.rrrr\r\r]create_namespaces zGlueCatalog.create_namespacec Cs||t}z |jj|d}|d}Wn|jjjy*}ztd||d}~wwt|dkrJ|d}||rBtd|dtd|d|jj |d dS) a[Drop a namespace. A Glue namespace can only be dropped if it is empty. Args: namespace: Namespace identifier. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid. NamespaceNotEmptyError: If the namespace is not empty. r TableListDatabase does not exist: NrzCannot drop namespace z) because it still contains Iceberg tablesz- because it still contains non-Iceberg tablesr) r0rr get_tablesrrlen_GlueCatalog__is_iceberg_tablerdelete_database)rlr.rtable_list_response table_listr first_tabler\r\r]drop_namespaces     zGlueCatalog.drop_namespacec s|tg}d}z$ |sjjdnjj|d}||d|d}|s,nq WnjjjyE}ztd|d}~wwfdd |DS) anList Iceberg tables under the given namespace in the catalog. Args: namespace (str | Identifier): Namespace identifier to search. Returns: List[Identifier]: list of table identifiers. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid. NTr3)r NextTokenr4r?r5cs"g|] }|r|dfqSr6)r9).0rrrlr\r] s"z+GlueCatalog.list_tables..)r0rrr7extendrTrr)rlr.r< next_tokenr;rr\rAr] list_tabless(   zGlueCatalog.list_tablesr\cs`|rgSg}d} |sjnjj|d}||d|d}|s&nq fdd|DS)zList namespaces from the given namespace. If not given, list top-level namespaces from the catalog. Returns: List[Identifier]: a List of namespace identifiers. NT)r? DatabaseListr?csg|] }|dqSr6)identifier_to_tuple)r@databaserlr\r]rBsz/GlueCatalog.list_namespaces..)r get_databasesrCrT)rlr. database_listrDdatabases_responser\rIr]list_namespacess zGlueCatalog.list_namespacesc Cs||t}z |jj|d}Wn-|jjjy&}ztd||d}~w|jjjy<}ztd||d}~ww|d}t|di}d|vrS|d|d<d |vr]|d |d <|S) a2Get properties for a namespace. Args: namespace: Namespace identifier. Returns: Properties: Properties for the given namespace. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist, or identifier is invalid. r6r5NzInvalid input for namespace DatabaserSrrr) r0rr get_databaserrInvalidInputExceptionrrT)rlr.rdatabase_responserrHrr\r\r]load_namespace_propertiess"   z%GlueCatalog.load_namespace_propertiesremovalscCsF|j|d}|j|||d\}}||t}|jj|t||d|S)uRemove provided property keys and updates properties for a namespace. Args: namespace: Namespace identifier. removals: Set of property keys that need to be removed. Optional Argument. updates: Properties to be updated for the given namespace. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist， or identifier is invalid. ValueError: If removals and updates have overlapping keys. )r.)current_propertiesrSr)rr/)rR%_get_updated_props_and_update_summaryr0rrupdate_databaser)rlr.rSrrTproperties_update_summaryupdated_propertiesrr\r\r]update_namespace_propertiess  z'GlueCatalog.update_namespace_propertiescCtrkNotImplementedError)rlr.r\r\r] list_views,rmzGlueCatalog.list_viewscCrZrkr[rlrr\r\r] drop_view/rmzGlueCatalog.drop_viewcCrZrkr[r^r\r\r] view_exists2rmzGlueCatalog.view_existscCs|ditdtkS)NrS)rTrrr )rr\r\r]__is_iceberg_table5szGlueCatalog.__is_iceberg_tablecCs |||S)zIOverride the default warehouse location to follow Hive-style conventions.)"_get_hive_style_warehouse_location)rlrrr\r\r]_get_default_warehouse_location9s z+GlueCatalog._get_default_warehouse_locationrk)r\)1rrr__annotations__rXrrrr%rrrr rr'r+r,rr rr(r-rrtupler)r*r$r"rr$r-r2r>rzrErMrRsetrrYr]r_rr` staticmethodr9rd __classcell__r\r\rr]r7sx  '$   ,   O<  r)NNN)NN)ztypingrrrrrrbotocore.configrpyiceberg.catalogrr r r r r rrrpyiceberg.exceptionsrrrrrrrr pyiceberg.iorrrrrpyiceberg.partitioningrrpyiceberg.schemar r!r"pyiceberg.serializersr#pyiceberg.tabler$r%pyiceberg.table.metadatar&pyiceberg.table.sortingr'r(pyiceberg.table.updater)r*pyiceberg.typedefr+r,r-pyiceberg.typesr.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@pyiceberg.utils.propertiesrArBpyarrowpamypy_boto3_glue.clientrCmypy_boto3_glue.type_defsrDrErFrGrHrIrrrrrrrrrrrrrrrrADAPTIVE_RETRY_MODELEGACY_RETRY_MODErrXr^rrhrzrrrrrrr\r\r\r]s ,(    T    (