o uyi0@sddlmZddlZddlmZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZddlmZmZddlmZdd lmZmZmZmZmZmZmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4e rddl5Z6ddl7m8Z8dZ9GdddeZ:eGdddZ;Gddde*dZe?fZ@Gdddee>ZAd+d!d"ZBd,d%d&ZCd-d)d*ZDdS).) annotationsN)copy) dataclass)Enum) TYPE_CHECKINGAnyDictListOptionalSetTupleUnion) ResolveErrorValidationError)literal)PartnerAccessorSchema SchemaVisitorSchemaWithPartnerVisitorassign_fresh_schema_idspromotevisitvisit_with_partner) NameMappingupdate_mapping)AddSchemaUpdateAssertCurrentSchemaIdSetCurrentSchemaUpdateSetPropertiesUpdateTableRequirement TableUpdateUpdatesAndRequirementsUpdateTableMetadata)L TableVersion) IcebergTypeListTypeMapType NestedField PrimitiveType StructType) Transactionc@seZdZdZdZdZdS)_MoveOperationN)__name__ __module__ __qualname__FirstBeforeAfterr7r7`/home/ubuntu/maya3_transcribe/venv/lib/python3.10/site-packages/pyiceberg/table/update/schema.pyr->sr-c@s2eZdZUded<ded<ded<dZded <dS) _Moveintfield_idstr full_namer-opN Optional[int]other_field_id)r1r2r3__annotations__r@r7r7r7r8r9Ds r9csneZdZUded<ded<ded<iZded<iZd ed <eZd ed <iZd ed<iZ ded<iZ ded<ded<ded<    dhdifd#d$ Z djd%d&Z 'dkdld,d-Z   dmdnd7d8Zdod9d:Zdpd;d<Zdqd@dAZdodBdCZdrdEdFZdsdGdHZdtdJdKZ   dudvdNdOZdwdRdSZdxdVdWZdodXdYZdyd[d\Zdzd^d_Zd{dadbZd|dcddZd}dfdgZZS)~ UpdateSchemar_schemazitertools.count[int]_last_column_idzSet[str]_identifier_field_namesDict[int, List[NestedField]]_addsDict[int, NestedField]_updatesSet[int]_deletesDict[int, List[_Move]]_moveszDict[str, int]_added_name_to_idzDict[int, str] _id_to_parentbool_allow_incompatible_changes_case_sensitiveFTN transactionr+allow_incompatible_changescase_sensitiveschemaOptional[Schema] name_mappingOptional[NameMapping]returnNonecst|t|tr|_td|j_nj j _tdj j j _|_ j_i_i_t_i_i_d fdd fdd jjD_|_|_|_ dS) Nr.r;r:rZr<cs(jj|d}|durtd||S)N) column_idzCould not find field-id: )rCfind_column_name ValueError)r; column_nameselfr7r8get_column_namewsz.UpdateSchema.__init__..get_column_namecsi|] \}}||qSr7r7).0r;parent_field_id)rbr7r8 }sz)UpdateSchema.__init__..)r;r:rZr<)super__init__ isinstancerrC itertoolscounthighest_field_idrD _transactiontable_metadatarVlast_column_id _name_mappingidentifier_field_namesrErGrIsetrKrMrN_lazy_id_to_parentitemsrOrQrR)rarSrTrUrVrX __class__)rbrar8rg\s(      zUpdateSchema.__init__cCs ||_|S)zDetermine if the case of schema needs to be considered when comparing column names. Args: case_sensitive: When false case is not considered in column name comparisons. Returns: This for method chaining )rR)rarUr7r7r8rUs zUpdateSchema.case_sensitiver/ new_schemaUnion[Schema, 'pa.Schema']format_versionr$cCsBddlm}t|j||ddt||j|jdt|j|jd|S)Nr)Catalog)rxr, update_schemaexisting_schemarUpartner_schemarU)pyiceberg.catalogryr_convert_schema_if_needed_UnionByNameVisitorrCrRPartnerIdByNameAccessor)rarvrxryr7r7r8 union_by_names  zUpdateSchema.union_by_namepathUnion[str, Tuple[str, ...]] field_typer%doc Optional[str]required default_value Optional[L]c Cst|trd|vrtd|d|f}|d}|dd}d|}d|} t} t|dkr]|j| |j} | j } t| t rD| j } nt| t rL| j } | j jsZtd|d| | j} d} z |j||j} Wn tyrYnw| dur| j|jvrtd ||}t||j}|durz t||j}Wnty}ztd ||d}~ww|}|r|dur|jstd d|||j|<| |j|<t|||||||d }| |jvr|j| ||S|g|j| <|S) a&Add a new column to a nested struct or Add a new top-level column. Because "." may be interpreted as a column path separator or may be used in field names, it is not allowed to add nested column by passing in a string. To add to nested structures or to add fields with names that contain "." use a tuple instead to indicate the path. If type is a nested type, its field IDs are reassigned when added to the existing schema. Args: path: Name for the new column. field_type: Type for the new column. doc: Documentation string for the new column. required: Whether the new column is required. default_value: Default value for the new column. Returns: This for method chaining. .z'Cannot add column with ambiguous name: z, provide a tuple insteadr,NrzCannot add column 'z' to non-struct type: z(Cannot add column, name already exists: Invalid default value: z1Incompatible change: cannot add required column: r;namerrrinitial_default write_default)rhr<r^join TABLE_ROOT_IDlenrC find_fieldrRrr' value_fieldr& element_field is_structr;rKassign_new_column_idrrtovaluerQrNrOr(rGappend)rarrrrrrparentr=parent_full_path parent_id parent_field parent_typeexisting_fieldnew_idnew_typerefieldr7r7r8 add_columnsn             zUpdateSchema.add_columncCsvt|tr|fn|}d|}|jj||jd}|j|jvr%td||j|j vr2td||j |j|S)zDelete a column from a table. Args: path: The path to the column. Returns: The UpdateSchema with the delete operation staged. rrUz+Cannot delete a column that has additions: z)Cannot delete a column that has updates: ) rhr<rrCrrRr;rGr^rIrKadd)rarrr=rr7r7r8 delete_columns   zUpdateSchema.delete_columncCs||||S)zSet the default value of a column. Args: path: The path to the column. Returns: The UpdateSchema with the delete operation staged. )_set_column_default_value)rarrr7r7r8set_default_values zUpdateSchema.set_default_value path_fromnew_namer<c Cst|tr d|n|}|j||j}|j|jvr!td||j |j}r@t |j||j |j |j|j|jd|j |j<nt |j||j |j |j|j|jd|j |j<|j|j}||jvr{|j||dt|j |}|j||S)zUpdate the name of a column. Args: path_from: The path to the column to be renamed. new_name: The new path of the column. Returns: The UpdateSchema with the rename operation staged. rz-Cannot rename a column that will be deleted: r;rrrrrrN)rhtuplerrCrrRr;rKr^rIgetr(rrrrrr]rEremoverrr)rarr field_fromupdatedfrom_field_correct_casingnew_identifier_pathr7r7r8 rename_column(s:      zUpdateSchema.rename_columncCs|j|dd|S)zMake a column optional. Args: path: The path to the field. Returns: The UpdateSchema with the requirement change staged. Fr)_set_column_requirement)rarr7r7r8make_column_optionalVs z!UpdateSchema.make_column_optionalfieldscGst||_dSN)rqrE)rarr7r7r8set_identifier_fieldsbsz"UpdateSchema.set_identifier_fieldsc Cst|tr|fn|}d|}|j||j}|jr|s!|jr#|s#dS|js0|r0t d|d|j |j vr=t d||j |j }r]t|j |j|j|j||j|jd|j |j <dSt|j |j|j|j||j|jd|j |j <dS)Nrz"Cannot change column nullability: z: optional -> required-Cannot update a column that will be deleted: r)rhr<rrCrrRroptionalrQr^r;rKrIrr(rrrrr)rarrrrrr7r7r8res8    z$UpdateSchema._set_column_requirementrc Cs.t|tr|fn|}d|}|j||j}|dur;z t||jj }Wnt y:}zt d||d}~ww|j rE||j krEdS|j sS|j rS|durSt d|j|jvr`t d||j|j}rt|j|j|j|j|j |j|d|j|j<dSt|j|j|j|j|j |j|d|j|j<dS)Nrrz?Cannot change change default-value of a required column to Nonerr)rhr<rrCrrRrrrrr^rrrQr;rKrIrr(rrr)rarrrrrrr7r7r8rsF   z&UpdateSchema._set_column_default_valueOptional[IcebergType]Optional[bool]c Cst|tr|fn|}d|}|dur|dur|dur|S|j||j}|j|jvr2td||duro|j j sCt d|j d|j so|j |krozt |j |Wntyn}zt d|d|j d||d}~ww|j|j}rt|j|j|p|j |dur|n|j|j|j|jd|j|j<nt|j|j|p|j |dur|n|j|j|j|jd|j|j<|dur|j||d |S) a7Update the type of column. Args: path: The path to the field. field_type: The new type required: If the field should be required doc: Documentation describing the column Returns: The UpdateSchema with the type update staged. rNrzCannot change column type: z is not a primitivez: z -> rr)rhr<rrCrrRr;rKr^r is_primitiverrQrrrIrr(rrrrrr) rarrrrr=rrrr7r7r8 update_columnsN    zUpdateSchema.update_columnrr?cCs4z |j||jjWStyYnw|j|Sr)rCrrRr;r^rNr)rarr7r7r8_find_for_moves   zUpdateSchema._find_for_movemover9cCs&|j|j}r[|jj||jd}|jjstd|j|j t j ks*|j t j krJ|j dur3td|j|j|j|j krJtd|jd|j|jg|g|j|j<dS|j t j ksg|j t j kr|j durptd|j|j }rtd|jd||jtg|g|jt<dS)Nrz'Cannot move fields in non-struct type: z6Expected other field when performing before/after movezCannot move field z to a different structz to a different struct: )rOrr;rCrrRrrr^r>r-r6r5r@r=rMr)rar parent_namer other_structr7r7r8_moves  " zUpdateSchema._movecCsRt|tr d|n|}||}|durtd||t||tjd|S)zMove the field to the first position of the parent struct. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNCannot move missing column: )r;r=r>) rhrrrr^rr9r-r4)rarr=r;r7r7r8 move_firsts  zUpdateSchema.move_first before_pathcCt|tr d|n|}||}|durtd|t|tr&d|n|}||}|dur;td|d|||krGtd|d|t|||tjd|S)zMove the field to before another field. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNr Cannot move z before missing column: z before itselfr;r=r@r>) rhrrrr^rr9r-r5)rarrr=r;before_full_namebefore_field_idr7r7r8 move_before-s"  zUpdateSchema.move_before after_namecCr)zMove the field to after another field. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNrrz after missing column: z after itselfr) rhrrrr^rr9r-r6)rarrr=r; after_pathafter_field_idr7r7r8 move_afterOs  zUpdateSchema.move_afterr!csddlm}|tfdd|jjjDd}d}d}||jjkr`|t |jjdf7}|dur>|t dt d d f7}n|t |d f7}|j }r`t ||j|j}|t|j|id f7}||fS) z%Apply the pending changes and commit.r)TablePropertiesc3s|] }|kr|jVqdSr schema_idrcrVrvr7r8 ssz'UpdateSchema._commit..Nr7)current_schema_id)rVr,r)updates)pyiceberg.tabler_applynextrlrmschemasrCrrrrrorrIrGrDEFAULT_NAME_MAPPINGmodel_dump_json)rarexisting_schema_id requirementsrrXupdated_name_mappingr7rr8_commitls*   zUpdateSchema._commitc Cs t|jt|j|j|j|j}|durtdt|j }ddl m }|j j jD]}|j D]}||j|j|j|tq,q't}|jD](}z |j||jd}Wntyc} z td|d| d} ~ ww||jqB|j } rd| j durtd d | j jDnd} nd} t|j | |d S) zApply the pending changes to the original schema and returns the result. Returns: the result Schema when all pending updates are applied NzCould not apply changesr)validate_partition_namerzCannot find identifier field z:. In case of deletion, update the identifier fields first.r.css|]}|jVqdSrrrr7r7r8rsz&UpdateSchema._apply..)ridentifier_field_ids)rrC _ApplyChangesrGrIrKrMr^rrpyiceberg.partitioningrrlrmpartition_specsr transform source_idrqrErrRrr;maxr) rastructrvrspecpartition_field field_idsrrrtxnnext_schema_idr7r7r8rs<      "zUpdateSchema._applyr:cCs t|jSr)rrDr`r7r7r8rs z!UpdateSchema.assign_new_column_id)FTNN) rSr+rTrPrUrPrVrWrXrYrZr[)rUrPrZrB)r/)rvrwrxr$rZrB)NFN) rrrr%rrrrPrrrZrB)rrrZrB)rrrrrZrB)rrrr<rZrB)rr<rZr[)rrrrPrZr[)rrrrrZr[)NNN) rrrrrrrrrZrB)rr<rZr?)rr9rZr[)rrrrrZrB)rrrrrZrB)rZr!)rZr)rZr:) r1r2r3rArGrIrqrKrMrNrOrgrUrrrrrrrrrrrrrrrrrr __classcell__r7r7rtr8rBLsV       )  b  .  %/ B    "  (rBc@steZdZUded<ded<ded<ded<d2ddZd3ddZd4ddZd5d d!Zd6d%d&Zd7d+d,Z d8d/d0Z d1S)9rrFrGrHrIrJrKrLrMaddsrdeletesmovesrZr[cCs||_||_||_||_dSr)rGrIrKrM)rarrrrr7r7r8rgs z_ApplyChanges.__init__rVr struct_resultrcCsf|jt}|jt}|dus|dur1t|ts td|t|j|p&g|p)g}r1t|S|SNz!Cannot add fields to non-struct: ) rGrrrMrhr*r^_add_and_move_fieldsr)rarVraddedr new_fieldsr7r7r8rVs   z_ApplyChanges.schemarr* field_resultsList[Optional[IcebergType]]c Csd}g}t|D]f\}}||}|durd}q|j|}|j}|j} |j} |j} |j|j} r=| j}| j} | j} | j} |j|kr\|j |kr\|j| kr\|j| kr\|j| kr\| |qd}| t |j||| | |j | dq|rut |S|S)NFTr) enumeraterrrrrrIrr;rrr(rr*) rarr has_changesridx result_typerrrrrupdater7r7r8rsL        z_ApplyChanges.structrr( field_resultcCs|j|jvrdS|j|j}r|j|jkr|jSt|trS|j|j}|j|j}|dus5|durSt|jtsBt d|t |j |pHg|pKg}rSt|S|Sr) r;rKrIrrrhr*rGrMr^rr)rarrrrrrr7r7r8r s   z_ApplyChanges.field list_typer&element_resultcCs6||j|}|durtd|t|j||jdS)Nz&Cannot delete element type from list: ) element_idelementelement_required)rrr^r&r r )rarr element_typer7r7r8list"sz_ApplyChanges.listmap_typer' key_result value_resultcCs|jj}||jvrtd|||jvrtd|||jvr(td||j|kr4td||j}|||}|durHtd|t |j |j|j ||j dS)NzCannot delete map keys: zCannot update map keys: zCannot add fields to map keys: zCannot alter map keys: z#Cannot delete value type from map: )key_idkey_typevalue_id value_typevalue_required) key_fieldr;rKr^rIrGrrrr'rrr)rarrrrrrr7r7r8map)s(     z_ApplyChanges.map primitiver)cCs|Srr7)rarr7r7r8rGz_ApplyChanges.primitiveN) rrFrrHrrJrrLrZr[)rVrrrrZr)rr*rrrZr)rr(rrrZr)rr&rrrZr)rr'rrrrrZr)rr)rZr) r1r2r3rArgrVrrr rrr7r7r7r8rs   5  rc@seZdZUded<ded<ded<d:d d Zd;ddZdddZd?d"d#Z d@d%d&Z dAd+d,Z dBd2d3Z dCd7d8Z d9S)DrrBr{rr|rPrUrZr[cCs||_||_||_dSrrz)rar{r|rUr7r7r8rgPs z_UnionByNameVisitor.__init__rV partner_idr?rcCs|Srr7)rarVrrr7r7r8rVUrz_UnionByNameVisitor.schemarr*missing_positions List[bool]c Cs|durdS|j}||}|jstd|t|D]$\}}|r+||||q||}|j|j|jd} r@| || qdS)NTzExpected a struct, got: rF) r_find_field_typerr^r _add_column field_by_namerrU_update_column) rarrrrpartner_structposmissingr nested_fieldr7r7r8rXs  z_UnionByNameVisitor.structrr:rr(cCsB|j|}r||jf}n|jf}|jj||j|j|jddS)N)rrrr)r|r]rr{rrrr)rarrrrr7r7r8rls z_UnionByNameVisitor._add_columnrcCs|j|j}|durtd||jr|jr|j||jj rD|j|jkrDz t |j|j}Wnt yC|jj ||jdYnw|j durZ|j |j kr\|jj ||j ddSdSdS)NzCould not find field: )r)r)r|r]r;r^rrr{rrrrrrr)rarrr=_r7r7r8r ts   z"_UnionByNameVisitor._update_columnr;r%cCs |dkr |jS|j|jSNr,)r| as_structrr)rar;r7r7r8rs z$_UnionByNameVisitor._find_field_typercC|duSrr7)rarrrr7r7r8rz_UnionByNameVisitor.fieldrr&list_partner_idelement_missingcCsN|durdS|r td||}t|tstd|||j|jdS)NTzAError traversing schemas: element is missing, but list is presentzExpected list-type, got: F)r^rrhr&r r)rarr*r+partner_list_typer7r7r8r s  z_UnionByNameVisitor.listrr'map_partner_id key_missing value_missingcCsj|durdS|r td|rtd||}t|ts#td|||j|j||j|jdS)NTzError traversing schemas: value is missing, but map is presentzExpected map-type, got: F)r^rrhr'r rr)rarr-r.r/partner_map_typer7r7r8rs  z_UnionByNameVisitor.maprr)primitive_partner_idcCr(rr7)rarr1r7r7r8rr)z_UnionByNameVisitor.primitiveN)r{rBr|rrUrPrZr[)rVrrr?rrPrZrP)rr*rr?rrrZrP)rr:rr(rZr[)rr(rr(rZr[)r;r:rZr%)rr(rr?rrPrZrP)rr&r*r?r+rPrZrP) rr'r-r?r.rPr/rPrZrP)rr)r1r?rZrP)r1r2r3rArgrVrrr rrr rrr7r7r7r8rKs          rc@sZeZdZUded<ded<dddZdd d ZdddZd ddZd!ddZd!ddZ dS)"rrr~rPrUrZr[cCs||_||_dSrr})rar~rUr7r7r8rgs z PartnerIdByNameAccessor.__init__partnerr?cCsdSr&r7)rar2r7r7r8schema_partnerrz&PartnerIdByNameAccessor.schema_partnerpartner_field_idr;r: field_namer<cCs\|dur,|dkr|j}n|j|j}|jstd||j||jd}r,|jSdS)Nr,zExpected StructType: )rrU) r~r'rrrr^rrUr;)rar4r;r5rrr7r7r8 field_partners z%PartnerIdByNameAccessor.field_partnerpartner_list_idcC@|dur|j|}rt|jtstd||jjjSdS)NzExpected ListType: )r~rrhrr&r^rr;)rar7rr7r7r8list_element_partner   z,PartnerIdByNameAccessor.list_element_partnerpartner_map_idcCr8NzExpected MapType: )r~rrhrr'r^rr;rar;rr7r7r8map_key_partnerr:z'PartnerIdByNameAccessor.map_key_partnercCr8r<)r~rrhrr'r^rr;r=r7r7r8map_value_partnerr:z)PartnerIdByNameAccessor.map_value_partnerN)r~rrUrPrZr[)r2r?rZr?)r4r?r;r:r5r<rZr?)r7r?rZr?)r;r?rZr?) r1r2r3rArgr3r6r9r>r?r7r7r7r8rs      rrTuple[NestedField, ...]rOptional[List[NestedField]]rZcCs|pg}|t|Sr)r)rrr7r7r8 _add_fieldss rBr List[_Move]cstt|}|D][tfdd|D}fdd|D}jtjkr*|g|}qjtjks6jtjkr\jtfddt |D}jtjkrS| ||q| |d|qt djt |S)Nc3s |] }|jjkr|VqdSrr;rcrrr7r8rsz_move_fields..csg|] }|jjkr|qSr7rDrErFr7r8 sz _move_fields..c3s"|] \}}|jkr|VqdSrrD)rcir)r@r7r8rs r.zUnknown operation: ) r rrr>r-r4r5r6r@rinsertr^r)rr reorderedrother_field_posr7)rr@r8 _move_fieldss    rLList[NestedField]!Optional[Tuple[NestedField, ...]]cCsht|dkrt||}t|dkrt||S|St|dkr#t||St|dkr+dStg||RS)Nr)rrBrLr)rrrrr7r7r8rs      "r)rr@rrArZr@)rr@rrCrZr@)rr@rrMrrCrZrN)E __future__rrir dataclassesrenumrtypingrrrr r r r r pyiceberg.exceptionsrrpyiceberg.expressionsrpyiceberg.schemarrrrrrrrpyiceberg.table.name_mappingrrpyiceberg.table.updaterrrrrr r!r"pyiceberg.typedefr#r$pyiceberg.typesr%r&r'r(r)r*pyarrowparr+rr-r9rBrr:rPrrrBrLrr7r7r7r8s@    ( ( (  pm 2