o d-d)d*Z?dS).) annotationsN)copy) dataclass)Enum) TYPE_CHECKINGAny) ResolveErrorValidationError)literal)PartnerAccessorSchema SchemaVisitorSchemaWithPartnerVisitorassign_fresh_schema_idspromotevisitvisit_with_partner) NameMappingupdate_mapping)AddSchemaUpdateAssertCurrentSchemaIdSetCurrentSchemaUpdateSetPropertiesUpdateTableRequirement TableUpdateUpdatesAndRequirementsUpdateTableMetadata)L TableVersion) IcebergTypeListTypeMapType NestedField PrimitiveType StructType) Transactionc@seZdZdZdZdZdS)_MoveOperationN)__name__ __module__ __qualname__FirstBeforeAfterr1r1Z/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/pyiceberg/table/update/schema.pyr'?sr'c@s2eZdZUded<ded<ded<dZded <dS) _Moveintfield_idstr full_namer'opN int | Noneother_field_id)r+r,r-__annotations__r:r1r1r1r2r3Es r3csneZdZUded<ded<ded<iZded<iZd ed <eZd ed <iZd ed<iZ ded<iZ ded<ded<ded<    dhdifd#d$ Z djd%d&Z 'dkdld,d-Z   dmdnd7d8Zdod9d:Zdpd;d<Zdqd@dAZdodBdCZdrdEdFZdsdGdHZdtdJdKZ   dudvdNdOZdwdRdSZdxdVdWZdodXdYZdyd[d\Zdzd^d_Zd{dadbZd|dcddZd}dfdgZZS)~ UpdateSchemar _schemazitertools.count[int]_last_column_idzset[str]_identifier_field_nameszdict[int, list[NestedField]]_addsdict[int, NestedField]_updatesset[int]_deleteszdict[int, list[_Move]]_moveszdict[str, int]_added_name_to_idzdict[int, str] _id_to_parentbool_allow_incompatible_changes_case_sensitiveFTN transactionr%allow_incompatible_changescase_sensitiveschema Schema | None name_mappingNameMapping | NonereturnNonecst|t|tr|_td|j_nj j _tdj j j _|_ j_i_i_t_i_i_d fdd fdd jjD_|_|_|_ dS) Nr(r5r4rRr6cs(jj|d}|durtd||S)N) column_idzCould not find field-id: )r=find_column_name ValueError)r5 column_nameselfr1r2get_column_namexsz.UpdateSchema.__init__..get_column_namecsi|] \}}||qSr1r1).0r5parent_field_id)rZr1r2 ~sz)UpdateSchema.__init__..)r5r4rRr6)super__init__ isinstancer r= itertoolscounthighest_field_idr> _transactiontable_metadatarNlast_column_id _name_mappingidentifier_field_namesr?r@rBsetrDrErF_lazy_id_to_parentitemsrGrIrJ)rYrKrLrMrNrP __class__)rZrYr2r_]s(      zUpdateSchema.__init__cCs ||_|S)zDetermine if the case of schema needs to be considered when comparing column names. Args: case_sensitive: When false case is not considered in column name comparisons. Returns: This for method chaining )rJ)rYrMr1r1r2rMs zUpdateSchema.case_sensitiver) new_schemaSchema | pa.Schemaformat_versionrcCsBddlm}t|j||ddt||j|jdt|j|jd|S)Nr)Catalog)rpr& update_schemaexisting_schemarMpartner_schemarM)pyiceberg.catalogrqr_convert_schema_if_needed_UnionByNameVisitorr=rJPartnerIdByNameAccessor)rYrnrprqr1r1r2 union_by_names  zUpdateSchema.union_by_namepathstr | tuple[str, ...] field_typerdoc str | Nonerequired default_valueL | Nonec Cst|trd|vrtd|d|f}|d}|dd}d|}d|} t} t|dkr]|j| |j} | j } t| t rD| j } nt| t rL| j } | j jsZtd|d| | j} d} z |j||j} Wn tyrYnw| dur| j|jvrtd ||}t||j}|durz t||j}Wnty}ztd ||d}~ww|}|r|dur|jstd d|||j|<| |j|<t|||||||d }| |jvr|j| ||S|g|j| <|S) a&Add a new column to a nested struct or Add a new top-level column. Because "." may be interpreted as a column path separator or may be used in field names, it is not allowed to add nested column by passing in a string. To add to nested structures or to add fields with names that contain "." use a tuple instead to indicate the path. If type is a nested type, its field IDs are reassigned when added to the existing schema. Args: path: Name for the new column. field_type: Type for the new column. doc: Documentation string for the new column. required: Whether the new column is required. default_value: Default value for the new column. Returns: This for method chaining. .z'Cannot add column with ambiguous name: z, provide a tuple insteadr&NrzCannot add column 'z' to non-struct type: z(Cannot add column, name already exists: Invalid default value: z1Incompatible change: cannot add required column: r5namer~rrinitial_default write_default)r`r6rVjoin TABLE_ROOT_IDlenr= find_fieldrJr~r! value_fieldr element_field is_structr5rDassign_new_column_idrr tovaluerIrFrGr"r@append)rYr|r~rrrrparentr7parent_full_path parent_id parent_field parent_typeexisting_fieldnew_idnew_typerefieldr1r1r2 add_columnsn             zUpdateSchema.add_columncCsvt|tr|fn|}d|}|jj||jd}|j|jvr%td||j|j vr2td||j |j|S)zDelete a column from a table. Args: path: The path to the column. Returns: The UpdateSchema with the delete operation staged. rrMz+Cannot delete a column that has additions: z)Cannot delete a column that has updates: ) r`r6rr=rrJr5r@rVrBrDadd)rYr|rr7rr1r1r2 delete_columns   zUpdateSchema.delete_columncCs||||S)zSet the default value of a column. Args: path: The path to the column. Returns: The UpdateSchema with the delete operation staged. )_set_column_default_value)rYr|rr1r1r2set_default_values zUpdateSchema.set_default_value path_fromnew_namer6c Cst|tr d|n|}|j||j}|j|jvr!td||j |j}r@t |j||j |j |j|j|jd|j |j<nt |j||j |j |j|j|jd|j |j<|j|j}||jvr{|j||dt|j |}|j||S)zUpdate the name of a column. Args: path_from: The path to the column to be renamed. new_name: The new path of the column. Returns: The UpdateSchema with the rename operation staged. rz-Cannot rename a column that will be deleted: r5rr~rrrrN)r`tuplerr=rrJr5rDrVrBgetr"r~rrrrrUr?removerrr)rYrr field_fromupdatedfrom_field_correct_casingnew_identifier_pathr1r1r2 rename_column)s:      zUpdateSchema.rename_columncCs|j|dd|S)zMake a column optional. Args: path: The path to the field. Returns: The UpdateSchema with the requirement change staged. Fr)_set_column_requirement)rYr|r1r1r2make_column_optionalWs z!UpdateSchema.make_column_optionalfieldscGst||_dSN)rir?)rYrr1r1r2set_identifier_fieldscsz"UpdateSchema.set_identifier_fieldsc Cst|tr|fn|}d|}|j||j}|jr|s!|jr#|s#dS|js0|r0t d|d|j |j vr=t d||j |j }r]t|j |j|j|j||j|jd|j |j <dSt|j |j|j|j||j|jd|j |j <dS)Nrz"Cannot change column nullability: z: optional -> required-Cannot update a column that will be deleted: r)r`r6rr=rrJroptionalrIrVr5rDrBrr"rr~rrr)rYr|rrrrr1r1r2rfs8    z$UpdateSchema._set_column_requirementrc Cs.t|tr|fn|}d|}|j||j}|dur;z t||jj }Wnt y:}zt d||d}~ww|j rE||j krEdS|j sS|j rS|durSt d|j|jvr`t d||j|j}rt|j|j|j|j|j |j|d|j|j<dSt|j|j|j|j|j |j|d|j|j<dS)Nrrz?Cannot change change default-value of a required column to Nonerr)r`r6rr=rrJr rr~rrVrrrIr5rDrBrr"rrr)rYr|rrrrrr1r1r2rsF   z&UpdateSchema._set_column_default_valueIcebergType | None bool | Nonec Cst|tr|fn|}d|}|dur|dur|dur|S|j||j}|j|jvr2td||duro|j j sCt d|j d|j so|j |krozt |j |Wntyn}zt d|d|j d||d}~ww|j|j}rt|j|j|p|j |dur|n|j|j|j|jd|j|j<nt|j|j|p|j |dur|n|j|j|j|jd|j|j<|dur|j||d |S) a7Update the type of column. Args: path: The path to the field. field_type: The new type required: If the field should be required doc: Documentation describing the column Returns: The UpdateSchema with the type update staged. rNrzCannot change column type: z is not a primitivez: z -> rr)r`r6rr=rrJr5rDrVr~ is_primitiver rIrrrBrr"rrrrrr) rYr|r~rrr7rrrr1r1r2 update_columnsN    zUpdateSchema.update_columnrr9cCs4z |j||jjWStyYnw|j|Sr)r=rrJr5rVrFr)rYrr1r1r2_find_for_moves   zUpdateSchema._find_for_movemover3cCs&|j|j}r[|jj||jd}|jjstd|j|j t j ks*|j t j krJ|j dur3td|j|j|j|j krJtd|jd|j|jg|g|j|j<dS|j t j ksg|j t j kr|j durptd|j|j }rtd|jd||jtg|g|jt<dS)Nrz'Cannot move fields in non-struct type: z6Expected other field when performing before/after movezCannot move field z to a different structz to a different struct: )rGrr5r=rrJr~rrVr8r'r0r/r:r7rEr)rYr parent_namer other_structr1r1r2_moves  " zUpdateSchema._movecCsRt|tr d|n|}||}|durtd||t||tjd|S)zMove the field to the first position of the parent struct. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNCannot move missing column: )r5r7r8) r`rrrrVrr3r'r.)rYr|r7r5r1r1r2 move_firsts  zUpdateSchema.move_first before_pathcCt|tr d|n|}||}|durtd|t|tr&d|n|}||}|dur;td|d|||krGtd|d|t|||tjd|S)zMove the field to before another field. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNr Cannot move z before missing column: z before itselfr5r7r:r8) r`rrrrVrr3r'r/)rYr|rr7r5before_full_namebefore_field_idr1r1r2 move_before.s"  zUpdateSchema.move_before after_namecCr)zMove the field to after another field. Args: path: The path to the field. Returns: The UpdateSchema with the move operation staged. rNrrz after missing column: z after itselfr) r`rrrrVrr3r'r0)rYr|rr7r5 after_pathafter_field_idr1r1r2 move_afterPs  zUpdateSchema.move_afterrcsddlm}|tfdd|jjjDd}d}d}||jjkr`|t |jjdf7}|dur>|t dt d d f7}n|t |d f7}|j }r`t ||j|j}|t|j|id f7}||fS) z%Apply the pending changes and commit.r)TablePropertiesc3s|] }|kr|jVqdSr schema_idr[rNrnr1r2 tsz'UpdateSchema._commit..Nr1)current_schema_id)rNr&r)updates)pyiceberg.tabler_applynextrdreschemasr=rrrrrgrrBr@rDEFAULT_NAME_MAPPINGmodel_dump_json)rYrexisting_schema_id requirementsrrPupdated_name_mappingr1rr2_commitms*   zUpdateSchema._commitc Cs t|jt|j|j|j|j}|durtdt|j }ddl m }|j j jD]}|j D]}||j|j|j|tq,q't}|jD](}z |j||jd}Wntyc} z td|d| d} ~ ww||jqB|j } rd| j durtd d | j jDnd} nd} t|j | |d S) zApply the pending changes to the original schema and returns the result. Returns: the result Schema when all pending updates are applied NzCould not apply changesr)validate_partition_namerzCannot find identifier field z:. In case of deletion, update the identifier fields first.r(css|]}|jVqdSrrrr1r1r2rsz&UpdateSchema._apply..)ridentifier_field_ids)rr= _ApplyChangesr@rBrDrErVr rpyiceberg.partitioningrrdrepartition_specsr transform source_idrir?rrJrr5maxr) rYstructrnrspecpartition_field field_idsrrrtxnnext_schema_idr1r1r2rs<      "zUpdateSchema._applyr4cCs t|jSr)rr>rXr1r1r2rs z!UpdateSchema.assign_new_column_id)FTNN) rKr%rLrHrMrHrNrOrPrQrRrS)rMrHrRr<)r))rnrorprrRr<)NFN) r|r}r~rrrrrHrrrRr<)r|r}rRr<)r|r}rrrRr<)rr}rr6rRr<)rr6rRrS)r|r}rrHrRrS)r|r}rrrRrS)NNN) r|r}r~rrrrrrRr<)rr6rRr9)rr3rRrS)r|r}rr}rRr<)r|r}rr}rRr<)rRr)rRr )rRr4) r+r,r-r;r@rBrirDrErFrGr_rMr{rrrrrrrrrrrrrrrrr __classcell__r1r1rlr2r<MsV       )  b  .  %/ B    "  (r<c@steZdZUded<ded<ded<ded<d2ddZd3ddZd4ddZd5d d!Zd6d%d&Zd7d+d,Z d8d/d0Z d1S)9r%dict[int, builtins.list[NestedField]]r@rArBrCrDdict[int, builtins.list[_Move]]rEaddsrdeletesmovesrRrScCs||_||_||_||_dSr)r@rBrDrE)rYrrrrr1r1r2r_s z_ApplyChanges.__init__rNr struct_resultrcCsf|jt}|jt}|dus|dur1t|ts td|t|j|p&g|p)g}r1t|S|SNz!Cannot add fields to non-struct: ) r@rrrEr`r$rV_add_and_move_fieldsr)rYrNraddedr new_fieldsr1r1r2rNs   z_ApplyChanges.schemarr$ field_results!builtins.list[IcebergType | None]c Csd}g}t|D]f\}}||}|durd}q|j|}|j}|j} |j} |j} |j|j} r=| j}| j} | j} | j} |j|kr\|j |kr\|j| kr\|j| kr\|j| kr\| |qd}| t |j||| | |j | dq|rut |S|S)NFTr) enumeraterrrrrrBrr5r~rr"rr$) rYrr has_changesridx result_typerrrrrupdater1r1r2rsL        z_ApplyChanges.structrr" field_resultcCs|j|jvrdS|j|j}r|j|jkr|jSt|trS|j|j}|j|j}|dus5|durSt|jtsBt d|t |j |pHg|pKg}rSt|S|Sr) r5rDrBrr~r`r$r@rErVrr)rYrrrrrrr1r1r2r s   z_ApplyChanges.field list_typer element_resultcCsR||j|}|durtd||j}|j|j}r!|j}t|j||dS)Nz&Cannot delete element type from list: ) element_idelementelement_required) rrrVrrBrrrr )rYrr element_typerrr1r1r2list#sz_ApplyChanges.listmap_typer! key_result value_resultc Cs|jj}||jvrtd|||jvrtd|||jvr(td||j|kr4td||j}|||}|durHtd||j }|j |j }rW|j }t |j|j|j ||dS)NzCannot delete map keys: zCannot update map keys: zCannot add fields to map keys: zCannot alter map keys: z#Cannot delete value type from map: )key_idkey_typevalue_id value_typevalue_required) key_fieldr5rDrVrBr@r rrrrr rr!r ) rYrr r r rrrrr1r1r2map.s.     z_ApplyChanges.map primitiver#cCs|Srr1)rYrr1r1r2rNz_ApplyChanges.primitiveN) rrrrArrCrrrRrS)rNr rrrRr)rr$rrrRr)rr"rrrRr)rr rrrRr)rr!r rr rrRr)rr#rRr) r+r,r-r;r_rNrrrrrr1r1r1r2rs   5   rc@seZdZUded<ded<ded<d:d d Zd;ddZdddZd?d"d#Z d@d%d&Z dAd+d,Z dBd2d3Z dCd7d8Z d9S)Dryr<rsr rtrHrMrRrScCs||_||_||_dSrrr)rYrsrtrMr1r1r2r_Ws z_UnionByNameVisitor.__init__rN partner_idr9rcCs|Srr1)rYrNrrr1r1r2rN\rz_UnionByNameVisitor.schemarr$missing_positionsbuiltins.list[bool]c Cs|durdS|j}||}|jstd|t|D]$\}}|r+||||q||}|j|j|jd} r@| || qdS)NTzExpected a struct, got: rF) r_find_field_typerrVr _add_column field_by_namerrM_update_column) rYrrrrpartner_structposmissingr nested_fieldr1r1r2r_s  z_UnionByNameVisitor.structrr4rr"cCsB|j|}r||jf}n|jf}|jj||j|j|jddS)N)r|r~rr)rtrUrrsrr~rr)rYrrrr|r1r1r2rss z_UnionByNameVisitor._add_columnrcCs|j|j}|durtd||jr|jr|j||jj rD|j|jkrDz t |j|j}Wnt yC|jj ||jdYnw|j durZ|j |j kr\|jj ||j ddSdSdS)NzCould not find field: )r~)r)rtrUr5rVrrrsrr~rrrrr)rYrrr7_r1r1r2r{s   z"_UnionByNameVisitor._update_columnr5rcCs |dkr |jS|j|jSNr&)rt as_structrr~)rYr5r1r1r2rs z$_UnionByNameVisitor._find_field_typercC|duSrr1)rYrrrr1r1r2rz_UnionByNameVisitor.fieldrr list_partner_idelement_missingcCsN|durdS|r td||}t|tstd|||j|jdS)NTzAError traversing schemas: element is missing, but list is presentzExpected list-type, got: F)rVrr`r rr)rYrr$r%partner_list_typer1r1r2rs  z_UnionByNameVisitor.listrr!map_partner_id key_missing value_missingcCsj|durdS|r td|rtd||}t|ts#td|||j|j||j|jdS)NTzError traversing schemas: value is missing, but map is presentzExpected map-type, got: F)rVrr`r!rrr)rYrr'r(r)partner_map_typer1r1r2rs  z_UnionByNameVisitor.maprr#primitive_partner_idcCr"rr1)rYrr+r1r1r2rr#z_UnionByNameVisitor.primitiveN)rsr<rtr rMrHrRrS)rNr rr9rrHrRrH)rr$rr9rrrRrH)rr4rr"rRrS)rr"rr"rRrS)r5r4rRr)rr"rr9rrHrRrH)rr r$r9r%rHrRrH) rr!r'r9r(rHr)rHrRrH)rr#r+r9rRrH)r+r,r-r;r_rNrrrrrrrrr1r1r1r2ryRs          ryc@sZeZdZUded<ded<dddZdd d ZdddZd ddZd!ddZd!ddZ dS)"rzr rvrHrMrRrScCs||_||_dSrru)rYrvrMr1r1r2r_s z PartnerIdByNameAccessor.__init__partnerr9cCsdSr r1)rYr,r1r1r2schema_partnerrz&PartnerIdByNameAccessor.schema_partnerpartner_field_idr5r4 field_namer6cCs\|dur,|dkr|j}n|j|j}|jstd||j||jd}r,|jSdS)Nr&zExpected StructType: )rrM) rvr!rr~rrVrrMr5)rYr.r5r/rrr1r1r2 field_partners z%PartnerIdByNameAccessor.field_partnerpartner_list_idcC@|dur|j|}rt|jtstd||jjjSdS)NzExpected ListType: )rvrr`r~r rVrr5)rYr1rr1r1r2list_element_partner   z,PartnerIdByNameAccessor.list_element_partnerpartner_map_idcCr2NzExpected MapType: )rvrr`r~r!rVrr5rYr5rr1r1r2map_key_partnerr4z'PartnerIdByNameAccessor.map_key_partnercCr2r6)rvrr`r~r!rVrr5r7r1r1r2map_value_partnerr4z)PartnerIdByNameAccessor.map_value_partnerN)rvr rMrHrRrS)r,r9rRr9)r.r9r5r4r/r6rRr9)r1r9rRr9)r5r9rRr9) r+r,r-r;r_r-r0r3r8r9r1r1r1r2rzs      rzrtuple[NestedField, ...]rlist[NestedField] | NonerRcCs|pg}|t|Sr)r)rrr1r1r2 _add_fieldss r<r list[_Move]cstt|}|D][tfdd|D}fdd|D}jtjkr*|g|}qjtjks6jtjkr\jtfddt |D}jtjkrS| ||q| |d|qt djt |S)Nc3s |] }|jjkr|VqdSrr5r[rrr1r2rsz_move_fields..csg|] }|jjkr|qSr1r>r?r@r1r2 sz _move_fields..c3s"|] \}}|jkr|VqdSrr>)r[ir)r:r1r2rs r(zUnknown operation: ) rrrr8r'r.r/r0r:rinsertrVr)rr reorderedrother_field_posr1)rr:r2 _move_fieldss    rFlist[NestedField]tuple[NestedField, ...] | NonecCsht|dkrt||}t|dkrt||S|St|dkr#t||St|dkr+dStg||RS)Nr)rr<rFr)rrrrr1r1r2r s      "r)rr:rr;rRr:)rr:rr=rRr:)rr:rrGrr=rRrH)@ __future__rbuiltinsrar dataclassesrenumrtypingrrpyiceberg.exceptionsrr pyiceberg.expressionsr pyiceberg.schemar r r rrrrrpyiceberg.table.name_mappingrrpyiceberg.table.updaterrrrrrrrpyiceberg.typedefrrpyiceberg.typesrr r!r"r#r$pyarrowparr%rr'r3r<rr4rHryrzr<rFrr1r1r1r2sB     ( (  pm 2