o Gd,d-d-e!ee?e?fZ@Gd.d/d/e!ZAGd0d1d1e!ZBGd2d3d3e!ZCGd4d5d5ZDdSdTd:d;ZEdUdCdDZFdVdJdKZGdWdNdOZHdXdQdRZIdS)Y) annotationsN) defaultdict)IterableMapping)Enum) TYPE_CHECKINGAny)Field PrivateAttrmodel_serializer)FileIO)DataFileDataFileContent ManifestFile _manifests)UNPARTITIONED_PARTITION_SPEC PartitionSpec)Schema) TableMetadata)IcebergBaseModelzadded-data-fileszadded-delete-fileszadded-equality-deleteszadded-files-sizezadded-position-deleteszadded-position-delete-filesz added-recordszdeleted-data-fileszdeleted-recordszadded-equality-delete-fileszremoved-delete-fileszremoved-equality-deleteszremoved-equality-delete-fileszremoved-files-sizezremoved-position-deleteszremoved-position-delete-filesztotal-equality-deletesztotal-position-deletesztotal-data-filesztotal-delete-filesz total-recordsztotal-files-sizezchanged-partition-countz partitions.zpartition-summaries-included operationc@s*eZdZdZdZdZdZdZd dd Zd S) Operationa+Describes the operation. Possible operation values are: - append: Only data files were added and no files were removed. - replace: Data and delete files were added and removed without changing table data; i.e., compaction, changing the data file format, or relocating data files. - overwrite: Data and delete files were added and removed in a logical overwrite operation. - delete: Data files were removed and their contents logically deleted and/or delete files were added to delete rows. appendreplace overwritedeletereturnstrcCs d|jS)z8Return the string representation of the Operation class.z Operation.)nameselfr!V/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/pyiceberg/table/snapshots.py__repr__Ts zOperation.__repr__Nrr) __name__ __module__ __qualname____doc__APPENDREPLACE OVERWRITEDELETEr#r!r!r!r"rCs rc@seZdZUded<ded<ded<ded<ded<ded<ded<ded <ded <ded <ded <ded <ded<ded<ded<ded<d ddZd!ddZd!ddZd"ddZdS)# UpdateMetricsintadded_file_sizeremoved_file_sizeadded_data_filesremoved_data_filesadded_eq_delete_filesremoved_eq_delete_filesadded_pos_delete_filesremoved_pos_delete_filesadded_delete_filesremoved_delete_files added_recordsdeleted_recordsadded_pos_deletesremoved_pos_deletesadded_eq_deletesremoved_eq_deletesrNonecCsdd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_dSNr)r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>rr!r!r"__init__ks  zUpdateMetrics.__init__ data_filer cC|j|j7_|jtjkr|jd7_|j|j7_dS|jtjkr=|j d7_ |j d7_ |j |j7_ dS|jtj kr[|j d7_ |j d7_ |j|j7_dStd|jNzUnknown data file content: )r/file_size_in_bytescontentrDATAr1r9 record_countPOSITION_DELETESr7r5r;EQUALITY_DELETESr3r= ValueErrorr rBr!r!r"add_file}   zUpdateMetrics.add_filecCrCrD)r0rFrGrrHr2r:rIrJr8r6r<rKr4r>rLrMr!r!r" remove_filerOzUpdateMetrics.remove_filedict[str, str]cCsi}t||jtt||jtt||jtt||jtt||j t t||j t t||j tt||jtt||jtt||jtt||jtt||jtt||jtt||jtt||jtt||jt |SN)!set_when_positiver/ADDED_FILE_SIZEr0REMOVED_FILE_SIZEr1ADDED_DATA_FILESr2DELETED_DATA_FILESr3ADDED_EQUALITY_DELETE_FILESr4REMOVED_EQUALITY_DELETE_FILESr5ADDED_POSITION_DELETE_FILESr6REMOVED_POSITION_DELETE_FILESr7ADDED_DELETE_FILESr8REMOVED_DELETE_FILESr9 ADDED_RECORDSr:DELETED_RECORDSr;ADDED_POSITION_DELETESr<REMOVED_POSITION_DELETESr=ADDED_EQUALITY_DELETESr>REMOVED_EQUALITY_DELETES)r propertiesr!r!r"to_dicts$zUpdateMetrics.to_dictN)rr?)rBr rr?rrQ)r%r&r'__annotations__rArNrPrer!r!r!r"r-Ys*    r-cseZdZUdZeZded<eZded<d$d%fd d Z d&ddZ d'ddZ d(ddZ e d)ddZed)ddZd*ddZd+d"d#ZZS),SummaryuA class that stores the summary information for a Snapshot. The snapshot summary’s operation field is used by some operations, like snapshot expiration, to skip processing certain snapshots. rrrQ_additional_propertiesNOperation | Nonedatarrr?c s<|durtjdddtj}tjdd|i|||_dS)NzSEncountered invalid snapshot summary: operation is missing, defaulting to overwrite) stacklevelrr!)warningswarnrr+superrAri)r rrk __class__r!r"rAs  zSummary.__init__ _Summary__keyr Any | NonecCs|dkr |jS|j|S)zReturn a key as it is a map.r)lowerrriget)r rsr!r!r" __getitem__s  zSummary.__getitem__keyvaluecCs$|dkr ||_dS||j|<dS)zSet a key as it is a map.rN)rurri)r rxryr!r!r" __setitem__s  zSummary.__setitem__r.cCsdt|jS)z)Return the number of keys in the summary.rE)lenrirr!r!r"__len__szSummary.__len__cCsdt|jji|jS)Nr)rrryrirr!r!r" ser_models zSummary.ser_modelcCs|jSrR)rirr!r!r"additional_propertiesszSummary.additional_propertiescCs0|jr dt|jnd}dt|j|dS)z6Return the string representation of the Summary class.z, **zSummary())rireprr)r repr_propertiesr!r!r"r#szSummary.__repr__otherboolcCs&t|tr|j|jko|j|jkSdS)z3Compare if the summary is equal to another summary.F) isinstancerhrr~)r rr!r!r"__eq__s zSummary.__eq__rR)rrjrkrrr?)rsrrrt)rxrryrrr?)rr.rfr$)rrrr)r%r&r'r(r rrgr rirArwrzr|r r}propertyr~r#r __classcell__r!r!rqr"rhs       rhc@seZdZUeddZded<edddZded <ed edZded <ed d ddZ ded<edddZ ded<eddZ ded<edddZ ded<eddddZ ded<eddd dZded!<d,d#d$Zd,d%d&Zd-d*d+ZdS).Snapshot snapshot-idaliasr. snapshot_idzparent-snapshot-idN)rdefaultz int | Noneparent_snapshot_idzsequence-numbersequence_number timestamp-mscCsttdS)Ni)r.timer!r!r!r"szSnapshot.)rdefault_factory timestamp_msz manifest-listz-Location of the snapshot's manifest list file)r descriptionr manifest_list)rzSummary | Nonesummaryz schema-id schema_idz first-row-idzFassigned to the first row in the first data file in the first manifest)rrr first_row_idz added-rowsz;The upper bound of the number of rows with assigned row IDs added_rowsrcCsd|jr |jjdnd}|jrd|jnd}|jdur"d|jnd}|d|j||}|S)7Return the string representation of the Snapshot class.z: rz , parent_id=Nz , schema_id=zid=)rrrrr)r r parent_idr result_strr!r!r"__str__s zSnapshot.__str__c Csd|jd|jd|jd|jd|jd|jr%dt|jnd|jdur1d |jnd|jdur=d |jnd|j durId |j ndg }d d |D}dd |dS)rz snapshot_id=zparent_snapshot_id=zsequence_number=z timestamp_ms=zmanifest_list=''zsummary=Nz schema_id=z first_row_id=z added_rows=cSsg|]}|dur|qSrRr!).0fieldr!r!r" sz%Snapshot.__repr__..z Snapshot(z, r) rrrrrrrrrrjoin)r fieldsfiltered_fieldsr!r!r"r#s      zSnapshot.__repr__ior list[ManifestFile]cCstt||jS)z,Return the manifests for the given snapshot.)listrr)r rr!r!r" manifestsszSnapshot.manifestsr$)rr rr)r%r&r'r rrgrINITIAL_SEQUENCE_NUMBERrrrrrrrrr#rr!r!r!r"rs"   rc@s2eZdZUeddZded<eddZded<dS) MetadataLogEntryz metadata-filerr metadata_filerr.rN)r%r&r'r rrgrr!r!r!r"r rc@s2eZdZUeddZded<eddZded<dS)SnapshotLogEntryrrr.rrrN)r%r&r'r rrgrr!r!r!r"r"rrc@sveZdZUded<ded<ded<d'd(d d Zd)ddZefd*ddZefd*ddZd+ddZ d,d d!Z d-d$d%Z d&S).SnapshotSummaryCollectorr-metricszdefaultdict[str, UpdateMetrics]partition_metricsr.$max_changed_partitions_for_summariesrpartition_summary_limitrr?cCst|_tt|_||_dSrR)r-rrrr)r rr!r!r"rA,s  z!SnapshotSummaryCollector.__init__limitcCs ||_dSrR)r)r rr!r!r"set_partition_summary_limit1s z4SnapshotSummaryCollector.set_partition_summary_limitrBr schemarpartition_specrcC4|j|t|jdkr|j||d|ddSdS)NrTrfile is_add_filer)rrNr{ partitionupdate_partition_metricsr rBrrr!r!r"rN4s z!SnapshotSummaryCollector.add_filecCr)NrFr)rrPr{rrrr!r!r"rP9s z$SnapshotSummaryCollector.remove_filerrrcCs8||j|}|j|}|r||dS||dSrR)partition_to_pathrrrNrP)r rrrrpartition_pathrr!r!r"r@s  z1SnapshotSummaryCollector.update_partition_metricsrQcCsx|j}t|j}t||t||jkr:|dkrd|t<|jD]\}}| |}r9t|dkr9||t |<q"|S)Nrtrue) rrer{rrSCHANGED_PARTITION_COUNT_PROPrPARTITION_SUMMARY_PROPitems_partition_summaryCHANGED_PARTITION_PREFIX)r rdchanged_partitions_sizerupdate_metrics_partitionrr!r!r"buildIs     zSnapshotSummaryCollector.buildupdate_metricsrcCsddd|DS)N,cSsg|] \}}|d|qS)=r!)rpropvalr!r!r"rWsz?SnapshotSummaryCollector._partition_summary..)rrer)r rr!r!r"rVsz+SnapshotSummaryCollector._partition_summaryN)r)rr.rr?)rr.rr?)rBr rrrrrr?) rrrr rrrrrr?rf)rr-rr) r%r&r'rgrArrrNrPrrrr!r!r!r"r's      rrprevious_summaryMapping[str, str] | Nonerc sjtjtjtjhvrtdjs#tdtdtdt dt dt did fd d }|tt t d |tttd |tttd |t ttd |t ttd |t ttd S) NzOperation not implemented: 0total_propertyradded_propertyremoved_propertyrr?c s|}rSz(t|}|dkr|}r|t|7}|dkr.|}r.|t|8}WntyF}z td|d||d}~ww|dkrUt||<dSdSdS)Nrz!Could not parse summary property z to an int: )rvr.rLr)rrrprevious_total_str new_totaladdedremovederrr!r"_update_totalshs    z1update_snapshot_summaries.._update_totals)rrr)rrrrrrrr?)rrr)r+r,rLTOTAL_DATA_FILESTOTAL_DELETE_FILES TOTAL_RECORDSTOTAL_FILE_SIZETOTAL_POSITION_DELETESTOTAL_EQUALITY_DELETESrVrWr\r]r^r_rTrUr`rarbrc)rrrr!rr"update_snapshot_summariesZsT rrdrQnumr. property_namerr?cCs|dkr t|||<dSdSr@)r)rdrrr!r!r"rSsrScurrent_snapshotSnapshot | Nonetable_metadatarIterable[Snapshot]ccs>|}|dur|V|jdurdS||j}|dusdSdS)z6Get the ancestors of and including the given snapshot.N)rsnapshot_by_id)rrsnapshotr!r!r" ancestors_ofs  r from_snapshot to_snapshotccsF|durt||D] }|V||krdSq dSt||EdHdS)zXGet the ancestors of and including the given snapshot between the to and from snapshots.N)r)rrrrr!r!r"ancestors_betweensrrcCsDd}d}t||D]}||jkr|krnq |}|j}q |S)aCFind the latest ancestor snapshot whose timestamp is before the provided timestamp. Args: table_metadata: The table metadata for a table timestamp_ms: lookup snapshots strictly before this timestamp Returns: The latest ancestor snapshot older than the timestamp, or None if not found. Nr)rrr)rrresultresult_timestampancestorr!r!r" latest_ancestor_before_timestamps rrR)rrhrrrrh)rdrQrr.rrrr?)rrrrrr)rrrrrrrr)rrrr.rr)J __future__rrrn collectionsrcollections.abcrrenumrtypingrrpydanticr r r pyiceberg.ior pyiceberg.manifestr rrrpyiceberg.partitioningrrpyiceberg.schemarpyiceberg.table.metadatarpyiceberg.typedefrrVr\rbrTr`rZr^rWr_rXr]rcrYrUrar[rrrrrrrrr OPERATIONrrr-rrhrrrrrrSrrrr!r!r!r"sl       [=, 3 >