o uyi3@sbUdZddlmZddlZddlZddlZddlmZmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZddlmZdd lmZmZeeZd Zd Zd Zd Z dZ!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2d Z3d!Z4d"Z5d#Z6d$Z7d%Z8d&Z9d'Z:d(Z;d)Zd,Z?d-Z@d.ZAd/ZBd0ZCd1ZDd2ZEd3ZFd4ZGd5ZHd6ZId7ZJd8ZKd9ZLd:ZMd;ZNdZQd?ZRd@ZSdAZTeGdBdCdCeZUeGdDdEdEeZVGdFdGdGeZWGdHdIdIeZXGdJdKdKeZYdLZZdMZ[dNZ\dOZ]e\e]ge\e]ge\e]ge\ge\ge\e]ge\ge\ge]e\ge]e\ge]e\ge]e\ge]gdP Z^dQe_dR<dbdYdZZ`d[Zadcd]d^Zbedfddd`daZcdS)eaBase FileIO classes for implementing reading and writing table files. The FileIO abstraction includes a subset of full filesystem implementations. Specifically, Iceberg needs to read or write a file at a given location (as a seekable stream), as well as check if a file exists. An implementation of the FileIO abstract base class is responsible for returning an InputFile instance, an OutputFile instance, and deleting a file given its location. ) annotationsN)ABCabstractmethod)SEEK_SET) TracebackType)DictListOptionalProtocolTypeUnionruntime_checkable)urlparse) EMPTY_DICT Propertiesz client.regionzclient.access-key-idzclient.secret-access-keyzclient.session-tokenzclient.role-arnzclient.role-session-namez s3.anonymousz s3.endpointzs3.access-key-idzs3.secret-access-keyzs3.session-tokenz s3.regionzs3.resolve-regionz s3.proxy-urizs3.connect-timeoutzs3.request-timeoutz s3.signerz s3.signer.urizs3.signer.endpointzv1/aws/s3/signz s3.role-arnzs3.role-session-namezs3.force-virtual-addressingzs3.retry-strategy-implz hdfs.hostz hdfs.portz hdfs.userzhdfs.kerberos_ticketzadls.connection-stringzadls.credentialzadls.account-namezadls.account-keyzadls.sas-tokenzadls.tenant-idzadls.client-idzadls.client-secretzadls.account-hostzadls.blob-storage-authorityzadls.dfs-storage-authorityzadls.blob-storage-schemezadls.dfs-storage-schemez adls.tokenzgcs.oauth2.tokenzgcs.oauth2.token-expires-atzgcs.project-idz gcs.accesszgcs.consistencyzgcs.cache-timeoutzgcs.requester-payszgcs.session-kwargszgcs.service.hostzgcs.default-bucket-locationzgcs.version-awarez hf.endpointzhf.tokenzpyarrow.use-large-types-on-readc@sfeZdZdZeddddZeefdd d Zed d dZed!ddZ d"ddZ ed#ddZ dS)$ InputStreama A protocol for the file-like object returned by InputFile.open(...). This outlines the minimally required methods for a seekable input stream returned from an InputFile implementation's `open(...)` method. These methods are a subset of IOBase/RawIOBase. rsizeintreturnbytescCdSN)selfrrrX/home/ubuntu/maya3_transcribe/venv/lib/python3.10/site-packages/pyiceberg/io/__init__.pyreadszInputStream.readoffsetwhencecCrrr)rrrrrrseekvrzInputStream.seekcCrrrrrrrtellyrzInputStream.tellNonecCrrrr rrrclose|rzInputStream.closecCdS)zCProvide setup when opening an InputStream using a 'with' statement.Nrr rrr __enter__zInputStream.__enter__exctypeOptional[Type[BaseException]]excinstOptional[BaseException]exctbOptional[TracebackType]cCr$z=Perform cleanup when exiting the scope of a 'with' statement.Nrrr'r)r+rrr__exit__r&zInputStream.__exit__N)r)rrrr)rrrrrrrrrr")rrr'r(r)r*r+r,rr") __name__ __module__ __qualname____doc__rrrrr!r#r%r/rrrrrks   rc@sHeZdZdZedddZedd d Zedd d ZedddZdS) OutputStreamaA protocol for the file-like object returned by OutputFile.create(...). This outlines the minimally required methods for a writable output stream returned from an OutputFile implementation's `create(...)` method. These methods are a subset of IOBase/RawIOBase. brrrcCrrr)rr8rrrwriterzOutputStream.writer"cCrrrr rrrr#rzOutputStream.closecCr$)zDProvide setup when opening an OutputStream using a 'with' statement.Nrr rrrr%r&zOutputStream.__enter__r'r(r)r*r+r,cCr$r-rr.rrrr/r&zOutputStream.__exit__N)r8rrrr1)rr7r2) r3r4r5r6rr9r#r%r/rrrrr7s   r7c@sTeZdZdZdddZeddd Zedd d Zedd dZ eddddZ dS) InputFileaA base class for InputFile implementations. Args: location (str): A URI or a path to a local file. Attributes: location (str): The URI or path to a local file for an InputFile instance. exists (bool): Whether the file exists or not. locationstrcC ||_dSr _locationrr;rrr__init__ zInputFile.__init__rrcCr$z.Return the total length of the file, in bytes.Nrr rrr__len__r&zInputFile.__len__cC|jS)z/The fully-qualified location of the input file.r>r rrrr;zInputFile.locationboolcCr$zCheck whether the location exists. Raises: PermissionError: If the file at self.location cannot be accessed due to a permission error. Nrr rrrexistsr&zInputFile.existsTseekablercCr$)aReturn an object that matches the InputStream protocol. Args: seekable: If the stream should support seek, or if it is consumed sequential. Returns: InputStream: An object that matches the InputStream protocol. Raises: PermissionError: If the file at self.location cannot be accessed due to a permission error. FileNotFoundError: If the file at self.location does not exist. Nr)rrJrrropenr&zInputFile.openNr;r<r0rr<rrG)T)rJrGrr) r3r4r5r6rArrDpropertyr;rIrKrrrrr:s    r:c@sbeZdZdZdddZeddd Zedd d Zedd dZ edddZ eddddZ dS) OutputFileaA base class for OutputFile implementations. Args: location (str): A URI or a path to a local file. Attributes: location (str): The URI or path to a local file for an OutputFile instance. exists (bool): Whether the file exists or not. r;r<cCr=rr>r@rrrrArBzOutputFile.__init__rrcCr$rCrr rrrrDr&zOutputFile.__len__cCrE)z0The fully-qualified location of the output file.r>r rrrr;rFzOutputFile.locationrGcCr$rHrr rrrrIr&zOutputFile.existsr:cCr$)z9Return an InputFile for the location of this output file.Nrr rrr to_input_filer&zOutputFile.to_input_fileF overwriter7cCr$)a Return an object that matches the OutputStream protocol. Args: overwrite (bool): If the file already exists at `self.location` and `overwrite` is False a FileExistsError should be raised. Returns: OutputStream: An object that matches the OutputStream protocol. Raises: PermissionError: If the file at self.location cannot be accessed due to a permission error. FileExistsError: If the file at self.location already exists and `overwrite=False`. Nr)rrRrrrcreater&zOutputFile.createNrLr0rMrN)rr:)F)rRrGrr7) r3r4r5r6rArrDrOr;rIrQrSrrrrrPs     rPc@sReZdZUdZded<efdddZedd d Zedd dZ edddZ dS)FileIOz(A base class for FileIO implementations.r propertiescCr=r)rU)rrUrrrrA rBzFileIO.__init__r;r<rr:cCr$)zGet an InputFile instance to read bytes from the file at the given location. Args: location (str): A URI or a path to a local file. Nrr@rrr new_input r&zFileIO.new_inputrPcCr$)zGet an OutputFile instance to write bytes to the file at the given location. Args: location (str): A URI or a path to a local file. Nrr@rrr new_outputr&zFileIO.new_output!Union[str, InputFile, OutputFile]r"cCr$)aDelete the file at the given path. Args: location (Union[str, InputFile, OutputFile]): A URI or a path to a local file--if an InputFile instance or an OutputFile instance is provided, the location attribute for that instance is used as the URI to delete. Raises: PermissionError: If the file at location cannot be accessed due to a permission error. FileNotFoundError: When the file at the provided location does not exist. Nrr@rrrdeleter&z FileIO.deleteN)rUr)r;r<rr:)r;r<rrP)r;rXrr") r3r4r5r6__annotations__rrArrVrWrYrrrrrTs   rTr; warehousez"pyiceberg.io.pyarrow.PyArrowFileIOz pyiceberg.io.fsspec.FsspecFileIO) s3s3as3nossgsfilehdfsviewfsabfsabfsswasbwasbshfzDict[str, List[str]]SCHEMA_TO_FILE_IOio_implr<rUrrOptional[FileIO]c Csz/|d}t|dkrtd|d|dd|d}}t|}t||}||WStyL}ztj d||dWYd}~dSd}~ww)N.z;py-io-impl should be full path (module.CustomFileIO), got: Could not initialize FileIO: )exc_info) splitlen ValueErrorjoin importlib import_modulegetattrModuleNotFoundErrorloggerwarning)rjrU path_parts module_name class_namemoduleclass_excrrr_import_file_ioDs     rz py-io-implpathcCsXt|}|jr*t|j}r!|D] }t||}r|SqdStd|jdS)Nz-No preferred file implementation for scheme: )rschemerigetrwarningswarn)rrU parsed_urlfile_ios file_io_pathfile_iorrr_infer_file_io_from_schemeUsr Optional[str]c Cs|t}rt||}rtd||Std||r(t||}r(|S|t}r8t||}r8|Sztdddlm }||WSt yY}zt d|d}~ww)NzLoaded FileIO: %srozDefaulting to PyArrow FileIOr) PyArrowFileIOzCould not load a FileIO, please consider installing one: pip3 install "pyiceberg[pyarrow]", for more options refer to the docs.) r PY_IO_IMPLrryinforsr WAREHOUSEpyiceberg.io.pyarrowrrx)rUr;rjrwarehouse_locationrerrr load_file_ioas.    r)rjr<rUrrrk)rr<rUrrrk)rUrr;rrrT)dr6 __future__rruloggingrabcrriortypesrtypingrrr r r r r urllib.parserpyiceberg.typedefrr getLoggerr3ry AWS_REGIONAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYAWS_SESSION_TOKEN AWS_ROLE_ARNAWS_ROLE_SESSION_NAME S3_ANONYMOUS S3_ENDPOINTS3_ACCESS_KEY_IDS3_SECRET_ACCESS_KEYS3_SESSION_TOKEN S3_REGIONS3_RESOLVE_REGION S3_PROXY_URIS3_CONNECT_TIMEOUTS3_REQUEST_TIMEOUT S3_SIGNER S3_SIGNER_URIS3_SIGNER_ENDPOINTS3_SIGNER_ENDPOINT_DEFAULT S3_ROLE_ARNS3_ROLE_SESSION_NAMES3_FORCE_VIRTUAL_ADDRESSINGS3_RETRY_STRATEGY_IMPL HDFS_HOST HDFS_PORT HDFS_USERHDFS_KERB_TICKETADLS_CONNECTION_STRINGADLS_CREDENTIALADLS_ACCOUNT_NAMEADLS_ACCOUNT_KEYADLS_SAS_TOKENADLS_TENANT_IDADLS_CLIENT_IDADLS_CLIENT_SECRETADLS_ACCOUNT_HOSTADLS_BLOB_STORAGE_AUTHORITYADLS_DFS_STORAGE_AUTHORITYADLS_BLOB_STORAGE_SCHEMEADLS_DFS_STORAGE_SCHEME ADLS_TOKEN GCS_TOKENGCS_TOKEN_EXPIRES_AT_MSGCS_PROJECT_ID GCS_ACCESSGCS_CONSISTENCYGCS_CACHE_TIMEOUTGCS_REQUESTER_PAYSGCS_SESSION_KWARGSGCS_SERVICE_HOSTGCS_DEFAULT_LOCATIONGCS_VERSION_AWARE HF_ENDPOINTHF_TOKENPYARROW_USE_LARGE_TYPES_ON_READrr7r:rPrTLOCATIONr ARROW_FILE_IOFSSPEC_FILE_IOrirZrrrrrrrrs   $  /4&