o ;i5@sdddlZddlZddlZddlmZddlmZmZddlm Z m Z m Z m Z m Z ddlmZddlZddlmZddlmZdd lmZmZmZmZmZdd lmZdd lmZmZm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2dZ3de4e5e6dfde4ej7fddZ8GdddeddZ9e"e9Z:dS)N) AsyncIterator)Path PurePosixPath)AnyBinaryIOCallableOptionalUnion)asynccontextmanager)api_pb2) LoadContext) EPHEMERAL_OBJECT_HEARTBEAT_SLEEP_get_environment_name_Object live_methodlive_method_gen)Resolver) TaskContextaclosing async_mapsync_or_async_itersynchronize_api)LARGE_FILE_LIMIT blob_iterblob_upload_file)warn_if_passing_namespace)get_sha256_hex)check_object_name)_Client) InvalidError) FileEntryiXvalidated_network_file_systems_NetworkFileSystemreturncCs,g}|D]\}}|tj||jdq|S)N) mount_pathshared_volume_id)appendr SharedVolumeMount object_id)r"network_file_system_mountspathvolumer-M/home/ubuntu/.local/lib/python3.10/site-packages/modal/network_file_system.py network_file_system_mount_protos%s r/c@seZdZdZeddddddedeededeed df d d Z e e dde fd e ddeedeed ed edf ddZe   d)dedeedeed efddZed*dedeedeefddZed+dededeedefd efddZeded eefddZeded eefddZe  d*d eeefdeeee dfdeedeffd!d"Z!e  d*d eeefdeeee dfdeedeffd#d$Z"eded e#efd%d&Z$ed,defd'd(Z%dS)-r#aA shared, writable file system accessible by one or more Modal functions. By attaching this file system as a mount to one or more functions, they can share and persist data with each other. **Note: `NetworkFileSystem` has been deprecated and will be removed.** **Usage** ```python import modal nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True) app = modal.App() @app.function(network_file_systems={"/root/foo": nfs}) def f(): pass @app.function(network_file_systems={"/root/goo": nfs}) def g(): pass ``` Also see the CLI methods for accessing network file systems: ``` modal nfs --help ``` A `NetworkFileSystem` can also be useful for some local scripting scenarios, e.g.: ```python notest nfs = modal.NetworkFileSystem.from_name("my-network-file-system") for chunk in nfs.read_file("my_db_dump.csv"): ... ``` NF) namespaceenvironment_namecreate_if_missingclientnamer1r2r3r$c sRtdt|ddtdtdtdttffdd }tj|d d t||d d S) aReference a NetworkFileSystem by its name, creating if necessary. This is a lazy method that defers hydrating the local object with metadata from Modal servers until the first time it is actually used. ```python notest nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True) @app.function(network_file_systems={"/data": nfs}) def f(): pass ``` NetworkFileSystemz!modal.NetworkFileSystem.from_nameselfresolver load_contextexisting_object_idc stj|jr tjndd}z|jj|IdH}||j|jdWdSt j j yE}z|j ddkr@t dddd}~ww)Ndeployment_namer1object_creation_typerzApp has wrong entity vozAttempted to mount: `z` as a NetworkFileSystem z which already exists as a Volume)r SharedVolumeGetOrCreateRequestr1&OBJECT_CREATION_TYPE_CREATE_IF_MISSINGr3stubSharedVolumeGetOrCreate_hydrater&modal exception NotFoundErrorargsr )r6r7r8r9reqresponseexcr2r4r-r._loadvs" z+_NetworkFileSystem.from_name.._loadzNetworkFileSystem()T)r1r3)hydrate_lazilyload_context_overrides)rrr#rr rstr _from_loader)r4r0r1r2r3rJr-rIr. from_name\s"   z_NetworkFileSystem.from_namecls_heartbeat_sleepc sdur tIdHtjtjt|djIdH}t4IdH+}tj |j d|j fdd|d|j |j dddd VWdIdHdS1IdHsWwYdS) asCreates a new ephemeral network filesystem within a context manager: Usage: ```python with modal.NetworkFileSystem.ephemeral() as nfs: assert nfs.listdir("/") == [] ``` ```python notest async with modal.NetworkFileSystem.ephemeral() as nfs: assert await nfs.listdir("/") == [] ``` N)r<r1r&cs jSN)r?SharedVolumeHeartbeatr-r3requestr-r.s z._NetworkFileSystem.ephemeral..)sleepTz#modal.NetworkFileSystem.ephemeral())is_another_apprep) rfrom_envr r=OBJECT_CREATION_TYPE_EPHEMERALrr?r@rSharedVolumeHeartbeatRequestr& infinite_loop _new_hydrated)rPr3r1rQrGtcr-rUr. ephemerals& .z_NetworkFileSystem.ephemeralr;csZt|dt|d|durtIdH}tj|t|tjd}|j |IdH}|j S)z mdmd:hiddenr5z'modal.NetworkFileSystem.create_deployedNr:) rrrr[r r=r*OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTSr?r@r&)r;r0r3r1rVrespr-r-r.create_deployeds  z"_NetworkFileSystem.create_deployedcsBtj||d|IdH}tj|jd}|jj|IdHdS)N)r1rR) r#rOhydrater SharedVolumeDeleteRequestr)_clientr?SharedVolumeDelete)r4r3r1objrFr-r-r.deletesz_NetworkFileSystem.delete remote_pathfp progress_cb.c s|pdd}t|}|dtj|}|d|tkrB|||d}t||jjt |||dIdH}t j |j |||dd}n|} t j |j || dd }t} t| tkrt|jj|IdH} | jrl |St| tks\tjd |d ) a)Write from a file object to a path on the network file system, atomically. Will create any needed parent directories automatically. If remote_path ends with `/` it's assumed to be a directory and the file will be uploaded with its current name to that directory. c_sdSrSr-)___r-r-r.rWsz/_NetworkFileSystem.write_file..r)r4size)progress_report_cb sha256_hexNT)r&r+ data_blob_idrr resumable)r&r+datartz Uploading of z timed out)rseekosSEEK_ENDtellrrrgr? functoolspartialr SharedVolumePutFileRequestr)readtime monotonic+NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUTSharedVolumePutFileexistsrBrC TimeoutError) r6rkrlrmsha_hash data_sizeprogress_task_idblob_idrFrut0rGr-r-r. write_filesB      z_NetworkFileSystem.write_filer+c Cstj|j|d}z |jj|IdH}Wntjjy*}zt |j dd}~ww| ddkr8|j VdSt |j|jj2z 3dHW}|Vq@6dS)z(Read a file from the network file systemr&r+Nr data_oneofru)r SharedVolumeGetFileRequestr)rgr?SharedVolumeGetFilerBrCrDFileNotFoundErrorrE WhichOneofrurrs)r6r+rFrGrHrur-r-r. read_files z_NetworkFileSystem.read_filecCsNtj|j|d}|jjj|2z3dHW}|jD]}t |Vqq6dS)aIterate over all files in a directory in the network file system. * Passing a directory path lists all files in the directory (names are relative to the directory) * Passing a file path returns a list containing only that file's listing description * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths) rN) r SharedVolumeListFilesRequestr)rgr?SharedVolumeListFilesStream unary_streamentriesr! _from_proto)r6r+rFbatchentryr-r-r.iterdir s  z_NetworkFileSystem.iterdir local_pathcstt|}|durtd|j}nt|}|d}|j|||dIdHWdS1s3wYdS)N/rb)rm)rrr4as_posixopenr)r6rrkrm local_filer-r-r.add_local_files  $z!_NetworkFileSystem.add_local_filec st|durtdjntsJfdd}dtttfdtffdd }ttt ||dd 4IdH}|2z3dHW}qH6WdIdHdS1IdHsawYdS) Nrc3s>dD]}|r q|}|t|fVqdS)N*)rglobis_dir relative_torr)subpath relpath_str) _local_pathrkr-r. gen_transfers:sz7_NetworkFileSystem.add_local_dir..gen_transferspathsr$cs|d|dIdHS)Nrr )r)r)rmr6r-r._add_local_fileAsz9_NetworkFileSystem.add_local_dir.._add_local_file) concurrency) rrr4rrtupleintrrr)r6rrkrmrrstreamrnr-)rrmrkr6r. add_local_dir+s   ".z _NetworkFileSystem.add_local_dircsdd||2IdHS)aList all files in a directory in the network file system. * Passing a directory path lists all files in the directory (names are relative to the directory) * Passing a file path returns a list containing only that file's listing description * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths) csg|z3dHW}|q6SrSr-).0rr-r-r. Qsz._NetworkFileSystem.listdir..N)r)r6r+r-r-r.listdirHs z_NetworkFileSystem.listdirc sZtj|j||d}z |jj|IdHWdStjjy,}zt |j dd}~ww)z'Remove a file in a network file system.)r&r+ recursiveNr) r SharedVolumeRemoveFileRequestr)rgr?SharedVolumeRemoveFilerBrCrDrrE)r6r+rrFrHr-r-r. remove_fileSsz_NetworkFileSystem.remove_file)NNN)NNrS)F)&__name__ __module__ __qualname____doc__ staticmethodrMrboolrrO classmethodr rtypefloatrrardrjrrrrrrrbytesrr!rr rrrrlistrrr-r-r-r.r#4s' 2% "*-      sv) type_prefix);rzrwr~collections.abcrpathlibrrtypingrrrrr synchronicity.async_wrapr rB modal_protor _load_contextr _objectrrrrr _resolverr_utils.async_utilsrrrrr_utils.blob_utilsrrr_utils.deprecationr_utils.hash_utilsr_utils.name_utilsrr3rrCr r,r!rrrrMr(r/r#r5r-r-r-r.s>             +