o
    biM"                     @   s@  d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$ d dl%m&Z& e 'e(Z)de*de+fddZ,e- e)dfdee*ef de
e* de
e j. dee*ef fddZ/G dd deZ0dS )    N)Path)
ModuleType)AnyDictListOptional)try_to_create_directory)RuntimeEnvContext)Protocoldelete_packagedownload_and_unpack_packageget_local_dir_from_uriget_uri_for_directoryget_uri_for_fileget_uri_for_packageinstall_wheel_package
is_whl_uripackage_exists	parse_uriupload_package_if_neededupload_package_to_gcs)RuntimeEnvPlugin)set_pythonpath_in_context)get_directory_size_bytes)	GcsClient)RuntimeEnvSetupErrorsreturnc                 C   s\   zt | \}}W n ty   d\}}Y nw |t v r*|ds*|ds*td|d uS )N)NNz.zip.whlz2Only .zip or .whl files supported for remote URIs.)r   
ValueErrorr
   remote_protocolsendswith)r   protocolpath r$   W/home/ubuntu/.local/lib/python3.10/site-packages/ray/_private/runtime_env/py_modules.py_check_is_uri!   s   r&   runtime_envscratch_dirloggerc                 C   sD  |  d}|du r| S t|tstdt| dg }|D ]}t|tr(|}n2t|tr2t|}n(t|trPt|ds@|j	}nt
|jdkrKtd|j\}n
tdt| dt|ra|}nt| snt|jd	krt| }	|  d
d}
|	rt||
d}nt|}|du rzt||||
|	|d W n~ ty } zddlm} | rtd| d| |td| d| |d}~ww |||
d nLt|jdkrtt|}|du rt|szt|t|  W n( ty } ztd| d| |d}~ww n||ddd ntd| || q|| d< | S )zUploads the entries in py_modules and replaces them with a list of URIs.

    For each entry that is already a URI, this is a no-op.
    
py_modulesNzIpy_modules must be a List of local paths, imported modules, or URIs, got .__path__   zZpy_modules only supports modules whose __path__ has length 1 or those who are single-file.zHpy_modules must be a list of file paths, URIs, or imported modules, got z.pyexcludes)r.   )r.   include_parent_dirr)   r   )is_in_databricks_runtimezFailed to upload module z to the Ray cluster, please ensure there are only files under the module path, notebooks under the path are not allowed, original exception: z to the Ray cluster: r   zFailed to upload T)r.   is_filezFpy_modules entry must be a .py file, a directory, or a .whl file; got )get
isinstancelist	TypeErrortypestrr   r   hasattr__file__lenr,   r   r&   is_dirsuffixr   r   r   	Exceptionray.util.spark.utilsr0   r   r   r   r   
read_bytesappend)r'   r(   r)   	upload_fnr*   py_modules_urismodulemodule_path
module_urir;   r.   er0   r$   r$   r%   upload_py_modules_if_needed1   s   











rG   c                   @   s   e Zd ZdZdedefddZdefddZefded	e	e
j d
efddZd
ee fddZefdeddded	e	e
j d
ef
ddZefdee deded	e	e
j fddZdS )PyModulesPluginr*   resources_dir
gcs_clientc                 C   s$   t j|d| _|| _t| j d S )Npy_modules_files)osr#   join_resources_dir_gcs_clientr   )selfrI   rJ   r$   r$   r%   __init__   s   zPyModulesPlugin.__init__uric                 C   s   t || jS N)r   rN   )rP   rR   r$   r$   r%   _get_local_dir_from_uri   s   z'PyModulesPlugin._get_local_dir_from_urir)   r   c                 C   sJ   | d| t|| j}t|}t|| j}|s#|d| d dS |S )z2Delete URI and return the number of bytes deleted.z%Got request to delete pymodule URI %sz!Tried to delete nonexistent URI: r+   r   )infor   rN   r   r   warning)rP   rR   r)   	local_dirlocal_dir_sizedeletedr$   r$   r%   
delete_uri   s   zPyModulesPlugin.delete_uric                 C   s   |  S rS   )r*   )rP   r'   r$   r$   r%   get_uris   s   zPyModulesPlugin.get_urisr'   
RuntimeEnvcontextc                    sN   t || j| j|dI d H }t|r#|}| |}t|||dI d H  t|S )N)r)   )	wheel_uri
target_dirr)   )r   rN   rO   r   rT   r   r   )rP   rR   r'   r]   r)   
module_dirr^   r$   r$   r%   create   s   
zPyModulesPlugin.createurisruntime_env_dictc                 C   sZ   g }|D ]}|  |}| std| d| d|t| qttj|| d S )NzLocal directory z	 for URI z~ does not exist on the cluster. Something may have gone wrong while downloading, unpacking or installing the py_modules files.)	rT   existsr   r@   r7   r   rL   pathseprM   )rP   rb   rc   r]   r)   module_dirsrR   r`   r$   r$   r%   modify_context   s   
zPyModulesPlugin.modify_contextN)__name__
__module____qualname__namer7   r   rQ   rT   default_loggerr   loggingLoggerintrZ   r   r[   r	   ra   r   rg   r$   r$   r$   r%   rH      sF    

rH   )1rm   rL   pathlibr   typesr   typingr   r   r   r   ray._common.utilsr    ray._private.runtime_env.contextr	   "ray._private.runtime_env.packagingr
   r   r   r   r   r   r   r   r   r   r   r   r   ray._private.runtime_env.pluginr   $ray._private.runtime_env.working_dirr   ray._private.utilsr   ray._rayletr   ray.exceptionsr   	getLoggerrh   rl   r7   boolr&   getcwdrn   rG   rH   r$   r$   r$   r%   <module>   s8    <



t