o
    z“©iŠ  ã                   @   s®   d dl Z d dlZd dlmZmZmZmZmZ d dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ erDd dlmZmZ G d	d
„ d
eƒZdeddfdd„ZdS )é    N)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚUnion)Úoverride)Ú_XLA_AVAILABLE)Ú	_Launcher)Ú_GlobalStateSnapshot)Úmove_data_to_device)ÚXLAFSDPStrategyÚXLAStrategyc                   @   s”   e Zd ZdZded ddfdd„Zeedefdd	„ƒƒZ	ed
e
dededefdd„ƒZ	dded
e
dededeejejf dee ddfdd„ZdS )Ú_XLALaunchera€  Launches processes that run a given function in parallel on XLA supported hardware, and joins them all at the
    end.

    The main process in which this launcher is invoked creates N so-called worker processes (using the
    `torch_xla` :func:`xmp.spawn`) that run the given function.
    Worker processes have a rank that ranges from 0 to N - 1.

    Note:
        - This launcher requires all objects to be pickleable.
        - It is important that the entry point to the program/script is guarded by ``if __name__ == "__main__"``.

    Args:
        strategy: A reference to the strategy that is used together with this launcher

    Ústrategy)r   r   ÚreturnNc                 C   s    t sttt ƒƒ‚|| _d| _d S )NÚfork)r   ÚModuleNotFoundErrorÚstrÚ	_strategyÚ_start_method)Úselfr   © r   ú]/home/ubuntu/.local/lib/python3.10/site-packages/lightning_fabric/strategies/launchers/xla.pyÚ__init__/   s   
z_XLALauncher.__init__c                 C   s   dS )NTr   )r   r   r   r   Úis_interactive_compatible5   s   z&_XLALauncher.is_interactive_compatibleÚfunctionÚargsÚkwargsc                 O   sf   t  ¡  ¡ }ddlm  m} i }| jj}|dkr||d< |j| j	f||||f| j
dœ|¤Ž | ¡ S )a  Launches processes that run the given function in parallel.

        The function is allowed to have a return value. However, when all processes join, only the return value
        of worker process 0 gets returned from this `launch` method in the main process.

        Arguments:
            function: The entry point for all launched processes.
            *args: Optional positional arguments to be passed to the given function.
            **kwargs: Optional keyword arguments to be passed to the given function.

        r   Né   Únprocs)r   Ústart_method)ÚmpÚManagerÚQueueÚ)torch_xla.distributed.xla_multiprocessingÚdistributedÚxla_multiprocessingr   Únum_processesÚspawnÚ_wrapping_functionr   Úget)r   r   r   r   Úreturn_queueÚxmpÚspawn_kwargsr   r   r   r   Úlaunch:   s   ÿ
ýüz_XLALauncher.launchÚprocess_idxr+   Úglobal_statesc           
      C   sz   dd l m  m} t| ¡ ƒdkr dd l}| |||f¡\}}}||i |¤Ž}	| jjdkr5| 	t
|	dƒ¡ t| jjƒ d S )Nr   r   Úcpu)Útorch_xla.core.xla_modelÚcoreÚ	xla_modelÚlenÚget_xla_supported_devicesÚcopyÚdeepcopyr   Ú
local_rankÚputr   Ú_rank_teardown)
r   r/   r   r   r   r+   r0   Úxmr7   Úresultsr   r   r   r)   [   s   z_XLALauncher._wrapping_function)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Úpropertyr   Úboolr   r   r   r.   Úintr!   ÚSimpleQueueÚqueuer#   r   r
   r)   r   r   r   r   r      s0    )÷üûúùø	÷
ör   Úrankr   c                 C   s6   dd l m  m} | d¡ | dkrt d¡ d S d S )Nr   zend-processr   )r2   r3   r4   Ú
rendezvousÚtimeÚsleep)rG   r<   r   r   r   r;   w   s
   
ÿr;   )rF   rI   Útypingr   r   r   r   r   Útorch.multiprocessingÚmultiprocessingr!   Útyping_extensionsr   Ú!lightning_fabric.accelerators.xlar   Ú.lightning_fabric.strategies.launchers.launcherr	   Ú5lightning_fabric.strategies.launchers.multiprocessingr
   Ú%lightning_fabric.utilities.apply_funcr   Úlightning_fabric.strategiesr   r   r   rD   r;   r   r   r   r   Ú<module>   s   Y