o
    zi                     @   s   d dl mZ d dlmZmZmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ G dd deeZdS )    )ABC)AnyDictListOptionalN)Tensor)override)Accelerator)ClusterEnvironment)CheckpointIO)	Precision)Strategy_all_gather_ddp_if_available)ReduceOpc                       sh  e Zd ZdZ					d'dee deeej  dee	 dee
 dee f
 fdd	Zed
efddZed
efddZed
efddZed
efddZeed
efddZed
eeej  fddZejdeeej  d
dfddZed
eeeef  fddZed(dedee ded
efddZed)d!ed"ed
efd#d$Zed* fd%d&Z  Z S )+ParallelStrategyz:Strategy for training with multiple processes in parallel.Nacceleratorparallel_devicescluster_environmentcheckpoint_io	precisionc                    s"   t  j|||d || _|| _d S )N)r   r   r   )super__init__r   r   )selfr   r   r   r   r   	__class__ X/home/ubuntu/.local/lib/python3.10/site-packages/lightning_fabric/strategies/parallel.pyr   !   s   
zParallelStrategy.__init__returnc                 C      | j d ur
| j  S dS Nr   )r   global_rankr   r   r   r   r!   -      zParallelStrategy.global_rankc                 C   r   r    )r   
local_rankr"   r   r   r   r$   1   r#   zParallelStrategy.local_rankc                 C   r   r    )r   	node_rankr"   r   r   r   r%   5   r#   zParallelStrategy.node_rankc                 C   r   )N   )r   
world_sizer"   r   r   r   r'   9   r#   zParallelStrategy.world_sizec                 C   s
   | j dkS r    )r!   r"   r   r   r   is_global_zero=   s   
zParallelStrategy.is_global_zeroc                 C   s   | j S N_parallel_devicesr"   r   r   r   r   B   s   z!ParallelStrategy.parallel_devicesc                 C   s
   || _ d S r)   r*   )r   r   r   r   r   r   F   s   
c                 C   s   | j | jdS )zArguments for the ``DistributedSampler``.

        If this method is not defined, or it returns ``None``, then the ``DistributedSampler`` will not be used.

        )num_replicasrank)r'   r!   r"   r   r   r   distributed_sampler_kwargsJ   s   z+ParallelStrategy.distributed_sampler_kwargsFtensorgroup
sync_gradsc                 C   s   t |||dS )z&Perform a all_gather on all processes.)r0   r1   r   )r   r/   r0   r1   r   r   r   
all_gatherS   s   zParallelStrategy.all_gatherTdecisionallc                 C   sF   t jt|| jd}| j|tjd}|rt|| jk}|S t|}|S )a  Reduces a boolean decision over distributed processes. By default is analagous to ``all`` from the standard
        library, returning ``True`` only if all input decisions evaluate to ``True``. If ``all`` is set to ``False``,
        it behaves like ``any`` instead.

        Args:
            decision: A single input decision.
            all: Whether to logically emulate ``all`` or ``any``. Defaults to True.

        Returns:
            bool: The reduced boolean decision.

        )device)	reduce_op)	torchr/   introot_device
all_reducer   SUMboolr'   )r   r3   r4   r   r   r   reduce_boolean_decisionX   s   z(ParallelStrategy.reduce_boolean_decisionc                    s"   | j d usJ | j   t  S r)   )r   teardownr   r"   r   r   r   r>   n   s   

zParallelStrategy.teardown)NNNNN)NF)T)r   N)!__name__
__module____qualname____doc__r   r	   r   r7   r5   r
   r   r   r   propertyr8   r!   r$   r%   r'   r   r<   r(   r   setterr   strr   r.   r   r2   r=   r>   __classcell__r   r   r   r   r      sR    "r   )abcr   typingr   r   r   r   r7   r   typing_extensionsr   )lightning_fabric.accelerators.acceleratorr	   9lightning_fabric.plugins.environments.cluster_environmentr
   )lightning_fabric.plugins.io.checkpoint_ior   "lightning_fabric.plugins.precisionr   $lightning_fabric.strategies.strategyr   &lightning_fabric.utilities.distributedr    lightning_fabric.utilities.typesr   r   r   r   r   r   <module>   s   