o #iA?@s4ddlZddlmZmZmZmZmZddlZddlm Z ddl m Z m Z gdZ eeZ dZGdddZGd d d eZeed d d ZdZGdddZGdddZddZ  ddeedfdeeeefdedeeedfdeeeefdeeeeeff ddZdeefddZdS) N)AnyDictListOptionalTuple map_aggregate) tree_flattentree_unflatten)TensorChunkSpecsplit_args_kwargs_into_chunks merge_chunksFc@seZdZdZddZdS)_CustomReducera$ Custom reducer class that can be used to specify a custom operation that reduces losses of multiple microbatches into one value. Example: >>> # xdoctest: +SKIP >>> sum_reducer = _CustomReducer( >>> torch.tensor(0.0), >>> lambda a, b: a + b >>> ) cCs||_||_dSN) init_value reduce_fn)selfrrre/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/distributed/pipelining/microbatch.py__init__(s z_CustomReducer.__init__N)__name__ __module__ __qualname____doc__rrrrrrs rc@ eZdZdS) _LossReducerNrrrrrrrr-rgcCs||Srr)abrrr1r c@sfeZdZUdZddZeed<ddZddZe d e ed ffd d Z e d e e effd dZdS)r z2 Class used to specify chunking of inputs cCs ||_dSr split_dim)rr#rrrr=s zTensorChunkSpec.__init__r#cCs |jjd|jjd|jdS)N.()) __class__rrr#rrrr__repr__BszTensorChunkSpec.__repr__cCsd|jdS)NzTensorChunkSpec(r&r"r(rrr__str__GszTensorChunkSpec.__str__ chunk_dims.cCt|dd}|S)a A helper for creating a tuple of `TensorChunkSpec` from a tuple of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # There are three positional arguments to the model, and >>> # we are chunking them along dimension 0, 0 and 1, respectively >>> args_chunk_spec = TensorChunkSpec.from_tuple((0, 0, 1)) cSt|Srr dimrrrr Yr!z,TensorChunkSpec.from_tuple..r)r+args_chunk_specrrr from_tupleJs  zTensorChunkSpec.from_tuplecCr,)a\ A helper for creating a dictionary of `TensorChunkSpec` from a dictionary of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # Chunk dimension 0 for the "id" argument, 1 for the "mask" argument >>> kwargs_chunk_spec = TensorChunkSpec.from_dict({"id": 0, "mask": 1}) cSr-rr.r/rrrr kr!z+TensorChunkSpec.from_dict..r)r+kwargs_chunk_specrrr from_dict]s  zTensorChunkSpec.from_dictN)rrrrrint__annotations__r)r* staticmethodrr2rstrr4rrrrr 8s    r c@r) _ReplicateNrrrrrr9qrr9c# Csi}g}|}d}t|t|ks"Jdt|dt||D]\}}t|\} } || ||} | dus?Jt| \} } t| t| krWtd|d| g}t| | D]\}}|tuslt |t j su||g|q^t |t rt |t j sJ|d| |j}||kr|rtd|d |d |d |}ntd |d |d|dt |||j}trg}d}|D]2}t |}|| |j}tdddg|j}t||||j<|||<|||| |j7}q||n||d}q^td||||<q&g}t|D]'}i}|D]\}}g}|D] }|||q'|||<q||qg}|D]+} i}!t|t| ksRJt| |D]\\}}}"t||"|!|<qY||!qC|S)aW Given a dictionary of args, and a dictionary of chunking specs, shard the args according to the chunking specs. Args: args_dict: Dictionary of args args_chunk_spec: Dictionary of chunking specs num_chunks: Number of chunks to shard the args into Returns: args_split: List of sharded args Tzargs_dict.keys() = z args_chunk_spec.keys() = NzArgument value z9 did not have the same number of values as as chunk spec z is not a tensorz%Tensor size on chunking dimension is z', downsizing the number of chunks from z to r$zArg z% on chunking dimension has a size of z$, smaller than the number of chunks z. PiPPy cannot reduce the number of chunks because other arguments have bigger chunk-dimension sizes. Please adjust your num_chunks setting.rFzUnrecognized chunk spec: )lenlistkeysitemsr append ValueErrorzipr9 isinstancetorchTensorr sizer#loggerwarning RuntimeError tensor_split_debug_mask_minibatches zeros_likeslicendim TypeErrorranger )# args_dictr1 num_chunksargs_sharded_replicated arg_specsreal_num_chunks first_tensorarg_keyargflatspec chunk_specchunk_spec_flat_sharded_arg_flatvchunk_vv_split_dim_size chunk_tensorsexpanded_chunks split_dim_idx chunk_tensornew_val upper_idx slice_indices chunks_flat chunk_idx chunk_argskeyarg_single_chunkv_flat args_splitchunkper_chunk_argsarg_specrrr_shard_dict_of_argsus              rqargs.kwargschunksr1r3returnc s|duri}|durttft|}|durt|tt}ttt|tt||}t|}t|||}t||krOt|}ttt|tt||}t|t|kretdt|dt|g}|D]|t fddt tDqi||fS)a Given a sequence of args and kwargs, split them into a number of chunks according to their respective chunking specs. Args: args: Tuple of args kwargs: Dict of kwargs chunks: Number of chunks to split the args and kwargs into args_chunk_spec: chunking specs for args, in same shape as args kwargs_chunk_spec: chunking specs for kwargs, in same shape as kwargs Returns: args_split: List of sharded args kwargs_split: List of sharded kwargs Nz;args and kwargs are split into different number of chunks: z, c3s|]}|VqdSrr).0irirr Ysz0split_args_kwargs_into_chunks..) r DEFAULT_CHUNK_DIMr:dictfromkeysrq enumeraterGr>tuplerN) rrrsrtr1r3args_split_dictrS kwargs_splitrmrrxrr sH8     &r cs0|dur t|\}}nt|d\}}ttgt|}g|D]}t|\}}t|t|kr:td|d||q g}t|D]\} t| trчfddttD} t r| dj } | ddD] } | j | kssJqjt j t j | dd it| | jd } g}d}t| t| ksJt| | D])\}}||| j}tdddg|j}t|||| j<||}|||}qn| }|t j|| jd qFt| tr| j}ttD] }| ||}q||qFd}tdtD] }||ks Jq||qFt||S) z Given a list of chunks, merge them into a single value according to the chunk spec. Args: chunks: list of chunks chunk_spec: Chunking spec for the chunks Returns: value: Merged value NrzChunk z did not match chunk spec csg|]}|qSrr)rvrharg_idxchunks_flattenedrr s z merge_chunks..devicemeta)sectionsr0r/)r r rzr:r?r>r}rArNrIshaperBrHemptyr#r@rDrKrLcatrrrr )rtrYspec_flattened flatten_spec chunk0_flatrnchunk_flattenedr[args_flattenedrVpartial_values overall_shapeval meta_chunks values_to_catchunk_start_idx partial_value meta_chunk chunk_end_idxrfsliced reduced_valrhvaluerrrr ^sd-              r )NN)loggingtypingrrrrrrB torch.fx.nodertorch.utils._pytreer r __all__ getLoggerrrErIrrtensor sum_reducerrzr r9rqr8r5r r rrrrsF  9  h