o ίiC? @s<ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZmZddlmZmZmZddlmZddlmZmZmZmZddlZddlZddlmZddl m!Z!ddl"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+dd l,m-Z.dd l/m0Z0m1Z1e2Z3ej4j5j6j7ej4j5j8j7ej4j5j9j7ej4j5j:j7ej4j;jj7hZ?eGdddZ@eGdddZAde@fddZBdKdeeCeDfde*fddZEdKdeeCeDfde*fddZFddZGedeHfddZIedeHfd d!ZJedeCfd"d#ZKedeCfd$d%ZLedeCfd&d'ZMedeHfd(d)ZNed*eCdeCfd+d,ZOedePfd-d.ZQd/d0ZRd1d2ZSd3d4ZTd5d6ZUd7eCdeeePeHffd8d9ZVdKd:d;ZWeX eDd?eDfd@dAZ[dBdCZ\dDdEZ]ej^dFdGZ_   dMde@dHeeeePeHffdIdJZ`dS)NN)asdict dataclass) lru_cachepartialreduce)chain)ListOptionalTupleUnion) distributed)ReduceOp)MixedPrecisionPolicy fully_shard)DTensor)checkpoint_wrapper) DeviceMeshinit_device_mesh)DistributedDataParallel)CheckpointPolicy$create_selective_checkpoint_contextsc@seZdZUdZeed<dZeed<dZeed<dZe ed<dZ e ed<dZ e ed<d Z e ed <d Ze ed <dZe ed <dZdZe ed<dZeed<dZe ed<dS)DistributedArgsdp_shard dp_replicatetp_sizeF"selective_activation_checkpointingfull_activation_checkpointingcompileno_shard fsdp_typebf16 model_dtypematmul_allow_tf32Tdetect_anomalycompile_cache_size_limit forkserver spawn_methodN)__name__ __module__ __qualname__rint__annotations__rrrboolrrr strr"r#&allow_bf16_reduced_precision_reductionr$r&r(r1r1D/home/ubuntu/.local/lib/python3.10/site-packages/core/distributed.pyr0s           rc@szeZdZUdZeed<dZeed<dZeed<dZeed<dZ eed<d Z eed <d Z eed <dZ eed <dZ eed<dS)EnvironmentArgsGNUMKL_SERVICE_FORCE_INTEL1OMP_NUM_THREADSMKL_NUM_THREADS0ENABLE_INTRA_NODE_COMMTORCH_NCCL_AVOID_RECORD_STREAMS22NCCL_IB_TIMEOUTWARN NCCL_DEBUGTORCH_NCCL_ASYNC_ERROR_HANDLINGzexpandable_segments:TruePYTORCH_CUDA_ALLOC_CONFN)r)r*r+r5r/r-r7r8r:r;r=r?r@rAr1r1r1r2r3Hs        r3distributed_argsc Cs|j}|j}|j}|||tks$Jd|d|d|dtd g}g}|dkr6|||d|dks?|jdkrI|||d|dkrW|||d t|}t|}td ||d S) Nz#dp_replicate * dp_shard * tp_size (z * z) != world_size ()rrrrtpcuda) mesh_shapemesh_dim_names)rrrget_world_sizeappendr tupler)rBrrrdimsnamesr1r1r2get_device_meshYs(      rMxmeshcC0t|}tj|tj|r|ndd|SN)opgroup)torchtensorrEdist all_reducer MAX get_grouprNrOrUr1r1r2dist_maxsr[cCrPrQ)rTrUrErVrWr AVGrYrZr1r1r2 dist_meanyr\r^cCsPt}|D] }t||||<||dkr||n||||<q|SNr)dictr^dimitemtolist)rNrkr1r1r2dist_mean_dicts .rfreturncCstjdduS)N LOCAL_RANK)osenvirongetr1r1r1r2get_is_torch_runsrlcCsdtjvot S)N SLURM_JOB_ID)rirjrlr1r1r1r2get_is_slurm_jobsrncC,tr ttjdStrttjdSdS)NRANK SLURM_PROCIDrrlr,rirjrnr1r1r1r2get_global_rank rscCro)Nrh SLURM_LOCALIDrrrr1r1r1r2get_local_rankrtrvcCro)N WORLD_SIZE SLURM_NTASKSrrrr1r1r1r2rHrtrHcCs tdkSr_)rsr1r1r1r2 get_is_masters ryjob_idcCs2tr ttjdSd\}}t|}|||S)N MASTER_PORT)i Ni`)rlr,rirjrandomRandomrandint)rzMIN_MASTER_PORTMAX_MASTER_PORTrngr1r1r2get_master_ports   rcCsDtrtjdStr tdddtjdg}|ddSdS) N MASTER_ADDRscontrolshow hostnamesSLURM_JOB_NODELISTrzutf-8z 127.0.0.1)rlrirjrn subprocess check_outputsplitdecode)rr1r1r2get_master_addrs rcCst|}t}tjtj|dd||d<tr*dtj d}tj |r*||d<| D] \}}tj |t|krNt|tj |<td|d|q.dS) NT) ignore_errorsTRITON_CACHE_DIRz/scratch/slurm_tmpdir/rmTMP_DIRzWARNING: Setting z to )rtempfilemkdtempatexitregistershutilrmtreernrirjpathexistsitemsrkr/loggerwarning)env_argsenv_varstriton_cache_dirnew_tmpnamevaluer1r1r2 setup_envs rcCsVt|jtWdn1swYt}tttjd<tt tjd<t tjd<tt t tj dddtjd<trPtd |ntr\td |ntd td tjd |krudksxJJ|jrdtjjj_td|jtjjj_tjdkrtj|tjjdddtj|j dS)z Handle single and multi-GPU / multi-node / SLURM jobs. Initialize the following variables: - global_rank - world_size Nrprwrrm)rzr{z(Run launched with torchrun, local rank: z%Run launched with slurm, local rank: zSingle GPU jobzENV: rr%Tz\WARNING: Setting torch.backends.matmul.allow_tf32 to True. This is faster but less accurate.rzenv://nccl) init_methodbackend)!mpset_start_methodr(Managerrvr/rsrirjrHrrr,rkrlrinfornr#rTbackendsrEmatmul allow_tf32rr0 device_count set_devicer init_process_groupautogradset_detect_anomalyr$) dist_args local_rankr1r1r2setup_torch_distributeds:        rcCs|jdd}tt||S)N.sep)rrgetattr)module access_stringrLr1r1r2 get_modules  rcCs4|jdd}tt|dd|}t||d|dS)Nrrr)rrrsetattr)rrrrLparentr1r1r2 set_modules rn_layerscsdgfddtDS)N) vision_modelTcs"g|] }d||dkfqS)zlayers.rr1).0irr1r2 sz.default_fsdp_grouping_plan..)rangerr1rr2default_fsdp_grouping_plansrcsptfdd}|S)Ncs|vrtjStjSN)r MUST_SAVEPREFER_RECOMPUTE)ctxfuncargskwargsno_recompute_opsr1r2default_policy's z*get_default_policy..default_policy)default_no_recompute_ops)rrr1rr2get_default_policy$s r@@modelrstdcCst|D]o\}}t|tr|}|dkr#td|dqt | s1t | r:td|d| | }|}||krVtd|d|d||krftd|d|d|dkrutd|dqdS) NrzModel parameter z, is empty, probably because of FSDP shardingz contains NaN or Infz! has a suspiciously large range (zE): please check initialization and init_weights is defined and calledz. has a suspiciously large standard deviation (z> is all zeros: it might be because of a missing initialization)rnamed_parameters isinstancerto_localnumelrrrTisnananyisinfmaxminrall)rrrrparam param_range param_stdr1r1r2check_model_value_range1s4     rcCsttj|tddS)zD Handle signals sent by SLURM for time limit / pre-emption. zSignal handler installed.N)signalSIGUSR2rr)callabler1r1r2init_signal_handlerSsrcCsttjd}tdt|f|dkr4tjdddkr4tdtjdtd tjdntd t ddS) NrqzHost: %s - Global rank: %ir LAUNCH_WITHDORAzRequeuing job rmzscontrol requeue z+Not the master process, no need to requeue.) r,rirjrrsocket gethostnamerksystemsysexit)prod_idr1r1r2requeue_slurm_job[s rc #sBdfddtjD}z dVWtj|dStj|w)N)rr{rprwrhLOCAL_WORLD_SIZETORCHELASTIC_RUN_IDDORA_FORCE_DISTRIBcs,i|]}|ds |vr|tj|qS))SLURM_SLURMD_SRUN_SBATCH_ SUBMITIT_WANDB_) startswithrirjpop)rrN distrib_namesr1r2 rs zclean_env..)rirjupdate) cluster_envr1rr2 clean_envfs rfsdp_grouping_planc Cs<|jdkr(|jdksJd|dusJd|jdksJd|||d||ttjtjtjd|j}|jdks?|jd kr|jd krY|j dksMJd |d  dksYJd tt |tjd |j dksj|jd krn|d n|dd}|durt t |j}|D]\} } t|| } t|| t| fi|d| iqt|fi|ddi}ntd|j|jr|jrJdt|ttt|d}|jr |jrJdtd|jD]\} } t| dd} |j| | qt|dr |jjj D]\} }t|dd}|jjj | |q|jr|j!tj"j#_$t|}|S)Nr full_shardz/Only full shard is supported for TP parallelismz&TP plan is required for TP parallelismFz+Compile is not supported for TP parallelismrD)fp32fp16r!rz)dp_shard must be 1 for no_shard fsdp_typer) param_dtype reduce_dtype)rrr) mp_policyrOreshard_after_forwardTzInvalid fsdp_type: zUSelective activation checkpointing is incompatible with full activation checkpointing) context_fnzUFull activation checkpointing is incompatible with selective activation checkpointingz7FULL ACTIVATION CHECKPOINTING on all transformer blocks)preserve_rng_stater)%rr rr`rTfloat32float16bfloat16r"rsizerrlenlayersrrr ValueErrorrrrrrrrdebugnamed_childrenregister_modulehasattrr transformer resblocksr&_dynamoconfigcache_size_limit)r device_mesh model_argsrBrtp_parallelizerr fsdp_configrr rlayer_idtransformer_blockresblockr1r1r2parallelize_models                r$r)rr)NNN)ar contextlibloggingmultiprocessingrrir|rrrrrr dataclassesrr functoolsrrr itertoolsrtypingrr r r rT xformers.opsxformersr rVtorch.distributedr "torch.distributed._composable.fsdprrtorch.distributed._tensorr;torch.distributed.algorithms._checkpoint.checkpoint_wrapperrtorch.distributed.device_meshrrtorch.nn.parallelrDDPtorch.utils.checkpointrr getLoggerropsatenmmdefault _scaled_mm'_scaled_dot_product_efficient_attention#_scaled_dot_product_flash_attentionc10d_functionalreduce_scatter_tensorxformers_flash flash_fwdrrr3rMr,floatr[r^rfr.rlrnrsrvrHryrr/rrrrrrrno_gradnnModulerrrcontextmanagerrr$r1r1r1r2s                  -   !