o Ti:@sHddlZddlZddlZddlZddlmZmZddlZddl Z ddl m Z ddZ ddZ d d Zd d Zd dZddZddZgfddZdefddZdefddZddZdefddZdd Zgfd!d"Zgfd#d$Zd%efd&d'Zd8d%efd(d)Zd9defd,d-Zdefd.d/Zd0e d1e!fd2d3Z"d:d4d5Z#d:d6d7Z$dS);N)GRADIENT_ACCUMULATION_STEPSTRAIN_MICRO_BATCH_SIZE_PER_GPU)loggercCstj|sdSt|4}|D](}dD]#}||}|dkr6||t|ddWdSqqWddS1sCwYdS)Nstderr.log does not exist)ErrorerrorERRORz: )ospathexistsopenfindlenlstripfilenameflinesidxrN/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/autotuning/utils.py search_errors     rcCsptj|sdSt|"}|D]}d}||}|dkr%WddSqWddS1s1wYdS)NrKeyboardInterruptr TF)r r r rrrrrrwas_interruptteds    rcCst|ts t|Std|}|D];}|dd}|dkr q||vs.Jd|d|t||tr>||||}qt|dksHJd||}q|S) Nz\$[\w]+$ nvme_pathzunknown var key: z, in z-unable to replace multiple non-string matches) isinstancestrrefindallreplacelowerr)value replace_dictmatchesvarvar_keyrrrfind_replace_str)s   r,cCst|tr=|D]1\}}t|trt||||<t|tr0tt|D] }t|||||<q$t|tr:t||q dSt|trVtt|D]}tt|||||<qHdSdSN) r!dictitemsr"r,listranger find_replace)targetr(keyr'irrrr2;s      r2cCst|ts|gS|Sr-)r!r0)valrrrget_listJs r7cs|D]?\}t|tjjrti|<qvr$|<qtts2g<fddt |DqS)Nc3s |] }|vr|VqdSr-r).0r5dkrr [szcombine_dict..) r/r! collectionsabcMapping combine_dictgetr0extendr7)r:uvrr9rr@Qs $r@cCs@||vr ||=dS|D]\}}t|tjjrt||q dS)zDeletes a key from a dictionary if it exists. Args: t (string): target key to delete d (dict): dictionary to delete from N)r/r!r=r>r? del_if_exists)tr:r;rDrrrrE_s rEcCsj|dur3|D]*\}}||vr2|durt||qt|tjjr.t||i||||<q|||<q|S)aReplaces values in dict d with values in dict u. Args: d (dict): the target dict to overwrite u (dict): the dict containing the values to overwrite the target dict Returns: dict d with values overwritten by the corresponding ones in dict u. N)r/rEr!r=r>r?r(rA)r:rC ignored_keysr;rDrrrr(ns  r(r:cCs:||vr||S|D]}t|trt||Sq dSr-)valuesr!r.get_val_by_key)r:r;rDrrrrIs  rIcCs8||vr|||<|D] }t|trt|||q dSr-)rHr!r.set_val_by_key)r:r;vvrDrrrrJs   rJc Cstj|s tddSt|dX}t}|D]F}| }|dkr&qz| \}}| d\}}t |}Wnt yL}zt d|d}~ww||vr]t dt d||||<qWd|S1smwY|S)NzNUnable to find hostfile, will proceed with training with local resources only.rr=zEHostfile is not formatted correctly, unable to proceed with training.zCHostfile contains duplicate hosts, unable to proceed with training.zhost {} is already defined)r r isfilerwarningrr= OrderedDict readlinesstripsplitint ValueErrorrformat) hostfile_pathfd resource_poolrhostnameslots_ slot_counterrrrrfetch_hostfiles6          r_configcCsdtfdd}|di}|sdS|d}d}|dkrdS|d kr/||d r.||d r.dSn%|d krRd dg}|did|vrDd}|did|vrQd}ndS|r]|ds]dSdS)Nr`cSs|durdSt||SNF)boolrA)r`r4rrris_Falsesz$validate_ds_config..is_Falsezero_optimizationTstageFr r cpu_offloadcpu_offload_paramscpunvmeoffload_optimizerdevice offload_param optimizer)r.rA)r`rc config_zeroreoffloadoffload_devicesrrrvalidate_ds_configs.  rrcCs$dd|D}t|}dd|DS)aF Removes duplicate dictionaries from a list. Uses list comprehension and the json library to sort and stringify each dictionary and the set data type to ensure unique values. Works with nested data structures. Args: l (list): a list of (nested) data structures. Returns: A list of unique values. cSsg|] }tj|ddqS)T) sort_keys)jsondumps)r8r:rrr sz%remove_dupe_dicts..cSsg|]}t|qSr)rtloads)r8rrrrrvs)set)llist_of_stringsrrrremove_dupe_dictss r{cs6|r|D]}dtdtffdd ||qdSdS)z Prunes the input configurations Args: configs (dict): A configuration dictionary. ignored_keys (list, optional): the keys of the sections to delete. Defaults to []. Returns: A configuration dictionary. r:r;cs8||vr ||=dS|D] }t|tr||q dSr-)rHr!r.)r:r;dd find_del_keyrrr~s    z"prune_config..find_del_keyN)r.r")r`rGr;rr}r prune_configs   rcCs*g}|D] }t||||qt|S)a Prunes the input list of configurations Args: configs (list): A list of configuration dictionaries. ignored_keys (list, optional): the keys of the sections to delete. Defaults to []. Returns: A list of valid and unique configuration dictionaries. )rappendr{)configsrG pruned_listr`rrr prune_configss    r tuning_spacecCsRg}|D] \}}t|tr|t|t|tr&t|dkr&||q|S)zOutputs the list of tunable parameters in the tuning space dict. Args: tuning_space (dict): a configuration dictionary containing tunable parameters as lists of values. Returns: A list of strings r )r/r!r.rBget_tuning_keysr0rr)r tuning_keysr4r6rrrrs   rcsddtffdd g}i}|D]}||i||<t||q|D] }t||||q#|S)z Splits the tuning space dictionary to result in all combinations of values. Args: tuning_space (dict): the tuning space where tunable parameters are lists of values. r:c3sd||}}|D] }t|ts|g}q fdd|D}tj|D] }tt||Vq%dS)Nc3s*|]}t|tr |nt|VqdSr-)r!r.r7)r8rDgen_combinationsrrr<,s(z.gen_combinations..)keysrHr!r0 itertoolsproductr.zip)r:rrHrDvalues_choicescombrrrr's z)get_all_configs..gen_combinations)r.rArEr(r)r ignore_keys all_configsignored_key_valsikcrrrget_all_configs s     rrFcsnt|vr |tt|vr|t|ddd dtffdd |||d}||d kr4|Sd S) a Generates a name from the acronyms of the tuning keys in the config dict. TRAIN_MICRO_BATCH_SIZE_PER_GPU is always included in the tuning keys. Args: config (dict): the config dict used to generate the name tuning_keys (list, optional): the tuning keys used to generate the name. Defaults to None. prefix (str, optional): a string added to the beginning of the name. Defaults to None. cSsd}|durdS|D]?\}}dtdd|d}t|ts't|tr7|dkr7||d|d7}q t|trB|r@dnd }|||d7}q |S) NrNone_cS|dSNrrrrrrMz:canonical_name..get_offload_name..r\(#{:.1e}TF) r/joinmaprSr!rTfloatrVrb)offload_configcnamer4r6rrrget_offload_nameHs  z(canonical_name..get_offload_nameNFr`csBd}|r|dur |S|D]\}}|dkrq|dkr(|d7}|s'||7}q|dkr9|d7}|s8||7}qt|trO|||d}|dkrO||d7}|rV||vrVqdtd d |d}|st|tsot|tr|d kr||d |d7}qt|t r|rd nd}|||d7}q||d7}q|ddS)Nr arg_mappingsrmop_rkoo_omit_valr\cSrrrrrrrrprz:canonical_name..get_name_by_keys..rrrrr ) r/r!r.rrrSrTrrVrb)r`rrrr4r6nkey_strget_name_by_keysrrrrVs>         z(canonical_name..get_name_by_keysrrexpra)rrrsortr.)r`rprefixrnamerrrcanonical_name;s  (rcCsr|sdSt|}|D])\}}t|tr%|dkr|||<nt|||<t|tr6t|dkr6|d||<q |S)Nrnr)copydeepcopyr/r!r.get_first_configr0r)r`cfgr4r6rrrrs     rexpsexps_dirc Csng}|D]0}|d}tj||d}t|d}t||||Wdn1s/wYq|S)Nrz.jsonw)r r rrrtdumpr)rr exp_pathsrexp_nameexp_pathrXrrrwrite_experimentss   rcCsL|durV|ddkrtt|d|d|S|ddkr*tt|d|d|S|ddkr=tt|d |d |S|d dkrPtt|d |d |St|dS|dkrgtt|d|d|S|d|krztt|d|d|S|d|krtt|d |d|S|d|krtt|d|d|St|dS)NlJ)rlz Tʚ;i@z G@Bi Mi K rGMKir"roundrpostfixunits precisionrrrmemory_to_strings&         rcCs|durC|ddkrtt|d|d|S|ddkr*tt|d|d|S|ddkr=tt|d|d|St|dS|d |krVtt|d|d|S|d |kritt|d|d|S|d |kr|tt|d|d|St|dS) Nrrz BrrrrrBrrrrrrrnumber_to_strings        rr-)NrF)rNr)%r#collections.abcr=r rtdeepspeed.runtime.constantsrrrrutilsrrrr,r2r7r@rEr(r.rIrJr_rrr{rrrrrrr0r"rrrrrrrs<     !  H