o پiVR@sRUddlmZddlZddlZddlZddlmZddlmZddl m Z m Z m Z ddl Z ddlZ ddlmmZddlmZddlmZe rTddlmZdd lmZeeZeGd d d Zdad e d <ddZ!ddZ"dDddZ#ddZ$ dEdFd#d$Z%dGd)d*Z&dHd-d.Z'dId0d1Z(dJd5d6Z)dKdd?d?Z+dLdBdCZ,dS)M) annotationsN) dataclass)Path) TYPE_CHECKINGListOptional)eplb_algorithms)get_model_architecture) ModelConfig) ServerArgsc@seZdZUded<ded<ded<ded<ded<ded<ed6d d Zed6d dZed6ddZed6ddZeddZ ddZ e d7ddZ e d8d7dd Z e d9d"d#Ze d:d$d%Ze d;d'd(Zdd4d5ZdS)?ExpertLocationMetadata torch.Tensorphysical_to_logical_mapphysical_to_logical_map_cpulogical_to_all_physical_maplogical_to_all_physical_map_cpu%logical_to_all_physical_map_num_validzOptional[torch.Tensor]%logical_to_rank_dispatch_physical_mapreturnintcC |jjdSNrrshapeselfrS/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/eplb/expert_location.py num_layers2 z!ExpertLocationMetadata.num_layerscCrNrrrrrnum_physical_experts6rz+ExpertLocationMetadata.num_physical_expertscCs"t|j|j\}}|dksJ|Sr)divmodr"ep_size)rans remainderrrrnum_local_physical_experts:s z1ExpertLocationMetadata.num_local_physical_expertscCrr )rrrrrrnum_logical_experts@rz*ExpertLocationMetadata.num_logical_expertscCs tjSN)torch distributedget_world_sizerrrrr$Ds zExpertLocationMetadata.ep_sizecCs^|jj\}}|jj\}}}|jj\}}||kr|ks!JJ||ks'J||ks-JdSr))rrrr)r num_layers_0num_physical_experts_0 num_layers_1num_logical_experts_0num_physical_experts_1 num_layers_2num_logical_experts_1rrr __post_init__Is  z$ExpertLocationMetadata.__post_init__ server_argsr model_configr moe_ep_rankc Cs^t||}|dur dS|d}|d}|j}|j}td||d|}tj||||dS)zDTrivial location - logical expert i corresponds to physical expert iNr" model_config_for_expert_locationrr!)rr7)r _init_commonrr(r*arangerepeatinit_by_mapping) r5r6r7commonr"r8rr(rrrr init_trivialWs" z#ExpertLocationMetadata.init_trivialNcCspt|tjs t|}||j}t||}|durdS|d}t|||j |d|d}tj ||d||dS)Nr8r$)r5rr(r$r7r5r$rr) isinstancer*Tensortensortodevicer r9$_compute_logical_to_all_physical_mapr( _init_raw)r5r6rr7r=r8rrrrr<rs(    z&ExpertLocationMetadata.init_by_mapping logical_countc Cst|tjs t|}t|jdkr|d}||j}t ||}|dur)dS|d}|d}|j }|j }t j||||d||t j|j||dd\}} } t j||d||j| |jdS) Nrr8r"r$) raw_algorithm num_groups num_nodes)tokens_per_expertr"r'rJrK algorithmr?)r@r*rArBlenr unsqueezerCrDr r9rJnnodesrrebalance_expertscompute_algorithmeplb_algorithmrF) r5r6rGr=r8r"rJrKrr expert_countrrr init_by_eplbsB       z#ExpertLocationMetadata.init_by_eplbcCsPt|}|dur dS|j|j}|j}||dksJ||}t||||dS)Nr)r8r"r'r$)ModelConfigForExpertLocationfrom_model_configr(ep_num_redundant_expertsr$dict)r5r6r8r"r$r'rrrr9s"z#ExpertLocationMetadata._init_commonr$c Cs|j\}}tj|d||jdfdd}tj|dkdd}t||||||jdkrsz1ExpertLocationMetadata.update..rZr!T) non_blocking.) getattrr*rBrangerviewr]rCrDwhere)rrjrlfield other_field self_field mask_updaterrprupdates  zExpertLocationMetadata.updateFlayer_idlogical_expert_idrequire_global_expertsboolcCsB|r|j|jd}tt|||jSdd|j||fDS)NrZcSg|]}|dkr|qSrZrrnphysical_expert_idrrrrq, zBExpertLocationMetadata.logical_to_all_physical..)rrlistrtr(tolist)rr|r}r~r"rrrlogical_to_all_physicals  z.ExpertLocationMetadata.logical_to_all_physical)rr)r5r r6r r7rr))r5r r6r rGr )r5r r6r )r5r r$rrr rr )rjrkrlrm)F)r|rr}rr~rrrm)__name__ __module__ __qualname____annotations__propertyrr"r'r(r$r4 staticmethodr>r<rUr9rFr{rrrrrr &s@        +   (#r Optional[ExpertLocationMetadata] _global_expert_location_metadatacCstSr)rrrrr#get_global_expert_location_metadata8srcCstdusJ|adSr)rr[rrr#set_global_expert_location_metadata<s rr5r rr r(rr$r7c s|j\}}fddt|D}t|D]}t|D]} ||| f} ||| | qq|durg|j|j} ||} | | } t|D]!}tD]} t||| | || | d}|dkre|g||| <qKqEt|dd}tj ||j dS)Ncsg|] }ddtDqS)cSsg|]}gqSrrrnrhrrrrqNszC_compute_logical_to_all_physical_map...)rtrr(rrrqMsz8_compute_logical_to_all_physical_map..candidate_physical_expert_idsnum_local_gpu_physical_expertsr7num_gpus_per_nodenum_local_node_physical_expertsrZ) pad_value)rD) rrtitemappendr$rP_find_nearest_expert_pad_nested_arrayr*rBrD)r5rr(r$r7rr"rr|rr}rrrnearest_expertrrrrEBsT         rEcs*tdd|Dfdd|D}|S)Ncss"|] }|D]}t|VqqdSr)rN)rnouterinnerrrr s z$_pad_nested_array..cs g|] }fdd|DqS)cs"g|] }|gt|qSrr)rnrmax_lenrrrrqs"z0_pad_nested_array...r)rnrrrrrqsz%_pad_nested_array..)max)arrrpaddedrrrr~s  r*rr"r_seedc Cst|}||}|j|j}||} |j\} } } |j} tj|| | fd| d}t| D]C}t| D]<}t |||}|dd||f}t|D] }t ||||| d||<qFt |dk }tj t|||d| d||dk<q1q+t|dksxJ|j}||ddddf|S)NrZ)size fill_valuedtyper)kr)r)randomRandomr$rPrrr*fullrt_logical_to_all_physical_rawrsumrrB _fair_choicesallrDrC)r5rr$r"r_rrrrrrr(rhrrr|r}routput_partialr7 num_remainrDrrrrfsH          rfr|r}rrmcCsdd|||fDS)NcSrrrrrrrrqrz0_logical_to_all_physical_raw..)r)rr|r}rrrrs rrrcC||Sr)r)rrrrr"_compute_gpu_id_of_physical_expertrnum_local_host_physical_expertscCrr)r)rrrrr#_compute_node_id_of_physical_expertrrrrrcspt|dkr |dSfdd|D}t|dkr|dS|fdd|D}t|dkr6|dSdS)Nr!rcg|] }t|kr|qSr)rr)r7rrrrqz(_find_nearest_expert..crr)rr) node_rankrrrrqrrZr)rrr7rrsame_gpu_physical_expert_idssame_node_physical_expert_idsr)r7rrrrrs     rrrrr random.RandomcCs6t|t|\}}|||j||d}|||S)N)r)r#rNsampleshuffle)rrrquotientr&r%rrrrs rc@s8eZdZUded<ded<dZded<ed d d ZdS) rVrrr(Nz Optional[int]rJr6r cCs&t|\}}t|dr||jSdS)N$get_model_config_for_expert_location)r hasattrr hf_config)r6 model_classrhrrrrW s  z.ModelConfigForExpertLocation.from_model_config)r6r )rrrrrJrrWrrrrrVs  rVr6r cCs|j}|dkrt|||S|drtj|dd}n|dr*tt| }nt|}d|vrFt dtj ||fi|d|iSd |vrYt d tj |||d d Std t|d )Ntrivialz.ptT) weights_onlyz.jsonrzOinit_expert_location from init_by_mapping using ServerArgs.init_expert_locationr7rGzLinit_expert_location from init_by_eplb using ServerArgs.init_expert_location)rGzendswithr*loadjsonloadsr read_textloggerinfor<rUNotImplementedErrorrkeys)r5r6r7data data_dictrrr(compute_initial_expert_location_metadatas@    r) r5r rr r(rr$rr7r)r) r5r rr r$rr"rr_rrr)r|rr}rrrm)rrrrrr)rrrrrr) rrmrrr7rrrrrrr)rrrrrrrr)r5r r6r r7rrr)- __future__rrloggingr dataclassesrpathlibrtypingrrrr*torch.distributedtorch.nn.functionalnn functionalrasglang.srt.eplbrsglang.srt.model_loaderr sglang.srt.configs.model_configr sglang.srt.server_argsr getLoggerrrr rrrrrErrfrrrrrrVrrrrrsF         < 6   (