o پi>#@s(ddlmZddlmZmZddlmZddlmZm Z m Z m Z m Z ddl Z ddlmZmZmZerHddlmZmZmZddlmZmZmZmZeGd d d ZeGd d d eZGd ddeZeGdddeZGdddeZGdddZ GdddZ!d$ddZ"d%dd Z#d&d"d#Z$dS)') annotations)ABCabstractmethod) dataclass) TYPE_CHECKINGCallableOptionalTuple TypeGuardN) MoeA2ABackendMoeRunnerBackendRoutingMethodType)TritonRunnerCoreTritonRunnerInputTritonRunnerOutput) CombineInputCombineInputFormatDispatchOutputDispatchOutputFormatc@seZdZUdZded<dZded<dZded<dZded<dZded<dZ ded<dZ ded <dZ d ed <dZ d ed <dZ ded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dS)MoeRunnerConfigNz Optional[int] num_expertsnum_local_experts hidden_sizeintermediate_size_per_partitionlayer_idtop_knum_fused_shared_expertszOptional[torch.dtype] params_dtypezOptional[RoutingMethodType]routing_method_typesilustr activationTboolis_gatedFapply_router_weight_on_inputinplace no_combinezOptional[float]routed_scaling_factor gemm1_alphagemm1_clamp_limit)__name__ __module__ __qualname__r__annotations__rrrrrrrrr!r#r$r%r&r'r(r)r.r.Y/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/layers/moe/moe_runner/base.pyrs$                rc@(eZdZeed ddZd ddZdS) RunnerInputreturnr cCdSNr.selfr.r.r/runner_backend7zRunnerInput.runner_backendTypeGuard[TritonRunnerInput]cC |jtjkSr4r7r TRITONr5r.r.r/runner_backend_is_triton; z$RunnerInput.runner_backend_is_tritonNr2r )r2r9r*r+r,propertyrr7r=r.r.r.r/r15s r1c@r0) RunnerOutputr2r cCr3r4r.r5r.r.r/r7@r8zRunnerOutput.runner_backendTypeGuard[TritonRunnerOutput]cCr:r4r;r5r.r.r/r=Dr>z%RunnerOutput.runner_backend_is_tritonNr?)r2rCr@r.r.r.r/rB?s rBc@seZdZdZdS) MoeQuantInfozMoe quantization data.N)r*r+r,__doc__r.r.r.r/rDHsrDc@s@eZdZdddZedd dZeedddZdddZdS) MoeRunnerCoreconfigrcCs ||_dSr4)rG)r6rGr.r.r/__init__Ps zMoeRunnerCore.__init__ runner_inputr1 quant_inforD running_statedictr2rBcCr3r4r.)r6rIrJrKr.r.r/runSszMoeRunnerCore.runr cCr3r4r.r5r.r.r/r7Yr8zMoeRunnerCore.runner_backendTypeGuard[TritonRunnerCore]cCr:r4r;r5r.r.r/r=]r>z&MoeRunnerCore.runner_backend_is_tritonN)rGr)rIr1rJrDrKrLr2rBr?)r2rN) r*r+r,rHrrMrAr7r=r.r.r.r/rFOs  rFc@s6eZdZUiZded<eddd ZedddZdS) FusedOpPoolzdict[str, Callable] _fused_funcsa2a_backend_namer runner_backend_name fused_funcrcCsb||f}||jvrtd|d|dt|sJd|t|s*Jd|||j|<dS)NzFused function for  to  is already registered.zInvalid dispatch name: zInvalid runner name: )rP ValueErrorr r )clsrQrRrSkeyr.r.r/register_fused_funcds  zFusedOpPool.register_fused_func dispatch_name runner_namer2Optional[Callable]cCs||f}|j|}|Sr4)rPget)rWrZr[rXrSr.r.r/get_fused_funcus zFusedOpPool.get_fused_funcN)rQr rRr rSr)rZr r[r r2r\)r*r+r,rPr- classmethodrYr^r.r.r.r/rOas  rOc@s^eZdZUiZded<iZded<edd d Zedd dZedddZ edddZ dS) PermuteMethodPoolz=dict[Tuple[DispatchOutputFormat, MoeRunnerBackend], Callable]_pre_permute_methodsz;dict[Tuple[MoeRunnerBackend, CombineInputFormat], Callable]_post_permute_methodsdispatch_output_namer rR permute_funcrcC6||f}||jvrtd|d|d||j|<dS)a8 Register a customized pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. :param dispatch_output_name: The DispatchOutputFormat name. :param runner_backend_name: The MoeRunnerBackend name. :param permute_func: The permute function to register. zPre-permute method for rTrUN)rarV)rWrcrRrdrXr.r.r/register_pre_permute  z&PermuteMethodPool.register_pre_permutecombine_input_namecCre)a3 Register a customized post-permute function for the given MoeRunnerBackend and CombineInputFormat. :param runner_backend_name: The MoeRunnerBackend name. :param combine_input_name: The CombineInputFormat name. :param permute_func: The permute function to register. zPost-permute method for rTrUN)rbrV)rWrRrhrdrXr.r.r/register_post_permutergz'PermuteMethodPool.register_post_permutedispatch_output_formatrrunner_input_formatr r2cC6||f}|j|}|dusJd|d|d|S)a9 Retrieve the pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. :param dispatch_output_format: The DispatchOutputFormat type. :param runner_input_format: The MoeRunnerBackend type. :return: The registered permute function or None if not found. NzPre-permute function for rT is not registered)rar])rWrjrkrXpre_permute_funcr.r.r/get_pre_permute   z!PermuteMethodPool.get_pre_permuterunner_output_formatcombine_input_formatrcCrl)a5 Retrieve the post-permute function for the given MoeRunnerBackend and CombineInputFormat. :param runner_output_format: The MoeRunnerBackend type. :param combine_input_format: The CombineInputFormat type. :return: The registered permute function or None if not found. NzPost-permute function for rTrm)rbr])rWrqrrrXpost_permute_funcr.r.r/get_post_permuterpz"PermuteMethodPool.get_post_permuteN)rcr rRr rdr)rRr rhr rdr)rjrrkr r2r)rqr rrrr2r) r*r+r,rar-rbr_rfrirortr.r.r.r/r`|s      r`rQr rRr2rcsdfdd }|S)z Decorator to register a fused function for the given DispatchOutputFormat and MoeRunnerBackend. :param a2a_backend_name: The A2A backend name. :param runner_backend_name: The MoeRunnerBackend name. :return: The decorator function. rSrct||Sr4)rOrY)rSrQrRr.r/ decoratorsz®ister_fused_func..decoratorN)rSrr.)rQrRrwr.rvr/rYs rYrccdfdd }|S) a Decorator to register a pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. :param dispatch_output_name: The DispatchOutputFormat name. :param runner_backend_name: The MoeRunnerBackend name. :return: The decorator function. rdLCallable[[DispatchOutput, MoeQuantInfo, MoeRunnerConfig, dict], RunnerInput]r2rcrur4)r`rfrdrcrRr.r/rwz'register_pre_permute..decoratorN)rdryr2rr.)rcrRrwr.r{r/rf  rfrhcrx) a  Decorator to register a post-permute function for the given MoeRunnerBackend and CombineInputFormat. :param runner_backend_name: The MoeRunnerBackend name. :param combine_input_name: The CombineInputFormat name. :return: The decorator function. rdKCallable[[RunnerOutput, MoeQuantInfo, MoeRunnerConfig, dict], CombineInput]r2rcst||Sr4)r`rirzrhrRr.r/rwr|z(register_post_permute..decoratorN)rdr~r2rr.)rRrhrwr.rr/rir}ri)rQr rRr r2r)rcr rRr r2r)rRr rhr r2r)% __future__rabcrr dataclassesrtypingrrrr r torchsglang.srt.layers.moe.utilsr r r 'sglang.srt.layers.moe.moe_runner.tritonrrr&sglang.srt.layers.moe.token_dispatcherrrrrrr1rBrDrFrOr`rYrfrir.r.r.r/s,     ]