o پiAH@sdZddlmZmZmZddlZddlmZddlmm Z ddl m Z e Ze Zer7ddlmZmZer=ddlZddlmZmZddlmZddlmZdd lmZmZdd lm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&e$'d Gddde$Z(e$'dGddde$Z)Gdddej)Z*deej+deej+fddZ,Gddde$Z-Gddde-Z.Gddde-Z/Gdd d e$Z0Gd!d"d"e0Z1Gd#d$d$e0Z2 %d5d&ej+d'ej+d(dd)dd*e3d+e4deej+ej+ffd,d-Z5d.ej+d/ddej+fd0d1Z6d.ej+d2e1fd3d4Z7dS)6zCustom normalization layers.)OptionalTupleUnionN)current_platform)fused_add_rmsnormrmsnorm) norm_infer rms_norm_fn)triton_one_pass_rms_norm)fuse_scale_shift_kernel)can_use_fused_inplace_qknormfused_inplace_qknorm)get_tensor_model_parallel_rank$get_tensor_model_parallel_world_size get_tp_group)CustomOp)get_bool_env_varrms_normc seZdZdZdejdfdededejde eddf fd d Z dd ej d e ej fd dZ dd ej d e ej de ej eej ej fffddZ dd ej d e ej de ej eej ej fffddZ dd ej d e ej de ej eej ej fffddZ dd ej d e ej de ej eej ej fffddZ dd ej d e ej de ej eej ej fffddZdefddZZS)RMSNormzRoot mean square normalization. Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight. Refer to https://arxiv.org/abs/1910.07467 ư>N hidden_sizeepsdtypevar_hidden_sizereturncsRttt||_||_||_||krdn||_ t dr'|j |_ dSdS)N%SGLANG_ENABLE_DETERMINISTIC_INFERENCE) super__init__nn Parametertorchonesweightvariance_epsilonrvariance_size_overriderforward_native_forward_method)selfrrrr __class__b/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/layers/layernorm.pyr-s  zRMSNorm.__init__xresidualcCst||jd||jdS)N)biasr-r)r r"r#r'r,r-r*r*r+forward_triton>szRMSNorm.forward_tritoncCs |j}|j}|d|d}|dur|j}|d|d}|jtjkrB|||}|dur;|d||d|fS||}|S|jdurM| ||S|duret |||j j |j ||||fS|jddkrvt||j j |j }n t||j j |j }||}|S)Nr)shapedevicereshapeviewrr floatr0r$r%rr"datar#r r)r'r,r-r4r5residual_shapeoutr*r*r+ forward_cudaCs0       zRMSNorm.forward_cudacCs|s|}|j}|tj}|dur"||tj}||}|jd}||jkr7td|jd||j dur?|}n||j krOtd|j d||dd|j f}| dj ddd}|t ||j }||j|}|durz|S||fS) Nr1zExpected hidden_size to be z , but found: z$Expected hidden_size to be at least .Tdimkeepdim) is_contiguous contiguousrtor float32r4r ValueErrorr$powmeanrsqrtr#r")r'r,r- orig_dtyperx_varvariancer*r*r+r%fsB      zRMSNorm.forward_nativecC |||SNr%r/r*r*r+ forward_cpu zRMSNorm.forward_cpucCsD|durt|||jj|j\}}}||fSt||jj|jdS)Nr) torch_npunpu_add_rms_normr"r9r# npu_rms_norm)r'r,r-r;_ residual_outr*r*r+ forward_npus  zRMSNorm.forward_npucCrLrMrNr/r*r*r+ forward_hips zRMSNorm.forward_hipcC(d|jjd}|d|j7}|SNz hidden_size=rz, eps=r"r9sizer#r'sr*r*r+ extra_reprzRMSNorm.extra_reprrM)__name__ __module__ __qualname____doc__r rDintr8rrrTensorr0rrr<r%rOrVrWstrr^ __classcell__r*r*r(r+r%st  & *    r layer_normc seZdZ     ddededdffdd Zd ejdejfd d Zd ejfd d Z d ejde eje ejejfffddZ ej ded dd ejdeejde eje ejejfffddZ dd ejdeejde eje ejejfffddZdefddZZS) LayerNormh㈵>TNrr.rcst||_||d}||_|r7tjtj|fi||_|r2tjtj|fi||_ dSd|_ dS| dd| ddd|_ dS)Nr5rr"r.) rrrrr rremptyr"r.register_parameter_weight_fallback_cache)r'rrr.elementwise_affiner5rfactory_kwargsr(r*r+rs     zLayerNorm.__init__r,cCsZt|dd}|dus|j|jks|j|jks||jkr+tj|j|j|jd}||_|S)Nrnrk)getattrr5rnumelrr r!rn)r'r,wfr*r*r+_get_weight_fallbacks   zLayerNorm._get_weight_fallbackcCs*t|d|j|j|j|jdd|jSNr1F)r is_rms_norm)rr7rr"r.rr4)r'r,r*r*r+r0s zLayerNorm.forward_tritoncCs$|j}|d|j}|||S)Nr1)r4r7rr0)r'r,r4r*r*r+r<szLayerNorm.forward_cudainductor)backenddisabler-cCsv|j}|jddd}||djddd}||t||j}|jdur,|j|}|jdur6||j}||S)Nr1T)r@r=) rrGrFr rHrr"r.rC)r'r,r- input_dtyperGrKr*r*r+r%s     zLayerNorm.forward_nativecCrLrMrNr/r*r*r+rOrPzLayerNorm.forward_cpucCrXrYrZr\r*r*r+r^r_zLayerNorm.extra_repr)rjTTNNrM)r`rarbrdboolrr rertr0rrr<compileris_npurr%rOrfr^rgr*r*r(r+risL     ric@s"eZdZdejdejfddZdS) FP32LayerNorminputsrcCsd|j}|j}t||j|jdur|jj|dnd|jdur*|jj|dnd|j |S)N)r5) rr5Frhr8normalized_shaper"rCr.r)r'r origin_dtyper5r*r*r+forward szFP32LayerNorm.forwardN)r`rarbr rerr*r*r*r+r~ sr~tensorrcCs|dur|SdSrM)rB)rr*r*r+_ensure_contiguoussrcseZdZUdZeed<ddejdfdede de d ej d ef fd d Z d ej dej dej eBdej dej deej ej ff ddZddZd ej dej dej eBdej dej deej ej ff ddZZS)_ScaleResidualNormScaleShiftz Fused kernel that combines: 1. residual_out = residual + gate * x 2. normed = layernorm(residual_out) or rmsnorm(residual_out) 3. out = normed * (1 + scale) + shift compute_dtype is always fp32 for higher precision. norm_typerFrrrorprefixcsft||_||_|jdkrt|||d|_dS|jdkr*t||||d|_dStd|jdNrms)rrlayer)rorrz Norm type z not implemented) rrrrrrnormr~NotImplementedErrorr'rrrorrr(r*r+r)s    z%_ScaleResidualNormScaleShift.__init__r-r,gateshiftscalerc Cs|jdddkr$|jddkr$ddl}|jddd||||||Sddlm}t|tr:|d kr:td ||| | t|t j rK| ndt t |jd dt t |jd d| | |j|j S) Nr1r zJFusedScaleResidualNormScaleShift cuda not available, using native fallbackr= stacklevel)%fused_scale_residual_norm_scale_shiftr2z8Only gate value of 1 is supported for int type, but got r"r.)r4warningswarnr%Csglang.jit_kernel.diffusion.cutedsl.scale_residual_norm_scale_shiftr isinstancerdrErBr rerrqrrr)r'r-r,rrrrrr*r*r+r<=s.  z)_ScaleResidualNormScaleShift.forward_cudacO|j|i|SrMrNr'argskwargsr*r*r+rWcz(_ScaleResidualNormScaleShift.forward_hipc Cst|tr|dks J||}n;t|tjrA|dkr:|jd}|jd|}||jd||fd|dd}n|||}n tdt |d| |} t | ||} | |fS)Nr2)r?sizesr=z Gate type z not supported) rrdr rer?r4 unflattenflattenrEtyperr ) r'r-r,rrrresidual_output num_frames frame_seqlen normalized modulatedr*r*r+r%hs       z+_ScaleResidualNormScaleShift.forward_native)r`rarbrcrf__annotations__r rDrdr8r{rrretupler<rWr%rgr*r*r(r+rsZ  &rc@eZdZdZdS) ScaleResidualLayerNormScaleShiftrNr`rarbrr*r*r*r+rrc@r)ScaleResidualRMSNormScaleShiftrNrr*r*r*r+rrrc seZdZUdZeed<ddejdfdede de d ej d ef fd d Z d ej dej dej dej fddZddZd ej dej dej dej fddZZS)_NormScaleShiftz Fused kernel that combines: 1. normed = layernorm(x) or rmsnorm(x) 2. out = normed * (1 + scale) + shift compute_dtype is always fp32 for higher precision. rrFrrrrorrcs`t||_|jdkrt|||d|_dS|jdkr't||||d|_dStd|jdr)rrrrrrr~rrr(r*r+rs    z_NormScaleShift.__init__r,rrrcCs|jdddkr"|jddkr"ddl}|jddd||||Sddlm}||tt|j d dtt|j d d|||j |j S) Nr1rrrz=FusedNormScaleShift cuda not available, using native fallbackr=r)fused_norm_scale_shiftr"r.) r4rrr%rrrBrrqrrr)r'r,rrrrr*r*r+r<s"  z_NormScaleShift.forward_cudacOrrMrNrr*r*r+rWrz_NormScaleShift.forward_hipcCs"||}t|||}||jSrM)rr rCr)r'r,rrrrr*r*r+r%s   z_NormScaleShift.forward_native)r`rarbrcrfrr rDrdr8r{rrrer<rWr%rgr*r*r(r+rsJ  rc@r)LayerNormScaleShiftrNrr*r*r*r+rrrc@r)RMSNormScaleShiftrNrr*r*r*r+rrrTqkq_normk_normhead_dim allow_inplacec Cs|d}|j}|j}tr3|r3||kr3t||jr3t||d|||d||j|j||d||fS|j} |j} ||d|| } ||d|| } | | fS)zApply QK normalization for query and key tensors. Uses JIT fused inplace kernel when available, falls back to standard RMSNorm. rr1)rrq_weightk_weightrr) r[r#_is_cudar rr r7r"r4) rrrrrr batch_sizeq_epsk_epsq_shapek_shapeq_outk_outr*r*r+ apply_qk_norms0    rr,rc Cs~t}t}|j}|j||}|}|djddd}tj |t j j j jd}|t ||j|}|j|dS)Nr=r1Tr>)op)r)rrrr" tensor_splitr8rFrGr all_reducer _C_distributed_c10dReduceOpAVGrHr#rC) r,rtp_ranktp_size src_dtyper"x_fp32rKoutputr*r*r+tensor_parallel_rms_norms  rlayernorm_scale_shiftcCs2t|d|jd|jj|jj|jdd|jSru)rr7r4rr"r.r)r,rr*r*r+apply_layernorm_onlysr)T)8rctypingrrrr torch.nnrtorch.nn.functional functionalr'sglang.multimodal_gen.runtime.platformsris_cudarr}_is_npu sgl_kernelrrrQ'sglang.jit_kernel.diffusion.triton.normrr 2sglang.jit_kernel.diffusion.triton.rmsnorm_onepassr .sglang.jit_kernel.diffusion.triton.scale_shiftr sglang.jit_kernel.normr r 8sglang.multimodal_gen.runtime.distributed.parallel_staterrr.sglang.multimodal_gen.runtime.layers.custom_opr*sglang.multimodal_gen.runtime.utils.commonrregisterrrir~rerrrrrrrrdr{rrrr*r*r*r+sd       ZiD  (