o پiM @s<ddlmZddlZddlZddlmZmZmZmZddl Z ddl Z ddl m Z ddl mZmZddlmZddlmZddlmZmZmZmZdd lmZdd lmZmZmZmZerddd l m!Z!m"Z"eZ#eZ$eZ%e#rdd l&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1ne$se2de3e4Z5GdddeZ6d4ddZ7e j8e j9e j:hZ;e je j?e j@e jAhZBe jCe jDe jEe jFe jGhZHe jIe jJe jKe jLe jMe jNe jOe jPe jQh ZReBeHBeRBZSeBeHBeRBZTeBeHBZUd5ddZVd6d%d&ZW d7d8d*d+ZXGd,d-d-eZYGd.d/d/eZZGd0d1d1eYZ[Gd2d3d3eZ\dS)9) annotationsN) TYPE_CHECKINGAnyListOptional)GGMLQuantizationType) ParameterUninitializedParameter) LinearBase)MoeRunnerConfig)FusedMoEMethodBaseLinearMethodBaseQuantizationConfigQuantizeMethodBase)UnquantizedLinearMethod)is_cudais_hipis_xpuset_weight_attrs) CombineInputStandardDispatchOutput) gelu_and_mulmoe_align_block_sizemoe_sum silu_and_mul)ggml_dequantize ggml_moe_a8ggml_moe_a8_vecggml_moe_get_block_sizeggml_mul_mat_a8ggml_mul_mat_vec_a8.Only CUDA support GGUF quantization currently.cseZdZdZd&d'fdd Zd(d d Zd)d dZd*ddZd+ddZe d,ddZ e d-ddZ e d.ddZ d/d$d%Z ZS)0 GGUFConfigzConfig class for GGUF.Nmodules_to_not_convertlist[str] | NonereturnNonecs&ttr td|pg|_dS)Nr!)super__init___is_hipwarningswarnr#)selfr# __class__W/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/layers/quantization/gguf.pyr(7s  zGGUFConfig.__init__strcCdS)Nz GGUFConfig()r/r,r/r/r0__repr__=zGGUFConfig.__repr__ List[str]cCgSNr/r3r/r/r0get_scaled_act_names@r5zGGUFConfig.get_scaled_act_names'str'cCr2)Nggufr/r3r/r/r0get_nameCr5zGGUFConfig.get_namelist[torch.dtype]cCstjtjtjgSr8)torchhalfbfloat16float32r3r/r/r0get_supported_act_dtypesFsz#GGUFConfig.get_supported_act_dtypesintcCr2)N<r/clsr/r/r0get_min_capabilityIzGGUFConfig.get_min_capability list[str]cCr7r8r/rEr/r/r0get_config_filenamesMrHzGGUFConfig.get_config_filenamesconfigdict[str, Any] 'GGUFConfig'cCs||dgd}||S)Nr#)get_from_keys_or)rFrKr#r/r/r0 from_configQszGGUFConfig.from_configlayertorch.nn.ModuleprefixOptional['QuantizeMethodBase']cCsdddlm}ddlm}t|trt||jrtSt |St||r't |St||r0t |SdS)Nr)FusedMoE)VocabParallelEmbedding) &sglang.srt.layers.moe.fused_moe_tritonrT*sglang.srt.layers.vocab_parallel_embeddingrU isinstancer is_layer_skipped_ggufr#rGGUFLinearMethodGGUFEmbeddingMethod GGUFMoEMethod)r,rPrRrTrUr/r/r0get_quant_methodXs      zGGUFConfig.get_quant_methodr8)r#r$r%r&)r%r1)r%r6)r%r:)r%r=)r%rC)r%rI)rKrLr%rM)rPrQrRr1r%rS)__name__ __module__ __qualname____doc__r(r4r9r<rB classmethodrGrJrOr] __classcell__r/r/r-r0r"4s       r"rRr1r#rIcstfdd|DS)Nc3s|]}|vVqdSr8r/).0 module_namerRr/r0 jsz(is_layer_skipped_gguf..)any)rRr#r/rfr0rYisrYx torch.Tensorqweight qweight_typerCr%c Cs4|tvr|jddkr dnd}n |jddkrdnd}|jddkr3tj|jd|jd|j|jdS|tvr<||jS|jd|krS|tvrSt ||||jd}|S|t vrct ||||jd}|S|t vrt j|\}}|jd|jd||f}t||g||jR}||j}|St|}td |) Nridtypedevice$Unsupported GGUF quantization type: )IMATRIX_QUANT_TYPESshaper>emptyrrrsUNQUANTIZED_TYPESTMMVQ_QUANT_TYPESr MMQ_QUANT_TYPESr DEQUANT_TYPESr;GGML_QUANT_SIZESr WeightTypeNotImplementedError) rirkrl mmvq_safey block_size type_sizerwweightr/r/r0fused_mul_mat_ggufs*"   rw1w2 topk_weightstopk_ids qweight_type2 activationc s6d fdd }t|} |tvrr|tvrr|jddkrr|j\} } |j\} } } |jd}t|}t||| \}}}t||||||| || }||}t|||||||jdd| | }|| ||jd| | |d}t || | S|t vr|t vr|j\} } |j\} } } |jd}t |||||| | }||}t |||d||jd| |}|| ||jd| | |d}t || | St dtt||D]L\}\}}||d |jdd}d}t||D],\}}||}t|||}||}||}t||||}|dur|}q||q|| |<q| S) Nrirjcsp|jdd}|jdd|f}tj||j|jd}dkr&t|||Sdkr1t|||Std)NrorqsilugeluzUnsupported activation: )rwr>rxrrrsrr ValueError)rid output_shapeoutrr/r0acts  zfused_moe_gguf..actr@rtznThere is no support for fast MoE kernel for current quantization method. Falling back to slow implementation. )rt)rirj)r> empty_liker|rwrrrreshapemul_viewrr{rlogger warning_once enumeratezipradd_)rirrrrrlrrrout_hidden_states num_tokens_ENtop_k BLOCK_SIZEsorted_token_ids expert_idsnum_tokens_post_paddedrtokwidxinpcurrent_hidden_statewwii expert_up expert_down current_stater/rr0fused_moe_ggufs        (         r hidden_sizerrtorch.dtype | Nonec Cs|tvr t||S|tvrDtj|\}}|}||jd||ks&Jtj|d|d}t ||||jd|} | j g|j|RSt |}t d|)Nrtr)dimindexru) ryr> embeddingr}r;r~flattenrw index_selectrrrr) rirkrlrrrrrx_flatquantdequantr/r/r0apply_gguf_embeddings rc@sFeZdZdZdddZdddZd ddZd ddZ d!d"ddZdS)#rZz[Linear method for GGUF. Args: quant_config: The GGUF quantization config. quant_configr"cC ||_dSr8rr,rr/r/r0r(< zGGUFLinearMethod.__init__rPrQinput_size_per_partitionrCoutput_partition_sizes list[int] input_size output_size params_dtype torch.dtypec Ks||_t|}||f} tdd} t| dd| dggidt| ||d| ttjt|tj ddd} t| ddidd t| ||d | dS) NF requires_gradrtrT) input_dim output_dim tensor_shapeis_gguf_weightdata_containershard_id shard_id_maprkrr)is_gguf_weight_type weight_typeshard_weight_typeignore_warningrl) rsumGGUFUninitializedParameterrregister_parameterrr>rxlenuint8) r,rPrrrrrextra_weight_attrsoutput_size_per_partitionrrkrlr/r/r0create_weights?s>    zGGUFLinearMethod.create_weightscCsD|jj}|tvs|tvst|}td|d|d||dS)Nz#Unsupported GGUF quantization type z in layer .)rlrryr}rr_create_padded_weight_param)r,rPrlr/r/r0process_weights_after_loadingmsz.GGUFLinearMethod.process_weights_after_loadingcCsf|j}|j}|j}t|j}dkrdd|D}t|dks(Jtd|tt|}tdd|D}t dd|D}t j ||f||j d} t tttttff} |D]6} || } t d d|d | D} | || d }|| d}|| | | |d |f<| ||f| | <qY|jt| d d }t|t|t|d| i|d|d Sd S)z;Create padded weight parameter for GGUF MergedLinear layer.rtcSsh|]}|jqSr/r)rddatar/r/r0 ~sz?GGUFLinearMethod._create_padded_weight_param..z!Data container has mixed dtypes: cs|]}|dVqdS)rtNsizerdrir/r/r0rgz?GGUFLinearMethod._create_padded_weight_param..csrrNrrr/r/r0rgrrqcsrrrrr/r/r0rgrNrFrshard_offset_maprk)rkrrrrrnextitermaxrr>zerosrsdictr1tuplerCrclearrrvarsr)r,rPrkrrrrr padded_side concat_side padded_datarrid_in_containerstartendr padded_paramr/r/r0rxs8    z,GGUFLinearMethod._create_padded_weight_paramNrirjbiastorch.Tensor | Noner%c Cs|jj}|rDd|vrgdn|}|j}g}|D]$}|jj|\}} } |jj|} |t|||| d| f| qtj |dd} n |j}|jj } t||| } |durZ| || S)Nq)rkvrt)axis) rkrrrlrappendr contiguousr>catrr) r,rPrirrrkresultrrroffsetrlrr/r/r0applys(   zGGUFLinearMethod.applyrr") rPrQrrCrrrrCrrCrr)rPrQr8)rPrQrirjrrr%rj) r^r_r`rar(rrrrr/r/r/r0rZ5s   . &rZc@s8eZdZdZdddZdddZdddZdddZdS)r\zXMoE method for GGUF. Args: quant_config: The GGUF quantization config. rr"cCrr8rrr/r/r0r(rzGGUFMoEMethod.__init__rPrQ num_expertsrCrintermediate_size_per_partitionrrc Ks|d||f}tdd}t|dd|dgdt|||d|ttjdtjd dd} t| dddd t| ||d | |||f}tdd} t| dd|dgdt| ||d | ttjdtjd dd} t| dddd t| ||d | dS)NroFrrtrT)rrrrr w13_qweightr)rrrw13_qweight_type w2_qweightw2_qweight_type)rrrrr>rxr) r,rPr rr rrrr r rrr/r/r0rsX          zGGUFMoEMethod.create_weightsmoe_runner_configr cCs ||_dSr8)r)r,rPrr/r/r0create_moe_runners zGGUFMoEMethod.create_moe_runnerdispatch_outputrr%rc Csz|jdusJddlm}|jjdksJd|j}|j}|j}|\}}} t||j|j |||j j |j j |jd} || dS)Nr)StandardCombineInputrz"Only SiLU activation is supported.)rirrrrrlrr) hidden_states) fused_experts&sglang.srt.layers.moe.token_dispatcherrrrr topk_outputrr rr rr) r,rPrrrirrrrroutputr/r/r0rs(   zGGUFMoEMethod.applyNr ) rPrQr rCrrCr rCrr)rPrQrr )rPrQrrr%r)r^r_r`rar(rrrr/r/r/r0r\s    >r\c@seZdZdZd ddZd S) r[z^Embedding method for GGUF. Args: quant_config: The GGUF quantization config. rPrQrirjr%cCs,|j}|jj}|jd}t|||||jdS)Nrtr)rkrlrrrr)r,rPrirkrlrr/r/r0r-s   zGGUFEmbeddingMethod.embeddingN)rPrQrirjr%rj)r^r_r`rarr/r/r/r0r[&sr[c@seZdZUeZded<dS)rzlist[torch.Tensor]rN)r^r_r`r cls_to_become__annotations__r/r/r/r0r7s  r)rRr1r#rI)rirjrkrjrlrCr%rj)rirjrrjrrjrrjrrjrlrCrrCrr1r%rjr8) rirjrkrjrlrCrrCrrrr%rj)] __future__rloggingr*typingrrrrr;r>rrtorch.nn.parameterrr sglang.srt.layers.linearr sglang.srt.layers.moer *sglang.srt.layers.quantization.base_configr r rr&sglang.srt.layers.quantization.unquantrsglang.srt.utilsrrrrrrr_is_cudar)_is_xpu sgl_kernelrrrrsgl_kernel.quantizationrrrrrr r+ getLoggerr^rr"rYF32F16BF16ryQ4_0Q4_1Q5_0Q5_1Q8_0Q8_1STANDARD_QUANT_TYPESQ2_KQ3_KQ4_KQ5_KQ6_K KQUANT_TYPESIQ1_MIQ1_SIQ2_XXSIQ2_XSIQ2_SIQ3_XXSIQ3_SIQ4_XSIQ4_NLrvr}r{r|rrrrZr\r[rr/r/r/r0s|     "   5     #p m