o پiL@srdZddlZddlmZddlmZmZmZddlZddl m Z ddl m Z ddl mZgdZeeZeZd ejd eefd d Zd ejdejfddZGddde ZGdddeZGdddeZGdddeeZGdddeeZGdddeZGdddeeZGdd d eZ Gd!d"d"eZ!Gd#d$d$eZ"d%ed&ed'ed efd(d)Z#d*d+Z$d,d-Z%dS).zdAdapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/parameter.pyN)Fraction)CallableOptionalUnion) Parameter)pad_or_narrow_weight)is_cpu) BasevLLMParameterPackedvLLMParameterPerTensorScaleParameterModelWeightParameterChannelQuantScaleParameterGroupQuantScaleParameterBlockQuantScaleParameterPackedColumnParameterRowvLLMParameterdtypereturncCsP|tjtjtjtjfvrdS|tjtjfvrdS|tjkrdS|tjkr&dSdS)Nr) torch float8_e4m3fnfloat8_e4m3fnuz float8_e5m2float8_e5m2fnuzfloat16bfloat16float32float64)rr O/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/layers/parameter.py _dtype_ranks  r"target loaded_weightcCs|j|jksJd|jd|j|j|jkr||dSt|j}t|j}|dus1|dur=td|jd|j||krMtd|jd|j||dS)z Copy `loaded_weight` into `target` while forbidding downcasts. bf16/fp16 share the same rank, and all fp8 variants share the same rank. z target.shape=, loaded_weight.shape=Nz.Unsupported copy between dtypes: target.dtype=z, loaded_weight.dtype=z&Downcasting not allowed: target.dtype=)shapercopy_r" ValueError)r#r$ target_rank loaded_rankr r r!copy_with_check/s"    r+cseZdZdZdejffdd ZdejdefddZe dd Z d ejfd d Z d ejfd dZ d ejfddZ d ejfddZd ejfddZZS)r z Base parameter for vLLM linear layers. Extends the torch.nn.parameter by taking in a linear weight loader. Will copy the loaded weight into the parameter when the provided weight loader is called. datac stj||ddS)NF)r, requires_grad)super__new__)clsr,kwargs __class__r r!r/SszBasevLLMParameter.__new__ weight_loadercCs ||_dS)z Initialize the BasevLLMParameter :param data: torch tensor with the parameter data :param weight_loader: weight loader callable :returns: a torch.nn.parameter N_weight_loader)selfr,r4r r r!__init__Ws zBasevLLMParameter.__init__cC|jSNr5r7r r r!r4czBasevLLMParameter.weight_loaderr$cCs"|jj|jks J|j|dSr:)r,r&r'r7r$r r r!_assert_and_loadgsz"BasevLLMParameter._assert_and_loadcC||dSr:r>r=r r r!load_column_parallel_weightkz-BasevLLMParameter.load_column_parallel_weightcCr?r:r@r=r r r!load_row_parallel_weightnrBz*BasevLLMParameter.load_row_parallel_weightcKr?r:r@r7r$r1r r r!load_merged_column_weightqrBz+BasevLLMParameter.load_merged_column_weightcKr?r:r@rDr r r!load_qkv_weighttrBz!BasevLLMParameter.load_qkv_weight)__name__ __module__ __qualname____doc__rTensorr/rr8propertyr4r>rArCrErF __classcell__r r r2r!r Ls r cszeZdZdZdeffdd ZeddZ ddej d ed e fd d Z dej fd dZ ddej d ed e fddZ ZS)_ColumnvLLMParametera Private class defining weight loading functionality (load_merged_column_weight, load_qkv_weight) for parameters being loaded into linear layers with column parallelism. This includes QKV and MLP layers which are not already fused on disk. Requires an output dimension to be defined. Called within the weight loader of each of the column parallel linear layers. output_dimc ||_tjdi|dSNr ) _output_dimr.r8)r7rOr1r2r r!r8z_ColumnvLLMParameter.__init__cCr9r:)rRr;r r r!rOr<z_ColumnvLLMParameter.output_dimFr$tp_rankuse_presharded_weightscCs|s9|jj|j}ddlm}tr/||j|d|||j|\}}|j|jks(J||dS||j|||}t|j|dS)Nr%narrow_padded_param_and_loaded_weight) r,r&rO$sglang.srt.model_loader.weight_utilsrW_is_cpur'narrowr+)r7r$rTrU shard_sizerW param_datar r r!rAs&   z0_ColumnvLLMParameter.load_column_parallel_weightc Ks|d}|d}|d}|d}t|ttfr*|j|jkr*|j||d\}}|j}||j||}ddl m }t rN|||d|||j|| \}}n#|sq||} | |} | |j |jkrit ||j| |}n||j| |}|j |j ksyJ||dS)N shard_offsetr[rTrUr]r[rrV)get isinstancerr packed_dimrO adjust_shard_indexes_for_packingr,rZrXrWrYr&rr') r7r$r1r]r[rTrUr\rW start_idxend_idxr r r!rEsF           z._ColumnvLLMParameter.load_merged_column_weightc Ks|d}|d}|d}|d}t|ttfr*|j|jkr*|j||d\}}|j} |dkr3|n||}| |j||} t rXddl m } | | |d|||j|| \} }n |sd||j|||}| j |j ksvJd | j d |j | |dS) Nr]r[shard_id num_headsr^qrrVzparam_data.shape=r%)r_r`rr rOrarbr,rZrYrXrWr&r') r7r$rTrUr1r]r[rerfr\rWr r r!rFs@           z$_ColumnvLLMParameter.load_qkv_weightF)rGrHrIrJintr8rLrOrrKboolrArErFrMr r r2r!rNxs,    5rNcsNeZdZdZdeffdd ZeddZ d dej d ed e fd d Z Z S)rz Parameter class defining weight_loading functionality (load_row_parallel_weight) for parameters being loaded into linear layers with row parallel functionality. Requires an input_dim to be defined. input_dimc rPrQ) _input_dimr.r8)r7rkr1r2r r!r8rSzRowvLLMParameter.__init__cCr9r:)rlr;r r r!rkr<zRowvLLMParameter.input_dimFr$rTrUc Cs|sP|jj|j}ddlm}tr/||j|d|||j|\}}|j|jks(J||dS||}||}||j|jkrHt||j||}n||j||}t |jdkr\| d}|jj|jkseJ|j|dS)NrrVr) r,r&rkrXrWrYr'rrZlenreshape) r7r$rTrUr[rWr\rcrdr r r!rCs8    z)RowvLLMParameter.load_row_parallel_weightrh) rGrHrIrJrir8rLrkrrKrjrCrMr r r2r!rs rc@eZdZdZdS)r z] Parameter class for linear layer weights. Uses both column and row parallelism. NrGrHrIrJr r r r!r Nr c@ro)rz Parameter class for weight scales loaded for weights with grouped quantization. Uses both column and row parallelism. Nrpr r r r!rWrqrc@ro)r z Parameter class for weight scales loaded for weights with channel-wise quantization. Equivalent to _ColumnvLLMParameter. Nrpr r r r!r `rqr c@ro)rz Parameter class for weight scales loaded for weights with block-wise quantization. Uses both column and row parallelism. Nrpr r r r!rirqrcs~eZdZdZfddZdeeefdefddZfdd Z d d Z d d Z fddZ de jdeeeffddZZS)r a Parameter class for scales where the number of scales is equivalent to the number of logical matrices in fused linear layers (e.g. for QKV, there are 3 scales loaded from disk). This is relevant to weights with per-tensor quantization. Adds functionality to map the scalers to a shard during weight loading. Note: additional parameter manipulation may be handled for each quantization config specifically, within process_weights_after_loading c s$dddd|_tjdi|dS)Nrrr)rgkvr )qkv_idxsr.r8)r7r1r2r r!r8sz PerTensorScaleParameter.__init__rercCs4t|tr|St|tsJ||jvsJ|j|Sr:)r`ristrrt)r7rer r r!_shard_id_as_ints  z(PerTensorScaleParameter._shard_id_as_intc.|dd|ddtj|i|dSNrTrUpopr.rCr7argsr1r2r r!rC  z0PerTensorScaleParameter.load_row_parallel_weightcO|j|i|dSr:_load_into_shard_idr{r r r!rEz1PerTensorScaleParameter.load_merged_column_weightcOr~r:rr{r r r!rFrz'PerTensorScaleParameter.load_qkv_weightcrwrxryr{r2r r!rAr}z3PerTensorScaleParameter.load_column_parallel_weightr$cKs^|j}||}t|jdkr|jddksJ|d}||}|j|jks(J||dS)zU Slice the parameter data based on the shard id for loading. rrN)r,rvrmr&r')r7r$rer1r\r r r!rs z+PerTensorScaleParameter._load_into_shard_id)rGrHrIrJr8rrurirvrCrErFrArrKrrMr r r2r!r rs    r cjeZdZdZ ddeeefdedeeffdd Ze dd Z e d d Z e d d Z ddZ ZS)rz Parameter for model parameters which are packed on disk and support column parallelism only. See PackedvLLMParameter for more details on the packed properties. N packed_factorramarlin_tile_sizec (||_||_||_tjdi|dSrQ_packed_factor _packed_dim_marlin_tile_sizer.r8r7rrarr1r2r r!r8zPackedColumnParameter.__init__cCr9r:rr;r r r!rar<z PackedColumnParameter.packed_dimcCr9r:rr;r r r!rr<z#PackedColumnParameter.packed_factorcCr9r:rr;r r r!rr<z&PackedColumnParameter.marlin_tile_sizecCt|||j|jdSNr[r]rr!_adjust_shard_indexes_for_packingrrr7r[r]r r r!rb z6PackedColumnParameter.adjust_shard_indexes_for_packingr:rGrHrIrJrrirrr8rLrarrrbrMr r r2r!rs"     rcr)r a Parameter for model weights which are packed on disk. Example: GPTQ Marlin weights are int4 or int8, packed into int32. Extends the ModelWeightParameter to take in the packed factor, the packed dimension, and optionally, marlin tile size for marlin kernels. Adjusts the shard_size and shard_offset for fused linear layers model weight loading by accounting for packing and optionally, marlin tile size. Nrrarc rrQrrr2r r!r8rzPackedvLLMParameter.__init__cCr9r:rr;r r r!rar<zPackedvLLMParameter.packed_dimcCr9r:rr;r r r!rr<z!PackedvLLMParameter.packed_factorcCr9r:rr;r r r!rr<z$PackedvLLMParameter.marlin_tile_sizecCrrrrr r r!rbrz4PackedvLLMParameter.adjust_shard_indexes_for_packingr:rr r r2r!r s"    r paramrkrOc s6t|ddt|dddusdur|jdksJddur1dus+JddddurCdus=Jdddfdd t|jD}||||d |vrtt|d rp|j||d kstJd |jj||_t|d r||_t|d r||_ d |vrt|dr|d |_ |S)a  Permute a parameter's layout to the specified input and output dimensions, useful for forcing the parameter into a known layout, for example, if I need a packed (quantized) weight matrix to be in the layout {input_dim = 0, output_dim = 1, packed_dim = 0} then I can call: permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) to ensure x is in the correct layout (permuting it to the correct layout if required, asserting if it cannot get it to the correct layout) rkNrOrz`permute_param_layout_ only supports 2D parameters when either input_dim or output_dim is not setz&either input or output dim must be setrcsg|] }|fvr|qSr r ).0icurr_input_dimcurr_output_dimr r! -sz)permute_param_layout_..raz9permute_param_layout_ currently doesn't support repackingrlrRr) getattrr,dimrangeinserthasattrrapermuterlrRr)rrkrOr1permr rr!permute_param_layout_ s>           rcCs||||fSr:r r[r]rr r r! _adjust_shard_indexes_for_marlinDsrcCs.||}||}|durt|||dS||fS)Nr)rrr r r!rHsr)&rJlogging fractionsrtypingrrrrtorch.nnrsglang.srt.layers.utilsrsglang.srt.utilsr__all__ getLoggerrGloggerrYrrir"rKr+r rNrr rr rr rr rrrr r r r!sF     ,?    D(, :