o پirZ@sddlZddlmZddlmZmZmZmZddlZddl m Z m Z ddl m Z mZmZmZmZddlmZddlmZddlmZdd lmZmZmZmZdd lmZdd lm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)m*Z*dZ+e&Z,e(Z-e)Z.e/e0Z1e+fde2de2de2fddZ3 d-de2de2de2dee2fddZ4 d-de2de2de2de2dee2f ddZ5eGdddZ6ej7de'e.d d!ej8d"e2d#e2d$e2d%e2d&e2deej8ej8ffd'd(Z9Gd)d*d*ej:j;Zr<r$r$r%num_org_vocab_paddinghr?z8VocabParallelEmbeddingShardIndices.num_org_vocab_paddingcCr;r*)rBr@r<r$r$r%num_added_vocab_paddinglr?z:VocabParallelEmbeddingShardIndices.num_added_vocab_paddingcCs |j|jSr*)rArBr<r$r$r%num_elements_paddedpr?z6VocabParallelEmbeddingShardIndices.num_elements_paddedcCs|j|jksJ|j|jksJ|j|jksJ|j|jks J|j|jks(J|j|jks0J|j|jks8J|j|jks@J|j|j ksHJ|j |j ksPJdSr*) r3r4r5r6r7r8r9r:r>rAr@rBr<r$r$r% __post_init__tsz0VocabParallelEmbeddingShardIndices.__post_init__N)__name__ __module__ __qualname____doc__int__annotations__propertyr>r@rArBrCrDrErFr$r$r$r%r2Js2  r2T)dynamicbackenddisableinput_r7r8rCr9r:c Cs^||k||k@}||k||k@}||||}||||} ||B} | || }|| fSr*r$) rQr7r8rCr9r:org_vocab_maskadded_vocab_mask added_offset valid_offset vocab_maskr$r$r%get_masked_input_and_masks    rWcseZdZdZddedddddddeded eejd eed ed ee d e de de de ffddZ e dededededededefddZdeeefddZdedejfd d!Zd"d#Zde fd$d%ZZS)&VocabParallelEmbeddingaEmbedding parallelized in the vocabulary dimension. Adapted from torch.nn.Embedding, note that we pad the vocabulary size to make sure it is divisible by the number of model parallel GPUs. In order to support various loading methods, we ensure that LoRA-added embeddings are always at the end of TP-sharded tensors. In other words, we shard base embeddings and LoRA embeddings separately (both padded), and place them in the same tensor. In this example, we will have the original vocab size = 1010, added vocab size = 16 and padding to 64. Therefore, the total vocab size with padding will be 1088 (because we first pad 1010 to 1024, add 16, and then pad to 1088). Therefore, the tensor format looks like the following: TP1, rank 0 (no sharding): |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 | index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | TP2, rank 0: |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 | index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 | TP2, rank 1: |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 | Args: num_embeddings: vocabulary size. embedding_dim: size of hidden state. params_dtype: type of the parameters. org_num_embeddings: original vocabulary size (without LoRA). padding_size: padding size for the vocabulary. quant_config: quant config for the layer prefix: full name of the layer in the state dict NTF) params_dtypeorg_num_embeddings padding_size quant_configprefix enable_tpuse_attn_tp_groupuse_presharded_weightsnum_embeddings embedding_dimrZr[r\r]r^r_r`rac st||_||_| |_|jr#| rt} t|_nt} t |_n | dus)Jd} d|_||_ |p4||_ t rHt |j ||jdkrH||j9}||_||j } | |_| r]| dks]Jdt |j |j|_t |j| |j|_|j|jkswJ||j|j|j |j | |j|_||_d} |dur|j||d} | durt} t|jtu}tt| }|r|stdt| jd| |_|durt}|j |j |_ t!|j|j|_"|jj#|j"ksJ|jj$|jj%|_&|jj'|jj(|_)|jj*||j|j"g|j|j||j+ddS) NFrr#z.Lora is not supported with presharded weights.)r^z The class zG must implement the 'embedding' method, see UnquantizedEmbeddingMethod.)rZ weight_loader),super__init__r]r_r`rrtp_sizer r rborg_vocab_size_is_cpur&r\raorg_vocab_size_paddednum_embeddings_padded _get_indices shard_indicesrcget_quant_methodrtype __class__rXrNotImplementedErrorrG quant_methodtorchget_default_dtypenum_added_embeddingsr num_embeddings_per_partitionrEr8r7 num_org_embeddings_per_partitionr:r9"num_added_embeddings_per_partitioncreate_weightsrd)r=rbrcrZr[r\r]r^r_r`ratp_rankrurris_embedding_layer!quant_method_implements_embeddingrpr$r%rfs          zVocabParallelEmbedding.__init__vocab_size_paddedrjr rhrzrgr"c Csj||}t|||\}} t||||d\} } t||} t| |} t| |}t| |}t|| | | | | ||S)zGet start and end indices for vocab parallel embedding, following the layout outlined in the class docstring, based on the given tp_rank and tp_size.r0)r1minr2)clsr~rjr rhrzrgnum_added_embeddings_paddedr3r4r5r6r7r8r9r:r$r$r%rl=s*     z#VocabParallelEmbedding._get_indicesc Cs|jdkrdSg}g}g}t|jD]g}||j|j|j|j||j}|j|}|j|d}|t|||j |t||j ||j |t||j ||j |j |t||j |j ||j |j ||j |j |ksyJq|||}t ||jksJ|S)aGet a mapping that can be used to reindex the gathered logits for sampling. During sampling, we gather logits from all ranks. The relationship of index->token_id will follow the same format as outlined in the class docstring. However, after the gather, we want to reindex the final logits tensor to map index->token_id one-to-one (the index is always equal the token_id it corresponds to). The indices returned by this method allow us to do that. Nr#)rgrangerlrkrjrbrhrvextendr>rAr@rBlen) r=base_embeddingsadded_embeddingspaddingrzrm range_start range_endretr$r$r%get_sharded_to_full_mappingcsv    z2VocabParallelEmbedding.get_sharded_to_full_mappingparam loaded_weightc Cst|dd}t|dd}t|ddr|j|||_dSt|tr@t|j}|dur6|||j ||<|j t ||j d|durU|jj|jksMJ|j|dS|j j}|j j|}|dur||krt|tro|jn|j}|j||j|jksJ||}||}n"|j||j|jr|j ndksJd|jd|jd|j||js||||}|d|jd j|||jd djd dS) N output_dim packed_dimis_gguf_weight_typedtyper#zself.org_vocab_size=z self.use_presharded_weights=z! loaded_weight.shape[output_dim]=r)getattrdatacopy_item weight_type isinstancerlistshaperg materializetuplerrmr7r8r packed_factorrhranarrowfill_) r=rrrrr start_idx shard_sizerr$r$r%rdsL           z$VocabParallelEmbedding.weight_loadercCs|jdkrt||jj|jj|jj|jj|jj\}}n|}tt t d|j || }Wdn1s:wY|jdkr^||ddtjs^|jrZt|}|St|}|S)Nr#)disabledr)rgrWrmr7r8rCr9r:rr rrr embeddinglong masked_fill_ unsqueezerinput_scatteredr`rr )r=rQ masked_input input_maskoutput_parallelr$r$r%forwards.     zVocabParallelEmbedding.forwardcCsVd|j}|d|j7}|d|j7}|d|j7}|jr)|d|j7}|S)Nznum_embeddings=z, embedding_dim=z, org_vocab_size=z, num_embeddings_padded=z , tp_size=)rvrcrhrkr_rg)r=sr$r$r% extra_reprs z!VocabParallelEmbedding.extra_repr)rGrHrIrJDEFAULT_VOCAB_PADDING_SIZErKrrsrrstrboolrf classmethodr2rlrrrTensorrdrr __classcell__r$r$r}r%rXsf+    u %A3rXcseZdZdZdddeddddddededed eej d eed ed ee d e dedeffddZ de fddZddZZS)ParallelLMHeadaParallelized LM head. Output logits weight matrices used in the Sampler. The weight and bias tensors are padded to make sure they are divisible by the number of model parallel GPUs. Args: num_embeddings: vocabulary size. embedding_dim: size of hidden state. bias: whether to use bias. params_dtype: type of the parameters. org_num_embeddings: original vocabulary size (without LoRA). padding_size: padding size for the vocabulary. FNrY)biasrZr[r\r]r^r`rarbrcrrZr[r\r]r^r`rac stj|||||||| | d ||_tr,tr,t|dr,|jjtj tj fvr,t dgd|_ |rEt tj|j|d|_t|jd|jddS|dddS)N)rZr[r\r]r^r`raweight) weight_namesrr)rrdr)rerfr]ri_is_cpu_amx_availablehasattrrrrsbfloat16float16rrrremptyrvrrrdregister_parameter) r=rbrcrrZr[r\r]r^r`rar}r$r%rfs: zParallelLMHead.__init__ embed_tokenscCs$|jr |jdkr |S|j|_|S)z%Tie the weights with word embeddings.gguf)r]get_namer)r=rr$r$r% tie_weightsAszParallelLMHead.tie_weightscCs ~td)Nz/LMHead's weights should be used in the sampler.) RuntimeError)r=rQr$r$r%rJszParallelLMHead.forward)rGrHrIrJrrKrrrsrrrrfrXrrrr$r$r}r%rsB    1 r)r)>logging dataclassesrtypingrrrrrstorch.nn.parameterrrsglang.srt.distributedr r r r r s           : a