o i.W @sddlZddlmZmZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZe r9ddlZeeZd d d d d ddde de dide didZeedZGdddeZGdddZGdddeZGdddeZGdddeZGdd d eZGd!d"d"eZGd#d$d$eZ Gd%d&d&eZ!Gd'd(d(eZ"Gd)d*d*eZ#eeeeeeee e!e"e"e#d+ Z$d,d-Z%   .d7d/ee&d0ee'd1e&fd2d3Z(d8d5d6Z)dS)9N) NamedTupleOptional)tqdm)GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPING_gguf_parse_value)is_torch_available)is_gguf_available) get_loggerversion tensor_countkv_count)r r r file_typequantization_version)rr)GGUFgeneral tokenizertokenizer_config)ignoreconfigrrrc@s(eZdZUejed<eed<eed<dS) GGUFTensorweightsnamemetadataN)__name__ __module__ __qualname__npndarray__annotations__strdictr#r#\/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_gguf_pytorch_utils.pyr7s  rc@seZdZdddZddZdS)TensorProcessorNcCs|pi|_dSNrselfrr#r#r$__init__>szTensorProcessor.__init__cKs t||iSr&rr)rrkwargsr#r#r$processAs zTensorProcessor.processr&)rrrr*r.r#r#r#r$r%=s  r%c sLeZdZd fdd ZddZ d dejdedeed ejfd d Z Z S) LlamaTensorProcessorNctj|ddSNr'superr*r( __class__r#r$r*FzLlamaTensorProcessor.__init__cKszd|vsd|vr7|jd}|jd}d||fvr t||iSd|vr,||||}n d|vr7||||}t||iS)Nz.attn_k.z.attn_q.num_attention_headsnum_key_value_heads)rgetr_reverse_permute_weights)r)rrr- num_heads num_kv_headsr#r#r$r.Is     zLlamaTensorProcessor.processrn_headr<returncCsZ|dur ||kr |}|jd|d}|j||dg|jddR}|dd|jS)Nrr)shapereshapeswapaxes)r)rr=r<dimwr#r#r$r:Vs  z-LlamaTensorProcessor._reverse_permute_weightsr&) rrrr*r.rrintrr: __classcell__r#r#r4r$r/Esr/csJeZdZd fdd ZddZdejdeeefded efd d Z Z S) Qwen2MoeTensorProcessorNcr0r1r2r(r4r#r$r*dr6z Qwen2MoeTensorProcessor.__init__cKs^d|vr|d}|d}|r|||||t|diSd|vr)tj|dd}t||iS)N_exptensor_key_mappingparsed_parametersffn_gate_inp_shexpraxis)r9_split_moe_expert_tensorrr expand_dims)r)rrr-rIrJr#r#r$r.gs    zQwen2MoeTensorProcessor.processrrJrrIc Cs^||}|jdd}td|D]}|dd|d}||}tt||d|<qdS)N num_experts<r mlp.experts..tensors)rr9rangereplacetorch from_numpyrcopy) r)rrJrrI w_counteri temp_name exp_weightr#r#r$rNtsz0Qwen2MoeTensorProcessor._split_moe_expert_tensorr&) rrrr*r.rrr"r!rNrFr#r#r4r$rGcs  rGcsVeZdZd fdd ZddZdejdedefd d Zdejdedefd d Z Z S)BloomTensorProcessorNcr0r1r2r(r4r#r$r*r6zBloomTensorProcessor.__init__cKsNd|vr!|jd}|jd}d|vr||||}n||||}t||iS)Nattn_qkvr= hidden_sizeweight)r_reverse_reshape_weights_reverse_reshape_biasr)r)rrr-r;n_embedr#r#r$r.s   zBloomTensorProcessor.processrr=rdcCsxtj|ddd\}}}|||||}|||||}|||||}tj|||gdd}||d|||S)NrrLr)r array_splitrAstack)r)rr=rdqkv qkv_weightsr#r#r$rbs z-BloomTensorProcessor._reverse_reshape_weightscCs^t|d\}}}||||}||||}||||}tj|||gdd}|S)NrerrL)rrfrArgflatten)r)rr=rdq_biask_biasv_biasqkv_biasr#r#r$rcs z*BloomTensorProcessor._reverse_reshape_biasr&) rrrr*r.rrrErbrcrFr#r#r4r$r^s  r^c&eZdZdfdd ZddZZS)T5TensorProcessorNcr0r1r2r(r4r#r$r*r6zT5TensorProcessor.__init__cKs8d}|dD] }|rt|}nqt||d|iS)NrSbid)splitisdigitrEr)r)rrr-rschunkr#r#r$r.szT5TensorProcessor.processr&rrrr*r.rFr#r#r4r$rrrrcrq)GPT2TensorProcessorNcr0r1r2r(r4r#r$r*r6zGPT2TensorProcessor.__init__cKsfd|vsd|vsd|vsd|vr|j}|dkr-d}|di}tt||d|<d}t||iS) Nzattn_qkv.weightzffn_down.weightz ffn_up.weightzattn_output.weight output.weightzlm_head.weightrJrT)Tr9rWrXrrYr)r)rrr-rJr#r#r$r.s  zGPT2TensorProcessor.processr&rwr#r#r4r$ryrxrycrq)MambaTensorProcessorNcr0r1r2r(r4r#r$r*r6zMambaTensorProcessor.__init__cKs6d|vr tj|dd}d|vrt| }t||iS)Nzssm_conv1d.weightrrLssm_a)rrOlogrr,r#r#r$r.s   zMambaTensorProcessor.processr&rwr#r#r4r$r|rxr|crq)NemotronTensorProcessorNcr0r1r2r(r4r#r$r*r6z NemotronTensorProcessor.__init__cKd|vr|d}t||iSNz norm.weightrr+r,r#r#r$r. zNemotronTensorProcessor.processr&rwr#r#r4r$rsrcrq)Gemma2TensorProcessorNcr0r1r2r(r4r#r$r*r6zGemma2TensorProcessor.__init__cKrrr+r,r#r#r$r.rzGemma2TensorProcessor.processr&rwr#r#r4r$rsrcrq)Lfm2TensorProcessorNcr0r1r2r(r4r#r$r*r6zLfm2TensorProcessor.__init__cKs"d|vr tj|dd}t||iS)Nzshortconv.conv.weightrrL)rrOrr,r#r#r$r.s zLfm2TensorProcessor.processr&rwr#r#r4r$rrxr) llamaqwen2moeqwen3moebloomt5 t5encodergpt2mambanemotrongemma2gemma3lfm2cs,||jvrgS|j|fddjDS)Ncg|] }tj|jqSr#rpartstypes.0 _data_indexvaluer#r$ zread_field..)fieldsdata)readerfieldr#rr$ read_fields  r model_type num_layers qual_namecstrtrddlm}m}n tdtd|dur |jj n|}|dur*|jj n|}|dkr3d}n|dkr:d }n|d krAd }n |d krHd }n|dkrNd}d}| D] \}}||kr`|}nqT|durmt d|d|||} i| } | D]>} |dvrd| vrtdd| } | d} } | ds| dr| dd\} } d| } | | }|durqz|| || <qz|}r|D]!\} }t||||| dd}fdd| D}|qS)aY GGUF uses this naming convention for their tensors from HF checkpoint: `blk.N.BB.weight` and `blk.N.BB.bias` where N signifies the block number of a layer, and BB signifies the attention/mlp layer components. See "Standardized tensor names" in https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details. r)MODEL_ARCH_NAMESget_tensor_name_mapLoading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions.KPlease install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.Ncoherez command-r qwen2_moer qwen3_moer gemma3_textrumt5rzUnknown gguf model_type: z in gguf-py. This might because you're using an outdated version of gguf-py package, you can install `gguf` package from source refer to https://github.com/ggerganov/llama.cpp/tree/master/gguf-py#development)rrrRzmlp.experts.\d+.rz.weightz.biasrSr)rcsi|] \}}|vr||qSr#r#)rrirjgguf_to_hf_name_mapr#r$ erz+get_gguf_hf_weights_map..)r r ggufrrloggererror ImportErrorrrnum_hidden_layersitemsNotImplementedError state_dictresubendswithrsplitget_namenamed_childrenget_gguf_hf_weights_mapupdate)hf_modelrrrrrarchkeyrname_maprhf_namersuffix gguf_namerchildsub_mapr#rr$rs`        rFc!s\trtrddlm}m}n tdtd||}|j}t | }ddt D}t |dd} t |d} d } d | vrEd | vrEd } n6d | vsMd | vryd|dd<| rkd| d vrkd} d | vrjdg|dd<nd | vrvdg|dd<d } n| } d| vrd} nd| vrd} d| vrhddtfdd|jD} tfdd|jD} | |dd<| |dd <| tvr| tvrtd!| d"d#d$g}td%d|jDp| |v|dd&<|jD]\}|| | }|d'}|d}d'|d(d }fd)d*jD}t|d(kr|d}t|tr$| |vr$|| | }t D]3\}}||vrZ|||vrZ|||}|d+krEq(|d urP||||<||vrZ||q(||vrltd,|d-|q|dd.d/kr|d0|dd.<|dd.d1kr|dd2}t||dd2<d3|dd4<d5d*t|D|dd6<d7|dvr|d8}d9|vrt|d9|dd7<ntd:|ri|d;<t |}|!di}t"!| t#}||d<}t$|jd=d>D]3}|j%}||j|j&}|j'||||d?} | j(}| j%}||vr q||}t)*t+,||d;|<qt|dkr,td@||S)Aa Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. Args: gguf_checkpoint_path (`str`): The path the to GGUF file to load return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. r) GGUFReader dequantizerrcSsi|]}|iqSr#r#)rrir#r#r$rsz(load_gguf_checkpoint..zgeneral.architecturez general.nameNrmistralrrTr is_gated_actrUMT5EncoderModel architecturesT5EncoderModelrrrrstablelm> attn_k.bias attn_q.bias attn_v.biasffn_normc3s$|] }D]}||jvVqqdSr&r)rtensor bias_name)attn_bias_namer#r$ s"z'load_gguf_checkpoint..c3s|]}|jvVqdSr&rrr) ffn_norm_namer#r$r use_qkv_biasuse_parallel_residualzGGUF model with architecture z is not supported yet.falconrcss|]}d|jkVqdS)rzNrrr#r#r$rrtie_word_embeddingsrSrcrr#rr)rr#r$rrz(load_gguf_checkpoint..z1Some keys were not parsed and added into account z | rrrrr8Fblock_auto_adjust_ff_dimcSsg|] \}}|dkr|qS)rr#)rr[r<r#r#r$rsfull_attn_idxs vocab_sizertokenszCan't find a way to retrieve missing config vocab_size from tokenizer parameters. This will use default value from model config class and cause unexpected behavior.rTr'z,Converting and de-quantizing GGUF tensors...)desc)rrrIrJz0Some keys of the GGUF file were not considered: )-r r rrrrrrrlistkeysGGUF_TO_TRANSFORMERS_MAPPINGrloweranyrTGGUF_SUPPORTED_ARCHITECTURES ValueErrorallrrVrtjoinrlen isinstancer!removeinfomax enumeratewarningrr9TENSOR_PROCESSORSr%rr tensor_typer.rrWrXrrY)!gguf_checkpoint_pathreturn_tensors model_to_loadrrrr reader_keysrJ architecture model_nameupdated_architecturerpr exceptionsgguf_keyrtprefix config_keyr parameterparameter_renamesrenamed_config_keygguf_num_key_value_headstokenizer_parametersrIrProcessorClass processorrrrresultr#)rrrr$load_gguf_checkpointks                        r )NNr)FN)*rtypingrrnumpyr tqdm.autor integrationsrrrutilsr utils.import_utilsr utils.loggingr rWrrrrrrrr%r/rGr^rrryr|rrrrrr!rErr r#r#r#r$st      '      R