o [۷i4@sddlmZddlmZddlmZddlmZddlZddl Z ddl Z d+d d Z d,d-ddZ d.ddZd/d0dd ZeGd!d"d"ZGd#d$d$ZeGd%d&d&ZeGd'd(d(ZGd)d*d*ZdS)1) annotations) dataclass)Path)LiteralNfilenamestr output_type str | NonereturncCs<|dur|nd}|dur|nd}|j|||||dS)N)outtypeftypeOUTTYPEFTYPE)lowerupperformat)rrftype_lowercaseftype_uppercaserB/home/ubuntu/vllm_env/lib/python3.10/site-packages/gguf/utility.pyfill_templated_filename s rmodel_params_countint min_digitscCs|dkr |d}d}n|dkr|d}d}n|dkr!|d}d }n|d }d }t|ttt|d d }|d|d|S)NgmBg-q=TgeAg& .>Bg.Agư>MgMbP?K0r.f)maxlenrroundlstrip)rrscaled_model_params scale_suffixfixrrr#model_weight_count_rounded_notations r* total_params shared_params expert_params expert_countcCsF|dkrtt|t|dd}|d|}|Stt|dd}|S)Nrr)rx)r*abs)r+r,r-r. pretty_size size_classrrr size_label,s r3 model_name base_namefinetune_stringversion_string model_typeLiteral['vocab', 'LoRA'] | Nonec Cs|dur|dddd}n|dur"|dddd}nd}|dur-d|nd}|dur>d|ddnd} |durOd|ddnd} |durbd|ddnd} |dursd|ddnd} ||| | | | S)N -/z ggml-modelr )stripreplacer) r4r5r6r7r3rr8name parametersfinetuneversionencodingkindrrrnaming_convention7s""&"rEc@s@eZdZUded<ded<ded<ded<ded<dd d Zd S) RemoteTensorrdtypetuple[int, ...]shaper offset_startsizeurlr bytearraycCsttj|j|j|jd}|S)N)rLstartrK)rMSafetensorRemoteget_data_by_rangerLrJrK)selfdatarrrrRVszRemoteTensor.dataN)r rM)__name__ __module__ __qualname____annotations__rRrrrrrFNs rFc@sjeZdZdZdZedddZedd d Zedd dZed d!ddZ ed"ddZ ed#ddZ dS)$rOa Uility class to handle remote safetensor files. This class is designed to work with Hugging Face model repositories. Example (one model has single safetensor file, the other has multiple): for model_id in ["ngxson/TEST-Tiny-Llama4", "Qwen/Qwen2.5-7B-Instruct"]: tensors = SafetensorRemote.get_list_tensors_hf_model(model_id) print(tensors) Example reading tensor data: tensors = SafetensorRemote.get_list_tensors_hf_model(model_id) for name, meta in tensors.items(): dtype, shape, offset_start, size, remote_safetensor_url = meta # read the tensor data data = SafetensorRemote.get_data_by_range(remote_safetensor_url, offset_start, size) print(data) zhttps://huggingface.comodel_idrr dict[str, RemoteTensor]cCs||jd|d}|r|jd|d}||S|jd|d}||}|r{||d}|d}t|}|ddusGJd|d} tt | } | i} | D]} |jd|d | }|| D]\} }|| | <qoq[| St d |d ) z Get list of tensors from a Hugging Face model repository. Returns a dictionary of tensor names and their metadata. Each tensor is represented as a tuple of (dtype, shape, offset_start, size, remote_safetensor_url) r<z/resolve/main/model.safetensorsz*/resolve/main/model.safetensors.index.jsonrutf-8 weight_mapNz"weight_map not found in index filez/resolve/main/z,No safetensor file has been found for model z.If the repo has safetensor files, make sure the model is public or you have a valid Hugging Face token set in the environment variable HF_TOKEN.)check_file_exist BASE_DOMAINget_list_tensorsrPdecodejsonloadsgetlistsetvaluessortitems ValueError)clsrWis_single_filerL index_urlis_multiple_files index_data index_str index_jsonrZ all_filestensorsfilekeyvalrrrget_list_tensors_hf_modelrs0        z*SafetensorRemote.get_list_tensors_hf_modelrLc Cs||\}}i}|D]V\}}|dkrq t|ts%td|d|z%|d}|d}|d\} } | | } || } t|t|| | |d||<Wq tyc} z td|d| d |d } ~ wwtt|d d d }|S)z Get list of tensors from a remote safetensor file. Returns a dictionary of tensor names and their metadata. Each tensor is represented as a tuple of (dtype, shape, offset_start, size) __metadata__zInvalid metadata for tensor 'z': rGrI data_offsets)rGrIrJrKrLz$Missing key in metadata for tensor 'z , meta = NcS|dSNrrtrrrz3SafetensorRemote.get_list_tensors..rr) get_metadatarf isinstancedictrgrFtupleKeyErrorsorted)rhrLmetadatadata_start_offsetresr?metarGrIoffset_start_relativeoffset_end_relativerKrJerrrr]s(  z!SafetensorRemote.get_list_tensorstuple[dict, int]c Csd}||d|}t|dkrtdtj|dddd}d|}t|d|kr8tdd|d t||dd|}|d }z t|}||fWStjyc} ztd | d} ~ ww) zz Get JSON metadata from a remote safetensor file. Returns tuple of (metadata, data_start_offset) iPrz%Not enough data to read metadata sizeNlittle byteorder'Could not read complete metadata. Need bytes, got rYz-Failed to parse safetensor metadata as JSON: ) rPr$rgr from_bytesr^r_r`JSONDecodeError) rhrL read_sizeraw_datametadata_lengthrmetadata_bytes metadata_strrrrrrr~s"    zSafetensorRemote.get_metadatarNrrKbytesc Csddl}ddlm}||}|jr|jstd||}|dkr/d|d|||d<|j|d |d }||j t |dkrF|SdS) z Get raw byte data from a remote file by range. If size is not specified, it will read the entire file. rNurlparse Invalid URL: rzbytes=r;RangeTallow_redirectsheaders) requests urllib.parserschemenetlocrg_get_request_headersraraise_for_statuscontentslice) rhrLrNrKrr parsed_urlrresponserrrrPs  z"SafetensorRemote.get_data_by_rangeboolcCsddl}ddlm}||}|jr|jstd|z|}d|d<|j|d|d}d |jko7d kWSWS|j yEYd Sw) zt Check if a file exists at the given URL. Returns True if the file exists, False otherwise. rNrrz bytes=0-0rTriF) rrrrrrgrhead status_codeRequestException)rhrLrrrrrrrrr[s  z!SafetensorRemote.check_file_existdict[str, str]cCs,ddi}tjdrdtjd|d<|S)z$Prepare common headers for requests.z User-Agentconvert_hf_to_ggufHF_TOKENzBearer Authorization)osenvironra)rhrrrrrs z%SafetensorRemote._get_request_headersN)rWrr rX)rLrr rX)rLrr r)r)rLrrNrrKrr r)rLrr r)r r) rSrTrU__doc__r\ classmethodrtr]r~rPr[rrrrrrO]s (    rOc@s&eZdZUded<ded<ded<dS)LocalTensorRangerrroffsetrKN)rSrTrUrVrrrrrs  rc@s0eZdZUded<ded<ded<d d d Zd S) LocalTensorrrGrHrIr data_ranger np.ndarraycCstj|jjd|jj|jjdS)Nc)moderrI)npmemmaprrrrK)rQrrr mmap_bytesszLocalTensor.mmap_bytesN)r r)rSrTrUrVrrrrrrs rc@s4eZdZUdZded<d ddZdd Zd d Zd S)SafetensorsLocalz Read a safetensors file from the local filesystem. Custom parsing gives a bit more control over the memory usage. The official safetensors library doesn't expose file ranges. zdict[str, LocalTensor]rprrc CsDt|d}tj|ddd}t|j}|d|kr(tdd|d|||d}zt |}Wnt j yK}ztd|d}~ww| }i} | D],\} } | d kr_qVt| d t| d t||| d d | d d| d d d| | <qVtt| ddd|_WddS1swYdS)NrbrrrrrrYz.Failed to parse safetensors metadata as JSON: rurGrIrvr)rGrIrcSrwrxrryrrrr{Mr|z+SafetensorsLocal.__init__..r})openrrreadrstatst_sizergr^r_r`rtellrfrrrrrrp) rQrr"r file_sizerrrrrpr?rrrr__init__,s8      "zSafetensorsLocal.__init__cOs ~~|jSN)rprQargskwargsrrr __enter__OszSafetensorsLocal.__enter__cOs~~dSrrrrrr__exit__SszSafetensorsLocal.__exit__N)rr)rSrTrUrrVrrrrrrrr"s  # r)rrrr r r)r)rrrrr r) r+rr,rr-rr.rr rr)r4r r5r r6r r7r r3r rr r8r9r r) __future__r dataclassesrpathlibrtypingrrr_numpyrrr*r3rErFrOrrrrrrrs(      5