o
    ºi‚  ã                   @  sŒ   d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
mZ d dlmZ ddlmZ d dlZe d¡ZeG d	d
„ d
ƒƒZdS )é    )ÚannotationsN)ÚPath)ÚAnyÚLiteralÚOptional)Ú	dataclassé   )ÚKeysÚmetadatac                   @  s¤  e Zd ZU dZded< dZded< dZded< dZded< dZded< dZ	ded< dZ
ded	< dZded
< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< ed>d?d%d&„ƒZed@dAd(d)„ƒZed@dBd*d+„ƒZed@dBd,d-„ƒZ ed.d/„ ƒZ!edCdDd2d3„ƒZ"ed>dEd8d9„ƒZ#dFd<d=„Z$dS )GÚMetadataNúOptional[str]ÚnameÚauthorÚversionÚorganizationÚfinetuneÚbasenameÚdescriptionÚquantized_byÚ
size_labelÚurlÚdoiÚuuidÚrepo_urlÚ
source_urlÚ
source_doiÚsource_uuidÚsource_repo_urlÚlicenseÚlicense_nameÚlicense_linkzOptional[list[dict]]Úbase_modelszOptional[list[str]]ÚtagsÚ	languagesÚdatasetsr   Úmetadata_override_pathúOptional[Path]Ú
model_pathÚ
model_nameÚtotal_paramsÚintÚreturnc                 C  s  t ƒ }t  |¡}t  |¡}t  |||||¡}t  | ¡}| tjj|j	¡|_	| tjj
|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj|j¡|_| tjj |j!¡|_!| tjj"|j#¡|_#| tjj$|j%¡|_%| tjj&|j'¡|_'| tjj(|j)¡|_)| tjj*|j+¡|_+| tjj,|j-¡|_-| d|j.¡|_.| d|j/¡|_/| tjj0|j1¡|_1| tjj2|j3¡|_3|d ur||_	|S )Nzgeneral.base_modelszgeneral.datasets)4r   Úload_model_cardÚload_hf_parametersÚapply_metadata_heuristicÚload_metadata_overrideÚgetr	   ÚGeneralÚNAMEr   ÚAUTHORr   ÚVERSIONr   ÚORGANIZATIONr   ÚFINETUNEr   ÚBASENAMEr   ÚDESCRIPTIONr   ÚQUANTIZED_BYr   Ú
SIZE_LABELr   ÚLICENSE_NAMEr   ÚLICENSE_LINKr    ÚURLr   ÚDOIr   ÚUUIDr   ÚREPO_URLr   Ú
SOURCE_URLr   Ú
SOURCE_DOIr   ÚSOURCE_UUIDr   ÚSOURCE_REPO_URLr   r!   r$   ÚTAGSr"   Ú	LANGUAGESr#   )r%   r'   r(   r)   r
   Ú
model_cardÚ	hf_paramsÚmetadata_override© rJ   úJ/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/gguf/metadata.pyÚload.   s>   



zMetadata.loadúdict[str, Any]c                 C  sP   | d u s|   ¡ s
i S t| ddd}t |¡W  d   ƒ S 1 s!w   Y  d S )NÚrúutf-8©Úencoding)Úis_fileÚopenÚjsonrL   )r%   ÚfrJ   rJ   rK   r/   j   s
   $ÿzMetadata.load_metadata_overridec           	      C  s2  | d u s|   ¡ s
i S | d }| ¡ si S d}t|dddQ}| ¡ }| ¡ }g }t|ƒdkr7i W  d   ƒ S t|ƒdkrL|d dkrLi W  d   ƒ S |dd … D ]}|dkrZ n| |¡ qRd	 |¡d	 }W d   ƒ n1 sqw   Y  | d
d¡}|r—t	 
|¡}t|tƒrŠ|S t dt|ƒ› d¡ i S i S )Nz	README.mdÚ rN   rO   rP   r   z---r   Ú
z- no
z- "no"
z3while reading YAML model card frontmatter, data is z instead of dict)Úis_dirrR   rS   ÚreadÚ
splitlinesÚlenÚappendÚjoinÚreplaceÚyamlÚ	safe_loadÚ
isinstanceÚdictÚloggerÚerrorÚtype)	r'   Úmodel_card_pathÚyaml_contentrU   ÚcontentÚlinesÚ
lines_yamlÚlineÚdatarJ   rJ   rK   r,   r   s<   ú÷
ñ

zMetadata.load_model_cardc                 C  sd   | d u s|   ¡ s
i S | d }| ¡ si S t|ddd}t |¡W  d   ƒ S 1 s+w   Y  d S )Nzconfig.jsonrN   rO   rP   )rX   rR   rS   rT   rL   )r'   Úconfig_pathrU   rJ   rJ   rK   r-   ž   s   $ÿzMetadata.load_hf_parametersc                 C  s$   d  dd„ |  ¡  dd¡ ¡ D ƒ¡S )Nú c                 S  s,   g | ]}|  ¡ rt d |¡s| ¡ n|‘qS )z^(v\d+(?:\.\d+)*|\d.*)$)ÚislowerÚreÚmatchÚtitle)Ú.0ÚwrJ   rJ   rK   Ú
<listcomp>®   s   , z(Metadata.id_to_title.<locals>.<listcomp>ú-)r]   Ústripr^   Úsplit)ÚstringrJ   rJ   rK   Úid_to_title«   s   $zMetadata.id_to_titleÚmodel_idúMtuple[str | None, str | None, str | None, str | None, str | None, str | None]c              	   C  s”  | d u rdS d| v r| d d d d d fS d| v r|   dd¡\}}nd | }}|d ur6t|ƒdkr6|d dkr6d }|  d¡}ttt|ƒƒƒD ]}t|| ƒdkrP||= qCdd	„ |D ƒ}t|ƒD ]\}}t d
|tj¡rq||  d¡ q\t d|tj¡r‡||  d¡ | 	¡ ||< q\|dkr:t d|tj¡r:| 
dd¡}|d  ¡ r±|d d… d |d  |d  }t|ƒdkrÏ|d  ¡ rÏ|d dv rÏ|d d… |d  	¡  }|dkr%zFt|d d… ƒtdd |d ¡ƒ }|dk rô|t|ƒd k s|dkrt|| ƒd| d kr||  d¡ |d d… |d  ¡  }W n
 ty$   Y nw t|| ƒdkr5||  d¡ |||< q\|dkrct d|tj¡rc|dk r\| ¡ dkr\||  d¡ q\||  d¡ q\tdd„ t||ƒD ƒƒrt||ƒD ]\}	}
d|
v rŽtdd„ |	D ƒƒrŽ|
 d¡ qvd}t||ƒD ]3\}}
|r¸t|
ƒdkr¬|d  ¡ s±d|
v r¸|
 d¡ q—|r½d }t|
ƒdkrÉ|
 d¡ q—tt|ƒt|ƒƒD ]\}}
d|
v rët|
ƒdkrë|
 d¡ qÔ d d!d„ t||ƒD ƒ¡pûd }d t d"d„ t||ƒD ƒ¡ ¡ ¡pd }d d#d„ t||ƒD ƒ¡p d }d d$d„ t||ƒD ƒ¡p0d }|d u rB|d u rB|d u rBd }||||||fS )%N)NNNNNNrn   ú/r   r   Ú.rv   c                 S  s   g | ]}t ƒ ‘qS rJ   )Úset)rs   Ú_rJ   rJ   rK   ru   Ó   s    z4Metadata.get_model_id_components.<locals>.<listcomp>z(v|iter)?\d+([.]\d+)*r   zi?q\d(_\w)*|b?fp?(16|32)re   zD(([A]|\d+[x])?\d+([._]\d+)?[KMBT][\d]?|small|mini|medium|large|x?xl)r€   éÿÿÿÿéþÿÿÿÚkmbtiè  z KMBTé   é   r   r   zchat|instruct|vision|loraÚlorac                 s  s.    | ]\}}d |v r|D ]}|  ¡ V  qqdS ©r   N)Ú	isdecimal)rs   ÚnÚtÚcrJ   rJ   rK   Ú	<genexpr>  s   €, z3Metadata.get_model_id_components.<locals>.<genexpr>c                 s  s    | ]}|  ¡ V  qd S ©N)Úisalpha)rs   r‹   rJ   rJ   rK   rŒ   
  s   € Tr   Fc                 s  ó     | ]\}}d |v r|V  qdS )r   NrJ   )rs   r‰   rŠ   rJ   rJ   rK   rŒ     ó   € c                 s  r   r‡   rJ   )rs   ÚsrŠ   rJ   rJ   rK   rŒ   !  r   c                 s  r   )r   NrJ   )rs   rU   rŠ   rJ   rJ   rK   rŒ   "  r   c                 s  s(    | ]\}}d |v rd|vr|V  qdS )r   r   NrJ   )rs   ÚvrŠ   rJ   rJ   rK   rŒ   %  s   €& )rx   r[   ÚreversedÚrangeÚ	enumeraterp   Ú	fullmatchÚ
IGNORECASEÚaddÚupperr^   rˆ   ÚfloatÚpowÚfindÚabsÚlowerÚ
ValueErrorÚanyÚzipÚallÚremoverŽ   r]   rb   ÚfromkeysÚkeys)r{   r)   Úorg_componentÚmodel_full_name_componentÚ
name_partsÚiÚ
name_typesÚpartÚlabel_paramsr‰   rŠ   Úat_startr   r   r   r   rJ   rJ   rK   Úget_model_id_components°   s”   
 
€þ 
$$€þ
€

€,
€ *  z Metadata.get_model_id_componentsr
   rG   úOptional[dict]rH   c              
     sF  ˆd ur´d@‡ ‡fdd„}d@‡ ‡fdd„}|ddƒ |d	d	ƒ |d
d
ƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |d	dƒ |d
dƒ |ddƒ |ddƒ |ddƒ |ddƒ |ddƒ |dd ƒ |dd!ƒ |dd"ƒ |dd#ƒ |ddƒ |d	d$ƒ |dd%ƒ d&ˆv s©d'ˆv s©d(ˆv r˜g }ˆ  d&ˆ  d'ˆ  d(d ¡¡¡}|d urÒt|tƒrÈ| |¡ n
t|tƒrÒ| |¡ ˆ jd u rÚg ˆ _|D ]»}	i }
t|	tƒr}|	 d)¡sö|	 d*¡sö|	 d+¡r;|	|
d< d,|	v r:t 	d-|	¡}|r:| 
d.¡}t ||¡\}}}}}}|d ur%t |¡|
d< |d ur1t |¡|
d< |d ur:||
d
< nVt |	|¡\}}}}}}|d urSt |¡|
d< |d ur_t |¡|
d< |d urh||
d
< |d ur||d ur|d/|› d0|› |
d< nt|	tƒr†|	}
nt d1t|	ƒ› d2¡ ˆ j |
¡ qÜd3ˆv s§d4ˆv s§d5ˆv r‘g }ˆ  d3ˆ  d4ˆ  d5d ¡¡¡}|d urÓt|tƒrÈ| |¡ nt|tƒrÓ| |¡ ˆ jd u rÜg ˆ _|D ]²}i }t|tƒru| d6¡r3||d< d,|v r2t 	d-|¡}|r2| 
d.¡}t ||¡\}}}}}}|d urt |¡|d< |d ur)t |¡|d< |d ur2||d
< nVt ||¡\}}}}}}|d urKt |¡|d< |d urWt |¡|d< |d ur`||d
< |d urt|d urtd/|› d0|› |d< nt|tƒr~|}nt d7t|ƒ› d2¡ ˆ j |¡ qÞ|d8d8ƒ |d9d9ƒ |d:d:ƒ |d;d;ƒ |d;d<ƒ |d=d=ƒ |d=d>ƒ |d ur3|  d?¡}|d ur3| d0¡d.kr3|}	t |	|¡\}}}}}}ˆ jd u rê|d urêt |¡ˆ _ˆ jd u rû|d urût |¡ˆ _ˆ jd u r	|d ur	|ˆ _ˆ jd u r|d ur|ˆ _ˆ jd u r%|d ur%|ˆ _ˆ jd u r3|d ur3|ˆ _|d ur¡|j}	t |	|¡\}}}}}}ˆ jd u rX|d urXt |¡ˆ _ˆ jd u ri|d urit |¡ˆ _ˆ jd u rw|d urw|ˆ _ˆ jd u r…|d ur…|ˆ _ˆ jd u r“|d ur“|ˆ _ˆ jd u r¡|d ur¡|ˆ _ˆ S )ANÚmetadata_keyÚstrÚmodel_card_keyc                   s6   |ˆv rt ˆ | d ƒd u rtˆ | ˆ |¡ƒ d S d S d S r   )ÚgetattrÚsetattrr0   )r°   r²   ©r
   rG   rJ   rK   Úuse_model_card_metadata5  s   ÿzBMetadata.apply_metadata_heuristic.<locals>.use_model_card_metadatac                   sj   ˆ  |d ¡}|d u rd S tˆ | d ƒ}|d u rg }t|tƒr#| |¡ n
t|tƒr-| |¡ tˆ | |ƒ d S r   )r0   r³   ra   r±   r\   ÚlistÚextendr´   )r°   r²   Ú
tags_valueÚcurrent_valuerµ   rJ   rK   Úuse_array_model_card_metadata9  s   


zHMetadata.apply_metadata_heuristic.<locals>.use_array_model_card_metadatar   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r(   Úmodel_authorÚmodel_versionÚmodel_organizationÚmodel_descriptionÚmodel_finetuneÚmodel_basenameÚmodel_size_labelÚ	model_urlÚ	model_doiÚ
model_uuidÚmodel_repo_urlÚmodel_creatorÚ
model_typeÚ
base_modelr!   Úbase_model_sourcesúhttp://úhttps://ússh://zhuggingface.coz&https?://huggingface.co/([^/]+/[^/]+)$r   zhttps://huggingface.co/r}   zbase model entry 'z' not in a known formatr$   ÚdatasetÚdataset_sources)rË   rÌ   rÍ   zdataset entry 'r   r   r    r"   Úpipeline_tagr#   ÚlanguageÚ_name_or_path)r°   r±   r²   r±   )r0   ra   r±   r\   r·   r¸   r!   Ú
startswithrp   rq   Úgroupr   r®   rz   rb   rc   rd   r$   Úcountr   r   r   r   r   r   )r
   rG   rH   r'   r)   r¶   r»   Úmetadata_base_modelsÚbase_model_valuer{   rÉ   rq   Úmodel_id_componentr§   r¦   r   r   r   r   Úmetadata_datasetsÚdataset_valueÚ
dataset_idrÎ   Údataset_id_componentÚdataset_name_componentÚhf_name_or_pathrJ   rµ   rK   r.   -  s0  































 




€


€






€


€









z!Metadata.apply_metadata_heuristicÚgguf_writerúgguf.GGUFWriterc                 C  sL  | j d usJ ‚| | j ¡ | jd ur| | j¡ | jd ur#| | j¡ | jd ur.| | j¡ | jd ur9| 	| j¡ | j
d urD| | j
¡ | jd urO| | j¡ | jd urZ| | j¡ | jd ure| | j¡ | jd ur€t| jtƒrz| d | j¡¡ n| | j¡ | jd ur‹| | j¡ | jd ur–| | j¡ | jd ur¡| | j¡ | jd ur¬| | j¡ | jd ur·|  | j¡ | j!d urÂ| "| j!¡ | j#d urÍ| $| j#¡ | j%d urØ| &| j%¡ | j'd urã| (| j'¡ | j)d urî| *| j)¡ | j+d ur|| ,t-| j+ƒ¡ t.| j+ƒD ]z\}}d|v r| /||d ¡ d|v r| 0||d ¡ d|v r,| 1||d ¡ d|v r9| 2||d ¡ d|v rF| 3||d ¡ d|v rS| 4||d ¡ d|v r`| 5||d ¡ d	|v rm| 6||d	 ¡ d
|v rz| 7||d
 ¡ q| j8d ur
| 9t-| j8ƒ¡ t.| j8ƒD ]z\}}d|v r | :||d ¡ d|v r­| ;||d ¡ d|v rº| <||d ¡ d|v rÇ| =||d ¡ d|v rÔ| >||d ¡ d|v rá| ?||d ¡ d|v rî| @||d ¡ d	|v rû| A||d	 ¡ d
|v r| B||d
 ¡ q| jCd ur| D| jC¡ | jEd ur$| F| jE¡ d S d S )Nú,r   r   r   r   r   r   r   r   r   )Gr   Úadd_namer   Ú
add_authorr   Úadd_versionr   Úadd_organizationr   Úadd_finetuner   Úadd_basenamer   Úadd_descriptionr   Úadd_quantized_byr   Úadd_size_labelr   ra   r·   Úadd_licenser]   r   Úadd_license_namer    Úadd_license_linkr   Úadd_urlr   Úadd_doir   Úadd_uuidr   Úadd_repo_urlr   Úadd_source_urlr   Úadd_source_doir   Úadd_source_uuidr   Úadd_source_repo_urlr!   Úadd_base_model_countr[   r•   Úadd_base_model_nameÚadd_base_model_authorÚadd_base_model_versionÚadd_base_model_organizationÚadd_base_model_descriptionÚadd_base_model_urlÚadd_base_model_doiÚadd_base_model_uuidÚadd_base_model_repo_urlr$   Úadd_dataset_countÚadd_dataset_nameÚadd_dataset_authorÚadd_dataset_versionÚadd_dataset_organizationÚadd_dataset_descriptionÚadd_dataset_urlÚadd_dataset_doiÚadd_dataset_uuidÚadd_dataset_repo_urlr"   Úadd_tagsr#   Úadd_languages)Úselfrß   ÚkeyÚbase_model_entryÚdataset_entryrJ   rJ   rK   Úset_gguf_meta_model  s¶   



























€








€ÿzMetadata.set_gguf_meta_model)NNNr   )
r%   r&   r'   r&   r(   r   r)   r*   r+   r   r   )r%   r&   r+   rM   )r'   r&   r+   rM   )Nr   )r{   r   r)   r*   r+   r|   )r
   r   rG   r¯   rH   r¯   r'   r&   r)   r*   r+   r   )rß   rà   )%Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   ÚstaticmethodrL   r/   r,   r-   rz   r®   r.   r  rJ   rJ   rJ   rK   r      sR   
 ;+
| rr   )Ú
__future__r   rp   rT   r_   ÚloggingÚpathlibr   Útypingr   r   r   Údataclassesr   Ú	constantsr	   ÚggufÚ	getLoggerrc   r   rJ   rJ   rJ   rK   Ú<module>   s    
