o
    پi                  	   @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZ ddlmZmZmZmZ e
G dd dZe
G d	d
 d
Zg dZdddedededdfddZdededdfddZdddedededefddZdedee fddZdedefddZd edeeef fd!d"Zdeeeef  fd#d$Zded%ee dedefd&d'Zdefd(d)ZG d*d+ d+e Z!d,d- Z"d.d/ Z#e$d0kre#  dS dS )1a.  
Model File Verifier - Verify model file integrity using SHA256 checksums.

Example commands:
    # Verify using HuggingFace model online metadata
    python -m sglang.srt.utils.model_file_verifier verify --model-path /path/to/model --model-checksum Qwen/Qwen3-0.6B

    # Verify using locally generated checksum
    python -m sglang.srt.utils.model_file_verifier generate --model-path <hf-id-or-model-path> --model-checksum checksums.json
    python -m sglang.srt.utils.model_file_verifier verify --model-path /path/to/model --model-checksum checksums.json
    N)ThreadPoolExecutor)asdict	dataclass)Path)DictListOptionalTuplec                   @   s   e Zd ZU eed< eed< dS )FileInfosha256sizeN)__name__
__module____qualname__str__annotations__int r   r   X/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/utils/model_file_verifier.pyr
      s   
 r
   c                   @   sB   e Zd ZU eeef ed< ededd fddZ	defddZ
dS )	Manifestfilesdatareturnc                 C   sP   d|v rt jdtdd | dd |d  D dS | dd |d	  D dS )
N	checksumsznThe 'checksums' format is deprecated. Please regenerate with the latest version to use the new 'files' format.   )
stacklevelc                 S   s   i | ]\}}|t |d dqS )r   r   r
   .0kvr   r   r   
<dictcomp>.   s    z&Manifest.from_dict.<locals>.<dictcomp>r   c                 S   s    i | ]\}}|t d i |qS )r   r   r   r   r   r   r#   2   s     r   )warningswarnDeprecationWarningitems)clsr   r   r   r   	from_dict$   s   
zManifest.from_dictc                 C   s   t | S N)r   )selfr   r   r   to_dict4   s   zManifest.to_dictN)r   r   r   r   r   r
   r   classmethoddictr*   r-   r   r   r   r   r       s
   
 r   )z	.DS_Storez*.lockz.gitattributesLICENSEz	LICENSE.*z	README.mdzREADME.*NOTICE   max_workers
model_pathchecksums_sourcer4   r   c                 C   sR   t |  } t|}t| t|j |d}t||d tdt	|j d d S )Nr5   	filenamesr4   )expectedactualz[ModelFileVerifier] All z files verified successfully.)
r   resolve_load_checksums_compute_manifest_from_folderlistr   keys_compare_manifestsprintlen)r5   r6   r4   r9   r:   r   r   r   verifyJ   s   rC   r9   r:   c                 C   s   g }| j  D ]D\}}||j vr|| d|j d q|j | j|jkrK|j | }|| d|jd d  d|j d|jd d  d|j d
 q|rWtdd| d S )	Nz: missing (expected size=)z: mismatch (expected=   z	... size=z	, actual=zIntegrity check failed: z; )r   r(   appendr   r   IntegrityErrorjoin)r9   r:   errorsfilenameexpactr   r   r   r@   V   s   

8r@   sourceoutput_pathc                 C   s   t |  r!t |  }t|}|std| t|||d}ntt| dd}t |t	j
| ddd tdt|j d	|  |S )
NzNo model files found in r7   repo_idr$      T)indent	sort_keysz,[ModelFileVerifier] Generated checksums for z
 files -> )r   is_dirr;   _discover_filesrG   r=   r   _load_file_infos_from_hf
write_textjsondumpsr-   rA   rB   r   )rM   rN   r4   r5   r   manifestr   r   r   generate_checksumsh   s    r[   c                 C   s   t dd |  D S )Nc                 3   s@    | ]    r jd st fddtD s jV  qdS ).c                 3   s    | ]
}t   j|V  qd S r+   )fnmatchname)r    per   r   	<genexpr>   s    z,_discover_files.<locals>.<genexpr>.<genexpr>N)is_filer^   
startswithanyIGNORE_PATTERNS)r    r   r`   r   rb      s    

z"_discover_files.<locals>.<genexpr>)sortediterdirr5   r   r   r   rU      s   rU   c                 C   s8   t |  rtt |  }t|S tt| ddS )NrO   r$   )r   rc   rX   loads	read_textr   r*   rV   )rM   r   r   r   r   r<      s   
r<   rP   c                    sX   ddl m} |   j| dd}tdd t fdd|D }|s*td	|  d
|S )Nr   )HfFileSystemT)detailc                 s   s    | ]}|r|V  qd S r+   r   )r    rr   r   r   rb      s    
z+_load_file_infos_from_hf.<locals>.<genexpr>c                    s
   t  | S r+   )#_get_filename_and_info_from_hf_file)ffsr   r   <lambda>   s   
 z*_load_file_infos_from_hf.<locals>.<lambda>zNo files found in HF repo r\   )huggingface_hubrl   lsr/   maprG   )rP   rl   r   
file_infosr   rq   r   rV      s   rV   c                    s   | ddkr	d S t| ddj t fddtD rd S | dd}| d	}|r:d
|v r: t|d
 |dfS d
|v rH t|d
 |dfS | | dd} tt|	 t
|dfS )Ntypefiler^    c                 3   s    | ]	}t   |V  qd S r+   )r]   )r    patrJ   r   r   rb      s    z6_get_filename_and_info_from_hf_file.<locals>.<genexpr>r   r   lfsr   r   )getr   r^   re   rf   r
   
read_byteshashlibr   	hexdigestrB   )rr   	file_infor   lfs_infocontentr   r|   r   ro      s   
ro   r8   c                    s   ddl m } dtdtttt f f fdd}t|d}t||||t|dd	}W d    n1 s6w   Y  t	d
d |D dS )Nr   )tqdmrJ   r   c                    s<    |  }|  s| d fS t|d}| j}| t||dfS )N)	file_pathr   )existscompute_sha256statst_sizer
   )rJ   	full_pathr   r   ri   r   r   compute_one   s   

z2_compute_manifest_from_folder.<locals>.compute_oner3   zComputing checksums)totaldescc                 S   s   i | ]\}}|d ur||qS r+   r   r   r   r   r   r#      s    z1_compute_manifest_from_folder.<locals>.<dictcomp>r$   )
r   r   r	   r   r
   r   r>   rv   rB   r   )r5   r8   r4   r   r   executorresultsr   ri   r   r=      s   "
	r=   c                 C   sz   t  }t| d(}|d }r'|| |d }sW d    | S W d    | S 1 s4w   Y  | S )Nrbi   )r   r   openreadupdater   )r   r   rp   chunkr   r   r   r      s   


r   c                   @   s   e Zd ZdS )rG   N)r   r   r   r   r   r   r   rG      s    rG   c                 C   s6   | j dddd | j dddd | j dtdd	d
 d S )Nz--model-pathTz,Local model directory or HuggingFace repo ID)requiredhelpz--model-checksumzChecksums JSON file pathz	--workersr2   zNumber of parallel workers)rx   defaultr   )add_argumentr   )parserr   r   r   _add_common_args   s   
r   c                  C   s|   t jdd} | jddd}|jddd}t| |jd	d
 d |jddd}t| |jdd
 d |  }|| d S )NzAModel File Verifier - Verify model file integrity using checksums)descriptioncommandT)destr   generatez#Generate checksums.json for a model)r   c                 S      t | j| j| jdS )N)rM   rN   r4   )r[   r5   model_checksumworkersargsr   r   r   rs     
    zmain.<locals>.<lambda>)funcrC   z$Verify model files against checksumsc                 S   r   )N)r5   r6   r4   )rC   r5   r   r   r   r   r   r   rs     r   )argparseArgumentParseradd_subparsers
add_parserr   set_defaults
parse_argsr   )r   
subparsers
gen_parserverify_parserr   r   r   r   main   s(   r   __main__)%__doc__r   r]   r   rX   r%   concurrent.futuresr   dataclassesr   r   pathlibr   typingr   r   r   r	   r
   r   rf   r   r   rC   r@   r[   rU   r<   rV   ro   r=   r   	ExceptionrG   r   r   r   r   r   r   r   <module>   sb     


"
