o
    ºiŽû  ã                   @  sø  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZmZ d dlZd dlZd dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& dej'vr‡e	e(ƒj)j)j)j)d  *¡ r‡ej+ ,d e-e	e(ƒj)j)j)ƒ¡ d dl.Z.d d	l.m/Z/m0Z0m1Z1m2Z2 d d
l3m4Z4m5Z5m6Z6m7Z7 e 8d¡Z9e.j:j;j<e4e.j:j=j>e5e.j:j?j@e6e.j:jAjBe7iZCe.j:j;jDe.j:j;j<e.j:j;jEgZFG dd„ de#ƒZGG dd„ de#ƒZHG dd„ de#ƒZIG dd„ deƒZJddd„ZKeLdkrúeKƒ  dS dS )é    )ÚannotationsN)ÚPath)ÚAnyÚOptionalÚTupleÚType)ÚQApplicationÚQMainWindowÚQWidgetÚQVBoxLayoutÚQHBoxLayoutÚQPushButtonÚQLabelÚ	QLineEditÚQFileDialogÚQTableWidgetÚQTableWidgetItemÚ	QComboBoxÚQMessageBoxÚ
QTabWidgetÚ	QTextEditÚQFormLayoutÚQHeaderViewÚQDialogÚQDialogButtonBox)ÚQtÚNO_LOCAL_GGUFzgguf-py)Ú
GGUFReaderÚ
GGUFWriterÚGGUFValueTypeÚReaderField)Ú	TokenTypeÚRopeScalingTypeÚPoolingTypeÚGGMLQuantizationTypezgguf-editor-guic                      sf   e Zd Zd‡ fdd„	Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Z‡  ZS )ÚTokenizerEditorDialogNc                   s  t ƒ  |¡ |  d¡ |  dd¡ |r| ¡ ng | _|r | ¡ ng | _|r)| ¡ ng | _tt	| jƒt	| jƒt	| jƒƒ}t	| jƒ|k rP| j 
dg|t	| jƒ  ¡ t	| jƒ|k re| j 
dg|t	| jƒ  ¡ t	| jƒ|k rz| j 
dg|t	| jƒ  ¡ t| ƒ}tƒ }| tdƒ¡ tƒ | _| j d¡ | jj | j¡ | | j¡ d	| _d| _td
t	| jƒ| j d
 | j ƒ| _td| j› ƒ| _| | j¡ tdƒ}|j | j¡ | |¡ tdƒ}	|	j | j¡ | |	¡ | |¡ tƒ | _| j  d¡ | j !g d¢¡ | j "¡  #dt$j%j&¡ | j "¡  #d
t$j%j'¡ | j "¡  #dt$j%j&¡ | j "¡  #dt$j%j&¡ | | j¡ tƒ }
tdƒ}|j | j(¡ |
 |¡ tdƒ}|j | j)¡ |
 |¡ |
 *¡  | |
¡ t+t+j,j-t+j,j.B ƒ}|j/ | j0¡ |j1 | j2¡ | |¡ t3t4t	| jƒƒƒ| _5|  6¡  d S )NzEdit Tokenizer Datai„  iX  Ú r   ç        úFilter:zType to filter tokens...éd   é   ú
Page 1 of ÚPreviousÚNexté   )ÚIndexÚTokenr   ÚScoreé   é   z	Add TokenúRemove Selected)7ÚsuperÚ__init__ÚsetWindowTitleÚresizeÚcopyÚtokensÚtoken_typesÚscoresÚmaxÚlenÚextendr   r   Ú	addWidgetr   r   Úfilter_editÚsetPlaceholderTextÚtextChangedÚconnectÚapply_filterÚ	page_sizeÚcurrent_pageÚtotal_pagesÚ
page_labelr   ÚclickedÚprevious_pageÚ	next_pageÚ	addLayoutr   Útokens_tableÚsetColumnCountÚsetHorizontalHeaderLabelsÚhorizontalHeaderÚsetSectionResizeModer   Ú
ResizeModeÚResizeToContentsÚStretchÚ	add_tokenÚremove_selectedÚ
addStretchr   ÚStandardButtonÚOkÚCancelÚacceptedÚacceptÚrejectedÚrejectÚlistÚrangeÚfiltered_indicesÚ	load_page)Úselfr:   r;   r<   ÚparentÚmax_lenÚlayoutÚfilter_layoutÚ	prev_pagerL   Úcontrols_layoutÚ
add_buttonÚremove_buttonÚbuttons©Ú	__class__© úY/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/gguf/scripts/gguf_editor_gui.pyr6   3   sn   
"






zTokenizerEditorDialog.__init__c                 C  s¦   | j  ¡  ¡ }|sttt| jƒƒƒ| _ng | _t| jƒD ]\}}|t	|ƒ ¡ v r.| j 
|¡ qtdt| jƒ| j d | j ƒ| _d| _| j d| j› ¡ |  ¡  dS )z+Filter the tokens based on the search text.r*   r   r+   N)rA   ÚtextÚlowerr`   ra   r>   r:   rb   Ú	enumerateÚstrÚappendr=   rF   rH   rG   rI   ÚsetTextrc   )rd   Úfilter_textÚiÚtokenrp   rp   rq   rE   ‡   s   €"z"TokenizerEditorDialog.apply_filterc                 C  óH   | j dkr"|  j d8  _ | j d| j d › d| j› ¡ |  ¡  dS dS ©z#Go to the previous page of results.r   r*   úPage ú of N©rG   rI   rw   rH   rc   ©rd   rp   rp   rq   rK   ›   ó
   
 ýz#TokenizerEditorDialog.previous_pagec                 C  óN   | j | jd k r%|  j d7  _ | j d| j d › d| j› ¡ |  ¡  dS dS ©zGo to the next page of results.r*   r}   r~   N©rG   rH   rI   rw   rc   r€   rp   rp   rq   rL   ¢   ó
    ýzTokenizerEditorDialog.next_pagec              
   C  s°  | j  d¡ | j| j }t|| j t| jƒƒ}| j  || ¡ tt||ƒƒD ]§\}}| j| }t	t
|ƒƒ}| tjj|¡ | | ¡ tjj @ ¡ | j  |d|¡ t	t
| j| ƒƒ}| j  |d|¡ |t| jƒk rn| j| nd}zt|ƒ}	|	j› d|› d}
W n ttfy   d|› d}
Y nw t	|
ƒ}| tjj|¡ | | ¡ tjj @ ¡ | j  |d|¡ |t| jƒk r½| j| nd}t	t
|ƒƒ}| j  |d|¡ q&| j j | j¡ d	S )
z(Load the current page of tokenizer data.r   r*   ú (ú)ú	Unknown (r2   r'   r3   N)rN   ÚsetRowCountrG   rF   Úminr>   rb   rt   ra   r   ru   ÚsetDatar   ÚItemDataRoleÚUserRoleÚsetFlagsÚflagsÚItemFlagÚItemIsEditableÚsetItemr:   r;   r!   ÚnameÚ
ValueErrorÚKeyErrorr<   ÚcellDoubleClickedrD   Úhandle_cell_double_click)rd   Ú	start_idxÚend_idxÚrowry   Úorig_idxÚ
index_itemÚ
token_itemÚ
token_typeÚenum_valÚdisplay_textÚ	type_itemÚscoreÚ
score_itemrp   rp   rq   rc   ©   s6   
ÿzTokenizerEditorDialog.load_pagec                 C  s@   |dkr| j  |d¡}|r| tjj¡}|  ||¡ dS dS dS )zCHandle double-click on a cell, specifically for token type editing.r2   r   N)rN   ÚitemÚdatar   rŒ   r   Úedit_token_type)rd   rš   ÚcolumnÚ	orig_itemr›   rp   rp   rq   r—   Ø   s   üz.TokenizerEditorDialog.handle_cell_double_clickc              	   C  s^  |t | jƒk r| j| nd}t| ƒ}| d¡ t|ƒ}tƒ }tD ]}| |j› d|j	› d|j	¡ q zt
|tƒrHt|ƒ}| |j› d|› d¡ W n ttfyT   Y nw | |¡ ttjjtjjB ƒ}|j |j¡ |j |j¡ | |¡ | ¡ tjjkr­| ¡ }	t|	ƒ}|j› d|	› d}
| j |d¡}|r¦| |
¡ |  t!j"j#|	¡ |	| j|< dS dS )zEEdit a token type using a dialog with a dropdown of all enum options.r   zSelect Token Typer†   r‡   r2   N)$r>   r;   r   r7   r   r   r!   ÚaddItemr“   ÚvalueÚ
isinstanceÚintÚsetCurrentTextr”   r•   r@   r   rY   rZ   r[   r\   rD   r]   r^   r_   ÚexecÚ
DialogCodeÚAcceptedÚcurrentDatarN   r¤   rw   r‹   r   rŒ   r   )rd   rš   r›   Úcurrent_valueÚdialogrg   ÚcomborŸ   rm   Ú	new_valuer    r¡   rp   rp   rq   r¦   à   s<   
 
€ÿ


óz%TokenizerEditorDialog.edit_token_typec                 C  s²   | j  d¡ | j d¡ | j d¡ t| j ƒd }| j ¡  ¡ }|r&|dv r,| j |¡ t	dt| jƒ| j
 d | j
 ƒ| _| jd | _| j d| jd › d| j› ¡ |  ¡  dS )z'Add a new token to the end of the list.r&   r   r'   r*   r}   r~   N)r:   rv   r;   r<   r>   rA   rr   rs   rb   r=   rF   rH   rG   rI   rw   rc   )rd   r›   rx   rp   rp   rq   rV     s   " zTokenizerEditorDialog.add_tokenc           
      C  sl  g }| j  ¡ D ]}| ¡ }||vr| |¡ q|sdS g }|D ]}| j  |d¡}|r4| | tjj¡¡ q|j	dd |D ]#}|t
| jƒk rJ| j|= |t
| jƒk rU| j|= |t
| jƒk r`| j|= q=g | _| j ¡  ¡ }t| jƒD ]\}}	|r~|t|	ƒ ¡ v r„| j |¡ qptdt
| jƒ| j d | j ƒ| _t| j| jd ƒ| _| j d| jd › d| j› ¡ |  ¡  dS )z'Remove selected tokens from all arrays.Nr   T©Úreverser*   r}   r~   )rN   ÚselectedItemsrš   rv   r¤   r¥   r   rŒ   r   Úsortr>   r:   r;   r<   rb   rA   rr   rs   rt   ru   r=   rF   rH   rŠ   rG   rI   rw   rc   )
rd   Úselected_rowsr¤   rš   Úorig_indicesr¨   Úidxrx   ry   rz   rp   rp   rq   rW   #  sB   
€€€€" z%TokenizerEditorDialog.remove_selectedc                 C  s   | j | j| jfS )z!Return the edited tokenizer data.)r:   r;   r<   r€   rp   rp   rq   Úget_dataO  s   zTokenizerEditorDialog.get_data©N)Ú__name__Ú
__module__Ú__qualname__r6   rE   rK   rL   rc   r—   r¦   rV   rW   r½   Ú__classcell__rp   rp   rn   rq   r%   2   s    T/+,r%   c                      sp   e Zd Zd‡ fdd„	Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zddd„Zdd„ Z‡  ZS )ÚArrayEditorDialogNc                   sN  t ƒ  |¡ |  d¡ |  dd¡ || _|| _|| _d | _|tv r+|t	j
kr+t| | _t| ƒ}| jd urhtƒ }td| jj› dƒ}| |¡ d dd„ | jD ƒ¡}td	|› ƒ}	|	 d
¡ | |	d¡ | |¡ tƒ }
|
 tdƒ¡ tƒ | _| j d¡ | jj | j¡ |
 | j¡ d| _d| _tdt|ƒ| j d | j ƒ| _td| j› ƒ| _|
 | j¡ tdƒ}|j  | j!¡ |
 |¡ tdƒ}|j  | j"¡ |
 |¡ | |
¡ t#ƒ | _$| jd ur| j$ %d¡ | j$ &g d¢¡ | j$ '¡  (dt)j*j+¡ | j$ '¡  (dt)j*j,¡ | j$ '¡  (dt)j*j+¡ n$| j$ %d¡ | j$ &ddg¡ | j$ '¡  (dt)j*j+¡ | j$ '¡  (dt)j*j,¡ | | j$¡ tƒ }tdƒ}|j  | j-¡ | |¡ tdƒ}|j  | j.¡ | |¡ | jd urqtdƒ}|j  | j/¡ | |¡ | 0¡  | |¡ t1t1j2j3t1j2j4B ƒ}|j5 | j6¡ |j7 | j8¡ | |¡ t9t:t| jƒƒƒ| _;|  <¡  d S )NzEdit Array Valuesi¼  iô  zEditing z values:ú, c                 S  s   g | ]}|j › d |j› ‘qS )ú=)r“   rª   )Ú.0Úerp   rp   rq   Ú
<listcomp>l  s    z.ArrayEditorDialog.__init__.<locals>.<listcomp>zAvailable values: Tr*   r(   zType to filter values...r)   r   r+   r,   r-   r3   )r/   ÚValueÚActionsr2   r/   rÉ   zAdd Itemr4   zBulk Edit Selected)=r5   r6   r7   r8   Úarray_valuesÚelement_typeÚkeyÚ	enum_typeÚKEY_TO_ENUM_TYPEr   ÚINT32r   r   r   r¿   r@   ÚjoinÚsetWordWraprM   r   rA   rB   rC   rD   rE   rF   rG   r=   r>   rH   rI   r   rJ   rK   rL   r   Úitems_tablerO   rP   rQ   rR   r   rS   rT   rU   Úadd_itemrW   Úbulk_edit_selectedrX   r   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   )rd   rË   rÌ   rÍ   re   rg   Úenum_info_layoutÚ
enum_labelÚenum_valuesÚenum_values_labelrh   ri   rL   rj   rk   rl   Úbulk_edit_buttonrm   rn   rp   rq   r6   U  sˆ   





 







zArrayEditorDialog.__init__c              
   C  s&  | j  ¡  ¡ }|sttt| jƒƒƒ| _n[g | _t| jƒD ]R\}}| j	dur`t
|tƒr`z|  	|¡}|j› d|› d ¡ }||v rE| j |¡ W q ttfy_   |t|ƒ ¡ v r]| j |¡ Y qw |t|ƒ ¡ v rn| j |¡ qtdt| jƒ| j d | j ƒ| _d| _| j d| j› ¡ |  ¡  dS )z1Filter the array values based on the search text.Nr†   r‡   r*   r   r+   )rA   rr   rs   r`   ra   r>   rË   rb   rt   rÎ   r«   r¬   r“   rv   r”   r•   ru   r=   rF   rH   rG   rI   rw   rc   )rd   rx   ry   rª   rŸ   r    rp   rp   rq   rE   À  s0   
€€ý€"zArrayEditorDialog.apply_filterc                 C  r{   r|   r   r€   rp   rp   rq   rK   á  r   zArrayEditorDialog.previous_pagec                 C  r‚   rƒ   r„   r€   rp   rp   rq   rL   è  r…   zArrayEditorDialog.next_pagec              
   C  sÚ  | j  d¡ | j| j }t|| j t| jƒƒ}| j  || ¡ tt||ƒƒD ]Ä\}}| j| }| j	| }t
t|ƒƒ}| tjj|¡ | | ¡ tjj @ ¡ | j  |d|¡ | jdurÜzt|ttjfƒrs|  |¡}|j› d|› d}	nt|ƒ}	W n ttfy‰   d|› d}	Y nw t
|	ƒ}
|
 tjj|¡ |
 |
 ¡ tjj @ ¡ | j  |d|
¡ tdƒ}| d|¡ |j  | j!¡ t"ƒ }t#|ƒ}| $d	d	d	d	¡ | %|¡ | &¡  | j  '|d	|¡ q&t
t|ƒƒ}
| j  |d|
¡ q&dS )
z&Load the current page of array values.r   Nr†   r‡   rˆ   r*   ÚEditrš   r2   )(rÓ   r‰   rG   rF   rŠ   r>   rb   rt   ra   rË   r   ru   r‹   r   rŒ   r   rŽ   r   r   r‘   r’   rÎ   r«   r¬   ÚnumpyÚsignedintegerr“   r”   r•   r   ÚsetPropertyrJ   rD   Úedit_array_enum_valuer
   r   ÚsetContentsMarginsr@   rX   ÚsetCellWidget)rd   r˜   r™   rš   ry   r›   rª   rœ   rŸ   r    Ú
value_itemÚedit_buttonÚbutton_widgetÚbutton_layoutrp   rp   rq   rc   ï  sJ   



€ÿ
ÕzArrayEditorDialog.load_pagec                 C  sš   |   ¡ }| d¡}| j |d¡}| j |d¡}|rC|rE| jrG|  || j¡rI| tjj	¡}| tjj	¡}t
|ttttfƒrK|| j|< dS dS dS dS dS dS )z1Handle editing an enum value in the array editor.rš   r   r*   N)ÚsenderÚpropertyrÓ   r¤   rÎ   Úedit_enum_valuer¥   r   rŒ   r   r«   r¬   Úfloatru   ÚboolrË   )rd   Úbuttonrš   r¨   Únew_itemr›   rµ   rp   rp   rq   rß   '  s   
ûz'ArrayEditorDialog.edit_array_enum_valuec                 C  s”  | j sdS tƒ }| j ¡ D ]	}| | ¡ ¡ q|s"t | dd¡ dS t| ƒ}| 	d| j j
› d¡ t|ƒ}| tdt|ƒ› dƒ¡ tƒ }| j D ]}| |j› d|j› d	|j¡ qH| |¡ ttjjtjjB ƒ}|j |j¡ |j |j¡ | |¡ | ¡ tjjkrÆ| ¡ }|   |¡}|j› d|› d	}	|D ]/}
| j  |
d
¡}| j  |
d¡}|rÅ|rÅ| !t"j#j$¡}|| j%|< | &|	¡ | 't"j#j$|¡ q˜dS dS )z"Edit multiple enum values at once.NzNo Selectionz'Please select at least one row to edit.z
Bulk Edit z ValueszSet z selected items to:r†   r‡   r   r*   )(rÎ   ÚsetrÓ   r¸   Úaddrš   r   Úinformationr   r7   r¿   r   r@   r   r>   r   r©   r“   rª   r   rY   rZ   r[   r\   rD   r]   r^   r_   r®   r¯   r°   r±   r¤   r¥   r   rŒ   r   rË   rw   r‹   )rd   rº   r¤   r³   rg   r´   rŸ   rm   rµ   r    rš   r¨   rì   r›   rp   rp   rq   rÕ   6  sF   
 




€ðz$ArrayEditorDialog.bulk_edit_selectedc                 C  s¾   t | jƒ}| jd urt| jƒd j}| j |¡ n| jtjkr&| j d¡ n| j d¡ | j	 |¡ t
dt | j	ƒ| j d | j ƒ| _| jd | _| j d| jd › d| j› ¡ |  ¡  d S )Nr   r&   r*   r}   r~   )r>   rË   rÎ   r`   rª   rv   rÌ   r   ÚSTRINGrb   r=   rF   rH   rG   rI   rw   rc   )rd   r›   Údefault_valuerp   rp   rq   rÔ   g  s   

" zArrayEditorDialog.add_itemc              
   C  sÂ  g }| j  ¡ D ]}| ¡ }||vr| |¡ q|sd S tƒ }|D ]}| j  |d¡}|r5| | tjj	¡¡ q |j
dd |D ]}| j|= q>g | _| j ¡  ¡ }t| jƒD ][\}}	|sa| j |¡ qT| jd ur¡t|	tƒr¡z|  |	¡}
|
j› d|	› d ¡ }||v r†| j |¡ W qT ttfy    |t|	ƒ ¡ v rž| j |¡ Y qTw |t|	ƒ ¡ v r¯| j |¡ qTtdt| jƒ| j d | j ƒ| _t| j| jd ƒ| _| j d| jd › d| j› ¡ |   ¡  d S )	Nr   Tr¶   r†   r‡   r*   r}   r~   )!rÓ   r¸   rš   rv   r`   r¤   r¥   r   rŒ   r   r¹   rË   rb   rA   rr   rs   rt   rÎ   r«   r¬   r“   r”   r•   ru   r=   r>   rF   rH   rŠ   rG   rI   rw   rc   )rd   rº   r¤   rš   r»   r¨   r¼   rx   ry   rª   rŸ   r    rp   rp   rq   rW   ƒ  sR   
€€

€€þ€" z!ArrayEditorDialog.remove_selectedrš   r¬   rÎ   úType[enum.Enum]c              	   C  s˜  | j  |d¡}|r| tjj¡}ndS | j| }t| ƒ}| d|j	› d¡ t
|ƒ}td|j	› dƒ}| |¡ tƒ }	|D ]}
|	 |
j› d|
j› d|
j¡ q=zt|tƒre||ƒ}
|	 |
j› d|› d¡ W n ttfyq   Y nw | |	¡ ttjjtjjB ƒ}|j |j¡ |j |j¡ | |¡ | ¡ tj j!krÊ|	 "¡ }||ƒ}
|
j› d|› d}| j  |d	¡}|rÃ| #|¡ | $tjj|¡ || j|< d
S dS )zFEdit an enum value using a dialog with a dropdown of all enum options.r   NúSelect ú Valuez	Select a z value:r†   r‡   r*   TF)%rÓ   r¤   r¥   r   rŒ   r   rË   r   r7   r¿   r   r   r@   r   r©   r“   rª   r«   r¬   r­   r”   r•   r   rY   rZ   r[   r\   rD   r]   r^   r_   r®   r¯   r°   r±   rw   r‹   )rd   rš   rÎ   r¨   r›   r²   r³   rg   Údescriptionr´   rŸ   rm   rµ   r    rì   rp   rp   rq   rè   ·  sJ   

 
€ÿ



z!ArrayEditorDialog.edit_enum_valuec                 C  s   | j S r¾   ©rË   r€   rp   rp   rq   Úget_array_valuesî  s   z"ArrayEditorDialog.get_array_values)NN)rš   r¬   rÎ   rò   )r¿   rÀ   rÁ   r6   rE   rK   rL   rc   rß   rÕ   rÔ   rW   rè   r÷   rÂ   rp   rp   rn   rq   rÃ   T  s    k!81
47rÃ   c                      s(   e Zd Zd‡ fdd„	Zd	dd„Z‡  ZS )
ÚAddMetadataDialogNc                   sÞ   t ƒ  |¡ |  d¡ |  dd¡ t| ƒ}tƒ }tƒ | _| d| j¡ t	ƒ | _
tD ]}|tjkr8| j
 |j|¡ q)| d| j
¡ tƒ | _| d| j¡ | |¡ ttjjtjjB ƒ}|j | j¡ |j | j¡ | |¡ d S )NúAdd Metadatai  éÈ   zKey:zType:zValue:)r5   r6   r7   r8   r   r   r   Úkey_editÚaddRowr   Ú
type_combor   ÚARRAYr©   r“   r   Ú
value_editrM   r   rY   rZ   r[   r\   rD   r]   r^   r_   r@   )rd   re   rg   Úform_layoutÚ
value_typerm   rn   rp   rq   r6   ô  s(   

€
zAddMetadataDialog.__init__ÚreturnúTuple[str, GGUFValueType, Any]c                 C  s
  | j  ¡ }| j ¡ }| j ¡ }|tjkrt 	t
|ƒ¡}nd|tjkr)t t
|ƒ¡}nW|tjkr6t t
|ƒ¡}nJ|tjkrCt t
|ƒ¡}n=|tjkrPt t
|ƒ¡}n0|tjkr]t t
|ƒ¡}n#|tjkrjt t|ƒ¡}n|tjkrv| ¡ dv }n
|tjkr~|}n|}|||fS )N©ÚtrueÚyesÚ1)rû   rr   rý   r±   rÿ   ÚtoPlainTextr   ÚUINT8ÚnpÚuint8r¬   ÚINT8Úint8ÚUINT16Úuint16ÚINT16Úint16ÚUINT32Úuint32rÐ   Úint32ÚFLOAT32Úfloat32ré   ÚBOOLrs   rð   )rd   rÍ   r  Ú
value_textrª   rp   rp   rq   r½     s.   












zAddMetadataDialog.get_datar¾   )r  r  )r¿   rÀ   rÁ   r6   r½   rÂ   rp   rp   rn   rq   rø   ó  s    rø   c                      s¬   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Zd0dd„Zd1dd„Z	d2dd„Z
d3dd„Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ Zd,d-„ Zd.d/„ Z‡  ZS )4ÚGGUFEditorWindowc                   sR   t ƒ  ¡  |  d¡ |  dd¡ d | _d | _d| _i | _tƒ | _	d| _
|  ¡  d S )NzGGUF Editoriè  i   F)r5   r6   r7   r8   Úcurrent_fileÚreaderÚmodifiedÚmetadata_changesrí   Úmetadata_to_removeÚ on_metadata_changed_is_connectedÚsetup_uir€   rn   rp   rq   r6   /  s   

zGGUFEditorWindow.__init__c           
      C  sP  t ƒ }|  |¡ t|ƒ}tƒ }tƒ | _| j d¡ | | j¡ tdƒ}|j	 
| j¡ | |¡ tdƒ}|j	 
| j¡ | |¡ | |¡ tƒ | _t ƒ | _t| jƒ}tƒ | _| j d¡ | j g d¢¡ | j ¡  dtjj¡ | j ¡  dtjj¡ | j ¡  dtjj¡ | j ¡  d	tjj¡ | | j¡ tƒ }td
ƒ}|j	 
| j¡ | |¡ | ¡  | |¡ t ƒ | _t| jƒ}	tƒ | _| j d¡ | j g d¢¡ | j ¡  dtjj¡ | j ¡  dtjj¡ | j ¡  dtjj¡ | j ¡  d	tjj¡ | j ¡  dtjj¡ |	 | j¡ | j | jd¡ | j | jd¡ | | j¡ |   ¡  !d¡ d S )NTz	Open GGUFz
Save As...r.   )ÚKeyr   rÉ   rÊ   r   r*   r2   r3   rù   é   )ÚNamer   ÚShapeÚElementszSize (bytes)ÚMetadataÚTensorsÚReady)"r
   ÚsetCentralWidgetr   r   r   Úfile_path_editÚsetReadOnlyr@   r   rJ   rD   Ú	open_fileÚ	save_filerM   r   ÚtabsÚmetadata_tabr   Úmetadata_tablerO   rP   rQ   rR   r   rS   rU   rT   Úadd_metadatarX   Útensors_tabÚtensors_tableÚaddTabÚ	statusBarÚshowMessage)
rd   Úcentral_widgetÚmain_layoutÚfile_layoutÚopen_buttonÚsave_buttonÚmetadata_layoutÚmetadata_controlsÚadd_metadata_buttonÚtensors_layoutrp   rp   rq   r   >  s\   







zGGUFEditorWindow.setup_uic              
   C  sÈ   z=|   ¡  d|› d¡ t ¡  t|dƒ| _|| _| j |¡ |  	¡  |  
¡  i | _tƒ | _d| _|   ¡  d|› ¡ W dS  tyc } zt | ddt|ƒ› ¡ |   ¡  d	¡ W Y d
}~dS d
}~ww )zLoad a GGUF file by pathzLoading ú...ÚrFzLoaded TÚErrorzFailed to open file: zError loading fileN)r5  r6  r   ÚprocessEventsr   r  r  r*  rw   Úload_metadataÚload_tensorsr  rí   r  r  Ú	Exceptionr   Úcriticalru   )rd   Ú	file_pathrÇ   rp   rp   rq   Ú	load_fileˆ  s&   €ýzGGUFEditorWindow.load_filec                 C  s*   t  | ddd¡\}}|sd S |  |¡ d S )NzOpen GGUF Filer&   ú"GGUF Files (*.gguf);;All Files (*))r   ÚgetOpenFileNamerI  )rd   rH  Ú_rp   rp   rq   r,     s   ÿzGGUFEditorWindow.open_filec                 C  sV  | j  d¡ | jsd S | jr2t ¡  t d¡ | j j | j	¡ W d   ƒ n1 s*w   Y  d| _t
| jj ¡ ƒD ]b\}\}}| j  |¡ t|ƒ}| | ¡ tjj @ ¡ | j  |d|¡ |jsed}nP|jd tjkr™t|jƒd }|jd j}|  |¡}|d urŽ|jd tjkrŽ|j}d| | d|  }nt|jd jƒ}|  |¡}|d urµ|jd tjkrµ|j}t|ƒ}	|	 |	 ¡ tjj @ ¡ | j  |d|	¡ |  |¡}
t|
ƒ}t|jƒdkrñ|jd tjkrñ| | ¡ tjjB ¡ n| | ¡ tjj @ ¡ | j  |d	|¡ tƒ }t |ƒ}| !d	d	d	d	¡ |jrM|jd tjkrMt"d
ƒ}| #d|¡ | #d|¡ |j$ %| j&¡ | '|¡ |t(v rL| )d¡ | *d¡ n,t|jƒdkry|  |¡d uryt"d
ƒ}| #d|¡ | #d|¡ |j$ %| j+¡ | '|¡ t"dƒ}| #d|¡ | #d|¡ |j$ %| j,¡ | '|¡ | j  -|d|¡ q:| j j %| j	¡ d| _d S )Nr   ÚignoreFúN/Ar*   éÿÿÿÿú[ú]r2   rÛ   rš   rÍ   zEdit Tokenizerz Edit all tokenizer data togetherÚRemover3   T).r0  r‰   r  r  ÚwarningsÚcatch_warningsÚfilterwarningsÚitemChangedÚ
disconnectÚon_metadata_changedrt   ÚfieldsÚitemsÚ	insertRowr   rŽ   r   r   r   r‘   r’   Útypesr   rþ   r>   r“   Úget_enum_for_keyrÐ   r¿   ru   Úformat_field_valuer
   r   rà   r   rÞ   rJ   rD   Úedit_array_metadatar@   ÚTOKENIZER_LINKED_KEYSrw   Ú
setToolTipÚedit_metadata_enumÚremove_metadatará   )rd   ry   rÍ   ÚfieldÚkey_itemÚtype_strÚ
nest_countrÌ   rÎ   r¡   Ú	value_strrâ   Úactions_widgetÚactions_layoutrã   rl   rp   rp   rq   rD  ª  s~   

þ






€ 


zGGUFEditorWindow.load_metadatard  r    r  r`   c                 C  sÂ   |j r|j d tjkrg S |j d }g }t|jƒ}|tjkr?t|ƒD ]}tt|j	d|| d d   ƒdd}| 
|¡ q"|S | jr_|| jjv r_t|ƒD ]}| 
|j	d|| d   d ¡ qL|S )z'Extract all values from an array field.r   r*   rO  r2   úutf-8©Úencoding)r\  r   rþ   r>   r¥   rð   ra   ru   ÚbytesÚpartsrv   r  Úgguf_scalar_to_np)rd   rd  Ú	curr_typerË   Útotal_elementsÚelement_posÚvalue_stringrp   rp   rq   Úextract_array_values  s   


&ü"z%GGUFEditorWindow.extract_array_valuesrÍ   ru   úOptional[Type[enum.Enum]]c                 C  s
   t  |¡S )z/Get the enum type for a given key if it exists.)rÏ   Úget)rd   rÍ   rp   rp   rq   r]    s   
z!GGUFEditorWindow.get_enum_for_keyrª   r   rÎ   rò   c              	   C  sR   zt |ttfƒr||ƒ}|j› d|› dW S W t|ƒS  ttfy(   Y t|ƒS w )z&Format a value as an enum if possible.r†   r‡   )r«   r¬   ru   r“   r”   r•   )rd   rª   rÎ   Ú
enum_valuerp   rp   rq   Úformat_enum_value  s   þþþz"GGUFEditorWindow.format_enum_valuec                   s>  |j sdS t|j ƒdkrE|j d }|tjkr!tt|jd ƒddS | jrE|| jjv rE|jd d }|  	|j
¡}|d urA|  ||¡S t|ƒS |j d tjkr|  |¡‰ tdtˆ ƒƒ}|  	|j
¡}|d urxg }t|ƒD ]}| |  ˆ | |¡¡ qin‡ fdd	„t|ƒD ƒ}d
d |¡ ¡ › tˆ ƒt|ƒkr˜d› dS d› dS dS )NrN  r*   r   rO  rk  rl  r"  c                   s   g | ]}t ˆ | ƒ‘qS rp   ©ru   )rÆ   ry   rö   rp   rq   rÈ   B  s    z7GGUFEditorWindow.format_field_value.<locals>.<listcomp>ú[ rÄ   ú, ...r&   ú ]zComplex value)r\  r>   r   rð   ru   rn  ro  r  rp  r]  r“   ry  rþ   ru  rŠ   ra   rv   rÑ   Ústrip)rd   rd  rq  rª   rÎ   Úrender_elementÚarray_elementsry   rp   rö   rq   r^  &  s0   


ÿ4z#GGUFEditorWindow.format_field_valuec           	      C  sT  | j  d¡ | jsd S t| jjƒD ]–\}}| j  |¡ t|jƒ}| | 	¡ t
jj @ ¡ | j  |d|¡ t|jjƒ}| | 	¡ t
jj @ ¡ | j  |d|¡ d dd„ |jD ƒ¡}t|ƒ}| | 	¡ t
jj @ ¡ | j  |d|¡ tt|jƒƒ}| | 	¡ t
jj @ ¡ | j  |d|¡ t|jd›ƒ}| | 	¡ t
jj @ ¡ | j  |d	|¡ qd S )
Nr   r*   u    Ã— c                 s  ó    | ]}t |ƒV  qd S r¾   rz  )rÆ   Údrp   rp   rq   Ú	<genexpr>\  ó   € z0GGUFEditorWindow.load_tensors.<locals>.<genexpr>r2   r3   ú,r.   )r3  r‰   r  rt   Útensorsr[  r   r“   rŽ   r   r   r   r‘   r’   Útensor_typerÑ   Úshaperu   Ú
n_elementsÚn_bytes)	rd   ry   ÚtensorÚ	name_itemr¡   Ú	shape_strÚ
shape_itemÚelements_itemÚ	size_itemrp   rp   rq   rE  H  s,   
åzGGUFEditorWindow.load_tensorsc                 C  s2  |  ¡ dkrd S | ¡ }| j |d¡}d }|r| ¡ }| ¡ }d }| jr,|r,| j |¡}|r3|jr3|s5d S |jd }|  |¡}|d urâ|t	j
krâz]z	|| }	|	j}
W n* ttfy|   d|v rvd|v rv| d¡d  d¡d  ¡ }t|ƒ}
nt|ƒ}
Y nw ||
ƒ ||
f| j|< d| _|  |
|¡}| |¡ |  ¡  d|› d|› ¡ W d S  ttfyá } z.t | d	|› dd
|› d|j› dd dd„ |D ƒ¡› ¡ |  |¡}| |¡ W Y d }~d S d }~ww z“|t	jkrðt t|ƒ¡}
nl|t	j krýt !t|ƒ¡}
n_|t	j"krt #t|ƒ¡}
nQ|t	j$krt %t|ƒ¡}
nC|t	j&kr't 't|ƒ¡}
n5|t	j
kr5t (t|ƒ¡}
n'|t	j)krCt *t+|ƒ¡}
n|t	j,krP| -¡ dv }
n|t	j.krY|}
nW d S ||
f| j|< d| _|  ¡  d|› d|› ¡ W d S  ty˜   t | dd|› d|j/› ¡ |  |¡}| |¡ Y d S w )Nr2   r   ú(r‡   r*   TúChanged ú to zInvalid Enum Value (ú'z' is not a valid z value.
Valid values are: rÄ   c                 s  s    | ]}|j V  qd S r¾   )r“   ©rÆ   Úvrp   rp   rq   rƒ  £  s   € z7GGUFEditorWindow.on_metadata_changed.<locals>.<genexpr>r  zInvalid ValuezThe value 'z' is not valid for type )0r§   rš   r0  r¤   rr   r  Ú	get_fieldr\  r]  r   rÐ   rª   r•   ÚAttributeErrorÚsplitr~  r¬   r  r  ry  rw   r5  r6  r”   r   Úwarningr¿   rÑ   r^  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  ré   r  rs   rð   r“   )rd   r¤   rš   r¨   rÍ   rµ   rd  r  rÎ   rŸ   Úconverted_valueÚ
value_partÚformatted_valuerÇ   Úoriginal_valuerp   rp   rq   rX  k  s–   




€ø

ÿý

€ö

 
ûz$GGUFEditorWindow.on_metadata_changedc                 C  s    |   ¡ }| d¡}| d¡}t | dd|› dtjjtjjB tjj¡}|tjjkrN| j |¡ | j	 
|¡ || jv r>| j|= d| _|  ¡  d|› d¡ d S d S )	NrÍ   rš   zConfirm Removalz2Are you sure you want to remove the metadata key 'z'?TzMarked z for removal)ræ   rç   r   ÚquestionrY   ÚYesÚNor0  Ú	removeRowr  rî   r  r  r5  r6  )rd   rë   rÍ   rš   Úreplyrp   rp   rq   rc  Î  s    


ý
÷z GGUFEditorWindow.remove_metadatac              	   C  sÈ  |   ¡ }| d¡}| d¡}d}| jr| j |¡}|r|js dS |  |¡}|du r+dS | ¡ }t| ƒ}| d|j	› d¡ t
|ƒ}tƒ }	|D ]}
|	 |
j› d|
j› d|
j¡ qFzt|ttfƒrp||ƒ}
|	 |
j› d|› d¡ W n ttfy|   Y nw | |	¡ ttjjtjjB ƒ}|j |j¡ |j |j¡ | |¡ | ¡ tj j!krâ|	 "¡ }||ƒ}
|jd |f| j#|< d	| _$|
j› d|› d}| j% &|d
¡}|rÓ| '|¡ |  (¡  )d|› d|› ¡ dS dS )zEdit an enum metadata field.rÍ   rš   Nró   rô   r†   r‡   r   Tr2   r’  r“  )*ræ   rç   r  r—  r\  r]  Úcontentsr   r7   r¿   r   r   r©   r“   rª   r«   r¬   ru   r­   r”   r•   r@   r   rY   rZ   r[   r\   rD   r]   r^   r_   r®   r¯   r°   r±   r  r  r0  r¤   rw   r5  r6  )rd   rë   rÍ   rš   rd  rÎ   r²   r³   rg   r´   rŸ   rm   rµ   r    Útarget_itemrp   rp   rq   rb  ä  sT   



 €ÿ


ñz#GGUFEditorWindow.edit_metadata_enumc                   s†  ˆ  ¡ }| d¡}| d¡}|tv rˆ |¡ d S d }ˆjr$ˆj |¡}|r1|jr1|jd tjkr3d S |jd }ˆ 	|¡}t
|||ˆƒ}| ¡ tjjkrÁ| ¡ }tj||ffˆj|< dˆ_ˆ |¡‰ ˆ d ur‹|tjkr‹dd ‡ ‡fdd	„|d d
… D ƒ¡› t|ƒd
kr…dnd› d}	ndd dd	„ |d d
… D ƒ¡› t|ƒd
kr¢dnd› d}	ˆj |d¡}
|
rµ|
 |	¡ ˆ ¡  d|› ¡ d S d S )NrÍ   rš   r   r*   Tr{  rÄ   c                 3  s    | ]	}ˆ  |ˆ ¡V  qd S r¾   )ry  r•  ©rÎ   rd   rp   rq   rƒ  B  s   € z7GGUFEditorWindow.edit_array_metadata.<locals>.<genexpr>r"  r|  r&   r}  c                 s  r  r¾   rz  r•  rp   rp   rq   rƒ  D  r„  r2   zUpdated array values for )ræ   rç   r`  Úedit_tokenizer_metadatar  r—  r\  r   rþ   ru  rÃ   r®   r   r¯   r°   r÷   r  r  r]  rÐ   rÑ   r>   r0  r¤   rw   r5  r6  )rd   rë   rÍ   rš   rd  rÌ   rË   r³   Ú
new_valuesrh  r¥  rp   r¦  rq   r_     s6   





@8
ïz$GGUFEditorWindow.edit_array_metadatac                 C  sØ  | j sdS | j  tjjj¡}| j  tjjj¡}| j  tjjj¡}|r'|  |¡ng }|r0|  |¡ng }|r9|  |¡ng }tjjj| j	v rO| j	tjjj \}\}}tjjj| j	v rc| j	tjjj \}\}}tjjj| j	v rw| j	tjjj \}\}}t
|||| ƒ}	|	 ¡ tjjkrê|	 ¡ \}
}}|rŸtj|jd |
ff| j	tjjj< |r±tj|jd |ff| j	tjjj< |rÃtj|jd |ff| j	tjjj< d| _|  tjjj|
¡ |  tjjj|¡ |  tjjj|¡ |  ¡  d¡ dS dS )z3Edit the linked tokenizer metadata arrays together.Nr*   TzUpdated tokenizer data)r  r—  ÚggufÚKeysÚ	TokenizerÚLISTÚ
TOKEN_TYPEÚSCORESru  r  r%   r®   r   r¯   r°   r½   r   rþ   r\  r  Úupdate_tokenizer_displayr5  r6  )rd   Útrigger_keyÚtokens_fieldÚtoken_types_fieldÚscores_fieldr:   r;   r<   rL  r³   Ú
new_tokensÚnew_token_typesÚ
new_scoresrp   rp   rq   r§  K  sF   þþþãz(GGUFEditorWindow.edit_tokenizer_metadatac                 C  s   t | j ¡ ƒD ]>}| j |d¡}|rE| ¡ |krEdd dd„ |dd… D ƒ¡› t|ƒdkr/dnd	› d
}| j |d¡}|rB| |¡  dS qdS )z>Update the display of a tokenizer field in the metadata table.r   r{  rÄ   c                 s  r  r¾   rz  r•  rp   rp   rq   rƒ  ˆ  r„  z<GGUFEditorWindow.update_tokenizer_display.<locals>.<genexpr>Nr"  r|  r&   r}  r2   )ra   r0  ÚrowCountr¤   rr   rÑ   r>   rw   )rd   rÍ   Úvaluesrš   re  rh  râ   rp   rp   rq   r¯  ƒ  s   8
€ùz)GGUFEditorWindow.update_tokenizer_displayc                 C  sÊ  t | ƒ}| ¡ tjjkrã| ¡ \}}}|st | dd¡ d S t| j	 
¡ ƒD ]}| j	 |d¡}|rD| ¡ |krDt | dd|› d¡  d S q%| j	 
¡ }| j	 |¡ t|ƒ}| | ¡ tjj @ ¡ | j	 |d|¡ t|jƒ}| | ¡ tjj @ ¡ | j	 |d|¡ tt|ƒƒ}	|	 |	 ¡ tjjB ¡ | j	 |d|	¡ tƒ }
t|
ƒ}| dddd¡ td	ƒ}| d
|¡ | d|¡ |j | j¡ | |¡ | j	  |d|
¡ ||f| j!|< d| _"|  #¡  $d|› ¡ d S d S )NzInvalid KeyzKey cannot be emptyr   zDuplicate KeyzKey 'z' already existsr*   r2   rR  rš   rÍ   r3   TzAdded new metadata key )%rø   r®   r   r¯   r°   r½   r   rš  ra   r0  r·  r¤   rr   r[  r   rŽ   r   r   r   r‘   r’   r“   ru   r
   r   rà   r   rÞ   rJ   rD   rc  r@   rá   r  r  r5  r6  )rd   r³   rÍ   r  rª   rš   r¨   re  r¡   râ   ri  rj  rl   rp   rp   rq   r1  Ž  sH   €


ÎzGGUFEditorWindow.add_metadatac              
   C  s   | j st | dd¡ d S | js| js| jst | dd¡ d S t | ddd¡\}}|s,d S z;|  	¡  
d|› d	¡ t ¡  d
}| j  tjjj¡}|rN| ¡ }t||| j jd}d }| j  tjjj¡}|ro| ¡ }|d uro||_| j j ¡ D ]S}|jtjjjks…|j d¡r†qu|j| jv rqud }|j| jv r§| j|j \}}	|tjkr¦|	\}}	n| ¡ }	|jd }|tjkrº|jd }|	d urÈ|j|j|	||d qu| j ¡ D ]#\}
\}}	| j  |
¡d urÝqÎd }|tjkrè|	\}}	|j|
|	||d qÎ| j j D ]}|j!|j|j"|j"j#|j$d qö| %t&|ƒ¡ | '¡  | (¡  |j)dd | *¡  |  	¡  
d|› ¡ t +| ddtj,j-tj,j.B tj,j-¡}|tj,j-krft/|dƒ| _ || _0| j1 2|¡ |  3¡  |  4¡  i | _t5ƒ | _d| _W d S W d S  t6y } zt 7| ddt8|ƒ› ¡ |  	¡  
d¡ W Y d }~d S d }~ww )NzNo File OpenzPlease open a GGUF file firstz
No ChangeszNo changes to savezSave GGUF File Asr&   rJ  z
Saving to r@  Úunknown)ÚarchÚ	endianesszGGUF.r   rO  )Úsub_type)Ú	raw_shapeÚ	raw_dtypeF)Úprogressz	Saved to zOpen Saved Filez,Would you like to open the newly saved file?rA  rB  zFailed to save file: zError saving file)9r  r   rš  r  r  r  rï   r   ÚgetSaveFileNamer5  r6  r   rC  r—  r©  rª  ÚGeneralÚARCHITECTUREr¤  r   r»  Ú	ALIGNMENTÚdata_alignmentrY  r¸  r“   Ú
startswithr   rþ   r\  Úadd_key_valuerZ  r†  Ú
add_tensorr¥   rˆ  r‡  Úopen_output_filer   Úwrite_header_to_fileÚwrite_kv_data_to_fileÚwrite_tensors_to_fileÚcloserŸ  rY   r   r¡  r   r  r*  rw   rD  rE  rí   rF  rG  ru   )rd   rH  rL  rº  rd  ÚwriterÚ	alignmentr¼  r  rª   rÍ   r‹  r£  rÇ   rp   rp   rq   r-  Ä  sœ   ÿ
€


€
ýö€þzGGUFEditorWindow.save_file)rd  r    r  r`   )rÍ   ru   r  rv  )rª   r   rÎ   rò   r  ru   )rd  r    r  ru   )r¿   rÀ   rÁ   r6   r   rI  r,  rD  ru  r]  ry  r^  rE  rX  rc  rb  r_  r§  r¯  r1  r-  rÂ   rp   rp   rn   rq   r  .  s&    J

[



"#c<+86r  r  ÚNonec                  C  sÎ   t jdd} | jdddd | jddd	d
 |  ¡ }tj|jr"tjntjd t	t
jƒ}tƒ }| ¡  |jr^tj |j¡rJ|j d¡rJ| |j¡ nt d|j› ¡ t |dd|j› ¡ t
 | ¡ ¡ d S )NzGUI GGUF Editor)rõ   Ú
model_pathú?z*path to GGUF model file to load at startup)ÚnargsÚhelpz	--verboseÚ
store_truezincrease output verbosity)ÚactionrÓ  )Úlevelz.ggufzInvalid model path: zInvalid Model Pathz9The specified file does not exist or is not a GGUF file: )ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsÚloggingÚbasicConfigÚverboseÚDEBUGÚINFOr   ÚsysÚargvr  ÚshowrÐ  ÚosÚpathÚisfileÚendswithrI  ÚloggerÚerrorr   rš  Úexitr®   )ÚparserÚargsÚappÚwindowrp   rp   rq   Úmain9  s$   

ýrî  Ú__main__)r  rÏ  )MÚ
__future__r   rÛ  r×  rã  rà  rÜ   ÚenumÚpathlibr   Útypingr   r   r   r   rS  r
  ÚPySide6.QtWidgetsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ÚPySide6.QtCorer   ÚenvironÚ__file__re   Úexistsrä  Úinsertru   r©  r   r   r   r    Úgguf.constantsr!   r"   r#   r$   Ú	getLoggerrç  rª  r«  r­  ÚRopeÚSCALING_TYPEÚLLMÚPOOLING_TYPErÁ  Ú	FILE_TYPErÏ   r¬  r®  r`  r%   rÃ   rø   r  rî  r¿   rp   rp   rp   rq   Ú<module>   s^   T"




ü	ý  $   ";      

ÿ