o
    հi                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZmZ d dlZd dlZd dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& dej'vre	e(j)j)j)j)d * rej+,d e-e	e(j)j)j) d dl.Z.d d	l.m/Z/m0Z0m1Z1m2Z2 d d
l3m4Z4m5Z5m6Z6m7Z7 e8dZ9e.j:j;j<e4e.j:j=j>e5e.j:j?j@e6e.j:jAjBe7iZCe.j:j;jDe.j:j;j<e.j:j;jEgZFG dd de#ZGG dd de#ZHG dd de#ZIG dd deZJdddZKeLdkreK  dS dS )    )annotationsN)Path)AnyOptionalTupleType)QApplicationQMainWindowQWidgetQVBoxLayoutQHBoxLayoutQPushButtonQLabel	QLineEditQFileDialogQTableWidgetQTableWidgetItem	QComboBoxQMessageBox
QTabWidget	QTextEditQFormLayoutQHeaderViewQDialogQDialogButtonBox)QtNO_LOCAL_GGUFzgguf-py)
GGUFReader
GGUFWriterGGUFValueTypeReaderField)	TokenTypeRopeScalingTypePoolingTypeGGMLQuantizationTypezgguf-editor-guic                      sf   e Zd Zd fdd	Zdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Z  ZS )TokenizerEditorDialogNc                   s  t  | | d | dd |r| ng | _|r | ng | _|r)| ng | _tt	| jt	| jt	| j}t	| j|k rP| j
dg|t	| j   t	| j|k re| j
dg|t	| j   t	| j|k rz| j
dg|t	| j   t| }t }|td t | _| jd | jj| j || j d	| _d| _td
t	| j| j d
 | j | _td| j | _|| j td}|j| j || td}	|	j| j ||	 || t | _| j d | j!g d | j" #dt$j%j& | j" #d
t$j%j' | j" #dt$j%j& | j" #dt$j%j& || j t }
td}|j| j( |
| td}|j| j) |
| |
*  ||
 t+t+j,j-t+j,j.B }|j/| j0 |j1| j2 || t3t4t	| j| _5| 6  d S )NzEdit Tokenizer Datai  iX   r           Filter:zType to filter tokens...d      
Page 1 of PreviousNext   )IndexTokenr   Score      z	Add TokenRemove Selected)7super__init__setWindowTitleresizecopytokenstoken_typesscoresmaxlenextendr   r   	addWidgetr   r   filter_editsetPlaceholderTexttextChangedconnectapply_filter	page_sizecurrent_pagetotal_pages
page_labelr   clickedprevious_page	next_page	addLayoutr   tokens_tablesetColumnCountsetHorizontalHeaderLabelshorizontalHeadersetSectionResizeModer   
ResizeModeResizeToContentsStretch	add_tokenremove_selected
addStretchr   StandardButtonOkCancelacceptedacceptrejectedrejectlistrangefiltered_indices	load_page)selfr:   r;   r<   parentmax_lenlayoutfilter_layout	prev_pagerL   controls_layout
add_buttonremove_buttonbuttons	__class__ P/home/ubuntu/.local/lib/python3.10/site-packages/gguf/scripts/gguf_editor_gui.pyr6   3   sn   
"






zTokenizerEditorDialog.__init__c                 C  s   | j   }|sttt| j| _ng | _t| jD ]\}}|t	| v r.| j
| qtdt| j| j d | j | _d| _| jd| j  |   dS )z+Filter the tokens based on the search text.r*   r   r+   N)rA   textlowerr`   ra   r>   r:   rb   	enumeratestrappendr=   rF   rH   rG   rI   setTextrc   )rd   filter_textitokenrp   rp   rq   rE      s   "z"TokenizerEditorDialog.apply_filterc                 C  H   | j dkr"|  j d8  _ | jd| j d  d| j  |   dS dS z#Go to the previous page of results.r   r*   Page  of NrG   rI   rw   rH   rc   rd   rp   rp   rq   rK      
   
 z#TokenizerEditorDialog.previous_pagec                 C  N   | j | jd k r%|  j d7  _ | jd| j d  d| j  |   dS dS zGo to the next page of results.r*   r}   r~   NrG   rH   rI   rw   rc   r   rp   rp   rq   rL      
    zTokenizerEditorDialog.next_pagec              
   C  s  | j d | j| j }t|| j t| j}| j ||  tt||D ]\}}| j| }t	t
|}|tjj| || tjj @  | j |d| t	t
| j| }| j |d| |t| jk rn| j| nd}zt|}	|	j d| d}
W n ttfy   d| d}
Y nw t	|
}|tjj| || tjj @  | j |d| |t| jk r| j| nd}t	t
|}| j |d| q&| j j| j d	S )
z(Load the current page of tokenizer data.r   r*    ()	Unknown (r2   r'   r3   N)rN   setRowCountrG   rF   minr>   rb   rt   ra   r   ru   setDatar   ItemDataRoleUserRolesetFlagsflagsItemFlagItemIsEditablesetItemr:   r;   r!   name
ValueErrorKeyErrorr<   cellDoubleClickedrD   handle_cell_double_click)rd   	start_idxend_idxrowry   orig_idx
index_item
token_item
token_typeenum_valdisplay_text	type_itemscore
score_itemrp   rp   rq   rc      s6   
zTokenizerEditorDialog.load_pagec                 C  s@   |dkr| j |d}|r|tjj}| || dS dS dS )zCHandle double-click on a cell, specifically for token type editing.r2   r   N)rN   itemdatar   r   r   edit_token_type)rd   r   column	orig_itemr   rp   rp   rq   r      s   z.TokenizerEditorDialog.handle_cell_double_clickc              	   C  s^  |t | jk r| j| nd}t| }|d t|}t }tD ]}||j d|j	 d|j	 q zt
|trHt|}||j d| d W n ttfyT   Y nw || ttjjtjjB }|j|j |j|j || | tjjkr| }	t|	}|j d|	 d}
| j|d}|r||
 | t!j"j#|	 |	| j|< dS dS )zEEdit a token type using a dialog with a dropdown of all enum options.r   zSelect Token Typer   r   r2   N)$r>   r;   r   r7   r   r   r!   addItemr   value
isinstanceintsetCurrentTextr   r   r@   r   rY   rZ   r[   r\   rD   r]   r^   r_   exec
DialogCodeAcceptedcurrentDatarN   r   rw   r   r   r   r   )rd   r   r   current_valuedialogrg   combor   rm   	new_valuer   r   rp   rp   rq   r      s<   
 



z%TokenizerEditorDialog.edit_token_typec                 C  s   | j d | jd | jd t| j d }| j  }|r&|dv r,| j| t	dt| j| j
 d | j
 | _| jd | _| jd| jd  d| j  |   dS )z'Add a new token to the end of the list.r&   r   r'   r*   r}   r~   N)r:   rv   r;   r<   r>   rA   rr   rs   rb   r=   rF   rH   rG   rI   rw   rc   )rd   r   rx   rp   rp   rq   rV     s   " zTokenizerEditorDialog.add_tokenc           
      C  sl  g }| j  D ]}| }||vr|| q|sdS g }|D ]}| j |d}|r4||tjj q|j	dd |D ]#}|t
| jk rJ| j|= |t
| jk rU| j|= |t
| jk r`| j|= q=g | _| j  }t| jD ]\}}	|r~|t|	 v r| j| qptdt
| j| j d | j | _t| j| jd | _| jd| jd  d| j  |   dS )z'Remove selected tokens from all arrays.Nr   Treverser*   r}   r~   )rN   selectedItemsr   rv   r   r   r   r   r   sortr>   r:   r;   r<   rb   rA   rr   rs   rt   ru   r=   rF   rH   r   rG   rI   rw   rc   )
rd   selected_rowsr   r   orig_indicesr   idxrx   ry   rz   rp   rp   rq   rW   #  sB   
" z%TokenizerEditorDialog.remove_selectedc                 C  s   | j | j| jfS )z!Return the edited tokenizer data.)r:   r;   r<   r   rp   rp   rq   get_dataO  s   zTokenizerEditorDialog.get_dataN)__name__
__module____qualname__r6   rE   rK   rL   rc   r   r   rV   rW   r   __classcell__rp   rp   rn   rq   r%   2   s    T/+,r%   c                      sp   e Zd Zd fdd	Zdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd ZdddZdd Z  ZS )ArrayEditorDialogNc                   sN  t  | | d | dd || _|| _|| _d | _|tv r+|t	j
kr+t| | _t| }| jd urht }td| jj d}|| ddd | jD }td	| }	|	d
 ||	d || t }
|
td t | _| jd | jj| j |
| j d| _d| _tdt|| j d | j | _td| j | _|
| j td}|j | j! |
| td}|j | j" |
| ||
 t# | _$| jd ur| j$%d | j$&g d | j$' (dt)j*j+ | j$' (dt)j*j, | j$' (dt)j*j+ n$| j$%d | j$&ddg | j$' (dt)j*j+ | j$' (dt)j*j, || j$ t }td}|j | j- || td}|j | j. || | jd urqtd}|j | j/ || |0  || t1t1j2j3t1j2j4B }|j5| j6 |j7| j8 || t9t:t| j| _;| <  d S )NzEdit Array Valuesi  i  zEditing z values:, c                 S  s   g | ]}|j  d |j qS )=)r   r   ).0erp   rp   rq   
<listcomp>l  s    z.ArrayEditorDialog.__init__.<locals>.<listcomp>zAvailable values: Tr*   r(   zType to filter values...r)   r   r+   r,   r-   r3   )r/   ValueActionsr2   r/   r   zAdd Itemr4   zBulk Edit Selected)=r5   r6   r7   r8   array_valueselement_typekey	enum_typeKEY_TO_ENUM_TYPEr   INT32r   r   r   r   r@   joinsetWordWraprM   r   rA   rB   rC   rD   rE   rF   rG   r=   r>   rH   rI   r   rJ   rK   rL   r   items_tablerO   rP   rQ   rR   r   rS   rT   rU   add_itemrW   bulk_edit_selectedrX   r   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   )rd   r   r   r   re   rg   enum_info_layout
enum_labelenum_valuesenum_values_labelrh   ri   rL   rj   rk   rl   bulk_edit_buttonrm   rn   rp   rq   r6   U  s   





 







zArrayEditorDialog.__init__c              
   C  s&  | j   }|sttt| j| _n[g | _t| jD ]R\}}| j	dur`t
|tr`z| 	|}|j d| d }||v rE| j| W q ttfy_   |t| v r]| j| Y qw |t| v rn| j| qtdt| j| j d | j | _d| _| jd| j  |   dS )z1Filter the array values based on the search text.Nr   r   r*   r   r+   )rA   rr   rs   r`   ra   r>   r   rb   rt   r   r   r   r   rv   r   r   ru   r=   rF   rH   rG   rI   rw   rc   )rd   rx   ry   r   r   r   rp   rp   rq   rE     s0   
"zArrayEditorDialog.apply_filterc                 C  r{   r|   r   r   rp   rp   rq   rK     r   zArrayEditorDialog.previous_pagec                 C  r   r   r   r   rp   rp   rq   rL     r   zArrayEditorDialog.next_pagec              
   C  s  | j d | j| j }t|| j t| j}| j ||  tt||D ]\}}| j| }| j	| }t
t|}|tjj| || tjj @  | j |d| | jdurzt|ttjfrs| |}|j d| d}	nt|}	W n ttfy   d| d}	Y nw t
|	}
|
tjj| |
|
 tjj @  | j |d|
 td}|d| |j | j! t" }t#|}|$d	d	d	d	 |%| |&  | j '|d	| q&t
t|}
| j |d|
 q&dS )
z&Load the current page of array values.r   Nr   r   r   r*   Editr   r2   )(r   r   rG   rF   r   r>   rb   rt   ra   r   r   ru   r   r   r   r   r   r   r   r   r   r   r   r   numpysignedintegerr   r   r   r   setPropertyrJ   rD   edit_array_enum_valuer
   r   setContentsMarginsr@   rX   setCellWidget)rd   r   r   r   ry   r   r   r   r   r   
value_itemedit_buttonbutton_widgetbutton_layoutrp   rp   rq   rc     sJ   




zArrayEditorDialog.load_pagec                 C  s   |   }|d}| j|d}| j|d}|rC|rE| jrG| || jrI|tjj	}|tjj	}t
|ttttfrK|| j|< dS dS dS dS dS dS )z1Handle editing an enum value in the array editor.r   r   r*   N)senderpropertyr   r   r   edit_enum_valuer   r   r   r   r   r   floatru   boolr   )rd   buttonr   r   new_itemr   r   rp   rp   rq   r   '  s   
z'ArrayEditorDialog.edit_array_enum_valuec                 C  s  | j sdS t }| j D ]	}||  q|s"t| dd dS t| }|	d| j j
 d t|}|tdt| d t }| j D ]}||j d|j d	|j qH|| ttjjtjjB }|j|j |j|j || | tjjkr| }|  |}|j d| d	}	|D ]/}
| j |
d
}| j |
d}|r|r|!t"j#j$}|| j%|< |&|	 |'t"j#j$| qdS dS )z"Edit multiple enum values at once.NzNo Selectionz'Please select at least one row to edit.z
Bulk Edit z ValueszSet z selected items to:r   r   r   r*   )(r   setr   r   addr   r   informationr   r7   r   r   r@   r   r>   r   r   r   r   r   rY   rZ   r[   r\   rD   r]   r^   r_   r   r   r   r   r   r   r   r   r   r   rw   r   )rd   r   r   r   rg   r   r   rm   r   r   r   r   r   r   rp   rp   rq   r   6  sF   
 




z$ArrayEditorDialog.bulk_edit_selectedc                 C  s   t | j}| jd urt| jd j}| j| n| jtjkr&| jd n| jd | j	| t
dt | j	| j d | j | _| jd | _| jd| jd  d| j  |   d S )Nr   r&   r*   r}   r~   )r>   r   r   r`   r   rv   r   r   STRINGrb   r=   rF   rH   rG   rI   rw   rc   )rd   r   default_valuerp   rp   rq   r   g  s   

" zArrayEditorDialog.add_itemc              
   C  s  g }| j  D ]}| }||vr|| q|sd S t }|D ]}| j |d}|r5||tjj	 q |j
dd |D ]}| j|= q>g | _| j  }t| jD ][\}}	|sa| j| qT| jd urt|	trz| |	}
|
j d|	 d }||v r| j| W qT ttfy   |t|	 v r| j| Y qTw |t|	 v r| j| qTtdt| j| j d | j | _t| j| jd | _| jd| jd  d| j  |    d S )	Nr   Tr   r   r   r*   r}   r~   )!r   r   r   rv   r`   r   r   r   r   r   r   r   rb   rA   rr   rs   rt   r   r   r   r   r   r   ru   r=   r>   rF   rH   r   rG   rI   rw   rc   )rd   r   r   r   r   r   r   rx   ry   r   r   r   rp   rp   rq   rW     sR   


" z!ArrayEditorDialog.remove_selectedr   r   r   Type[enum.Enum]c              	   C  s  | j |d}|r|tjj}ndS | j| }t| }|d|j	 d t
|}td|j	 d}|| t }	|D ]}
|	|
j d|
j d|
j q=zt|tre||}
|	|
j d| d W n ttfyq   Y nw ||	 ttjjtjjB }|j|j |j|j || | tj j!kr|	" }||}
|
j d| d}| j |d	}|r|#| |$tjj| || j|< d
S dS )zFEdit an enum value using a dialog with a dropdown of all enum options.r   NSelect  Valuez	Select a z value:r   r   r*   TF)%r   r   r   r   r   r   r   r   r7   r   r   r   r@   r   r   r   r   r   r   r   r   r   r   rY   rZ   r[   r\   rD   r]   r^   r_   r   r   r   r   rw   r   )rd   r   r   r   r   r   r   rg   descriptionr   r   rm   r   r   r   rp   rp   rq   r     sJ   

 




z!ArrayEditorDialog.edit_enum_valuec                 C  s   | j S r   r   r   rp   rp   rq   get_array_values  s   z"ArrayEditorDialog.get_array_values)NN)r   r   r   r   )r   r   r   r6   rE   rK   rL   rc   r   r   r   rW   r   r   r   rp   rp   rn   rq   r   T  s    k!81
47r   c                      s(   e Zd Zd fdd	Zd	ddZ  ZS )
AddMetadataDialogNc                   s   t  | | d | dd t| }t }t | _|d| j t	 | _
tD ]}|tjkr8| j
|j| q)|d| j
 t | _|d| j || ttjjtjjB }|j| j |j| j || d S )NAdd Metadatai     zKey:zType:zValue:)r5   r6   r7   r8   r   r   r   key_editaddRowr   
type_combor   ARRAYr   r   r   
value_editrM   r   rY   rZ   r[   r\   rD   r]   r^   r_   r@   )rd   re   rg   form_layout
value_typerm   rn   rp   rq   r6     s(   


zAddMetadataDialog.__init__returnTuple[str, GGUFValueType, Any]c                 C  s
  | j  }| j }| j }|tjkrt	t
|}nd|tjkr)tt
|}nW|tjkr6tt
|}nJ|tjkrCtt
|}n=|tjkrPtt
|}n0|tjkr]tt
|}n#|tjkrjtt|}n|tjkrv| dv }n
|tjkr~|}n|}|||fS )Ntrueyes1)r   rr   r   r   r   toPlainTextr   UINT8npuint8r   INT8int8UINT16uint16INT16int16UINT32uint32r   int32FLOAT32float32r   BOOLrs   r   )rd   r   r  
value_textr   rp   rp   rq   r     s.   












zAddMetadataDialog.get_datar   )r  r  )r   r   r   r6   r   r   rp   rp   rn   rq   r     s    r   c                      s   e Zd Z fddZdd Zdd Zdd Zd	d
 Zd0ddZd1ddZ	d2ddZ
d3ddZdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Z  ZS )4GGUFEditorWindowc                   sR   t    | d | dd d | _d | _d| _i | _t | _	d| _
|   d S )NzGGUF Editori  i   F)r5   r6   r7   r8   current_filereadermodifiedmetadata_changesr   metadata_to_remove on_metadata_changed_is_connectedsetup_uir   rn   rp   rq   r6   /  s   

zGGUFEditorWindow.__init__c           
      C  sP  t  }| | t|}t }t | _| jd || j td}|j	
| j || td}|j	
| j || || t | _t  | _t| j}t | _| jd | jg d | j dtjj | j dtjj | j dtjj | j d	tjj || j t }td
}|j	
| j || |  || t  | _t| j}	t | _| jd | jg d | j dtjj | j dtjj | j dtjj | j d	tjj | j dtjj |	| j | j| jd | j| jd || j |   !d d S )NTz	Open GGUFz
Save As...r.   )Keyr   r   r   r   r*   r2   r3   r      )Namer   ShapeElementszSize (bytes)MetadataTensorsReady)"r
   setCentralWidgetr   r   r   file_path_editsetReadOnlyr@   r   rJ   rD   	open_file	save_filerM   r   tabsmetadata_tabr   metadata_tablerO   rP   rQ   rR   r   rS   rU   rT   add_metadatarX   tensors_tabtensors_tableaddTab	statusBarshowMessage)
rd   central_widgetmain_layoutfile_layoutopen_buttonsave_buttonmetadata_layoutmetadata_controlsadd_metadata_buttontensors_layoutrp   rp   rq   r   >  s\   







zGGUFEditorWindow.setup_uic              
   C  s   z=|   d| d t  t|d| _|| _| j| | 	  | 
  i | _t | _d| _|   d|  W dS  tyc } zt| ddt|  |   d	 W Y d
}~dS d
}~ww )zLoad a GGUF file by pathzLoading ...rFzLoaded TErrorzFailed to open file: zError loading fileN)r5  r6  r   processEventsr   r  r  r*  rw   load_metadataload_tensorsr  r   r  r  	Exceptionr   criticalru   )rd   	file_pathr   rp   rp   rq   	load_file  s&   zGGUFEditorWindow.load_filec                 C  s*   t | ddd\}}|sd S | | d S )NzOpen GGUF Filer&   "GGUF Files (*.gguf);;All Files (*))r   getOpenFileNamerI  )rd   rH  _rp   rp   rq   r,    s   zGGUFEditorWindow.open_filec                 C  sV  | j d | jsd S | jr2t  td | j j| j	 W d    n1 s*w   Y  d| _t
| jj D ]b\}\}}| j | t|}|| tjj @  | j |d| |jsed}nP|jd tjkrt|jd }|jd j}| |}|d ur|jd tjkr|j}d| | d|  }nt|jd j}| |}|d ur|jd tjkr|j}t|}	|	|	 tjj @  | j |d|	 | |}
t|
}t|jdkr|jd tjkr|| tjjB  n|| tjj @  | j |d	| t }t |}|!d	d	d	d	 |jrM|jd tjkrMt"d
}|#d| |#d| |j$%| j& |'| |t(v rL|)d |*d n,t|jdkry| |d uryt"d
}|#d| |#d| |j$%| j+ |'| t"d}|#d| |#d| |j$%| j, |'| | j -|d| q:| j j%| j	 d| _d S )Nr   ignoreFN/Ar*   []r2   r   r   r   zEdit Tokenizerz Edit all tokenizer data togetherRemover3   T).r0  r   r  r  warningscatch_warningsfilterwarningsitemChanged
disconnecton_metadata_changedrt   fieldsitems	insertRowr   r   r   r   r   r   r   typesr   r   r>   r   get_enum_for_keyr   r   ru   format_field_valuer
   r   r   r   r   rJ   rD   edit_array_metadatar@   TOKENIZER_LINKED_KEYSrw   
setToolTipedit_metadata_enumremove_metadatar   )rd   ry   r   fieldkey_itemtype_str
nest_countr   r   r   	value_strr   actions_widgetactions_layoutr   rl   rp   rp   rq   rD    s~   








 


zGGUFEditorWindow.load_metadatard  r    r  r`   c                 C  s   |j r|j d tjkrg S |j d }g }t|j}|tjkr?t|D ]}tt|j	d|| d d   dd}|
| q"|S | jr_|| jjv r_t|D ]}|
|j	d|| d   d  qL|S )z'Extract all values from an array field.r   r*   rO  r2   utf-8encoding)r\  r   r   r>   r   r   ra   ru   bytespartsrv   r  gguf_scalar_to_np)rd   rd  	curr_typer   total_elementselement_posvalue_stringrp   rp   rq   extract_array_values  s   


&"z%GGUFEditorWindow.extract_array_valuesr   ru   Optional[Type[enum.Enum]]c                 C  s
   t |S )z/Get the enum type for a given key if it exists.)r   get)rd   r   rp   rp   rq   r]    s   
z!GGUFEditorWindow.get_enum_for_keyr   r   r   r   c              	   C  sR   zt |ttfr||}|j d| dW S W t|S  ttfy(   Y t|S w )z&Format a value as an enum if possible.r   r   )r   r   ru   r   r   r   )rd   r   r   
enum_valuerp   rp   rq   format_enum_value  s   z"GGUFEditorWindow.format_enum_valuec                   s>  |j sdS t|j dkrE|j d }|tjkr!tt|jd ddS | jrE|| jjv rE|jd d }| 	|j
}|d urA| ||S t|S |j d tjkr| | tdt }| 	|j
}|d urxg }t|D ]}||  | | qin fdd	t|D }d
d|  t t|krd dS d dS dS )NrN  r*   r   rO  rk  rl  r"  c                   s   g | ]}t  | qS rp   ru   )r   ry   r   rp   rq   r   B  s    z7GGUFEditorWindow.format_field_value.<locals>.<listcomp>[ r   , ...r&    ]zComplex value)r\  r>   r   r   ru   rn  ro  r  rp  r]  r   ry  r   ru  r   ra   rv   r   strip)rd   rd  rq  r   r   render_elementarray_elementsry   rp   r   rq   r^  &  s0   


4z#GGUFEditorWindow.format_field_valuec           	      C  sT  | j d | jsd S t| jjD ]\}}| j | t|j}||	 t
jj @  | j |d| t|jj}||	 t
jj @  | j |d| ddd |jD }t|}||	 t
jj @  | j |d| tt|j}||	 t
jj @  | j |d| t|jd}||	 t
jj @  | j |d	| qd S )
Nr   r*   u    × c                 s      | ]}t |V  qd S r   rz  )r   drp   rp   rq   	<genexpr>\      z0GGUFEditorWindow.load_tensors.<locals>.<genexpr>r2   r3   ,r.   )r3  r   r  rt   tensorsr[  r   r   r   r   r   r   r   r   tensor_typer   shaperu   
n_elementsn_bytes)	rd   ry   tensor	name_itemr   	shape_str
shape_itemelements_item	size_itemrp   rp   rq   rE  H  s,   
zGGUFEditorWindow.load_tensorsc                 C  s2  |  dkrd S | }| j|d}d }|r| }| }d }| jr,|r,| j|}|r3|jr3|s5d S |jd }| |}|d ur|t	j
krz]z	|| }	|	j}
W n* ttfy|   d|v rvd|v rv|dd dd  }t|}
nt|}
Y nw ||
 ||
f| j|< d| _| |
|}|| |  d| d|  W d S  ttfy } z.t| d	| dd
| d|j dddd |D   | |}|| W Y d }~d S d }~ww z|t	jkrtt|}
nl|t	j krt!t|}
n_|t	j"krt#t|}
nQ|t	j$krt%t|}
nC|t	j&kr't't|}
n5|t	j
kr5t(t|}
n'|t	j)krCt*t+|}
n|t	j,krP|- dv }
n|t	j.krY|}
nW d S ||
f| j|< d| _|  d| d|  W d S  ty   t| dd| d|j/  | |}|| Y d S w )Nr2   r   (r   r*   TChanged  to zInvalid Enum Value ('z' is not a valid z value.
Valid values are: r   c                 s  s    | ]}|j V  qd S r   )r   r   vrp   rp   rq   r    s    z7GGUFEditorWindow.on_metadata_changed.<locals>.<genexpr>r  zInvalid ValuezThe value 'z' is not valid for type )0r   r   r0  r   rr   r  	get_fieldr\  r]  r   r   r   r   AttributeErrorsplitr~  r   r  r  ry  rw   r5  r6  r   r   warningr   r   r^  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r   r  rs   r   r   )rd   r   r   r   r   r   rd  r  r   r   converted_value
value_partformatted_valuer   original_valuerp   rp   rq   rX  k  s   










 
z$GGUFEditorWindow.on_metadata_changedc                 C  s   |   }|d}|d}t| dd| dtjjtjjB tjj}|tjjkrN| j| | j	
| || jv r>| j|= d| _|  d| d d S d S )	Nr   r   zConfirm Removalz2Are you sure you want to remove the metadata key 'z'?TzMarked z for removal)r   r   r   questionrY   YesNor0  	removeRowr  r   r  r  r5  r6  )rd   r   r   r   replyrp   rp   rq   rc    s    



z GGUFEditorWindow.remove_metadatac              	   C  s  |   }|d}|d}d}| jr| j|}|r|js dS | |}|du r+dS | }t| }|d|j	 d t
|}t }	|D ]}
|	|
j d|
j d|
j qFzt|ttfrp||}
|	|
j d| d W n ttfy|   Y nw ||	 ttjjtjjB }|j|j |j|j || | tj j!kr|	" }||}
|jd |f| j#|< d	| _$|
j d| d}| j%&|d
}|r|'| | ( )d| d|  dS dS )zEdit an enum metadata field.r   r   Nr   r   r   r   r   Tr2   r  r  )*r   r   r  r  r\  r]  contentsr   r7   r   r   r   r   r   r   r   r   ru   r   r   r   r@   r   rY   rZ   r[   r\   rD   r]   r^   r_   r   r   r   r   r  r  r0  r   rw   r5  r6  )rd   r   r   r   rd  r   r   r   rg   r   r   rm   r   r   target_itemrp   rp   rq   rb    sT   



 


z#GGUFEditorWindow.edit_metadata_enumc                   s    }|d}|d}|tv r| d S d }jr$j|}|r1|jr1|jd tjkr3d S |jd }	|}t
|||}| tjjkr| }tj||ffj|< d_|  d ur|tjkrdd fdd	|d d
 D  t|d
krdnd d}	ndddd	 |d d
 D  t|d
krdnd d}	j|d}
|
r|
|	  d|  d S d S )Nr   r   r   r*   Tr{  r   c                 3  s    | ]	} | V  qd S r   )ry  r  r   rd   rp   rq   r  B  s    z7GGUFEditorWindow.edit_array_metadata.<locals>.<genexpr>r"  r|  r&   r}  c                 s  r  r   rz  r  rp   rp   rq   r  D  r  r2   zUpdated array values for )r   r   r`  edit_tokenizer_metadatar  r  r\  r   r   ru  r   r   r   r   r   r   r  r  r]  r   r   r>   r0  r   rw   r5  r6  )rd   r   r   r   rd  r   r   r   
new_valuesrh  r  rp   r  rq   r_     s6   





@8
z$GGUFEditorWindow.edit_array_metadatac                 C  s  | j sdS | j tjjj}| j tjjj}| j tjjj}|r'| |ng }|r0| |ng }|r9| |ng }tjjj| j	v rO| j	tjjj \}\}}tjjj| j	v rc| j	tjjj \}\}}tjjj| j	v rw| j	tjjj \}\}}t
|||| }	|	 tjjkr|	 \}
}}|rtj|jd |
ff| j	tjjj< |rtj|jd |ff| j	tjjj< |rtj|jd |ff| j	tjjj< d| _| tjjj|
 | tjjj| | tjjj| |  d dS dS )z3Edit the linked tokenizer metadata arrays together.Nr*   TzUpdated tokenizer data)r  r  ggufKeys	TokenizerLIST
TOKEN_TYPESCORESru  r  r%   r   r   r   r   r   r   r   r\  r  update_tokenizer_displayr5  r6  )rd   trigger_keytokens_fieldtoken_types_fieldscores_fieldr:   r;   r<   rL  r   
new_tokensnew_token_types
new_scoresrp   rp   rq   r  K  sF   z(GGUFEditorWindow.edit_tokenizer_metadatac                 C  s   t | j D ]>}| j|d}|rE| |krEdddd |dd D  t|dkr/dnd	 d
}| j|d}|rB||  dS qdS )z>Update the display of a tokenizer field in the metadata table.r   r{  r   c                 s  r  r   rz  r  rp   rp   rq   r    r  z<GGUFEditorWindow.update_tokenizer_display.<locals>.<genexpr>Nr"  r|  r&   r}  r2   )ra   r0  rowCountr   rr   r   r>   rw   )rd   r   valuesr   re  rh  r   rp   rp   rq   r    s   8
z)GGUFEditorWindow.update_tokenizer_displayc                 C  s  t | }| tjjkr| \}}}|st| dd d S t| j	
 D ]}| j	|d}|rD| |krDt| dd| d  d S q%| j	
 }| j	| t|}|| tjj @  | j	|d| t|j}|| tjj @  | j	|d| tt|}	|	|	 tjjB  | j	|d|	 t }
t|
}|dddd td	}|d
| |d| |j| j || | j	 |d|
 ||f| j!|< d| _"| # $d|  d S d S )NzInvalid KeyzKey cannot be emptyr   zDuplicate KeyzKey 'z' already existsr*   r2   rR  r   r   r3   TzAdded new metadata key )%r   r   r   r   r   r   r   r  ra   r0  r  r   rr   r[  r   r   r   r   r   r   r   r   ru   r
   r   r   r   r   rJ   rD   rc  r@   r   r  r  r5  r6  )rd   r   r   r  r   r   r   re  r   r   ri  rj  rl   rp   rp   rq   r1    sH   


zGGUFEditorWindow.add_metadatac              
   C  s   | j st| dd d S | js| js| jst| dd d S t| ddd\}}|s,d S z;| 	 
d| d	 t  d
}| j tjjj}|rN| }t||| j jd}d }| j tjjj}|ro| }|d uro||_| j j D ]S}|jtjjjks|jdrqu|j| jv rqud }|j| jv r| j|j \}}	|tjkr|	\}}	n| }	|jd }|tjkr|jd }|	d ur|j|j|	||d qu| j D ]#\}
\}}	| j |
d urqd }|tjkr|	\}}	|j|
|	||d q| j j D ]}|j!|j|j"|j"j#|j$d q|%t&| |'  |(  |j)dd |*  | 	 
d|  t+| ddtj,j-tj,j.B tj,j-}|tj,j-krft/|d| _ || _0| j12| | 3  | 4  i | _t5 | _d| _W d S W d S  t6y } zt7| ddt8|  | 	 
d W Y d }~d S d }~ww )NzNo File OpenzPlease open a GGUF file firstz
No ChangeszNo changes to savezSave GGUF File Asr&   rJ  z
Saving to r@  unknown)arch	endianesszGGUF.r   rO  )sub_type)	raw_shape	raw_dtypeF)progressz	Saved to zOpen Saved Filez,Would you like to open the newly saved file?rA  rB  zFailed to save file: zError saving file)9r  r   r  r  r  r  r   r   getSaveFileNamer5  r6  r   rC  r  r  r  GeneralARCHITECTUREr  r   r  	ALIGNMENTdata_alignmentrY  r  r   
startswithr   r   r\  add_key_valuerZ  r  
add_tensorr   r  r  open_output_filer   write_header_to_filewrite_kv_data_to_filewrite_tensors_to_filecloser  rY   r  r  r   r  r*  rw   rD  rE  r   rF  rG  ru   )rd   rH  rL  r  rd  writer	alignmentr  r  r   r   r  r  r   rp   rp   rq   r-    s   




zGGUFEditorWindow.save_file)rd  r    r  r`   )r   ru   r  rv  )r   r   r   r   r  ru   )rd  r    r  ru   )r   r   r   r6   r   rI  r,  rD  ru  r]  ry  r^  rE  rX  rc  rb  r_  r  r  r1  r-  r   rp   rp   rn   rq   r  .  s&    J

[



"#c<+86r  r  Nonec                  C  s   t jdd} | jdddd | jddd	d
 |  }tj|jr"tjntjd t	t
j}t }|  |jr^tj|jrJ|jdrJ||j ntd|j  t|dd|j  t
|  d S )NzGUI GGUF Editor)r   
model_path?z*path to GGUF model file to load at startup)nargshelpz	--verbose
store_truezincrease output verbosity)actionr  )levelz.ggufzInvalid model path: zInvalid Model Pathz9The specified file does not exist or is not a GGUF file: )argparseArgumentParseradd_argument
parse_argsloggingbasicConfigverboseDEBUGINFOr   sysargvr  showr  ospathisfileendswithrI  loggererrorr   r  exitr   )parserargsappwindowrp   rp   rq   main9  s$   

r  __main__)r  r  )M
__future__r   r  r  r  r  r   enumpathlibr   typingr   r   r   r   rS  r
  PySide6.QtWidgetsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   PySide6.QtCorer   environ__file__re   existsr  insertru   r  r   r   r   r    gguf.constantsr!   r"   r#   r$   	getLoggerr  r  r  r  RopeSCALING_TYPELLMPOOLING_TYPEr  	FILE_TYPEr   r  r  r`  r%   r   r   r  r  r   rp   rp   rp   rq   <module>   s^   T"




	  $   ";      

