
    `i                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZmZ d dlZd dlZd dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& dej'        vrp e	e(          j)        j)        j)        j)        dz  *                                r<ej+        ,                    d  e- e	e(          j)        j)        j)                             d dl.Z.d d	l.m/Z/m0Z0m1Z1m2Z2 d d
l3m4Z4m5Z5m6Z6m7Z7  ej8        d          Z9e.j:        j;        j<        e4e.j:        j=        j>        e5e.j:        j?        j@        e6e.j:        jA        jB        e7iZCe.j:        j;        jD        e.j:        j;        j<        e.j:        j;        jE        gZF G d de#          ZG G d de#          ZH G d de#          ZI G d de          ZJddZKeLdk    r eK             dS dS )    )annotationsN)Path)AnyOptionalTupleType)QApplicationQMainWindowQWidgetQVBoxLayoutQHBoxLayoutQPushButtonQLabel	QLineEditQFileDialogQTableWidgetQTableWidgetItem	QComboBoxQMessageBox
QTabWidget	QTextEditQFormLayoutQHeaderViewQDialogQDialogButtonBox)QtNO_LOCAL_GGUFzgguf-py)
GGUFReader
GGUFWriterGGUFValueTypeReaderField)	TokenTypeRopeScalingTypePoolingTypeGGMLQuantizationTypezgguf-editor-guic                  V     e Zd Zd fd	Zd Zd Zd Zd Zd Zd Z	d	 Z
d
 Zd Z xZS )TokenizerEditorDialogNc                   t                                          |           |                     d           |                     dd           |r|                                ng | _        |r|                                ng | _        |r|                                ng | _        t          t          | j                  t          | j                  t          | j                            }t          | j                  |k     r3| j        
                    dg|t          | j                  z
  z             t          | j                  |k     r3| j        
                    dg|t          | j                  z
  z             t          | j                  |k     r3| j        
                    dg|t          | j                  z
  z             t          |           }t                      }|                    t          d                     t                      | _        | j                            d           | j        j                            | j                   |                    | j                   d	| _        d| _        t          d
t          | j                  | j        z   d
z
  | j        z            | _        t          d| j                   | _        |                    | j                   t3          d          }|j                            | j                   |                    |           t3          d          }	|	j                            | j                   |                    |	           |                    |           t=                      | _        | j                             d           | j        !                    g d           | j        "                                #                    dtH          j%        j&                   | j        "                                #                    d
tH          j%        j'                   | j        "                                #                    dtH          j%        j&                   | j        "                                #                    dtH          j%        j&                   |                    | j                   t                      }
t3          d          }|j                            | j(                   |
                    |           t3          d          }|j                            | j)                   |
                    |           |
*                                 |                    |
           tW          tV          j,        j-        tV          j,        j.        z            }|j/                            | j0                   |j1                            | j2                   |                    |           tg          ti          t          | j                                      | _5        | 6                                 d S )NzEdit Tokenizer Datai  iX   r           Filter:zType to filter tokens...d      
Page 1 of PreviousNext   )IndexTokenr   Score      z	Add TokenRemove Selected)7super__init__setWindowTitleresizecopytokenstoken_typesscoresmaxlenextendr   r   	addWidgetr   r   filter_editsetPlaceholderTexttextChangedconnectapply_filter	page_sizecurrent_pagetotal_pages
page_labelr   clickedprevious_page	next_page	addLayoutr   tokens_tablesetColumnCountsetHorizontalHeaderLabelshorizontalHeadersetSectionResizeModer   
ResizeModeResizeToContentsStretch	add_tokenremove_selected
addStretchr   StandardButtonOkCancelacceptedacceptrejectedrejectlistrangefiltered_indices	load_page)selfr=   r>   r?   parentmax_lenlayoutfilter_layout	prev_pagerO   controls_layout
add_buttonremove_buttonbuttons	__class__s                 p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/gguf/scripts/gguf_editor_gui.pyr9   zTokenizerEditorDialog.__init__3   sa      1222C'-5fkkmmm21<D;++---"'-5fkkmmm2 c$+&&D,<(=(=s4;?O?OPPt{g%%KtwT[1A1A'ABCCCt  7**##QC7S9I5J5J+J$KLLLt{g%%Ku#dk2B2B(BCDDDT"" $y 1 1222$;;++,FGGG$,,T->??? 0111 q3t{#3#3dn#Dq#HT^"[\\ !@d.>!@!@AA000
++	!!$"4555	***''	!!$.111	***''' )NN((+++334W4W4WXXX**,,AA![E[Elmmm**,,AA![E[Ecddd**,,AA![E[Elmmm**,,AA![E[Elmmm*+++ &-- --
""4>222!!*---#$566%%d&:;;;!!-000""$$$))) ##3#B#EHXHgHn#noo  ---  ---!!! !%U3t{+;+;%<%< = = 	    c                r   | j                                                                         }|s4t          t	          t          | j                                      | _        n^g | _        t          | j                  D ]B\  }}|t          |                                          v r| j        
                    |           Ct          dt          | j                  | j        z   dz
  | j        z            | _        d| _        | j                            d| j                    |                                  dS )z+Filter the tokens based on the search text.r-   r   r.   N)rD   textlowerrc   rd   rA   r=   re   	enumeratestrappendr@   rI   rK   rJ   rL   setTextrf   )rg   filter_textitokens       rr   rH   z"TokenizerEditorDialog.apply_filter   s    &++--3355 	4$(s4;/?/?)@)@$A$AD!! %'D!%dk22 4 45#e**"2"2"4"444)00333 q3t'<#=#=#NQR#RW[We"eff ?T-= ? ?@@@rs   c                    | j         dk    rS| xj         dz  c_         | j                            d| j         dz    d| j                    |                                  dS dS z#Go to the previous page of results.r   r-   Page  of NrJ   rL   rz   rK   rf   rg   s    rr   rN   z#TokenizerEditorDialog.previous_page   t    q  "O##$YD,=,A$Y$YtGW$Y$YZZZNN ! rs   c                    | j         | j        dz
  k     rS| xj         dz  c_         | j                            d| j         dz    d| j                    |                                  dS dS zGo to the next page of results.r-   r   r   NrJ   rK   rL   rz   rf   r   s    rr   rO   zTokenizerEditorDialog.next_page   |    t/!333"O##$YD,=,A$Y$YtGW$Y$YZZZNN 43rs   c                   | j                             d           | j        | j        z  }t	          || j        z   t          | j                            }| j                             ||z
             t          t          ||                    D ]8\  }}| j        |         }t          t          |                    }|                    t          j        j        |           |                    |                                t          j        j         z             | j                             |d|           t          t          | j        |                             }| j                             |d|           |t          | j                  k     r| j        |         nd}	 t-          |          }	|	j         d| d}
n# t0          t2          f$ r	 d| d}
Y nw xY wt          |
          }|                    t          j        j        |           |                    |                                t          j        j         z             | j                             |d|           |t          | j                  k     r| j        |         nd}t          t          |                    }| j                             |d|           :| j         j                            | j                   d	S )
z(Load the current page of tokenizer data.r   r-    ()	Unknown (r5   r*   r6   N)rQ   setRowCountrJ   rI   minrA   re   rw   rd   r   rx   setDatar   ItemDataRoleUserRolesetFlagsflagsItemFlagItemIsEditablesetItemr=   r>   r"   name
ValueErrorKeyErrorr?   cellDoubleClickedrG   handle_cell_double_click)rg   	start_idxend_idxrowr|   orig_idx
index_item
token_item
token_typeenum_valdisplay_text	type_itemscore
score_items                 rr   rf   zTokenizerEditorDialog.load_page   s   %%a((( %6	i$.0#d6K2L2LMM 	%%g	&9:::i 9 9:: 	: 	:FC,Q/H *#h--88Jr7BBB
 0 0 2 2bk6P5P PQQQ%%c1j999 *#dk(.C*D*DEEJ%%c1j999 8@#dFVBWBW7W7W)(33]^J9$Z00"*-@@:@@@) 9 9 98:8889 )66Ibo6
CCC y00BK4N3NNOOO%%c1i888 .6DK8H8H-H-HDK))cE)#e**55J%%c1j9999 	+33D4QRRRRRs   #G  GGc                    |dk    rY| j                             |d          }|r>|                    t          j        j                  }|                     ||           dS dS dS )zCHandle double-click on a cell, specifically for token type editing.r5   r   N)rQ   itemdatar   r   r   edit_token_type)rg   r   column	orig_itemr   s        rr   r   z.TokenizerEditorDialog.handle_cell_double_click   sn    Q;;)..sA66I 4$>>"/*BCC$$S(33333	 ;4 4rs   c                   |t          | j                  k     r| j        |         nd}t          |           }|                    d           t	          |          }t                      }t          D ]-}|                    |j         d|j	         d|j	                   .	 t          |t                    r/t          |          }|                    |j         d| d           n# t          t          f$ r Y nw xY w|                    |           t!          t           j        j        t           j        j        z            }|j                            |j                   |j                            |j                   |                    |           |                                t          j        j        k    r|                                }	t          |	          }|j         d|	 d}
| j                            |d          }|r:|                    |
           |                     tB          j"        j#        |	           |	| j        |<   dS dS )zEEdit a token type using a dialog with a dropdown of all enum options.r   zSelect Token Typer   r   r5   N)$rA   r>   r   r:   r   r   r"   addItemr   value
isinstanceintsetCurrentTextr   r   rC   r   r\   r]   r^   r_   rG   r`   ra   rb   exec
DialogCodeAcceptedcurrentDatarQ   r   rz   r   r   r   r   )rg   r   r   current_valuedialogrj   combor   rp   	new_valuer   r   s               rr   r   z%TokenizerEditorDialog.edit_token_type   sM   6>TEUAVAV6V6V(22\] 1222V$$! 	Q 	QHMMX]??hn???PPPP	--- K$]33$$%I%I%I%I%IJJJH% 	 	 	D	 	"#3#B#EHXHgHn#noo  ///  ///!!!;;==G.777))++I ++H&m;;y;;;L )..sA66I G!!,///!!"/":IFFF *3DX&&& 87s   AC$ $C87C8c                z   | j                             d           | j                            d           | j                            d           t	          | j                   dz
  }| j                                                                        }|r|dv r| j                            |           t          dt	          | j                  | j
        z   dz
  | j
        z            | _        | j        dz
  | _        | j                            d| j        dz    d| j                    |                                  dS )z'Add a new token to the end of the list.r)   r   r*   r-   r   r   N)r=   ry   r>   r?   rA   rD   ru   rv   re   r@   rI   rK   rJ   rL   rz   rf   )rg   r   r{   s      rr   rY   zTokenizerEditorDialog.add_token  s7    	2"""3t{##a' &++--3355 	3kR//!((222 q3t'<#=#=#NQR#RW[We"eff !,q0 U(9A(= U U4CS U UVVV 	rs   c                   g }| j                                         D ]/}|                                }||vr|                    |           0|sdS g }|D ]V}| j                             |d          }|r7|                    |                    t          j        j                             W|	                    d           |D ]b}|t          | j                  k     r| j        |= |t          | j                  k     r| j        |= |t          | j                  k     r| j        |= cg | _        | j                                                                        }t%          | j                  D ]D\  }}	|r#|t'          |	                                          v r| j                            |           Et)          dt          | j                  | j        z   dz
  | j        z            | _        t/          | j        | j        dz
            | _        | j                            d| j        dz    d| j                    |                                  dS )z'Remove selected tokens from all arrays.Nr   Treverser-   r   r   )rQ   selectedItemsr   ry   r   r   r   r   r   sortrA   r=   r>   r?   re   rD   ru   rv   rw   rx   r@   rI   rK   r   rJ   rL   rz   rf   )
rg   selected_rowsr   r   orig_indicesr   idxr{   r|   r}   s
             rr   rZ   z%TokenizerEditorDialog.remove_selected#  se   %3355 	* 	*D((**C-''$$S))) 	F   	N 	NC)..sA66I N##INN2?3K$L$LMMM$'''   	% 	%CS%%%%K$S)****$S)S%%%%K$ !#&++--3355!$+.. 	0 	0HAu 0+U1A1A1C1C"C"C%,,Q/// q3t'<#=#=#NQR#RW[We"eff 143Ca3GHH U(9A(= U U4CS U UVVV 	rs   c                *    | j         | j        | j        fS )z!Return the edited tokenizer data.)r=   r>   r?   r   s    rr   get_datazTokenizerEditorDialog.get_dataO  s    {D,dk99rs   N)__name__
__module____qualname__r9   rH   rN   rO   rf   r   r   rY   rZ   r   __classcell__rq   s   @rr   r'   r'   2   s        R R R R R Rh  (    -S -S -S^4 4 4)3 )3 )3V  0* * *X: : : : : : :rs   r'   c                  ^     e Zd Zd fd	Zd Zd Zd Zd Zd Zd Z	d	 Z
d
 ZddZd Z xZS )ArrayEditorDialogNc                `   t                                          |           |                     d           |                     dd           || _        || _        || _        d | _        |t          v r"|t          j
        k    rt          |         | _        t          |           }| j        t                      }t          d| j        j         d          }|                    |           d                    d | j        D                       }t          d|           }	|	                    d	           |                    |	d
           |                    |           t                      }
|
                    t          d                     t'                      | _        | j                            d           | j        j                            | j                   |
                    | j                   d| _        d| _        t7          d
t9          |          | j        z   d
z
  | j        z            | _        t          d| j                   | _        |
                    | j                   t?          d          }|j                             | j!                   |
                    |           t?          d          }|j                             | j"                   |
                    |           |                    |
           tG                      | _$        | j        | j$        %                    d           | j$        &                    g d           | j$        '                                (                    dtR          j*        j+                   | j$        '                                (                    d
tR          j*        j,                   | j$        '                                (                    dtR          j*        j+                   n| j$        %                    d           | j$        &                    ddg           | j$        '                                (                    dtR          j*        j+                   | j$        '                                (                    d
tR          j*        j,                   |                    | j$                   t                      }t?          d          }|j                             | j-                   |                    |           t?          d          }|j                             | j.                   |                    |           | j        Ct?          d          }|j                             | j/                   |                    |           |0                                 |                    |           tc          tb          j2        j3        tb          j2        j4        z            }|j5                            | j6                   |j7                            | j8                   |                    |           ts          tu          t9          | j                                      | _;        | <                                 d S )NzEdit Array Valuesi  i  zEditing z values:, c                0    g | ]}|j          d |j         S )=)r   r   ).0es     rr   
<listcomp>z.ArrayEditorDialog.__init__.<locals>.<listcomp>l  s*    $S$S$Sq%:%:%:%:$S$S$Srs   zAvailable values: Tr-   r+   zType to filter values...r,   r   r.   r/   r0   r6   )r2   ValueActionsr5   r2   r   zAdd Itemr7   zBulk Edit Selected)=r8   r9   r:   r;   array_valueselement_typekey	enum_typeKEY_TO_ENUM_TYPEr    INT32r   r   r   r   rC   joinsetWordWraprP   r   rD   rE   rF   rG   rH   rI   rJ   r@   rA   rK   rL   r   rM   rN   rO   r   items_tablerR   rS   rT   rU   r   rV   rW   rX   add_itemrZ   bulk_edit_selectedr[   r   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   )rg   r   r   r   rh   rj   enum_info_layout
enum_labelenum_valuesenum_values_labelrk   rl   rO   rm   rn   ro   bulk_edit_buttonrp   rq   s                     rr   r9   zArrayEditorDialog.__init__U  s      /000C(( """|}7J'J'J-c2DNT"" >%*}} L4>+B L L LMMJ&&z222 ))$S$SDN$S$S$STTK &'IK'I'I J J))$///&&'8!<<<-... $y 1 1222$;;++,FGGG$,,T->??? 0111 q3|#4#4t~#E#Idn"\]] !@d.>!@!@AA000
++	!!$"4555	***''	!!$.111	***''' (>> >%++A...667T7T7TUUU--//DDQH^Hoppp--//DDQH^Hfggg--//DDQH^Hopppp++A...667IJJJ--//DDQH^Hoppp--//DDQH^Hfggg)*** &-- ,,
""4=111!!*---#$566%%d&:;;;!!-000 >%*+?@@$,,T-DEEE%%&6777""$$$))) ##3#B#EHXHgHn#noo  ---  ---!!! !%U3t/@+A+A%B%B C C 	rs   c                    | j                                                                         }|s5t          t	          t          | j                                      | _        n$g | _        t          | j                  D ]\  }}| j	        t          |t                    r	 | 	                    |          }|j         d| d                                }||v r| j                            |           v# t          t          f$ r@ |t!          |                                          v r| j                            |           Y w xY w|t!          |                                          v r| j                            |           	t#          dt          | j                  | j        z   dz
  | j        z            | _        d| _        | j                            d| j                    |                                  dS )z1Filter the array values based on the search text.Nr   r   r-   r   r.   )rD   ru   rv   rc   rd   rA   r   re   rw   r   r   r   r   ry   r   r   rx   r@   rI   rK   rJ   rL   rz   rf   )rg   r{   r|   r   r   r   s         rr   rH   zArrayEditorDialog.apply_filter  s   &++--3355 	8$(s43D/E/E)F)F$G$GD!! %'D!%d&788 8 85>-*UC2H2H-<#'>>%#8#8*2-'C'C5'C'C'C'I'I'K'K&,66 188;;;&1 < < <&#e***:*:*<*<<< 188;;;< #c%jj&6&6&8&888-44Q777 q3t'<#=#=#NQR#RW[We"eff ?T-= ? ?@@@s   !AC44AEEc                    | j         dk    rS| xj         dz  c_         | j                            d| j         dz    d| j                    |                                  dS dS r   r   r   s    rr   rN   zArrayEditorDialog.previous_page  r   rs   c                    | j         | j        dz
  k     rS| xj         dz  c_         | j                            d| j         dz    d| j                    |                                  dS dS r   r   r   s    rr   rO   zArrayEditorDialog.next_page  r   rs   c                   | j                             d           | j        | j        z  }t	          || j        z   t          | j                            }| j                             ||z
             t          t          ||                    D ]\  }}| j        |         }| j	        |         }t          t          |                    }|                    t          j        j        |           |                    |                                t          j        j         z             | j                             |d|           | j        	 t-          |t.          t0          j        f          r#|                     |          }|j         d| d}	nt          |          }	n# t6          t8          f$ r	 d| d}	Y nw xY wt          |	          }
|
                    t          j        j        |           |
                    |
                                t          j        j         z             | j                             |d|
           t;          d          }|                    d|           |j                             | j!                   tE                      }tG          |          }|$                    d	d	d	d	           |%                    |           |&                                 | j         '                    |d	|           {t          t          |                    }
| j                             |d|
           dS )
z&Load the current page of array values.r   Nr   r   r   r-   Editr   r5   )(r   r   rJ   rI   r   rA   re   rw   rd   r   r   rx   r   r   r   r   r   r   r   r   r   r   r   r   numpysignedintegerr   r   r   r   setPropertyrM   rG   edit_array_enum_valuer   r   setContentsMarginsrC   r[   setCellWidget)rg   r   r   r   r|   r   r   r   r   r   
value_itemedit_buttonbutton_widgetbutton_layouts                 rr   rf   zArrayEditorDialog.load_page  s   $$Q''' %6	i$.0#d6K2L2LMM 	$$Wy%8999i 9 9:: +	= +	=FC,Q/H%h/E *#h--88Jr7BBB
 0 0 2 2bk6P5P PQQQ$$S!Z888 ~)8!%#u/B)CDD 2#'>>%#8#8*2-'C'C5'C'C'C'*5zz"H- 8 8 8#7u#7#7#7LLL8 .l;;
""2?#;UCCC##J$4$4$6$6"+:T9T$TUUU ((a<<< *&11''s333#++D,FGGG !(		 +M : :00Aq!<<<''444((*** ..sA}EEEE-c%jj99
 ((a<<<<W+	= +	=s   AF""F<;F<c                   |                                  }|                    d          }| j                            |d          }| j                            |d          }|r|r| j        r|                     || j                  r|                    t          j        j	                  }|                    t          j        j	                  }t          |t          t          t          t          f          r|| j        |<   dS dS dS dS dS dS )z1Handle editing an enum value in the array editor.r   r   r-   N)senderpropertyr   r   r   edit_enum_valuer   r   r   r   r   r   floatrx   boolr   )rg   buttonr   r   new_itemr   r   s          rr   r   z'ArrayEditorDialog.edit_array_enum_value'  s   ooe$$ $))#q11	#((a00 	8 	8dn 	89M9McSWSa9b9b 	8 ~~bo&>??H bo&>??I)c5#t%<== 8.7!(+++	8 	8 	8 	8 	8 	8 	8 	88 8rs   c                   | j         sdS t                      }| j                                        D ])}|                    |                                           *|st          j        | dd           dS t          |           }|	                    d| j         j
         d           t          |          }|                    t          dt          |           d                     t                      }| j         D ]-}|                    |j         d|j         d	|j                   .|                    |           t'          t&          j        j        t&          j        j        z            }|j                            |j                   |j                            |j                   |                    |           |                                t          j        j        k    r|                                }|                      |          }|j         d| d	}	|D ]}
| j                             |
d
          }| j                             |
d          }|rj|rh|!                    tD          j#        j$                  }|| j%        |<   |&                    |	           |'                    tD          j#        j$        |           dS dS )z"Edit multiple enum values at once.NzNo Selectionz'Please select at least one row to edit.z
Bulk Edit z ValueszSet z selected items to:r   r   r   r-   )(r   setr   r   addr   r   informationr   r:   r   r   rC   r   rA   r   r   r   r   r   r\   r]   r^   r_   rG   r`   ra   rb   r   r   r   r   r   r   r   r   r   r   rz   r   )rg   r   r   r   rj   r   r   rp   r   r   r   r   r   r   s                 rr   r   z$ArrayEditorDialog.bulk_edit_selected6  s   ~ 	F$2244 	* 	*Ddhhjj)))) 	#D.:cdddF K4>+BKKKLLLV$$ Ns='9'9 N N NOOPPP 	Q 	QHMMX]??hn???PPPP"#3#B#EHXHgHn#noo  ///  ///!!!;;==G.777))++I~~i00H&m;;y;;;L % 	J 	J ,11#q99	+00a88 J J(~~bo.FGGH2;D%h/ $$\222$$R_%=yIII! 87	J 	Jrs   c                   t          | j                  }| j        :t          | j                  d         j        }| j                            |           nJ| j        t          j        k    r| j                            d           n| j                            d           | j	                            |           t          dt          | j	                  | j        z   dz
  | j        z            | _        | j        dz
  | _        | j                            d| j        dz    d| j                    |                                  d S )Nr   r)   r-   r   r   )rA   r   r   rc   r   ry   r   r    STRINGre   r@   rI   rK   rJ   rL   rz   rf   )rg   r   default_values      rr   r   zArrayEditorDialog.add_itemg  s9   t()) >% 0039M$$]3333 M$888!((,,,,!((+++ 	$$X... q3t'<#=#=#NQR#RW[We"eff !,q0 U(9A(= U U4CS U UVVV 	rs   c                   g }| j                                         D ]/}|                                }||vr|                    |           0|sd S t	                      }|D ]V}| j                             |d          }|r7|                    |                    t          j        j	                             W|
                    d           |D ]
}| j        |= g | _        | j                                                                        }t!          | j                  D ]$\  }}	|s| j                            |           #| j        t%          |	t&                    r	 |                     |	          }
|
j         d|	 d                                }||v r| j                            |           # t*          t,          f$ r@ |t/          |	                                          v r| j                            |           Y w xY w|t/          |	                                          v r| j                            |           &t1          dt3          | j                  | j        z   dz
  | j        z            | _        t9          | j        | j        dz
            | _        | j                            d| j        dz    d| j                    |                                   d S )	Nr   Tr   r   r   r-   r   r   )!r   r   r   ry   rc   r   r   r   r   r   r   r   re   rD   ru   rv   rw   r   r   r   r   r   r   rx   r@   rA   rI   rK   r   rJ   rL   rz   rf   )rg   r   r   r   r   r   r   r{   r|   r   r   r   s               rr   rZ   z!ArrayEditorDialog.remove_selected  s   $2244 	* 	*D((**C-''$$S))) 	F vv  	N 	NC(--c155I N##INN2?3K$L$LMMM$'''   	' 	'C!#&& !#&++--3355!$"344 	8 	8HAu 8%,,Q//// >-*UC2H2H-<#'>>%#8#8*2-'C'C5'C'C'C'I'I'K'K&,66 188;;;&1 < < <&#e***:*:*<*<<< 188;;;< #c%jj&6&6&8&888-44Q777 q3t'<#=#=#NQR#RW[We"eff 143Ca3GHH U(9A(= U U4CS U UVVV 	s    AF33AHHr   r   r   Type[enum.Enum]c                D   | j                             |d          }|r%|                    t          j        j                  }ndS | j        |         }t          |           }|                    d|j	         d           t          |          }t          d|j	         d          }|                    |           t                      }	|D ]-}
|	                    |
j         d|
j         d|
j                   .	 t#          |t$                    r+ ||          }
|	                    |
j         d| d           n# t(          t*          f$ r Y nw xY w|                    |	           t-          t,          j        j        t,          j        j        z            }|j                            |j                   |j                            |j                   |                    |           |                                t          j         j!        k    r|	"                                } ||          }
|
j         d| d}| j                             |d	          }|r:|#                    |           |$                    t          j        j        |           || j        |<   d
S dS )zFEdit an enum value using a dialog with a dropdown of all enum options.r   NSelect  Valuez	Select a z value:r   r   r-   TF)%r   r   r   r   r   r   r   r   r:   r   r   r   rC   r   r   r   r   r   r   r   r   r   r   r\   r]   r^   r_   rG   r`   ra   rb   r   r   r   r   rz   r   )rg   r   r   r   r   r   r   rj   descriptionr   r   rp   r   r   r   s                  rr   r   z!ArrayEditorDialog.edit_enum_value  s    $))#q11	 	 ~~bo&>??HHF)(3 B	(:BBBCCCV$$ D);DDDEE%%% ! 	Q 	QHMMX]??hn???PPPP	--- K$9]33$$%I%I%I%I%IJJJH% 	 	 	D	 	"#3#B#EHXHgHn#noo  ///  ///!!!;;==G.777))++I y++H&m;;y;;;L',,S!44H F  ...  !99EEE +4Dh'4us   :A D; ;EEc                    | j         S r   )r   r   s    rr   get_array_valuesz"ArrayEditorDialog.get_array_values  s      rs   )NN)r   r   r   r  )r   r   r   r9   rH   rN   rO   rf   r   r   r   rZ   r   r  r   r   s   @rr   r   r   T  s        i i i i i iV  B    6= 6= 6=p8 8 8/J /J /Jb  82 2 2h5 5 5 5n! ! ! ! ! ! !rs   r   c                  (     e Zd Zd fd	ZddZ xZS )AddMetadataDialogNc                   t                                          |           |                     d           |                     dd           t	          |           }t                      }t                      | _        |                    d| j                   t                      | _
        t          D ]2}|t          j        k    r | j
                            |j        |           3|                    d| j
                   t                      | _        |                    d| j                   |                    |           t%          t$          j        j        t$          j        j        z            }|j                            | j                   |j                            | j                   |                    |           d S )NAdd Metadatai     zKey:zType:zValue:)r8   r9   r:   r;   r   r   r   key_editaddRowr   
type_combor    ARRAYr   r   r   
value_editrP   r   r\   r]   r^   r_   rG   r`   ra   rb   rC   )rg   rh   rj   form_layout
value_typerp   rq   s         rr   r9   zAddMetadataDialog.__init__  s|      N+++CT""!mm!64=111#++' 	E 	EJ]000''
DDD7DO444#++8T_555%%%"#3#B#EHXHgHn#noo  ---  ---!!!!!rs   returnTuple[str, GGUFValueType, Any]c                   | j                                         }| j                                        }| j                                        }|t          j        k    r#t          j	        t          |                    }nj|t          j        k    r#t          j        t          |                    }n7|t          j        k    r#t          j        t          |                    }n|t          j        k    r"t          j        t          |                    }n|t          j        k    r"t          j        t          |                    }n|t          j        k    r"t          j        t          |                    }nn|t          j        k    r"t          j        t/          |                    }n<|t          j        k    r|                                dv }n|t          j        k    r|}n|}|||fS )Ntrueyes1)r  ru   r  r   r  toPlainTextr    UINT8npuint8r   INT8int8UINT16uint16INT16int16UINT32uint32r   int32FLOAT32float32r   BOOLrv   r  )rg   r   r  
value_textr   s        rr   r   zAddMetadataDialog.get_data  s   m  ""_0022
_0022
 ,,,HS__--EE=---GC
OO,,EE=///Ic*oo..EE=...HS__--EE=///Ic*oo..EE=...HS__--EE=000JuZ0011EE=---$$&&*>>EE=///EEEJ%%rs   r   )r  r  )r   r   r   r9   r   r   r   s   @rr   r  r    sQ        " " " " " "8& & & & & & & &rs   r  c                       e Zd Z fdZd Zd Zd Zd Zdd
ZddZ	d dZ
d!dZd Zd Zd Zd Zd Zd Zd Zd Zd Z xZS )"GGUFEditorWindowc                2   t                                                       |                     d           |                     dd           d | _        d | _        d| _        i | _        t                      | _	        d| _
        |                                  d S )NzGGUF Editori  i   F)r8   r9   r:   r;   current_filereadermodifiedmetadata_changesr   metadata_to_remove on_metadata_changed_is_connectedsetup_ui)rg   rq   s    rr   r9   zGGUFEditorWindow.__init__/  s    M***D#  ""%%%05-rs   c                
   t                      }|                     |           t          |          }t                      }t	                      | _        | j                            d           |                    | j                   t          d          }|j	        
                    | j                   |                    |           t          d          }|j	        
                    | j                   |                    |           |                    |           t                      | _        t                      | _        t          | j                  }t#                      | _        | j                            d           | j                            g d           | j                                                            dt.          j        j                   | j                                                            dt.          j        j                   | j                                                            dt.          j        j                   | j                                                            d	t.          j        j                   |                    | j                   t                      }t          d
          }|j	        
                    | j                   |                    |           |                                 |                    |           t                      | _        t          | j                  }	t#                      | _        | j                            d           | j                            g d           | j                                                            dt.          j        j                   | j                                                            dt.          j        j                   | j                                                            dt.          j        j                   | j                                                            d	t.          j        j                   | j                                                            dt.          j        j                   |	                    | j                   | j                            | j        d           | j                            | j        d           |                    | j                   |                                  !                    d           d S )NTz	Open GGUFz
Save As...r1   )Keyr   r   r   r   r-   r5   r6   r     )Namer   ShapeElementszSize (bytes)MetadataTensorsReady)"r   setCentralWidgetr   r   r   file_path_editsetReadOnlyrC   r   rM   rG   	open_file	save_filerP   r   tabsmetadata_tabr   metadata_tablerR   rS   rT   rU   r   rV   rX   rW   add_metadatar[   tensors_tabtensors_tableaddTab	statusBarshowMessage)
rg   central_widgetmain_layoutfile_layoutopen_buttonsave_buttonmetadata_layoutmetadata_controlsadd_metadata_buttontensors_layouts
             rr   r;  zGGUFEditorWindow.setup_ui>  s[    n---!.11 "mm'kk''---d1222!+..##DN333k***!,//##DN333k***k*** LL	 $II%d&788 +nn**1---556Y6Y6YZZZ,,..CCA{G]Gefff,,..CCA{G]Gnooo,,..CCA{G]Gefff,,..CCA{G]Gnooo!!$"5666 (MM).99#++D,=>>>##$7888$$&&&!!"3444 #99$T%566)^^))!,,,445j5j5jkkk++--BB1kF\Fdeee++--BB1kF\Fmnnn++--BB1kF\Fmnnn++--BB1kF\Fmnnn++--BB1kF\Fmnnn  !3444 		*J777	)9555di((( 	$$W-----rs   c           	        	 |                                                      d| d           t          j                     t	          |d          | _        || _        | j                            |           | 	                                 | 
                                 i | _        t                      | _        d| _        |                                                      d|            dS # t          $ rX}t!          j        | ddt%          |                      |                                                      d	           Y d
}~dS d
}~ww xY w)zLoad a GGUF file by pathzLoading ...rFzLoaded TErrorzFailed to open file: zError loading fileN)rQ  rR  r	   processEventsr   r6  r5  rF  rz   load_metadataload_tensorsr8  r   r9  r7  	Exceptionr   criticalrx   )rg   	file_pathr   s      rr   	load_filezGGUFEditorWindow.load_file  sJ   	NN(()BI)B)B)BCCC&((($Y44DK )D''	222   $&D!&)eeD#!DMNN(()>9)>)>???4 	 	 	 w0PA0P0PQQQNN(()=>>>55555	s   C'C+ +
E5AEEc                l    t          j        | ddd          \  }}|sd S |                     |           d S )NzOpen GGUF Filer)   "GGUF Files (*.gguf);;All Files (*))r   getOpenFileNamerf  )rg   re  _s      rr   rH  zGGUFEditorWindow.open_file  sJ    "2"B(L
 
	1  	Fy!!!!!rs   c                   | j                             d           | j        sd S | j        rjt	          j                    5  t	          j        d           | j         j                            | j	                   d d d            n# 1 swxY w Y   d| _        t          | j        j                                                  D ]\  }\  }}| j                             |           t          |          }|                    |                                t"          j        j         z             | j                             |d|           |j        sd}n|j        d         t,          j        k    rqt1          |j                  dz
  }|j        d         j        }|                     |          }|"|j        d         t,          j        k    r|j        }d|z  |z   d|z  z   }nXt;          |j        d         j                  }|                     |          }|"|j        d         t,          j        k    r|j        }t          |          }	|	                    |	                                t"          j        j         z             | j                             |d|	           |                     |          }
t          |
          }t1          |j                  dk    rU|j        d         t,          j        k    r:|                    |                                t"          j        j        z             n:|                    |                                t"          j        j         z             | j                             |d	|           t?                      }tA          |          }|!                    d	d	d	d	           |j        r|j        d         t,          j        k    rtE          d
          }|#                    d|           |#                    d|           |j$        %                    | j&                   |'                    |           |tP          v r*|)                    d           |*                    d           nt1          |j                  dk    r|                     |          otE          d
          }|#                    d|           |#                    d|           |j$        %                    | j+                   |'                    |           tE          d          }|#                    d|           |#                    d|           |j$        %                    | j,                   |'                    |           | j         -                    |d|           | j         j        %                    | j	                   d| _        d S )Nr   ignoreFN/Ar-   []r5   r   r   r   zEdit Tokenizerz Edit all tokenizer data togetherRemover6   T).rL  r   r6  r:  warningscatch_warningsfilterwarningsitemChanged
disconnecton_metadata_changedrw   fieldsitems	insertRowr   r   r   r   r   r   r   typesr    r  rA   r   get_enum_for_keyr   r   rx   format_field_valuer   r   r   r   r   rM   rG   edit_array_metadatarC   TOKENIZER_LINKED_KEYSrz   
setToolTipedit_metadata_enumremove_metadatar   )rg   r|   r   fieldkey_itemtype_str
nest_countr   r   r   	value_strr   actions_widgetactions_layoutr   ro   s                   rr   ra  zGGUFEditorWindow.load_metadata  s   ''***{ 	F 0 	:(** U U'111#/::4;STTTU U U U U U U U U U U U U U U 5:D1();)A)A)C)CDD H	D H	DOA|U))!,,, (,,Hhnn.."+2L1LLMMM''1h777 ; 2 Q=#666 --1
${23 11#66	(U[_@S-S-S#,#5L+l:S:=MMu{1~233 11#66	(U[^}?R-R-R(1H(22Iy00BK4N3NNOOO''1i888 //66I))44J 5;1$$Q=;N)N)N##J$4$4$6$69S$STTTT##J$4$4$6$6"+:T9T$TUUU''1j999 %YYN(88N--aAq999 { 6u{1~1DDD)&11''q111''s333#++D,DEEE((555 ///''(8999**+MNNNU[!!Q&&4+@+@+E+E+Q)&11''q111''s333#++D,CDDD((555'11M%%eQ///%%eS111!))$*>???$$]333--aNCCCC 	'//0HIII04---s   9BB
Br  r!   r  rc   c           	        |j         r|j         d         t          j        k    rg S |j         d         }g }t          |j                  }|t          j        k    r]t          |          D ]L}t          t          |j	        d||z
  dz
  dz  z
                     d          }|
                    |           MnV| j        rO|| j        j        v rAt          |          D ]1}|
                    |j	        d||z
  dz
  z
           d                    2|S )z'Extract all values from an array field.r   r-   rn  r5   utf-8encoding)r{  r    r  rA   r   r  rd   rx   bytespartsry   r6  gguf_scalar_to_np)rg   r  	curr_typer   total_elementselement_posvalue_strings          rr   extract_array_valuesz%GGUFEditorWindow.extract_array_values  s)   { 	ek!n0CCCIKN	UZ,,,$^44 2 2"5R>K;WZ[;[_`:`5`)a#b#bmtuuu##L11112 [ 	]Y$+*GGG$^44 ] ]##EKn{6RUV6V0W$XYZ$[\\\\rs   r   rx   Optional[Type[enum.Enum]]c                6    t                               |          S )z/Get the enum type for a given key if it exists.)r   get)rg   r   s     rr   r|  z!GGUFEditorWindow.get_enum_for_key  s    ##C(((rs   r   r   r   r  c                    	 t          |t          t          f          r ||          }|j         d| dS n# t          t
          f$ r Y nw xY wt          |          S )z&Format a value as an enum if possible.r   r   )r   r   rx   r   r   r   )rg   r   r   
enum_values       rr   format_enum_valuez"GGUFEditorWindow.format_enum_value  s}    	%#s,, 6&Yu--
$/55U55556 H% 	 	 	D	5zzs   37 A
Ac                   |j         sdS t          |j                   dk    r|j         d         }|t          j        k    r)t	          t          |j        d                   d          S | j        rb|| j        j        v rT|j        d         d         }| 	                    |j
                  }||                     ||          S t	          |          S |j         d         t          j        k    r|                     |          t          dt                              }| 	                    |j
                  }|Dg }t          |          D ]1}|                    |                     |         |                     2nfdt          |          D             }d	d
                    |                                           t                    t          |          k    rdnd dS dS )Nrm  r-   r   rn  r  r  r>  c                :    g | ]}t          |                   S  rx   )r   r|   r   s     rr   r   z7GGUFEditorWindow.format_field_value.<locals>.<listcomp>B  s%    !V!V!V1#l1o"6"6!V!V!Vrs   [ r   , ...r)    ]zComplex value)r{  rA   r    r  rx   r  r  r6  r  r|  r   r  r  r  r   rd   ry   r   strip)	rg   r  r  r   r   render_elementarray_elementsr|   r   s	           @rr   r}  z#GGUFEditorWindow.format_field_value&  s   { 	5u{q  AIM0005R11GDDDD "dk.K!K!KB* 11%*==	(11%CCC5zz!;q>]00044U;;L C$5$566N --ej99I$!#~.. ^ ^A"))$*@*@aR[*\*\]]]]^ "W!V!V!Vn@U@U!V!V!Vy		.117799yc,FWFWZ]^lZmZmFmFm77suyyyyrs   c                L   | j                             d           | j        sd S t          | j        j                  D ]e\  }}| j                             |           t          |j                  }|                    |	                                t          j        j         z             | j                             |d|           t          |j        j                  }|                    |	                                t          j        j         z             | j                             |d|           d                    d |j        D                       }t          |          }|                    |	                                t          j        j         z             | j                             |d|           t          t#          |j                            }|                    |	                                t          j        j         z             | j                             |d|           t          |j        d          }|                    |	                                t          j        j         z             | j                             |d|           gd S )	Nr   r-   u    × c              3  4   K   | ]}t          |          V  d S r   r  )r   ds     rr   	<genexpr>z0GGUFEditorWindow.load_tensors.<locals>.<genexpr>\  s(      #A#AqCFF#A#A#A#A#A#Ars   r5   r6   ,r1   )rO  r   r6  rw   tensorsrz  r   r   r   r   r   r   r   r   tensor_typer   shaperx   
n_elementsn_bytes)	rg   r|   tensor	name_itemr   	shape_str
shape_itemelements_item	size_items	            rr   rb  zGGUFEditorWindow.load_tensorsH  sV   &&q))){ 	F"4;#677 	8 	8IAv((+++ )55Iy00BK4N3NNOOO&&q!Y777 ));)@AAIy00BK4N3NNOOO&&q!Y777 #A#AFL#A#A#AAAI))44J
 0 0 2 2bk6P5P PQQQ&&q!Z888 -S1B-C-CDDM""=#6#6#8#8BK<V;V#VWWW&&q!];;; )FN)>)>??Iy00BK4N3NNOOO&&q!Y77777	8 	8rs   c                l	   |                                 dk    rd S |                                }| j                            |d          }d }|r|                                }|                                }d }| j        r|r| j                            |          }|r	|j        r|sd S |j        d         }|                     |          }||t          j
        k    r	 	 ||         }	|	j        }
n# t          t          f$ rp d|v rZd|v rV|                    d          d                             d          d                                         }t!          |          }
nt!          |          }
Y nw xY w ||
           ||
f| j        |<   d| _        |                     |
|          }|                    |           |                                                     d| d|            d S # t.          t          f$ rz}t1          j        | d	| dd
| d|j         dd                    d |D                                   |                     |          }|                    |           Y d }~d S d }~ww xY w	 |t          j        k    r#t=          j        t!          |                    }
nj|t          j         k    r#t=          j!        t!          |                    }
n7|t          j"        k    r#t=          j#        t!          |                    }
n|t          j$        k    r"t=          j%        t!          |                    }
n|t          j&        k    r"t=          j'        t!          |                    }
n|t          j
        k    r"t=          j(        t!          |                    }
nn|t          j)        k    r"t=          j*        tW          |                    }
n<|t          j,        k    r|-                                dv }
n|t          j.        k    r|}
nd S ||
f| j        |<   d| _        |                                                     d| d|            d S # t.          $ rO t1          j        | dd| d|j/                    |                     |          }|                    |           Y d S w xY w)Nr5   r   (r   r-   TChanged  to zInvalid Enum Value ('z' is not a valid z value.
Valid values are: r   c              3  $   K   | ]}|j         V  d S r   )r   r   vs     rr   r  z7GGUFEditorWindow.on_metadata_changed.<locals>.<genexpr>  s$      2M2Ma162M2M2M2M2M2Mrs   r  zInvalid ValuezThe value 'z' is not valid for type )0r   r   rL  r   ru   r6  	get_fieldr{  r|  r    r   r   r   AttributeErrorsplitr  r   r8  r7  r  rz   rQ  rR  r   r   warningr   r   r}  r"  r#  r$  r%  r&  r'  r(  r)  r*  r+  r,  r-  r.  r/  r   r0  rv   r  r   )rg   r   r   r   r   r   r  r  r   r   converted_value
value_partformatted_valuer   original_values                  rr   rw  z$GGUFEditorWindow.on_metadata_changedk  s   ;;==AFhhjj',,S!44	 	#..""CIIKK	; 	/3 	/K))#..E 	EK 	s 	F[^
 ))#..	 Z=3F%F%F&9(3H&.nOO .1 9 9 9i''C9,<,<%.__S%9%9!%<%B%B3%G%G%J%P%P%R%R
*-j// +.i..9 	/*** /9/-J%c* $ #'"8"8)"T"T_---  ,,-R-R-R-R-RSSS) 
 
 
#/1///P	 P PI4F P P)-2M2M92M2M2M)M)MP PQ Q Q "&!8!8!?!?^,,,
"	)]000"$(3y>>":":}111"$'#i.."9"9}333"$)C	NN";";}222"$(3y>>":":}333"$)C	NN";";}222"$(3y>>":":}444"$*U9-=-=">">}111"+//"3"37K"K}333"+  +5o)FD!#& DMNN(()HC)H)HY)H)HIIIII 	) 	) 	)o7yY7y7yhrhw7y7yzzz "44U;;NLL((((((	)sT    C0 /G, 0A>E1.G, 0E11A9G, ,I7=A/I22I7;FQ A Q AR32R3c                6   |                                  }|                    d          }|                    d          }t          j        | dd| dt          j        j        t          j        j        z  t          j        j                  }|t          j        j        k    ry| j                            |           | j	        
                    |           || j        v r| j        |= d| _        |                                                     d| d           d S d S )	Nr   r   zConfirm Removalz2Are you sure you want to remove the metadata key 'z'?TzMarked z for removal)r   r   r   questionr\   YesNorL  	removeRowr9  r  r8  r7  rQ  rR  )rg   r   r   r   replys        rr   r  z GGUFEditorWindow.remove_metadata  s   ooe$$ooe$$$#HHHH&*[-G-JJKLfLi
 
 K.222))#...#'',,, d+++)#. DMNN(()D3)D)D)DEEEEE 32rs   c                   |                                  }|                    d          }|                    d          }d}| j        r| j                            |          }|r|j        sdS |                     |          }|dS |                                }t          |           }|                    d|j	         d           t          |          }t                      }	|D ]-}
|	                    |
j         d|
j         d|
j                   .	 t          |t           t"          f          r+ ||          }
|	                    |
j         d| d           n# t&          t(          f$ r Y nw xY w|                    |	           t-          t,          j        j        t,          j        j        z            }|j                            |j                   |j                            |j                   |                    |           |                                t          j         j!        k    r|	"                                } ||          }
|j        d         |f| j#        |<   d	| _$        |
j         d| d}| j%        &                    |d
          }|r|'                    |           | (                                )                    d| d|            dS dS )zEdit an enum metadata field.r   r   Nr	  r
  r   r   r   Tr5   r  r  )*r   r   r6  r  r{  r|  contentsr   r:   r   r   r   r   r   r   r   r   rx   r   r   r   rC   r   r\   r]   r^   r_   rG   r`   ra   rb   r   r   r   r   r8  r7  rL  r   rz   rQ  rR  )rg   r   r   r   r  r   r   r   rj   r   r   rp   r   r   target_items                  rr   r  z#GGUFEditorWindow.edit_metadata_enum  s   ooe$$ooe$$; 	/K))#..E 	EK 	F))#..	F (( B	(:BBBCCCV$$! 	Q 	QHMMX]??hn???PPPP	-#s44 K$9]33$$%I%I%I%I%IJJJH% 	 	 	D	 	"#3#B#EHXHgHn#noo  ///  ///!!!;;==G.777))++I y++H +0+a.))DD!#& DM 'm;;y;;;L-223::K 2##L111NN(()KC)K)K\)K)KLLLLL 87s   AE E10E1c                                                      }|                    d          }|                    d          }|t          v r                     |           d S d } j        r j                            |          }|r"|j        r|j        d         t          j        k    rd S |j        d         } 	                    |          }t          |||           }|                                t          j        j        k    r@|                                }t          j        ||ff j        |<   d _                             |          V|t          j        k    rFdd                     fd|d d	         D                        t+          |          d	k    rd
nd d}	nBdd                    d |d d	         D                        t+          |          d	k    rd
nd d}	 j                            |d          }
|
r|
                    |	                                                                d|            d S d S )Nr   r   r   r-   Tr  r   c              3  D   K   | ]}                     |          V  d S r   )r  )r   r  r   rg   s     rr   r  z7GGUFEditorWindow.edit_array_metadata.<locals>.<genexpr>B  s4      *h*hTU4+A+A!Y+O+O*h*h*h*h*h*hrs   r>  r  r)   r  c              3  4   K   | ]}t          |          V  d S r   r  r  s     rr   r  z7GGUFEditorWindow.edit_array_metadata.<locals>.<genexpr>D  s(      *J*Ja3q66*J*J*J*J*J*Jrs   r5   zUpdated array values for )r   r   r  edit_tokenizer_metadatar6  r  r{  r    r  r  r   r   r   r   r   r  r8  r7  r|  r   r   rA   rL  r   rz   rQ  rR  )rg   r   r   r   r  r   r   r   
new_valuesr  r  r   s   `          @rr   r~  z$GGUFEditorWindow.edit_array_metadata   s   ooe$$ooe$$ '''((---F; 	/K))#..E 	EK 	5;q>]=P+P+PF {1~ 0077 #<sDII;;==G.7770022J +8*=j?Y)ZD!#& DM --c22I$9L)L)L U*h*h*h*h*hYcdfefdfYg*h*h*h!h!h  Uux  zD  vE  vE  HI  vI  vIjqjq  OQ  U  U  U		v*J*J:bqb>*J*J*J!J!JvWZ[eWfWfijWjWjGGprvvv	-223::K /##I...NN(()JS)J)JKKKKK# 87rs   c                   | j         sdS | j                             t          j        j        j                  }| j                             t          j        j        j                  }| j                             t          j        j        j                  }|r|                     |          ng }|r|                     |          ng }|r|                     |          ng }t          j        j        j        | j	        v r'| j	        t          j        j        j                 \  }\  }}t          j        j        j        | j	        v r'| j	        t          j        j        j                 \  }\  }}t          j        j        j        | j	        v r'| j	        t          j        j        j                 \  }\  }}t          ||||           }	|	                                t          j        j        k    rq|	                                \  }
}}|r7t           j        |j        d         |
ff| j	        t          j        j        j        <   |r7t           j        |j        d         |ff| j	        t          j        j        j        <   |r7t           j        |j        d         |ff| j	        t          j        j        j        <   d| _        |                     t          j        j        j        |
           |                     t          j        j        j        |           |                     t          j        j        j        |           |                                                     d           dS dS )z3Edit the linked tokenizer metadata arrays together.Nr-   TzUpdated tokenizer data)r6  r  ggufKeys	TokenizerLIST
TOKEN_TYPESCORESr  r8  r'   r   r   r   r   r   r    r  r{  r7  update_tokenizer_displayrQ  rR  )rg   trigger_keytokens_fieldtoken_types_fieldscores_fieldr=   r>   r?   rj  r   
new_tokensnew_token_types
new_scoress                rr   r  z(GGUFEditorWindow.edit_tokenizer_metadataK  s   { 	F {,,TY-@-EFF K11$)2E2PQQ{,,TY-@-GHH =IP**<888bFW_d//0ABBB]_<HP**<888b 9#t'<<<!2493F3KLNA{69)T-BBB"&"7	8K8V"WA;9%)>>>!2493F3MNNA{6 'v{FDII;;==G.7776<oo6G6G3J  !'!'*J7C%di&9&>?
 ! !'&,Q/AI%di&9&DE
  !'!'*J7E%di&9&@A
 !DM ))$)*=*BJOOO))$)*=*H/ZZZ))$)*=*DjQQQNN(()ABBBBB; 87rs   c                   t          | j                                                  D ]}| j                            |d          }|r|                                |k    rwdd                    d |dd         D                        t          |          dk    rdnd d	}| j                            |d
          }|r|                    |            dS dS )z>Update the display of a tokenizer field in the metadata table.r   r  r   c              3  4   K   | ]}t          |          V  d S r   r  r  s     rr   r  z<GGUFEditorWindow.update_tokenizer_display.<locals>.<genexpr>  s(      *F*Fa3q66*F*F*F*F*F*Frs   Nr>  r  r)   r  r5   )rd   rL  rowCountr   ru   r   rA   rz   )rg   r   valuesr   r  r  r   s          rr   r  z)GGUFEditorWindow.update_tokenizer_display  s    ,557788 	 	C*//Q77H HMMOOs22n*F*F6"1":*F*F*F!F!FnSVW]S^S^abSbSbhjnnn	!055c1==
 2&&y111	 	rs   c                   t          |           }|                                t          j        j        k    r|                                \  }}}|st          j        | dd           d S t          | j	        
                                          D ]T}| j	                            |d          }|r5|                                |k    rt          j        | dd| d            d S U| j	        
                                }| j	                            |           t          |          }|                    |                                t"          j        j         z             | j	                            |d|           t          |j                  }|                    |                                t"          j        j         z             | j	                            |d|           t          t-          |                    }	|	                    |	                                t"          j        j        z             | j	                            |d|	           t/                      }
t1          |
          }|                    dddd           t5          d	          }|                    d
|           |                    d|           |j                            | j                   |                    |           | j	                             |d|
           ||f| j!        |<   d| _"        | #                                $                    d|            d S d S )NzInvalid KeyzKey cannot be emptyr   zDuplicate KeyzKey 'z' already existsr-   r5   rq  r   r   r6   TzAdded new metadata key )%r  r   r   r   r   r   r   r  rd   rL  r  r   ru   rz  r   r   r   r   r   r   r   r   rx   r   r   r   r   r   rM   rG   r  rC   r   r8  r7  rQ  rR  )rg   r   r   r  r   r   r   r  r   r   r  r  ro   s                rr   rM  zGGUFEditorWindow.add_metadata  s   "4((;;==G.777%+__%6%6"CU #D-9NOOO T099;;<<   /44S!<<	 !1!1S!8!8'o?\s?\?\?\]]]FF %..00C))#... (,,Hhnn.."+2L1LLMMM''Q999 )99Iy00BK4N3NNOOO''Q	::: *#e**55J
 0 0 2 2R[5O OPPP''Q
;;; %YYN(88N--aAq999'11M%%eS111%%eS111!))$*>???$$]333--c1nEEE +5e)<D!#& DMNN(()H3)H)HIIIIIe 87rs   c           	     	   | j         st          j        | dd           d S | j        s&| j        s| j        st          j        | dd           d S t          j        | ddd          \  }}|sd S 	 | 	                                
                    d| d	           t          j                     d
}| j                             t          j        j        j                  }|r|                                }t'          ||| j         j                  }d }| j                             t          j        j        j                  }|r|                                }|||_        | j         j                                        D ]}|j        t          j        j        j        k    s|j                            d          r<|j        | j        v rKd }|j        | j        v r+| j        |j                 \  }}	|t6          j        k    r|	\  }}	n>|                                }	|j        d         }|t6          j        k    r|j        d         }|	|                    |j        |	||           | j                                        D ]S\  }
\  }}	| j                             |
          #d }|t6          j        k    r|	\  }}	|                    |
|	||           T| j         j         D ]4}|!                    |j        |j"        |j"        j#        |j$                   5|%                    tM          |                     |'                                 |(                                 |)                    d           |*                                 | 	                                
                    d|            t          j+        | ddt          j,        j-        t          j,        j.        z  t          j,        j-                  }|t          j,        j-        k    rt_          |d          | _         || _0        | j1        2                    |           | 3                                 | 4                                 i | _        tk                      | _        d| _        d S d S # tl          $ rX}t          j7        | ddtq          |                      | 	                                
                    d           Y d }~d S d }~ww xY w)NzNo File OpenzPlease open a GGUF file firstz
No ChangeszNo changes to savezSave GGUF File Asr)   rh  z
Saving to r]  unknown)arch	endianesszGGUF.r   rn  )sub_type)	raw_shape	raw_dtypeF)progressz	Saved to zOpen Saved Filez,Would you like to open the newly saved file?r^  r_  zFailed to save file: zError saving file)9r6  r   r  r7  r8  r9  r  r   getSaveFileNamerQ  rR  r	   r`  r  r  r  GeneralARCHITECTUREr  r   r  	ALIGNMENTdata_alignmentrx  r  r   
startswithr    r  r{  add_key_valuery  r  
add_tensorr   r  r  open_output_filer   write_header_to_filewrite_kv_data_to_filewrite_tensors_to_filecloser  r\   r  r  r   r5  rF  rz   ra  rb  r   rc  rd  rx   )rg   re  rj  r  r  writer	alignmentr  r  r   r   r  r  r   s                 rr   rI  zGGUFEditorWindow.save_file  s   { 	n6UVVVF} 	T%: 	4CZ 	#D,8LMMMF"2%r+O
 
	1  	Fb	>NN(()Di)D)D)DEEE&((( DK))$)*;*HIIE (~~''  	@UVVVF IK))$)*;*EFFE 6!NN,,	(,5F) +2244 [ [:!2!???5:CXCXY`CaCa? :!888  :!666(,(=ej(I%J!]%888*/% "NN,,E!&QJ!]%888#(;r?$((UJQY(ZZZ -1,A,G,G,I,I 
P 
P((j%;((--9!444&+OHe$$S%h$OOOO +- w w!!&+v{fkFWcicu!vvvv ##DOO444'')))((*** ((%(888LLNNNNN(()@Y)@)@AAA  ('>*.1K1NNP[PjPn E 2666(C88$-!#++I666""$$$!!###(*%*-%%' % 76  	> 	> 	> w0PA0P0PQQQNN(()<=========	>s   ,PR
 

S,AS''S,)r  r!   r  rc   )r   rx   r  r  )r   r   r   r  r  rx   )r  r!   r  rx   )r   r   r   r9   r;  rf  rH  ra  r  r|  r  r}  rb  rw  r  r  r~  r  r  rM  rI  r   r   s   @rr   r3  r3  .  ss           H. H. H.T  0" " "Y5 Y5 Y5v   &) ) ) )          D!8 !8 !8Fa) a) a)FF F F,:M :M :Mx)L )L )LV6C 6C 6Cp	 	 	4J 4J 4Jlr> r> r> r> r> r> r>rs   r3  r  Nonec                    t          j        d          } |                     ddd           |                     ddd	
           |                                 }t	          j        |j        rt          j        nt          j                   t          t          j                  }t                      }|                                 |j        rt          j                            |j                  r5|j                            d          r|                    |j                   n@t(                              d|j                    t-          j        |dd|j                    t          j        |                                           d S )NzGUI GGUF Editor)r  
model_path?z*path to GGUF model file to load at startup)nargshelpz	--verbose
store_truezincrease output verbosity)actionr  )levelz.ggufzInvalid model path: zInvalid Model Pathz9The specified file does not exist or is not a GGUF file: )argparseArgumentParseradd_argument
parse_argsloggingbasicConfigverboseDEBUGINFOr	   sysargvr3  showr  ospathisfileendswithrf  loggererrorr   r  exitr   )parserargsappwindows       rr   mainr  9  sh   $1BCCCF
C6bccc
L?Z[[[Dt|MgmmNNNN
sx
 
 CF
KKMMM  _7>>$/** 	_t/G/G/P/P 	_T_----LLAAABBB$]DO]]_ _ _
 HSXXZZrs   __main__)r  r  )M
__future__r   r  r  r  r  r   enumpathlibr   typingr   r   r   r   rr  r#  PySide6.QtWidgetsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   PySide6.QtCorer   environ__file__rh   existsr  insertrx   r  r   r   r    r!   gguf.constantsr"   r#   r$   r%   	getLoggerr  r  r  r  RopeSCALING_TYPELLMPOOLING_TYPEr  	FILE_TYPEr   r  r  r  r'   r   r  r3  r  r   r  rs   rr   <module>r2     s   " " " " " "   				 



         - - - - - - - - - - - -                                                     "*$$$$x..*?*F*M*TW`*`)h)h)j)j$HOOAss44>>07>??@@@  C C C C C C C C C C C C X X X X X X X X X X X X		,	-	- 	I"IINIMI!5	  	II"I _: _: _: _: _:G _: _: _:D	\! \! \! \! \! \! \! \!~8& 8& 8& 8& 8& 8& 8& 8&vH> H> H> H> H>{ H> H> H>V   6 zDFFFFF rs   