
    PiD                        d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ  G d d	ej                  Zd
 Zd Zd Zd Zej         e
j        d          k     rd Zn e            rd Z ee          d             Zd Zd Zd Zd Zd Zd Zej         e
j        d          k     r ee          d             ZdS  e            r ee          d             ZdS dS )zPExtends `dill` to support pickling more types and produce more consistent dumps.    N)BytesIO)CodeTypeFunctionType)version   )configc                        e Zd Zej                            ej        j                                                  ZdZ	ddZ
 fdZd Z xZS )PicklerFTc                    t          |          }|| j        vrdt          j        v r*dd l}||j        u r t          |          t                     dt          j        v r6dd l}t          ||j
                  r t          |          t                     dt          j        v r*dd l}||j        u r t          |          t                     dt          j        v rdd l}t          ||j                  r t          |          t"                     ||j        u r t          |          t&                     t          ||j        j                  rt-          |d|          }dt          j        v r6dd l}t          ||j                  r t          |          t2                     |t4          u rt-          |d|          }t6          j                            | ||	           d S )
Nregexr   spacytiktokentorch	_orig_modtransformers_torchdynamo_orig_callable)save_persistent_id)typedispatchsysmodulesr   Patternpklregister_save_regexPatternr   
issubclassLanguage_save_spacyLanguager   Encoding_save_tiktokenEncodingr   Tensor_save_torchTensor	Generator_save_torchGeneratornnModulegetattrr   PreTrainedTokenizerBase)_save_transformersPreTrainedTokenizerBaser   dillr
   save)	selfobjr   obj_typer   r   r   r   r   s	            h/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/utils/_dill.pyr*   zPickler.save   s   994=((#+%%u},,)K))*<===#+%%h77 ?)K))*=>>>S[((x000)K))*@AAA#+%%h55 =)K))*;<<<u..)K))*>??? h88 9!#{C88C,,####h(LMM U)K))*STTT |###;SAAC$8JKKKKK    c                     	 t          |          }n*# t          $ r ddlm t          |fd          }Y nw xY w t	                      j        |g|R i |S )Nr   Hasherc                 :                         | d                   S )Nr   )hash)xr2   s    r.   <lambda>z)Pickler._batch_setitems.<locals>.<lambda>P   s    AaD0A0A r/   key)sorted	Exceptiondatasets.fingerprintr2   super_batch_setitems)r+   itemsargskwargsr2   	__class__s       @r.   r=   zPickler._batch_setitemsH   s    	C5MMEE 	C 	C 	C3333335&A&A&A&ABBBEEE	C 'uww&u>t>>>v>>>s    $;;c                 v    t          |          t          ur"t          j                            | |           d S d S N)r   strr)   r
   memoize)r+   r,   s     r.   rE   zPickler.memoizeS   s9    99CL  s+++++  r/   )T)__name__
__module____qualname__r)   _dillMetaCatchingDictr
   r   copy_legacy_no_dict_keys_sortingr*   r=   rE   __classcell__)rA   s   @r.   r
   r
      s        z**4<+@+E+E+G+GHHH#( 'L 'L 'L 'LR	? 	? 	? 	? 	?, , , , , , ,r/   r
   c                       fd}|S )z'Register a custom reducer for the type.c                 &    | t           j        <   | S rC   )r
   r   )functs    r.   proxyzpklregister.<locals>.proxy\   s    "r/    )rQ   rR   s   ` r.   r   r   Y   s#         Lr/   c                  (   t           j        j        dd         t          j        d          j        t          j        d          j        t          j        d          j        t          j        d          j        t          j        d          j        fv S )z<Check if the current dill version is in the supported range.N   0.3.6z0.3.7z0.3.8z0.3.9z0.4.0)r   DILL_VERSIONreleaser   parserS   r/   r.   _is_supported_dill_versionrZ   c   ss    &rr*g&g&g&g&g&/  r/   c                 N    t          |d                              |            dS )zPickle an object to a file.T)recurseN)r
   dumpr,   files     r.   r]   r]   n   s)    D$$$S)))))r/   c                 f    t                      }t          | |           |                                S )zPickle an object to a string.)r   r]   getvaluer^   s     r.   dumpsrb   s   s&    99DdOOO==??r/   rV   c                 N    t           j        j                            |           d S rC   )r)   rI   loginfopicklermsgs     r.   rd   rd   |   s     
C     r/   c                 P    t           j        j                            | |           d S rC   )r)   rI   loggertracerf   s     r.   rd   rd      s#    
-----r/   c                    t          | d|            	 t          |          f}n-# t          $ r  ddlm} t          ||j                  f}Y nw xY w|                     t          ||           t          | d           d S )NzSe: r   r1   r7   r,   z# Se)rd   r9   r:   r;   r2   r4   save_reduceset)rg   r,   r?   r2   s       r.   	_save_setrp      s    /s~ / / ///////s,,,./
 Ts+++s   & 'AAc                     dd l }t          | d|            |j        |j        f}|                     |j        ||           t          | d           d S )Nr   zRe: rm   z# Re)r   rd   patternflagsrn   compile)rg   r,   r   r?   s       r.   r   r      sb    LLLK#Dt555r/   c                     dd l }t          | d|            |j        |j        |j        |j        f}|                     |j        ||           t          | d           d S )Nr   zEnc: rm   z# Enc)r   rd   name_pat_str_mergeable_ranks_special_tokensrn   r   )rg   r,   r   r?   s       r.   r   r      sm    OOOHclC$8#:MND)4S999r/   c                    dd l dfd	}t          | d|            |j        j        k    rX|                                                    j                                                                                  j        f}n9|                                                                                                f}| 	                    |||           t          | d           d S )Nr   c                 `                         |           }|r|                    |          }|S rC   )
from_numpyr   )np_arraydtypetensorr   s      r.   create_torchTensorz-_save_torchTensor.<locals>.create_torchTensor   s4    !!(++ 	([[''Fr/   zTo: rm   z# TorC   )
r   rd   r~   bfloat16detachtofloatcpunumpyrn   )rg   r,   r   r?   r   s       @r.   r!   r!      s    LLL      
yEN""

,,002288::ENK

  ""((**,*Dc:::r/   c                     dd l fd}t          | d|            |                                f}|                     |||           t          | d           d S )Nr   c                 Z                                     }|                    |            |S rC   )r"   	set_state)state	generatorr   s     r.   create_torchGeneratorz3_save_torchGenerator.<locals>.create_torchGenerator   s,    OO%%	E"""r/   zGe: rm   z# Ge)r   rd   	get_statern   )rg   r,   r   r?   r   s       @r.   r#   r#      s    LLL    
 MMOOD-t===r/   c                     dd l fd}t          | d|            |j        |                                f}|                     |||           t          | d           d S )Nr   c                     j                             | d         d                   }|                    |           }|                    |          S )Nnlplang)utilget_lang_classfrom_config
from_bytes)r   byteslang_cls	lang_instr   s       r.   create_spacyLanguagez1_save_spacyLanguage.<locals>.create_spacyLanguage   sH    :,,VE]6-BCC((00	##E***r/   zSp: rm   z# Sp)r   rd   r   to_bytesrn   )rg   r,   r   r?   r   s       @r.   r   r      s    LLL+ + + + +
 J'D,d<<<r/   c                     t          | d|            |j        }d|v r t          |d         t                    ri |d<   |                     t          |          d||           t          | d           d S )NzTok: cacherS   )r   r,   z# Tok)rd   __dict__
isinstancedictrn   r   )rg   r,   r   s      r.   r(   r(      s    LE%JuW~t<<gS		2U<<<r/   c                    t           j        j                            d|            |j                            d          st          |j                            t          j	        j
                            dk    rB|j                            t          j	        j
                  d                             d          s|j        dk    rdn#t          j	                            |j                  }d}t           j        j        rt          |d          rp|j        |j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        |t2          j        d	k    r|j        n|j        |j        |j        f}n|j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        ||j        |j        |j        f}nL|j        |j        |j        |j        |j        |j        |j        |j        ||j        ||j        |j        |j        f}|                     t@          ||
           t           j        j                            d           dS )z
        From dill._dill.save_code
        This is a modified version that removes the origin (filename + line no.)
        of functions created in notebooks or shells for example.
        zCo: <   
ipykernel_<lambda> co_posonlyargcount)rU   
   rm   # CoN)!r)   rI   rd   re   co_filename
startswithlensplitospathsepco_namebasenamePY3hasattrco_argcountr   co_kwonlyargcount
co_nlocalsco_stacksizeco_flagsco_code	co_constsco_namesco_varnamesr   version_infoco_linetable	co_lnotabco_freevarsco_cellvarsrn   r   rg   r,   r   co_firstlinenor?   s        r.   
_save_coder      s3    	
L3LL)))& ))#..3 CO))"'+6677!;;O))"'+66r:EElSS < {j(( B !!#/22 	 :> 6	s011 $O*)N$LKMLOK"(+(8G(C(CC$$OO!( O)N$LKMLOK"MOO&  D  	Hd444
F###r/   c                    t           j        j                            | d|           |j                            d          st          |j                            t          j	        j
                            dk    rB|j                            t          j	        j
                  d                             d          s|j        dk    rdn#t          j	                            |j                  }d}t          |d          rx|j        |j        |j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        |j        ||j        |j        |j        |j        |j        |j        f}nt          |d	          rl|j        |j        |j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        |j        ||j        |j        |j        |j        f}n*t          |d
          r_|j        |j        |j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        ||j        |j        |j        f}nt          |d          rY|j        |j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        ||j        |j        |j        f}nR|j        |j        |j        |j        |j        |j        |j        |j        |j        ||j        ||j        |j        |j        f}|                      t           j        j!        ||           t           j        j                            | d           d S )NzCo: %sr   r   r   r   r   r   co_endlinetableco_exceptiontabler   r   rm   r   )"r)   rI   rj   rk   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   co_qualnamer   co_columntabler   r   r   r   rn   _create_coder   s        r.   	save_coder   B  sJ   
3777, ))#..3 CO))"'+6677!;;O))"'+66r:EElSS < {j(( B !!#/22 	  3)** f	 &%  #"%+DD. S-.. N	 &%  %'DD* S.)) 8	 &%  #DD& S.// $	&% !DD( % D$ 	DJ3TsCCC
000r/   )__doc__r   r   ior   typesr   r   r)   	packagingr   r   r   r
   r   rZ   r]   rb   rW   rY   rd   ro   rp   r   r   r!   r#   r   r(   r   r   rS   r/   r.   <module>r      sU   W V 				 



       ( ( ( ( ( ( ( (             ;, ;, ;, ;, ;,dl ;, ;, ;,|    * * *
   
w////! ! ! !  !! .. . . S        &       
w////[\ \ \ \ \|  !! S[P P P P PS Sr/   