
    Pi                         d dl Z d dlZd dlmZ d dlmZ d dlZd dlZd dl	m
Z
 ej        j                            e          Z G d dej                  ZdS )    N)islice)Iterable)Keyc                   N    e Zd ZdZd Zd ZdefdZdee         fdZ	d Z
d	 Zd
S )Eval   c                 (    t          j                    S N)datasetsDatasetInfo)selfs    w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/packaged_modules/eval/eval.py_infoz
Eval._info   s    #%%%    c           	         | j         j        st          d| j         j                   dj        _                            | j         j                  }                    |          }g }|                                D ]E\  }}fd|D             }|                    t          j
        ||||         d                     F| j        j        st          t          |                     |d                   | j                            }d |D             }	t#          j        |	d	          j        }
t          j                            |
          | j        _        |S )
z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=Tc                 :    g | ]}                     |          S  )
iter_files).0log
dl_managers     r   
<listcomp>z*Eval._split_generators.<locals>.<listcomp>   s'    #O#O#O3J$9$9#$>$>#O#O#Or   )logs_files_iterables
base_files)name
gen_kwargsr   c                 N    g | ]"}t           j                            |g          #S r   )paTablefrom_pylist)r   examples     r   r   z*Eval._split_generators.<locals>.<listcomp>/   s*    WWWW--wi88WWWr   default)promote_options)config
data_files
ValueErrordownload_configextract_on_the_flydownloadextractitemsappendr   SplitGeneratorinfofeatureslistr   _iter_samples_from_log_files#NUM_EXAMPLES_FOR_FEATURES_INFERENCEr   concat_tablesschemaFeaturesfrom_arrow_schema)r   r   base_data_filesextracted_data_filessplits
split_namelogsr   first_examples	pa_tablesinferred_arrow_schemas    `         r   _split_generatorszEval._split_generators   s   {% 	wu]a]h]suuvvv8<
"5$--dk.DEE)11/BB 4 : : < < 
	 
	J#O#O#O#O$#O#O#O MM'#0D&5j&A         y! 		\!556J16MNN<  N XWWWWI$&$4YPY$Z$Z$Z$a!!)!2!D!DEZ![![DIr   sample_pathc                     t           j                            t           j                            |                    d                             d          \  }}t          |          t          |          fS )Nr   _epoch_)ospathsplitextbasenamesplitint)r   r@   sample_idx_strepoch_idx_strs       r   _sort_samples_keyzEval._sort_samples_key5   s\    *,'*:*:27;K;KK;X;X*Y*YZ[*\*b*bcl*m*m'M""C$7$788r   	log_filesc              #     K   d |D             }|                     | j                   |D ]}t          |          5 }t          j        |          }|D ]j}t          ||         t                    rt          j        ||                   ||<   t          ||         t                    rd ||         D             ||<   k|V  d d d            n# 1 swxY w Y   d S )Nc                     g | ]D}t           j                            t           j                            |                    d k    B|ES )samples)rC   rD   rF   dirname)r   log_files     r   r   z5Eval._iter_samples_from_log_files.<locals>.<listcomp>;   sB    wwwXbg>N>Nrw_gOhOh>i>imv>v>v>v>v>vr   )keyc                 6    g | ]}t          j        |          S r   )jsondumps)r   xs     r   r   z5Eval._iter_samples_from_log_files.<locals>.<listcomp>D   s     (N(N(N1A(N(N(Nr   )	sortrK   openrT   load
isinstancedictrU   r0   )r   rL   sample_filessample_filefsamplefields          r   r1   z!Eval._iter_samples_from_log_files:   s5     wwwwwd4555' 	 	Kk"" a1# O OE!&-66 B(,
6%=(A(Au!&-66 O(N(Nu(N(N(Nu              	 	s   BCC	C	c              #      K   |E d {V  d S r
   r   )r   r   r   s      r   _generate_shardszEval._generate_shardsG   s$      r   c              #      K   t          |          D ]A\  }}t          |                     |                    D ]\  }}t          ||          |fV  Bd S r
   )	enumerater1   r   )r   r   r   file_idxrL   
sample_idxr_   s          r   _generate_exampleszEval._generate_examplesJ   s{      #,-A#B#B 	8 	8Hi&/0Q0QR[0\0\&]&] 8 8"
F(J//777778	8 	8r   N)__name__
__module____qualname__r2   r   r?   strrK   r   r1   rb   rg   r   r   r   r   r      s        *+'& & &  @9S 9 9 9 9
hsm      8 8 8 8 8r   r   )rT   rC   	itertoolsr   typingr   pyarrowr   r   datasets.builderr   utilslogging
get_loggerrh   loggerGeneratorBasedBuilderr   r   r   r   <module>ru      s     				                              
		*	*8	4	4>8 >8 >8 >8 >88) >8 >8 >8 >8 >8r   