
    Pi(                     ,   U d dl Z d dlZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ  ej        e                    dd          d                    ZdZd	Zd
Zej                            dd          Zedz   ZdZdZ ej         ej                              Zh dZh dZe                    dh          Z e                    dh          Z! ej        e j"                            d                    Z# ej        e j"                            d                    Z$ ej        e j"                            d                    Z% ej        e j"                            d                    Z& ej        e j"                            d                    Z'ej                            dd          (                                Z)ej                            dd          (                                Z*ej                            dd          (                                Z+dZ,dZ-e*e v r{e)evrwe j.        /                    d          duZ-e-rX	  ej        e j"                            d                    Z,e0                    de, d           n+# e j"        j1        $ r Y nw xY wne2                    d            dZ3e j.        /                    d!          duZ4e4rX	  ej        e j"                            d!                    Z3e0                    d"e3 d           n# e j"        j1        $ r Y nw xY wdZ5e j.        /                    d#          duZ6e6rX	  ej        e j"                            d#                    Z5e0                    d$e5 d           n# e j"        j1        $ r Y nw xY wdZ7dZ8e)e v re*evre j.        /                    d%          duZ8e8rFd&D ]AZ9	  ej        e j"                            e9                    Z7 n# e j"        j1        $ r Y >w xY wdZ8e8r@e7j:        d'k     re2                    d(e7 d)           dZ8n/e2                    d*e7 d           ne2                    d+           dZ;dZ<e+e v re j.        /                    d,          duoe j.        /                    d-          duZ<e<rX	  ej        e j"                            d,                    Z;e2                    d.e; d           n+# e j"        j1        $ r Y nw xY wne2                    d/           e j.        /                    d0          duZ=e j.        /                    d1          duZ>d2Z?d2Z@e j.        /                    d3          duZAe j.        /                    d4          duZBe j.        /                    d5          duZCe j.        /                    d6          duZDe j.        /                    d7          duZEe j.        /                    d8          duZFe j.        /                    d9          duZGe j.        /                    d:          duZHd;ZI ejJ        d<eI          ZKejL        M                    eKd=          ZNejL        O                     ejJ        d>eN                    ZPejL        M                    ePd?          ZQ e ejJ        d@eQ                    ZRejL        M                    ePdA          ZS e ejJ        dBeS                    ZTdCZUejL        M                    eReU          ZV e ejJ        dDeV                    ZWdEZXejL        M                    eVeX          ZY e ejJ        dFeY                    ZZdZ[ej                            dGd          (                                e v Z\dHZ]d2Z^dIZ_dJdKd dLZ`dMZadNZbdOZcdZddZedZfdZgdPZhdPZidPZjdMZkej                            dQ          Zlele
jm        nel(                                ev ZmemZnej                            dR          Zoeoeo(                                ev ndZpeeq         erdR<   d Zs etej                            dSes                    ZudTZvdUZwdVZxdWZydXZzdYZ{dZZ|d[Z}d\Z~d]Zd^Zd_Zd`ZdaZdbZdaZdcZdaZdbZddZdMZddZdIZdeZdfZdgZdS )h    N)Path)Optional)	constants)version.   zBhttps://s3.amazonaws.com/datasets.huggingface.co/datasets/datasetsz5https://cdn-datasets.huggingface.co/datasets/datasetszXhttps://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}HF_ENDPOINTzhttps://huggingface.coz-/datasets/{repo_id}/resolve/{revision}/{path}z)hf://datasets/{repo_id}@{revision}/{path}main>   1ONYESTRUE>   0NOOFFFALSEAUTOdillfsspecpandaspyarrowhuggingface_hubUSE_TF	USE_TORCHUSE_JAXzN/AFtorchzPyTorch version z available.z'Disabling PyTorch because USE_TF is setpolarszPolars version duckdbzDuckdb version 
tensorflow)	r   ztensorflow-cpuztensorflow-gpuz
tf-nightlyztf-nightly-cpuztf-nightly-gpuzintel-tensorflowztensorflow-rocmztensorflow-macos   z"TensorFlow found but with version z(. `datasets` requires version 2 minimum.zTensorFlow version z-Disabling Tensorflow because USE_TORCH is setjaxjaxlibzJAX version z-Disabling JAX because USE_JAX is set to False
sqlalchemyPILT
torchcodectorchvision
pdfplumbernibabelrarfile	zstandardlz4py7zrz~/.cacheXDG_CACHE_HOMEhuggingfaceHF_HOMEdatasetsHF_DATASETS_CACHEmodulesHF_MODULES_CACHE	downloads$HF_DATASETS_DOWNLOADED_DATASETS_PATH	extracted#HF_DATASETS_EXTRACTED_DATASETS_PATHHF_UPDATE_DOWNLOAD_COUNTS   i  i   i   )min_chunk_sizemax_chunk_size
norm_level
   500MB100MBd   HF_DATASETS_OFFLINE!HF_DATASETS_DISABLE_PROGRESS_BARSHF_DATASETS_IN_MEMORY_MAX_SIZEzdataset.arrowzindices.arrowz
state.jsonzdataset_info.jsonzdataset_infos.jsonLICENSEzdataset_dict.jsonconfigsz	README.mdz.huggingface.yamldatasets_modules   zhf_datasets-      <      g?2   l        )	importlibimportlib.metadataloggingosplatformpathlibr   typingr   r   r   	packagingr   	getLogger__name__splitloggerS3_DATASETS_BUCKET_PREFIX"CLOUDFRONT_DATASETS_DISTRIB_PREFIXREPO_DATASETS_URLenvirongetr	   HUB_DATASETS_URLHUB_DATASETS_HFFS_URLHUB_DEFAULT_VERSIONparsepython_version
PY_VERSIONENV_VARS_TRUE_VALUESENV_VARS_FALSE_VALUESunionENV_VARS_TRUE_AND_AUTO_VALUESENV_VARS_FALSE_AND_AUTO_VALUESmetadataDILL_VERSIONFSSPEC_VERSIONPANDAS_VERSIONPYARROW_VERSIONHF_HUB_VERSIONupperr   r   r   TORCH_VERSIONTORCH_AVAILABLEutil	find_specdebugPackageNotFoundErrorinfoPOLARS_VERSIONPOLARS_AVAILABLEDUCKDB_VERSIONDUCKDB_AVAILABLE
TF_VERSIONTF_AVAILABLEpackagemajorJAX_VERSIONJAX_AVAILABLESQLALCHEMY_AVAILABLEPIL_AVAILABLEIS_OPUS_SUPPORTEDIS_MP3_SUPPORTEDTORCHCODEC_AVAILABLETORCHVISION_AVAILABLEPDFPLUMBER_AVAILABLENIBABEL_AVAILABLERARFILE_AVAILABLEZSTANDARD_AVAILABLELZ4_AVAILABLEPY7ZR_AVAILABLEDEFAULT_XDG_CACHE_HOMEgetenvr-   pathjoinDEFAULT_HF_CACHE_HOME
expanduserHF_CACHE_HOMEDEFAULT_HF_DATASETS_CACHEr1   DEFAULT_HF_MODULES_CACHEr3   DOWNLOADED_DATASETS_DIR DEFAULT_DOWNLOADED_DATASETS_PATHDOWNLOADED_DATASETS_PATHEXTRACTED_DATASETS_DIRDEFAULT_EXTRACTED_DATASETS_PATHEXTRACTED_DATASETS_PATHSAVE_ORIGINAL_SHARD_LENGTHSr8   &HF_DATASETS_MULTITHREADING_MAX_WORKERSUSE_PARQUET_EXPORTDEFAULT_MAX_BATCH_SIZEDEFAULT_CDC_OPTIONS'ARROW_READER_BATCH_SIZE_IN_DATASET_ITERMAX_SHARD_SIZEMAX_ROW_GROUP_SIZE)PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS)PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS*PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS)PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS+ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS_offlineHF_HUB_OFFLINErA   #__HF_DATASETS_DISABLE_PROGRESS_BARSrB   bool__annotations__DEFAULT_IN_MEMORY_MAX_SIZEfloatIN_MEMORY_MAX_SIZEDATASET_ARROW_FILENAMEDATASET_INDICES_FILENAMEDATASET_STATE_JSON_FILENAMEDATASET_INFO_FILENAMEDATASETDICT_INFOS_FILENAMELICENSE_FILENAMEDATASETDICT_JSON_FILENAMEMETADATA_CONFIGS_FIELDREPOCARD_FILENAMEREPOYAML_FILENAMEMODULE_NAME_FOR_DYNAMIC_MODULES%MAX_DATASET_CONFIG_ID_READABLE_LENGTHTEMP_CACHE_DIR_PREFIXSTREAMING_READ_MAX_RETRIESSTREAMING_READ_RETRY_INTERVAL0STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL(STREAMING_READ_RATE_LIMIT_RETRY_INTERVALSTREAMING_OPEN_MAX_RETRIESSTREAMING_OPEN_RETRY_INTERVAL*DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE2GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE3ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE/MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLELPBAR_REFRESH_TIME_INTERVALUPLOADS_MAX_NUMBER_PER_COMMITMAX_TABLE_NBYTES_FOR_PICKLING     c/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/config.py<module>r      s
             				              % % % % % %       
	8>>#q11!4	5	5 a %\ "n  jnn],DEE!PP C  W]2824455
 211 333  4 : :F8 D D !6!<!<fX!F!F  w}Y/77??@@y199(CCDDy199(CCDD'-	 2 : :9 E EFFy199:KLLMM	&	)	)	/	/	1	1JNN;//5577	
*..F
+
+
1
1
3
3---&@T2T2Tn..w77tCO 	)GM)*<*D*DW*M*MNNMLLFMFFFGGGG!6 	 	 	D		 KK9:::>++H55TA  &y'9'A'A('K'KLLB~BBBCCCC2    >++H55TA  &y'9'A'A('K'KLLB~BBBCCCC2    
	***y@T/T/T>++L99EL !

 	! 	!G*W]9+=+E+Eg+N+NOO
  %:   
 !L GaKKqZqqqrrr LLKKEjEEEFFF
KK?@@@ 
+++N,,U334?rIND\D\]eDfDfnrDrM 	''-	(:(B(B5(I(IJJKKK?{???@@@@!6 	 	 	D		 KK?@@@ !~//==TI  ((//t;   ~//==TI !00??tK  ~//==TI N,,Y77tC  N,,Y77tC n..{;;4G ((//t;.**7334? $ +-CDD^]CC ""929Y8M#N#NOOGLL
CC D#68QRRSS 7<<yAA 4		"46NOOPP % #%7<<0ACZ#[#[  4		*PRr s stt $ "$',,/OQg"h"h $yry)NPoppqq  $  JNN.77==??C`` 
 *, &    )3{bcdd  +- '    -1 ),0 )-1 *,0 ) .1 *-0 *.1 +-/ * :>>/00-5-=))8>>CSCSWkCk$  ')jnn5X&Y&Y # +6 (--//3GGG	 "8D>     U2:>>*JLfgghh  ) * * + 1  / "  ' "4 (+ % '     ! 35 0+- (  !  .1 *57 269 3 37 / "  !#  !(   s^   AJ J! J!AL L/.L/AN N'&N'(PPP5AS7 7T	T	