
    Pi                         d dl Z d dlmZ d dlmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z#  e!e$          Z%erdd	l&m'Z' dd
ee         fdZ(ddZ)dS )    N)wraps)TYPE_CHECKINGOptional   )DownloadConfig)	xbasenamexdirname	xet_parsexexistsxgetsizexglob
xgzip_openxisdirxisfilexjoinxlistdirxnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxpyarrow_parquet_read_tablexrelpathxsio_loadmatxsplit	xsplitextxwalkxxml_dom_minidom_parse)
get_logger)patch_submoduleDatasetBuilderdownload_configc                 	   t          j        |           }t          |d          rE|j        r>t	          |j        t
                    r"j        |j        _        j        |j        _        dS fd}t          |d |t                              
                                 t          |d |t                              
                                 t          |d |t                              
                                 t          |d |t                              
                                 t          |dt                    
                                 t          |d	t                    
                                 t          |d
t                     
                                 t          |dt"                    
                                 t          |dt$                    
                                 t          |dt&                    
                                 t          |d |t(                              
                                 t          |d |t*                              
                                 t          |d |t,                              
                                 t          |d |t.                              
                                 t          |dt0                    
                                 t          |d |t2                              
                                 t          |d |t4                              
                                 t          |d |t6                    dg          
                                 t          |d |t8                    dg          
                                 t          |d |t:                    dg          
                                 t          |d |t<                              
                                 t          |d |t>                              
                                 |j         !                    d          s1t          |d |tD                              
                                 |_        dS )a?  Extend the module to support streaming.

    We patch some functions in the module to use `fsspec` to support data streaming:
    - We use `fsspec.open` to open and read remote files. We patch the module function:
      - `open`
    - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
      functions:
      - `os.path.join`
      - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)

    The patched functions are replaced with custom functions defined to work with the
    :class:`~download.streaming_download_manager.StreamingDownloadManager`.

    Args:
        module_path: Path to the module to be extended.
        download_config: Mainly use `token` or `storage_options` to support different platforms and auth types.
    _patched_for_streamingNc                 N     t                      fd            }d|_        |S )Nc                       | di|S )Nr#    )argskwargsr#   functions     f/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/streaming.pywrapperz?extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapperG   s    8TM?MfMMM    	wrap_auth)r   _decorator_name_)r+   r-   r#   s   ` r,   r/   z.extend_module_for_streaming.<locals>.wrap_authF   sG    	x	N 	N 	N 	N 	N 
	N $/ r.   openz
os.listdirzos.walkz	glob.globzos.path.joinzos.path.dirnamezos.path.basenamezos.path.relpathzos.path.splitzos.path.splitextzos.path.existszos.path.isdirzos.path.isfilezos.path.getsizezpathlib.Pathz	gzip.openz
numpy.loadzpandas.read_csv__version__)attrszpandas.read_excelzscipy.io.loadmatzxml.etree.ElementTree.parsezxml.dom.minidom.parsezdatasets.packaged_modules.zpyarrow.parquet.read_table)#	importlibimport_modulehasattrr%   
isinstancer   tokenstorage_optionsr    r   startr   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   __name__
startswithr   )module_pathr#   moduler/   s    `  r,   extend_module_for_streamingr?   *   s+   & $[11F v/00 V5R f3^DD 	\2A2GF)/<K<[F)9     FFIIe$4$455;;===FL))H*=*=>>DDFFFFIyy'7'788>>@@@FK5)9)9::@@BBBFNE2288:::F-x88>>@@@F.	::@@BBBF-x88>>@@@FOV44::<<<F.	::@@BBBF,ii.@.@AAGGIIIFOYYv->->??EEGGGF,ii.@.@AAGGIIIF-yy/B/BCCIIKKKFNE2288:::FK:)>)>??EEGGGFL))K*@*@AAGGIIIF-yy9I/J/JS`RabbbhhjjjF/;M1N1NWdVefffllnnnF.		,0G0GP]___eegggF999Y;O;OPPVVXXXF3YY?U5V5VWW]]___?%%&BCC n <iiHc>d>deekkmmm$3F!!!r.   builderr"   c                     t          | j        | j                  }t          | j        |           ddlm fdt          |           j        dd         D             }|D ]}t          ||           dS )zExtend the dataset builder module and the modules imported by it to support streaming.

    Args:
        builder (:class:`DatasetBuilder`): Dataset builder instance.
    )r9   r8   )r#   r   r!   c                 ^    g | ])}t          |          r|j        j        k    "|j        *S r(   )
issubclass
__module__).0clsr"   s     r,   
<listcomp>z8extend_dataset_builder_for_streaming.<locals>.<listcomp>}   sI       c>** 03~AZ/Z/Z 	/Z/Z/Zr.   N)	r   r9   r8   r?   rD   r@   r"   type__mro__)r@   r#   parent_builder_modulesr>   r"   s       @r,   $extend_dataset_builder_for_streamingrK   n   s     %W5LT[TabbbO 2OTTTT
 ('''''   ==(,  
 ) M M#FOLLLLLM Mr.   )N)r@   r"   )*r4   	functoolsr   typingr   r   download.download_configr   utils.file_utilsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.loggingr   utils.patchingr    r;   loggerr@   r"   r?   rK   r(   r.   r,   <module>rS      s             * * * * * * * * 4 4 4 4 4 4                                                 2 & % % % % % + + + + + + 
H		  (''''''A4 A4h~>V A4 A4 A4 A4HM M M M M Mr.   