
    Pi                     j    d dl mZ d dlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ  G d	 d
e          ZdS )    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                        e Zd ZdZ	 	 	 	 	 	 	 	 ddej        j        dee         dee	         d	e
d
ede
dede
def fdZd Z xZS )SparkDatasetReaderzA dataset reader that reads from a Spark DataFrame.

    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
    provided. Streaming is not currently supported.
    NTFarrowdfsplitfeatures	streaming	cache_dirkeep_in_memoryworking_dirload_from_cache_filefile_formatc
           	           t                      j        d|||||d|
 || _        |	| _        t	          d||||d|
| _        d S )N)r   r   r   r   r   )r   r   r   r    )super__init___load_from_cache_file_file_formatr   builder)selfr   r   r   r   r   r   r   r   r   kwargs	__class__s              e/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/io/spark.pyr   zSparkDatasetReader.__init__   s     	 	
)	
 	
 	
 	
 	
 &:"' 
#	
 

 
 
    c                     | j         r | j                            | j                  S | j        rd nt
          j        }| j                            || j                   | j        	                    | j                  S )N)r   )download_moder   )
r   r   as_streaming_datasetr   r   r   FORCE_REDOWNLOADdownload_and_preparer   
as_dataset)r   r$   s     r!   readzSparkDatasetReader.read1   s    > 	G<444:4FFF $ :]@]))') 	* 	
 	
 	
 |&&TZ&888r"   )NNTNFNTr   )__name__
__module____qualname____doc__pysparksql	DataFramer   r   r   boolstrr   r)   __classcell__)r    s   @r!   r   r      s          '+'+$%)"
 
K!
 
#
 8$	

 
 
 
 
 #
 
 
 
 
 
 
>9 9 9 9 9 9 9r"   r   )typingr   r.    r   r   downloadr   packaged_modules.spark.sparkr   abcr
   r   r   r"   r!   <module>r9      s           # # # # # # # # # # # # # # 0 0 0 0 0 0 & & & & & &.9 .9 .9 .9 .9. .9 .9 .9 .9 .9r"   