
    &`i                     f    d dl mZmZmZ d dlmZ d dlmZ d dlm	Z	 erd dl
Z
 G d de	          ZdS )    )TYPE_CHECKINGIteratorList)DelegatingBlockBuilder)Block)FileBasedDatasourceNc                   h     e Zd ZdZdZddddee         dedef fd	Zd
ddede	e
         fdZ xZS )TextDatasourcez4Text datasource, for reading and writing text files.textFzutf-8)drop_empty_linesencodingpathsr   r   c                X     t                      j        |fi | || _        || _        d S )N)super__init__r   r   )selfr   r   r   file_based_datasource_kwargs	__class__s        /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/datasource/text_datasource.pyr   zTextDatasource.__init__   s8     	??">??? 0     fzpyarrow.NativeFilepathreturnc              #   \  K   |                                 }t                      }|                    | j                                                  }|D ]@}| j        r|                                dk    r"| j        |i}|                    |           A|	                                }|V  d S )N )
readallr   decoder   
splitlinesr   strip_COLUMN_NAMEaddbuild)	r   r   r   databuilderlineslineitemblocks	            r   _read_streamzTextDatasource._read_stream   s      yy{{(**DM**5577 	 	D$ ););%t,DKKr   )__name__
__module____qualname____doc__r    r   strboolr   r   r   r)   __classcell__)r   s   @r   r
   r
      s        >>L "'! ! !Cy! 	!
 ! ! ! ! ! !2 # (5/        r   r
   )typingr   r   r   +ray.data._internal.delegating_block_builderr   ray.data.blockr   )ray.data.datasource.file_based_datasourcer   pyarrowr
    r   r   <module>r7      s    0 0 0 0 0 0 0 0 0 0 N N N N N N             I I I I I I NNN    (     r   