
    &`i                         d dl Z d dlmZmZ d dlmZ d dlmZmZ d dl	m
Z
mZ  e j        e          ZdefdZd Zd	 Zd
efdZd Z G d de
          ZdS )    N)ListOptional)_check_import)BlockBlockMetadata)
DatasourceReadTaskreturnc                      dd l } d| j         S )Nr   zray/)ray__version__)r   s    /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/datasource/bigquery_datasource.py_create_user_agentr      s    JJJ##/###    c                  >    ddl m}   | t                                S Nr   
ClientInfo)
user_agent)google.api_core.client_infor   r   r   s    r   _create_client_infor      s6    666666:%''   r   c                  >    ddl m}   | t                                S r   )$google.api_core.gapic_v1.client_infor   r   r   s    r   _create_client_info_gapicr      s6    ??????:%''   r   
project_idc                 T    ddl m} |                    | t                                S )Nr   )bigquery)projectclient_info)google.cloudr   Clientr   )r   r   s     r   _create_clientr"   !   s;    %%%%%%??'))    r   c                  R    ddl m}  |                     t                                S )Nr   bigquery_storage)r   )r    r%   BigQueryReadClientr   r$   s    r   _create_read_clientr'   *   s:    ------..-// /   r   c                       e Zd Z	 	 ddedee         dee         fdZ	 ddedee         dee         fd	Z	dee         fd
Z
dededdfdZdS )BigQueryDatasourceNr   datasetqueryc                     t          | dd           t          | dd           t          | dd           || _        || _        || _        ||t	          d          d S d S )Nzgoogle.cloudr   )modulepackager%   zgoogle.api_core
exceptionszNQuery and dataset kwargs cannot both be provided (must be mutually exclusive).)r   _project_id_dataset_query
ValueError)selfr   r*   r+   s       r   __init__zBigQueryDatasource.__init__3   s     	d>:FFFFd>;MNNNNd#4lKKKK%!42   !4!4r   parallelismper_task_row_limitr
   c                 h   ddl m} dt          fd| j        rt	          | j                  }|                    | j                  }|                                 t          |j	                  }|
                    d          d         }|
                    d          d         }n`|                     | j        | j                   | j        
                    d          d         }| j        
                    d          d	         }t                      }	d
| j         d| d| }
|dk    rd }|j                            |
|j        j        j                  }|	                    d
| j         ||          }g }t&                              dt          t+          |j                            z              t+          |j                  |k     rt&                              d           |j        D ]A}t/          d d d d           }t1          |ffd	||          }|                    |           B|S )Nr   r$   r
   c                 z    t                      }|                    | j                  }|                                S N)r'   	read_rowsnameto_arrow)streamclientreaders      r   _read_single_partitionzABigQueryDatasource.get_read_tasks.<locals>._read_single_partitionL   s2    (**F%%fk22F??$$$r   r   .   z	projects/z
/datasets/z/tables/)tabledata_format)parentread_sessionmax_stream_countzCreated streams: zThe number of streams created by the BigQuery Storage Read API is less than the requested parallelism due to the size of the dataset.)num_rows
size_bytesinput_files
exec_statsc                      |           gS r:    )r>   rA   s    r   <lambda>z3BigQueryDatasource.get_read_tasks.<locals>.<lambda>   s    '='=f'E'E&F r   )r7   )r    r%   r   r2   r"   r0   r+   resultstrdestinationsplit_validate_dataset_table_existr1   r'   typesReadSession
DataFormatARROWcreate_read_sessionloggerinfolenstreamsr   r	   append)r4   r6   r7   r%   query_client	query_jobrU   
dataset_idtable_id
bqs_clientrG   requested_sessionrJ   
read_tasksr>   metadata	read_taskrA   s                    @r   get_read_tasksz!BigQueryDatasource.get_read_tasksG   s    	211111	%e 	% 	% 	% 	%
 ; 
	3)T5EFFFL$**4;77Ii344K$**3//3J"((--b1HH..t/?OOO,,S11!4J}**3//2H(**
VD,VV
VVHVV"K,2>>(.9? ? 
 
 "551t/11*( 6 
 
 
'#c,2F.G.G*H*HHIII|#$${22KK@   #* 	) 	)F$ 	  H !$FFFFF#5  I
 i((((r   c                     d S r:   rQ   )r4   s    r   estimate_inmemory_data_sizez.BigQueryDatasource.estimate_inmemory_data_size   s    tr   c                    ddl m} t          |          }|                    d          d         }	 |                    |           n0# |j        $ r# t          d                    |                    w xY w	 |                    |           d S # |j        $ r# t          d                    |                    w xY w)Nr   )r/   rB   rC   z6Dataset {} is not found. Please ensure that it exists.z4Table {} is not found. Please ensure that it exists.)	google.api_corer/   r"   rV   get_datasetNotFoundr3   format	get_table)r4   r   r*   r/   r?   rd   s         r   rW   z0BigQueryDatasource._validate_dataset_table_exist   s    ......:666]]3''*
	z****" 	 	 	HOO   		W%%%%%" 	 	 	FMMgVV  	s   A	 	-A6:B -B>)NNr:   )__name__
__module____qualname__rT   r   r5   intr   r	   rk   rm   rW   rQ   r   r   r)   r)   2   s         "&#	  # }	   * EI? ??4<SM?	h? ? ? ?BXc]     c d      r   r)   )loggingtypingr   r   ray.data._internal.utilr   ray.data.blockr   r   ray.data.datasource.datasourcer   r	   	getLoggerrt   r]   rT   r   r   r   r"   r'   r)   rQ   r   r   <module>r~      s    ! ! ! ! ! ! ! ! 1 1 1 1 1 1 / / / / / / / / ? ? ? ? ? ? ? ?		8	$	$$C $ $ $ $    s      l l l l l l l l l lr   