
    &`i                         d dl Z d dlZd dlmZmZmZmZ d dlZd dl	m
Z
 d dlmZmZmZ d dlmZ d dlmZmZ  G d de          ZdS )	    N)IterableListOptionalTuple)_check_pyarrow_version)BlockBlockAccessorBlockMetadata)DataContext)
DatasourceReadTaskc            
           e Zd ZdZ	 	 	 ddedededee         fd	Zd
ee         fdZ		 ddedee         d
e
e         fdZej        d             ZdS )RangeDatasourcezCAn example datasource that generates ranges of numbers from [0..n).arrow   Nnblock_formattensor_shapecolumn_namec                 X    t          |          | _        || _        || _        || _        d S N)int_n_block_format_tensor_shape_column_name)selfr   r   r   r   s        /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/data/_internal/datasource/range_datasource.py__init__zRangeDatasource.__init__   s/     a&&))'    returnc                     | j         dk    r't          t          j        | j                            }nd}d| j        z  |z  S )Ntensorr      )r   r   npprodr   r   )r   element_sizes     r   estimate_inmemory_data_sizez+RangeDatasource.estimate_inmemory_data_size   sB    ))rwt'9::;;LLL47{\))r!   parallelismper_task_row_limitc           	           j         dk    rg S g } j         } j         j        t          d||z            }t	          j                    }|j        |nD                                  j         z  }t          |d          }t          d|j        |z            dt          dt          dt          f fddt          dt          dt          dt          t                   ffdd	k    r"t          t          j                            }nd}d}	|	|k     rit          |||	z
            }
t          |
d
|
z  |z  d d           }|                    t!          |	|
ffd	| j        |                     |	|z  }	|	|k     i|S )Nr   r   startcountr"   c                 "   dk    r?dd l }|j                            t          j        | | |z             gj        pdg          S dk    rdd l }t          j        t          j                  t          j        t          j        | | |z             t          t          ddt                    z                                 z  }t          j        j        r	j        |in|          S t          t          j
        | | |z                       S )Nr   r   value)namesr$   dtyper   )pyarrowTablefrom_arraysr&   aranger   onesint64expand_dimstuplerangelenr	   batch_to_blocklistbuiltins)r-   r.   par$   r   r   r   s       r   
make_blockz2RangeDatasource.get_read_tasks.<locals>.make_block?   s%   w&&$$$$x++Yueem445,78 ,    ))$$$$RX>>>IeUU]33%1s<'8'8#899::B B  %3373DPT&//&   HN5%%-@@AAAr!   target_rows_per_blockc              3   x   K   |dk    r0t          ||          } | |          V  | |z  } ||z  }|dk    .d S d S )Nr   )min)r-   r.   rC   num_rowsrB   s       r   make_blocksz3RangeDatasource.get_read_tasks.<locals>.make_blocksT   sa       !))u&;<< j11111!!	 !))))))r!   r$   r%   )rF   
size_bytesinput_files
exec_statsc                      | |          S r    )ir.   rG   rC   s     r   <lambda>z0RangeDatasource.get_read_tasks.<locals>.<lambda>m   s    [[5"7. . r!   )schemar+   )r   r   r   maxr   get_currenttarget_max_block_sizer)   r   r   r   r&   r'   rE   r
   appendr   _schema)r   r*   r+   
read_tasksr   
block_sizectxrow_size_bytesr(   rM   r.   metar   rB   rG   rC   r   s   `           @@@@@r   get_read_taskszRangeDatasource.get_read_tasks#   s&   
 7a<<I%'
G))A,--
 %''$, %&!!!==??47JN 33N$'3+D+V$W$W!	Bc 	B# 	B% 	B 	B 	B 	B 	B 	B 	B 	B*	"	""	";>	"e_	" 	" 	" 	" 	" 	" 8##rw|4455LLL!ee
AE**E u9|3 	  D e       <'9  	 	 	 OA% !ee( r!   c                    | j         dk    rd S | j        dk    r<t                       dd l}|j                            | j        pddgi          j        }n| j        dk    rt                       dd l}t          j	        | j
        t          j                  t          j        t          j        dd          t          t          ddt!          | j
                  z                                 z  }t#          j        | j        r	| j        |in|          j        }n(| j        dk    rt&          }nt)          d	| j                  |S )
Nr   r   r0   r$   r2   
   r   r?   zUnsupported block type)r   r   r   r4   r5   from_pydictr   rO   r&   r8   r   r9   r:   r7   r;   r<   r=   r	   r>   r   
ValueError)r   rA   rO   r$   s       r   rT   zRangeDatasource._schemay   sL   7a<<4(("$$$    X))4+<+G!*MNNUFF8++"$$$    WT/rx@@@2>	!R  %aS9K5L5L1L(M(M"N"ND D F #1/3/@L"F++f  F 6))FF5t7IJJJr!   )r   r   Nr   )__name__
__module____qualname____doc__r   strr   r   r    r)   r   r   rZ   	functoolscached_propertyrT   rL   r!   r   r   r      s        MM
 $"%)
( 
(
( 
( 	
(
 c]
( 
( 
( 
(*Xc] * * * * -1T TT %SMT 
h	T T T Tl     r!   r   )r@   rd   typingr   r   r   r   numpyr&   ray.data._internal.utilr   ray.data.blockr   r	   r
   ray.data.contextr   ray.data.datasourcer   r   r   rL   r!   r   <module>rl      s         2 2 2 2 2 2 2 2 2 2 2 2     : : : : : : > > > > > > > > > > ( ( ( ( ( ( 4 4 4 4 4 4 4 4D D D D Dj D D D D Dr!   